diff --git a/cld/hits/v3.2.0/README.md b/cld/hits/v3.2.0/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d5d99c2645d410f644a959bce40884f7fef93f6c --- /dev/null +++ b/cld/hits/v3.2.0/README.md @@ -0,0 +1,47 @@ +# CLD hits: ParticleFlow v3.2.0 + +Attention and HEPTv2 checkpoints from the 28 October 2026 backbone campaign. +Both complete 50,000 optimizer steps with seed 12345 and global batch 128. +Training code: [`6b80b983477321cf708e5043fb1d2ec15dd398bc`](https://github.com/jpata/particleflow/commit/6b80b983477321cf708e5043fb1d2ec15dd398bc). +The v3.2.0 directory is a release label; this upload does not create a code tag. + +| Backbone | Original run | Parameters | Validation F1 | +| --- | --- | ---: | ---: | +| attention | [cld_hits_attention_seed12345_20261028_111201_979187](cld_hits_attention_seed12345_20261028_111201_979187/) | 7,260,244 | 0.8458 | +| heptv2 | [cld_hits_heptv2_seed12345_20261028_111202_875252](cld_hits_heptv2_seed12345_20261028_111202_875252/) | 12,786,964 | 0.8624 | + +## Files and checkpoint loading + +Each run preserves `checkpoints/checkpoint-50000.pth`, configs, the resolved +`particleflow_spec.yaml`, `model_kwargs.pkl`, scenario manifest, every training +log, complete available histories, separate `runs/train` and `runs/valid` +TensorBoard event streams, and `plots_step_50000`. The checkpoint contains +`model_state_dict`, optimizer state and `extra_state` with step 50000 and resume +state. Use the accompanying configuration with compatible ParticleFlow code. +The original full checkpoint bytes are retained. No best-checkpoint selection +or inference conversion is implied by the filename. + +[release.json](release.json) records dataset versions, model dimensions, +checkpoint hashes and metadata validation. [SHA256SUMS](SHA256SUMS) covers the +release contents except the checksum file itself. + +## Interpretation + +PF/clusters refers to the track/cluster-input MLPF model, not Pandora output. +Hit models reconstruct particle sets from raw hits. HEPTv2 and attention have +different parameter counts; hits and PF also differ in decoder, objectives and +target populations. All measurements are single-seed observations. + +Particle F1 uses capped matching cost and requires deltaR < 0.1 and relative +pT error < 0.5, without requiring PID agreement. PF uses TFDS 3.3.0 and hits +3.2.2. Validation uses 1530 PF entries or 1536 padded hit entries. Detailed +momentum uses a 512-entry subset with different selection across representations. +CLD hit HEPTv2 is missing its 30k history JSON; the final 50k result is present. + +[Comparison summaries](comparison_metrics/summary.json), +[metric definitions](comparison_metrics/provenance.json), and the archived +[study methodology](comparison_metrics/study_methodology.md) explain sampling +and timing caveats. Relative artifact links in that archived methodology refer +to the original local study; use the release links here for uploaded files. +Predictions, batch parquet dumps and dataset caches are not part of this release. +Training timing does not measure inference or upstream tracking/clustering cost. diff --git a/cld/hits/v3.2.0/SHA256SUMS b/cld/hits/v3.2.0/SHA256SUMS new file mode 100644 index 0000000000000000000000000000000000000000..068a094dd5f34df15d0517d6b6e274b05f9dc70b --- /dev/null +++ b/cld/hits/v3.2.0/SHA256SUMS @@ -0,0 +1,214 @@ +ac92ed34ecb92da09b2f56f149280e1decb4e7144a038daf3cae6ab9636e96c2 README.md +55342abbe8a974130f663b6c44fa5a5387d715e0946dfa7e2b4e281c6f1e3578 cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-50000.pth +b5ede80381b2488acc3530ebb6b3d8831305c87f53de8783e6107aa7a88fb414 cld_hits_attention_seed12345_20261028_111201_979187/history/step_1000.json +1ac334f3769fe3406bfdd52591d05c61af2ecaa8a6588f3bbc2f25381adaa51a cld_hits_attention_seed12345_20261028_111201_979187/history/step_10000.json +6090c6f2d069e49efdc4c7ddd2ed0517dd5f13f754aaf9bc31092f2d9ce38cc4 cld_hits_attention_seed12345_20261028_111201_979187/history/step_11000.json +b3458826594a34ee506717f0e7b6e7355d36eee8955eedc08aae63715a11b7ff cld_hits_attention_seed12345_20261028_111201_979187/history/step_12000.json +8e3f143c4d24d56aca78cea46e99a6690c8e499b4ae4a4871dc9b2edecdda523 cld_hits_attention_seed12345_20261028_111201_979187/history/step_13000.json +0e7c778ed9a49aa7433cf9283ac6adab2b14630d6eebe3be339520e8c25ef382 cld_hits_attention_seed12345_20261028_111201_979187/history/step_14000.json +a7ce88a552335055f691206ad6f06a587f72e860f6890a79ea9f9274efd16859 cld_hits_attention_seed12345_20261028_111201_979187/history/step_15000.json +5b9c98435d0eb03305e045974cf6fdc18952fa6525f1c3a53f4ff28cc0fd69cb cld_hits_attention_seed12345_20261028_111201_979187/history/step_16000.json +e286dbe54ad48513eed29f23669896e18b9e35627c6ffcb43c2859001e1cec2c cld_hits_attention_seed12345_20261028_111201_979187/history/step_17000.json +285c7c4f164758be3cf9080fc43de2d31ed59844f45b5721558c8fe7ed38499a cld_hits_attention_seed12345_20261028_111201_979187/history/step_18000.json +c797b8bfbe9e81cad958f4027c4a464fef1a8a2b10f82d83158d4a00bf3f89e2 cld_hits_attention_seed12345_20261028_111201_979187/history/step_19000.json +38d362c37223a25c0f88eb4edf7c7dd8263f91d89dc80621d1c84a6a62382836 cld_hits_attention_seed12345_20261028_111201_979187/history/step_2000.json +72ea4f487ad208ae58185f261ae95e62896583485fcaec9494bcae592c822c11 cld_hits_attention_seed12345_20261028_111201_979187/history/step_20000.json +784547c95af9e79cee85c6eca769db2fbcf988ef2f5df09793fc10e0733f2526 cld_hits_attention_seed12345_20261028_111201_979187/history/step_21000.json +44cf9ddda50228324d43e4180ca6a815c087b351d9eba658447f1497c694d231 cld_hits_attention_seed12345_20261028_111201_979187/history/step_22000.json +fbdf5d7d7ccefecd81b3b823d0553c6b7a757d15599454eafe651ab81f433327 cld_hits_attention_seed12345_20261028_111201_979187/history/step_23000.json +be7685cae374fe3af683d4c15b4738181e6f710ba3f8b13b33b13b8e7548ba3e cld_hits_attention_seed12345_20261028_111201_979187/history/step_24000.json +6a38318bc5a1b317499558ccfca903008cac54675f72500c7c00ac6ab6ed2933 cld_hits_attention_seed12345_20261028_111201_979187/history/step_25000.json +5ee9ed0e41f79851d836b96b9f6714c2f04d57b7afa0e6b0c2b7b03bb8cd56cb cld_hits_attention_seed12345_20261028_111201_979187/history/step_26000.json +83bfd7796f400facbf9418791401cf5d6aa3d496a7e8445e25cf515b5199c9d4 cld_hits_attention_seed12345_20261028_111201_979187/history/step_27000.json +f1cf609337ae219b99243120e3aa72eed9851dfe2c91b0b18a228360ebc2c4e7 cld_hits_attention_seed12345_20261028_111201_979187/history/step_28000.json +0ee44dabdae59842db071bd07898b78c856069c5c25569bb103bdd48a150c735 cld_hits_attention_seed12345_20261028_111201_979187/history/step_29000.json +4178246f8547b99c478f657a30abf3c20380da2f6a71306cdc7efdf95ebba487 cld_hits_attention_seed12345_20261028_111201_979187/history/step_3000.json +b19776e63c07778b601f6243ece4f45f3b45c318adc5b431e73be8f1630472f1 cld_hits_attention_seed12345_20261028_111201_979187/history/step_30000.json +f912c2ef23ee476f8f2f7c3df9699b400f2bb379f1bfc9a6f5378a0fe4130fa6 cld_hits_attention_seed12345_20261028_111201_979187/history/step_31000.json +e90ba20189e1bcd07b2d508a3ca93fd61ef1069f60279fabb95af43caac5066f cld_hits_attention_seed12345_20261028_111201_979187/history/step_32000.json +61c0f65392407b5bf091b56c8f9bf4b1ee0216481c2f9669c17f7a0af3c67463 cld_hits_attention_seed12345_20261028_111201_979187/history/step_33000.json +4bd74286ed95728d4f28c4427ee5f60f2fea908f4362d0f4e59081548a949b58 cld_hits_attention_seed12345_20261028_111201_979187/history/step_34000.json +68b20963addc5210b1a0db2ad9be9c090380c8af5b836204ee8735eff9451790 cld_hits_attention_seed12345_20261028_111201_979187/history/step_35000.json +d7f4efa891aa42ae252e647f7e927bbaed9e76fff54564db03f89723244c5555 cld_hits_attention_seed12345_20261028_111201_979187/history/step_36000.json +7d8989bdeba656a694952893d2ddecf9c82f5eddb6020cb2fc3d6d4f44fd5fff cld_hits_attention_seed12345_20261028_111201_979187/history/step_37000.json +b70c85abd30088aab32c1b49f87dd5e0d2f520a004a0c51b03c7434327612459 cld_hits_attention_seed12345_20261028_111201_979187/history/step_38000.json +3f6e3e068724697b675d26d7ecf345c76dda186b41197b88959ed6ee60b4ff98 cld_hits_attention_seed12345_20261028_111201_979187/history/step_39000.json +0f39ac18b88e32f54e8176472896d63aac2bc6b3309f2bc5d78ab9a6a63a2a16 cld_hits_attention_seed12345_20261028_111201_979187/history/step_4000.json +c3243c262c606b0f43a46d7d802800c4f85fce0b072dfe04e8bae748d09b1bfa cld_hits_attention_seed12345_20261028_111201_979187/history/step_40000.json +7cd7c2fafe4fa9cc39a290aad818d2f89ce2f18d5d7ba19b8bcb10a7445f22a7 cld_hits_attention_seed12345_20261028_111201_979187/history/step_41000.json +5849cc9a7fe93b5d7a18dab601eda492036c7b923303bf108c8a8ba3961f818e cld_hits_attention_seed12345_20261028_111201_979187/history/step_42000.json +babb2b473b056aa0b2655c575677bad5bd4304725e3675c8a580cafe5e243d39 cld_hits_attention_seed12345_20261028_111201_979187/history/step_43000.json +08178ce01132a56922ab42ca2c60b2e6b42dfd3cc839acdb050bae2a3bf9e1de cld_hits_attention_seed12345_20261028_111201_979187/history/step_44000.json +2372600786a097bfa53eaea900254d69f3b4518d6d68b8c453612058be6471dd cld_hits_attention_seed12345_20261028_111201_979187/history/step_45000.json +41046a9eb772183201c449ed00cd29e9bf46cec3ddb8c288f0296ca80fd3cf23 cld_hits_attention_seed12345_20261028_111201_979187/history/step_46000.json +baca346a8de2fbd167a65d05295ead916c67735b000cf50dd879100fe9fc4fae cld_hits_attention_seed12345_20261028_111201_979187/history/step_47000.json +a0c3f05a2821e60992ce1777cbe951f88a7ab4f2496b54f28e933058d7f47f12 cld_hits_attention_seed12345_20261028_111201_979187/history/step_48000.json +19fcf14e51ab4c9a0ce5290932b1846116ebd6950cde412e5251693a853070cf cld_hits_attention_seed12345_20261028_111201_979187/history/step_49000.json +f971bdd4ba6e0b2ce3017effd9c3d7973fd2666dc48573523ee6cd7873c7c382 cld_hits_attention_seed12345_20261028_111201_979187/history/step_5000.json +e6c1e45b3a5662956f5890e1e10b78484e6b2746ce8d480a9e38dcf104543407 cld_hits_attention_seed12345_20261028_111201_979187/history/step_50000.json +ed520135a5d3509d3a82d7ee8c7b9c1e27440b9029be37f8b6271bb2c4ebf2e2 cld_hits_attention_seed12345_20261028_111201_979187/history/step_6000.json +1dbfcb71a4abb72f0e68aca86bd0320e3ad2c0a55c3a71d89939307d1ab2988a cld_hits_attention_seed12345_20261028_111201_979187/history/step_7000.json +91ad6f42f305059d0aa43c97cfd15f11945fdc77c12691672bb9e460c886fe8f cld_hits_attention_seed12345_20261028_111201_979187/history/step_8000.json +171aec54162f785be9fb75d764956e6f943699e475de220f43d9454e9ac6cc3c cld_hits_attention_seed12345_20261028_111201_979187/history/step_9000.json +e488e70ef1a2bfc6849add05ab4f3db38787dbf481ecb742474bc474767b9219 cld_hits_attention_seed12345_20261028_111201_979187/hyperparameters.json +72c3356bd75892977dcf280e91b3c88edba3d42e5979931e6d5452199c3f7382 cld_hits_attention_seed12345_20261028_111201_979187/model_kwargs.pkl +0b760998f19c1189d9d765e69de13a138fa6c0aa679cda57a5cba137f70ddff1 cld_hits_attention_seed12345_20261028_111201_979187/particleflow_spec.yaml +b0390a1312f7f3ee1cf28c0d6dc3e16187e8681489cde649524278c053c238d6 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_eta.pdf +fffd8aaea0a3134c061cea00108f762082fbf1b8e20ec4b32f253204a5f5a103 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_eta.png +5049019dac2c7724d75f5cdda84d16359fd8e44fb41dad0af2c6e11d47db9906 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt.pdf +ef3e531c4aa477c71cc63d8431dfb1e55859076cd2e22744300bdc9a3bbb5215 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt.png +8068722ddbc6b35493f5604d5f983ac43f322e9397be182e14a4e127b83a8fc6 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt_log.pdf +d7424880726b453357a140e659db7c2cbb5eb671bb15f7b3b6fdefe96a3d251e cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt_log.png +9654ea5f93540f24a5e41a0ac7321ce9ff78383c534fc2a3e4717b6fd581ce14 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res.pdf +27e5ed95d6b0ee77df0374ad98e988ab7b68b65f5cd4862e7b4627fd1c03561f cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res.png +d8703422e49557bff41f039734068ef2f508a20013ea8e6aa1fc7ce77a75e8c9 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.pdf +b01eecef6b455097f9058984d811647f13b384274f5673db1e46d6591a248a13 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.png +24ef6c7a35a461e7226b927e3793cfb2db554c4900f7246ead6f0c45d0dfe345 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/num_elements.pdf +ad9fa480f42c813423a94f6fb6b034df731c4006b7e7562b8d3572d16c142464 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/num_elements.png +d47f544b92c3db424932252d68071654272387253156470a6fd7fedd16738176 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_eta.pdf +42abf9f429913ef79673b5b18efd48b95a9f79b7056140f31a5f141daf84e12b cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_eta.png +40cadea29a211de792633d1fa09336c1ddaa28866704208969519197b485131b cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt.pdf +6d43cf4c0e5e12531ee47e726e88e68411bedecb2b088e1678da79a7a25ea5bf cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt.png +bc2ee33948459ac122fd74adfbfd47d67dff5086a64ec1f3a01d47927a4aec95 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.pdf +6a02770da6c15a74c5ca86491442a6c46e4521607426fda584c4e9f5d156c401 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.png +5f3f83df62258ca21e9fcde54c4c76fb751829b636cbb33b9f90af000ee48650 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res.pdf +9e619e5081bfe638fbcd1211a177c47888d8d99bae937192f790fba407acf4a0 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res.png +a9a1017248faca8387cf9e01a0909fd17dda820a44c72b8ce8adce8b55a01b22 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.pdf +9e9380f6150dae2a6d8b61661b0712d64db4ede7959ebb5e7050faf04b80c507 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.png +7cc2052599bc7e9e1037451aec501f5cbec839f4eb4f34b90771dd9b820601ed cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/num_elements.pdf +9291c720cb644128b7dd82cea0b6673901c16ce2cb92095926b459aff72d57e3 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/num_elements.png +36027854065b5488a52f7e7047d40dfc71daa1386f1e42f5458f7328f0846e35 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.pdf +ba8750b1a9e6cef4c17d8c06352327efd53b5295423256e4487d67c10162c542 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.png +f9ca23e7e0cdf646718ce73fb4c9080ecb2c9ac0ce2c386c24f09c10590d1675 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.pdf +5f97c0fc4e72b7b4765cb665dc9dd21657629fbdd2e41a77fc5ec7c4151e57c8 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.png +611fab8f368c163dc83516da17771c74863afb6d555b64ca2704437ae3b54883 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.pdf +674fcdeca2ca0b0bb454a5b9d63932a74131630fbe2d63d068e414ebc12468c0 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.png +cc1ddcefe3a02d506f8d5ea6afc94f65c4094b075770487c6c53af864b226778 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.pdf +3ed142dead3808fc3aebea697f1a74dd11bc1066208419e78830ca8f0541b6ef cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.png +18a7a2dcc2eaf5bbfa0a1bd641cd00f86154070c3e01da0e19336aa76b6b4b62 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.pdf +01738278630f8be18fe99d5c5664b978110ca219a6ad69915818e6e679ad2598 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.png +9f3ca5da47b45c2b6a0a9553b3cb034aee87c22fa4cf8c34df24a3e7c8305e8e cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.pdf +489c64f2a8778727cdb14f9dd8877aebb75e557afcac6741fed3f7c4f24de944 cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.png +6611e6819aa90e72376b199874b3c2c2e9e7cce7a1fedf3c2130e512550d140f cld_hits_attention_seed12345_20261028_111201_979187/runs/train/events.out.tfevents.1793178801.nid006207.95745.0 +cb7b2a3d26a80285ac27dd86de4eb0622c12dd40cd848d69ec168d89fabc2b8b cld_hits_attention_seed12345_20261028_111201_979187/runs/train/events.out.tfevents.1793278150.nid005379.42183.0 +44f9e14fcef344f9156f797267b20d24c7aa892cec3f809ea2c6758af6fbfd78 cld_hits_attention_seed12345_20261028_111201_979187/runs/valid/events.out.tfevents.1793178801.nid006207.95745.1 +e6b67c7f5067e538b2f5c96216eb325ac55c9f03ebe263ebcb4d10039226e6a9 cld_hits_attention_seed12345_20261028_111201_979187/runs/valid/events.out.tfevents.1793278150.nid005379.42183.1 +0198bb8b8a119cd7814736797e8c56c282f7aedf9fd3300965a38593fd6d22b4 cld_hits_attention_seed12345_20261028_111201_979187/scenario-manifest.json +fb1237f89aea62faa31186808365890dfbb4470da8e0b3dc27e62acd8e542862 cld_hits_attention_seed12345_20261028_111201_979187/train-config.yaml +de5bcdd1b9841e56e018ac4984b7904fd0fdeecb2a45ef552414dc7f48b2a401 cld_hits_attention_seed12345_20261028_111201_979187/train.log +f45d8cdadf0a4d2067d33e95c100b41841de75b62656aa45e5aabf9fe692e466 cld_hits_attention_seed12345_20261028_111201_979187/train.log.0 +990fefb1e23b584d6dfaa3d09271d94ada7b92f8ea8213959b49d42d10a79d9c cld_hits_attention_seed12345_20261028_111201_979187/train.log.1 +ea97927c1502c6bb609b2d718aa8d3c1c8535259a03285e2a85482622140f745 cld_hits_attention_seed12345_20261028_111201_979187/train.log.2 +beaba90db230e75a1d6224750c9211bb266df40a32d64fa9c7a1e587d56c1cdf cld_hits_attention_seed12345_20261028_111201_979187/train.log.3 +b74a4de27097b48d7ebeb415af82e5b43b8c77086234955e3491a8c216bea5b8 cld_hits_attention_seed12345_20261028_111201_979187/train.log.4 +5dcd450070a98c6cb3dac398c19f1a565b2082c9e8eaf6413aa684095d4b7491 cld_hits_attention_seed12345_20261028_111201_979187/train.log.5 +3dece4c80f9795b5b09ecd0ba8a5ec2abebbb0780a6a22aaab98c99b097462f2 cld_hits_attention_seed12345_20261028_111201_979187/train.log.6 +f245c85aa7a84a09662e3fb0c91cb52fa4142ea837325ed26a75ff8fcfa27267 cld_hits_attention_seed12345_20261028_111201_979187/train.log.7 +f162da4e4e90066cc41fd492f7279c8aa89180040a65358d25eb6c9f45660883 cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-50000.pth +a7bdf9c4c4f903d71cf249456dcd1793ad553b7bb56b871886712b8f845a6c97 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_1000.json +713edac4d97734cd0661a77e8659179483930470c89587583551c0fe819ef552 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_10000.json +2f8f701429f3f2f1b15c8a842b268d7cce505dd838bfb7a183945bef93b7e3a7 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_11000.json +409609de47b44a5c4c5e0188ef0a2416bf57a64a21cf8d5ce75d31704537f44f cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_12000.json +bb86b8dff9d31e1d639ac0246a170a27f4705dec790e8221a8d228a37456da22 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_13000.json +a44744b9a389d64fc57a0ece6e81b7ab55cc118159e573a21dde6ccbf26ae562 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_14000.json +0bd613f99ab02ab6000af5c5f5b53de4be0dc48030ee8e4a531a39b2212b942f cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_15000.json +ba674fa352490201a327d998cfec3337fec1f27a203b0921686b9dc9b254ece4 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_16000.json +74f4ed9a7e63f92122e37fdadb178e2b0278ef2d56c39875a785895d7a8c6836 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_17000.json +01bea75decc45d1de507764d1e2b5b00bc5d764e254dae50805aa6a898c34918 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_18000.json +f7da9c45eb055715dd02bd58af5717a609006939c765cf0049ce145a50fdf6f6 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_19000.json +6b14bca5b1f651d1d4cddf489147ee24cc4f22f9b70a0cf9e6abecf3c97d2031 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_2000.json +3a96bb3d4f902b0eaf4d17bfa3cb278751e63eceb947469a8d03027d6dc9c99c cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_20000.json +60a0cd26b71b08bc6e7d40c062879db86b8f6c9c5eb9c7a3189dc8e4e60931a0 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_21000.json +b065f9eaa235d74e0d037bbb0fc05f27bb4f84a1492a08762901eb3a07163363 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_22000.json +6b297e5dc93541f6abf573a14053d515aff404e88e6df45c1836797b2aa12c4c cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_23000.json +68b38bb3fc415de0d7d81c3353f479117f826d2cec4a0251a1939c94310562a5 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_24000.json +b6b268a455ca8bc817daadefe795d6e73a451c10a54d01801b2da33f4e646ce9 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_25000.json +a5ade9d574c6af5333ba0112d0086cb67a2ee7112b2efc8e88b82ac50f0151d1 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_26000.json +e674cf3f255364300ff8a7fc23cf5f309bc7973debc814d383c0dde4970c2475 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_27000.json +52c113a829a9f5f6cf11a7af3abdd2a58998a407aba4e90a3dbff5e26e0fd71c cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_28000.json +29e8d3945233c2fc8ea913510dcf60fafc575ee06cf65bf470bf298c24483032 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_29000.json +7f639133674e1e92d591724589e8f927a3219566842c1e1a8b92fa893bfa0915 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_3000.json +810d04723f0eb77d1171f7a06e0b35150a6d09126dc625b4ecb13566f906f025 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_31000.json +95e47e962aeb6f11ba391b571452b860024b1a85ff57c572280733f44e972199 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_32000.json +3d009f314fe11698cf2dbafc71b275d7adeb8c98f8aac8a4b9a1d88ad218f9c4 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_33000.json +54bf7e051b3baf793101ad46fc187c33ba09265a64ab95eb4eff098fbd3f1400 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_34000.json +1518108d6635e163f2bd6b2a9b241c864a1ac13a79e416c126ebca14df2c6bc4 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_35000.json +15909329273c98858a7374b3b2df1e12a388beeca55d1db6e8a37ef8cd9e79ab cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_36000.json +829619de0ba47cdf3fd25ff02878962474cac4963bf5fe1e9950b64322ff4cca cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_37000.json +29cf7ca793098b73e5eaba9cf3e1a13fe03e090cd82cea5e46a52f412df144d3 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_38000.json +6b1b91b3afe1020e7325baad738cd524126fb5e6c29216403b821daa51433202 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_39000.json +fb8cdab41a077e50168c7beace72c15332154bbe945c3d932dc4a9f5f9bbb9ee cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_4000.json +38e9374047894a242957fd26937fdf30afab3069edf9f389593c885201e3ea3f cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_40000.json +501d721e11500b3ffc618747fe1e0172c684cd2811a963bf3a1f14c5bd6858d4 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_41000.json +91b3fb68f78861fbe56b7fadf5741489830d14a50fa0aeacbed50fac6fe524bf cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_42000.json +f03d5a5327e086d5581ec2a0a11656fec23539b51b093fd2e382654c62600fee cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_43000.json +cd011219c6b49f021ec32d524dfd14b2b67d185099ab6f653671d92470429b4e cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_44000.json +3f435c8b158d2d22122bf7bfbc1b2e0737da4ce15dec988e12e2cff281776a70 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_45000.json +aa1cda788919f7ea8200bb4b21d21caa0883e157aa7c2265f77f53b754982bb7 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_46000.json +f14a6623ea8cebb716d797120c8fb27b51e387d32aa5405059b7a259b58d2ced cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_47000.json +fa295773026e65e1470a416bfbf640c88ccc5582a8ff57a02d363e38af733eaa cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_48000.json +e57fadd4e60887a292e3a7959d5177dbb7ed538c3f9ae24a0fcd016b9514d1d6 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_49000.json +f6d645a1db131e21d57017f69b15308948c66f90dfa022a278dfcd8c8fdc537f cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_5000.json +320814f2c5465845b4e451436298411f22ec64880b5e4f9e96291b891a517320 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_50000.json +a798afe2fe62ac7382165139395ce8e61203178aadf0909f08a1f02ed3df800b cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_6000.json +20e64730aa0ca389f1943e7395a39248a9667cdecfb093044aed2927286fea27 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_7000.json +db197308d07ff16b55691acac3c6cf162b7b5cab181453ed531ce022c975b4e2 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_8000.json +c92a0bc4b7df6325ed4ed27674222b9a07387cc59dd7334a707b6e5d45c849b9 cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_9000.json +992c15bf954887b836ad44c2b5094e2b85ff9bfb42b9d6bb17e0c167ba3b27e7 cld_hits_heptv2_seed12345_20261028_111202_875252/hyperparameters.json +e69f18484342dae306407316f3aae0085bd7f3db28e1fbb1153a839dc3d3c6eb cld_hits_heptv2_seed12345_20261028_111202_875252/model_kwargs.pkl +0b760998f19c1189d9d765e69de13a138fa6c0aa679cda57a5cba137f70ddff1 cld_hits_heptv2_seed12345_20261028_111202_875252/particleflow_spec.yaml +30b0e60f0adb2c5fbc1275b73aa61b1cd558d69e34106f585dd96bbf3b5a9259 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_eta.pdf +c8677049a9df30f99dc8bf7e03683d92e16024d1c4e2c49219ac366a767e57aa cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_eta.png +ee3c2c95c6ef581a19f462bfff3cf549b61569a48c037270493c71f5eaaefb7e cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt.pdf +0554cf0335540e14840b1371712cbaafc84fbf011acc7b0febcd6f4edfa636e1 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt.png +699f8b49de46e2293517354ec6285dc195a02bdc8d10f8e05ef16012e05f14d8 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt_log.pdf +b9a900f186a076bc6cdb25bdf56b75e478950c388f7bc94559b1de6fccc0df0f cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt_log.png +a88e15f07a57f5bee1f32bcc25207763e438784eb89079d1864dca13af68e28b cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res.pdf +6b8cb22fd1246432e4cbd9f4d48a514f9d6a2689da168e0b65fd6ea3d1837b8a cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res.png +442fa8a1a17cb852fe3113e743389dc1af4be4ad037c8b1b1553e912555a73f1 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.pdf +c4500a90e46022a1dadb7fbaf46eb4a94e7dc45806e50f838d9665f745234c21 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.png +8b580db3e3070312243960f6755751f1ebeef9e03c42c2f7b3ad5a421c5d77d8 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/num_elements.pdf +ad9fa480f42c813423a94f6fb6b034df731c4006b7e7562b8d3572d16c142464 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/num_elements.png +9e783199874e4e4456868d2184c9d1a77dabe0c8d8777cd15f9657aa0a2900d3 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_eta.pdf +12b5429913aefb6d1179403d4b57e3743ee001d476dc2b88b5b08c8cf47680d8 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_eta.png +1911353cb9718a4b88808fdfceaad2e604ec338323eedaffb78ba323975ed0c5 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt.pdf +389b9c3196f7dd799251e5231619402a4bfef877767bb5d2eab1c22c36f2f4ea cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt.png +0f7e68614c0d51016bdd04c24284c3cd398ddbe2a9d9f298febf65b2d0bb774c cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.pdf +5380bb90631d9a73b557b3e25c84de1eb4984042b2351603738f54d0d60383a0 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.png +510d4c1972a2c32f5ce46e0d854165f85a10411e26e938c322725ae96f087e11 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res.pdf +908ae880888b040a374edb32faa28934a88788590a22342883ad9fb2c64d94dd cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res.png +db64687fa7ede6d971ea1d04d59e88052e057062fc1355c87b226ba8787e6cdf cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.pdf +7724f75471cd73806aab30144f304822f981e926b0b662cb2acc8915b4118d73 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.png +9aa361de9ac849cbc83d1ceddff0999f5fd2926b486dcc174c4a8de75b76db35 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/num_elements.pdf +9291c720cb644128b7dd82cea0b6673901c16ce2cb92095926b459aff72d57e3 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/num_elements.png +b785780cfd0816b6087915a49d04e5e1d9c32ad447c0b9d90076e75a22cb7e67 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.pdf +3c042bfbe04e5bee1f22835c1ad07bef7190f7aa680dc55b5e94ab08b2f9f2cf cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.png +a73eb7b8946e431cc4f2515624ac8e845ba1945c3a9eb2c00bc8fc238f1ad940 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.pdf +1134ab01137fa2a5ed1433fbbbb58b466e663cf5ea4d331bd91c2af7fc0ed0cb cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.png +6efb271ae06ea3eae60b373e0f9244a61ff48a4766daa142105b73e45e4dea9d cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.pdf +be166bf55cf3966e31e54d274f14ed34d748f91f32523d3cecea4344ca2567a9 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.png +7e078819d3230fc0a57567124f9274c5f53f2659a28b5674d1f4646cd2f146e4 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.pdf +5b588c00e46661e6ea727813ba2da98b9909e1b1eb558520ddb2b79c7413c2cc cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.png +e72669d1c645124a704a3a898f00be7e13836909ed110250629ba8aec99710d1 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.pdf +817bc7dfdebf8a24bdc08695c57157e8989be6415f03c63406e9d55ef2922aeb cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.png +dcf1ec91042cabfbd20a12b3939cd3a33e15624b4d86efd46365331eb0a545a8 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.pdf +489c64f2a8778727cdb14f9dd8877aebb75e557afcac6741fed3f7c4f24de944 cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.png +645b68e0082731ba25bfe794170c7bc75e4e9442a5a9f6413cf413593180175d cld_hits_heptv2_seed12345_20261028_111202_875252/runs/train/events.out.tfevents.1793178801.nid006495.97037.0 +638cf4bdb226a5d634248919fad0b405923c3aa66215e3f2e2b42d0e39cc243a cld_hits_heptv2_seed12345_20261028_111202_875252/runs/train/events.out.tfevents.1793278187.nid005451.121603.0 +82907733e494fe149fb45be8d01b947a9e5cce07e82acb36d293b68bd69e1a24 cld_hits_heptv2_seed12345_20261028_111202_875252/runs/valid/events.out.tfevents.1793178801.nid006495.97037.1 +14decc8db4cb9b59728c3068619a73cc08250359c1170fcd1813d2032e8d2d58 cld_hits_heptv2_seed12345_20261028_111202_875252/runs/valid/events.out.tfevents.1793278187.nid005451.121603.1 +981edf44d343d85570b2b2ff91e7c130fb283b01b800bccca72b1c58f264da82 cld_hits_heptv2_seed12345_20261028_111202_875252/scenario-manifest.json +1dc4f601558bdb814db1aa6b432dd015a82288281b6d34d8c734aeae779e5dff cld_hits_heptv2_seed12345_20261028_111202_875252/train-config.yaml +956f4a0c9c17e7e0b2fa950e6fe42ab66f13acedc3331e436d78c9ad89dd549d cld_hits_heptv2_seed12345_20261028_111202_875252/train.log +6e5f9ed71457c119f6e02f28a6c2ef10bf8b7bd2fd350dcd02e35a2f6d31b721 cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.0 +90b685fbc34565170cdabe26daee23b64b342ecff6b21cde14de363cd91d9519 cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.1 +b8c252b66c60345109251325322dac0074f879d7d7ba60465dfcbfbc412dad4c cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.2 +e8cdae0775346be888041fb9b6741e6b49952569abecda354d627b13078ad030 cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.3 +97e731a67f3d3933b583a003c481e4c21c921336b0287596c9e4eefc82421c64 cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.4 +1e18e3f7d1f476d03cf206726fc065e8ae841c2f2d1464a2c6a164e2cf8aa671 cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.5 +aa146e05898262631c97357d858a3d8377fffdbacf55945649df1e9d3eddbb1e cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.6 +32190d14e9350ca9833f01258bef47db7a300d8b18c1f80ded108354be72029c cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.7 +a9a813daf88031a86204a1100e12ecad2caf059954dd8256d7d98e46b2171105 comparison_metrics/provenance.json +4b00f93f03bf042116ccedcd743f4f78b0d26cc18501fca9bc294975f2e4beb1 comparison_metrics/study_methodology.md +52360d2381619be1d13d267d1e32cb282338c94283422bc3cff4920db8e93263 comparison_metrics/summary.json +43b896b26944dea592d018c22de2662822b4f91d04f435d543344b21eb9bc1f7 release.json diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-50000.pth b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..f4653c880de99381deda09c085414376367e16ab --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55342abbe8a974130f663b6c44fa5a5387d715e0946dfa7e2b4e281c6f1e3578 +size 87327154 diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_1000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..422509e111108786a1553bab94d1712b0dc6bf07 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 0.0473664266767715, "train": {"Classification_binary": 0.42323631048202515, "Cardinality": 0.27602678537368774, "Classification": 0.36031362414360046, "Regression_pt": 0.5425720810890198, "Regression_eta": 0.006576064042747021, "Regression_sin_phi": 0.0031143072992563248, "Regression_cos_phi": 0.003107067197561264, "Regression_energy": 0.5411424040794373, "HitGrouping": 0.31772199273109436, "Auxiliary": 0.5942466855049133, "Total": 3.0580549240112305, "Total_unweighted": 1.8800618583336473}, "train_group": {}, "valid": {"Classification_binary": 0.43120908737182617, "Cardinality": 0.30045372247695923, "Classification": 0.36404744784037274, "Regression_pt": 0.5429916381835938, "Regression_eta": 0.00664433774848779, "Regression_sin_phi": 0.0031611897672216096, "Regression_cos_phi": 0.003119909204542637, "Regression_energy": 0.5409567753473917, "HitGrouping": 0.3256998260815938, "Auxiliary": 0.6058096885681152, "Total": 3.11918036142985, "Total_unweighted": 1.8921303854634364, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8298329710960388, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.07929839193820953, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.17355918884277344, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.07649821043014526, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.04790894687175751, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.45462289452552795, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7934097051620483, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.07282496243715286, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.13441655039787292, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.06939450651407242, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.04914263263344765, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4689807891845703, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8251253962516785, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.07307685911655426, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.15394584834575653, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.07101236283779144, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.04583999142050743, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4685952365398407, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8240892291069031, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.07494213432073593, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.15837155282497406, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.07254871726036072, "metrics/hit_grouping/calorimeter/all/object_purity": 0.04672319442033768, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.46439221501350403, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003688529133796692, "metrics/hit_grouping/calorimeter/background_recall": 0.005313451401889324, "metrics/hit_grouping/calorimeter/completeness": 0.1769300103187561, "metrics/hit_grouping/calorimeter/energy_completeness": 0.2125452309846878, "metrics/hit_grouping/calorimeter/energy_purity": 0.2100602239370346, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5491029620170593, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8157714009284973, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.06023187190294266, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.11066226661205292, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.0575084388256073, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.054964546114206314, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4762744605541229, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8285413384437561, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.07674842327833176, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.16812528669834137, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.07426891475915909, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.038786280900239944, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.45750805735588074, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8365638852119446, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.10311637818813324, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.2458236664533615, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.10148254781961441, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.037915103137493134, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4470997750759125, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8173553347587585, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.08634190261363983, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.20730093121528625, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.08246250450611115, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.04787034913897514, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.48959586024284363, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8654728531837463, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3195295035839081, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5265054106712341, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3219758868217468, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.1682736575603485, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3762223422527313, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.82602459192276, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.03418007493019104, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.06462689489126205, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.03272509574890137, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.03107249177992344, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4491998255252838, "metrics/hit_grouping/calorimeter/purity": 0.17444755136966705, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9181592464447021, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.017513424158096313, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.04135338217020035, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.018530702218413353, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.006212644744664431, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.028195489197969437, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8602698445320129, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.02931227907538414, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.04296645149588585, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.028901459649205208, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.01924670673906803, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.29487934708595276, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8206736445426941, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.078252412378788, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.16628380119800568, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.07570024579763412, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.04881312698125839, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4803209900856018, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6862586736679077, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.2582935094833374, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.1324390023946762, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.0855736956000328, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6904244422912598, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7382919192314148, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.19831223785877228, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.13568679988384247, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.08714301139116287, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5406871438026428, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6120246052742004, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.24406389892101288, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.10562252998352051, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.07124453783035278, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7531299591064453, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6441668272018433, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.2450578808784485, "metrics/hit_grouping/tracker/all/object_completeness": 0.11608515679836273, "metrics/hit_grouping/tracker/all/object_purity": 0.07682288438081741, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7181656360626221, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.029018424451351166, "metrics/hit_grouping/tracker/background_recall": 0.05628354847431183, "metrics/hit_grouping/tracker/completeness": 0.12266474217176437, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6018080115318298, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6176235675811768, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.24364370107650757, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.12221569567918777, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.10556057840585709, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7603892683982849, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.663356602191925, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.23597748577594757, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.10821612179279327, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.053131103515625, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.694862961769104, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6797009110450745, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.2569882869720459, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.11129271239042282, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.04124774783849716, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6544634699821472, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6810081005096436, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.2633098363876343, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.12298690527677536, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.06233547255396843, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6890549659729004, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6844231486320496, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.37376847863197327, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.19980168342590332, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.08002140372991562, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6453201770782471, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5890552997589111, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.19825588166713715, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.09210487455129623, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.09508360922336578, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7690694332122803, "metrics/hit_grouping/tracker/purity": 0.11785778403282166, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8988468050956726, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.015837104991078377, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.009098600596189499, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0035119333770126104, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.03054298646748066, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7627593278884888, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.11409395933151245, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.06030265614390373, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.022311335429549217, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.4259163737297058, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5935191512107849, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.2932684123516083, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.13800780475139618, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.09361039847135544, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8518829345703125, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 0.25, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.25607097148895264, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.5, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5943992137908936, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6799145340919495, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5222816467285156, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.26722458004951477, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.05219316482543945, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5232955813407898, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.699999988079071, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.334001541137695, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.09403984248638153, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.7749999761581421, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.002416672185063362, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.37347903847694397, "metrics/particle/class_1/pid_accuracy": 0.7358422875404358, "metrics/particle/class_2/efficiency": 0.3580246865749359, "metrics/particle/class_2/pid_accuracy": 0.08351787179708481, "metrics/particle/class_3/efficiency": 0.3728220760822296, "metrics/particle/class_3/pid_accuracy": 0.8592119216918945, "metrics/particle/class_4/efficiency": 0.39178356528282166, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 0.3927203118801117, "metrics/particle/class_5/pid_accuracy": 0.0, "metrics/particle/count/bias_mean": -10.060546875, "metrics/particle/count/mae": 15.102213859558105, "metrics/particle/count/prediction_mean": 60.59440231323242, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.20401592552661896, "metrics/particle/event/energy_response_mean": 0.987055242061615, "metrics/particle/event/met_abs_error": 16.359539031982422, "metrics/particle/event/scalar_pt_relative_abs_error": 0.20462052524089813, "metrics/particle/event/scalar_pt_response_mean": 1.0634068250656128, "metrics/particle/event/vector_pt_closure": 0.13921378552913666, "metrics/particle/matched/delta_eta_abs_mean": 0.025067534297704697, "metrics/particle/matched/delta_phi_abs_mean": 0.022892536595463753, "metrics/particle/matched/delta_r_mean": 0.037981025874614716, "metrics/particle/matched/energy_relative_abs_mean": 0.20883215963840485, "metrics/particle/matched/pid_accuracy": 0.7260236740112305, "metrics/particle/matched/pt_relative_abs_mean": 0.20569676160812378, "metrics/particle/matching/duplicate_fraction": 0.10976330190896988, "metrics/particle/matching/efficiency": 0.3722057342529297, "metrics/particle/matching/f1": 0.40073612332344055, "metrics/particle/matching/purity": 0.43400341272354126, "metrics/particle/matching/target_coverage_dr0p05": 0.3027016520500183, "metrics/particle/matching/target_coverage_dr0p10": 0.4592263698577881, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.23188912868499756, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.45287764072418213, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.038804277777671814, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.21155235171318054, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.20687852799892426, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.01960526593029499, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24971796572208405, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.021911298856139183, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.6321666240692139, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.12201311439275742, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.328425645828247, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 3788.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.8529841303825378, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.847576379776001, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 4.393032073974609, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 4.219529151916504, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.4472668468952179, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.087011337280273, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03822687268257141, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.3239383697509766, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.01362658105790615, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.450756072998047, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8031.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.35165244340896606, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.5895135402679443, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 20.626888275146484, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 9.371707916259766, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.19859737157821655, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2883639335632324, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 17.363683700561523, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.3714172840118408, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 26.31459617614746, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03897826373577118, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.22928635776042938, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.22530703246593475, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0035363612696528435, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2644941806793213, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.06843267381191254, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 3.2104580402374268, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.09880755096673965, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.350367546081543, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 453.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.6783627271652222, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.957777500152588, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 9.942119598388672, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 9.80321216583252, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.606039822101593, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 45.86104965209961, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09677419066429138, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.535980224609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.39151307940483093, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.82144546508789, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1457.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.164249449968338, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -18.259042739868164, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 19.988922119140625, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 3.1472201347351074, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.11709842830896378, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.047380805015563965, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 19.62332534790039, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.1900951862335205, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 40.52527618408203, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.038166023790836334, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.2070183902978897, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.20470012724399567, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.016464922577142715, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2454962134361267, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0030603630002588034, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.5305298566818237, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0549544095993042, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.2857239246368408, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 9476.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.7698261141777039, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.752849578857422, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 4.315516948699951, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 4.130932331085205, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.44003063440322876, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 19.370750427246094, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.02196802943944931, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.45313560962677, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.03464069589972496, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.365882635116577, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20894.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.40723708271980286, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 18.258163452148438, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 23.65266990661621, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 19.765892028808594, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.8283767700195312, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 5.505089282989502, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 10.72720718383789, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.948301374912262, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 15.860970497131348, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03836910426616669, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20900583267211914, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.20598222315311432, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.016671601682901382, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24731703102588654, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.010425020009279251, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.9232547283172607, "metrics/particle/momentum/all/accepted/pz_bias": -0.0683949738740921, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.268636703491211, "metrics/particle/momentum/all/accepted/samples": 13717.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.8353774547576904, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 4.1631364822387695, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4.605836868286133, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4.426371097564697, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.44990456104278564, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 21.104116439819336, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.029853202402591705, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.8311593532562256, "metrics/particle/momentum/all/assigned/pz_bias": -0.008649305440485477, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.201584339141846, "metrics/particle/momentum/all/assigned/samples": 30382.0, "metrics/particle/momentum/all/efficiency": 0.37275469303131104, "metrics/particle/momentum/all/event/energy_bias": 3.5886335372924805, "metrics/particle/momentum/all/event/p3_error_mean": 46.759124755859375, "metrics/particle/momentum/all/event/pt_vector_error_mean": 22.83543586730957, "metrics/particle/momentum/all/event/px_bias": 1.909875750541687, "metrics/particle/momentum/all/event/py_bias": 5.84083366394043, "metrics/particle/momentum/all/event/pz_abs_mean": 35.060340881347656, "metrics/particle/momentum/all/event/pz_bias": -4.5098137855529785, "metrics/particle/momentum/all/event/pz_residual_rms": 50.56886291503906, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.05106397345662117, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.22811609506607056, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.22441837191581726, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.03332636505365372, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2641819715499878, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.01252074260264635, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.15355384349823, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.15445230901241302, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.5369699001312256, "metrics/particle/momentum/class_1/accepted/samples": 6629.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.7607857584953308, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 3.299699306488037, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 3.9392800331115723, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 3.723767042160034, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.42116039991378784, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 21.414730072021484, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.032764505594968796, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 2.0035276412963867, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.07901585847139359, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.476022720336914, "metrics/particle/momentum/class_1/assigned/samples": 14650.0, "metrics/particle/momentum/class_1/efficiency": 0.37365424633026123, "metrics/particle/momentum/class_1/event/energy_bias": 11.317193984985352, "metrics/particle/momentum/class_1/event/p3_error_mean": 42.661827087402344, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 28.36625099182129, "metrics/particle/momentum/class_1/event/px_bias": 3.095595121383667, "metrics/particle/momentum/class_1/event/py_bias": 9.83344841003418, "metrics/particle/momentum/class_1/event/pz_abs_mean": 26.099912643432617, "metrics/particle/momentum/class_1/event/pz_bias": -6.745318412780762, "metrics/particle/momentum/class_1/event/pz_residual_rms": 37.348289489746094, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.041519757360219955, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24705836176872253, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.24224162101745605, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08355452865362167, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2811652421951294, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029822926968336105, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6219525337219238, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0799926295876503, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6094720363616943, "metrics/particle/momentum/class_2/accepted/samples": 1073.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7599439024925232, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7735041975975037, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1816802024841309, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8156617283821106, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3359185755252838, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.2326226234436035, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05089675262570381, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5788021087646484, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.01539560779929161, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.944889068603516, "metrics/particle/momentum/class_2/assigned/samples": 2063.0, "metrics/particle/momentum/class_2/efficiency": 0.35934361815452576, "metrics/particle/momentum/class_2/event/energy_bias": -27.706104278564453, "metrics/particle/momentum/class_2/event/p3_error_mean": 36.31089401245117, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 18.95164680480957, "metrics/particle/momentum/class_2/event/px_bias": -1.2039746046066284, "metrics/particle/momentum/class_2/event/py_bias": -1.9805970191955566, "metrics/particle/momentum/class_2/event/pz_abs_mean": 26.323434829711914, "metrics/particle/momentum/class_2/event/pz_bias": -1.5991274118423462, "metrics/particle/momentum/class_2/event/pz_residual_rms": 40.43087387084961, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02321499213576317, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17909269034862518, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17734074592590332, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01703157275915146, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2182786762714386, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0036144577898085117, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.491958349943161, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.031925223767757416, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.4081988334655762, "metrics/particle/momentum/class_3/accepted/samples": 5810.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.9307210445404053, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.461559772491455, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.7220139503479, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.615982532501221, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.49893924593925476, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.785472869873047, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.022048795595765114, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.4862158298492432, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.06456504762172699, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.417891502380371, "metrics/particle/momentum/class_3/assigned/samples": 13198.0, "metrics/particle/momentum/class_3/efficiency": 0.3740423619747162, "metrics/particle/momentum/class_3/event/energy_bias": 32.04690170288086, "metrics/particle/momentum/class_3/event/p3_error_mean": 32.281105041503906, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 18.72406005859375, "metrics/particle/momentum/class_3/event/px_bias": -0.6973806619644165, "metrics/particle/momentum/class_3/event/py_bias": -0.921883761882782, "metrics/particle/momentum/class_3/event/pz_abs_mean": 22.181615829467773, "metrics/particle/momentum/class_3/event/pz_bias": 3.3301758766174316, "metrics/particle/momentum/class_3/event/pz_residual_rms": 36.919429779052734, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.039633333683013916, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.2463788539171219, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.23363865911960602, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.08994891494512558, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2679859399795532, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.036496348679065704, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 2.2962751388549805, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.3266395628452301, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.72336483001709, "metrics/particle/momentum/class_4/accepted/samples": 137.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.921720027923584, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 12.054741859436035, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 11.097558975219727, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 10.899778366088867, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.4883720874786377, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 34.612483978271484, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.049418605864048004, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.417450428009033, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.07523234188556671, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.28054666519165, "metrics/particle/momentum/class_4/assigned/samples": 344.0, "metrics/particle/momentum/class_4/efficiency": 0.36533331871032715, "metrics/particle/momentum/class_4/event/energy_bias": -6.539658546447754, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.854221343994141, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.774823188781738, "metrics/particle/momentum/class_4/event/px_bias": 0.6257258057594299, "metrics/particle/momentum/class_4/event/py_bias": -0.5165429711341858, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.601644277572632, "metrics/particle/momentum/class_4/event/pz_bias": -0.0370001345872879, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.958283424377441, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.04332936182618141, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2261100709438324, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2280241847038269, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.06971102952957153, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.26771363615989685, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.029411764815449715, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5316743850708008, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.4527447819709778, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.796447277069092, "metrics/particle/momentum/class_5/accepted/samples": 68.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.5230895280838013, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 2.5164475440979004, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 3.5398752689361572, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 2.967336654663086, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.4173228442668915, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 28.110300064086914, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.11023622006177902, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.061891078948975, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.2896714210510254, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 8.62219524383545, "metrics/particle/momentum/class_5/assigned/samples": 127.0, "metrics/particle/momentum/class_5/efficiency": 0.4146341383457184, "metrics/particle/momentum/class_5/event/energy_bias": -5.529701232910156, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.964344501495361, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.7691142559051514, "metrics/particle/momentum/class_5/event/px_bias": 0.08990992605686188, "metrics/particle/momentum/class_5/event/py_bias": -0.5735896825790405, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.5832419395446777, "metrics/particle/momentum/class_5/event/pz_bias": 0.5414551496505737, "metrics/particle/momentum/class_5/event/pz_residual_rms": 9.030928611755371, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.7631270289421082, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.7677819132804871, "metrics/set_slot/angular_recall_all_dr0p10": 0.8108103275299072, "metrics/set_slot/angular_recall_selected_dr0p10": 0.5046993494033813, "metrics/set_slot/soft_count_bias_mean": 0.3381604850292206}, "step_train_time": 5.077341556549072, "step_valid_time": 2246.567012310028, "step_total_time": 2251.6443543434143, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.2285583717366404, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.44204554515392336, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3175, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2439, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1862, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.797319385420072, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7681889763779528, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.782483156881617, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.23181102362204725, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6086956521739131, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.5864566929133859, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.5973692653192172, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.797319385420072, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1150488531229712, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3571894748816704, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2260, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1752, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1546, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7667396061269146, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7752212389380531, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7709570957095709, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2247787610619469, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6765864332603939, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.684070796460177, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6803080308030803, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7667396061269146, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0158187422677112, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.38133492375147393, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1251, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 934, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 815, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7065052950075643, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7466027178257394, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7260007773027595, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.2533972821742606, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6164901664145235, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6514788169464428, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6335017489312087, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7065052950075643} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_10000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..aa334fd02ae939278b5d35e71a93a6bceda15db6 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.47366426676771506, "train": {"Classification_binary": 0.32250267267227173, "Cardinality": 0.26476266980171204, "Classification": 0.1818925142288208, "Regression_pt": 0.30981555581092834, "Regression_eta": 0.003663061885163188, "Regression_sin_phi": 0.0019139533396810293, "Regression_cos_phi": 0.0019429113017395139, "Regression_energy": 0.3077285587787628, "HitGrouping": 0.24716420471668243, "Auxiliary": 0.4020433723926544, "Total": 2.0705742835998535, "Total_unweighted": 1.1294592280173674}, "train_group": {}, "valid": {"Classification_binary": 0.29654566446940106, "Cardinality": 0.21864751974741617, "Classification": 0.18283418814341226, "Regression_pt": 0.29241760571797687, "Regression_eta": 0.0036919855823119483, "Regression_sin_phi": 0.0018751515696446102, "Regression_cos_phi": 0.0018110059512158234, "Regression_energy": 0.2889363765716553, "HitGrouping": 0.2489048639933268, "Auxiliary": 0.3792925675710042, "Total": 1.9468620618184407, "Total_unweighted": 1.0681119780056179, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7470483779907227, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.30842748284339905, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.44809311628341675, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.29941946268081665, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.2936634421348572, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.603577196598053, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7190110683441162, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2707444429397583, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.40509602427482605, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.2629571855068207, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.25340279936790466, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6244460940361023, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7636227607727051, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.2890201508998871, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.4093496799468994, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.2816019058227539, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.2725561559200287, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5738486647605896, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7551379203796387, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.2934807538986206, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.42075687646865845, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.2855522632598877, "metrics/hit_grouping/calorimeter/all/object_purity": 0.2774636447429657, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.586783230304718, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.001016941387206316, "metrics/hit_grouping/calorimeter/background_recall": 0.0441988967359066, "metrics/hit_grouping/calorimeter/completeness": 0.3506433069705963, "metrics/hit_grouping/calorimeter/energy_completeness": 0.37846192717552185, "metrics/hit_grouping/calorimeter/energy_purity": 0.37317103147506714, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.556419312953949, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7846293449401855, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.31125253438949585, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3548589050769806, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3064238131046295, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.2985997498035431, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5228152275085449, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7438257336616516, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.307930052280426, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.46706587076187134, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.29715436697006226, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.29128703474998474, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6146937012672424, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7064017057418823, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.24268792569637299, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5084686875343323, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.23130732774734497, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.22044150531291962, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.689095139503479, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7222045063972473, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3165476620197296, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5241760611534119, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.30814358592033386, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2976686656475067, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6513110399246216, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7217407822608948, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4193495512008667, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6695831418037415, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.40384045243263245, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3915598392486572, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6479670405387878, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7926943898200989, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.25496706366539, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2856704890727997, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.24842660129070282, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2432703673839569, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5137040615081787, "metrics/hit_grouping/calorimeter/purity": 0.3447989523410797, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8989303112030029, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05161551386117935, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14473684132099152, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05369674414396286, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.016811992973089218, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8395366668701172, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.09955376386642456, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10712183266878128, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.09635532647371292, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.08233650028705597, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.33078280091285706, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7484029531478882, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.30751344561576843, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4415755569934845, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.2991757094860077, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.29186493158340454, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6092376708984375, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7076202034950256, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4571925103664398, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.37618857622146606, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.33105406165122986, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6317543387413025, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7631196975708008, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3580470085144043, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3626360595226288, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.29874441027641296, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4575045108795166, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6349075436592102, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45438191294670105, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.30584079027175903, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.27377110719680786, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7188084721565247, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6667259335517883, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4481300115585327, "metrics/hit_grouping/tracker/all/object_completeness": 0.3316571116447449, "metrics/hit_grouping/tracker/all/object_purity": 0.29322147369384766, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6727515459060669, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.014056543819606304, "metrics/hit_grouping/tracker/background_recall": 0.10041048377752304, "metrics/hit_grouping/tracker/completeness": 0.3373918831348419, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6207831501960754, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6744082570075989, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4452919661998749, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.38441750407218933, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.347537100315094, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6741188764572144, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6627288460731506, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4654202163219452, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3176177144050598, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.27591007947921753, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.670176088809967, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6548943519592285, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4367899000644684, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.23707780241966248, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.19869384169578552, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6724977493286133, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6927016377449036, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4894285500049591, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3601005971431732, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3055305480957031, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.643542468547821, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6920390129089355, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5314039587974548, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.34443163871765137, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.2804480195045471, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6231527328491211, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6284187436103821, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37951210141181946, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.29238951206207275, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.27950918674468994, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7196158766746521, "metrics/hit_grouping/tracker/purity": 0.3192509114742279, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8906481862068176, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.018099548295140266, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.014363904483616352, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.009076636284589767, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06447964161634445, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6697497367858887, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.21270005404949188, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1657760590314865, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.12498827278614044, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5745999217033386, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6331687569618225, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5374629497528076, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.3967219293117523, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.3536221385002136, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7736556529998779, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.40809229016304016, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5094788670539856, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5714470744132996, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5932630300521851, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.14106044173240662, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.01878821663558483, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.803125381469727, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.25165408849716187, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16003093123435974, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6716252565383911, "metrics/particle/class_1/pid_accuracy": 0.9258859753608704, "metrics/particle/class_2/efficiency": 0.370710164308548, "metrics/particle/class_2/pid_accuracy": 0.32630613446235657, "metrics/particle/class_3/efficiency": 0.5157510638237, "metrics/particle/class_3/pid_accuracy": 0.9136995673179626, "metrics/particle/class_4/efficiency": 0.5711422562599182, "metrics/particle/class_4/pid_accuracy": 0.23508772253990173, "metrics/particle/class_5/efficiency": 0.7375478744506836, "metrics/particle/class_5/pid_accuracy": 0.2519480586051941, "metrics/particle/count/bias_mean": -8.572916984558105, "metrics/particle/count/mae": 9.682291984558105, "metrics/particle/count/prediction_mean": 62.08203125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.1104779839515686, "metrics/particle/event/energy_response_mean": 1.0007684230804443, "metrics/particle/event/met_abs_error": 10.7738037109375, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09721794724464417, "metrics/particle/event/scalar_pt_response_mean": 1.021333932876587, "metrics/particle/event/vector_pt_closure": 0.08811697363853455, "metrics/particle/matched/delta_eta_abs_mean": 0.02192629687488079, "metrics/particle/matched/delta_phi_abs_mean": 0.018293937668204308, "metrics/particle/matched/delta_r_mean": 0.03193940594792366, "metrics/particle/matched/energy_relative_abs_mean": 0.18859440088272095, "metrics/particle/matched/pid_accuracy": 0.8798305988311768, "metrics/particle/matched/pt_relative_abs_mean": 0.1872674971818924, "metrics/particle/matching/duplicate_fraction": 0.10772038251161575, "metrics/particle/matching/efficiency": 0.5808377861976624, "metrics/particle/matching/f1": 0.6183516383171082, "metrics/particle/matching/purity": 0.6610457301139832, "metrics/particle/matching/target_coverage_dr0p05": 0.5179957151412964, "metrics/particle/matching/target_coverage_dr0p10": 0.6623297929763794, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.36615189909935, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6373864412307739, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03143339604139328, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.19063019752502441, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.19029881060123444, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.007165136281400919, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2313755303621292, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.015898767858743668, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.1345560550689697, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.22623328864574432, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.73260760307312, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6164.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.41342613101005554, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.7186472415924072, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.9275165796279907, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.7341172695159912, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.2452356517314911, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.821206092834473, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.030126668512821198, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.7057414054870605, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.1449108123779297, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.294970989227295, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8763.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5722242593765259, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.995612144470215, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.94024658203125, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.961032390594482, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.27588796615600586, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.491891086101532, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.942051887512207, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.8207170963287354, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.793949127197266, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03168446943163872, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.19290600717067719, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1903448849916458, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.014685206115245819, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.23152370750904083, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.042241379618644714, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.925560712814331, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.35071036219596863, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.392380714416504, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1160.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6445683836936951, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.017650842666626, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.439927577972412, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.240418434143066, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3224773108959198, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 39.470497131347656, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0827549397945404, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.4362752437591553, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.31254512071609497, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.144826889038086, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1873.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.42059463262557983, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.966020584106445, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.401395797729492, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.761311650276184, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.012178860604763031, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.18510709702968597, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.187773704528809, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.365928888320923, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 27.58580207824707, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03230773285031319, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.18872569501399994, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.18725289404392242, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.011901384219527245, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.22792135179042816, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0011422859970480204, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3908992111682892, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.07989523559808731, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0988202095031738, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 14007.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.46287521719932556, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.941911458969116, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.412247657775879, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.2157692909240723, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.27373388409614563, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 23.44828224182129, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.011306297965347767, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.877403974533081, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.06682232767343521, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.772632122039795, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21404.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6019597053527832, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.076011657714844, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 15.881128311157227, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.990439414978027, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.8121575713157654, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.06438323855400085, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.517799377441406, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -2.3407185077667236, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 12.126136779785156, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0320211797952652, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.18950337171554565, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.188301220536232, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.010684140957891941, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2291213572025299, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007641460746526718, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6892493963241577, "metrics/particle/momentum/all/accepted/pz_bias": -0.1369095891714096, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.999947190284729, "metrics/particle/momentum/all/accepted/samples": 21331.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.45997223258018494, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.5533156394958496, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.456665515899658, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.260852098464966, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.26878902316093445, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 22.284423828125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.02063046209514141, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.2535426616668701, "metrics/particle/momentum/all/assigned/pz_bias": -0.10254418104887009, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.0245680809021, "metrics/particle/momentum/all/assigned/samples": 32040.0, "metrics/particle/momentum/all/efficiency": 0.5796625018119812, "metrics/particle/momentum/all/event/energy_bias": 4.105603218078613, "metrics/particle/momentum/all/event/p3_error_mean": 29.268463134765625, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.33664321899414, "metrics/particle/momentum/all/event/px_bias": 1.07586669921875, "metrics/particle/momentum/all/event/py_bias": 0.24240078032016754, "metrics/particle/momentum/all/event/pz_abs_mean": 21.072025299072266, "metrics/particle/momentum/all/event/pz_bias": -7.527364253997803, "metrics/particle/momentum/all/event/pz_residual_rms": 32.66856002807617, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.037493620067834854, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1989910900592804, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.19945861399173737, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.022141994908452034, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2388991117477417, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0073535628616809845, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7609105110168457, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.204293891787529, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.0092568397521973, "metrics/particle/momentum/class_1/accepted/samples": 11831.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.3024391233921051, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.1293540000915527, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.2427111864089966, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.0280314683914185, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.2149515450000763, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.738398551940918, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.014779835939407349, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1083225011825562, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.2047879695892334, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.4477415084838867, "metrics/particle/momentum/class_1/assigned/samples": 16306.0, "metrics/particle/momentum/class_1/efficiency": 0.6668733358383179, "metrics/particle/momentum/class_1/event/energy_bias": 21.931713104248047, "metrics/particle/momentum/class_1/event/p3_error_mean": 27.190885543823242, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 14.74790096282959, "metrics/particle/momentum/class_1/event/px_bias": 0.7779841423034668, "metrics/particle/momentum/class_1/event/py_bias": -0.08637672662734985, "metrics/particle/momentum/class_1/event/pz_abs_mean": 19.69095230102539, "metrics/particle/momentum/class_1/event/pz_bias": -11.150629997253418, "metrics/particle/momentum/class_1/event/pz_residual_rms": 28.547861099243164, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04179506003856659, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22014795243740082, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22024594247341156, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05886410176753998, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2594629228115082, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.043478261679410934, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9141207933425903, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.3193582594394684, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.262784004211426, "metrics/particle/momentum/class_2/accepted/samples": 1127.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5860556960105896, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8389447927474976, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1889714002609253, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8287040591239929, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3002038598060608, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 7.741048812866211, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06371049582958221, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7471938133239746, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.15775172412395477, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.443849086761475, "metrics/particle/momentum/class_2/assigned/samples": 1962.0, "metrics/particle/momentum/class_2/efficiency": 0.37742799520492554, "metrics/particle/momentum/class_2/event/energy_bias": -19.293170928955078, "metrics/particle/momentum/class_2/event/p3_error_mean": 25.59161376953125, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 13.404841423034668, "metrics/particle/momentum/class_2/event/px_bias": 0.018155083060264587, "metrics/particle/momentum/class_2/event/py_bias": 0.24916675686836243, "metrics/particle/momentum/class_2/event/pz_abs_mean": 18.528656005859375, "metrics/particle/momentum/class_2/event/pz_bias": 2.753117799758911, "metrics/particle/momentum/class_2/event/pz_residual_rms": 29.332447052001953, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02256491780281067, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1707848459482193, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16693955659866333, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01362165343016386, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20852795243263245, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0023623027373105288, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3771240711212158, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.010715887881815434, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2887468338012695, "metrics/particle/momentum/class_3/accepted/samples": 8043.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.6390440464019775, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.460308074951172, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.042004108428955, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.894096612930298, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.32959675788879395, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 32.68861389160156, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.021140536293387413, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.1925116777420044, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03984491899609566, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.217758655548096, "metrics/particle/momentum/class_3/assigned/samples": 13292.0, "metrics/particle/momentum/class_3/efficiency": 0.5178008079528809, "metrics/particle/momentum/class_3/event/energy_bias": 3.69620418548584, "metrics/particle/momentum/class_3/event/p3_error_mean": 17.170652389526367, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 10.25147819519043, "metrics/particle/momentum/class_3/event/px_bias": 0.04636644199490547, "metrics/particle/momentum/class_3/event/py_bias": 0.4917064607143402, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.643043518066406, "metrics/particle/momentum/class_3/event/pz_bias": 1.4941511154174805, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.574960708618164, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03315321356058121, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.19930848479270935, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.19968092441558838, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0028936839662492275, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.24091936647891998, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.009708737954497337, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.141829013824463, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.17684632539749146, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.368403911590576, "metrics/particle/momentum/class_4/accepted/samples": 206.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.45058122277259827, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 7.136924743652344, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 6.879257678985596, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 6.711531162261963, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.33128833770751953, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 32.28616714477539, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.012269938364624977, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.281523585319519, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.09922073036432266, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.6493747234344482, "metrics/particle/momentum/class_4/assigned/samples": 326.0, "metrics/particle/momentum/class_4/efficiency": 0.5493333339691162, "metrics/particle/momentum/class_4/event/energy_bias": -0.7070455551147461, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.208687782287598, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.1812257766723633, "metrics/particle/momentum/class_4/event/px_bias": 0.31290391087532043, "metrics/particle/momentum/class_4/event/py_bias": 0.030178707093000412, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.2694687843322754, "metrics/particle/momentum/class_4/event/pz_bias": -0.20979700982570648, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.820746421813965, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.032536569982767105, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.20359386503696442, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.20009908080101013, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.07868274301290512, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.23773391544818878, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04838709533214569, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.212965488433838, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.1684068888425827, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.546746253967285, "metrics/particle/momentum/class_5/accepted/samples": 124.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09762530773878098, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.8696220517158508, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9493011832237244, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6520381569862366, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.18831168115139008, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.867215633392334, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06493506580591202, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.808894634246826, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.8701732754707336, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.648731708526611, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.7560975551605225, "metrics/particle/momentum/class_5/event/energy_bias": -1.522099494934082, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.304147481918335, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.5087790489196777, "metrics/particle/momentum/class_5/event/px_bias": -0.07954281568527222, "metrics/particle/momentum/class_5/event/py_bias": -0.44227439165115356, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.703321933746338, "metrics/particle/momentum/class_5/event/pz_bias": -0.41420605778694153, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.240097999572754, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5943505764007568, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5975061655044556, "metrics/set_slot/angular_recall_all_dr0p10": 0.8618948459625244, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6786760687828064, "metrics/set_slot/soft_count_bias_mean": 0.6280924677848816}, "step_train_time": 6.477054119110107, "step_valid_time": 25348.384278535843, "step_total_time": 25354.86133289337, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0659839224739631, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2491232509960929, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3013, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2742, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2635, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8963713631905852, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9100564221705941, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9031620553359684, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0899435778294059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8613926119646943, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8745436442084301, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8679183135704874, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8963713631905852, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0290911669809868, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2232254380014712, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2228, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2053, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2022, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8984682713347921, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9214542190305206, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9098160868601817, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07854578096947935, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8849015317286653, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.90754039497307, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8960779968978506, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8984682713347921, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.013340675140845, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2223416242292845, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1266, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1142, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1118, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8638426626323752, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9020537124802528, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8825347758887172, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09794628751974724, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8456883509833586, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8830963665086888, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8639876352395672, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8638426626323752} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_11000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..b3c768227061ae6e4c07536d6ba129fa18f7ade7 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 0.5210306934444866, "train": {"Classification_binary": 0.3087218403816223, "Cardinality": 0.2077820599079132, "Classification": 0.1806342899799347, "Regression_pt": 0.27576538920402527, "Regression_eta": 0.0036648318637162447, "Regression_sin_phi": 0.001724114059470594, "Regression_cos_phi": 0.002081784186884761, "Regression_energy": 0.27401402592658997, "HitGrouping": 0.24673613905906677, "Auxiliary": 0.3721325099468231, "Total": 1.9315295219421387, "Total_unweighted": 1.0466062756022438}, "train_group": {}, "valid": {"Classification_binary": 0.2755904197692871, "Cardinality": 0.2052040696144104, "Classification": 0.16161165634791055, "Regression_pt": 0.2668023109436035, "Regression_eta": 0.0030208835378289223, "Regression_sin_phi": 0.0015045444791515668, "Regression_cos_phi": 0.0015174755826592445, "Regression_energy": 0.2650829553604126, "HitGrouping": 0.24367193380991617, "Auxiliary": 0.35162969430287677, "Total": 1.768771489461263, "Total_unweighted": 0.9751302460208535, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7602211236953735, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3291371464729309, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4356014132499695, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.32213541865348816, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3164840042591095, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5702659487724304, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7155498266220093, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.28638845682144165, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3862629234790802, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.27825015783309937, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.27943482995033264, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6322008967399597, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7783944010734558, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.30631545186042786, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.40323999524116516, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.30155277252197266, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.292372465133667, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5381624698638916, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7680096626281738, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.31168225407600403, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.4117259085178375, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3059791922569275, "metrics/hit_grouping/calorimeter/all/object_purity": 0.2986726760864258, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5551891922950745, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0019457346061244607, "metrics/hit_grouping/calorimeter/background_recall": 0.08597361296415329, "metrics/hit_grouping/calorimeter/completeness": 0.36679965257644653, "metrics/hit_grouping/calorimeter/energy_completeness": 0.39108455181121826, "metrics/hit_grouping/calorimeter/energy_purity": 0.38578271865844727, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5299988389015198, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7948102355003357, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3346153795719147, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3441624939441681, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.33180364966392517, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3231748044490814, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4985775947570801, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7560184001922607, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3239804208278656, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4565868377685547, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3149891495704651, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.31587132811546326, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5829110741615295, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7254440784454346, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2525326609611511, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.504292368888855, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.24424681067466736, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.23138706386089325, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.642691433429718, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7355729341506958, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.33301058411598206, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5166586637496948, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3270944058895111, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.32026922702789307, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.61787348985672, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7428092360496521, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.433803915977478, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6793618202209473, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.42028680443763733, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3982822000980377, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5928975939750671, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8040831685066223, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.27538764476776123, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2728554904460907, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.27083271741867065, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.26478657126426697, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.486786425113678, "metrics/hit_grouping/calorimeter/purity": 0.36102166771888733, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9005475044250488, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05896848812699318, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13533835113048553, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.0588032603263855, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.019470494240522385, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.020676691085100174, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8403488397598267, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.10436822474002838, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1112419068813324, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.10417249798774719, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.08790767937898636, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.31135961413383484, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7620855569839478, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3265884220600128, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.43186619877815247, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3205084800720215, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3141900300979614, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5765873193740845, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7110618352890015, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.46037954092025757, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.39619535207748413, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.3674566149711609, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6188613772392273, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7584437727928162, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3273056149482727, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.377478688955307, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.3415399193763733, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4472573697566986, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6449673175811768, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4584832489490509, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.33235281705856323, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.30307644605636597, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6970067620277405, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6736631393432617, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4493766725063324, "metrics/hit_grouping/tracker/all/object_completeness": 0.35530778765678406, "metrics/hit_grouping/tracker/all/object_purity": 0.32570457458496094, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6545414328575134, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.025100592523813248, "metrics/hit_grouping/tracker/background_recall": 0.1450110524892807, "metrics/hit_grouping/tracker/completeness": 0.35992440581321716, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5840821862220764, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6837661266326904, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4461686909198761, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4071444571018219, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3727605640888214, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6533403396606445, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6656218767166138, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4657832682132721, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.34180405735969543, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.31842100620269775, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6667271852493286, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6608705520629883, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43967539072036743, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.26209646463394165, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.23614725470542908, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6449053287506104, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6965929865837097, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.48467352986335754, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3857095241546631, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.34724000096321106, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6285980939865112, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7033735513687134, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4963054060935974, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3682582974433899, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3389686346054077, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5966748595237732, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6385273933410645, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.395076721906662, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.31346291303634644, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.29532989859580994, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6986422538757324, "metrics/hit_grouping/tracker/purity": 0.34443002939224243, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8810128569602966, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01998491771519184, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.016781583428382874, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.013295166194438934, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6577625870704651, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2085699588060379, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.17387662827968597, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.1509590595960617, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5735673904418945, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6446165442466736, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.539197564125061, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4252089560031891, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.39100542664527893, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7511051297187805, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.3943397104740143, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7562499046325684, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5166156888008118, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.582301139831543, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5793178081512451, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.15481308102607727, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.004843013361096382, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.855390548706055, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.25008276104927063, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1584596037864685, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7030171155929565, "metrics/particle/class_1/pid_accuracy": 0.9302805066108704, "metrics/particle/class_2/efficiency": 0.3836221396923065, "metrics/particle/class_2/pid_accuracy": 0.4080307185649872, "metrics/particle/class_3/efficiency": 0.5602605938911438, "metrics/particle/class_3/pid_accuracy": 0.9255111813545227, "metrics/particle/class_4/efficiency": 0.5871743559837341, "metrics/particle/class_4/pid_accuracy": 0.18430034816265106, "metrics/particle/class_5/efficiency": 0.7796934843063354, "metrics/particle/class_5/pid_accuracy": 0.2702702581882477, "metrics/particle/count/bias_mean": -7.186849117279053, "metrics/particle/count/mae": 8.672526359558105, "metrics/particle/count/prediction_mean": 63.46809768676758, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09882599115371704, "metrics/particle/event/energy_response_mean": 0.967779815196991, "metrics/particle/event/met_abs_error": 9.677370071411133, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08246458321809769, "metrics/particle/event/scalar_pt_response_mean": 0.9996193051338196, "metrics/particle/event/vector_pt_closure": 0.07958034425973892, "metrics/particle/matched/delta_eta_abs_mean": 0.01896180398762226, "metrics/particle/matched/delta_phi_abs_mean": 0.016989117488265038, "metrics/particle/matched/delta_r_mean": 0.028454197570681572, "metrics/particle/matched/energy_relative_abs_mean": 0.18134787678718567, "metrics/particle/matched/pid_accuracy": 0.8914461135864258, "metrics/particle/matched/pt_relative_abs_mean": 0.18031682074069977, "metrics/particle/matching/duplicate_fraction": 0.10295731574296951, "metrics/particle/matching/efficiency": 0.6161656975746155, "metrics/particle/matching/f1": 0.6491823196411133, "metrics/particle/matching/purity": 0.6859375834465027, "metrics/particle/matching/target_coverage_dr0p05": 0.5700477361679077, "metrics/particle/matching/target_coverage_dr0p10": 0.696211040019989, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.399185448884964, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6685771346092224, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02660173922777176, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1815630942583084, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.18153730034828186, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009059729054570198, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.22262504696846008, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.015955021604895592, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.0749613046646118, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0077424258925020695, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.7078003883361816, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6581.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3629915714263916, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.6788372993469238, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.0115926265716553, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.8401775360107422, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.22309653460979462, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.451114654541016, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02859359420835972, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5437579154968262, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03126917779445648, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.88266921043396, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9023.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6109357476234436, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.7039275169372559, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.503096580505371, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.285467624664307, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.22876709699630737, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.383146196603775, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.774476051330566, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.2434263527393341, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.650348663330078, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027478860691189766, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1900932490825653, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.19232936203479767, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02182457596063614, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.233938530087471, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04728132486343384, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.0734975337982178, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.05393574386835098, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.1305108070373535, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1269.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5724630355834961, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6282917261123657, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.69646692276001, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.516750335693359, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.29738059639930725, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 46.24271011352539, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08577298372983932, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.287978410720825, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.07206563651561737, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.920856475830078, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1947.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.46011602878570557, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.154654026031494, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.302324295043945, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5836175680160522, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.07183924317359924, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.17238067090511322, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.10911750793457, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.5821636915206909, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.05384635925293, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.029537932947278023, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.18183380365371704, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.18038377165794373, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.003725223010405898, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.22009995579719543, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0010840842733159661, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3599773943424225, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.024149268865585327, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9775066375732422, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 14759.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.4187757074832916, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.459091901779175, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.9647595882415771, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.7794809341430664, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.2549751102924347, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.588492393493652, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00962778739631176, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7805870771408081, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03436388820409775, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.5418412685394287, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21708.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.63427734375, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.0247589349746704, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.335248947143555, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 11.366697311401367, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.5384887456893921, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.031987011432647705, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.714913368225098, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.8162657022476196, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.117152214050293, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02856769785284996, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.18221858143806458, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.18139001727104187, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.001430266653187573, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.22163496911525726, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.008005661889910698, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6642705798149109, "metrics/particle/momentum/all/accepted/pz_bias": 0.01048351265490055, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.058011293411255, "metrics/particle/momentum/all/accepted/samples": 22609.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.4125296175479889, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.1941487789154053, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.1404497623443604, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.9593305587768555, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.24869942665100098, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 16.56146812438965, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01940143294632435, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1407065391540527, "metrics/particle/momentum/all/assigned/pz_bias": 0.035755705088377, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.6326465606689453, "metrics/particle/momentum/all/assigned/samples": 32678.0, "metrics/particle/momentum/all/efficiency": 0.6143916845321655, "metrics/particle/momentum/all/event/energy_bias": -5.833823204040527, "metrics/particle/momentum/all/event/p3_error_mean": 25.463991165161133, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.589363098144531, "metrics/particle/momentum/all/event/px_bias": 0.8390949964523315, "metrics/particle/momentum/all/event/py_bias": -0.5875139832496643, "metrics/particle/momentum/all/event/pz_abs_mean": 18.044048309326172, "metrics/particle/momentum/all/event/pz_bias": -0.47752845287323, "metrics/particle/momentum/all/event/pz_residual_rms": 28.95587921142578, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03192344680428505, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.18717217445373535, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1872604936361313, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.010345504619181156, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.22704645991325378, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.008066467940807343, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7146798372268677, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.02329765446484089, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.07179594039917, "metrics/particle/momentum/class_1/accepted/samples": 12397.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2601628601551056, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9834227561950684, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.1748075485229492, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.9968726634979248, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.19809778034687042, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.56724739074707, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.014902768656611443, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.0068751573562622, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.06411368399858475, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.1852450370788574, "metrics/particle/momentum/class_1/assigned/samples": 16507.0, "metrics/particle/momentum/class_1/efficiency": 0.6987768411636353, "metrics/particle/momentum/class_1/event/energy_bias": 8.87373161315918, "metrics/particle/momentum/class_1/event/p3_error_mean": 23.306869506835938, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 13.391478538513184, "metrics/particle/momentum/class_1/event/px_bias": -0.19069011509418488, "metrics/particle/momentum/class_1/event/py_bias": -0.1720310002565384, "metrics/particle/momentum/class_1/event/pz_abs_mean": 16.193910598754883, "metrics/particle/momentum/class_1/event/pz_bias": 4.300797462463379, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.759801864624023, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04035491868853569, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2211448699235916, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.223353311419487, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06621259450912476, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26176026463508606, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04396551847457886, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9996557235717773, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.17085489630699158, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.52242374420166, "metrics/particle/momentum/class_2/accepted/samples": 1160.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6060017943382263, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.789851188659668, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0421173572540283, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6702402830123901, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30497512221336365, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.715575218200684, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06417910754680634, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7416138648986816, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.13684022426605225, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.350769519805908, "metrics/particle/momentum/class_2/assigned/samples": 2010.0, "metrics/particle/momentum/class_2/efficiency": 0.3884795606136322, "metrics/particle/momentum/class_2/event/energy_bias": -14.770587921142578, "metrics/particle/momentum/class_2/event/p3_error_mean": 24.47833251953125, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 13.159438133239746, "metrics/particle/momentum/class_2/event/px_bias": -1.0336799621582031, "metrics/particle/momentum/class_2/event/py_bias": -0.608508825302124, "metrics/particle/momentum/class_2/event/pz_abs_mean": 17.57467269897461, "metrics/particle/momentum/class_2/event/pz_bias": -2.6378750801086426, "metrics/particle/momentum/class_2/event/pz_residual_rms": 27.370325088500977, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022223392501473427, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1697845757007599, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16733448207378387, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.000557230319827795, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20735515654087067, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0022943674121052027, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3821179270744324, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0042183795012533665, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2940113544464111, "metrics/particle/momentum/class_3/accepted/samples": 8717.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.569342851638794, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.771137237548828, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.3992972373962402, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.243347644805908, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.30027809739112854, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.434127807617188, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.018003512173891068, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.0539947748184204, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.018986230716109276, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.6009297370910645, "metrics/particle/momentum/class_3/assigned/samples": 13664.0, "metrics/particle/momentum/class_3/efficiency": 0.5611922740936279, "metrics/particle/momentum/class_3/event/energy_bias": 5.546422004699707, "metrics/particle/momentum/class_3/event/p3_error_mean": 17.60734748840332, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 10.901583671569824, "metrics/particle/momentum/class_3/event/px_bias": 1.8474639654159546, "metrics/particle/momentum/class_3/event/py_bias": 0.07058489322662354, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.562768936157227, "metrics/particle/momentum/class_3/event/pz_bias": -2.271085262298584, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.074968338012695, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03168756514787674, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1880980134010315, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18546155095100403, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.025152452290058136, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22880683839321136, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004807692486792803, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0632059574127197, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.0212569460272789, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0801048278808594, "metrics/particle/momentum/class_4/accepted/samples": 208.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4925510883331299, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.584938049316406, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.5133771896362305, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.341032028198242, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3333333432674408, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 23.92416763305664, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005899704992771149, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0541185140609741, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.06232904642820358, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.0270862579345703, "metrics/particle/momentum/class_4/assigned/samples": 339.0, "metrics/particle/momentum/class_4/efficiency": 0.5546666383743286, "metrics/particle/momentum/class_4/event/energy_bias": -3.5191359519958496, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.801208972930908, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.970297336578369, "metrics/particle/momentum/class_4/event/px_bias": 0.14471107721328735, "metrics/particle/momentum/class_4/event/py_bias": 0.16661621630191803, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8292458057403564, "metrics/particle/momentum/class_4/event/pz_bias": -0.11946950852870941, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.122228622436523, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.023685667663812637, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18694457411766052, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18313615024089813, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.07102850079536438, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22388656437397003, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.07086614519357681, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.2593681812286377, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.8979702591896057, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.116966247558594, "metrics/particle/momentum/class_5/accepted/samples": 127.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.1366751492023468, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.7488757967948914, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.8948490619659424, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6122439503669739, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.17721518874168396, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.048343181610107, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06962025165557861, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.4414174556732178, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.9294354319572449, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.099365711212158, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.7743902206420898, "metrics/particle/momentum/class_5/event/energy_bias": -1.9642527103424072, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.852203369140625, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.1847422122955322, "metrics/particle/momentum/class_5/event/px_bias": 0.07129009813070297, "metrics/particle/momentum/class_5/event/py_bias": -0.04417519271373749, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.423921823501587, "metrics/particle/momentum/class_5/event/pz_bias": 0.25010374188423157, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.535617828369141, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5320529341697693, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5355451107025146, "metrics/set_slot/angular_recall_all_dr0p10": 0.8893076181411743, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7063745260238647, "metrics/set_slot/soft_count_bias_mean": 1.133851170539856}, "step_train_time": 4.889329671859741, "step_valid_time": 27974.419896364212, "step_total_time": 27979.30922627449, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.037533366606553, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21549035834849706, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3019, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2780, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2703, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9087937234390323, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9208347134812852, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9147745969068772, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07916528651871481, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8836220987250736, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8953295793309043, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8894373149062191, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9087937234390323, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.008021750160589, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20162910454128236, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2258, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2075, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2055, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9080962800875274, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9189548272807795, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9134932863746423, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08104517271922054, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.899343544857768, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9100974313551816, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9046885318071759, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9080962800875274, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0016075379881477, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19487288801557345, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1154, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1138, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8729198184568835, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9022673964034402, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8873510188389081, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09773260359655982, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8608169440242057, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8897576231430805, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8750480584390619, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8729198184568835} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_12000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..cb3a95841d2144dc4a72ffd843b73e8dcbc2d8ee --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 0.5683971201212581, "train": {"Classification_binary": 0.24587731063365936, "Cardinality": 0.1972426027059555, "Classification": 0.14806142449378967, "Regression_pt": 0.23849882185459137, "Regression_eta": 0.0025373119860887527, "Regression_sin_phi": 0.0013242254499346018, "Regression_cos_phi": 0.0013636662624776363, "Regression_energy": 0.2329653650522232, "HitGrouping": 0.22862780094146729, "Auxiliary": 0.31810688972473145, "Total": 1.603585124015808, "Total_unweighted": 0.8706281257327646}, "train_group": {}, "valid": {"Classification_binary": 0.26637468735376996, "Cardinality": 0.19786673784255981, "Classification": 0.14394261439641318, "Regression_pt": 0.24099657932917276, "Regression_eta": 0.002674512875576814, "Regression_sin_phi": 0.0013920677204926808, "Regression_cos_phi": 0.0013481265244384606, "Regression_energy": 0.23748471339543661, "HitGrouping": 0.2331110636393229, "Auxiliary": 0.3278493682543437, "Total": 1.648034890492757, "Total_unweighted": 0.8942133015953004, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7727153301239014, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3573843240737915, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4247184693813324, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3528890311717987, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3476855754852295, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5405507683753967, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7275696396827698, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.31690919399261475, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3818316161632538, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3124816119670868, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.31803053617477417, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5941654443740845, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7943547964096069, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3370375633239746, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3903726041316986, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.33312439918518066, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.31478777527809143, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5033556818962097, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7826140522956848, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.34163898229599, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.40011468529701233, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.33750948309898376, "metrics/hit_grouping/calorimeter/all/object_purity": 0.32500600814819336, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5216743350028992, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0023294531274586916, "metrics/hit_grouping/calorimeter/background_recall": 0.09994080662727356, "metrics/hit_grouping/calorimeter/completeness": 0.4066028296947479, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4339209496974945, "metrics/hit_grouping/calorimeter/energy_purity": 0.42809608578681946, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5099238157272339, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8060080409049988, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3625012934207916, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.32948338985443115, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.36091840267181396, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.351082444190979, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.46711423993110657, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.77588951587677, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3566415011882782, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.451404869556427, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3508175015449524, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3421759009361267, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5339705348014832, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7416885495185852, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2839871644973755, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.49245938658714294, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.27637240290641785, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.254539430141449, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6205336451530457, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7576079964637756, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.37007224559783936, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5049915909767151, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.3661178946495056, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.35249969363212585, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5758359432220459, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7675104141235352, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4876171052455902, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6582604050636292, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.47508928179740906, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.42199018597602844, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.548121452331543, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8099085688591003, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.29525822401046753, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.262431800365448, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2919507920742035, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2854202389717102, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.46330246329307556, "metrics/hit_grouping/calorimeter/purity": 0.40035176277160645, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8858641982078552, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06360270082950592, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12781955301761627, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06387396901845932, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.025251206010580063, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04511278122663498, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8278462886810303, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.11885613948106766, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09829311072826385, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.11769561469554901, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.10795772820711136, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3537375032901764, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7785778045654297, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3577626645565033, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.42025789618492126, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3534070551395416, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.34118518233299255, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5381788015365601, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7199491858482361, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44386497139930725, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.40918511152267456, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.380621075630188, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6032159924507141, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7606887817382812, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.34237492084503174, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.40035584568977356, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.35723453760147095, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4581072926521301, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.648684561252594, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4509281814098358, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.33432668447494507, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3049449324607849, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6985897421836853, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6788606643676758, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44073909521102905, "metrics/hit_grouping/tracker/all/object_completeness": 0.3622113764286041, "metrics/hit_grouping/tracker/all/object_purity": 0.33206602931022644, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6515138149261475, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.015126005746424198, "metrics/hit_grouping/tracker/background_recall": 0.11323808133602142, "metrics/hit_grouping/tracker/completeness": 0.3659966289997101, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5706299543380737, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6865184307098389, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.43433281779289246, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4092043936252594, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.37916451692581177, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6502717733383179, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6733204126358032, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4590669870376587, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3508693277835846, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3230503499507904, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6596478223800659, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6686129570007324, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43570783734321594, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2768157422542572, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2441422939300537, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6459873914718628, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7046516537666321, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4833149313926697, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.39901480078697205, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.35614603757858276, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6220599412918091, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7112129330635071, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5270935893058777, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3939671516418457, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.34421175718307495, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5935960412025452, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6395317316055298, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3699083924293518, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.30867281556129456, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.298583984375, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7001876831054688, "metrics/hit_grouping/tracker/purity": 0.3466936945915222, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8826946020126343, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.020361991599202156, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.02544489875435829, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.022649012506008148, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0682503804564476, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6673145294189453, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.23644810914993286, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.20716696977615356, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.16879072785377502, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5715023279190063, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6498637795448303, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5252643823623657, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4289913773536682, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.39567962288856506, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7467405200004578, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4493423402309418, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.49098992347717285, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5479090213775635, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5923895835876465, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.09981036186218262, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0179147832095623, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.543701171875, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2663875222206116, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17476435005664825, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7109127640724182, "metrics/particle/class_1/pid_accuracy": 0.9380030035972595, "metrics/particle/class_2/efficiency": 0.39528825879096985, "metrics/particle/class_2/pid_accuracy": 0.4590258002281189, "metrics/particle/class_3/efficiency": 0.5810288190841675, "metrics/particle/class_3/pid_accuracy": 0.9319738149642944, "metrics/particle/class_4/efficiency": 0.5991984009742737, "metrics/particle/class_4/pid_accuracy": 0.21237458288669586, "metrics/particle/class_5/efficiency": 0.7796934843063354, "metrics/particle/class_5/pid_accuracy": 0.27272728085517883, "metrics/particle/count/bias_mean": -6.25, "metrics/particle/count/mae": 7.983072757720947, "metrics/particle/count/prediction_mean": 64.40494537353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.10144555568695068, "metrics/particle/event/energy_response_mean": 0.9776430726051331, "metrics/particle/event/met_abs_error": 12.138083457946777, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09121119976043701, "metrics/particle/event/scalar_pt_response_mean": 1.0266270637512207, "metrics/particle/event/vector_pt_closure": 0.09345653653144836, "metrics/particle/matched/delta_eta_abs_mean": 0.01729450933635235, "metrics/particle/matched/delta_phi_abs_mean": 0.016453281044960022, "metrics/particle/matched/delta_r_mean": 0.026694413274526596, "metrics/particle/matched/energy_relative_abs_mean": 0.1768854856491089, "metrics/particle/matched/pid_accuracy": 0.9008910059928894, "metrics/particle/matched/pt_relative_abs_mean": 0.17983825504779816, "metrics/particle/matching/duplicate_fraction": 0.10483593493700027, "metrics/particle/matching/efficiency": 0.6297937631607056, "metrics/particle/matching/f1": 0.6589379906654358, "metrics/particle/matching/purity": 0.6909103989601135, "metrics/particle/matching/target_coverage_dr0p05": 0.5961889028549194, "metrics/particle/matching/target_coverage_dr0p10": 0.7150912880897522, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4077363908290863, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6789064407348633, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02501150593161583, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17493455111980438, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.180348739027977, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.026506977155804634, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.22191818058490753, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01448193471878767, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.0356309413909912, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0006455015973187983, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.6524691581726074, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6698.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.37265411019325256, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.634084701538086, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.131242513656616, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.9702725410461426, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.23912577331066132, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.35161018371582, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.026248125359416008, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5237566232681274, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04831041395664215, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.7886791229248047, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9334.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6217972636222839, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.4889146685600281, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.041973114013672, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.707869529724121, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.8332650065422058, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.174979567527771, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.142963409423828, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.965578556060791, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.987885475158691, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02746572718024254, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17834192514419556, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18977563083171844, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.044394154101610184, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22931940853595734, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0352664589881897, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.8458539247512817, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.032365549355745316, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.263471603393555, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1276.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5925253033638, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.2937896251678467, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.638404846191406, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.495492935180664, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3052472174167633, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 60.48240661621094, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07366296648979187, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2060563564300537, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.0007573424954898655, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.257816314697266, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1982.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.46265408396720886, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.374765396118164, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.054938316345215, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.6665955781936646, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0036881789565086365, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07054001837968826, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.867175102233887, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.0994250774383545, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.464570999145508, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.027381746098399162, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.17672200500965118, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.17728911340236664, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010487276129424572, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21785882115364075, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0013837638543918729, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.343132346868515, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.017955996096134186, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9278750419616699, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15176.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.393511563539505, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.337738037109375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.908402919769287, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.7386687994003296, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.24673135578632355, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.021541595458984, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00933898240327835, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7505180835723877, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0191703699529171, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.6987404823303223, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21951.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6521981954574585, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.69191312789917, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 15.882183074951172, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 13.76583480834961, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.8408769369125366, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 6.629326820373535, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.504096984863281, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5965526103973389, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 12.406911849975586, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026700591668486595, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.17629413306713104, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1788625866174698, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01699117012321949, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2196849286556244, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007041036617010832, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6263214349746704, "metrics/particle/momentum/all/accepted/pz_bias": 0.009800350293517113, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.8978707790374756, "metrics/particle/momentum/all/accepted/samples": 23150.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.39951637387275696, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.1376895904541016, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.19315505027771, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.027477741241455, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2480836808681488, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.86504554748535, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.017915653064846992, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1137696504592896, "metrics/particle/momentum/all/assigned/pz_bias": 0.026159169152379036, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.733567476272583, "metrics/particle/momentum/all/assigned/samples": 33267.0, "metrics/particle/momentum/all/efficiency": 0.6290931701660156, "metrics/particle/momentum/all/event/energy_bias": -2.193937301635742, "metrics/particle/momentum/all/event/p3_error_mean": 28.954742431640625, "metrics/particle/momentum/all/event/pt_vector_error_mean": 16.52447509765625, "metrics/particle/momentum/all/event/px_bias": 2.6704537868499756, "metrics/particle/momentum/all/event/py_bias": 7.733766555786133, "metrics/particle/momentum/all/event/pz_abs_mean": 19.750497817993164, "metrics/particle/momentum/all/event/pz_bias": -0.4627060890197754, "metrics/particle/momentum/all/event/pz_residual_rms": 30.990541458129883, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.029729731380939484, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.17686660587787628, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.18400414288043976, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.03645997494459152, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.22454698383808136, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006849315017461777, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6757413744926453, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.00836918130517006, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8992359638214111, "metrics/particle/momentum/class_1/accepted/samples": 12556.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2545633614063263, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.8744938373565674, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.1328233480453491, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.9800392389297485, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.20227545499801636, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.293050765991211, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.013353293761610985, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.9762963056564331, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.05124661326408386, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.2747464179992676, "metrics/particle/momentum/class_1/assigned/samples": 16700.0, "metrics/particle/momentum/class_1/efficiency": 0.7077391147613525, "metrics/particle/momentum/class_1/event/energy_bias": 12.955041885375977, "metrics/particle/momentum/class_1/event/p3_error_mean": 24.652509689331055, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 14.744250297546387, "metrics/particle/momentum/class_1/event/px_bias": 3.587588310241699, "metrics/particle/momentum/class_1/event/py_bias": 5.683602333068848, "metrics/particle/momentum/class_1/event/pz_abs_mean": 16.361379623413086, "metrics/particle/momentum/class_1/event/pz_bias": 1.1211073398590088, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.080251693725586, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03772994875907898, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20531070232391357, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2041199952363968, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.057384785264730453, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24408188462257385, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03731976076960564, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8626168966293335, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.018069595098495483, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.272623062133789, "metrics/particle/momentum/class_2/accepted/samples": 1179.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6272270679473877, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7832175493240356, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0538374185562134, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7149417996406555, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.32261162996292114, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.009288787841797, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05280844867229462, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5271923542022705, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.018529998138546944, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.277576923370361, "metrics/particle/momentum/class_2/assigned/samples": 2083.0, "metrics/particle/momentum/class_2/efficiency": 0.39484259486198425, "metrics/particle/momentum/class_2/event/energy_bias": -15.551886558532715, "metrics/particle/momentum/class_2/event/p3_error_mean": 23.037700653076172, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.402130126953125, "metrics/particle/momentum/class_2/event/px_bias": 1.03863525390625, "metrics/particle/momentum/class_2/event/py_bias": 0.8183214664459229, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.5157470703125, "metrics/particle/momentum/class_2/event/pz_bias": -2.3918566703796387, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.205068588256836, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02118019014596939, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1714411973953247, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16831865906715393, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0006159633048810065, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20919610559940338, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0028637514915317297, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3702833354473114, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.00199818960390985, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2275004386901855, "metrics/particle/momentum/class_3/accepted/samples": 9079.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5406293272972107, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.713416814804077, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.5146708488464355, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.358999252319336, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.29119622707366943, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 26.065311431884766, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01786479912698269, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.0492311716079712, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.011832187883555889, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.9636878967285156, "metrics/particle/momentum/class_3/assigned/samples": 13994.0, "metrics/particle/momentum/class_3/efficiency": 0.5844975113868713, "metrics/particle/momentum/class_3/event/energy_bias": 4.034667491912842, "metrics/particle/momentum/class_3/event/p3_error_mean": 16.38713836669922, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.861895561218262, "metrics/particle/momentum/class_3/event/px_bias": -1.8093394041061401, "metrics/particle/momentum/class_3/event/py_bias": 1.5926811695098877, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.118886947631836, "metrics/particle/momentum/class_3/event/pz_bias": 0.828267514705658, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.790348052978516, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02570706605911255, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1803024411201477, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1789570301771164, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0010317020351067185, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21821191906929016, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.009661835618317127, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9541299939155579, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.011327879503369331, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0445690155029297, "metrics/particle/momentum/class_4/accepted/samples": 207.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4451199471950531, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 8.609171867370605, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 7.801320552825928, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 7.6647772789001465, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2978723347187042, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 39.355464935302734, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.009118541143834591, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.133715271949768, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1470157951116562, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.2813429832458496, "metrics/particle/momentum/class_4/assigned/samples": 329.0, "metrics/particle/momentum/class_4/efficiency": 0.5519999861717224, "metrics/particle/momentum/class_4/event/energy_bias": -2.4941160678863525, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.542405843734741, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.7194437980651855, "metrics/particle/momentum/class_4/event/px_bias": -0.12453452497720718, "metrics/particle/momentum/class_4/event/py_bias": -0.33292356133461, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7630817890167236, "metrics/particle/momentum/class_4/event/pz_bias": -0.11168596148490906, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.163793563842773, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02118043601512909, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.19049108028411865, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18950620293617249, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.020116958767175674, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22860273718833923, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03875968977808952, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.0108580589294434, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.6201875805854797, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.9624366760253906, "metrics/particle/momentum/class_5/accepted/samples": 129.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.1302924007177353, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.5034790635108948, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.5928817987442017, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.40173184871673584, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.18633539974689484, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.0955920219421387, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06211180239915848, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.655597686767578, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.577838122844696, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.600019931793213, "metrics/particle/momentum/class_5/assigned/samples": 161.0, "metrics/particle/momentum/class_5/efficiency": 0.7865853905677795, "metrics/particle/momentum/class_5/event/energy_bias": -1.1376426219940186, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.844305992126465, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.1080334186553955, "metrics/particle/momentum/class_5/event/px_bias": -0.021896231919527054, "metrics/particle/momentum/class_5/event/py_bias": -0.027914201840758324, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.504479169845581, "metrics/particle/momentum/class_5/event/pz_bias": 0.09146150946617126, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.186539649963379, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5144219398498535, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.519646167755127, "metrics/set_slot/angular_recall_all_dr0p10": 0.9010283350944519, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7212557196617126, "metrics/set_slot/soft_count_bias_mean": 0.5839119553565979}, "step_train_time": 4.420576333999634, "step_valid_time": 30549.58877658844, "step_total_time": 30554.009353399277, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0600351398270418, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2350553317394546, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3013, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2799, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2721, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9150049035632559, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9289744440756721, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9219367588932806, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07102555592432791, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8895063746322328, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.903086624626618, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8962450592885376, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9150049035632559, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0365945265541412, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2194557028015769, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2251, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2086, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2058, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.912910284463895, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9266992447800977, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9197530864197531, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07330075521990227, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9006564551422319, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.914260328742781, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9074074074074074, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.912910284463895, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0331573146862587, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2237487165086045, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1174, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1151, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8880484114977307, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.918622848200313, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9030769230769231, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.081377151799687, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8706505295007564, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9006259780907668, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8853846153846154, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8880484114977307} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_13000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..7b86b130eb11e07d318092dff87a764b2d5dd54a --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 0.6157635467980296, "train": {"Classification_binary": 0.21349741518497467, "Cardinality": 0.1692606508731842, "Classification": 0.13822244107723236, "Regression_pt": 0.2511154115200043, "Regression_eta": 0.0024097501300275326, "Regression_sin_phi": 0.0012227558763697743, "Regression_cos_phi": 0.0012039131252095103, "Regression_energy": 0.24698272347450256, "HitGrouping": 0.22922618687152863, "Auxiliary": 0.3141074776649475, "Total": 1.5217247009277344, "Total_unweighted": 0.8546544103883207}, "train_group": {}, "valid": {"Classification_binary": 0.24933083852132162, "Cardinality": 0.19352696339289346, "Classification": 0.13917462031046549, "Regression_pt": 0.22773625453313193, "Regression_eta": 0.002325500672062238, "Regression_sin_phi": 0.0012416366177300613, "Regression_cos_phi": 0.0012590873520821333, "Regression_energy": 0.22385605176289877, "HitGrouping": 0.22628537813822427, "Auxiliary": 0.31673715511957806, "Total": 1.5630710919698079, "Total_unweighted": 0.8449239897696923, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7844955921173096, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.37793809175491333, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.42083844542503357, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.37607014179229736, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3662185072898865, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5118765830993652, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7296028733253479, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.314164400100708, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3844165503978729, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3116001486778259, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.31785279512405396, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5974889397621155, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.801270604133606, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.35534027218818665, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3821800649166107, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.35287126898765564, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3357788026332855, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.48655402660369873, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7906244397163391, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.35898956656455994, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.39406538009643555, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3566952347755432, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3436088562011719, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5028383135795593, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002281145891174674, "metrics/hit_grouping/calorimeter/background_recall": 0.09658642113208771, "metrics/hit_grouping/calorimeter/completeness": 0.42517539858818054, "metrics/hit_grouping/calorimeter/energy_completeness": 0.44894716143608093, "metrics/hit_grouping/calorimeter/energy_purity": 0.44289857149124146, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.498089462518692, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8165753483772278, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3827754259109497, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3222576379776001, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.38356366753578186, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.37315425276756287, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.44185253977775574, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7848788499832153, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.37319818139076233, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4350529611110687, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.36800217628479004, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3557414412498474, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5252187848091125, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7434993982315063, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.296176552772522, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4991879463195801, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2905201315879822, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2711438536643982, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6046403646469116, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7698764204978943, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3939891755580902, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5006014108657837, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.39251962304115295, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.37308672070503235, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5486528873443604, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7675093412399292, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.49369823932647705, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6469377279281616, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4817136824131012, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.44223445653915405, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5383427739143372, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8145321607589722, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3072566092014313, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.25531914830207825, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3052758276462555, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3018045127391815, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4518977105617523, "metrics/hit_grouping/calorimeter/purity": 0.4186185300350189, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8761837482452393, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0730801373720169, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12030075490474701, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07421679049730301, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0331181064248085, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0357142873108387, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8249914050102234, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.12638521194458008, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09947028011083603, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.12555043399333954, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.11939109116792679, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3396115303039551, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7874418497085571, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.37575265765190125, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.41385647654533386, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.37332645058631897, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3603360652923584, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5189439058303833, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7313454151153564, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43734607100486755, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.41832610964775085, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.39160478115081787, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5806171298027039, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7531894445419312, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3182640075683594, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.38861334323883057, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.35836270451545715, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4683544337749481, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6608392596244812, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4444524347782135, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3475320041179657, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.320142924785614, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6789466142654419, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6893303990364075, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43294745683670044, "metrics/hit_grouping/tracker/all/object_completeness": 0.37232476472854614, "metrics/hit_grouping/tracker/all/object_purity": 0.344929575920105, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6331700682640076, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.019414443522691727, "metrics/hit_grouping/tracker/background_recall": 0.12571045756340027, "metrics/hit_grouping/tracker/completeness": 0.3768613934516907, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5620934367179871, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.693877637386322, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41837629675865173, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.41789039969444275, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.39999285340309143, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6334385275840759, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6865079998970032, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4516246020793915, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3572831451892853, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3257569372653961, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6356870532035828, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6827810406684875, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4443642795085907, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.29354071617126465, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2507132291793823, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6301172375679016, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7193273901939392, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4757578372955322, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4125487208366394, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.366085022687912, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5950581431388855, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7275972366333008, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5166256427764893, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3990752398967743, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.34725379943847656, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.567118227481842, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6434733271598816, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3622916340827942, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.31523674726486206, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3170100748538971, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6945579051971436, "metrics/hit_grouping/tracker/purity": 0.358546644449234, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8844954967498779, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01772247441112995, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.02635875903069973, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.025397762656211853, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06485671550035477, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6711017489433289, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.240061953663826, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2237122654914856, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.19110974669456482, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5699535608291626, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6623443365097046, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5154719948768616, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4397728145122528, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4090192914009094, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7243579030036926, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4571378529071808, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5201143622398376, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5841662883758545, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6043506264686584, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.09201490134000778, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.029875831678509712, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.26778221130371, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.27103784680366516, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17941465973854065, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.749113142490387, "metrics/particle/class_1/pid_accuracy": 0.9530283212661743, "metrics/particle/class_2/efficiency": 0.39494845271110535, "metrics/particle/class_2/pid_accuracy": 0.5314023494720459, "metrics/particle/class_3/efficiency": 0.5832367539405823, "metrics/particle/class_3/pid_accuracy": 0.9340305328369141, "metrics/particle/class_4/efficiency": 0.608216404914856, "metrics/particle/class_4/pid_accuracy": 0.2388797402381897, "metrics/particle/class_5/efficiency": 0.7567049860954285, "metrics/particle/class_5/pid_accuracy": 0.3113923966884613, "metrics/particle/count/bias_mean": -7.641275882720947, "metrics/particle/count/mae": 8.582682609558105, "metrics/particle/count/prediction_mean": 63.013671875, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09283614903688431, "metrics/particle/event/energy_response_mean": 0.9714882373809814, "metrics/particle/event/met_abs_error": 8.853475570678711, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08097099512815475, "metrics/particle/event/scalar_pt_response_mean": 1.019086241722107, "metrics/particle/event/vector_pt_closure": 0.0770220160484314, "metrics/particle/matched/delta_eta_abs_mean": 0.016359493136405945, "metrics/particle/matched/delta_phi_abs_mean": 0.016384974122047424, "metrics/particle/matched/delta_r_mean": 0.025896765291690826, "metrics/particle/matched/energy_relative_abs_mean": 0.17398177087306976, "metrics/particle/matched/pid_accuracy": 0.915212869644165, "metrics/particle/matched/pt_relative_abs_mean": 0.1747882217168808, "metrics/particle/matching/duplicate_fraction": 0.08485468477010727, "metrics/particle/matching/efficiency": 0.6491255760192871, "metrics/particle/matching/f1": 0.6862333416938782, "metrics/particle/matching/purity": 0.7278409600257874, "metrics/particle/matching/target_coverage_dr0p05": 0.6135764718055725, "metrics/particle/matching/target_coverage_dr0p10": 0.7239370942115784, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4282752573490143, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6981276273727417, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024452058598399162, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17398056387901306, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17600280046463013, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019202696159482002, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21736523509025574, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.013916785828769207, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9673742651939392, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.019866181537508965, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.516752243041992, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6970.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.31555598974227905, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.115687608718872, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2465306520462036, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0754951238632202, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1941937655210495, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.287607192993164, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.027182776480913162, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.4603396654129028, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.13244134187698364, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.7848379611968994, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9197.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6470478773117065, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.2646368443965912, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.539782524108887, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.939037322998047, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.11991771310567856, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.29642802476882935, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.88833999633789, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.557185173034668, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.355269432067871, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02584853209555149, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17807962000370026, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.181004136800766, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03219708055257797, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22283105552196503, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03539822995662689, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7461735010147095, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06546522676944733, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.302416801452637, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1356.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5427659749984741, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.3467514514923096, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.499231815338135, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.324587345123291, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2631053328514099, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 74.39276885986328, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07838242501020432, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1459455490112305, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.13545307517051697, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.849072456359863, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2003.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.49166062474250793, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.549115180969238, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.93152141571045, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4226045608520508, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05096195638179779, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.10164828598499298, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.765871047973633, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.1942664384841919, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.334226608276367, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026668904349207878, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.17432528734207153, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1747237592935562, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0058746724389493465, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21513305604457855, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005142049049027264, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3119748830795288, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.02193950116634369, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8438467979431152, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15558.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3614463210105896, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.954728126525879, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.4939614534378052, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.3000011444091797, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.21680983901023865, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.992157936096191, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.008683538064360619, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7003514170646667, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.09223314374685287, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.4822165966033936, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21535.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.66861492395401, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.7528433799743652, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.475285530090332, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 11.367454528808594, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.71368408203125, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.20927751064300537, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.031074523925781, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.5495107173919678, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.86760425567627, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025975391268730164, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.17443782091140747, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1754535585641861, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011258590966463089, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.21622991561889648, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006405962165445089, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.584663987159729, "metrics/particle/momentum/all/accepted/pz_bias": 0.01637209765613079, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.8339112997055054, "metrics/particle/momentum/all/accepted/samples": 23884.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3596479296684265, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.7429842948913574, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.6695209741592407, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.4831831455230713, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.21328853070735931, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 20.58186912536621, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.018145715817809105, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.0635143518447876, "metrics/particle/momentum/all/assigned/pz_bias": 0.08959802985191345, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.5873947143554688, "metrics/particle/momentum/all/assigned/samples": 32735.0, "metrics/particle/momentum/all/efficiency": 0.6490393877029419, "metrics/particle/momentum/all/event/energy_bias": -3.5316355228424072, "metrics/particle/momentum/all/event/p3_error_mean": 25.822650909423828, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.660804748535156, "metrics/particle/momentum/all/event/px_bias": -0.8845638036727905, "metrics/particle/momentum/all/event/py_bias": -0.6073538064956665, "metrics/particle/momentum/all/event/pz_abs_mean": 18.53026008605957, "metrics/particle/momentum/all/event/pz_bias": 4.300961971282959, "metrics/particle/momentum/all/event/pz_residual_rms": 30.13334083557129, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02804936096072197, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.17589272558689117, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.17847244441509247, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02518598362803459, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.21914362907409668, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005955970846116543, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6201712489128113, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.04027076065540314, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8421911001205444, "metrics/particle/momentum/class_1/accepted/samples": 13264.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.21098703145980835, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.7353304028511047, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.8856497406959534, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.7183449268341064, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.15988460183143616, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.37955093383789, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.013944821432232857, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.9298112392425537, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.11588384211063385, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.007965326309204, "metrics/particle/momentum/class_1/assigned/samples": 16637.0, "metrics/particle/momentum/class_1/efficiency": 0.747646689414978, "metrics/particle/momentum/class_1/event/energy_bias": 15.721929550170898, "metrics/particle/momentum/class_1/event/p3_error_mean": 23.461301803588867, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 13.643796920776367, "metrics/particle/momentum/class_1/event/px_bias": 2.0196633338928223, "metrics/particle/momentum/class_1/event/py_bias": -1.7109339237213135, "metrics/particle/momentum/class_1/event/pz_abs_mean": 16.139415740966797, "metrics/particle/momentum/class_1/event/pz_bias": 6.174065589904785, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.016603469848633, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0350242555141449, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20329216122627258, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20540356636047363, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.019816745072603226, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24542954564094543, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03652174025774002, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8551702499389648, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1396121382713318, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.1291584968566895, "metrics/particle/momentum/class_2/accepted/samples": 1150.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5814983248710632, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7332769632339478, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9728493690490723, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6194298267364502, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30781009793281555, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.278123378753662, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.060745276510715485, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.7288951873779297, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.10560320317745209, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.096860408782959, "metrics/particle/momentum/class_2/assigned/samples": 1959.0, "metrics/particle/momentum/class_2/efficiency": 0.3851306140422821, "metrics/particle/momentum/class_2/event/energy_bias": -11.699114799499512, "metrics/particle/momentum/class_2/event/p3_error_mean": 23.723390579223633, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.537553787231445, "metrics/particle/momentum/class_2/event/px_bias": -1.931269884109497, "metrics/particle/momentum/class_2/event/py_bias": 1.085808277130127, "metrics/particle/momentum/class_2/event/pz_abs_mean": 17.189538955688477, "metrics/particle/momentum/class_2/event/pz_bias": -1.9119954109191895, "metrics/particle/momentum/class_2/event/pz_residual_rms": 27.743837356567383, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02185467816889286, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16855190694332123, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16733945906162262, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.005548435728996992, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20786282420158386, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0026324449572712183, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3454448878765106, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004159091971814632, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1850016117095947, "metrics/particle/momentum/class_3/accepted/samples": 9117.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5113000273704529, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.0515589714050293, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.682241678237915, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.4951648712158203, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.26339319348335266, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 29.435165405273438, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.016636131331324577, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9688541889190674, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.08769598603248596, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.744713544845581, "metrics/particle/momentum/class_3/assigned/samples": 13645.0, "metrics/particle/momentum/class_3/efficiency": 0.5869439244270325, "metrics/particle/momentum/class_3/event/energy_bias": -4.594078540802002, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.599331855773926, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.058220863342285, "metrics/particle/momentum/class_3/event/px_bias": -0.9809362888336182, "metrics/particle/momentum/class_3/event/py_bias": 0.4028283953666687, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.746904373168945, "metrics/particle/momentum/class_3/event/pz_bias": 0.1256631314754486, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.563644409179688, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026397012174129486, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.175821915268898, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17092439532279968, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04931788146495819, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21376526355743408, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004566209856420755, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8851227760314941, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.06192464753985405, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7745885848999023, "metrics/particle/momentum/class_4/accepted/samples": 219.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4035305678844452, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.884814262390137, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.8988161087036133, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.771480083465576, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.29640719294548035, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 24.179950714111328, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011976048350334167, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.080473780632019, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07880236208438873, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.404726982116699, "metrics/particle/momentum/class_4/assigned/samples": 334.0, "metrics/particle/momentum/class_4/efficiency": 0.5839999914169312, "metrics/particle/momentum/class_4/event/energy_bias": -2.1046528816223145, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.336583137512207, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.5928332805633545, "metrics/particle/momentum/class_4/event/px_bias": 0.08348716795444489, "metrics/particle/momentum/class_4/event/py_bias": -0.1041238009929657, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6732982397079468, "metrics/particle/momentum/class_4/event/pz_bias": -0.14813697338104248, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.598574161529541, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02269887365400791, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1809961050748825, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17905952036380768, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.01935037225484848, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2205815315246582, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05223880708217621, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9511570930480957, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.5142814517021179, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.06507682800293, "metrics/particle/momentum/class_5/accepted/samples": 134.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07663041353225708, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.7272685766220093, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.6875291466712952, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.507716715335846, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.16249999403953552, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.8057448863983154, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.07500000298023224, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.612943172454834, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.06889495998620987, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.395426273345947, "metrics/particle/momentum/class_5/assigned/samples": 160.0, "metrics/particle/momentum/class_5/efficiency": 0.8170731663703918, "metrics/particle/momentum/class_5/event/energy_bias": -0.8557199239730835, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.0223262310028076, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.3268535137176514, "metrics/particle/momentum/class_5/event/px_bias": -0.07550787925720215, "metrics/particle/momentum/class_5/event/py_bias": -0.2809327244758606, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4819103479385376, "metrics/particle/momentum/class_5/event/pz_bias": 0.06136557087302208, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.6263837814331055, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.48861929774284363, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.49266815185546875, "metrics/set_slot/angular_recall_all_dr0p10": 0.9160846471786499, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7308478951454163, "metrics/set_slot/soft_count_bias_mean": 0.4614010155200958}, "step_train_time": 4.50003981590271, "step_valid_time": 33092.89616346359, "step_total_time": 33097.396203517914, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.048638747354274, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20385663816439448, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2800, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2739, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9153318077803204, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9250082590023125, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9201445941505094, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07499174099768748, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8953906505393919, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9048562933597621, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9000985869208018, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9153318077803204, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0326036344027785, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19491003973634524, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2273, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2105, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2079, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9212253829321663, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9260888693356797, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9236507240017552, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07391113066432028, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9098468271334792, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9146502419709635, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9122422114962703, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9212253829321663, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0185657945578548, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1910047157125605, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1288, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1186, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1169, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.897125567322239, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9208074534161491, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9088122605363985, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07919254658385093, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8842662632375189, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.907608695652174, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8957854406130268, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.897125567322239} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_14000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..78ec7ab4257ec9e5bf8645540866af994bba892a --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 0.6631299734748011, "train": {"Classification_binary": 0.19970418512821198, "Cardinality": 0.17069627344608307, "Classification": 0.09737302362918854, "Regression_pt": 0.21807606518268585, "Regression_eta": 0.0022534567397087812, "Regression_sin_phi": 0.0009586975211277604, "Regression_cos_phi": 0.0009555590804666281, "Regression_energy": 0.21492376923561096, "HitGrouping": 0.22205981612205505, "Auxiliary": 0.2809135317802429, "Total": 1.3658543825149536, "Total_unweighted": 0.7342447565170005}, "train_group": {}, "valid": {"Classification_binary": 0.23766446113586426, "Cardinality": 0.18931645154953003, "Classification": 0.12848751743634543, "Regression_pt": 0.21049495538075766, "Regression_eta": 0.0023465176733831563, "Regression_sin_phi": 0.0012117366616924603, "Regression_cos_phi": 0.0011685766900579135, "Regression_energy": 0.20707150300343832, "HitGrouping": 0.22581998507181802, "Auxiliary": 0.3029521306355794, "Total": 1.5004323323567708, "Total_unweighted": 0.7884452679815392, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7772749662399292, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3867698609828949, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.42254188656806946, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3837084472179413, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3804035186767578, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5252200365066528, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.72233647108078, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.34241801500320435, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.38995569944381714, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.33832690119743347, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3455105423927307, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6022894978523254, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8076671957969666, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.36602693796157837, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3867160379886627, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.363914430141449, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.34397557377815247, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4718815088272095, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7918348908424377, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.370478093624115, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3978210985660553, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3679245710372925, "metrics/hit_grouping/calorimeter/all/object_purity": 0.355130672454834, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4981651306152344, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0026436990592628717, "metrics/hit_grouping/calorimeter/background_recall": 0.11359792202711105, "metrics/hit_grouping/calorimeter/completeness": 0.4300481081008911, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4523153603076935, "metrics/hit_grouping/calorimeter/energy_purity": 0.44629329442977905, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.48729273676872253, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8153203129768372, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.3997161388397217, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3310195803642273, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3994385600090027, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3877885639667511, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.44077152013778687, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7853294014930725, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3855072855949402, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.44449561834335327, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3811691105365753, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.37161287665367126, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5141639709472656, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7505027651786804, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2957216501235962, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4870069622993469, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2903251647949219, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.27193737030029297, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5990719199180603, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7705039381980896, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.40260055661201477, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4938657581806183, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4011630713939667, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3858560025691986, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5463074445724487, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7731257677078247, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4913533329963684, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6057642698287964, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.47951602935791016, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.43559691309928894, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5409160852432251, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8158121705055237, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3233266770839691, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2751241624355316, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.320741206407547, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3142179250717163, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.44398799538612366, "metrics/hit_grouping/calorimeter/purity": 0.42357003688812256, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8672327399253845, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07777409255504608, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14097744226455688, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07775689661502838, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.031087370589375496, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03759398311376572, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8272956013679504, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14462189376354218, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11359623074531555, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14177939295768738, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.13345573842525482, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.32077693939208984, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7887610793113708, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.3870007395744324, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.41679683327674866, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.38442090153694153, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.37194645404815674, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5149009227752686, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7208831906318665, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4485006630420685, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4287067651748657, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.41238975524902344, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5972765684127808, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7632428407669067, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3170584738254547, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4186280369758606, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.3933209478855133, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4345991611480713, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6448239684104919, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44783422350883484, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.34856197237968445, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3410244584083557, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6942725777626038, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6769474744796753, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43837934732437134, "metrics/hit_grouping/tracker/all/object_completeness": 0.37836962938308716, "metrics/hit_grouping/tracker/all/object_purity": 0.36682116985321045, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6452804803848267, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.025164125487208366, "metrics/hit_grouping/tracker/background_recall": 0.15385222434997559, "metrics/hit_grouping/tracker/completeness": 0.3808225393295288, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.547206699848175, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6866132020950317, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4275819659233093, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.42670562863349915, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.41731706261634827, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6408907771110535, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6707590818405151, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45997458696365356, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3643516004085541, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3542396128177643, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6545652747154236, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6632134318351746, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4391343593597412, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.29287001490592957, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2754516899585724, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6450856924057007, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7026634812355042, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4721066355705261, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.41407912969589233, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3951234817504883, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6142480969429016, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7160429358482361, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4802955687046051, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3955160677433014, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3542405366897583, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5837438702583313, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6365073323249817, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38701844215393066, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3288722038269043, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3322826027870178, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6966552734375, "metrics/hit_grouping/tracker/purity": 0.3644522428512573, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8772539496421814, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01923076994717121, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.020688461139798164, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.020923428237438202, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6632319092750549, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2586473822593689, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.23941396176815033, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.20425398647785187, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5580794811248779, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6487092971801758, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5200604200363159, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4465094804763794, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.435771644115448, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7411448955535889, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.43881022930145264, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5297947525978088, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5957062840461731, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6006835103034973, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.11034253984689713, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.026208721101284027, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.495792388916016, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.27132412791252136, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17970094084739685, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7757943272590637, "metrics/particle/class_1/pid_accuracy": 0.9605191946029663, "metrics/particle/class_2/efficiency": 0.385774165391922, "metrics/particle/class_2/pid_accuracy": 0.5102759599685669, "metrics/particle/class_3/efficiency": 0.5976433753967285, "metrics/particle/class_3/pid_accuracy": 0.9402297139167786, "metrics/particle/class_4/efficiency": 0.625250518321991, "metrics/particle/class_4/pid_accuracy": 0.25961539149284363, "metrics/particle/class_5/efficiency": 0.8180076479911804, "metrics/particle/class_5/pid_accuracy": 0.3255269229412079, "metrics/particle/count/bias_mean": -7.532552242279053, "metrics/particle/count/mae": 8.360676765441895, "metrics/particle/count/prediction_mean": 63.12239456176758, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09874270111322403, "metrics/particle/event/energy_response_mean": 0.9381933808326721, "metrics/particle/event/met_abs_error": 10.6797513961792, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08013855665922165, "metrics/particle/event/scalar_pt_response_mean": 0.9767614006996155, "metrics/particle/event/vector_pt_closure": 0.08204341679811478, "metrics/particle/matched/delta_eta_abs_mean": 0.015500817447900772, "metrics/particle/matched/delta_phi_abs_mean": 0.015353245660662651, "metrics/particle/matched/delta_r_mean": 0.024417925626039505, "metrics/particle/matched/energy_relative_abs_mean": 0.1671425998210907, "metrics/particle/matched/pid_accuracy": 0.9219295382499695, "metrics/particle/matched/pt_relative_abs_mean": 0.16982214152812958, "metrics/particle/matching/duplicate_fraction": 0.07761252671480179, "metrics/particle/matching/efficiency": 0.667793869972229, "metrics/particle/matching/f1": 0.7053951025009155, "metrics/particle/matching/purity": 0.7474833726882935, "metrics/particle/matching/target_coverage_dr0p05": 0.6261172294616699, "metrics/particle/matching/target_coverage_dr0p10": 0.7338794469833374, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.45347657799720764, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7168697118759155, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022823043167591095, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1655457615852356, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1683894395828247, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014061407186090946, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20967422425746918, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.012373141013085842, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9184936285018921, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.003638531779870391, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.4002130031585693, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7193.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2860300540924072, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0470762252807617, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2070423364639282, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.032659649848938, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.17312072217464447, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.150298118591309, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.022887514904141426, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.305109977722168, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07269838452339172, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.4214985370635986, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9219.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6677497029304504, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -4.259343147277832, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.755123138427734, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.983395576477051, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.0349164009094238, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.680235743522644, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.109619140625, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.1710212230682373, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.499571800231934, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024899793788790703, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17574264109134674, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17428328096866608, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012185519561171532, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21473756432533264, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.031095406040549278, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.678901195526123, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.08373181521892548, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.403985500335693, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1415.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5006078481674194, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8300625085830688, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.219746112823486, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.035106658935547, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23371456563472748, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 99.15361022949219, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06613624840974808, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.116075277328491, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.17470039427280426, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.285078048706055, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2011.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5130529403686523, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.559500694274902, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.326128005981445, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5172216892242432, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.1929427981376648, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07914448529481888, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.12232494354248, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.6711724996566772, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.308977127075195, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.025387506932020187, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16778303682804108, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1707545518875122, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010753273032605648, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21204791963100433, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0011848341673612595, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.30799686908721924, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.010515731759369373, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.926246166229248, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16036.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3276037871837616, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.6808065176010132, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.290878415107727, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.1017807722091675, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.19938069581985474, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.680535316467285, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00734852347522974, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6093879342079163, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0543486624956131, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.1660351753234863, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21637.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.689157247543335, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -3.4423367977142334, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.920104026794434, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.018157005310059, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 4.126402854919434, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.733081579208374, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.845076560974121, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.38271597027778625, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.168266296386719, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02461099810898304, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1675870716571808, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.17026686668395996, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011801074258983135, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2115136981010437, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006167829968035221, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5649005174636841, "metrics/particle/momentum/all/accepted/pz_bias": 0.003096970496699214, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.8312257528305054, "metrics/particle/momentum/all/accepted/samples": 24644.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.32652804255485535, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5121811628341675, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.5077544450759888, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.3230571746826172, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.19411568343639374, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 25.973772048950195, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.015304104425013065, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.957908034324646, "metrics/particle/momentum/all/assigned/pz_bias": 0.045481059700250626, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.5839529037475586, "metrics/particle/momentum/all/assigned/samples": 32867.0, "metrics/particle/momentum/all/efficiency": 0.6696920990943909, "metrics/particle/momentum/all/event/energy_bias": -14.261181831359863, "metrics/particle/momentum/all/event/p3_error_mean": 27.22496795654297, "metrics/particle/momentum/all/event/pt_vector_error_mean": 14.691633224487305, "metrics/particle/momentum/all/event/px_bias": 5.354261875152588, "metrics/particle/momentum/all/event/py_bias": 3.3341729640960693, "metrics/particle/momentum/all/event/pz_abs_mean": 19.216327667236328, "metrics/particle/momentum/all/event/pz_bias": 2.224909782409668, "metrics/particle/momentum/all/event/pz_residual_rms": 31.6651611328125, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.026093745604157448, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.16318866610527039, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1679132878780365, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01802188530564308, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.208551824092865, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005735443439334631, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5904178023338318, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.01274137757718563, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7598600387573242, "metrics/particle/momentum/class_1/accepted/samples": 13774.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.18573305010795593, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5858834981918335, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6934115886688232, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.5249617099761963, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.13408491015434265, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.0516581535339355, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010434157215058804, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.799993634223938, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.033704061061143875, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.098694324493408, "metrics/particle/momentum/class_1/assigned/samples": 16676.0, "metrics/particle/momentum/class_1/efficiency": 0.7763936519622803, "metrics/particle/momentum/class_1/event/energy_bias": 8.770697593688965, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.216209411621094, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.963294982910156, "metrics/particle/momentum/class_1/event/px_bias": 3.895798921585083, "metrics/particle/momentum/class_1/event/py_bias": 3.5453622341156006, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.815376281738281, "metrics/particle/momentum/class_1/event/pz_bias": -0.22597366571426392, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.339521408081055, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035523246973752975, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2084118276834488, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21217387914657593, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.017035111784934998, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2532636821269989, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031141867861151695, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7398443222045898, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.12437267601490021, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9663233757019043, "metrics/particle/momentum/class_2/accepted/samples": 1156.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5765555500984192, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6259258389472961, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7400925159454346, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3822033405303955, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.29707542061805725, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.744127035140991, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05284761264920235, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.67838191986084, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1251731961965561, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.812215328216553, "metrics/particle/momentum/class_2/assigned/samples": 1949.0, "metrics/particle/momentum/class_2/efficiency": 0.3871399760246277, "metrics/particle/momentum/class_2/event/energy_bias": -16.80975341796875, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.8563175201416, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.785026550292969, "metrics/particle/momentum/class_2/event/px_bias": -0.3970499336719513, "metrics/particle/momentum/class_2/event/py_bias": -0.49850085377693176, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.787086486816406, "metrics/particle/momentum/class_2/event/pz_bias": 0.8150384426116943, "metrics/particle/momentum/class_2/event/pz_residual_rms": 28.10338592529297, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02110113576054573, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16863912343978882, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16816586256027222, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.00656755268573761, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20988181233406067, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0029924120754003525, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3488559424877167, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.006961306091398001, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3617314100265503, "metrics/particle/momentum/class_3/accepted/samples": 9357.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.46222031116485596, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.708908796310425, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.565297842025757, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.3846218585968018, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2502728998661041, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 39.30984878540039, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.015501054935157299, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8819507360458374, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.07030942291021347, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.4567997455596924, "metrics/particle/momentum/class_3/assigned/samples": 13741.0, "metrics/particle/momentum/class_3/efficiency": 0.6023948788642883, "metrics/particle/momentum/class_3/event/energy_bias": -3.179699182510376, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.254261016845703, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.90945291519165, "metrics/particle/momentum/class_3/event/px_bias": 1.154078483581543, "metrics/particle/momentum/class_3/event/py_bias": 0.27308133244514465, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.100446701049805, "metrics/particle/momentum/class_3/event/pz_bias": 1.4580379724502563, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.44659996032715, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026235999539494514, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1715582013130188, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17996318638324738, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03309006616473198, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21861182153224945, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00909090880304575, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9927667379379272, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08818485587835312, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1055662631988525, "metrics/particle/momentum/class_4/accepted/samples": 220.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4344998300075531, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.205761909484863, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.6468865871429443, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.510444402694702, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3008849620819092, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 16.7896785736084, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014749262481927872, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2460792064666748, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2772725820541382, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.1013026237487793, "metrics/particle/momentum/class_4/assigned/samples": 339.0, "metrics/particle/momentum/class_4/efficiency": 0.5866666436195374, "metrics/particle/momentum/class_4/event/energy_bias": -1.5015170574188232, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.0463368892669678, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.3646774291992188, "metrics/particle/momentum/class_4/event/px_bias": 0.3214923143386841, "metrics/particle/momentum/class_4/event/py_bias": 0.2041407823562622, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4760496616363525, "metrics/particle/momentum/class_4/event/pz_bias": -0.10693071782588959, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.7713189125061035, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.020569823682308197, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18709082901477814, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18120959401130676, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.04706297069787979, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21945294737815857, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.051094889640808105, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.153867483139038, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.6593655347824097, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.032240867614746, "metrics/particle/momentum/class_5/accepted/samples": 137.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07618116587400436, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3819067180156708, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.39224934577941895, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.17662088572978973, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.14814814925193787, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.4706602096557617, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04938271641731262, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.354301929473877, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.7198916077613831, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.056512355804443, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.8353658318519592, "metrics/particle/momentum/class_5/event/energy_bias": -1.5409083366394043, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.5089287757873535, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.9270970821380615, "metrics/particle/momentum/class_5/event/px_bias": 0.37994202971458435, "metrics/particle/momentum/class_5/event/py_bias": -0.18991076946258545, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2396466732025146, "metrics/particle/momentum/class_5/event/pz_bias": 0.28473758697509766, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.178206443786621, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.462431937456131, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4678548276424408, "metrics/set_slot/angular_recall_all_dr0p10": 0.9194663166999817, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7397213578224182, "metrics/set_slot/soft_count_bias_mean": -1.4512921571731567}, "step_train_time": 5.196303606033325, "step_valid_time": 35694.94109582901, "step_total_time": 35700.13739967346, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0113211013276941, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21499025326529742, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2812, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2760, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9192546583850931, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9289725801123224, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9240880709825829, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07102741988767756, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9022556390977443, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9117938553022795, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9069996713769306, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9192546583850931, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9785865626977961, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20296963863478856, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2255, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2104, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2084, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9207877461706784, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9330376940133037, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9268722466960353, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06696230598669622, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.912035010940919, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9241685144124169, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9180616740088106, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9207877461706784, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9789653196694587, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20869627154892445, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1187, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1179, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8978819969742814, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9222999222999223, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9099271751628977, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0777000777000777, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8918305597579426, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.916083916083916, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9037945573016481, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8978819969742814} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_15000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..0506a0fc21cdab8d2caa20c66311af9b2636d6dc --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 0.7104964001515726, "train": {"Classification_binary": 0.22523565590381622, "Cardinality": 0.16140533983707428, "Classification": 0.11821959912776947, "Regression_pt": 0.19330470263957977, "Regression_eta": 0.0023210172075778246, "Regression_sin_phi": 0.0009725135751068592, "Regression_cos_phi": 0.0010633021593093872, "Regression_energy": 0.1929694563150406, "HitGrouping": 0.22024807333946228, "Auxiliary": 0.2827412784099579, "Total": 1.3835994005203247, "Total_unweighted": 0.7340862469282001}, "train_group": {}, "valid": {"Classification_binary": 0.23649356762568155, "Cardinality": 0.17483921845753989, "Classification": 0.1242376168568929, "Regression_pt": 0.19916260242462158, "Regression_eta": 0.002150743268430233, "Regression_sin_phi": 0.0011082404914001625, "Regression_cos_phi": 0.001066002141063412, "Regression_energy": 0.1936418612798055, "HitGrouping": 0.21978958447774252, "Auxiliary": 0.29378145933151245, "Total": 1.432325839996338, "Total_unweighted": 0.7578606340878954, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7876945734024048, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.41337141394615173, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.41572821140289307, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.41184061765670776, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.40444520115852356, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4991009831428528, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.738842248916626, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.35700666904449463, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.39660266041755676, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.35499873757362366, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3554001748561859, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5690546631813049, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8135377168655396, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3901345729827881, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3781069219112396, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3893599212169647, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.371269553899765, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4549409747123718, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7999092936515808, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3946022391319275, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3909403681755066, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.393502801656723, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3800881505012512, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4771789014339447, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0019190908642485738, "metrics/hit_grouping/calorimeter/background_recall": 0.0881018117070198, "metrics/hit_grouping/calorimeter/completeness": 0.4562678039073944, "metrics/hit_grouping/calorimeter/energy_completeness": 0.47766733169555664, "metrics/hit_grouping/calorimeter/energy_purity": 0.47115403413772583, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.46769070625305176, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8254693746566772, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.419321209192276, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3111060559749603, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.42082375288009644, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4074326753616333, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4151115119457245, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7965726256370544, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4126715064048767, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.43758636713027954, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4105760455131531, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.39794403314590454, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4885997176170349, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7511543035507202, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.32599735260009766, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5067285299301147, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.32059580087661743, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.30634477734565735, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5922273993492126, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7793700098991394, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.429696649312973, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.48418331146240234, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4302816390991211, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.41645684838294983, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5227327346801758, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7711993455886841, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5106950402259827, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6433350443840027, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5006361603736877, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.47159117460250854, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5378280878067017, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8242707252502441, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3449905216693878, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2658041715621948, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.34324103593826294, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.33210670948028564, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.42350849509239197, "metrics/hit_grouping/calorimeter/purity": 0.44906851649284363, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8670225143432617, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07077890634536743, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.13345864415168762, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07077290862798691, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03156845644116402, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04135338217020035, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.834754467010498, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.14932723343372345, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09829311072826385, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.14925213158130646, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1415204256772995, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3178340196609497, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7970024347305298, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4126425087451935, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4103647768497467, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4114718735218048, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3981817662715912, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.49257251620292664, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7224604487419128, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.434448778629303, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.43722039461135864, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4351094663143158, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.594668984413147, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.760480523109436, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3182640075683594, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4210117757320404, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4038839340209961, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.44786015152931213, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6448891162872314, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44560369849205017, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3562546670436859, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.35807204246520996, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6990214586257935, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6772685050964355, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4327693581581116, "metrics/hit_grouping/tracker/all/object_completeness": 0.38592246174812317, "metrics/hit_grouping/tracker/all/object_purity": 0.38513311743736267, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6483971476554871, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.018591169267892838, "metrics/hit_grouping/tracker/background_recall": 0.12432901561260223, "metrics/hit_grouping/tracker/completeness": 0.3889178931713104, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5264129042625427, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6859701871871948, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41469401121139526, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4334314465522766, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.43944594264030457, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6460634469985962, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6760595440864563, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45325830578804016, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3736879229545593, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3682611584663391, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6445816159248352, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6605702042579651, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4495942294597626, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.30035215616226196, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2901747524738312, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.656988263130188, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7015640139579773, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.46675723791122437, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4230159819126129, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4147659242153168, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6165407299995422, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7089829444885254, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.511083722114563, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40496137738227844, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.37485578656196594, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6028324961662292, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6399980783462524, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3745446503162384, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3342866003513336, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3484519124031067, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6979799270629883, "metrics/hit_grouping/tracker/purity": 0.3697068393230438, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8796383142471313, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02224736101925373, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.024441266432404518, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.022484533488750458, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06711915880441666, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6665631532669067, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.25141972303390503, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2413220852613449, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2158757448196411, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5725348591804504, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6483977437019348, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5133456587791443, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4552380442619324, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4572944641113281, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7428795099258423, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4868188798427582, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.534223735332489, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6015823483467102, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6226780414581299, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.06233389303088188, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04820328205823898, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.5773983001709, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2844739556312561, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.19285079836845398, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8018652200698853, "metrics/particle/class_1/pid_accuracy": 0.9542157053947449, "metrics/particle/class_2/efficiency": 0.4026503562927246, "metrics/particle/class_2/pid_accuracy": 0.5232067704200745, "metrics/particle/class_3/efficiency": 0.6435957551002502, "metrics/particle/class_3/pid_accuracy": 0.949796199798584, "metrics/particle/class_4/efficiency": 0.6523045897483826, "metrics/particle/class_4/pid_accuracy": 0.2519201338291168, "metrics/particle/class_5/efficiency": 0.8237547874450684, "metrics/particle/class_5/pid_accuracy": 0.33023256063461304, "metrics/particle/count/bias_mean": -3.9453125, "metrics/particle/count/mae": 6.134114742279053, "metrics/particle/count/prediction_mean": 66.70963287353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09844493120908737, "metrics/particle/event/energy_response_mean": 0.9422186017036438, "metrics/particle/event/met_abs_error": 9.55133056640625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08144209533929825, "metrics/particle/event/scalar_pt_response_mean": 0.973898708820343, "metrics/particle/event/vector_pt_closure": 0.08217373490333557, "metrics/particle/matched/delta_eta_abs_mean": 0.016508951783180237, "metrics/particle/matched/delta_phi_abs_mean": 0.015527760609984398, "metrics/particle/matched/delta_r_mean": 0.025313766673207283, "metrics/particle/matched/energy_relative_abs_mean": 0.1699099987745285, "metrics/particle/matched/pid_accuracy": 0.9228464961051941, "metrics/particle/matched/pt_relative_abs_mean": 0.1713367998600006, "metrics/particle/matching/duplicate_fraction": 0.08829270303249359, "metrics/particle/matching/efficiency": 0.7014079689979553, "metrics/particle/matching/f1": 0.7215534448623657, "metrics/particle/matching/purity": 0.7428902983665466, "metrics/particle/matching/target_coverage_dr0p05": 0.6538525223731995, "metrics/particle/matching/target_coverage_dr0p10": 0.7653834223747253, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.47336122393608093, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7528795003890991, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023719780147075653, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1730652004480362, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17380763590335846, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009731100872159004, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21374747157096863, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.012200026772916317, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9295181632041931, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.24872131645679474, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.416686773300171, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7459.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3500725328922272, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2783938646316528, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.5081326961517334, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3235194683074951, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1886521875858307, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.607147216796875, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.022736582905054092, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3433523178100586, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.24905383586883545, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.6183483600616455, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9764.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6924433708190918, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -4.129565238952637, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.81811809539795, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.045705318450928, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.8280808329582214, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.0135200023651123, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.307032585144043, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -6.101472854614258, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.35385513305664, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02625872753560543, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17953628301620483, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18031607568264008, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0009385886369273067, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22195351123809814, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03641456738114357, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7234084606170654, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.5926835536956787, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.347996711730957, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1428.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6661249399185181, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7202773094177246, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.623532772064209, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.4295806884765625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2709014415740967, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 132.43179321289062, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0728631466627121, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.08937668800354, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.6227715015411377, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.540831565856934, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2141.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5177664756774902, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.408559799194336, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.632518768310547, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.622654914855957, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.10491708666086197, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.025943025946617126, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.422533988952637, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.9490697383880615, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.016124725341797, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026092035695910454, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16947436332702637, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1714196354150772, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0025834308471530676, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21179041266441345, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008857919019646943, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.29538294672966003, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.042487550526857376, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8286910057067871, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16934.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.30996665358543396, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.5215681791305542, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.2229503393173218, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0206408500671387, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1853732466697693, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.613564491271973, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006057366728782654, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5663326382637024, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.08133789151906967, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.005155563354492, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22452.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7277493476867676, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.0727310180664062, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 14.459747314453125, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.223816871643066, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.4729368686676025, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.3109939098358154, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.507199287414551, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -2.829495668411255, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.6458101272583, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025415973737835884, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.17106813192367554, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.17260146141052246, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.004453422036021948, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.212930828332901, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0061190505512058735, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5575429797172546, "metrics/particle/momentum/all/accepted/pz_bias": -0.13249094784259796, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7841033935546875, "metrics/particle/momentum/all/accepted/samples": 25821.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3435588777065277, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4648425579071045, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.578224778175354, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.3814653158187866, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1916348934173584, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 35.123680114746094, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014960560947656631, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9443824887275696, "metrics/particle/momentum/all/assigned/pz_bias": -0.16274161636829376, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4670915603637695, "metrics/particle/momentum/all/assigned/samples": 34357.0, "metrics/particle/momentum/all/efficiency": 0.7016766667366028, "metrics/particle/momentum/all/event/energy_bias": -13.610855102539062, "metrics/particle/momentum/all/event/p3_error_mean": 29.16790771484375, "metrics/particle/momentum/all/event/pt_vector_error_mean": 14.94400405883789, "metrics/particle/momentum/all/event/px_bias": -3.405935049057007, "metrics/particle/momentum/all/event/py_bias": 3.2985708713531494, "metrics/particle/momentum/all/event/pz_abs_mean": 21.692455291748047, "metrics/particle/momentum/all/event/pz_bias": -12.88003921508789, "metrics/particle/momentum/all/event/pz_residual_rms": 32.56407928466797, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.026510121300816536, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1653204709291458, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.17084774374961853, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.004503163043409586, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.21031449735164642, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005627462174743414, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.58964604139328, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.1658925712108612, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7432376146316528, "metrics/particle/momentum/class_1/accepted/samples": 14216.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1988515853881836, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5074841976165771, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5995409488677979, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.40113064646720886, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.12130974978208542, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.487558841705322, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010367853567004204, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8044424653053284, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.19463790953159332, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.949256181716919, "metrics/particle/momentum/class_1/assigned/samples": 17072.0, "metrics/particle/momentum/class_1/efficiency": 0.8013076782226562, "metrics/particle/momentum/class_1/event/energy_bias": 0.7853532433509827, "metrics/particle/momentum/class_1/event/p3_error_mean": 22.464885711669922, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 13.310179710388184, "metrics/particle/momentum/class_1/event/px_bias": -4.217739105224609, "metrics/particle/momentum/class_1/event/py_bias": 4.176103591918945, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.203714370727539, "metrics/particle/momentum/class_1/event/pz_bias": -7.7675652503967285, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.505064010620117, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03590831533074379, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20469890534877777, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20812323689460754, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03998642787337303, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2488912045955658, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0331914909183979, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7696260213851929, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.3097946047782898, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.981396436691284, "metrics/particle/momentum/class_2/accepted/samples": 1175.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7467231154441833, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7806441783905029, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9889757037162781, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5580835342407227, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3492496609687805, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.004597187042236, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06093678995966911, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.789214849472046, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.4650789499282837, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.8587236404418945, "metrics/particle/momentum/class_2/assigned/samples": 2199.0, "metrics/particle/momentum/class_2/efficiency": 0.39350301027297974, "metrics/particle/momentum/class_2/event/energy_bias": -15.030654907226562, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.47179412841797, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.126773834228516, "metrics/particle/momentum/class_2/event/px_bias": -0.3480198383331299, "metrics/particle/momentum/class_2/event/py_bias": -1.9796082973480225, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.082155227661133, "metrics/particle/momentum/class_2/event/pz_bias": -2.6270368099212646, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.642187118530273, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02272224985063076, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17519794404506683, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17101681232452393, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.021456347778439522, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21209265291690826, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0029821074567735195, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.34573307633399963, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.06424561887979507, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2965950965881348, "metrics/particle/momentum/class_3/accepted/samples": 10060.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.45242229104042053, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.5877957344055176, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.7566041946411133, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.5952632427215576, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.24795770645141602, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 53.60378646850586, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012768586166203022, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8090176582336426, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0805743858218193, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.2566349506378174, "metrics/particle/momentum/class_3/assigned/samples": 14567.0, "metrics/particle/momentum/class_3/efficiency": 0.6476534008979797, "metrics/particle/momentum/class_3/event/energy_bias": 3.101824998855591, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.804914474487305, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.946098327636719, "metrics/particle/momentum/class_3/event/px_bias": 0.6745039224624634, "metrics/particle/momentum/class_3/event/py_bias": 1.023396372795105, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.828457832336426, "metrics/particle/momentum/class_3/event/pz_bias": -1.9288368225097656, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.921844482421875, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026337480172514915, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1774103045463562, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17304940521717072, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04638278856873512, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21363183856010437, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.008547008968889713, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9013662338256836, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.13978999853134155, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2461330890655518, "metrics/particle/momentum/class_4/accepted/samples": 234.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4571543037891388, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.094625473022461, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.417939186096191, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.284494876861572, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3072625696659088, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.418533325195312, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016759777441620827, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1828707456588745, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.20273756980895996, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.8899428844451904, "metrics/particle/momentum/class_4/assigned/samples": 358.0, "metrics/particle/momentum/class_4/efficiency": 0.6240000128746033, "metrics/particle/momentum/class_4/event/energy_bias": -1.9332904815673828, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.479094982147217, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.6392180919647217, "metrics/particle/momentum/class_4/event/px_bias": 0.28693321347236633, "metrics/particle/momentum/class_4/event/py_bias": -0.036176301538944244, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.740828275680542, "metrics/particle/momentum/class_4/event/pz_bias": -0.5804996490478516, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.64990234375, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.018065320327878, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16491387784481049, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16546525061130524, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.005232265219092369, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.20843744277954102, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05147058889269829, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8059104681015015, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.14477360248565674, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.085172176361084, "metrics/particle/momentum/class_5/accepted/samples": 136.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07899193465709686, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4279250204563141, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.471289724111557, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.3020413815975189, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1428571492433548, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.0338668823242188, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06832297891378403, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.3030714988708496, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.0034756630193442106, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.711258411407471, "metrics/particle/momentum/class_5/assigned/samples": 161.0, "metrics/particle/momentum/class_5/efficiency": 0.8292682766914368, "metrics/particle/momentum/class_5/event/energy_bias": -0.534088134765625, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.7734296321868896, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.2340235710144043, "metrics/particle/momentum/class_5/event/px_bias": 0.19838720560073853, "metrics/particle/momentum/class_5/event/py_bias": 0.11485540866851807, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2795501947402954, "metrics/particle/momentum/class_5/event/pz_bias": 0.02390032634139061, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.177938461303711, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4476088881492615, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4534026086330414, "metrics/set_slot/angular_recall_all_dr0p10": 0.9266166687011719, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7608130574226379, "metrics/set_slot/soft_count_bias_mean": 1.2703808546066284}, "step_train_time": 4.433207750320435, "step_valid_time": 38321.01673603058, "step_total_time": 38325.44994401932, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0027298808881864, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22480776035594108, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3010, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2797, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2746, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9143510951291272, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9292358803986711, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9217333992420498, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0707641196013289, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8976789800588427, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9122923588039867, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9049266765529741, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9143510951291272, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9761810190735647, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21675992284093992, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2248, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2116, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2105, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.926039387308534, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9412811387900356, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9335980586807854, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05871886120996441, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9212253829321663, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9363879003558719, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.928744760644165, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.926039387308534, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9846816496507879, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19459090155269054, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1290, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1195, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1186, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9039334341906202, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9263565891472868, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9150076569678407, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07364341085271318, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.897125567322239, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9193798449612403, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9081163859111792, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9039334341906202} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_16000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..20b76f7ed471ed169c29ccd28b9a4595a3ff59c0 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 0.757862826828344, "train": {"Classification_binary": 0.25797027349472046, "Cardinality": 0.2093275636434555, "Classification": 0.11901750415563583, "Regression_pt": 0.19688300788402557, "Regression_eta": 0.002113169291988015, "Regression_sin_phi": 0.0011023283004760742, "Regression_cos_phi": 0.0010891451966017485, "Regression_energy": 0.19195151329040527, "HitGrouping": 0.2151159942150116, "Auxiliary": 0.3126499652862549, "Total": 1.4978320598602295, "Total_unweighted": 0.770126941613853}, "train_group": {}, "valid": {"Classification_binary": 0.22826814651489258, "Cardinality": 0.17097453276316324, "Classification": 0.12425909439722697, "Regression_pt": 0.18896323442459106, "Regression_eta": 0.002118729675809542, "Regression_sin_phi": 0.001049830811098218, "Regression_cos_phi": 0.0010616673001398642, "Regression_energy": 0.18563711643218994, "HitGrouping": 0.21568850676218668, "Auxiliary": 0.2875448266665141, "Total": 1.3948911031087239, "Total_unweighted": 0.7313578195559481, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7931401133537292, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4103671908378601, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.40645405650138855, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.40776151418685913, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.40522944927215576, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4925712049007416, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7496713995933533, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3502345681190491, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3666912913322449, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3517615497112274, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3606337308883667, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5568685531616211, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8113713264465332, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.39490872621536255, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.38023605942726135, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.39236578345298767, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3786860406398773, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4595695436000824, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8010579347610474, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3961235284805298, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3871272802352905, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3938775658607483, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3853258788585663, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4771215617656708, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002776917302981019, "metrics/hit_grouping/calorimeter/background_recall": 0.11248449981212616, "metrics/hit_grouping/calorimeter/completeness": 0.4581905007362366, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4780007004737854, "metrics/hit_grouping/calorimeter/energy_purity": 0.47166240215301514, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.47152653336524963, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8279286623001099, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.42109355330467224, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3101388216018677, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.42132022976875305, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4118594527244568, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.40925124287605286, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.796494722366333, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4114942252635956, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4309074282646179, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4078938961029053, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4003896713256836, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.49470290541648865, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7508661150932312, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.32972538471221924, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.32380205392837524, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.31604883074760437, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.597795844078064, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7840603590011597, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4370487630367279, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.48634833097457886, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4356079697608948, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4229643642902374, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5190041065216064, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7674605250358582, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5103709101676941, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6186310052871704, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.49915844202041626, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.47466692328453064, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5589295029640198, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8223905563354492, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.34078678488731384, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.25838494300842285, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3387880325317383, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3363074064254761, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.42467349767684937, "metrics/hit_grouping/calorimeter/purity": 0.45134878158569336, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8557286262512207, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07832323014736176, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10902255773544312, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07932330667972565, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05146065354347229, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03947368264198303, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8169883489608765, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1578359752893448, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10005886107683182, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.15674623847007751, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1588384211063385, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3237198293209076, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7993380427360535, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4137020409107208, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.40659743547439575, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.411342978477478, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4025520980358124, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.49223560094833374, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7270947694778442, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.42590177059173584, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4327874779701233, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4302141070365906, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5878603458404541, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7635580897331238, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3086196482181549, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.41707298159599304, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.39019855856895447, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.43942132592201233, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6563881635665894, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43812060356140137, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.35909444093704224, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3574749529361725, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6847028136253357, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6860355734825134, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4247996509075165, "metrics/hit_grouping/tracker/all/object_completeness": 0.3860263228416443, "metrics/hit_grouping/tracker/all/object_purity": 0.3822481632232666, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6368210315704346, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.020264189690351486, "metrics/hit_grouping/tracker/background_recall": 0.13182823359966278, "metrics/hit_grouping/tracker/completeness": 0.39036500453948975, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5330796837806702, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6953747272491455, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40838155150413513, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.43059444427490234, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.42921626567840576, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6287041902542114, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6821176409721375, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4436376690864563, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.37524694204330444, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.37539857625961304, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6434924602508545, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.670717716217041, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4398557245731354, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.305059552192688, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.29244059324264526, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6468890905380249, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7100514769554138, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.46268150210380554, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.42342498898506165, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.411827951669693, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6047380566596985, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7131220102310181, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4889162480831146, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40328744053840637, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3764403760433197, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5935960412025452, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6499583721160889, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.36405783891677856, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.334312379360199, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.34483471512794495, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6862788200378418, "metrics/hit_grouping/tracker/purity": 0.37171614170074463, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8873896598815918, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.031674209982156754, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03281145542860031, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.026426145806908607, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.061463046818971634, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6748069524765015, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.24832214415073395, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.24570755660533905, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.22733132541179657, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5467217564582825, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6573723554611206, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5022662281990051, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4536511301994324, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.45184221863746643, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7319679856300354, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.46589505672454834, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5702195167541504, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6487463116645813, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6107404232025146, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.08325766772031784, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.03626558929681778, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.25986671447754, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29052484035491943, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1989016979932785, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7916809916496277, "metrics/particle/class_1/pid_accuracy": 0.9549035429954529, "metrics/particle/class_2/efficiency": 0.41397666931152344, "metrics/particle/class_2/pid_accuracy": 0.529685378074646, "metrics/particle/class_3/efficiency": 0.6657193303108215, "metrics/particle/class_3/pid_accuracy": 0.9533692598342896, "metrics/particle/class_4/efficiency": 0.6182364821434021, "metrics/particle/class_4/pid_accuracy": 0.25283631682395935, "metrics/particle/class_5/efficiency": 0.7892720103263855, "metrics/particle/class_5/pid_accuracy": 0.33252426981925964, "metrics/particle/count/bias_mean": -3.1263020038604736, "metrics/particle/count/mae": 5.39453125, "metrics/particle/count/prediction_mean": 67.52864837646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09734194725751877, "metrics/particle/event/energy_response_mean": 1.0019038915634155, "metrics/particle/event/met_abs_error": 9.070716857910156, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08242196589708328, "metrics/particle/event/scalar_pt_response_mean": 1.0251879692077637, "metrics/particle/event/vector_pt_closure": 0.07955311983823776, "metrics/particle/matched/delta_eta_abs_mean": 0.016787497326731682, "metrics/particle/matched/delta_phi_abs_mean": 0.0161837600171566, "metrics/particle/matched/delta_r_mean": 0.026011301204562187, "metrics/particle/matched/energy_relative_abs_mean": 0.16830812394618988, "metrics/particle/matched/pid_accuracy": 0.9250199198722839, "metrics/particle/matched/pt_relative_abs_mean": 0.17087256908416748, "metrics/particle/matching/duplicate_fraction": 0.08958389610052109, "metrics/particle/matching/efficiency": 0.7062547206878662, "metrics/particle/matching/f1": 0.722233235836029, "metrics/particle/matching/purity": 0.7389514446258545, "metrics/particle/matching/target_coverage_dr0p05": 0.6631959080696106, "metrics/particle/matching/target_coverage_dr0p10": 0.7792233824729919, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.47310322523117065, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7539852261543274, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024935944005846977, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.17263758182525635, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17829006910324097, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.027710696682333946, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21893753111362457, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010420842096209526, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9191713333129883, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.2372167408466339, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.3403851985931396, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7485.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.33910855650901794, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5319242477416992, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.8882631063461304, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.7237529754638672, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.20305435359477997, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 19.04206085205078, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02280719392001629, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3965429067611694, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.26528316736221313, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.9010283946990967, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9953.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6948570609092712, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.753884792327881, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.610286712646484, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.898119926452637, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.269045889377594, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.013918276876211166, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.150395393371582, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -5.95077657699585, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.86090087890625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02631041780114174, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.18176309764385223, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1822032481431961, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02077794261276722, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2238212525844574, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0354815348982811, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.82108736038208, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.5349342226982117, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.994672775268555, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1381.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6775781512260437, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.9677709341049194, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.054764270782471, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.869935512542725, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.31335774064064026, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 66.11213684082031, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07868252694606781, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.339953660964966, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.7343618869781494, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.230031967163086, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2186.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5007251501083374, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.477223873138428, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.445337295532227, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5786726474761963, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04280734062194824, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07043059170246124, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.23933219909668, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -4.343608856201172, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 27.33262062072754, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026467759162187576, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16577622294425964, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16752061247825623, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014241533353924751, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20795096457004547, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009317493531852961, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.28833457827568054, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.04271845147013664, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8213547468185425, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17172.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.29055255651474, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.581416130065918, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.253388524055481, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0776395797729492, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1842338740825653, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.348403930664062, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006033182609826326, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5521973371505737, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.03526363521814346, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9380724430084229, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22542.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7379775643348694, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.575379848480225, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.996844291687012, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.713913917541504, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.8332915902137756, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.12833580374717712, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.198490619659424, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -2.0059871673583984, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 13.415552139282227, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02601907216012478, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16859650611877441, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1713951975107193, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018460117280483246, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.21202656626701355, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005491973366588354, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5509716272354126, "metrics/particle/momentum/all/accepted/pz_bias": -0.12473586946725845, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.8282763957977295, "metrics/particle/momentum/all/accepted/samples": 26038.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3288823366165161, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5915653705596924, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.7382277250289917, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.5651319026947021, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1977739930152893, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 20.88934326171875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.015426314435899258, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9702304601669312, "metrics/particle/momentum/all/assigned/pz_bias": -0.14534159004688263, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.661320924758911, "metrics/particle/momentum/all/assigned/samples": 34681.0, "metrics/particle/momentum/all/efficiency": 0.7075735926628113, "metrics/particle/momentum/all/event/energy_bias": 4.8520402908325195, "metrics/particle/momentum/all/event/p3_error_mean": 28.112083435058594, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.312700271606445, "metrics/particle/momentum/all/event/px_bias": -1.1451448202133179, "metrics/particle/momentum/all/event/py_bias": 0.07182355225086212, "metrics/particle/momentum/all/event/pz_abs_mean": 21.73080062866211, "metrics/particle/momentum/all/event/pz_bias": -12.300374031066895, "metrics/particle/momentum/all/event/pz_residual_rms": 34.578590393066406, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.026389969512820244, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.16514869034290314, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.16988985240459442, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.033527933061122894, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2103603184223175, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0054088677279651165, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5841764807701111, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.1399572491645813, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.66004478931427, "metrics/particle/momentum/class_1/accepted/samples": 14051.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.18210850656032562, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.6070985794067383, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.7072259187698364, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.5560635328292847, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.14122648537158966, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.437412261962891, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010832217521965504, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.838026762008667, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.18232671916484833, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.17136287689209, "metrics/particle/momentum/class_1/assigned/samples": 17171.0, "metrics/particle/momentum/class_1/efficiency": 0.7920072078704834, "metrics/particle/momentum/class_1/event/energy_bias": 14.663811683654785, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.845746994018555, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.850911140441895, "metrics/particle/momentum/class_1/event/px_bias": 1.495703935623169, "metrics/particle/momentum/class_1/event/py_bias": 0.9110292196273804, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.782255172729492, "metrics/particle/momentum/class_1/event/pz_bias": -8.109546661376953, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.101329803466797, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035352982580661774, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20732878148555756, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20879621803760529, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.015399663709104061, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24856139719486237, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029983792454004288, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6788039207458496, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.465187668800354, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8139452934265137, "metrics/particle/momentum/class_2/accepted/samples": 1234.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7430652976036072, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7086644172668457, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0276455879211426, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6400550603866577, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.33009278774261475, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.131570816040039, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0596553236246109, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.6876955032348633, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.28852447867393494, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.001147270202637, "metrics/particle/momentum/class_2/assigned/samples": 2263.0, "metrics/particle/momentum/class_2/efficiency": 0.41326189041137695, "metrics/particle/momentum/class_2/event/energy_bias": -13.661766052246094, "metrics/particle/momentum/class_2/event/p3_error_mean": 21.317440032958984, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.27889347076416, "metrics/particle/momentum/class_2/event/px_bias": -1.6335623264312744, "metrics/particle/momentum/class_2/event/py_bias": 0.09320829063653946, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.522456169128418, "metrics/particle/momentum/class_2/event/pz_bias": -2.3955459594726562, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.743499755859375, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.024532200768589973, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1681794673204422, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16870303452014923, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.002006215276196599, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20916928350925446, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0021155879367142916, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3470916152000427, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.06419605016708374, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.6316684484481812, "metrics/particle/momentum/class_3/accepted/samples": 10399.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4356535077095032, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.815375804901123, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.0064845085144043, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.8395774364471436, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.24058108031749725, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 31.264448165893555, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013441042974591255, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8387553095817566, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.07741885632276535, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.715816020965576, "metrics/particle/momentum/class_3/assigned/samples": 14731.0, "metrics/particle/momentum/class_3/efficiency": 0.6694778800010681, "metrics/particle/momentum/class_3/event/energy_bias": 6.702531337738037, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.438244819641113, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.730079650878906, "metrics/particle/momentum/class_3/event/px_bias": -1.4600721597671509, "metrics/particle/momentum/class_3/event/py_bias": -0.5743322372436523, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.892587661743164, "metrics/particle/momentum/class_3/event/pz_bias": -1.498598337173462, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.97154426574707, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.024247806519269943, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1847689002752304, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1824152171611786, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.025613559409976006, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22519998252391815, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.009009009227156639, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9835745096206665, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.334661602973938, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0729176998138428, "metrics/particle/momentum/class_4/accepted/samples": 222.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.47815757989883423, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.468810558319092, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.031512260437012, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.8967788219451904, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3239436745643616, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.618759155273438, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.019718309864401817, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1926465034484863, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.362093985080719, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.1092188358306885, "metrics/particle/momentum/class_4/assigned/samples": 355.0, "metrics/particle/momentum/class_4/efficiency": 0.5920000076293945, "metrics/particle/momentum/class_4/event/energy_bias": -2.3972320556640625, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.1481759548187256, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.462712287902832, "metrics/particle/momentum/class_4/event/px_bias": 0.22098314762115479, "metrics/particle/momentum/class_4/event/py_bias": -0.26594647765159607, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5209532976150513, "metrics/particle/momentum/class_4/event/pz_bias": -0.3006199598312378, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.271101474761963, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.019395455718040466, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17917458713054657, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17554476857185364, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.015286747366189957, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2183648943901062, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04545454680919647, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8070608377456665, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.26195162534713745, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.6137781143188477, "metrics/particle/momentum/class_5/accepted/samples": 132.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06254403293132782, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.6779448390007019, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.586521327495575, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4382765293121338, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.17391304671764374, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.8830759525299072, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05590062215924263, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.4686779975891113, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.07498166710138321, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.911608695983887, "metrics/particle/momentum/class_5/assigned/samples": 161.0, "metrics/particle/momentum/class_5/efficiency": 0.8048780560493469, "metrics/particle/momentum/class_5/event/energy_bias": -0.4553031921386719, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.7412517070770264, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.16810941696167, "metrics/particle/momentum/class_5/event/px_bias": 0.23180262744426727, "metrics/particle/momentum/class_5/event/py_bias": -0.09213527292013168, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2810217142105103, "metrics/particle/momentum/class_5/event/pz_bias": 0.003937922418117523, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.070056438446045, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.44746971130371094, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4528905153274536, "metrics/set_slot/angular_recall_all_dr0p10": 0.9303576946258545, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7717689871788025, "metrics/set_slot/soft_count_bias_mean": 1.310887098312378}, "step_train_time": 5.186636209487915, "step_valid_time": 40808.37364077568, "step_total_time": 40813.560277462006, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0485582434676353, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2057764561158647, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3043, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2821, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2747, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9221967963386728, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9270456786066382, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9246148803670927, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07295432139336182, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8980058842759071, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9027275714755176, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9003605375286792, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9221967963386728, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0225560067031922, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20316867802379357, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2266, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2113, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2090, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9247264770240701, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9324801412180053, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9285871237090749, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06751985878199471, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9146608315098468, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9223300970873787, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9184794550648209, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9247264770240701, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0438676115935166, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2038708537515036, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1296, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1190, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1170, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9001512859304085, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9182098765432098, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9090909090909091, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08179012345679013, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8850226928895613, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9027777777777778, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8938120702826585, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9001512859304085} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_17000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..7e12770ad6f8a0089638baf18e17bbd72cdb6db2 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 0.8052292535051155, "train": {"Classification_binary": 0.23252248764038086, "Cardinality": 0.17813695967197418, "Classification": 0.11792967468500137, "Regression_pt": 0.17916949093341827, "Regression_eta": 0.0018031983636319637, "Regression_sin_phi": 0.0009253136813640594, "Regression_cos_phi": 0.0008984990417957306, "Regression_energy": 0.17822806537151337, "HitGrouping": 0.20885489881038666, "Auxiliary": 0.29059451818466187, "Total": 1.3575423955917358, "Total_unweighted": 0.7114767297171056}, "train_group": {}, "valid": {"Classification_binary": 0.2118450403213501, "Cardinality": 0.16205783685048422, "Classification": 0.1177991231282552, "Regression_pt": 0.18066527446111044, "Regression_eta": 0.001975466807683309, "Regression_sin_phi": 0.0010319269883135955, "Regression_cos_phi": 0.0010034959608068068, "Regression_energy": 0.1785105069478353, "HitGrouping": 0.21254998445510864, "Auxiliary": 0.2774263024330139, "Total": 1.3345157305399578, "Total_unweighted": 0.6928308346153548, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7980688214302063, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4253961145877838, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.40815746784210205, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.42392396926879883, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4180629253387451, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4728872776031494, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7465562224388123, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.369852215051651, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.38774001598358154, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.36800727248191833, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.36496222019195557, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5409896373748779, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8222815990447998, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.40616944432258606, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3691737949848175, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.40535688400268555, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3865055739879608, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4364730417728424, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8090671300888062, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4091746509075165, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.382425457239151, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.40808212757110596, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3943934142589569, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.45561927556991577, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002521187998354435, "metrics/hit_grouping/calorimeter/background_recall": 0.1096515953540802, "metrics/hit_grouping/calorimeter/completeness": 0.47495874762535095, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4957166612148285, "metrics/hit_grouping/calorimeter/energy_purity": 0.48908907175064087, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4577842652797699, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8318701982498169, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4333816170692444, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.31383705139160156, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.43458953499794006, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4196358621120453, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3975876271724701, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8063684701919556, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.42683935165405273, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.41812530159950256, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4252058267593384, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.41335657238960266, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.46683555841445923, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7652910351753235, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.34202122688293457, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4863108992576599, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.33678314089775085, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.323820561170578, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5626450181007385, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7944803833961487, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.45145660638809204, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.47221553325653076, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4513200521469116, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.43499794602394104, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.49212172627449036, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7806411981582642, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.529579222202301, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6037055850028992, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5210651159286499, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.48503822088241577, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5239320397377014, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8273259401321411, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.35172098875045776, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.26451653242111206, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3505380153656006, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3421955108642578, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4102642834186554, "metrics/hit_grouping/calorimeter/purity": 0.46774670481681824, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8430309295654297, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07339296489953995, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1560150384902954, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07569817453622818, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04327906668186188, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05451127886772156, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8220751285552979, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.16372767090797424, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11065332591533661, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.16292165219783783, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1569487601518631, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.31665685772895813, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8078368306159973, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.42741867899894714, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.40025728940963745, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4262559115886688, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.41247084736824036, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4693865180015564, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7315600514411926, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44096770882606506, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.456910103559494, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4482206404209137, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5816311836242676, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7680978775024414, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.31163352727890015, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4361037313938141, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4179477095603943, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4412296712398529, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6539732217788696, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4556770622730255, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3724757134914398, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3698166608810425, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6866455674171448, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6862491369247437, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44051647186279297, "metrics/hit_grouping/tracker/all/object_completeness": 0.40312618017196655, "metrics/hit_grouping/tracker/all/object_purity": 0.39746907353401184, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6362422108650208, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013140618801116943, "metrics/hit_grouping/tracker/background_recall": 0.0990290492773056, "metrics/hit_grouping/tracker/completeness": 0.40613579750061035, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5313389897346497, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6947499513626099, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42889708280563354, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.44907864928245544, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.44174227118492126, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6330001950263977, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6828057765960693, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4510800540447235, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.39106059074401855, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.39299389719963074, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6425848603248596, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6721837520599365, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.45392245054244995, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32058972120285034, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.31084561347961426, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6366095542907715, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7142559289932251, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4742294251918793, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.44332432746887207, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4275272786617279, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5980300307273865, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7259204983711243, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5043103694915771, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.42139318585395813, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3801272213459015, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5683497786521912, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6427273154258728, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3852522373199463, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.34759247303009033, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.36150115728378296, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6980903148651123, "metrics/hit_grouping/tracker/purity": 0.3839418888092041, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8872146010398865, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.023378582671284676, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.030049553140997887, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.027364101260900497, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6667166352272034, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.26587507128715515, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2615917921066284, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.23696154356002808, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5658234357833862, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6585435271263123, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5213474631309509, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.47383013367652893, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.469788521528244, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.728946328163147, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.47831249237060547, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5658714175224304, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6413173079490662, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6057273149490356, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0708402544260025, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.031252529472112656, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.185340881347656, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2722061276435852, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1805829554796219, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8187435865402222, "metrics/particle/class_1/pid_accuracy": 0.961775004863739, "metrics/particle/class_2/efficiency": 0.41250425577163696, "metrics/particle/class_2/pid_accuracy": 0.553816556930542, "metrics/particle/class_3/efficiency": 0.655575692653656, "metrics/particle/class_3/pid_accuracy": 0.9556488990783691, "metrics/particle/class_4/efficiency": 0.625250518321991, "metrics/particle/class_4/pid_accuracy": 0.3044871687889099, "metrics/particle/class_5/efficiency": 0.8544061183929443, "metrics/particle/class_5/pid_accuracy": 0.37443944811820984, "metrics/particle/count/bias_mean": -4.899739742279053, "metrics/particle/count/mae": 6.194010257720947, "metrics/particle/count/prediction_mean": 65.75521087646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09301827102899551, "metrics/particle/event/energy_response_mean": 0.954402506351471, "metrics/particle/event/met_abs_error": 8.123557090759277, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07406337559223175, "metrics/particle/event/scalar_pt_response_mean": 0.9750723838806152, "metrics/particle/event/vector_pt_closure": 0.07173983752727509, "metrics/particle/matched/delta_eta_abs_mean": 0.01609315536916256, "metrics/particle/matched/delta_phi_abs_mean": 0.014947832562029362, "metrics/particle/matched/delta_r_mean": 0.024562636390328407, "metrics/particle/matched/energy_relative_abs_mean": 0.16313616931438446, "metrics/particle/matched/pid_accuracy": 0.9316114783287048, "metrics/particle/matched/pt_relative_abs_mean": 0.16405585408210754, "metrics/particle/matching/duplicate_fraction": 0.07315841317176819, "metrics/particle/matching/efficiency": 0.7153124809265137, "metrics/particle/matching/f1": 0.7410058975219727, "metrics/particle/matching/purity": 0.768613874912262, "metrics/particle/matching/target_coverage_dr0p05": 0.6753957867622375, "metrics/particle/matching/target_coverage_dr0p10": 0.7825313806533813, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.49534672498703003, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7591821551322937, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023196175694465637, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15951021015644073, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1617242991924286, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016416635364294052, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20117886364459991, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010162866674363613, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8337079882621765, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.09607069939374924, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.3061087131500244, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7675.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27546948194503784, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9087994694709778, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0744904279708862, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9060622453689575, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16453740000724792, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.549819946289062, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019163403660058975, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.184784173965454, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.05656885728240013, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2806854248046875, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9706.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7124953866004944, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.6593286991119385, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.8118314743042, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.292681694030762, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.011418554931879044, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.33938324451446533, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.426247596740723, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.9332025051116943, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.173868179321289, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02457783930003643, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17686408758163452, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1785384565591812, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02807065285742283, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2190880924463272, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.031271714717149734, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5950268507003784, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.18956056237220764, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.388238906860352, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1439.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5645787119865417, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7001744508743286, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.359739303588867, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.184018135070801, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25991401076316833, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 114.41410827636719, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06688962876796722, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9630744457244873, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3248359262943268, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.194818496704102, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2093.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5217549204826355, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.012168884277344, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.371465682983398, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4448171854019165, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.011912696063518524, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.005400974303483963, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.188867568969727, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.642719268798828, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.960712432861328, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02533646672964096, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16339543461799622, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16412483155727386, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02542329952120781, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20515389740467072, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009340338874608278, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2854236364364624, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.022399678826332092, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9433035254478455, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17130.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.27868589758872986, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2617217302322388, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9782060980796814, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8038405776023865, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.17753639817237854, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.404155731201172, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00450714398175478, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5071645379066467, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.021725136786699295, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9452431201934814, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22187.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.736172616481781, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.0010135509073734283, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.267950057983398, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.501349449157715, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.27331554889678955, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.7263786196708679, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.963868618011475, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.9759939312934875, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.087909698486328, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024668946862220764, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16299772262573242, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1642131358385086, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.022934481501579285, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.204792320728302, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005296448711305857, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5175758004188538, "metrics/particle/momentum/all/accepted/pz_bias": -0.05311029776930809, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7866041660308838, "metrics/particle/momentum/all/accepted/samples": 26244.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.295373797416687, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1879329681396484, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2755368947982788, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1027835607528687, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1788971871137619, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 29.22026824951172, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01253457274287939, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8519299030303955, "metrics/particle/momentum/all/assigned/pz_bias": -0.05034291744232178, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2788188457489014, "metrics/particle/momentum/all/assigned/samples": 33986.0, "metrics/particle/momentum/all/efficiency": 0.7131715416908264, "metrics/particle/momentum/all/event/energy_bias": -10.670483589172363, "metrics/particle/momentum/all/event/p3_error_mean": 24.23520278930664, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.919095039367676, "metrics/particle/momentum/all/event/px_bias": 0.2499842792749405, "metrics/particle/momentum/all/event/py_bias": 1.0603609085083008, "metrics/particle/momentum/all/event/pz_abs_mean": 17.977413177490234, "metrics/particle/momentum/all/event/pz_bias": -5.551916122436523, "metrics/particle/momentum/all/event/pz_residual_rms": 30.324586868286133, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.026033099740743637, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1563609093427658, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15637846291065216, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.016050787642598152, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19577936828136444, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005387856625020504, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5415745377540588, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.07087534666061401, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.6419365406036377, "metrics/particle/momentum/class_1/accepted/samples": 14477.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.16987833380699158, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.47995179891586304, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5239292979240417, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3607251048088074, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.11126123368740082, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.627559661865234, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009281559847295284, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7306191325187683, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.07034704834222794, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8663978576660156, "metrics/particle/momentum/class_1/assigned/samples": 17023.0, "metrics/particle/momentum/class_1/efficiency": 0.8160194158554077, "metrics/particle/momentum/class_1/event/energy_bias": 7.20242977142334, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.016231536865234, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.662322044372559, "metrics/particle/momentum/class_1/event/px_bias": -0.8981605172157288, "metrics/particle/momentum/class_1/event/py_bias": -1.3511114120483398, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.243619918823242, "metrics/particle/momentum/class_1/event/pz_bias": -1.0504769086837769, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.420536041259766, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033843450248241425, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20165076851844788, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20680493116378784, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.012562556192278862, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24586546421051025, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.032867707312107086, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7262123823165894, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.3481540083885193, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.939168930053711, "metrics/particle/momentum/class_2/accepted/samples": 1217.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.650005578994751, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.690081775188446, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9818229675292969, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6043269634246826, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.32192105054855347, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.577495098114014, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05373276397585869, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.589275360107422, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0665842592716217, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.0284223556518555, "metrics/particle/momentum/class_2/assigned/samples": 2103.0, "metrics/particle/momentum/class_2/efficiency": 0.40756866335868835, "metrics/particle/momentum/class_2/event/energy_bias": -18.03729248046875, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.483661651611328, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.62336540222168, "metrics/particle/momentum/class_2/event/px_bias": 0.3456585109233856, "metrics/particle/momentum/class_2/event/py_bias": 1.4885560274124146, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.085290908813477, "metrics/particle/momentum/class_2/event/pz_bias": -3.1518845558166504, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.107810974121094, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021722447127103806, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16769860684871674, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17036665976047516, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.037230778485536575, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2118762582540512, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001374570420011878, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3175757825374603, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0003801614511758089, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.506003737449646, "metrics/particle/momentum/class_3/accepted/samples": 10185.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.39151421189308167, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.046926498413086, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.17423152923584, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.0191962718963623, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.23598244786262512, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 44.595458984375, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010029950179159641, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7231545448303223, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.026262205094099045, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.153640031814575, "metrics/particle/momentum/class_3/assigned/samples": 14357.0, "metrics/particle/momentum/class_3/efficiency": 0.6557007431983948, "metrics/particle/momentum/class_3/event/energy_bias": 2.754362106323242, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.277694702148438, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.821636199951172, "metrics/particle/momentum/class_3/event/px_bias": 0.27094757556915283, "metrics/particle/momentum/class_3/event/py_bias": 1.1029072999954224, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.255560874938965, "metrics/particle/momentum/class_3/event/pz_bias": -1.3948945999145508, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.164758682250977, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.023603389039635658, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17091922461986542, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16297367215156555, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.028737878426909447, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20426692068576813, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.008888889104127884, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8135258555412292, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08494726568460464, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.9150094985961914, "metrics/particle/momentum/class_4/accepted/samples": 225.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4214385151863098, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.7961819171905518, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.1651275157928467, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.0257391929626465, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2994185984134674, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.82421588897705, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011627906933426857, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1053236722946167, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.02499968186020851, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.228851556777954, "metrics/particle/momentum/class_4/assigned/samples": 344.0, "metrics/particle/momentum/class_4/efficiency": 0.6000000238418579, "metrics/particle/momentum/class_4/event/energy_bias": -1.3136115074157715, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.8974404335021973, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.2018771171569824, "metrics/particle/momentum/class_4/event/px_bias": 0.30334219336509705, "metrics/particle/momentum/class_4/event/py_bias": 0.08319127559661865, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4702770709991455, "metrics/particle/momentum/class_4/event/pz_bias": -0.052626270800828934, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.5854902267456055, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01992364414036274, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15856632590293884, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1584489494562149, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0060547832399606705, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19995838403701782, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0357142873108387, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6038044691085815, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.23538605868816376, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.713728666305542, "metrics/particle/momentum/class_5/accepted/samples": 140.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08697381615638733, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3649265468120575, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3932342827320099, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.2340937703847885, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.11320754885673523, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.450067162513733, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.044025156646966934, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.940589189529419, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.07695911079645157, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.125877857208252, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.8536585569381714, "metrics/particle/momentum/class_5/event/energy_bias": -1.2763721942901611, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.452988624572754, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.9214298725128174, "metrics/particle/momentum/class_5/event/px_bias": 0.22819650173187256, "metrics/particle/momentum/class_5/event/py_bias": -0.26318228244781494, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.183480143547058, "metrics/particle/momentum/class_5/event/pz_bias": 0.09796668589115143, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.8530499935150146, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4259645342826843, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.42997172474861145, "metrics/set_slot/angular_recall_all_dr0p10": 0.9345963001251221, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7790206670761108, "metrics/set_slot/soft_count_bias_mean": -0.8131389021873474}, "step_train_time": 5.028367280960083, "step_valid_time": 43354.41306567192, "step_total_time": 43359.4414331913, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9997105287496733, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17289411684808886, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3036, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2843, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2820, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9293886891140896, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.936429512516469, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.932895816242822, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06357048748353096, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9218698921216084, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9288537549407114, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9253486464315013, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9293886891140896, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9843682074493438, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.177060398056188, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2265, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2125, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2114, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9299781181619255, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9381898454746137, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9340659340659341, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06181015452538632, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.925164113785558, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9333333333333333, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9292307692307692, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9299781181619255, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9927363087317198, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17969583094804376, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1298, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1188, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1182, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8986384266263238, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9152542372881356, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9068702290076336, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0847457627118644, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8940998487140696, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9106317411402157, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9022900763358779, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8986384266263238} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_18000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..00a3e8a2d256e7ef17478a0fbc76c7558620b818 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 0.852595680181887, "train": {"Classification_binary": 0.2275436818599701, "Cardinality": 0.1599428951740265, "Classification": 0.12474710494279861, "Regression_pt": 0.17688404023647308, "Regression_eta": 0.0020652133971452713, "Regression_sin_phi": 0.0011629249202087522, "Regression_cos_phi": 0.0009743586415424943, "Regression_energy": 0.17490461468696594, "HitGrouping": 0.21217553317546844, "Auxiliary": 0.2780870795249939, "Total": 1.359738826751709, "Total_unweighted": 0.7082819386851043}, "train_group": {}, "valid": {"Classification_binary": 0.20888876914978027, "Cardinality": 0.15418549378712973, "Classification": 0.11674726009368896, "Regression_pt": 0.17165716489156088, "Regression_eta": 0.0019896842228869596, "Regression_sin_phi": 0.0009743379584203163, "Regression_cos_phi": 0.000971123343333602, "Regression_energy": 0.16967256863911948, "HitGrouping": 0.2118277152379354, "Auxiliary": 0.2710872491200765, "Total": 1.3001858393351238, "Total_unweighted": 0.6709009082987905, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7986956238746643, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.439597487449646, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.41023942828178406, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4378846287727356, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4335232377052307, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.47127851843833923, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7455252408981323, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.37710845470428467, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.37924668192863464, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.37744390964508057, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3760523796081543, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5480058789253235, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8150296211242676, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.414151132106781, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.38204118609428406, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.41275519132614136, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.39854705333709717, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4445730149745941, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8046850562095642, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4189842939376831, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.39036697149276733, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4176267683506012, "metrics/hit_grouping/calorimeter/all/object_purity": 0.40739676356315613, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4606938064098358, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0025291561614722013, "metrics/hit_grouping/calorimeter/background_recall": 0.10880595445632935, "metrics/hit_grouping/calorimeter/completeness": 0.47439146041870117, "metrics/hit_grouping/calorimeter/energy_completeness": 0.49222463369369507, "metrics/hit_grouping/calorimeter/energy_purity": 0.48565635085105896, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4470823109149933, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8317016363143921, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4462178349494934, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.314690500497818, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.44636833667755127, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.43371251225471497, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3945152461528778, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8024522662162781, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4361853301525116, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.42779824137687683, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4343475103378296, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.42259907722473145, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.47535696625709534, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7518482208251953, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3461267948150635, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5069605708122253, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3421784043312073, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.33842432498931885, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5808584690093994, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7855068445205688, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4554857015609741, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.484123170375824, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4553685486316681, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4448157548904419, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5052922964096069, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7635079622268677, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5144500732421875, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6165723204612732, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5049847960472107, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.489057719707489, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5362840890884399, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8291440606117249, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.37039539217948914, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.267827570438385, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.36873912811279297, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.35951706767082214, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.40621742606163025, "metrics/hit_grouping/calorimeter/purity": 0.46719175577163696, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8521661162376404, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07751015573740005, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12030075490474701, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07847744226455688, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04586656391620636, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03759398311376572, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8254775404930115, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.18056081235408783, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11477339267730713, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.17904897034168243, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.17377455532550812, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3090052902698517, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.802829384803772, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.43695563077926636, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4091089963912964, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4355682134628296, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.425445020198822, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.47548162937164307, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7188637256622314, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4383601248264313, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.45328959822654724, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.46576136350631714, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5982906222343445, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7548432350158691, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3080168664455414, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4295484721660614, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.42537954449653625, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.44424352049827576, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6387214660644531, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4569002687931061, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.37138742208480835, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3912080228328705, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7055691480636597, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.671930193901062, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44020479917526245, "metrics/hit_grouping/tracker/all/object_completeness": 0.40085580945014954, "metrics/hit_grouping/tracker/all/object_purity": 0.4166457951068878, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6532947421073914, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01925825886428356, "metrics/hit_grouping/tracker/background_recall": 0.12827597558498383, "metrics/hit_grouping/tracker/completeness": 0.40258893370628357, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5054097175598145, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.684729278087616, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42872172594070435, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4484454393386841, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4615364074707031, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6466771960258484, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.666936457157135, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45362135767936707, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3905835747718811, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4116097390651703, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6558359265327454, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6505640149116516, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4504959285259247, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.31317010521888733, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3293096125125885, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6643823385238647, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6963011622428894, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4698989689350128, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43730294704437256, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.44674214720726013, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6192578673362732, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7010489702224731, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4889162480831146, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40982964634895325, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.39729011058807373, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6052955389022827, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6350271105766296, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3928689658641815, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.35186466574668884, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3809893727302551, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7061485648155212, "metrics/hit_grouping/tracker/purity": 0.3829628825187683, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8828248381614685, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02752639539539814, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03469695523381233, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.030967114493250847, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06108597293496132, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.66276615858078, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2762003242969513, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.27417483925819397, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.26652440428733826, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.562725841999054, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6416274309158325, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5192210674285889, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.46892333030700684, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4901506006717682, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7509932518005371, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5094358921051025, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5429748892784119, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6146761775016785, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6149913668632507, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.03971688821911812, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04051658511161804, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.0886173248291, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2865240275859833, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.19490087032318115, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8345157504081726, "metrics/particle/class_1/pid_accuracy": 0.9574010968208313, "metrics/particle/class_2/efficiency": 0.4450107514858246, "metrics/particle/class_2/pid_accuracy": 0.5925171971321106, "metrics/particle/class_3/efficiency": 0.6862689256668091, "metrics/particle/class_3/pid_accuracy": 0.956294596195221, "metrics/particle/class_4/efficiency": 0.6312625408172607, "metrics/particle/class_4/pid_accuracy": 0.30000001192092896, "metrics/particle/class_5/efficiency": 0.8639847040176392, "metrics/particle/class_5/pid_accuracy": 0.337028831243515, "metrics/particle/count/bias_mean": -3.1295573711395264, "metrics/particle/count/mae": 5.093099117279053, "metrics/particle/count/prediction_mean": 67.525390625, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08707109838724136, "metrics/particle/event/energy_response_mean": 0.991205632686615, "metrics/particle/event/met_abs_error": 8.567570686340332, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07040330767631531, "metrics/particle/event/scalar_pt_response_mean": 1.0048657655715942, "metrics/particle/event/vector_pt_closure": 0.07205135375261307, "metrics/particle/matched/delta_eta_abs_mean": 0.016167975962162018, "metrics/particle/matched/delta_phi_abs_mean": 0.01468698401004076, "metrics/particle/matched/delta_r_mean": 0.0244545117020607, "metrics/particle/matched/energy_relative_abs_mean": 0.158813014626503, "metrics/particle/matched/pid_accuracy": 0.930425763130188, "metrics/particle/matched/pt_relative_abs_mean": 0.15818817913532257, "metrics/particle/matching/duplicate_fraction": 0.08022638410329819, "metrics/particle/matching/efficiency": 0.7386156320571899, "metrics/particle/matching/f1": 0.755344033241272, "metrics/particle/matching/purity": 0.7728477716445923, "metrics/particle/matching/target_coverage_dr0p05": 0.6878904700279236, "metrics/particle/matching/target_coverage_dr0p10": 0.7980115413665771, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5256067514419556, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7821996808052063, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02361362613737583, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15939071774482727, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1587030589580536, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00900229997932911, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1993361860513687, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009252313524484634, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8152461051940918, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.08727482706308365, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1794698238372803, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7998.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2918817698955536, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3553966283798218, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.5095771551132202, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3430778980255127, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1593017429113388, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.064308166503906, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020847881212830544, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2301620244979858, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07232566177845001, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.446500778198242, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10025.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7424805164337158, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.087148666381836, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.755518913269043, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.552145004272461, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.535719096660614, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.27008914947509766, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.223567008972168, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7334463000297546, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.154539108276367, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02626659721136093, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16832990944385529, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16850219666957855, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0031599891372025013, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20790673792362213, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.027450980618596077, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4888733625411987, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.1913376897573471, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.882840394973755, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1530.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5597460269927979, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3611091375350952, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.89469051361084, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.710430145263672, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24831004440784454, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 30.06822967529297, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06894997507333755, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8828818798065186, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.017042018473148346, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.210062980651855, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2219.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5547498464584351, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.473244667053223, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.981844902038574, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5383474826812744, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.01100260391831398, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.11491610109806061, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.799120903015137, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.7052697539329529, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.110557556152344, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.025000669062137604, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1586601883172989, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15825018286705017, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.004630260169506073, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19856317341327667, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006785027799196541, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2733860909938812, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.017424486577510834, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7911320924758911, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17686.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.27557241916656494, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.5985504388809204, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.2907921075820923, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.1111011505126953, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.16722838580608368, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.13798999786377, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006075387820601463, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5565193891525269, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.006441463716328144, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.271920919418335, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22550.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7600670456886292, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.6113436222076416, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.268178939819336, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.34765625, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.44439375400543213, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.99573814868927, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.484342575073242, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.2983340919017792, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.981553077697754, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024664198979735374, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15941853821277618, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15895964205265045, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.005477195605635643, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1993270069360733, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004703461658209562, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5009708404541016, "metrics/particle/momentum/all/accepted/pz_bias": 0.04773058742284775, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.62800133228302, "metrics/particle/momentum/all/accepted/samples": 27214.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2983947992324829, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.513348937034607, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.5198941230773926, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.3437124490737915, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.17011553049087524, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.398114204406738, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01434155274182558, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8989768624305725, "metrics/particle/momentum/all/assigned/pz_bias": 0.023926639929413795, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.489621877670288, "metrics/particle/momentum/all/assigned/samples": 34794.0, "metrics/particle/momentum/all/efficiency": 0.7395309805870056, "metrics/particle/momentum/all/event/energy_bias": 0.22524742782115936, "metrics/particle/momentum/all/event/p3_error_mean": 23.73971939086914, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.557021141052246, "metrics/particle/momentum/all/event/px_bias": 0.9691101908683777, "metrics/particle/momentum/all/event/py_bias": 1.1509112119674683, "metrics/particle/momentum/all/event/pz_abs_mean": 16.928977966308594, "metrics/particle/momentum/all/event/pz_bias": -0.27015742659568787, "metrics/particle/momentum/all/event/pz_residual_rms": 30.00760269165039, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.025876371189951897, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14910879731178284, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1496567726135254, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0010098597267642617, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1880454272031784, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0046681552194058895, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5249270796775818, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06608603894710541, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5724678039550781, "metrics/particle/momentum/class_1/accepted/samples": 14781.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.17431952059268951, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5853615999221802, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.640162467956543, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.47320905327796936, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10175009816884995, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.028295516967773, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010058724321424961, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7514199018478394, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.07807732373476028, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.997645616531372, "metrics/particle/momentum/class_1/assigned/samples": 17199.0, "metrics/particle/momentum/class_1/efficiency": 0.8331548571586609, "metrics/particle/momentum/class_1/event/energy_bias": 5.656077861785889, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.246187210083008, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.746963500976562, "metrics/particle/momentum/class_1/event/px_bias": -2.130275011062622, "metrics/particle/momentum/class_1/event/py_bias": -0.2699870467185974, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.522587776184082, "metrics/particle/momentum/class_1/event/pz_bias": 6.022106647491455, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.18657112121582, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03285807743668556, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19945324957370758, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20066094398498535, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.006137130782008171, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2399120032787323, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.024719100445508957, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5591857433319092, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.004385422915220261, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4971959590911865, "metrics/particle/momentum/class_2/accepted/samples": 1335.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6351170539855957, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8226862549781799, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1185556650161743, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7741912007331848, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3165436387062073, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.821924686431885, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05297438055276871, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.6008963584899902, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.23379774391651154, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.585668563842773, "metrics/particle/momentum/class_2/assigned/samples": 2303.0, "metrics/particle/momentum/class_2/efficiency": 0.4470863938331604, "metrics/particle/momentum/class_2/event/energy_bias": -7.241789817810059, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.760421752929688, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.088774681091309, "metrics/particle/momentum/class_2/event/px_bias": 2.7063817977905273, "metrics/particle/momentum/class_2/event/py_bias": 1.3955906629562378, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.093474388122559, "metrics/particle/momentum/class_2/event/pz_bias": -5.369820594787598, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.445777893066406, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022089600563049316, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16863809525966644, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1664568930864334, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013578872196376324, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20850129425525665, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016769145149737597, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.309289813041687, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.013521194458007812, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1836813688278198, "metrics/particle/momentum/class_3/accepted/samples": 10734.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3892695903778076, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.6202926635742188, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.532209873199463, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.3711206912994385, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.22358356416225433, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.407001495361328, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012861301191151142, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.78128582239151, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.010841408744454384, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.299852132797241, "metrics/particle/momentum/class_3/assigned/samples": 14773.0, "metrics/particle/momentum/class_3/efficiency": 0.6910448670387268, "metrics/particle/momentum/class_3/event/energy_bias": 4.0069122314453125, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.03892993927002, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.754724025726318, "metrics/particle/momentum/class_3/event/px_bias": 0.003700248897075653, "metrics/particle/momentum/class_3/event/py_bias": 0.3000866770744324, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.902280807495117, "metrics/particle/momentum/class_3/event/pz_bias": -0.6668728590011597, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.780352592468262, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.023713450878858566, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15815001726150513, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15938489139080048, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.011324099265038967, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2024868279695511, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.008810572326183319, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8901298642158508, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3184375762939453, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1679928302764893, "metrics/particle/momentum/class_4/accepted/samples": 227.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.45665669441223145, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.2250590324401855, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.759080410003662, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.615151882171631, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.31179773807525635, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.544647216796875, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014044944196939468, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1530555486679077, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.015227923169732094, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.305828094482422, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.6053333282470703, "metrics/particle/momentum/class_4/event/energy_bias": -0.835266649723053, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.6797661781311035, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.7033987045288086, "metrics/particle/momentum/class_4/event/px_bias": 0.30298393964767456, "metrics/particle/momentum/class_4/event/py_bias": 0.1516183614730835, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.0184943675994873, "metrics/particle/momentum/class_4/event/pz_bias": -0.6056972742080688, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.192790031433105, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01733248680830002, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1613704115152359, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16817685961723328, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.04382089525461197, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.20558591187000275, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.043795619159936905, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.977995753288269, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.8069716691970825, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.791588306427002, "metrics/particle/momentum/class_5/accepted/samples": 137.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.050942666828632355, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.7575427293777466, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.1928530931472778, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.9808914065361023, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.15950919687747955, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 10.706801414489746, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05521472543478012, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.5340020656585693, "metrics/particle/momentum/class_5/assigned/pz_bias": 1.1216362714767456, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.264086723327637, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.8353658318519592, "metrics/particle/momentum/class_5/event/energy_bias": -1.3606860637664795, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.659731864929199, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.0492405891418457, "metrics/particle/momentum/class_5/event/px_bias": 0.08631932735443115, "metrics/particle/momentum/class_5/event/py_bias": -0.4263973832130432, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.3162407875061035, "metrics/particle/momentum/class_5/event/pz_bias": 0.35012727975845337, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.264763355255127, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4178526997566223, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.42000633478164673, "metrics/set_slot/angular_recall_all_dr0p10": 0.9358494877815247, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7896540760993958, "metrics/set_slot/soft_count_bias_mean": 0.41135847568511963}, "step_train_time": 5.3461997509002686, "step_valid_time": 46005.802800655365, "step_total_time": 46011.14900112152, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0211274819402951, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1857415610596258, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3029, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2834, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2801, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.92644655116051, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9356223175965666, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.931011826544021, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06437768240343347, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9156587119973848, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9247276328821393, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9201708278580815, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.92644655116051, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003414353981785, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17507125939597312, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2253, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2132, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2123, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9330415754923413, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9462938304482912, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9396209784045835, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05370616955170883, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9291028446389497, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9422991566799822, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9356544733362715, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9330415754923413, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.007742578422214, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18029766283826942, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1303, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1198, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1186, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9062027231467473, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9194167306216423, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9127619047619048, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08058326937835764, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.897125567322239, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9102072141212586, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9036190476190477, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9062027231467473} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_19000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..b9e36695a3eab8e7c8c308ffed120d341e00a86d --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 0.8999621068586586, "train": {"Classification_binary": 0.18658176064491272, "Cardinality": 0.12767721712589264, "Classification": 0.11880750954151154, "Regression_pt": 0.16217030584812164, "Regression_eta": 0.0020559225231409073, "Regression_sin_phi": 0.0010392728727310896, "Regression_cos_phi": 0.0010425504297018051, "Regression_energy": 0.16002927720546722, "HitGrouping": 0.208957239985466, "Auxiliary": 0.257254958152771, "Total": 1.2396893501281738, "Total_unweighted": 0.6317265990655869}, "train_group": {}, "valid": {"Classification_binary": 0.20237712065378824, "Cardinality": 0.15627378225326538, "Classification": 0.1112117866675059, "Regression_pt": 0.16893418629964194, "Regression_eta": 0.00187003534908096, "Regression_sin_phi": 0.0009961253963410854, "Regression_cos_phi": 0.0009411113957564036, "Regression_energy": 0.16850252946217856, "HitGrouping": 0.20997472604115805, "Auxiliary": 0.2676175038019816, "Total": 1.281154950459798, "Total_unweighted": 0.6548328952242931, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8019314408302307, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4335482716560364, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.39717990159988403, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4339793026447296, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4272894859313965, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.47260338068008423, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7586914896965027, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3802255690097809, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.37296897172927856, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.37961599230766296, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3735780417919159, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5343427062034607, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8205334544181824, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4180847406387329, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.36815550923347473, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4178553819656372, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4031360149383545, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4314742088317871, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8100966215133667, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4198301434516907, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.377322256565094, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.41977137327194214, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4081585705280304, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.45192086696624756, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0024317947681993246, "metrics/hit_grouping/calorimeter/background_recall": 0.11006031930446625, "metrics/hit_grouping/calorimeter/completeness": 0.481484979391098, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4968900978565216, "metrics/hit_grouping/calorimeter/energy_purity": 0.49024417996406555, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4478302597999573, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8364404439926147, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.44548580050468445, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3001820743083954, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.44742482900619507, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.43259891867637634, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.38922393321990967, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.807961642742157, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.43653273582458496, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.41098570823669434, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4357404112815857, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.42678865790367126, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.46395671367645264, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.758532702922821, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3506922721862793, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.500696063041687, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.34729889035224915, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.33955684304237366, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5676333904266357, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7928767204284668, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.46305131912231445, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4686071574687958, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4642915725708008, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.45045265555381775, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.49254271388053894, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7715812921524048, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5190941095352173, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6242923140525818, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5094132423400879, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4856194257736206, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5141533613204956, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8322418928146362, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3639376759529114, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.25482863187789917, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.36370065808296204, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3558087646961212, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.40309032797813416, "metrics/hit_grouping/calorimeter/purity": 0.47413134574890137, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8641604781150818, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07563887536525726, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10902255773544312, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07371553778648376, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03895803540945053, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.046992480754852295, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8135861754417419, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.16528461873531342, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10359034687280655, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.16467387974262238, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.16982001066207886, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.323131263256073, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8090340495109558, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.43868473172187805, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.39593860507011414, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.43868502974510193, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.42657724022865295, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.46522098779678345, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7275141477584839, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.42952340841293335, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4569390118122101, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4621102213859558, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5951035618782043, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7712075710296631, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3062085509300232, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4372367262840271, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4202585518360138, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4321880638599396, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6494113206863403, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4435170590877533, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.37447676062583923, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3828263282775879, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6918981075286865, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6824123859405518, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42907389998435974, "metrics/hit_grouping/tracker/all/object_completeness": 0.4044569730758667, "metrics/hit_grouping/tracker/all/object_purity": 0.4099588394165039, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6429652571678162, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01721198670566082, "metrics/hit_grouping/tracker/background_recall": 0.12081623077392578, "metrics/hit_grouping/tracker/completeness": 0.40723106265068054, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5194594860076904, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6934601664543152, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4075048267841339, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4495912194252014, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.45916467905044556, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6388742923736572, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6787706613540649, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4503539800643921, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3943469226360321, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4010484218597412, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6447631120681763, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6633053421974182, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4522993564605713, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32166069746017456, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3175957500934601, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.649594247341156, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7054084539413452, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4620022177696228, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.44051769375801086, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.43907737731933594, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6094081401824951, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7120799422264099, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5104680061340332, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4251864552497864, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.38911622762680054, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5929803252220154, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6471982002258301, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3716745674610138, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.35386067628860474, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.37584036588668823, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6955513954162598, "metrics/hit_grouping/tracker/purity": 0.3865721523761749, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.886074960231781, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.0290346909314394, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.0358264185488224, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03214201331138611, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06410256773233414, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6683878302574158, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2632937431335449, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2698524594306946, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.26111671328544617, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5549819469451904, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6537096500396729, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5064070224761963, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.47375014424324036, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.482158362865448, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7384030222892761, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.48918285965919495, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.554686963558197, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6317580342292786, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6158707141876221, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.059969931840896606, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04139586538076401, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.333606719970703, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28046053647994995, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18883734941482544, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.831540584564209, "metrics/particle/class_1/pid_accuracy": 0.9636476039886475, "metrics/particle/class_2/efficiency": 0.4186204671859741, "metrics/particle/class_2/pid_accuracy": 0.6079545617103577, "metrics/particle/class_3/efficiency": 0.688345730304718, "metrics/particle/class_3/pid_accuracy": 0.9595071077346802, "metrics/particle/class_4/efficiency": 0.625250518321991, "metrics/particle/class_4/pid_accuracy": 0.31891027092933655, "metrics/particle/class_5/efficiency": 0.8448275923728943, "metrics/particle/class_5/pid_accuracy": 0.35147392749786377, "metrics/particle/count/bias_mean": -3.41015625, "metrics/particle/count/mae": 5.220052242279053, "metrics/particle/count/prediction_mean": 67.24478912353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09067090600728989, "metrics/particle/event/energy_response_mean": 0.9871525168418884, "metrics/particle/event/met_abs_error": 9.0482759475708, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07137492299079895, "metrics/particle/event/scalar_pt_response_mean": 0.9897271990776062, "metrics/particle/event/vector_pt_closure": 0.07464388757944107, "metrics/particle/matched/delta_eta_abs_mean": 0.015745393931865692, "metrics/particle/matched/delta_phi_abs_mean": 0.014657420106232166, "metrics/particle/matched/delta_r_mean": 0.024086881428956985, "metrics/particle/matched/energy_relative_abs_mean": 0.16292116045951843, "metrics/particle/matched/pid_accuracy": 0.9371313452720642, "metrics/particle/matched/pt_relative_abs_mean": 0.1601453423500061, "metrics/particle/matching/duplicate_fraction": 0.07279644906520844, "metrics/particle/matching/efficiency": 0.7357591986656189, "metrics/particle/matching/f1": 0.7539539337158203, "metrics/particle/matching/purity": 0.7730714082717896, "metrics/particle/matching/target_coverage_dr0p05": 0.6957963705062866, "metrics/particle/matching/target_coverage_dr0p10": 0.8006929159164429, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5195068717002869, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7795090675354004, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02281857095658779, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16368189454078674, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1622709184885025, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.042288586497306824, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20307379961013794, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009326689876616001, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7917826175689697, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.11701830476522446, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.0909037590026855, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7827.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.30334028601646423, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1896387338638306, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.225713849067688, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.075554609298706, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.17070727050304413, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.184488296508789, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020637147128582, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2457047700881958, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.170017808675766, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.9437777996063232, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9982.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.726606011390686, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.198979377746582, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.62944221496582, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.8689188957214355, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.13710784912109375, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.51899653673172, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.085195541381836, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.756751775741577, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.623348236083984, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026032080873847008, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.175288587808609, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17642131447792053, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03664369881153107, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2183496057987213, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02958579920232296, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5937384366989136, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.30823567509651184, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.381039619445801, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1521.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5683575868606567, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.499126672744751, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.580961227416992, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.409128189086914, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2562020719051361, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 29.195528030395508, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07036536186933517, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9809787273406982, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.4481486678123474, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.241324424743652, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2217.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.551486611366272, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.186924934387207, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.660101890563965, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5413687229156494, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04906760901212692, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07167468965053558, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.449979782104492, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.2110023498535156, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.271886825561523, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02469608373939991, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16120795905590057, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.158005952835083, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.023515932261943817, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19859011471271515, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006185682723298669, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2590077817440033, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00526602054014802, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7578626275062561, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17783.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2596486210823059, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2173407077789307, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9687496423721313, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7911264300346375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15917786955833435, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.715555191040039, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005427529104053974, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4962073564529419, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.017878321930766106, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.102583646774292, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22478.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7642356753349304, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.5466766357421875, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.29397964477539, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.385986328125, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.2873250246047974, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.032049298286438, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.433481216430664, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.887403666973114, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.37579345703125, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024229340255260468, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.162711039185524, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16026875376701355, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.029667600989341736, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20104587078094482, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004754708614200354, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4875343441963196, "metrics/particle/momentum/all/accepted/pz_bias": -0.05449022725224495, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6473356485366821, "metrics/particle/momentum/all/accepted/samples": 27131.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.291962206363678, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2273818254470825, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2097246646881104, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0403772592544556, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.168699711561203, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.396769523620605, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013957378454506397, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8708136081695557, "metrics/particle/momentum/all/assigned/pz_bias": -0.0891810804605484, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.5780982971191406, "metrics/particle/momentum/all/assigned/samples": 34677.0, "metrics/particle/momentum/all/efficiency": 0.7372754812240601, "metrics/particle/momentum/all/event/energy_bias": -0.4412692189216614, "metrics/particle/momentum/all/event/p3_error_mean": 24.889673233032227, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.056351661682129, "metrics/particle/momentum/all/event/px_bias": 1.3753653764724731, "metrics/particle/momentum/all/event/py_bias": -1.62272047996521, "metrics/particle/momentum/all/event/pz_abs_mean": 17.897605895996094, "metrics/particle/momentum/all/event/pz_bias": -7.855157852172852, "metrics/particle/momentum/all/event/pz_residual_rms": 29.15013313293457, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.024063481017947197, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15353424847126007, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15048064291477203, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.03550441563129425, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1904185712337494, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0043410430662333965, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5031101703643799, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.07573157548904419, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5534789562225342, "metrics/particle/momentum/class_1/accepted/samples": 14743.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1630779504776001, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.4612214267253876, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5443362593650818, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4009733200073242, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10540713369846344, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 9.103203773498535, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01065130066126585, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7445816993713379, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.09450238198041916, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.1902015209198, "metrics/particle/momentum/class_1/assigned/samples": 17181.0, "metrics/particle/momentum/class_1/efficiency": 0.8310129046440125, "metrics/particle/momentum/class_1/event/energy_bias": 9.252283096313477, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.185535430908203, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.554783821105957, "metrics/particle/momentum/class_1/event/px_bias": 0.37099653482437134, "metrics/particle/momentum/class_1/event/py_bias": 1.268733024597168, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.713915824890137, "metrics/particle/momentum/class_1/event/pz_bias": -3.78944993019104, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.627967834472656, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03249617666006088, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20066340267658234, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2008238583803177, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.00778920715674758, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24107690155506134, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02905569039285183, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6635178327560425, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.14816583693027496, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.735248565673828, "metrics/particle/momentum/class_2/accepted/samples": 1239.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.700413167476654, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7033933401107788, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7946365475654602, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3894217908382416, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3264014422893524, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.6378989219665527, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.056509945541620255, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.6423370838165283, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.23580527305603027, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.246406555175781, "metrics/particle/momentum/class_2/assigned/samples": 2212.0, "metrics/particle/momentum/class_2/efficiency": 0.414936363697052, "metrics/particle/momentum/class_2/event/energy_bias": -11.313783645629883, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.972875595092773, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.873358726501465, "metrics/particle/momentum/class_2/event/px_bias": 0.20004093647003174, "metrics/particle/momentum/class_2/event/py_bias": -2.1970889568328857, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.328749656677246, "metrics/particle/momentum/class_2/event/pz_bias": -3.610107183456421, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.325464248657227, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023635242134332657, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17076385021209717, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16906584799289703, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.025680270045995712, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20994076132774353, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0019469682592898607, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3103002905845642, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.019337406381964684, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2966060638427734, "metrics/particle/momentum/class_3/accepted/samples": 10786.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.37835314869880676, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.095701217651367, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.9475353956222534, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.7817842960357666, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2152773141860962, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.042375564575195, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010902688838541508, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7305189967155457, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.05753910541534424, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.4405291080474854, "metrics/particle/momentum/class_3/assigned/samples": 14767.0, "metrics/particle/momentum/class_3/efficiency": 0.6943925619125366, "metrics/particle/momentum/class_3/event/energy_bias": 3.2583799362182617, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.42724323272705, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.799570083618164, "metrics/particle/momentum/class_3/event/px_bias": 0.7412797808647156, "metrics/particle/momentum/class_3/event/py_bias": -0.9059042930603027, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.25394058227539, "metrics/particle/momentum/class_3/event/pz_bias": -0.08274725079536438, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.277135848999023, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.021542519330978394, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17091546952724457, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15986619889736176, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0523965060710907, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20276705920696259, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.013392857275903225, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8704608082771301, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.08460485935211182, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.124594211578369, "metrics/particle/momentum/class_4/accepted/samples": 224.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4753660261631012, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.7187933921813965, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.541215896606445, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.419683456420898, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3258427083492279, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 36.7047119140625, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016853932291269302, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1661347150802612, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.10937277972698212, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.2110698223114014, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.5973333120346069, "metrics/particle/momentum/class_4/event/energy_bias": -0.9702913165092468, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.2632150650024414, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.3711740970611572, "metrics/particle/momentum/class_4/event/px_bias": 0.07978294044733047, "metrics/particle/momentum/class_4/event/py_bias": 0.2642472982406616, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8016458749771118, "metrics/particle/momentum/class_4/event/pz_bias": -0.47379228472709656, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.843137741088867, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.018563201650977135, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15965557098388672, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15496619045734406, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.01724250800907612, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19779588282108307, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.035971224308013916, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4889259338378906, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.354233056306839, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.409175157546997, "metrics/particle/momentum/class_5/accepted/samples": 139.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.10462857782840729, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.6128230094909668, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.6690402030944824, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.5317918062210083, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.13664595782756805, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.51124906539917, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05590062215924263, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.2172560691833496, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.36440297961235046, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.1838579177856445, "metrics/particle/momentum/class_5/assigned/samples": 161.0, "metrics/particle/momentum/class_5/efficiency": 0.8475610017776489, "metrics/particle/momentum/class_5/event/energy_bias": -0.6678568720817566, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.3976759910583496, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.7632478475570679, "metrics/particle/momentum/class_5/event/px_bias": -0.01673474721610546, "metrics/particle/momentum/class_5/event/py_bias": -0.05270739644765854, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2895004749298096, "metrics/particle/momentum/class_5/event/pz_bias": 0.100939080119133, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.346010684967041, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4127679169178009, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4130435585975647, "metrics/set_slot/angular_recall_all_dr0p10": 0.9408068060874939, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7918010354042053, "metrics/set_slot/soft_count_bias_mean": -0.6982465386390686}, "step_train_time": 4.85127329826355, "step_valid_time": 48417.89451789856, "step_total_time": 48422.74579167366, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0099976412710734, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1792997455244203, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3041, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2870, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2819, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9382151029748284, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9437684972048668, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.940983606557377, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05623150279513318, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9215429879045439, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9269976981256166, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9242622950819672, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9382151029748284, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9961893232082331, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18211890603973047, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2255, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2130, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2115, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9321663019693655, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9445676274944568, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9383259911894273, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05543237250554324, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.925601750547046, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9379157427937915, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9317180616740088, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9321663019693655, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.996694605738884, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18533360526025122, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1303, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1201, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1192, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9084720121028744, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9217191097467383, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.915047619047619, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07828089025326171, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9016641452344932, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9148119723714505, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9081904761904762, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9084720121028744} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_2000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..45897261a934d4ccca3a23ce716bc772a653f52c --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 0.094732853353543, "train": {"Classification_binary": 0.44824451208114624, "Cardinality": 0.30913954973220825, "Classification": 0.35189157724380493, "Regression_pt": 0.49956822395324707, "Regression_eta": 0.00617800559848547, "Regression_sin_phi": 0.002852251986041665, "Regression_cos_phi": 0.00308591453358531, "Regression_energy": 0.4921742081642151, "HitGrouping": 0.3137361407279968, "Auxiliary": 0.5673047304153442, "Total": 3.0006682872772217, "Total_unweighted": 1.8039946935605258}, "train_group": {}, "valid": {"Classification_binary": 0.4044543107350667, "Cardinality": 0.2908800045649211, "Classification": 0.3387953042984009, "Regression_pt": 0.498080571492513, "Regression_eta": 0.005675564830501874, "Regression_sin_phi": 0.002851571887731552, "Regression_cos_phi": 0.002853380205730597, "Regression_energy": 0.49624156951904297, "HitGrouping": 0.30980898936589557, "Auxiliary": 0.5534594456354777, "Total": 2.880671819051107, "Total_unweighted": 1.7489522729689875, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8409088253974915, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.09224721789360046, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.19721774756908417, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.09070124477148056, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.05850187689065933, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.419986754655838, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8119106292724609, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.0918630063533783, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.16248153150081635, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.08778608590364456, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.060447998344898224, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.44571638107299805, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.83783358335495, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.0834396705031395, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.16977551579475403, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.08179205656051636, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.05283857509493828, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.43633419275283813, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.836752712726593, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.08676190674304962, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.1775229424238205, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.08495648950338364, "metrics/hit_grouping/calorimeter/all/object_purity": 0.05514506250619888, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.43211010098457336, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0014036480570212007, "metrics/hit_grouping/calorimeter/background_recall": 0.013135640881955624, "metrics/hit_grouping/calorimeter/completeness": 0.19382601976394653, "metrics/hit_grouping/calorimeter/energy_completeness": 0.23648492991924286, "metrics/hit_grouping/calorimeter/energy_purity": 0.2332991659641266, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5756056904792786, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.840030312538147, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.07430171966552734, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.13205507397651672, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.07252177596092224, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.06398747861385345, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4151684045791626, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8405025005340576, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.08896636962890625, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.18701060116291046, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.08683910965919495, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.04826918989419937, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.42848917841911316, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8262919783592224, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.10994714498519897, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.2606728672981262, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.10841399431228638, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.04404247924685478, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.47030162811279297, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8223608136177063, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.10024493932723999, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.22726726531982422, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.09729433804750443, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.05953434482216835, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.47185471653938293, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8523194789886475, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3636670708656311, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5898095965385437, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3634442389011383, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.1918192058801651, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4029850661754608, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8495714068412781, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.04002559930086136, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.07768716663122177, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.03919919580221176, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.034387025982141495, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.39505794644355774, "metrics/hit_grouping/calorimeter/purity": 0.1906692385673523, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9057464599609375, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.022260386496782303, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0714285746216774, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.02130325883626938, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.007568382658064365, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03759398311376572, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8762062788009644, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.023672331124544144, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.04178928956389427, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.02314707636833191, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.01464136503636837, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2666274309158325, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8335753679275513, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.09109601378440857, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.18631504476070404, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.089210145175457, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.05802805349230766, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.44714999198913574, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6886792182922363, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3360857665538788, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.19359023869037628, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.13812817633152008, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.679704487323761, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7422991394996643, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2531645596027374, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.1946691870689392, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.14242923259735107, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5316455960273743, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6140236854553223, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.32162901759147644, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.15483301877975464, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.11275099962949753, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7516189217567444, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.646443784236908, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.3210151493549347, "metrics/hit_grouping/tracker/all/object_completeness": 0.1696874052286148, "metrics/hit_grouping/tracker/all/object_purity": 0.12274275720119476, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7132680416107178, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.008148030377924442, "metrics/hit_grouping/tracker/background_recall": 0.025339437648653984, "metrics/hit_grouping/tracker/completeness": 0.1754477620124817, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6348801255226135, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6331847906112671, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.32588112354278564, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.19819803535938263, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.1652919352054596, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7422409057617188, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6546246409416199, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.30931204557418823, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.14945952594280243, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.09331034868955612, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7062987685203552, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6655897498130798, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.3226329982280731, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.1311379373073578, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.06446090340614319, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6605951189994812, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.679953932762146, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.3483060300350189, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.175917848944664, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.1114581972360611, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6780164837837219, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6879421472549438, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.42241379618644714, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.22213037312030792, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.12298497557640076, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6453201770782471, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5954402685165405, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.2673584222793579, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.15218622982501984, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.13736964762210846, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7712771892547607, "metrics/hit_grouping/tracker/purity": 0.1650252789258957, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.9017890691757202, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.018099548295140266, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.01028355024755001, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0028798545245081186, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.04675716534256935, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.7038066387176514, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.15126484632492065, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.08123542368412018, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.03687772899866104, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5198761224746704, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6023339033126831, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.3843657374382019, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.2029295563697815, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.14983676373958588, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8331375122070312, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 0.5, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.25339770317077637, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.550000011920929, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6151204705238342, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7011361122131348, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5259832143783569, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.27189692854881287, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.04849160090088844, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5252946019172668, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.699999988079071, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.081295013427734, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.07367834448814392, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": -0.0179448239505291, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.45529618859291077, "metrics/particle/class_1/pid_accuracy": 0.8107066750526428, "metrics/particle/class_2/efficiency": 0.34194132685661316, "metrics/particle/class_2/pid_accuracy": 0.0735342800617218, "metrics/particle/class_3/efficiency": 0.4199768304824829, "metrics/particle/class_3/pid_accuracy": 0.8615897297859192, "metrics/particle/class_4/efficiency": 0.44088175892829895, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 0.515325665473938, "metrics/particle/class_5/pid_accuracy": 0.09665427356958389, "metrics/particle/count/bias_mean": -3.310546875, "metrics/particle/count/mae": 10.784504890441895, "metrics/particle/count/prediction_mean": 67.34439849853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.17356769740581512, "metrics/particle/event/energy_response_mean": 0.994208812713623, "metrics/particle/event/met_abs_error": 14.808181762695312, "metrics/particle/event/scalar_pt_relative_abs_error": 0.16851478815078735, "metrics/particle/event/scalar_pt_response_mean": 1.0652191638946533, "metrics/particle/event/vector_pt_closure": 0.1293148547410965, "metrics/particle/matched/delta_eta_abs_mean": 0.02283284068107605, "metrics/particle/matched/delta_phi_abs_mean": 0.02207673341035843, "metrics/particle/matched/delta_r_mean": 0.035556282848119736, "metrics/particle/matched/energy_relative_abs_mean": 0.20915783941745758, "metrics/particle/matched/pid_accuracy": 0.7723233103752136, "metrics/particle/matched/pt_relative_abs_mean": 0.20683853328227997, "metrics/particle/matching/duplicate_fraction": 0.10216452181339264, "metrics/particle/matching/efficiency": 0.4313436448574066, "metrics/particle/matching/f1": 0.44169139862060547, "metrics/particle/matching/purity": 0.4525478184223175, "metrics/particle/matching/target_coverage_dr0p05": 0.3726388216018677, "metrics/particle/matching/target_coverage_dr0p10": 0.5313934087753296, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.2654663324356079, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5186960101127625, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.0357995331287384, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.2096136212348938, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.2070617526769638, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.030505776405334473, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24874502420425415, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0200803205370903, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.5291210412979126, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.041947800666093826, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.2539031505584717, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4233.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.8898840546607971, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.5165367126464844, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.79899263381958, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.603789806365967, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.4496636986732483, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 15.608407020568848, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03872857615351677, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.2702486515045166, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.2419254034757614, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.728292942047119, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9218.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.3929632306098938, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 5.680276870727539, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 19.625694274902344, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 8.867280006408691, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -1.7483783960342407, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.6583302021026611, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 16.559803009033203, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 5.949862480163574, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 25.37361717224121, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03852822631597519, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.22123228013515472, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.2202877700328827, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.013040730729699135, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2613608241081238, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.062157221138477325, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.983354330062866, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.4484507441520691, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 6.408515453338623, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 547.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.6522659063339233, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.8458197116851807, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10.163363456726074, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10.001277923583984, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.6044198870658875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 74.70895385742188, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.1016574576497078, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.282243728637695, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.27663367986679077, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.454068183898926, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1810.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.19833213090896606, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -16.195735931396484, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 18.978151321411133, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 3.0300283432006836, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.5021986961364746, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.2302316576242447, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 18.61460304260254, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.5067601203918457, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 40.51272201538086, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.035536881536245346, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20924797654151917, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.20640802383422852, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.02937665954232216, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.24711784720420837, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0028174135368317366, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.488095223903656, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03714173287153244, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.2894395589828491, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 11003.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.6901492476463318, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.831695318222046, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.310328483581543, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.0932533740997314, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.4053785800933838, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.335113525390625, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.019989170134067535, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.287785530090332, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.188927561044693, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.3802499771118164, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22162.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.4728609025478363, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 12.276283264160156, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 21.22559928894043, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 17.089757919311523, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.0346834659576416, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.930260181427002, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 10.674352645874023, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 3.601747512817383, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 15.576562881469727, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.035711001604795456, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20976139605045319, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.2070644348859787, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.029113324359059334, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24806174635887146, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.009503896348178387, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8537778854370117, "metrics/particle/momentum/all/accepted/pz_bias": 0.05268567055463791, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.3285470008850098, "metrics/particle/momentum/all/accepted/samples": 15783.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.7980909943580627, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.4126665592193604, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.8197739124298096, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.6117725372314453, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.4285326898097992, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 22.153602600097656, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.02964748442173004, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.7239503860473633, "metrics/particle/momentum/all/assigned/pz_bias": 0.20842990279197693, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.3258585929870605, "metrics/particle/momentum/all/assigned/samples": 33190.0, "metrics/particle/momentum/all/efficiency": 0.42889752984046936, "metrics/particle/momentum/all/event/energy_bias": 1.7608250379562378, "metrics/particle/momentum/all/event/p3_error_mean": 44.483463287353516, "metrics/particle/momentum/all/event/pt_vector_error_mean": 21.316659927368164, "metrics/particle/momentum/all/event/px_bias": -4.285260200500488, "metrics/particle/momentum/all/event/py_bias": 3.818821907043457, "metrics/particle/momentum/all/event/pz_abs_mean": 34.10807800292969, "metrics/particle/momentum/all/event/pz_bias": 9.044849395751953, "metrics/particle/momentum/all/event/pz_residual_rms": 50.29450988769531, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.04508080333471298, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.22755257785320282, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.225215882062912, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.041822485625743866, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2650982737541199, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.011469891294836998, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.005103349685669, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06141451746225357, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.3900880813598633, "metrics/particle/momentum/class_1/accepted/samples": 8021.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.6560896039009094, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 2.445478916168213, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.7412502765655518, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 2.493217706680298, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.39380258321762085, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 12.914300918579102, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.031420089304447174, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.7864441871643066, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.23591794073581696, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.349007606506348, "metrics/particle/momentum/class_1/assigned/samples": 16168.0, "metrics/particle/momentum/class_1/efficiency": 0.45211657881736755, "metrics/particle/momentum/class_1/event/energy_bias": 16.686235427856445, "metrics/particle/momentum/class_1/event/p3_error_mean": 38.05699157714844, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 23.250507354736328, "metrics/particle/momentum/class_1/event/px_bias": -6.319447040557861, "metrics/particle/momentum/class_1/event/py_bias": 2.451077938079834, "metrics/particle/momentum/class_1/event/pz_abs_mean": 25.725561141967773, "metrics/particle/momentum/class_1/event/pz_bias": 12.327751159667969, "metrics/particle/momentum/class_1/event/pz_residual_rms": 34.05847930908203, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.041205115616321564, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2396155297756195, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23699204623699188, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08497937768697739, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2756538391113281, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028237584978342056, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7082496881484985, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.11710895597934723, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7568390369415283, "metrics/particle/momentum/class_2/accepted/samples": 1027.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9187418818473816, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7547116279602051, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1740591526031494, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7571251392364502, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.38336268067359924, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.83888840675354, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05985915660858154, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8739089965820312, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.16354720294475555, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.682362079620361, "metrics/particle/momentum/class_2/assigned/samples": 2272.0, "metrics/particle/momentum/class_2/efficiency": 0.3439383804798126, "metrics/particle/momentum/class_2/event/energy_bias": -30.906763076782227, "metrics/particle/momentum/class_2/event/p3_error_mean": 33.848960876464844, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 18.373287200927734, "metrics/particle/momentum/class_2/event/px_bias": -1.0480107069015503, "metrics/particle/momentum/class_2/event/py_bias": -0.02948063611984253, "metrics/particle/momentum/class_2/event/pz_abs_mean": 23.96776008605957, "metrics/particle/momentum/class_2/event/pz_bias": -1.769740104675293, "metrics/particle/momentum/class_2/event/pz_residual_rms": 37.85617446899414, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023102296516299248, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1813409924507141, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17840789258480072, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.0015344031853601336, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21850982308387756, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0024698982015252113, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4788758158683777, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.016418058425188065, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.7557770013809204, "metrics/particle/momentum/class_3/accepted/samples": 6478.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.9432212114334106, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.812671661376953, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.376847267150879, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.250827789306641, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4758494794368744, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 30.5587158203125, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.021552935242652893, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.4323971271514893, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.1791280210018158, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.7125236988067627, "metrics/particle/momentum/class_3/assigned/samples": 14244.0, "metrics/particle/momentum/class_3/efficiency": 0.4170475900173187, "metrics/particle/momentum/class_3/event/energy_bias": 26.376253128051758, "metrics/particle/momentum/class_3/event/p3_error_mean": 29.01740074157715, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 16.912275314331055, "metrics/particle/momentum/class_3/event/px_bias": 1.8515139818191528, "metrics/particle/momentum/class_3/event/py_bias": 2.9022374153137207, "metrics/particle/momentum/class_3/event/pz_abs_mean": 19.81871795654297, "metrics/particle/momentum/class_3/event/pz_bias": -2.2017099857330322, "metrics/particle/momentum/class_3/event/pz_residual_rms": 34.018253326416016, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.039734043180942535, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.24544234573841095, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.2428663671016693, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.11488920450210571, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2792595624923706, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.03750000149011612, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 2.0036754608154297, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.27503857016563416, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.739044189453125, "metrics/particle/momentum/class_4/accepted/samples": 160.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.8274898529052734, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 8.895049095153809, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 8.340968132019043, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 8.131784439086914, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.46348315477371216, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 26.287012100219727, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03651685267686844, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.1364939212799072, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2601924538612366, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.063804626464844, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.4266666769981384, "metrics/particle/momentum/class_4/event/energy_bias": -6.539658546447754, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.854221343994141, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.774823188781738, "metrics/particle/momentum/class_4/event/px_bias": 0.6257258057594299, "metrics/particle/momentum/class_4/event/py_bias": -0.5165429711341858, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.601644277572632, "metrics/particle/momentum/class_4/event/pz_bias": -0.0370001345872879, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.958283424377441, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03816261887550354, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.26166900992393494, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.24396926164627075, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08702586591243744, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2815989553928375, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.07216494530439377, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.4342708587646484, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.7041131258010864, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.612485408782959, "metrics/particle/momentum/class_5/accepted/samples": 97.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.4251510500907898, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 1.9657580852508545, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.5540071725845337, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 1.0432723760604858, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.2800000011920929, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 8.77192211151123, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.13333334028720856, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.276724338531494, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.5850644707679749, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 8.805756568908691, "metrics/particle/momentum/class_5/assigned/samples": 150.0, "metrics/particle/momentum/class_5/efficiency": 0.5914633870124817, "metrics/particle/momentum/class_5/event/energy_bias": -3.8552393913269043, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.9259209632873535, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.7930140495300293, "metrics/particle/momentum/class_5/event/px_bias": 0.6049578189849854, "metrics/particle/momentum/class_5/event/py_bias": -0.9884699583053589, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.4828572273254395, "metrics/particle/momentum/class_5/event/pz_bias": 0.7255479693412781, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.376890182495117, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.7354034781455994, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.740631639957428, "metrics/set_slot/angular_recall_all_dr0p10": 0.824724018573761, "metrics/set_slot/angular_recall_selected_dr0p10": 0.5746180415153503, "metrics/set_slot/soft_count_bias_mean": 1.6569472551345825}, "step_train_time": 4.823162317276001, "step_valid_time": 4700.194816112518, "step_total_time": 4705.017978668213, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1564128045959583, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3776129941435805, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3120, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2466, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2075, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8061457992808108, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7903846153846154, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.798187408965852, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.20961538461538462, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6783262504086303, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6650641025641025, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6716297135458812, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8061457992808108, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.096321085339533, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3438178407527306, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1803, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1640, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7890590809628009, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8031180400890868, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7960264900662252, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.19688195991091315, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7177242888402626, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7305122494432071, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7240618101545254, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7890590809628009, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0518641993698297, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.353873813204886, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1238, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 976, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 879, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7382753403933434, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7883683360258481, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7625, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.21163166397415187, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6649016641452344, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.710016155088853, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.68671875, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7382753403933434} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_20000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..1ca70e76b475b3fb0a62551592c90b7e34668dc0 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.9473285335354301, "train": {"Classification_binary": 0.17290301620960236, "Cardinality": 0.14869296550750732, "Classification": 0.10298150777816772, "Regression_pt": 0.1652108132839203, "Regression_eta": 0.0018566218204796314, "Regression_sin_phi": 0.0008942846907302737, "Regression_cos_phi": 0.0008548786863684654, "Regression_energy": 0.16393055021762848, "HitGrouping": 0.21052996814250946, "Auxiliary": 0.2556267976760864, "Total": 1.2077257633209229, "Total_unweighted": 0.6086316726868972}, "train_group": {}, "valid": {"Classification_binary": 0.19640292723973593, "Cardinality": 0.16042436162630716, "Classification": 0.11423376202583313, "Regression_pt": 0.16708834966023764, "Regression_eta": 0.0021731994735697904, "Regression_sin_phi": 0.0009893004316836596, "Regression_cos_phi": 0.0009667991350094477, "Regression_energy": 0.16499706109364828, "HitGrouping": 0.20688859621683756, "Auxiliary": 0.26788872480392456, "Total": 1.2857789993286133, "Total_unweighted": 0.6468513990597179, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8103452920913696, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4495753049850464, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3909340500831604, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4503706395626068, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.44296327233314514, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.45225703716278076, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7529471516609192, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3957580029964447, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.37813884019851685, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3920959234237671, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3913920521736145, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5288035273551941, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8346788287162781, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.42620325088500977, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.35542699694633484, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.42688581347465515, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.40599122643470764, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4043971300125122, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8209615349769592, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4309201240539551, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3679472506046295, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.43129962682724, "metrics/hit_grouping/calorimeter/all/object_purity": 0.41605857014656067, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.42855504155158997, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002443996025249362, "metrics/hit_grouping/calorimeter/background_recall": 0.10910192877054214, "metrics/hit_grouping/calorimeter/completeness": 0.4929647743701935, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5086261034011841, "metrics/hit_grouping/calorimeter/energy_purity": 0.5018160343170166, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4360862374305725, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8416211605072021, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4589846432209015, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.29614245891571045, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4604541063308716, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.44552794098854065, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3742603659629822, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8189745545387268, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4508405923843384, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4066098630428314, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4502548575401306, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.43546122312545776, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4433440864086151, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.780838668346405, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3536289632320404, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4754060208797455, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3527582287788391, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3364243507385254, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5243619680404663, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8075271248817444, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.47002044320106506, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4557974636554718, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.47265857458114624, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.45106735825538635, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.46307432651519775, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7962747812271118, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5363736748695374, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5820895433425903, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5306538343429565, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4860677421092987, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.47246524691581726, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8375996351242065, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.37849169969558716, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2528665065765381, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.37729495763778687, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3720242977142334, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.38812926411628723, "metrics/hit_grouping/calorimeter/purity": 0.48544174432754517, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8464464545249939, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08966735005378723, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12781955301761627, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08928347378969193, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04617512598633766, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04135338217020035, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8177006244659424, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.18649613857269287, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11006474494934082, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.18692468106746674, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18420085310935974, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.32136550545692444, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8207159638404846, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4492001533508301, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3852798044681549, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.44958946108818054, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.43415114283561707, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.44044229388237, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7294649481773376, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43198609352111816, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.45518168807029724, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4527738690376282, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5838041305541992, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7643994688987732, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3122362792491913, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.43723002076148987, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.41723868250846863, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4309825301170349, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6571268439292908, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43833646178245544, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.37372738122940063, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.37560099363327026, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6823284029960632, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6872833371162415, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42707034945487976, "metrics/hit_grouping/tracker/all/object_completeness": 0.40345266461372375, "metrics/hit_grouping/tracker/all/object_purity": 0.4023953676223755, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6334817409515381, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.024984117597341537, "metrics/hit_grouping/tracker/background_recall": 0.1500236839056015, "metrics/hit_grouping/tracker/completeness": 0.4059229791164398, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5225039124488831, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6967973709106445, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41872698068618774, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4515582323074341, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.45250824093818665, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.626512348651886, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6830554604530334, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44381919503211975, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3964892625808716, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3964115083217621, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6433109641075134, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6719136834144592, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4275924265384674, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.31141817569732666, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3052586317062378, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6380522847175598, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7139036059379578, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4602190852165222, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43996864557266235, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.42727091908454895, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5962468981742859, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7214841842651367, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.47044333815574646, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.41501522064208984, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3843516707420349, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5738916397094727, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6465451121330261, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37620046734809875, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3539077937602997, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.37329086661338806, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6925709247589111, "metrics/hit_grouping/tracker/purity": 0.38840198516845703, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8774420619010925, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.0214932132512331, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03611614555120468, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03524470701813698, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06523378938436508, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6675541400909424, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2762003242969513, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2778918147087097, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2615709602832794, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5487867593765259, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6612028479576111, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5036091804504395, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.47157347202301025, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4721430242061615, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7269878387451172, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5068252086639404, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.59639573097229, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6818786263465881, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6028075814247131, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.04232751205563545, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.02833285555243492, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.281208038330078, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.289822518825531, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.19819936156272888, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8313689827919006, "metrics/particle/class_1/pid_accuracy": 0.9640530347824097, "metrics/particle/class_2/efficiency": 0.4373088777065277, "metrics/particle/class_2/pid_accuracy": 0.6083915829658508, "metrics/particle/class_3/efficiency": 0.6910565495491028, "metrics/particle/class_3/pid_accuracy": 0.9568504691123962, "metrics/particle/class_4/efficiency": 0.6322645545005798, "metrics/particle/class_4/pid_accuracy": 0.339144229888916, "metrics/particle/class_5/efficiency": 0.8563218116760254, "metrics/particle/class_5/pid_accuracy": 0.3624161183834076, "metrics/particle/count/bias_mean": -4.028645992279053, "metrics/particle/count/mae": 5.4140625, "metrics/particle/count/prediction_mean": 66.62630462646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08491426706314087, "metrics/particle/event/energy_response_mean": 0.9803231358528137, "metrics/particle/event/met_abs_error": 7.509986877441406, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0680939108133316, "metrics/particle/event/scalar_pt_response_mean": 0.9933468699455261, "metrics/particle/event/vector_pt_closure": 0.07029733806848526, "metrics/particle/matched/delta_eta_abs_mean": 0.018558284267783165, "metrics/particle/matched/delta_phi_abs_mean": 0.014983825385570526, "metrics/particle/matched/delta_r_mean": 0.02665071189403534, "metrics/particle/matched/energy_relative_abs_mean": 0.15786653757095337, "metrics/particle/matched/pid_accuracy": 0.9358014464378357, "metrics/particle/matched/pt_relative_abs_mean": 0.15818965435028076, "metrics/particle/matching/duplicate_fraction": 0.06802947074174881, "metrics/particle/matching/efficiency": 0.7384589910507202, "metrics/particle/matching/f1": 0.7601297497749329, "metrics/particle/matching/purity": 0.7831108570098877, "metrics/particle/matching/target_coverage_dr0p05": 0.6704937219619751, "metrics/particle/matching/target_coverage_dr0p10": 0.7974586486816406, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5269796848297119, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7829920649528503, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02472352609038353, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1584317535161972, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15952059626579285, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015657732263207436, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19957448542118073, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009205548092722893, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8053376078605652, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.10453418642282486, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2031033039093018, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7930.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2705846428871155, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1541236639022827, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.259140968322754, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0926828384399414, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15265239775180817, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.997232437133789, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019576022401452065, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2107923030853271, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.10832429677248001, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.4501523971557617, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9859.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.736167848110199, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.3363403379917145, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.561994552612305, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.554005146026611, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.24316559731960297, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.03971449285745621, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.017607688903809, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.0072922706604004, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.475872993469238, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026977209374308586, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1795470267534256, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17646707594394684, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02029898762702942, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21728745102882385, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.034393250942230225, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.558930516242981, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.021846886724233627, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.74955153465271, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1541.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5723059177398682, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8925485610961914, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.208300590515137, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6.023791313171387, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23883317410945892, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 149.70799255371094, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07702825963497162, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9860053062438965, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.20049798488616943, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.222455024719238, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2194.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5587382316589355, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.727993011474609, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.54252815246582, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3682297468185425, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04866914078593254, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.052771829068660736, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.378545761108398, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.5231649875640869, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.039112091064453, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.027715591713786125, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15607133507728577, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15591993927955627, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.004961871542036533, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1965351402759552, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008452133042737842, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.27000340819358826, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.005427523050457239, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7893771529197693, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17747.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2608831524848938, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.317487359046936, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9921504259109497, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8089325428009033, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15609341859817505, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.436323165893555, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006039727944880724, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5162215232849121, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.01317775808274746, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.0012850761413574, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22352.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7626885771751404, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.308172345161438, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.0330171585083, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.100016593933105, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.775197744369507, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.5760338306427002, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.519489765167236, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.22816908359527588, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.671188354492188, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026802044361829758, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15808816254138947, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15813229978084564, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.008946466259658337, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19865305721759796, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00518039520829916, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.498948872089386, "metrics/particle/momentum/all/accepted/pz_bias": 0.035231996327638626, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6175284385681152, "metrics/particle/momentum/all/accepted/samples": 27218.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2835225760936737, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3073457479476929, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.4012913703918457, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.222793698310852, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1603836715221405, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 38.8125114440918, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014445574954152107, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8727534413337708, "metrics/particle/momentum/all/assigned/pz_bias": 0.02681662328541279, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.3818540573120117, "metrics/particle/momentum/all/assigned/samples": 34405.0, "metrics/particle/momentum/all/efficiency": 0.7396396398544312, "metrics/particle/momentum/all/event/energy_bias": -3.0834805965423584, "metrics/particle/momentum/all/event/p3_error_mean": 23.180431365966797, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.361632347106934, "metrics/particle/momentum/all/event/px_bias": -3.0670325756073, "metrics/particle/momentum/all/event/py_bias": 1.562976598739624, "metrics/particle/momentum/all/event/pz_abs_mean": 16.691143035888672, "metrics/particle/momentum/all/event/pz_bias": 1.2559583187103271, "metrics/particle/momentum/all/event/pz_residual_rms": 27.191450119018555, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.028715278953313828, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1466062068939209, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14893515408039093, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01052505150437355, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.188697949051857, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005080612376332283, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.530975878238678, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.057792384177446365, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5918426513671875, "metrics/particle/momentum/class_1/accepted/samples": 14762.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.16360649466514587, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5632880330085754, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6400823593139648, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4813666045665741, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09638907015323639, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.5817694664001465, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010709896683692932, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7575045228004456, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.10779831558465958, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.1471915245056152, "metrics/particle/momentum/class_1/assigned/samples": 17087.0, "metrics/particle/momentum/class_1/efficiency": 0.8320838809013367, "metrics/particle/momentum/class_1/event/energy_bias": 7.751537799835205, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.19734001159668, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.209101676940918, "metrics/particle/momentum/class_1/event/px_bias": 0.6927634477615356, "metrics/particle/momentum/class_1/event/py_bias": 2.700425386428833, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.10899829864502, "metrics/particle/momentum/class_1/event/pz_bias": 4.585384845733643, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.233657836914062, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.034153152257204056, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18773876130580902, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19147157669067383, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.012801380828022957, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2312985062599182, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031037092208862305, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5657318830490112, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.005757596809417009, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.604048252105713, "metrics/particle/momentum/class_2/accepted/samples": 1321.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6304593086242676, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7687324285507202, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9868870377540588, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6416282057762146, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30334538221359253, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 7.069331645965576, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06283906102180481, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.689035177230835, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.21414241194725037, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.522512435913086, "metrics/particle/momentum/class_2/assigned/samples": 2212.0, "metrics/particle/momentum/class_2/efficiency": 0.4423978626728058, "metrics/particle/momentum/class_2/event/energy_bias": -11.969552040100098, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.893583297729492, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.510583877563477, "metrics/particle/momentum/class_2/event/px_bias": -1.93739652633667, "metrics/particle/momentum/class_2/event/py_bias": -1.1421663761138916, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.475691795349121, "metrics/particle/momentum/class_2/event/pz_bias": -3.127537965774536, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.89068603515625, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023353897035121918, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16981934010982513, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16640006005764008, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.008669612929224968, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2070261836051941, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001670998870395124, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.29969239234924316, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0040303622372448444, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0997692346572876, "metrics/particle/momentum/class_3/accepted/samples": 10772.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3695419728755951, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.1610329151153564, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.2916383743286133, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.1135168075561523, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.21012605726718903, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 58.845497131347656, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011304466985166073, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7117736339569092, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.042975589632987976, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.922410249710083, "metrics/particle/momentum/class_3/assigned/samples": 14596.0, "metrics/particle/momentum/class_3/efficiency": 0.6934912800788879, "metrics/particle/momentum/class_3/event/energy_bias": 2.1023056507110596, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.863908767700195, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.542037010192871, "metrics/particle/momentum/class_3/event/px_bias": -1.9364138841629028, "metrics/particle/momentum/class_3/event/py_bias": -0.42015284299850464, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.834712028503418, "metrics/particle/momentum/class_3/event/pz_bias": -0.49686914682388306, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.639975547790527, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.0271526537835598, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16501504182815552, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15965688228607178, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.046319231390953064, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20552486181259155, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.008849557489156723, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8473575115203857, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2468520998954773, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0225539207458496, "metrics/particle/momentum/class_4/accepted/samples": 226.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.45660048723220825, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.939370155334473, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.5672736167907715, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.457616329193115, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3141210377216339, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 23.537582397460938, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014409221708774567, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1494280099868774, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1824984848499298, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.5159173011779785, "metrics/particle/momentum/class_4/assigned/samples": 347.0, "metrics/particle/momentum/class_4/efficiency": 0.6026666760444641, "metrics/particle/momentum/class_4/event/energy_bias": -0.2314896285533905, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.187450885772705, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.456892251968384, "metrics/particle/momentum/class_4/event/px_bias": 0.30193743109703064, "metrics/particle/momentum/class_4/event/py_bias": 0.32908833026885986, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5727676153182983, "metrics/particle/momentum/class_4/event/pz_bias": -0.08310817927122116, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.43074369430542, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02030719630420208, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17556731402873993, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17508365213871002, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00866782944649458, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21900330483913422, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.036496348679065704, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.854042649269104, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.7375584244728088, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.435201644897461, "metrics/particle/momentum/class_5/accepted/samples": 137.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07484111189842224, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.30977556109428406, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.35431361198425293, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.18491804599761963, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.14723926782608032, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.9328741431236267, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.030674846842885017, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.1322474479675293, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.7257907390594482, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.489542007446289, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.8353658318519592, "metrics/particle/momentum/class_5/event/energy_bias": -0.7362821102142334, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.3685336112976074, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.8276232481002808, "metrics/particle/momentum/class_5/event/px_bias": -0.18792317807674408, "metrics/particle/momentum/class_5/event/py_bias": 0.09578203409910202, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.1891248226165771, "metrics/particle/momentum/class_5/event/pz_bias": 0.3780885338783264, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.915574789047241, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.42081472277641296, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.42435723543167114, "metrics/set_slot/angular_recall_all_dr0p10": 0.9361351132392883, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7898567914962769, "metrics/set_slot/soft_count_bias_mean": -1.2082197666168213}, "step_train_time": 4.586990594863892, "step_valid_time": 50861.94976043701, "step_total_time": 50866.53675150871, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0045938656302957, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18201378918411248, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3009, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2842, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2795, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9290617848970252, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9444998338318378, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.936717205009888, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05550016616816218, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9136972866949984, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9288800265869059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9212261041529334, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9290617848970252, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9939697345152024, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1686373184975285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2260, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2143, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2134, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9378555798687089, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9482300884955752, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.943014301430143, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05176991150442478, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9339168490153172, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9442477876106194, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.939053905390539, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9378555798687089, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9906070233641553, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.174846414392532, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1295, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1206, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1196, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9122541603630863, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9312741312741313, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9216660298051204, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06872586872586872, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9046898638426626, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9235521235521236, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9140236912495223, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9122541603630863} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_21000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..c0a372aede35fdead8b478395513e8fb48bb2bd5 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 0.9946949602122016, "train": {"Classification_binary": 0.17010076344013214, "Cardinality": 0.12360506504774094, "Classification": 0.09665361791849136, "Regression_pt": 0.15900923311710358, "Regression_eta": 0.0017140156123787165, "Regression_sin_phi": 0.0007947888225317001, "Regression_cos_phi": 0.0008380975341424346, "Regression_energy": 0.1582554131746292, "HitGrouping": 0.20982958376407623, "Auxiliary": 0.23987874388694763, "Total": 1.1402060985565186, "Total_unweighted": 0.5873659296194091}, "train_group": {}, "valid": {"Classification_binary": 0.1973656415939331, "Cardinality": 0.14384292562802634, "Classification": 0.11005964875221252, "Regression_pt": 0.15997598568598428, "Regression_eta": 0.0017960495315492153, "Regression_sin_phi": 0.0009387456811964512, "Regression_cos_phi": 0.0009212445778151354, "Regression_energy": 0.15861048301060995, "HitGrouping": 0.2051012913386027, "Auxiliary": 0.2592203617095947, "Total": 1.2320545514424641, "Total_unweighted": 0.6296677988333007, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8086954355239868, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4577447175979614, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3935838043689728, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4593009352684021, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.45407989621162415, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4472414255142212, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7553103566169739, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3998245298862457, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3755539059638977, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.398786723613739, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3960503339767456, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5258493423461914, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8306638598442078, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4345717132091522, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.35542699694633484, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.43465152382850647, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.41828978061676025, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4075908362865448, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8181582093238831, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4388943314552307, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.36854931712150574, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4393346905708313, "metrics/hit_grouping/calorimeter/all/object_purity": 0.42740583419799805, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.42878440022468567, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002795094856992364, "metrics/hit_grouping/calorimeter/background_recall": 0.1237456277012825, "metrics/hit_grouping/calorimeter/completeness": 0.49145326018333435, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5037568807601929, "metrics/hit_grouping/calorimeter/energy_purity": 0.49707964062690735, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4294058084487915, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8424768447875977, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4671172499656677, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.29688212275505066, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4695419371128082, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.45538705587387085, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3693673312664032, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8183043003082275, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4556555449962616, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.40211883187294006, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.45535793900489807, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4465925991535187, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4348226487636566, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7684255838394165, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3644627034664154, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4808584749698639, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3616006374359131, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.35102376341819763, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5438514947891235, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8028486371040344, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.47969090938568115, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4544142484664917, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.48195940256118774, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.46844199299812317, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.46764492988586426, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7770151495933533, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5141029357910156, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5970149040222168, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5065962672233582, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.4992746412754059, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5079773664474487, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8386688232421875, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.38833969831466675, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.25378626585006714, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3878629207611084, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3770049214363098, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3797289729118347, "metrics/hit_grouping/calorimeter/purity": 0.48412367701530457, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8462204337120056, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08909766376018524, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12030075490474701, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.088209368288517, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04348382726311684, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.043233081698417664, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8103082180023193, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.20088481903076172, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11006474494934082, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19985173642635345, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.20081202685832977, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3219541013240814, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8181094527244568, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.45697975158691406, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.38604551553726196, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.45751842856407166, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4454532563686371, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4406260550022125, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7248349189758301, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.42836448550224304, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4569142758846283, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.467489093542099, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5969868302345276, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.76398104429245, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2923447787761688, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4336027503013611, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.433952271938324, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4508740305900574, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.654954731464386, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4404950439929962, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.37938398122787476, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.39233478903770447, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6954957842826843, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6844853162765503, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4258236885070801, "metrics/hit_grouping/tracker/all/object_completeness": 0.40721753239631653, "metrics/hit_grouping/tracker/all/object_purity": 0.41850730776786804, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6471505165100098, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013047967106103897, "metrics/hit_grouping/tracker/background_recall": 0.10767287760972977, "metrics/hit_grouping/tracker/completeness": 0.4084738790988922, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5060528516769409, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6948915123939514, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42109414935112, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.456597238779068, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4661926329135895, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6409784555435181, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6791815757751465, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.442548543214798, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3967772126197815, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4113560914993286, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6581956744194031, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6683493256568909, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41893598437309265, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.31601640582084656, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.327523797750473, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6488728523254395, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7049903869628906, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45121848583221436, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43970155715942383, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4502045214176178, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6179842352867126, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.720632016658783, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4766009747982025, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.41371139883995056, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.38598158955574036, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5812807679176331, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6513480544090271, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38370680809020996, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3638230264186859, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.38313060998916626, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.696876049041748, "metrics/hit_grouping/tracker/purity": 0.3861159682273865, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8894491195678711, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.028657617047429085, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.045667149126529694, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.044356994330883026, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.062217194586992264, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6639567613601685, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2607124447822571, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2857924997806549, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.28251150250434875, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5751161575317383, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6562944054603577, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.502657949924469, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.47403138875961304, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4887702167034149, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7417603731155396, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4745180308818817, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5905563831329346, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6770756244659424, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6250366568565369, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.07463474571704865, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.050561852753162384, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.56386947631836, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29655084013938904, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20492765307426453, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8574398159980774, "metrics/particle/class_1/pid_accuracy": 0.966235876083374, "metrics/particle/class_2/efficiency": 0.44965454936027527, "metrics/particle/class_2/pid_accuracy": 0.5853904485702515, "metrics/particle/class_3/efficiency": 0.7112345099449158, "metrics/particle/class_3/pid_accuracy": 0.9614864587783813, "metrics/particle/class_4/efficiency": 0.6583166122436523, "metrics/particle/class_4/pid_accuracy": 0.31811264157295227, "metrics/particle/class_5/efficiency": 0.9061302542686462, "metrics/particle/class_5/pid_accuracy": 0.36152219772338867, "metrics/particle/count/bias_mean": -2.1731770038604736, "metrics/particle/count/mae": 4.35546875, "metrics/particle/count/prediction_mean": 68.48177337646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08859986066818237, "metrics/particle/event/energy_response_mean": 0.997336208820343, "metrics/particle/event/met_abs_error": 8.499457359313965, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07421866059303284, "metrics/particle/event/scalar_pt_response_mean": 1.0199683904647827, "metrics/particle/event/vector_pt_closure": 0.07129371166229248, "metrics/particle/matched/delta_eta_abs_mean": 0.015362919308245182, "metrics/particle/matched/delta_phi_abs_mean": 0.01454125065356493, "metrics/particle/matched/delta_r_mean": 0.023684373125433922, "metrics/particle/matched/energy_relative_abs_mean": 0.15384361147880554, "metrics/particle/matched/pid_accuracy": 0.9374402165412903, "metrics/particle/matched/pt_relative_abs_mean": 0.15390124917030334, "metrics/particle/matching/duplicate_fraction": 0.07896337658166885, "metrics/particle/matching/efficiency": 0.7610434293746948, "metrics/particle/matching/f1": 0.7729302048683167, "metrics/particle/matching/purity": 0.7851940989494324, "metrics/particle/matching/target_coverage_dr0p05": 0.7130733728408813, "metrics/particle/matching/target_coverage_dr0p10": 0.8179053664207458, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5506330132484436, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8018078804016113, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02265777997672558, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1528821438550949, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15267837047576904, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.005521389655768871, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19280238449573517, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010037948377430439, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7867839336395264, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.03711036220192909, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.16367769241333, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8169.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2787742614746094, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2698955535888672, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4137465953826904, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.245322346687317, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1537022441625595, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.350863456726074, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.022252855822443962, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2145310640335083, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.048059552907943726, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.527132749557495, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10156.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7583550214767456, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.1342108249664307, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.611173629760742, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.533398151397705, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.07600390166044235, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.3996712267398834, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.119596481323242, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.9088109731674194, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.389272689819336, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02514025755226612, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16594454646110535, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16577380895614624, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.015039770863950253, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20602908730506897, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02222222276031971, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4093338251113892, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.024917999282479286, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4905991554260254, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1575.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6221933960914612, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.1908459663391113, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 9.683626174926758, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 9.494235038757324, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24382716417312622, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 303.2095947265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0714285746216774, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.006615161895752, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1265864223241806, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.80253791809082, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2268.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5710659623146057, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.522640705108643, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.871577262878418, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2998327016830444, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0647406280040741, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.039051007479429245, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.705301284790039, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.4606730937957764, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.52991485595703, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02404782734811306, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1534942239522934, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15370716154575348, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.004057776648551226, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19325587153434753, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006601749337278306, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.25735461711883545, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0016757129924371839, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7732551693916321, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18177.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.24856887757778168, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4549834728240967, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.1614596843719482, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.9824621677398682, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15542897582054138, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.193485260009766, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005519494879990816, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5012657046318054, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.011035486124455929, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.925528883934021, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22647.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7811681032180786, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.375607490539551, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.78436279296875, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.018479347229004, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.4121607840061188, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.2820961475372314, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.008476734161377, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6397949457168579, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.038121223449707, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02370275743305683, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15401744842529297, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15408684313297272, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.005105479154735804, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19386671483516693, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004620178136974573, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.477234810590744, "metrics/particle/momentum/all/accepted/pz_bias": -0.01117226853966713, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5640507936477661, "metrics/particle/momentum/all/accepted/samples": 27921.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2814777195453644, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4489723443984985, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.7856364250183105, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.609028697013855, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16064554452896118, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 77.74412536621094, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014627469703555107, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8698339462280273, "metrics/particle/momentum/all/assigned/pz_bias": -0.029229603707790375, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.494401454925537, "metrics/particle/momentum/all/assigned/samples": 35071.0, "metrics/particle/momentum/all/efficiency": 0.7587434649467468, "metrics/particle/momentum/all/event/energy_bias": 1.9871774911880493, "metrics/particle/momentum/all/event/p3_error_mean": 23.266326904296875, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.334664344787598, "metrics/particle/momentum/all/event/px_bias": 0.27141624689102173, "metrics/particle/momentum/all/event/py_bias": -2.720818519592285, "metrics/particle/momentum/all/event/pz_abs_mean": 16.54045867919922, "metrics/particle/momentum/all/event/pz_bias": -4.009279251098633, "metrics/particle/momentum/all/event/pz_residual_rms": 28.263643264770508, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.024674447253346443, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14277900755405426, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14347010850906372, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007252424955368042, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18150931596755981, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004349545110017061, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4908038377761841, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.040130872279405594, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5202205181121826, "metrics/particle/momentum/class_1/accepted/samples": 15174.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1493552327156067, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.4338451325893402, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4997774362564087, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3441923260688782, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09087757766246796, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.091533660888672, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010168120265007019, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7220377922058105, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.035703714936971664, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.354001760482788, "metrics/particle/momentum/class_1/assigned/samples": 17309.0, "metrics/particle/momentum/class_1/efficiency": 0.8553069233894348, "metrics/particle/momentum/class_1/event/energy_bias": 11.405051231384277, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.69716453552246, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.824468612670898, "metrics/particle/momentum/class_1/event/px_bias": 1.1191463470458984, "metrics/particle/momentum/class_1/event/py_bias": -2.1028358936309814, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.723325729370117, "metrics/particle/momentum/class_1/event/pz_bias": 0.8602590560913086, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.168926239013672, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0319930799305439, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19807399809360504, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19924797117710114, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.009130619466304779, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24075159430503845, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.032575756311416626, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.728895902633667, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.0029142985586076975, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9642179012298584, "metrics/particle/momentum/class_2/accepted/samples": 1320.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7179127931594849, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8645318150520325, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1744163036346436, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7973470091819763, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.32964223623275757, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.883203983306885, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0660136267542839, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.807070732116699, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.10700777918100357, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.356226921081543, "metrics/particle/momentum/class_2/assigned/samples": 2348.0, "metrics/particle/momentum/class_2/efficiency": 0.44206297397613525, "metrics/particle/momentum/class_2/event/energy_bias": -12.11984920501709, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.714153289794922, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.151732444763184, "metrics/particle/momentum/class_2/event/px_bias": -1.5847779512405396, "metrics/particle/momentum/class_2/event/py_bias": 0.025570787489414215, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.465438842773438, "metrics/particle/momentum/class_2/event/pz_bias": -2.527264356613159, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.417667388916016, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021447835490107536, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16383837163448334, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16305401921272278, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0031440232414752245, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20345930755138397, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013590649468824267, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.28738245368003845, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01848716288805008, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9601070284843445, "metrics/particle/momentum/class_3/accepted/samples": 11037.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3653199374675751, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.60562801361084, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.2871484756469727, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.1158242225646973, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.21270310878753662, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 119.09278106689453, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011682556942105293, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7229434847831726, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.01718060113489628, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.003265857696533, "metrics/particle/momentum/class_3/assigned/samples": 14894.0, "metrics/particle/momentum/class_3/efficiency": 0.7105517387390137, "metrics/particle/momentum/class_3/event/energy_bias": 4.984291076660156, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.347006797790527, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.8975982666015625, "metrics/particle/momentum/class_3/event/px_bias": 0.18902899324893951, "metrics/particle/momentum/class_3/event/py_bias": -0.45532482862472534, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.92340087890625, "metrics/particle/momentum/class_3/event/pz_bias": -2.147277593612671, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.094568252563477, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.023459069430828094, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16535885632038116, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15801094472408295, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04945924133062363, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2031603902578354, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004201680887490511, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8066175580024719, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.20500534772872925, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.707334280014038, "metrics/particle/momentum/class_4/accepted/samples": 238.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.413894385099411, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.447043418884277, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.671741485595703, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.5752644538879395, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.30337077379226685, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 29.251998901367188, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.00561797758564353, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9381019473075867, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.31300199031829834, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9710417985916138, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.6346666812896729, "metrics/particle/momentum/class_4/event/energy_bias": -1.0737746953964233, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.844295024871826, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.2421481609344482, "metrics/particle/momentum/class_4/event/px_bias": 0.26147645711898804, "metrics/particle/momentum/class_4/event/py_bias": -0.05412103235721588, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3646045923233032, "metrics/particle/momentum/class_4/event/pz_bias": -0.17925837635993958, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.30133581161499, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.018820440396666527, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16247110068798065, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16448833048343658, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.012616321444511414, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.20290564000606537, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02631578966975212, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.522749423980713, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.2652890682220459, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.2667934894561768, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0758267194032669, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 1.0621497631072998, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.4509230852127075, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 1.2717328071594238, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.06707317382097244, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 12.529552459716797, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03658536449074745, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.925032138824463, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.06952732056379318, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.239542484283447, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9268292784690857, "metrics/particle/momentum/class_5/event/energy_bias": -1.2085411548614502, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.183797836303711, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.6677796840667725, "metrics/particle/momentum/class_5/event/px_bias": 0.28654250502586365, "metrics/particle/momentum/class_5/event/py_bias": -0.13410766422748566, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.097520112991333, "metrics/particle/momentum/class_5/event/pz_bias": -0.015737786889076233, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.8008875846862793, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3918446898460388, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3945978283882141, "metrics/set_slot/angular_recall_all_dr0p10": 0.9435895681381226, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8073272705078125, "metrics/set_slot/soft_count_bias_mean": 0.7264936566352844}, "step_train_time": 4.696591377258301, "step_valid_time": 53434.72806477547, "step_total_time": 53439.42465662956, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.026999433979777, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18122280499032606, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2854, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2822, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.932984635501798, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9431592861863847, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9380443714050944, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056840713813615336, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9225237005557372, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9325842696629213, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9275267050123254, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.932984635501798, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0245083837683018, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1685058691990433, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2249, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2135, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2123, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9343544857768052, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9493108048021343, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9417732686369652, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050689195197865716, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9291028446389497, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9439751000444642, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9364799294221438, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9343544857768052, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0176814624369195, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1836384035222386, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1288, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1196, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1182, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9046898638426626, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9285714285714286, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9164750957854406, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07142857142857142, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8940998487140696, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9177018633540373, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9057471264367816, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9046898638426626} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_22000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..799924b5e0780bb95edf423f3dc3bb8e8bd2dac4 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 1.0420613868889732, "train": {"Classification_binary": 0.1578821986913681, "Cardinality": 0.12475566565990448, "Classification": 0.1037018895149231, "Regression_pt": 0.15052804350852966, "Regression_eta": 0.0016981236403807998, "Regression_sin_phi": 0.0008077860111370683, "Regression_cos_phi": 0.0008081175619736314, "Regression_energy": 0.14966358244419098, "HitGrouping": 0.20163029432296753, "Auxiliary": 0.23438292741775513, "Total": 1.1093641519546509, "Total_unweighted": 0.5650897413725033}, "train_group": {}, "valid": {"Classification_binary": 0.19462581475575766, "Cardinality": 0.14463828007380167, "Classification": 0.10992891589800517, "Regression_pt": 0.1610351800918579, "Regression_eta": 0.001875480016072591, "Regression_sin_phi": 0.000905385318522652, "Regression_cos_phi": 0.0008948334337522587, "Regression_energy": 0.15858720739682516, "HitGrouping": 0.20463124910990396, "Auxiliary": 0.26072641213734943, "Total": 1.2288506825764973, "Total_unweighted": 0.6278528169107934, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8092129230499268, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4672294855117798, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3759818375110626, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.46991777420043945, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4704149663448334, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4475253224372864, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7554023265838623, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4187072217464447, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3755539059638977, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.42009907960891724, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.42814651131629944, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5192023515701294, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8266626000404358, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4409007132053375, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.34811386466026306, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4419705271720886, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4331303536891937, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.41272854804992676, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8158437013626099, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4471540153026581, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3586869239807129, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4487392008304596, "metrics/hit_grouping/calorimeter/all/object_purity": 0.44403892755508423, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4315367043018341, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0031217909418046474, "metrics/hit_grouping/calorimeter/background_recall": 0.12980607151985168, "metrics/hit_grouping/calorimeter/completeness": 0.5024983286857605, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5123095512390137, "metrics/hit_grouping/calorimeter/energy_purity": 0.5056155323982239, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.41116881370544434, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8396807312965393, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4728598892688751, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.27344104647636414, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4764302670955658, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4720723330974579, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.367489755153656, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8165019750595093, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4692511260509491, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3978581428527832, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.46996551752090454, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.46158474683761597, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.44345924258232117, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7665770649909973, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3724791705608368, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.49303942918777466, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.37089380621910095, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.369203120470047, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5501160025596619, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8009545207023621, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.48847725987434387, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.44713735580444336, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.49187636375427246, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.48836642503738403, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.47281694412231445, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7768356800079346, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5204374194145203, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5980442762374878, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5184049010276794, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5079877376556396, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5095213651657104, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8356713056564331, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.39629173278808594, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23999018967151642, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3964585065841675, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3912256956100464, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3801581859588623, "metrics/hit_grouping/calorimeter/purity": 0.49516627192497253, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8447301387786865, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0809161365032196, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.129699245095253, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08195488899946213, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03734736517071724, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8029223084449768, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.18794289231300354, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1059446707367897, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.18660099804401398, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18543489277362823, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3025309145450592, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8160454034805298, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4666106104850769, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.37557047605514526, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.46835699677467346, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.46412307024002075, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4447609484195709, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.725718080997467, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4273504316806793, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4645450711250305, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.48195335268974304, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5939446687698364, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7651964426040649, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.30922242999076843, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4364730417728424, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4440124034881592, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.45207956433296204, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6533492207527161, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4418621361255646, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3845438063144684, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4004416763782501, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6859260201454163, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6838530898094177, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.427604615688324, "metrics/hit_grouping/tracker/all/object_completeness": 0.4129676818847656, "metrics/hit_grouping/tracker/all/object_purity": 0.4287123382091522, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6403828859329224, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.010676090605556965, "metrics/hit_grouping/tracker/background_recall": 0.0920034721493721, "metrics/hit_grouping/tracker/completeness": 0.41240665316581726, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.49638432264328003, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6959772109985352, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4154830873012543, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.45984920859336853, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4764236807823181, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6321234703063965, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6788290143013, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4400072693824768, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.40650704503059387, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.42392921447753906, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6456707119941711, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6639053225517273, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.44021642208099365, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3229515552520752, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3353227972984314, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6521190404891968, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7063804864883423, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4553791284561157, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4485054612159729, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.458756685256958, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6052474975585938, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7188205122947693, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.49876847863197327, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.43101072311401367, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.39703816175460815, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5849753618240356, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6482980847358704, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3787393867969513, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.36353281140327454, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.39533185958862305, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6959929466247559, "metrics/hit_grouping/tracker/purity": 0.3888988494873047, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.883870005607605, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03469080105423927, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.048943664878606796, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.04551666975021362, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06184012070298195, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6638685464859009, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.272586464881897, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2871570289134979, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.28785479068756104, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5653071999549866, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6563372611999512, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5027139186859131, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4806240499019623, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5008446574211121, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7343741059303284, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4880656599998474, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5983805656433105, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.686294674873352, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6095204949378967, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.06108709052205086, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.03504569083452225, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.481468200683594, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2606848478317261, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.16906167566776276, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8624175190925598, "metrics/particle/class_1/pid_accuracy": 0.9576514959335327, "metrics/particle/class_2/efficiency": 0.4700419008731842, "metrics/particle/class_2/pid_accuracy": 0.6325300931930542, "metrics/particle/class_3/efficiency": 0.7166123986244202, "metrics/particle/class_3/pid_accuracy": 0.961653470993042, "metrics/particle/class_4/efficiency": 0.6623246669769287, "metrics/particle/class_4/pid_accuracy": 0.31164902448654175, "metrics/particle/class_5/efficiency": 0.8850574493408203, "metrics/particle/class_5/pid_accuracy": 0.36796537041664124, "metrics/particle/count/bias_mean": -2.037109375, "metrics/particle/count/mae": 4.503255367279053, "metrics/particle/count/prediction_mean": 68.61783599853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08390427380800247, "metrics/particle/event/energy_response_mean": 0.9807419776916504, "metrics/particle/event/met_abs_error": 7.5315093994140625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0680241659283638, "metrics/particle/event/scalar_pt_response_mean": 0.9802007675170898, "metrics/particle/event/vector_pt_closure": 0.06971171498298645, "metrics/particle/matched/delta_eta_abs_mean": 0.017795883119106293, "metrics/particle/matched/delta_phi_abs_mean": 0.01474065613001585, "metrics/particle/matched/delta_r_mean": 0.02578660473227501, "metrics/particle/matched/energy_relative_abs_mean": 0.15883100032806396, "metrics/particle/matched/pid_accuracy": 0.9346246719360352, "metrics/particle/matched/pt_relative_abs_mean": 0.15908677875995636, "metrics/particle/matching/duplicate_fraction": 0.07833240181207657, "metrics/particle/matching/efficiency": 0.7673091888427734, "metrics/particle/matching/f1": 0.7785324454307556, "metrics/particle/matching/purity": 0.7900888919830322, "metrics/particle/matching/target_coverage_dr0p05": 0.7084016799926758, "metrics/particle/matching/target_coverage_dr0p10": 0.8200154900550842, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5400088429450989, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8078156113624573, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025601180270314217, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1565653681755066, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15684489905834198, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0075478036887943745, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19445419311523438, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008898098953068256, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8224886655807495, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.04817821830511093, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.206556797027588, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8204.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2909933924674988, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3102343082427979, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.446747899055481, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.266292691230774, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1491963267326355, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.467856407165527, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020905235782265663, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2222529649734497, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12827393412590027, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.405259132385254, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10141.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7616041302680969, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.682443618774414, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.121597290039062, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.542144775390625, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2184939980506897, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.13028499484062195, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.687381744384766, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.4036104679107666, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.079689979553223, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02802285924553871, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17572498321533203, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17381715774536133, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0018957367865368724, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21505548059940338, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03139013424515724, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5937656164169312, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0707295835018158, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.8603029251098633, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1561.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6146566867828369, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8390458822250366, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.3541998863220215, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.1659674644470215, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25885438919067383, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 81.29945373535156, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06908614188432693, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.016292095184326, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.17050257325172424, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.287553787231445, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2287.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5659898519515991, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.2901201248168945, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.697237968444824, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3994140625, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.01859629899263382, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.06782463192939758, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.516022682189941, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 2.3070430755615234, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.8410701751709, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02577902190387249, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15849563479423523, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15945258736610413, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.021758463233709335, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19791077077388763, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009212095174007118, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.27334311604499817, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.041414692997932434, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8243668675422668, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18454.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.24117805063724518, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.469889760017395, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0737398862838745, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8673223257064819, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1434243619441986, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.162359237670898, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006132262758910656, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5069900155067444, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.07433276623487473, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9181809425354004, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22667.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7930723428726196, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.8336892127990723, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.897418975830078, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.085875511169434, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.7750608921051025, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.3461687564849854, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.973874568939209, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.150738000869751, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.518828392028809, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02585144154727459, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15888753533363342, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15948906540870667, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.016318567097187042, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1979040950536728, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004925759043544531, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5060362219810486, "metrics/particle/momentum/all/accepted/pz_bias": 0.04500265046954155, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6383730173110962, "metrics/particle/momentum/all/accepted/samples": 28219.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.27991071343421936, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4478121995925903, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.4604638814926147, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.2627336978912354, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15261432528495789, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 22.774324417114258, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014503490179777145, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8771924376487732, "metrics/particle/momentum/all/assigned/pz_bias": 0.09618652611970901, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.368720531463623, "metrics/particle/momentum/all/assigned/samples": 35095.0, "metrics/particle/momentum/all/efficiency": 0.766841471195221, "metrics/particle/momentum/all/event/energy_bias": -3.441365957260132, "metrics/particle/momentum/all/event/p3_error_mean": 23.59015655517578, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.399839401245117, "metrics/particle/momentum/all/event/px_bias": -2.0121512413024902, "metrics/particle/momentum/all/event/py_bias": 1.408629059791565, "metrics/particle/momentum/all/event/pz_abs_mean": 17.766433715820312, "metrics/particle/momentum/all/event/pz_bias": 6.861392021179199, "metrics/particle/momentum/all/event/pz_residual_rms": 29.84792709350586, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.028304772451519966, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14802412688732147, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14985954761505127, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.03357860818505287, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18594390153884888, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004517480731010437, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5323531627655029, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.03337833657860756, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.6164882183074951, "metrics/particle/momentum/class_1/accepted/samples": 15274.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.16149640083312988, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5387340188026428, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6268647909164429, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4267405569553375, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08369993418455124, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.724215507507324, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009377170354127884, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7382566332817078, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.06161265820264816, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.1531319618225098, "metrics/particle/momentum/class_1/assigned/samples": 17276.0, "metrics/particle/momentum/class_1/efficiency": 0.8609435558319092, "metrics/particle/momentum/class_1/event/energy_bias": -1.695419430732727, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.331832885742188, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.861505508422852, "metrics/particle/momentum/class_1/event/px_bias": -1.296964406967163, "metrics/particle/momentum/class_1/event/py_bias": 4.264811038970947, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.382052421569824, "metrics/particle/momentum/class_1/event/pz_bias": 1.6189701557159424, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.499454498291016, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03291209042072296, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20448513329029083, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20378583669662476, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.019219782203435898, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24433626234531403, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03316866606473923, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6560444831848145, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.0555713027715683, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9050514698028564, "metrics/particle/momentum/class_2/accepted/samples": 1417.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6876609921455383, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.776921272277832, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.02841317653656, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6553916335105896, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30474990606307983, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.9178125858306885, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06052963435649872, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.567354679107666, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.17989635467529297, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.732985973358154, "metrics/particle/momentum/class_2/assigned/samples": 2379.0, "metrics/particle/momentum/class_2/efficiency": 0.4745478928089142, "metrics/particle/momentum/class_2/event/energy_bias": -4.7107696533203125, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.522977828979492, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.241018295288086, "metrics/particle/momentum/class_2/event/px_bias": -1.1206674575805664, "metrics/particle/momentum/class_2/event/py_bias": -2.4975156784057617, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.061111450195312, "metrics/particle/momentum/class_2/event/pz_bias": 4.736318111419678, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.744274139404297, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0216282457113266, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1677335649728775, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16701272130012512, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.007777126505970955, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20691484212875366, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015246636467054486, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.29954102635383606, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.047893039882183075, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9868695735931396, "metrics/particle/momentum/class_3/accepted/samples": 11150.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.35013294219970703, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.5382227897644043, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.441262722015381, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.2734909057617188, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20513507723808289, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 34.053985595703125, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012938258238136768, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7499369978904724, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.11382143199443817, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0955851078033447, "metrics/particle/momentum/class_3/assigned/samples": 14917.0, "metrics/particle/momentum/class_3/efficiency": 0.7178265452384949, "metrics/particle/momentum/class_3/event/energy_bias": 5.822383880615234, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.372838973999023, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.964041709899902, "metrics/particle/momentum/class_3/event/px_bias": 0.3410523533821106, "metrics/particle/momentum/class_3/event/py_bias": -0.6372873187065125, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.720353126525879, "metrics/particle/momentum/class_3/event/pz_bias": 0.33264583349227905, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.120433807373047, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026805967092514038, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16548123955726624, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15775850415229797, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.0011501017725095153, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19686955213546753, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004291845485568047, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8787055015563965, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.3048821687698364, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.20707106590271, "metrics/particle/momentum/class_4/accepted/samples": 233.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.44107532501220703, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.6129865646362305, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.7866740226745605, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.6315701007843018, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.30083566904067993, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.352860450744629, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008356546051800251, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1031451225280762, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2673591375350952, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.53542423248291, "metrics/particle/momentum/class_4/assigned/samples": 359.0, "metrics/particle/momentum/class_4/efficiency": 0.6213333606719971, "metrics/particle/momentum/class_4/event/energy_bias": -1.5742013454437256, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.868543863296509, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.200845718383789, "metrics/particle/momentum/class_4/event/px_bias": 0.031018635258078575, "metrics/particle/momentum/class_4/event/py_bias": 0.33100953698158264, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4431772232055664, "metrics/particle/momentum/class_4/event/pz_bias": -0.11534755676984787, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.974048137664795, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02163832075893879, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16679584980010986, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16519401967525482, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.04707963764667511, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2055547833442688, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03448275849223137, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7754554748535156, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.5263419151306152, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.924546718597412, "metrics/particle/momentum/class_5/accepted/samples": 145.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09896247088909149, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.8340102434158325, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.2374353408813477, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 1.0164449214935303, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1036585345864296, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 11.634576797485352, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04268292710185051, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.075427293777466, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.5452218055725098, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.3213887214660645, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.8841463327407837, "metrics/particle/momentum/class_5/event/energy_bias": -1.2833601236343384, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.265040636062622, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.7283166646957397, "metrics/particle/momentum/class_5/event/px_bias": 0.033409517258405685, "metrics/particle/momentum/class_5/event/py_bias": -0.05238795280456543, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.1635855436325073, "metrics/particle/momentum/class_5/event/pz_bias": 0.2888047397136688, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.803579330444336, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.41633865237236023, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4201987683773041, "metrics/set_slot/angular_recall_all_dr0p10": 0.942207396030426, "metrics/set_slot/angular_recall_selected_dr0p10": 0.805880606174469, "metrics/set_slot/soft_count_bias_mean": 0.7238358855247498}, "step_train_time": 5.175967693328857, "step_valid_time": 55991.281991004944, "step_total_time": 55996.45795893669, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9875634202084036, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16981288434590358, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3004, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2860, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2834, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9349460608041844, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952063914780293, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9434273461982516, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047936085219707054, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.92644655116051, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9434087882822902, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9348507339600858, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9349460608041844, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9826809156673693, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17705395025417148, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2237, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2128, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2119, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9312910284463894, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9512740277156907, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9411764705882353, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048725972284309346, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9273522975929979, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9472507822977202, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9371959310039806, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9312910284463894, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9896444128104707, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16436632098876625, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1291, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1214, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1204, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9183055975794251, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9403563129357088, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9292001530807501, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05964368706429125, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9107413010590015, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9326103795507359, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9215461155759663, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9183055975794251} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_23000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..a9ab90dd9847647b4900de4aa2ae833157681723 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 1.0894278135657447, "train": {"Classification_binary": 0.15192684531211853, "Cardinality": 0.11285638809204102, "Classification": 0.08362405002117157, "Regression_pt": 0.15511351823806763, "Regression_eta": 0.0016542822122573853, "Regression_sin_phi": 0.000667557236738503, "Regression_cos_phi": 0.0007250968483276665, "Regression_energy": 0.1546511948108673, "HitGrouping": 0.2042495608329773, "Auxiliary": 0.22475272417068481, "Total": 1.0575381517410278, "Total_unweighted": 0.5483625446795486}, "train_group": {}, "valid": {"Classification_binary": 0.1883986989657084, "Cardinality": 0.14425894618034363, "Classification": 0.10554058353106181, "Regression_pt": 0.14962677160898843, "Regression_eta": 0.001690991533299287, "Regression_sin_phi": 0.000864134170114994, "Regression_cos_phi": 0.0008648076715568701, "Regression_energy": 0.14827932914098105, "HitGrouping": 0.2050450841585795, "Auxiliary": 0.25125686327616376, "Total": 1.1888365745544434, "Total_unweighted": 0.5952653166217109, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7944916486740112, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.46454742550849915, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.40333113074302673, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4631268084049225, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4605334997177124, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4793224334716797, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7384030818939209, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.40077871084213257, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3744460940361023, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.39602452516555786, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.40384018421173096, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5601920485496521, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8178523778915405, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4371833801269531, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.36810922622680664, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.43624788522720337, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4246222972869873, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.43536219000816345, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8046069145202637, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4426470100879669, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3792717754840851, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.44126808643341064, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4338882267475128, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4583715498447418, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002796588698402047, "metrics/hit_grouping/calorimeter/background_recall": 0.1203489676117897, "metrics/hit_grouping/calorimeter/completeness": 0.49684539437294006, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5141089558601379, "metrics/hit_grouping/calorimeter/energy_purity": 0.5072907209396362, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.43339869379997253, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.829306423664093, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4726068675518036, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.30296996235847473, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4733692705631256, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4640860855579376, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3969617784023285, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8003674149513245, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4575742185115814, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.41455549001693726, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.45532748103141785, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.44965285062789917, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.47708427906036377, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.758516252040863, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3665214776992798, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.499303936958313, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.36165064573287964, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3564337491989136, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5647332072257996, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7890046238899231, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.48408159613609314, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4680057764053345, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4834895730018616, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4761035740375519, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4969930350780487, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7732809782028198, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5267279744148254, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5887802243232727, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.51999431848526, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5125262141227722, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5146680474281311, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8242465257644653, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.39038485288619995, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.26384204626083374, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.38884153962135315, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.381478488445282, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4122876822948456, "metrics/hit_grouping/calorimeter/purity": 0.4894361197948456, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8461891412734985, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0786760076880455, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12406015396118164, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07885338366031647, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03834816813468933, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04511278122663498, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8041830658912659, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1843862384557724, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11006474494934082, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.18505001068115234, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18464012444019318, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3325485587120056, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8039515018463135, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4620172381401062, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3974394202232361, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4605066478252411, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.45330384373664856, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4716530442237854, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7288264036178589, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.415761262178421, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4729907512664795, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.49131840467453003, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5849630832672119, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7696369290351868, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2772754728794098, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.44819867610931396, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.45241299271583557, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4388185739517212, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6551664471626282, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4368973970413208, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39152559638023376, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.41457682847976685, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6849187016487122, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6862608790397644, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4186108708381653, "metrics/hit_grouping/tracker/all/object_completeness": 0.4207497239112854, "metrics/hit_grouping/tracker/all/object_purity": 0.44095778465270996, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6360195875167847, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01277265977114439, "metrics/hit_grouping/tracker/background_recall": 0.10565993189811707, "metrics/hit_grouping/tracker/completeness": 0.42027217149734497, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4914410710334778, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7039383053779602, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4079431891441345, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.47465890645980835, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4849986732006073, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6233561038970947, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6781560778617859, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42838990688323975, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4069213569164276, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.44115737080574036, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6433109641075134, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6579509973526001, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43083858489990234, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32359781861305237, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3501681089401245, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6548241376876831, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7045850157737732, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44391611218452454, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4518842399120331, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.47098761796951294, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6050776839256287, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.706662654876709, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.44889163970947266, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.416500449180603, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.42153382301330566, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6003694534301758, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6587815880775452, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38028478622436523, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.38103562593460083, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40540027618408203, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6826360821723938, "metrics/hit_grouping/tracker/purity": 0.3971576690673828, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8883890509605408, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02677224762737751, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.04377424716949463, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.04514171555638313, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6671632528305054, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2674238383769989, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.29112762212753296, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2908802330493927, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5560144782066345, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6583356261253357, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.49314531683921814, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.49074116349220276, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5159622430801392, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7297297120094299, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.49035966396331787, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6099839806556702, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.69780433177948, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6168459057807922, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.05879310518503189, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.042371075600385666, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.40555763244629, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2957119047641754, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.2040887027978897, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.86808180809021, "metrics/particle/class_1/pid_accuracy": 0.9692422747612, "metrics/particle/class_2/efficiency": 0.44931477308273315, "metrics/particle/class_2/pid_accuracy": 0.6163347363471985, "metrics/particle/class_3/efficiency": 0.7190389633178711, "metrics/particle/class_3/pid_accuracy": 0.9645496010780334, "metrics/particle/class_4/efficiency": 0.6663326621055603, "metrics/particle/class_4/pid_accuracy": 0.3368421196937561, "metrics/particle/class_5/efficiency": 0.9003831148147583, "metrics/particle/class_5/pid_accuracy": 0.38297873735427856, "metrics/particle/count/bias_mean": -3.404296875, "metrics/particle/count/mae": 4.856119632720947, "metrics/particle/count/prediction_mean": 67.25064849853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08307308703660965, "metrics/particle/event/energy_response_mean": 0.951938807964325, "metrics/particle/event/met_abs_error": 7.485447406768799, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0672493502497673, "metrics/particle/event/scalar_pt_response_mean": 0.9679014086723328, "metrics/particle/event/vector_pt_closure": 0.06532052159309387, "metrics/particle/matched/delta_eta_abs_mean": 0.01473990734666586, "metrics/particle/matched/delta_phi_abs_mean": 0.013977508060634136, "metrics/particle/matched/delta_r_mean": 0.022785089910030365, "metrics/particle/matched/energy_relative_abs_mean": 0.15022605657577515, "metrics/particle/matched/pid_accuracy": 0.9422943592071533, "metrics/particle/matched/pt_relative_abs_mean": 0.1489274948835373, "metrics/particle/matching/duplicate_fraction": 0.06218960881233215, "metrics/particle/matching/efficiency": 0.769493043422699, "metrics/particle/matching/f1": 0.788488507270813, "metrics/particle/matching/purity": 0.8084455728530884, "metrics/particle/matching/target_coverage_dr0p05": 0.7220850586891174, "metrics/particle/matching/target_coverage_dr0p10": 0.8187715411186218, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.569329023361206, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.808875322341919, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022243661805987358, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1480754166841507, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14638176560401917, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00012321396206971258, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1860392540693283, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008374802768230438, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7449437379837036, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.021548615768551826, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.0725197792053223, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8239.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2549871504306793, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9807784557342529, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.004101037979126, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8338401913642883, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13467879593372345, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.49880599975586, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01818363554775715, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1090301275253296, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.021202700212597847, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2613697052001953, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10009.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.764853298664093, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.2155399322509766, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.91323471069336, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.094545364379883, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.13730578124523163, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.4457206726074219, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.581611156463623, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.0048738718032837, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.135031700134277, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025196101516485214, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16832631826400757, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16629831492900848, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0006585537921637297, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2067858725786209, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.033438485115766525, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4779019355773926, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.17860394716262817, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5123300552368164, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1585.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5696663856506348, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.475522041320801, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 11.357090950012207, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 11.16091251373291, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22849462926387787, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 310.599365234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06854838877916336, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8588855266571045, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2102624773979187, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.744828224182129, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2232.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5746918320655823, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.178444862365723, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.128460884094238, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2366276979446411, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.13589493930339813, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.06364554911851883, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.980653762817383, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.391938328742981, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.530467987060547, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02295120619237423, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1494586169719696, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1487802267074585, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.00038096526986919343, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18883465230464935, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008121277787722647, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.24687784910202026, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005550779402256012, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7934350967407227, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18470.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23294571042060852, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0558059215545654, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8049440979957581, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6174213290214539, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1338241845369339, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.124351978302002, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0041499328799545765, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.43602320551872253, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.012480128556489944, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.625332236289978, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22410.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7937599420547485, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -4.078099727630615, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.53304672241211, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.872993469238281, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.14230090379714966, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.5763722062110901, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.702276706695557, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.6876417994499207, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.8105058670043945, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022870929911732674, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15011277794837952, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14906315505504608, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.00017591926734894514, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18907992541790009, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004842015914618969, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.46087154746055603, "metrics/particle/momentum/all/accepted/pz_bias": -0.01990348845720291, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5338846445083618, "metrics/particle/momentum/all/accepted/samples": 28294.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2610018253326416, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.125583291053772, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.542173981666565, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.3590798377990723, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14016911387443542, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 79.16949462890625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01235173549503088, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7864879965782166, "metrics/particle/momentum/all/assigned/pz_bias": 0.0006519786547869444, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1155595779418945, "metrics/particle/momentum/all/assigned/samples": 34651.0, "metrics/particle/momentum/all/efficiency": 0.7688795924186707, "metrics/particle/momentum/all/event/energy_bias": -11.472084999084473, "metrics/particle/momentum/all/event/p3_error_mean": 22.01247787475586, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.273591995239258, "metrics/particle/momentum/all/event/px_bias": -0.1308998167514801, "metrics/particle/momentum/all/event/py_bias": -1.0857385396957397, "metrics/particle/momentum/all/event/pz_abs_mean": 16.034149169921875, "metrics/particle/momentum/all/event/pz_bias": 0.3005772531032562, "metrics/particle/momentum/all/event/pz_residual_rms": 27.096176147460938, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.022735491394996643, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13478922843933105, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1338980793952942, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004795069806277752, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17181719839572906, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0047476585023105145, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4761005938053131, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.010191519744694233, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5166041851043701, "metrics/particle/momentum/class_1/accepted/samples": 15376.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1313001662492752, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3754202425479889, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.41721072793006897, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.26618432998657227, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07502903789281845, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.90950870513916, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008478512987494469, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6486222147941589, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.005630061496049166, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7534401416778564, "metrics/particle/momentum/class_1/assigned/samples": 17220.0, "metrics/particle/momentum/class_1/efficiency": 0.8666929602622986, "metrics/particle/momentum/class_1/event/energy_bias": 3.5759472846984863, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.88892364501953, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.878996849060059, "metrics/particle/momentum/class_1/event/px_bias": -0.4219900369644165, "metrics/particle/momentum/class_1/event/py_bias": -0.7181844711303711, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.496173858642578, "metrics/particle/momentum/class_1/event/pz_bias": 0.8604360818862915, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.60400390625, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.032130930572748184, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1960500329732895, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1956089586019516, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.025632623583078384, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23659372329711914, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03125, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6043635606765747, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.21670836210250854, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.662184000015259, "metrics/particle/momentum/class_2/accepted/samples": 1344.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6479553580284119, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7366431951522827, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0545779466629028, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.645897388458252, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.299954891204834, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 7.547325134277344, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.060442037880420685, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.595669746398926, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.17743906378746033, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.047718524932861, "metrics/particle/momentum/class_2/assigned/samples": 2217.0, "metrics/particle/momentum/class_2/efficiency": 0.45010048151016235, "metrics/particle/momentum/class_2/event/energy_bias": -16.54471206665039, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.358680725097656, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.787153244018555, "metrics/particle/momentum/class_2/event/px_bias": -0.25924551486968994, "metrics/particle/momentum/class_2/event/py_bias": 0.16585084795951843, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.868746757507324, "metrics/particle/momentum/class_2/event/pz_bias": -1.5298407077789307, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.380783081054688, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022064704447984695, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16535265743732452, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16427098214626312, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.00460023945197463, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20447708666324615, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001608866616152227, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.28436318039894104, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.010142642073333263, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.00032639503479, "metrics/particle/momentum/class_3/accepted/samples": 11188.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.35191747546195984, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.006542921066284, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.9068145751953125, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.7184653282165527, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18916712701320648, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 121.43521881103516, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009254219010472298, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6531947255134583, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.022409291937947273, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.797548294067383, "metrics/particle/momentum/class_3/assigned/samples": 14696.0, "metrics/particle/momentum/class_3/efficiency": 0.720272958278656, "metrics/particle/momentum/class_3/event/energy_bias": 3.628169536590576, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.847190856933594, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.701314926147461, "metrics/particle/momentum/class_3/event/px_bias": 0.4721340239048004, "metrics/particle/momentum/class_3/event/py_bias": -0.3337790369987488, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.311777114868164, "metrics/particle/momentum/class_3/event/pz_bias": 1.227638840675354, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.19824504852295, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.021931203082203865, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16329626739025116, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1510505974292755, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.023244623094797134, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19767218828201294, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0041841003112494946, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7153081893920898, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.06809507310390472, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.749780297279358, "metrics/particle/momentum/class_4/accepted/samples": 239.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.45815712213516235, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.8686416149139404, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.2351620197296143, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.104423999786377, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2909604609012604, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.174654006958008, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014124293811619282, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1314711570739746, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.09562770277261734, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.124744415283203, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.637333333492279, "metrics/particle/momentum/class_4/event/energy_bias": -1.2398728132247925, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.041063070297241, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.345078229904175, "metrics/particle/momentum/class_4/event/px_bias": -0.05598113685846329, "metrics/particle/momentum/class_4/event/py_bias": 0.0038694068789482117, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5076210498809814, "metrics/particle/momentum/class_4/event/pz_bias": -0.31074684858322144, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.227071285247803, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.015263979323208332, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15161612629890442, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14906612038612366, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.011263946071267128, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18945759534835815, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.020408162847161293, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.433325171470642, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.0990632101893425, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.918142557144165, "metrics/particle/momentum/class_5/accepted/samples": 147.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07623273879289627, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.28694137930870056, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3152661919593811, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.17252972722053528, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1036585345864296, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.8519777655601501, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04268292710185051, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.005065441131592, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.26923635601997375, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.730952739715576, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.8963414430618286, "metrics/particle/momentum/class_5/event/energy_bias": -0.8916155099868774, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.29850697517395, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.7453534603118896, "metrics/particle/momentum/class_5/event/px_bias": 0.13418300449848175, "metrics/particle/momentum/class_5/event/py_bias": -0.20349520444869995, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.1545829772949219, "metrics/particle/momentum/class_5/event/pz_bias": 0.05308988317847252, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.9064061641693115, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.38078704476356506, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.38284382224082947, "metrics/set_slot/angular_recall_all_dr0p10": 0.9472476840019226, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8073456883430481, "metrics/set_slot/soft_count_bias_mean": -0.7111636996269226}, "step_train_time": 4.465530872344971, "step_valid_time": 58530.60748887062, "step_total_time": 58535.073019981384, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9921517940472486, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15913891629522325, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3008, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2868, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2839, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9375612945406996, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9534574468085106, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9454425581012033, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04654255319148936, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9280810722458319, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9438164893617021, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9358826438107797, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9375612945406996, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.972507756283434, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1557740662777376, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2249, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2139, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2133, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9361050328227571, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.951089373054691, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9435377150419056, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048910626945309024, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9334792122538293, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.948421520675856, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.940891045434495, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9361050328227571, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.982596536890226, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1535168668795852, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1293, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1217, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1211, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9205748865355522, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9412219644238206, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9307839388145316, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05877803557617943, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.916036308623298, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9365815931941222, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9261950286806884, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9205748865355522} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_24000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..15e05de4e97fdfa6708dc667024859da4bfe4576 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 1.1367942402425162, "train": {"Classification_binary": 0.17915752530097961, "Cardinality": 0.1170029565691948, "Classification": 0.10070189833641052, "Regression_pt": 0.1453891098499298, "Regression_eta": 0.0014751101844012737, "Regression_sin_phi": 0.0007557228091172874, "Regression_cos_phi": 0.0007928238483145833, "Regression_energy": 0.14310817420482635, "HitGrouping": 0.2021602988243103, "Auxiliary": 0.23764321208000183, "Total": 1.107356071472168, "Total_unweighted": 0.5713803645339794}, "train_group": {}, "valid": {"Classification_binary": 0.18528602520624796, "Cardinality": 0.13618760307629904, "Classification": 0.10521569848060608, "Regression_pt": 0.1471479833126068, "Regression_eta": 0.001755026790002982, "Regression_sin_phi": 0.0008642650985469421, "Regression_cos_phi": 0.0008499912607173125, "Regression_energy": 0.14469014604886374, "HitGrouping": 0.2013643185297648, "Auxiliary": 0.2480080525080363, "Total": 1.1681230862935383, "Total_unweighted": 0.5858091361975918, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8138732314109802, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4730803072452545, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.37777987122535706, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4739890694618225, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.46860039234161377, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4424150586128235, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7615535855293274, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4220485985279083, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3618907034397125, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4195259213447571, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4293772280216217, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5173559784889221, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8352810144424438, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.44541943073272705, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3483452796936035, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.44623252749443054, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4309188723564148, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4018514156341553, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8230714797973633, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4519849419593811, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3583142161369324, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.45256802439689636, "metrics/hit_grouping/calorimeter/all/object_purity": 0.44221487641334534, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4231078028678894, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0025037575978785753, "metrics/hit_grouping/calorimeter/background_recall": 0.11720599979162216, "metrics/hit_grouping/calorimeter/completeness": 0.5114037394523621, "metrics/hit_grouping/calorimeter/energy_completeness": 0.526526927947998, "metrics/hit_grouping/calorimeter/energy_purity": 0.5195077657699585, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4138105511665344, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8465558290481567, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.47949156165122986, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.28362539410591125, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.48247644305229187, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4714842736721039, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.361743301153183, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8208894729614258, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4657134711742401, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3980884253978729, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.46543779969215393, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4571719169616699, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.43908336758613586, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.777385413646698, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3820689022541046, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4705336391925812, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3786199390888214, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.36746710538864136, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5321345925331116, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8097025156021118, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4968123137950897, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4492422342300415, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.49941539764404297, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4840901494026184, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4557974636554718, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7918410301208496, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5350363850593567, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5743695497512817, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5297596454620361, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5010087490081787, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4837879538536072, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8404225707054138, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3963862359523773, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23986755311489105, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3956078588962555, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.39251604676246643, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.382549524307251, "metrics/hit_grouping/calorimeter/purity": 0.5036295056343079, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8731292486190796, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09334380179643631, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12218045443296432, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09387531876564026, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05754594877362251, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8246444463729858, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19185630977153778, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10712183266878128, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19191928207874298, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.19436237215995789, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.314302533864975, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8221738934516907, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4713655114173889, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.37523353099823, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4719764292240143, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4613807201385498, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.43514350056648254, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7317355871200562, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.426770955324173, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4707898795604706, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4825797975063324, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5855425000190735, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.776792049407959, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2935503423213959, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.45031842589378357, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.44327402114868164, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4177215099334717, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6594968438148499, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44035112857818604, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3859705328941345, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3958379924297333, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6845589280128479, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6903631091117859, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4253339171409607, "metrics/hit_grouping/tracker/all/object_completeness": 0.41679248213768005, "metrics/hit_grouping/tracker/all/object_purity": 0.426001638174057, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6344167590141296, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013905654661357403, "metrics/hit_grouping/tracker/background_recall": 0.11667192727327347, "metrics/hit_grouping/tracker/completeness": 0.415431410074234, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5030342936515808, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7023627161979675, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41977906227111816, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.46830448508262634, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.47374916076660156, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6226547360420227, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.684257984161377, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44109639525413513, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.40508756041526794, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.42220339179039, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6456707119941711, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6717456579208374, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42110008001327515, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32246190309524536, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.33155912160873413, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6474301218986511, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7129690051078796, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4477371275424957, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4489695429801941, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.45441049337387085, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6043134927749634, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7248598337173462, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4612068831920624, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.426861047744751, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4016178548336029, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5646551847457886, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6547905802726746, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38977810740470886, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.37315627932548523, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3934403955936432, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6860580444335938, "metrics/hit_grouping/tracker/purity": 0.3930342197418213, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8853085041046143, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02677224762737751, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.04556215554475784, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.04692106693983078, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06711915880441666, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6769877076148987, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27774909138679504, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.30463820695877075, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2939416170120239, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5513681173324585, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6628836393356323, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5004756450653076, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4840380549430847, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4965696632862091, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7276033759117126, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4902153015136719, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5938982367515564, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6786265969276428, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6300822496414185, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.058937445282936096, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.055607449263334274, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.584026336669922, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2790941298007965, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.187470942735672, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8583743572235107, "metrics/particle/class_1/pid_accuracy": 0.966072678565979, "metrics/particle/class_2/efficiency": 0.4844263195991516, "metrics/particle/class_2/pid_accuracy": 0.6579378247261047, "metrics/particle/class_3/efficiency": 0.7384955286979675, "metrics/particle/class_3/pid_accuracy": 0.966874897480011, "metrics/particle/class_4/efficiency": 0.6613226532936096, "metrics/particle/class_4/pid_accuracy": 0.32575756311416626, "metrics/particle/class_5/efficiency": 0.8908045887947083, "metrics/particle/class_5/pid_accuracy": 0.4172042906284332, "metrics/particle/count/bias_mean": -2.5006511211395264, "metrics/particle/count/mae": 4.387369632720947, "metrics/particle/count/prediction_mean": 68.154296875, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08643743395805359, "metrics/particle/event/energy_response_mean": 0.9721501469612122, "metrics/particle/event/met_abs_error": 7.397477626800537, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06917354464530945, "metrics/particle/event/scalar_pt_response_mean": 1.0004204511642456, "metrics/particle/event/vector_pt_closure": 0.06807465851306915, "metrics/particle/matched/delta_eta_abs_mean": 0.01716518960893154, "metrics/particle/matched/delta_phi_abs_mean": 0.01371482852846384, "metrics/particle/matched/delta_r_mean": 0.02450384385883808, "metrics/particle/matched/energy_relative_abs_mean": 0.1527467668056488, "metrics/particle/matched/pid_accuracy": 0.942689836025238, "metrics/particle/matched/pt_relative_abs_mean": 0.1546519696712494, "metrics/particle/matching/duplicate_fraction": 0.06524334847927094, "metrics/particle/matching/efficiency": 0.7757680416107178, "metrics/particle/matching/f1": 0.7897434830665588, "metrics/particle/matching/purity": 0.8042317628860474, "metrics/particle/matching/target_coverage_dr0p05": 0.7245268225669861, "metrics/particle/matching/target_coverage_dr0p10": 0.828511118888855, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5581058859825134, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8134824633598328, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022719543427228928, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15046779811382294, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15247370302677155, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.023536991328001022, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19214315712451935, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01017842162400484, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7605381608009338, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0397004671394825, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1789231300354004, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8351.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25895625352859497, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9268541932106018, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1010326147079468, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9418148398399353, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14217032492160797, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.082731246948242, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01893641985952854, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1256839036941528, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.09468109905719757, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.274541139602661, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10192.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7752506732940674, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.08716896176338196, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.792060852050781, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.4841837882995605, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.7812218070030212, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.5430387854576111, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.302838325500488, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.0004043579101562, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.953248023986816, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02431492693722248, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1638568490743637, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16495166718959808, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.00723555451259017, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20534834265708923, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02993280440568924, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5131964683532715, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.013384455814957619, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.0506696701049805, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1637.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6034268736839294, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4290094375610352, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.335916757583618, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.1368744373321533, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23076923191547394, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 24.219289779663086, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06866579502820969, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8671505451202393, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.12793530523777008, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.277551651000977, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2301.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5935460329055786, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.608339309692383, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.971412658691406, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4039583206176758, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.12200149893760681, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.19498305022716522, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.814118385314941, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.039257436990737915, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.376108169555664, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02553890459239483, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15376527607440948, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15580570697784424, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.025268813595175743, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19601601362228394, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000699827738571912, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2535631060600281, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.019604653120040894, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7761634588241577, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18576.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.227250337600708, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.212082028388977, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9561417698860168, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7906233072280884, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13960996270179749, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.04378604888916, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004643346648663282, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4563242495059967, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0029873987659811974, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7223265171051025, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22613.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7983153462409973, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.3543219566345215, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.927250862121582, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.254383087158203, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.656029462814331, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9259966015815735, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.76503849029541, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.10022330284118652, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.813233375549316, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024644486606121063, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15337957441806793, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15535570681095123, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02372901327908039, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19544166326522827, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005146338138729334, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.47397229075431824, "metrics/particle/momentum/all/accepted/pz_bias": -0.00037554456503130496, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6492902040481567, "metrics/particle/momentum/all/accepted/samples": 28564.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.26111146807670593, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1434928178787231, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1541873216629028, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9883008599281311, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14632825553417206, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.983845710754395, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012989232316613197, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8086693286895752, "metrics/particle/momentum/all/assigned/pz_bias": 0.037797607481479645, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.26578688621521, "metrics/particle/momentum/all/assigned/samples": 35106.0, "metrics/particle/momentum/all/efficiency": 0.7762167453765869, "metrics/particle/momentum/all/event/energy_bias": -4.3411865234375, "metrics/particle/momentum/all/event/p3_error_mean": 23.016939163208008, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.843111991882324, "metrics/particle/momentum/all/event/px_bias": -3.5592527389526367, "metrics/particle/momentum/all/event/py_bias": -1.6640186309814453, "metrics/particle/momentum/all/event/pz_abs_mean": 16.664615631103516, "metrics/particle/momentum/all/event/pz_bias": 1.0613701343536377, "metrics/particle/momentum/all/event/pz_residual_rms": 28.805849075317383, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.027132419869303703, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13919851183891296, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1439523547887802, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.050842948257923126, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1827968955039978, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004535594489425421, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.48514848947525024, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.002776017179712653, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5237677097320557, "metrics/particle/momentum/class_1/accepted/samples": 15213.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.15740880370140076, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.429723858833313, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5440097451210022, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.42142027616500854, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09027817845344543, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.569238662719727, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00884853396564722, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.693648636341095, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.036106668412685394, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.114471197128296, "metrics/particle/momentum/class_1/assigned/samples": 17291.0, "metrics/particle/momentum/class_1/efficiency": 0.857505202293396, "metrics/particle/momentum/class_1/event/energy_bias": 6.999361991882324, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.27417755126953, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.041409492492676, "metrics/particle/momentum/class_1/event/px_bias": -1.1852078437805176, "metrics/particle/momentum/class_1/event/py_bias": -1.0399634838104248, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.850863456726074, "metrics/particle/momentum/class_1/event/pz_bias": 1.9852650165557861, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.20224380493164, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.031934939324855804, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19629435241222382, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.197804257273674, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.019663160666823387, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23847372829914093, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03690807893872261, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6473991870880127, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.046224791556596756, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.065646648406982, "metrics/particle/momentum/class_2/accepted/samples": 1436.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6467587351799011, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7431119084358215, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0790512561798096, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7042632102966309, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.29202038049697876, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 7.487839698791504, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05730050802230835, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.482224941253662, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.003480849787592888, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.7587385177612305, "metrics/particle/momentum/class_2/assigned/samples": 2356.0, "metrics/particle/momentum/class_2/efficiency": 0.48091092705726624, "metrics/particle/momentum/class_2/event/energy_bias": -12.033731460571289, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.270627975463867, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.42656421661377, "metrics/particle/momentum/class_2/event/px_bias": -1.554450273513794, "metrics/particle/momentum/class_2/event/py_bias": -0.2205713838338852, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.33498764038086, "metrics/particle/momentum/class_2/event/pz_bias": -1.9199965000152588, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.042325973510742, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020514197647571564, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16652090847492218, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16491660475730896, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.007524657994508743, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20504426956176758, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0017329520778730512, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2930445969104767, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.009223103523254395, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.174759030342102, "metrics/particle/momentum/class_3/accepted/samples": 11541.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3178543150424957, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.927032709121704, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7897452116012573, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.604629635810852, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18439194560050964, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.144076347351074, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010574928484857082, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6596595644950867, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04051561653614044, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.8702516555786133, "metrics/particle/momentum/class_3/assigned/samples": 14941.0, "metrics/particle/momentum/class_3/efficiency": 0.7429987788200378, "metrics/particle/momentum/class_3/event/energy_bias": 3.506134510040283, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.769094467163086, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.492908954620361, "metrics/particle/momentum/class_3/event/px_bias": -0.8515912890434265, "metrics/particle/momentum/class_3/event/py_bias": -0.1488010138273239, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.535213470458984, "metrics/particle/momentum/class_3/event/pz_bias": 0.7777843475341797, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.734323501586914, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02496558427810669, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1695519983768463, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16990317404270172, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05894267559051514, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2120721936225891, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004310344811528921, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7922846078872681, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.044834110885858536, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0586607456207275, "metrics/particle/momentum/class_4/accepted/samples": 232.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4514249265193939, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.9117231369018555, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.941850185394287, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.840987682342529, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.32303372025489807, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 32.12677764892578, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01123595517128706, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0672553777694702, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07952502369880676, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.7699368000030518, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.6186666488647461, "metrics/particle/momentum/class_4/event/energy_bias": -1.7378473281860352, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.8438720703125, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1318488121032715, "metrics/particle/momentum/class_4/event/px_bias": -0.038581591099500656, "metrics/particle/momentum/class_4/event/py_bias": -0.02409040927886963, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4712485074996948, "metrics/particle/momentum/class_4/event/pz_bias": -0.06398145854473114, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.349879741668701, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.019539985805749893, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14418895542621613, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14694848656654358, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0403200164437294, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19527436792850494, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.028169013559818268, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.594910740852356, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.012964530847966671, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.4896082878112793, "metrics/particle/momentum/class_5/accepted/samples": 142.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06970078498125076, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4071691930294037, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.43396085500717163, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.31547340750694275, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1111111119389534, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.9465861320495605, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03703703731298447, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9211561679840088, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.37498003244400024, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.115640640258789, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.8658536672592163, "metrics/particle/momentum/class_5/event/energy_bias": -1.0751041173934937, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.165088176727295, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.6965082883834839, "metrics/particle/momentum/class_5/event/px_bias": 0.07057801634073257, "metrics/particle/momentum/class_5/event/py_bias": -0.23059216141700745, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.022168755531311, "metrics/particle/momentum/class_5/event/pz_bias": 0.28229865431785583, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3926002979278564, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3899642527103424, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3932931125164032, "metrics/set_slot/angular_recall_all_dr0p10": 0.9463077783584595, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8149936199188232, "metrics/set_slot/soft_count_bias_mean": 0.22218485176563263}, "step_train_time": 5.888675689697266, "step_valid_time": 61031.792385578156, "step_total_time": 61037.68106150627, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0294655035435634, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16789161389397478, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3051, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2877, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2848, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9405034324942791, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9429695181907571, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9417348608837971, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05703048180924287, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9310232101994116, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9334644378892166, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9322422258592471, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9405034324942791, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023182031366518, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1651630051322326, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2265, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2143, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2132, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9378555798687089, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9461368653421633, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9419780219780219, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053863134657836646, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9330415754923413, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.941280353200883, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9371428571428572, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9378555798687089, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0008038207504335, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15920401121287486, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1301, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1205, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1195, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9114977307110439, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9262106072252114, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9187952725886389, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07378939277478862, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9039334341906202, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9185242121445042, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9111704155547083, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9114977307110439} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_25000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..f4e88c5af333350f5ce9c517c96c6bba0780bab4 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 1.1841606669192877, "train": {"Classification_binary": 0.20088139176368713, "Cardinality": 0.1590391993522644, "Classification": 0.0890524759888649, "Regression_pt": 0.14401009678840637, "Regression_eta": 0.001840720884501934, "Regression_sin_phi": 0.0008472695481032133, "Regression_cos_phi": 0.0008000294910743833, "Regression_energy": 0.14208629727363586, "HitGrouping": 0.19757291674613953, "Auxiliary": 0.2603418827056885, "Total": 1.1994997262954712, "Total_unweighted": 0.5795182817382738}, "train_group": {}, "valid": {"Classification_binary": 0.18318063020706177, "Cardinality": 0.15539586544036865, "Classification": 0.10268984238306682, "Regression_pt": 0.14754789074261984, "Regression_eta": 0.0016185780987143517, "Regression_sin_phi": 0.0008568085419634978, "Regression_cos_phi": 0.0008279522104809681, "Regression_energy": 0.14609993497530618, "HitGrouping": 0.2007798751195272, "Auxiliary": 0.2526382803916931, "Total": 1.1823693911234539, "Total_unweighted": 0.5828216371592134, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8181958198547363, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.471824586391449, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.37371060252189636, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.47440189123153687, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4659916162490845, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4308696985244751, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7653289437294006, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.41577017307281494, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.35672083497047424, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4177907407283783, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4125158488750458, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.49852290749549866, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8394553065299988, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4438665211200714, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3462624251842499, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.44578155875205994, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4250873625278473, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.38606804609298706, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8272597193717957, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4501551687717438, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.35538992285728455, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4522790312767029, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4365033805370331, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.40837156772613525, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003006999148055911, "metrics/hit_grouping/calorimeter/background_recall": 0.1252959817647934, "metrics/hit_grouping/calorimeter/completeness": 0.5125675201416016, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5239599347114563, "metrics/hit_grouping/calorimeter/energy_purity": 0.5170677304267883, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.419153094291687, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8493220806121826, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.47794049978256226, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2819185256958008, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.48179352283477783, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.46516743302345276, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.35144513845443726, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8261153101921082, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.468281090259552, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3916397988796234, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4688980281352997, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.45228224992752075, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4175495207309723, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7834278345108032, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.37524086236953735, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.46867749094963074, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.37535715103149414, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3621619939804077, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.515197217464447, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8156991600990295, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4943280816078186, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4438296854496002, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.49861279129981995, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.47820571064949036, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4404618740081787, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7987186312675476, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5452109575271606, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5702521800994873, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5407492518424988, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5096880197525024, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.459598571062088, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8424466848373413, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.39379358291625977, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2396222949028015, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.39449891448020935, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.38526642322540283, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.36955055594444275, "metrics/hit_grouping/calorimeter/purity": 0.5050303339958191, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8382205367088318, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08520044386386871, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.09962406009435654, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08607009053230286, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04935051128268242, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.818680465221405, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.197428897023201, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09417304396629333, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1967625916004181, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.19673015177249908, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3090052902698517, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8275275826454163, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.46925339102745056, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3731507956981659, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4715428650379181, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4552892744541168, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.419767826795578, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.732610821723938, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4227147698402405, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.47240012884140015, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4824966788291931, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5735187530517578, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7719770073890686, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2935503423213959, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.45282113552093506, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.44363364577293396, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4261603355407715, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6621933579444885, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44286948442459106, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3922840356826782, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4037938416004181, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.675780713558197, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6919450759887695, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4256455898284912, "metrics/hit_grouping/tracker/all/object_completeness": 0.421379029750824, "metrics/hit_grouping/tracker/all/object_purity": 0.4309256374835968, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6259127259254456, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0135191660374403, "metrics/hit_grouping/tracker/background_recall": 0.10258130729198456, "metrics/hit_grouping/tracker/completeness": 0.42189520597457886, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5024334788322449, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7016217708587646, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41478168964385986, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4704788625240326, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4848438501358032, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6204628944396973, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6895922422409058, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4385550916194916, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4134388267993927, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4244678318500519, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6306044459342957, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6743775606155396, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4351668059825897, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.32826969027519226, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.32643231749534607, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.632461667060852, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7150523066520691, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45682263374328613, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4567272663116455, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.45476335287094116, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5923410058021545, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.740969181060791, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5043103694915771, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.44543612003326416, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.393020898103714, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5295566320419312, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6531162261962891, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37101224064826965, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3711124062538147, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40673092007637024, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6868307590484619, "metrics/hit_grouping/tracker/purity": 0.3989931643009186, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8878501653671265, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.026395173743367195, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.04713026434183121, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.04802943021059036, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6714767217636108, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2694889008998871, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.28985339403152466, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2878990173339844, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5482705235481262, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6650918126106262, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5018185973167419, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4911721348762512, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5032485127449036, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7178669571876526, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.509730875492096, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6006827354431152, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6872045993804932, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6321600675582886, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.039421893656253815, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.057685233652591705, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.203248977661133, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2666389048099518, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17501570284366608, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8708280920982361, "metrics/particle/class_1/pid_accuracy": 0.973259449005127, "metrics/particle/class_2/efficiency": 0.45769622921943665, "metrics/particle/class_2/pid_accuracy": 0.638950765132904, "metrics/particle/class_3/efficiency": 0.7009378671646118, "metrics/particle/class_3/pid_accuracy": 0.9610454440116882, "metrics/particle/class_4/efficiency": 0.6673346757888794, "metrics/particle/class_4/pid_accuracy": 0.3498498499393463, "metrics/particle/class_5/efficiency": 0.9003831148147583, "metrics/particle/class_5/pid_accuracy": 0.3957446813583374, "metrics/particle/count/bias_mean": -4.410807132720947, "metrics/particle/count/mae": 5.426432132720947, "metrics/particle/count/prediction_mean": 66.244140625, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08527607470750809, "metrics/particle/event/energy_response_mean": 0.9531324505805969, "metrics/particle/event/met_abs_error": 7.566328048706055, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0663134828209877, "metrics/particle/event/scalar_pt_response_mean": 0.9725953936576843, "metrics/particle/event/vector_pt_closure": 0.06689637899398804, "metrics/particle/matched/delta_eta_abs_mean": 0.014636981301009655, "metrics/particle/matched/delta_phi_abs_mean": 0.013691466301679611, "metrics/particle/matched/delta_r_mean": 0.022395865991711617, "metrics/particle/matched/energy_relative_abs_mean": 0.15086153149604797, "metrics/particle/matched/pid_accuracy": 0.9439572691917419, "metrics/particle/matched/pt_relative_abs_mean": 0.15316441655158997, "metrics/particle/matching/duplicate_fraction": 0.05910506844520569, "metrics/particle/matching/efficiency": 0.7638814449310303, "metrics/particle/matching/f1": 0.788493275642395, "metrics/particle/matching/purity": 0.8147438168525696, "metrics/particle/matching/target_coverage_dr0p05": 0.7285811901092529, "metrics/particle/matching/target_coverage_dr0p10": 0.818845272064209, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.552641749382019, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7983524799346924, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021648025140166283, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14935103058815002, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15564978122711182, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03266085684299469, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19576914608478546, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008686077781021595, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7589322328567505, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.07768034189939499, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.0936505794525146, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8174.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.22872216999530792, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7859598994255066, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9116389751434326, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7638038992881775, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13734646141529083, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.297522068023682, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01685108058154583, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0666522979736328, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12747301161289215, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.0435991287231445, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9851.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7588191628456116, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.484133005142212, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.67354965209961, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.8331923484802246, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.06600768864154816, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.40224847197532654, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.4206318855285645, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.0561113357543945, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.142716407775879, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025765080004930496, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16355843842029572, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17084486782550812, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.028870655223727226, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2121000587940216, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.029318036511540413, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.506635069847107, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.08915171772241592, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.898892402648926, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1569.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5106505751609802, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5154484510421753, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.1609601974487305, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.986910820007324, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23006416857242584, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 96.64838409423828, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0659945011138916, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.807215452194214, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.09499501436948776, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.899540901184082, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2182.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5688905119895935, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.005691051483154, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.932981491088867, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2584936618804932, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05739966779947281, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.01931682601571083, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.774107933044434, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.16382944583892822, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.260801315307617, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.022382302209734917, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15070363879203796, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1514160931110382, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.025914890691637993, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19117380678653717, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008198961731977761, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.24797049164772034, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.026519304141402245, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7747777700424194, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18295.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22116336226463318, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9759693741798401, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7631893754005432, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.607184648513794, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13884393870830536, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.502356052398682, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0036407767329365015, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4248170554637909, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03922359272837639, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5190118551254272, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22248.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7862392067909241, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.6398675441741943, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.518941879272461, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.848417282104492, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5476149916648865, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 3.0990281105041504, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.750619888305664, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.0483009815216064, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.25381851196289, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022357536479830742, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15102866291999817, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15373758971691132, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02804696187376976, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1937474012374878, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004707896616309881, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4673672020435333, "metrics/particle/momentum/all/accepted/pz_bias": 0.044939324259757996, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5875294208526611, "metrics/particle/momentum/all/accepted/samples": 28038.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2417614459991455, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9557061791419983, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.0857678651809692, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9309622049331665, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14421983063220978, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 25.0119571685791, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011405735276639462, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7608957886695862, "metrics/particle/momentum/all/assigned/pz_bias": 0.06813284754753113, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0332601070404053, "metrics/particle/momentum/all/assigned/samples": 34281.0, "metrics/particle/momentum/all/efficiency": 0.7619228959083557, "metrics/particle/momentum/all/event/energy_bias": -12.129691123962402, "metrics/particle/momentum/all/event/p3_error_mean": 21.632381439208984, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.975326538085938, "metrics/particle/momentum/all/event/px_bias": -0.5390070080757141, "metrics/particle/momentum/all/event/py_bias": 3.5205931663513184, "metrics/particle/momentum/all/event/pz_abs_mean": 15.979107856750488, "metrics/particle/momentum/all/event/pz_bias": 4.268241882324219, "metrics/particle/momentum/all/event/pz_residual_rms": 26.177846908569336, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.022802729159593582, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1391221582889557, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14207644760608673, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.030122380703687668, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18005111813545227, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0044184536673128605, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4806194007396698, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.053953737020492554, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5301963090896606, "metrics/particle/momentum/class_1/accepted/samples": 15390.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12591446936130524, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.40566372871398926, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.49753135442733765, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.36812323331832886, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07656896114349365, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.129573345184326, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008449391461908817, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6551495790481567, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.08355539292097092, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.899038791656494, "metrics/particle/momentum/class_1/assigned/samples": 17161.0, "metrics/particle/momentum/class_1/efficiency": 0.8674821257591248, "metrics/particle/momentum/class_1/event/energy_bias": 7.562182426452637, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.366430282592773, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.39335823059082, "metrics/particle/momentum/class_1/event/px_bias": 0.41627106070518494, "metrics/particle/momentum/class_1/event/py_bias": 4.394192695617676, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.793933868408203, "metrics/particle/momentum/class_1/event/pz_bias": 2.0126540660858154, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.981460571289062, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.031019318848848343, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19853976368904114, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1975889652967453, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02240668050944805, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23785144090652466, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.026986505836248398, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5326194763183594, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11761032044887543, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.63950777053833, "metrics/particle/momentum/class_2/accepted/samples": 1334.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.611260712146759, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6065627932548523, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9489913582801819, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5731500387191772, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2869080901145935, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.775761604309082, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0533890426158905, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.487607717514038, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.02376365102827549, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.901306629180908, "metrics/particle/momentum/class_2/assigned/samples": 2154.0, "metrics/particle/momentum/class_2/efficiency": 0.44675150513648987, "metrics/particle/momentum/class_2/event/energy_bias": -15.34782600402832, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.319862365722656, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.947283744812012, "metrics/particle/momentum/class_2/event/px_bias": -0.2820262610912323, "metrics/particle/momentum/class_2/event/py_bias": -1.0696985721588135, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.718306541442871, "metrics/particle/momentum/class_2/event/pz_bias": -0.2490132749080658, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.695737838745117, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02076953649520874, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16175749897956848, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16477784514427185, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03122907504439354, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20587505400180817, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0019218450179323554, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.29559895396232605, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.03896797075867653, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1020615100860596, "metrics/particle/momentum/class_3/accepted/samples": 10927.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.32219555974006653, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6367573738098145, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7853189706802368, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6319557428359985, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2004982978105545, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 38.00108337402344, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008374282158911228, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6147724390029907, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.05259368196129799, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5160202980041504, "metrics/particle/momentum/class_3/assigned/samples": 14449.0, "metrics/particle/momentum/class_3/efficiency": 0.7034700512886047, "metrics/particle/momentum/class_3/event/energy_bias": -2.206362247467041, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.820940017700195, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.708380699157715, "metrics/particle/momentum/class_3/event/px_bias": -0.8171674013137817, "metrics/particle/momentum/class_3/event/py_bias": 0.3548789620399475, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.331396102905273, "metrics/particle/momentum/class_3/event/pz_bias": 2.170916795730591, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.73739242553711, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02241392619907856, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15870995819568634, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15607134997844696, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.047056958079338074, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19888174533843994, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00826446246355772, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7661612629890442, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2472817301750183, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7601040601730347, "metrics/particle/momentum/class_4/accepted/samples": 242.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.40708065032958984, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.2204196453094482, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.182913064956665, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.084808826446533, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.28651684522628784, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.075577735900879, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008426966145634651, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8152240514755249, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.15256042778491974, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.8179298639297485, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.6453333497047424, "metrics/particle/momentum/class_4/event/energy_bias": -0.9816612005233765, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.850832462310791, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1007301807403564, "metrics/particle/momentum/class_4/event/px_bias": 0.1097973883152008, "metrics/particle/momentum/class_4/event/py_bias": 0.028949886560440063, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5527403354644775, "metrics/particle/momentum/class_4/event/pz_bias": 0.0017903149127960205, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.300802707672119, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014992374926805496, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15632687509059906, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1521206796169281, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0004138995718676597, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1929098516702652, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03448275849223137, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.706038475036621, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.6959158182144165, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.49973201751709, "metrics/particle/momentum/class_5/accepted/samples": 145.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06227618828415871, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.33823734521865845, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.40839841961860657, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.24887093901634216, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.08074533939361572, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.2495012283325195, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.043478261679410934, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9246649742126465, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.8615987300872803, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.677041530609131, "metrics/particle/momentum/class_5/assigned/samples": 161.0, "metrics/particle/momentum/class_5/efficiency": 0.8841463327407837, "metrics/particle/momentum/class_5/event/energy_bias": -1.1560261249542236, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.2348949909210205, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.6411932706832886, "metrics/particle/momentum/class_5/event/px_bias": 0.03411825746297836, "metrics/particle/momentum/class_5/event/py_bias": -0.1877296417951584, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2146023511886597, "metrics/particle/momentum/class_5/event/pz_bias": 0.3318934440612793, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.176545143127441, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.37701117992401123, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3805999159812927, "metrics/set_slot/angular_recall_all_dr0p10": 0.9480677247047424, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8116396069526672, "metrics/set_slot/soft_count_bias_mean": -1.8269575834274292}, "step_train_time": 4.9030609130859375, "step_valid_time": 63536.337589502335, "step_total_time": 63541.24065065384, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9927730486037609, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16616208932862975, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3039, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2879, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2850, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.941157240928408, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9473511023362948, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.944244014430961, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05264889766370517, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9316770186335404, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9378084896347483, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9347326992456543, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.941157240928408, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9772236169471773, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16167506546454646, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2261, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2134, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2127, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9339168490153172, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9438301636444051, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9388473383194017, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05616983635559487, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9308533916849016, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.940734188412207, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.935767707875055, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9339168490153172, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9850230719699331, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16015554918251718, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1304, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1210, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1200, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9152798789712556, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9279141104294478, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9215536938309216, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07208588957055215, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9077155824508321, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9202453987730062, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.913937547600914, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9152798789712556} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_26000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..75257514474fc7b2582d514e57c00396caaf0905 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 1.2315270935960592, "train": {"Classification_binary": 0.20210900902748108, "Cardinality": 0.1352473944425583, "Classification": 0.09990616887807846, "Regression_pt": 0.13184328377246857, "Regression_eta": 0.0020696851424872875, "Regression_sin_phi": 0.0009196998435072601, "Regression_cos_phi": 0.0007472819415852427, "Regression_energy": 0.13037165999412537, "HitGrouping": 0.19938866794109344, "Auxiliary": 0.24975863099098206, "Total": 1.1708428859710693, "Total_unweighted": 0.5679667885997333}, "train_group": {}, "valid": {"Classification_binary": 0.1803176005681356, "Cardinality": 0.13875113924344382, "Classification": 0.10291402538617452, "Regression_pt": 0.1455071767171224, "Regression_eta": 0.0020813352117935815, "Regression_sin_phi": 0.0008193346826980511, "Regression_cos_phi": 0.0008010605815798044, "Regression_energy": 0.14430389801661173, "HitGrouping": 0.20181731383005777, "Auxiliary": 0.25033364693323773, "Total": 1.1674613157908122, "Total_unweighted": 0.5767444311641157, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8143978714942932, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4735601842403412, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.37872621417045593, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.47537416219711304, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.47256845235824585, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.428882360458374, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7597731351852417, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4148266017436981, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.35155096650123596, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4129946529865265, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4227362871170044, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5125554203987122, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8350609540939331, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4425344467163086, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.34186530113220215, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.44409266114234924, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4317092299461365, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3939828872680664, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.822955846786499, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4497826397418976, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3537844121456146, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4511550962924957, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4433910548686981, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4137614667415619, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0032398197799921036, "metrics/hit_grouping/calorimeter/background_recall": 0.1389390081167221, "metrics/hit_grouping/calorimeter/completeness": 0.5072985887527466, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5191617608070374, "metrics/hit_grouping/calorimeter/energy_purity": 0.5124253034591675, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.40613853931427, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8474642038345337, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4811742305755615, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.28038233518600464, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.484267920255661, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.47571027278900146, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3498520851135254, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8211723566055298, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4644218683242798, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.38737908005714417, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4650760293006897, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.457670658826828, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.42054352164268494, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7747805714607239, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.37102794647216797, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.46960556507110596, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.36961451172828674, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.36310726404190063, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5372389554977417, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8093534708023071, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4898768663406372, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4351094663143158, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.49317023158073425, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4850744605064392, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4504450261592865, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7927120327949524, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5342751741409302, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5784868597984314, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5298652648925781, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5047721266746521, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4770972728729248, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8404275178909302, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.3988380432128906, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24409835040569305, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.39894089102745056, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3935796022415161, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.36881476640701294, "metrics/hit_grouping/calorimeter/purity": 0.4999556541442871, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8392723202705383, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07988370209932327, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14097744226455688, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08191683888435364, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05498537793755531, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8128377795219421, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.21567198634147644, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11536198109388351, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.21626350283622742, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2100597620010376, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2984108328819275, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8232164978981018, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4679926931858063, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.36965909600257874, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4693950414657593, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.46186214685440063, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4259854853153229, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7275429368019104, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.421266108751297, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4723432958126068, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.48996052145957947, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5855425000190735, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7609025239944458, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.26160338521003723, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4443055987358093, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4565494954586029, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4327908456325531, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6572998762130737, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44884157180786133, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39523985981941223, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4123273491859436, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.687005341053009, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6865413784980774, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42653605341911316, "metrics/hit_grouping/tracker/all/object_completeness": 0.42256149649620056, "metrics/hit_grouping/tracker/all/object_purity": 0.4394541382789612, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6370436549186707, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01786584034562111, "metrics/hit_grouping/tracker/background_recall": 0.12492106109857559, "metrics/hit_grouping/tracker/completeness": 0.4239949584007263, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.501160204410553, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6979121565818787, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.412326842546463, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4746301770210266, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4921952486038208, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.628353476524353, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6835520267486572, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4463604986667633, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.413108229637146, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4308966398239136, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6462152600288391, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6661216616630554, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4360685348510742, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3248486816883087, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3394680917263031, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6458070278167725, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7075445055961609, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4518977701663971, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.45183268189430237, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4627786874771118, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6076250076293945, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.707665741443634, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.47413793206214905, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.41488248109817505, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.41214507818222046, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5979064106941223, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6554495692253113, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38503146171569824, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3858845829963684, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.41402700543403625, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6823049187660217, "metrics/hit_grouping/tracker/purity": 0.4027535617351532, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8794682621955872, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01772247441112995, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03573150560259819, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03638375550508499, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06523378938436508, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6682479381561279, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2534847557544708, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.28654611110687256, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2891884744167328, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5539494156837463, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6598944664001465, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5059593915939331, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.49470821022987366, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5155554413795471, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7309048175811768, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5069272518157959, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6073510646820068, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6963669657707214, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.626024067401886, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.04222549498081207, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.051549237221479416, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.082313537597656, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2811252176761627, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1895020455121994, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8646488785743713, "metrics/particle/class_1/pid_accuracy": 0.9698700904846191, "metrics/particle/class_2/efficiency": 0.4572431743144989, "metrics/particle/class_2/pid_accuracy": 0.6056477427482605, "metrics/particle/class_3/efficiency": 0.7337734699249268, "metrics/particle/class_3/pid_accuracy": 0.9676448702812195, "metrics/particle/class_4/efficiency": 0.6492986083030701, "metrics/particle/class_4/pid_accuracy": 0.3333333432674408, "metrics/particle/class_5/efficiency": 0.8927202820777893, "metrics/particle/class_5/pid_accuracy": 0.4034334719181061, "metrics/particle/count/bias_mean": -2.5279948711395264, "metrics/particle/count/mae": 4.326171875, "metrics/particle/count/prediction_mean": 68.126953125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08385620266199112, "metrics/particle/event/energy_response_mean": 0.980116069316864, "metrics/particle/event/met_abs_error": 8.0210599899292, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06433465331792831, "metrics/particle/event/scalar_pt_response_mean": 0.9857122302055359, "metrics/particle/event/vector_pt_closure": 0.06702103465795517, "metrics/particle/matched/delta_eta_abs_mean": 0.02198186330497265, "metrics/particle/matched/delta_phi_abs_mean": 0.013293091207742691, "metrics/particle/matched/delta_r_mean": 0.0283599141985178, "metrics/particle/matched/energy_relative_abs_mean": 0.1539377123117447, "metrics/particle/matched/pid_accuracy": 0.9434404373168945, "metrics/particle/matched/pt_relative_abs_mean": 0.1525764763355255, "metrics/particle/matching/duplicate_fraction": 0.06377875059843063, "metrics/particle/matching/efficiency": 0.7744964361190796, "metrics/particle/matching/f1": 0.7886043190956116, "metrics/particle/matching/purity": 0.8032357692718506, "metrics/particle/matching/target_coverage_dr0p05": 0.7025966048240662, "metrics/particle/matching/target_coverage_dr0p10": 0.8295615911483765, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5651640892028809, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8139800429344177, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.027185484766960144, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1544111967086792, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15444910526275635, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0243845172226429, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19502073526382446, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00920502096414566, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7542877197265625, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.04916847497224808, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.08707332611084, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8365.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24663685262203217, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1585822105407715, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3099051713943481, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1575676202774048, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13482370972633362, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.52136993408203, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.018219420686364174, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.104320764541626, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.01318004447966814, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.27791428565979, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10154.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.77655029296875, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.0482231378555298, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.936264038085938, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.633388519287109, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.630692720413208, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.6966375708580017, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.355941772460938, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5097071528434753, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.698678970336914, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028194092214107513, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16215646266937256, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16370461881160736, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.014576616697013378, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2033299207687378, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02996254712343216, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4789766073226929, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.023660242557525635, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.8641164302825928, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1602.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5643599629402161, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8617596626281738, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.803617238998413, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.6154544353485107, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22835689783096313, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 28.385234832763672, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06625441461801529, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.776905059814453, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.011083744466304779, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.790773391723633, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2264.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.580855667591095, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.193004608154297, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.83025074005127, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4213567972183228, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.09956429898738861, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.21585649251937866, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.66126537322998, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.0807617902755737, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.80579376220703, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.029282644391059875, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15338605642318726, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15129928290843964, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017008107155561447, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1915208101272583, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004852013662457466, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.25718268752098083, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.055693961679935455, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7578524947166443, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18549.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22500967979431152, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1780049800872803, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9671905040740967, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7995057702064514, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13750220835208893, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.366047859191895, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004558329004794359, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4603950083255768, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.04164717346429825, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.9010645151138306, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22596.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7971550226211548, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.4988226890563965, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.220491409301758, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.98951530456543, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.578394889831543, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9668466448783875, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.873702049255371, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.667310893535614, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.677109718322754, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02860630489885807, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15417949855327606, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1529201865196228, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019035333767533302, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19323308765888214, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0046991161070764065, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.47164469957351685, "metrics/particle/momentum/all/accepted/pz_bias": -0.05198013037443161, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5780354738235474, "metrics/particle/momentum/all/accepted/samples": 28516.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25322386622428894, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2165839672088623, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.249980092048645, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0854219198226929, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14260010421276093, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.650062561035156, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012509281747043133, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7969176173210144, "metrics/particle/momentum/all/assigned/pz_bias": -0.029982183128595352, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2318742275238037, "metrics/particle/momentum/all/assigned/samples": 35014.0, "metrics/particle/momentum/all/efficiency": 0.7749123573303223, "metrics/particle/momentum/all/event/energy_bias": -2.645958662033081, "metrics/particle/momentum/all/event/p3_error_mean": 23.003381729125977, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.910345077514648, "metrics/particle/momentum/all/event/px_bias": 3.308651924133301, "metrics/particle/momentum/all/event/py_bias": -1.879340648651123, "metrics/particle/momentum/all/event/pz_abs_mean": 16.63434600830078, "metrics/particle/momentum/all/event/pz_bias": -2.257779836654663, "metrics/particle/momentum/all/event/pz_residual_rms": 27.800344467163086, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03359779715538025, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14034788310527802, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13956624269485474, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.015845512971282005, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17762671411037445, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004249476827681065, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4891885817050934, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.09439954906702042, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5043374300003052, "metrics/particle/momentum/class_1/accepted/samples": 15296.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14583568274974823, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.48256969451904297, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5598863959312439, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4142497777938843, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07968035340309143, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.036147117614746, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007933290675282478, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6698349714279175, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.05172240361571312, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.965237617492676, "metrics/particle/momentum/class_1/assigned/samples": 17269.0, "metrics/particle/momentum/class_1/efficiency": 0.8621836304664612, "metrics/particle/momentum/class_1/event/energy_bias": 10.541006088256836, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.273862838745117, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.036223411560059, "metrics/particle/momentum/class_1/event/px_bias": 2.467317581176758, "metrics/particle/momentum/class_1/event/py_bias": -1.3708927631378174, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.0761079788208, "metrics/particle/momentum/class_1/event/pz_bias": -3.9253687858581543, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.656984329223633, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033627238124608994, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19603577256202698, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19566430151462555, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.020897889509797096, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23604905605316162, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03143274784088135, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5734400749206543, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.13110779225826263, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7652652263641357, "metrics/particle/momentum/class_2/accepted/samples": 1368.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6780876517295837, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7164422869682312, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.023780345916748, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6413515210151672, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30345430970191956, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.947568416595459, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05815478786826134, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.518831491470337, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.02863394096493721, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.444024562835693, "metrics/particle/momentum/class_2/assigned/samples": 2287.0, "metrics/particle/momentum/class_2/efficiency": 0.45813798904418945, "metrics/particle/momentum/class_2/event/energy_bias": -17.473711013793945, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.05428695678711, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.388638496398926, "metrics/particle/momentum/class_2/event/px_bias": -0.4008032977581024, "metrics/particle/momentum/class_2/event/py_bias": -0.6867884397506714, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.173445701599121, "metrics/particle/momentum/class_2/event/pz_bias": -0.6935856342315674, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.96555519104004, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021364685148000717, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16716381907463074, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16532652080059052, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02767126075923443, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20661668479442596, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0017427675193175673, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2936759293079376, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.010442725382745266, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1258577108383179, "metrics/particle/momentum/class_3/accepted/samples": 11476.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30860885977745056, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0975356101989746, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.043022871017456, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8886446952819824, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1869853436946869, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.38304901123047, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010309968143701553, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6597403287887573, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.007251088507473469, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0744454860687256, "metrics/particle/momentum/class_3/assigned/samples": 14937.0, "metrics/particle/momentum/class_3/efficiency": 0.738814115524292, "metrics/particle/momentum/class_3/event/energy_bias": 6.275243759155273, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.955513000488281, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.779963493347168, "metrics/particle/momentum/class_3/event/px_bias": 0.7915127277374268, "metrics/particle/momentum/class_3/event/py_bias": 0.2704732120037079, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.41897964477539, "metrics/particle/momentum/class_3/event/pz_bias": 2.4344327449798584, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.738572120666504, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.029611647129058838, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16831208765506744, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15846335887908936, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04080977663397789, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20338395237922668, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004310344811528921, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7892829775810242, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.1229810118675232, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.735368251800537, "metrics/particle/momentum/class_4/accepted/samples": 232.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.48810192942619324, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.455935001373291, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.2698864936828613, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.1583783626556396, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.31284916400909424, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.292146682739258, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.013966480270028114, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.031455397605896, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07142331451177597, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.3519835472106934, "metrics/particle/momentum/class_4/assigned/samples": 358.0, "metrics/particle/momentum/class_4/efficiency": 0.6186666488647461, "metrics/particle/momentum/class_4/event/energy_bias": -1.0846275091171265, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.2229857444763184, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.428537607192993, "metrics/particle/momentum/class_4/event/px_bias": 0.3644183874130249, "metrics/particle/momentum/class_4/event/py_bias": 0.20264682173728943, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6547818183898926, "metrics/particle/momentum/class_4/event/pz_bias": -0.11977964639663696, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.126895427703857, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.026196833699941635, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16821855306625366, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16768942773342133, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.013914676383137703, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21107709407806396, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0347222238779068, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8123984336853027, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.34525391459465027, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.4915919303894043, "metrics/particle/momentum/class_5/accepted/samples": 144.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07807686924934387, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3518766164779663, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.4263222813606262, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.26459741592407227, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.11042945086956024, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.835777997970581, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05521472543478012, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.1565797328948975, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.05138746649026871, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.241718292236328, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.8780487775802612, "metrics/particle/momentum/class_5/event/energy_bias": -0.9038698673248291, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.235684633255005, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.7221754789352417, "metrics/particle/momentum/class_5/event/px_bias": 0.08620630204677582, "metrics/particle/momentum/class_5/event/py_bias": -0.2947796583175659, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0941998958587646, "metrics/particle/momentum/class_5/event/pz_bias": 0.046521805226802826, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.5974879264831543, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4179679751396179, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4204105734825134, "metrics/set_slot/angular_recall_all_dr0p10": 0.946031391620636, "metrics/set_slot/angular_recall_selected_dr0p10": 0.814754068851471, "metrics/set_slot/soft_count_bias_mean": -0.3413604497909546}, "step_train_time": 4.894542217254639, "step_valid_time": 66030.8094086647, "step_total_time": 66035.70395112038, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0047346739820249, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16422723549552276, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3029, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2880, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2840, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9414841451454724, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.950808847804556, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9461235216819974, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04919115219544404, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9284079764628964, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9376031693628261, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9329829172141918, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9414841451454724, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9969161060097644, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15278548178504814, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2250, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2132, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2122, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9330415754923413, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9475555555555556, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9402425578831312, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052444444444444446, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9286652078774617, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9431111111111111, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.935832414553473, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9330415754923413, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9948915446129398, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15410051629696386, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1296, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1204, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1192, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9107413010590015, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9290123456790124, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9197860962566845, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07098765432098765, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9016641452344932, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9197530864197531, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9106187929717342, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9107413010590015} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_27000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..8b62d19019a2901fe22528948349a1fe4025c9de --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 1.2788935202728307, "train": {"Classification_binary": 0.19119086861610413, "Cardinality": 0.11960610002279282, "Classification": 0.09567538648843765, "Regression_pt": 0.12481829524040222, "Regression_eta": 0.0015476350672543049, "Regression_sin_phi": 0.0007770049851387739, "Regression_cos_phi": 0.000804517010692507, "Regression_energy": 0.12474600970745087, "HitGrouping": 0.1927197128534317, "Auxiliary": 0.2369586080312729, "Total": 1.094516396522522, "Total_unweighted": 0.5395597171154805}, "train_group": {}, "valid": {"Classification_binary": 0.17754781246185303, "Cardinality": 0.13581424951553345, "Classification": 0.09982498486836751, "Regression_pt": 0.14223015308380127, "Regression_eta": 0.0015376238152384758, "Regression_sin_phi": 0.0008120177468905846, "Regression_cos_phi": 0.0007807499108215173, "Regression_energy": 0.1411362886428833, "HitGrouping": 0.199028750260671, "Auxiliary": 0.2480474909146627, "Total": 1.1341528097788494, "Total_unweighted": 0.5638696305298557, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8178179264068604, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.48101478815078735, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3745623230934143, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.48370954394340515, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4764852523803711, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4271789491176605, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7637408375740051, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.41871294379234314, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3552437126636505, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.41863831877708435, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.42398494482040405, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4959379732608795, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8375639319419861, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.45665401220321655, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3451515734195709, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.45786380767822266, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4445258677005768, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3889840245246887, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8258504271507263, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.46108850836753845, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.35484519600868225, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.46264851093292236, "metrics/hit_grouping/calorimeter/all/object_purity": 0.45261332392692566, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4088589549064636, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002962675876915455, "metrics/hit_grouping/calorimeter/background_recall": 0.1278047114610672, "metrics/hit_grouping/calorimeter/completeness": 0.5195330381393433, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5299007296562195, "metrics/hit_grouping/calorimeter/energy_purity": 0.5229163765907288, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4039682149887085, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8516440391540527, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4887823164463043, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2808375060558319, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.49159860610961914, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4782184660434723, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.34518662095069885, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8253944516181946, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4773271977901459, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.38737908005714417, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4791823625564575, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.46960288286209106, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4152464270591736, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7737171649932861, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3882623314857483, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.47296983003616333, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3869631886482239, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3832891583442688, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5322505831718445, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8107365965843201, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5044832229614258, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4405220150947571, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5085300803184509, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.49898865818977356, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.44936251640319824, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7805725932121277, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5453904867172241, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5702521800994873, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.542320966720581, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5301371812820435, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.48944929242134094, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8466541171073914, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.40680167078971863, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24182966351509094, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4063776433467865, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3960948884487152, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.35796186327934265, "metrics/hit_grouping/calorimeter/purity": 0.5118706822395325, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8420404195785522, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09380827844142914, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14097744226455688, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09515976905822754, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05541331693530083, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03947368264198303, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8223848938941956, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19509154558181763, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11653914302587509, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19525261223316193, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18453830480575562, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2842848598957062, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8257669806480408, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.48091524839401245, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3707311153411865, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4825513958930969, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.47303566336631775, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4213605225086212, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7331079244613647, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.42155584692955017, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4770281910896301, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.48545074462890625, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5822106599807739, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.770672619342804, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28209763765335083, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4505918323993683, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4463540017604828, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4243520200252533, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6580899953842163, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44495609402656555, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4004383981227875, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4193585216999054, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6827601194381714, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6894623637199402, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42573463916778564, "metrics/hit_grouping/tracker/all/object_completeness": 0.4276825487613678, "metrics/hit_grouping/tracker/all/object_purity": 0.4416657090187073, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6327693462371826, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01284942775964737, "metrics/hit_grouping/tracker/background_recall": 0.10167350620031357, "metrics/hit_grouping/tracker/completeness": 0.4273318648338318, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4940046966075897, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.704285204410553, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4151323735713959, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.47763335704803467, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4864661991596222, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6216903328895569, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6804666519165039, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4372844398021698, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.41602301597595215, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4423733353614807, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6483935117721558, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6679091453552246, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4360685348510742, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.33651816844940186, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3488086462020874, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6400360465049744, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7090704441070557, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4507090151309967, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.46102312207221985, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4762861132621765, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6055871844291687, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7163916230201721, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4747537076473236, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.43381717801094055, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.41378116607666016, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.564039409160614, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6591435074806213, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3844795227050781, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.38323894143104553, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.401656836271286, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6804283261299133, "metrics/hit_grouping/tracker/purity": 0.403860479593277, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8863784074783325, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.018853696063160896, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.04371737688779831, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.046372368931770325, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.062217194586992264, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6692942976951599, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.24935467541217804, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.28431665897369385, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2891763746738434, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5601445436477661, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6624266505241394, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5052319169044495, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5002008676528931, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.516853928565979, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7253091335296631, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5605975985527039, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999761581421, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6188742518424988, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7105661034584045, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6193668246269226, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.011444788426160812, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.044892024248838425, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.246219635009766, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28460025787353516, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.19297710061073303, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8813174366950989, "metrics/particle/class_1/pid_accuracy": 0.9681028127670288, "metrics/particle/class_2/efficiency": 0.47547852993011475, "metrics/particle/class_2/pid_accuracy": 0.6712720394134521, "metrics/particle/class_3/efficiency": 0.7474805116653442, "metrics/particle/class_3/pid_accuracy": 0.969437301158905, "metrics/particle/class_4/efficiency": 0.6783567070960999, "metrics/particle/class_4/pid_accuracy": 0.3766617476940155, "metrics/particle/class_5/efficiency": 0.9061302542686462, "metrics/particle/class_5/pid_accuracy": 0.4122621417045593, "metrics/particle/count/bias_mean": -1.4375, "metrics/particle/count/mae": 4.009114742279053, "metrics/particle/count/prediction_mean": 69.21744537353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08164439350366592, "metrics/particle/event/energy_response_mean": 0.9543363451957703, "metrics/particle/event/met_abs_error": 6.735998630523682, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06288067996501923, "metrics/particle/event/scalar_pt_response_mean": 0.9785820841789246, "metrics/particle/event/vector_pt_closure": 0.06367337703704834, "metrics/particle/matched/delta_eta_abs_mean": 0.013929221779108047, "metrics/particle/matched/delta_phi_abs_mean": 0.013078493066132069, "metrics/particle/matched/delta_r_mean": 0.02137627825140953, "metrics/particle/matched/energy_relative_abs_mean": 0.14886637032032013, "metrics/particle/matched/pid_accuracy": 0.9463679790496826, "metrics/particle/matched/pt_relative_abs_mean": 0.14963170886039734, "metrics/particle/matching/duplicate_fraction": 0.06511597335338593, "metrics/particle/matching/efficiency": 0.7901424765586853, "metrics/particle/matching/f1": 0.7982629537582397, "metrics/particle/matching/purity": 0.8065520524978638, "metrics/particle/matching/target_coverage_dr0p05": 0.7565744519233704, "metrics/particle/matching/target_coverage_dr0p10": 0.8434016108512878, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5798057317733765, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8252492547035217, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021333929151296616, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15112127363681793, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15144972503185272, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01751464605331421, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19199301302433014, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009128144942224026, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7528955936431885, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.006021874025464058, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1512413024902344, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8545.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24263082444667816, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.060137152671814, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.192041039466858, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0322893857955933, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13411559164524078, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.716158866882324, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017067493870854378, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.083018183708191, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06403616815805435, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2669920921325684, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10312.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7932602763175964, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.1271424293518066, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.575366020202637, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.886199474334717, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.6936412453651428, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.47101882100105286, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.2922210693359375, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.1571561098098755, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.098746299743652, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024104773998260498, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15810304880142212, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15926586091518402, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.008507180958986282, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1997077316045761, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.029253730550408363, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4417378902435303, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2807561755180359, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.845139741897583, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1675.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5886503458023071, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3802621364593506, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.9210991859436035, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.7338051795959473, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22041861712932587, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 24.25774383544922, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06108500808477402, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5625200271606445, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.44162097573280334, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.291414260864258, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2341.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6073241233825684, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.284643650054932, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.132291793823242, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2741109132766724, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0006783641874790192, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.11057383567094803, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.972654342651367, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.2073404788970947, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.351314544677734, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021277153864502907, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1462002545595169, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14739882946014404, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.019495880231261253, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18763603270053864, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006902043824084103, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.24170836806297302, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.019761763513088226, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7784830927848816, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18835.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.215765580534935, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0407060384750366, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8325398564338684, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6702709197998047, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13589371740818024, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.0431294441223145, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00409132894128561, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4259158968925476, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.010323844850063324, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5959633588790894, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22731.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8094460368156433, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.0050528049468994, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.809106826782227, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.235061645507812, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.7811620235443115, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.15281476080417633, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.610870361328125, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.4925059974193573, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.844771862030029, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02145686000585556, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14833371341228485, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1492743194103241, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01827971264719963, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18963991105556488, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00481844786554575, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.46122807264328003, "metrics/particle/momentum/all/accepted/pz_bias": -0.0016037470195442438, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6143885850906372, "metrics/particle/momentum/all/accepted/samples": 29055.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24826496839523315, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0688337087631226, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.0754884481430054, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9122974276542664, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14096766710281372, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.52627182006836, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01164368074387312, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7587733864784241, "metrics/particle/momentum/all/assigned/pz_bias": -0.04124758392572403, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.048790216445923, "metrics/particle/momentum/all/assigned/samples": 35384.0, "metrics/particle/momentum/all/efficiency": 0.7895594835281372, "metrics/particle/momentum/all/event/energy_bias": -10.416838645935059, "metrics/particle/momentum/all/event/p3_error_mean": 21.26871109008789, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.016561508178711, "metrics/particle/momentum/all/event/px_bias": -2.4754815101623535, "metrics/particle/momentum/all/event/py_bias": -0.7344074845314026, "metrics/particle/momentum/all/event/pz_abs_mean": 15.27071762084961, "metrics/particle/momentum/all/event/pz_bias": -3.8570022583007812, "metrics/particle/momentum/all/event/pz_residual_rms": 25.64430809020996, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021578844636678696, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13431960344314575, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13514582812786102, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02420201525092125, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17351916432380676, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004028133116662502, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.46621501445770264, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.03270037844777107, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5166585445404053, "metrics/particle/momentum/class_1/accepted/samples": 15640.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13159753382205963, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.37589502334594727, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.44403085112571716, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.30998125672340393, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0739142969250679, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.532342910766602, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006960023194551468, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6247555613517761, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.012854519300162792, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7461044788360596, "metrics/particle/momentum/class_1/assigned/samples": 17385.0, "metrics/particle/momentum/class_1/efficiency": 0.8815737366676331, "metrics/particle/momentum/class_1/event/energy_bias": 1.5110344886779785, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.132766723632812, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.181303977966309, "metrics/particle/momentum/class_1/event/px_bias": -0.6931189894676208, "metrics/particle/momentum/class_1/event/py_bias": -0.1417771577835083, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.33910083770752, "metrics/particle/momentum/class_1/event/pz_bias": 1.6672627925872803, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.732423782348633, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029233593493700027, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19533763825893402, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19797787070274353, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.01597519963979721, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23787571489810944, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03467798978090286, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.686933159828186, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.3121030628681183, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.082824230194092, "metrics/particle/momentum/class_2/accepted/samples": 1413.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.681914746761322, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7433364987373352, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9964785575866699, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6118721961975098, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30250000953674316, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.580060005187988, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.059166666120290756, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.59108304977417, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.34207040071487427, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.029077529907227, "metrics/particle/momentum/class_2/assigned/samples": 2400.0, "metrics/particle/momentum/class_2/efficiency": 0.47320830821990967, "metrics/particle/momentum/class_2/event/energy_bias": -11.390411376953125, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.00068473815918, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.197442054748535, "metrics/particle/momentum/class_2/event/px_bias": -1.384490728378296, "metrics/particle/momentum/class_2/event/py_bias": -0.835348904132843, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.221038818359375, "metrics/particle/momentum/class_2/event/pz_bias": -2.907573699951172, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.59056282043457, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020420558750629425, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.161153182387352, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1621486395597458, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.014250864274799824, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2030995786190033, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018949181539937854, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.28220921754837036, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.016218319535255432, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0717254877090454, "metrics/particle/momentum/class_3/accepted/samples": 11610.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31272533535957336, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.884097695350647, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7944523096084595, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6325299739837646, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18932007253170013, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.176684379577637, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009220563806593418, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6044438481330872, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.06577225774526596, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6485605239868164, "metrics/particle/momentum/class_3/assigned/samples": 15075.0, "metrics/particle/momentum/class_3/efficiency": 0.7474409341812134, "metrics/particle/momentum/class_3/event/energy_bias": 1.807202696800232, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.979798316955566, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.739362716674805, "metrics/particle/momentum/class_3/event/px_bias": -0.678480327129364, "metrics/particle/momentum/class_3/event/py_bias": 0.3889019787311554, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.462559700012207, "metrics/particle/momentum/class_3/event/pz_bias": -2.7096190452575684, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.480584144592285, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022370439022779465, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16222795844078064, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1606450080871582, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.02798200212419033, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20116384327411652, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004081632476300001, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.868074893951416, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04029995575547218, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.989954948425293, "metrics/particle/momentum/class_4/accepted/samples": 245.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3689248561859131, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.9122588634490967, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.345757007598877, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.198791742324829, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.29916897416114807, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.622135162353516, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.002770083025097847, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8650387525558472, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.023711932823061943, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.936325192451477, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.653333306312561, "metrics/particle/momentum/class_4/event/energy_bias": -1.2426345348358154, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.6996259689331055, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0597891807556152, "metrics/particle/momentum/class_4/event/px_bias": 0.15544798970222473, "metrics/particle/momentum/class_4/event/py_bias": 0.03905123472213745, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3396880626678467, "metrics/particle/momentum/class_4/event/pz_bias": -0.3124261498451233, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.918998718261719, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014050663448870182, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15190976858139038, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14855676889419556, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0194717925041914, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18872037529945374, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03401360660791397, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6096222400665283, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.4176328182220459, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.585066318511963, "metrics/particle/momentum/class_5/accepted/samples": 147.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07774332910776138, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.285717248916626, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.28143957257270813, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.1169447973370552, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.0920245423913002, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.049979329109192, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05521472543478012, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.111611843109131, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.7419971823692322, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.6036272048950195, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.8963414430618286, "metrics/particle/momentum/class_5/event/energy_bias": -1.1020296812057495, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.163182258605957, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.6294759511947632, "metrics/particle/momentum/class_5/event/px_bias": 0.12516053020954132, "metrics/particle/momentum/class_5/event/py_bias": -0.18523453176021576, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0956662893295288, "metrics/particle/momentum/class_5/event/pz_bias": 0.405353844165802, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.727140188217163, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.36395418643951416, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3657471835613251, "metrics/set_slot/angular_recall_all_dr0p10": 0.9518179893493652, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8291929960250854, "metrics/set_slot/soft_count_bias_mean": 1.0476878881454468}, "step_train_time": 4.879790782928467, "step_valid_time": 68554.19219112396, "step_total_time": 68559.07198238373, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058251614835445, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15523245385717255, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2883, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2858, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9424648577966656, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9514851485148514, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9469535227459352, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048514851485148516, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9342922523700555, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9432343234323433, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.938741993759238, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9424648577966656, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9870313650134838, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1500304821580004, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2258, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2148, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2140, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9400437636761488, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9512843224092117, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9456306405458947, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048715677590788306, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.936542669584245, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9477413640389726, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9421087387189082, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9400437636761488, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9805089291781899, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15810828118523879, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1289, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1206, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1201, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9122541603630863, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9356089992242048, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9237839908081195, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0643910007757952, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9084720121028744, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9317300232738557, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9199540405974722, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9122541603630863} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_28000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..bfe5052fbeeaaf20bc2907ddc14552a857ccae77 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 1.3262599469496021, "train": {"Classification_binary": 0.16992367804050446, "Cardinality": 0.12736321985721588, "Classification": 0.1035318672657013, "Regression_pt": 0.13231295347213745, "Regression_eta": 0.0016300572315230966, "Regression_sin_phi": 0.0008204197511076927, "Regression_cos_phi": 0.0008458447409793735, "Regression_energy": 0.1310025453567505, "HitGrouping": 0.20040372014045715, "Auxiliary": 0.23862284421920776, "Total": 1.11112642288208, "Total_unweighted": 0.5400673658587039}, "train_group": {}, "valid": {"Classification_binary": 0.17507372299830118, "Cardinality": 0.13426714142163595, "Classification": 0.10057824850082397, "Regression_pt": 0.139162411292394, "Regression_eta": 0.001498798994968335, "Regression_sin_phi": 0.0008141157062103351, "Regression_cos_phi": 0.0007997856009751558, "Regression_energy": 0.1381708880265554, "HitGrouping": 0.1963468591372172, "Auxiliary": 0.24366913239161173, "Total": 1.121238072713216, "Total_unweighted": 0.5560979711202284, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8202714920043945, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.48608940839767456, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3667076826095581, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.48810169100761414, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.47899922728538513, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.41847261786460876, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7718203067779541, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4248977303504944, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3559822738170624, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4258858859539032, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4199945032596588, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.485228955745697, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8410203456878662, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.46160951256752014, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.33820873498916626, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4632059335708618, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.447898268699646, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3791714906692505, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8293618559837341, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.46617552638053894, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3482224643230438, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4678507149219513, "metrics/hit_grouping/calorimeter/all/object_purity": 0.45515406131744385, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.39931192994117737, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002904906403273344, "metrics/hit_grouping/calorimeter/background_recall": 0.12525369226932526, "metrics/hit_grouping/calorimeter/completeness": 0.5251601338386536, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5353677868843079, "metrics/hit_grouping/calorimeter/energy_purity": 0.5282953381538391, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4048226773738861, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8541752696037292, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.49380573630332947, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.27344104647636414, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4970746338367462, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.483624130487442, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3372212052345276, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8277795314788818, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4857875406742096, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3810456097126007, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.48621898889541626, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.47339537739753723, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4115614891052246, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.780362069606781, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3900805711746216, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4676333963871002, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3897590935230255, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.37872743606567383, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5135731101036072, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8146892786026001, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5083395838737488, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4317416548728943, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5127599239349365, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.49849557876586914, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4394395053386688, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.796503484249115, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.552980899810791, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5795162320137024, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5489479899406433, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5196836590766907, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.45393720269203186, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8482361435890198, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.41284507513046265, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23551413416862488, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4124014973640442, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4032769203186035, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3518916070461273, "metrics/hit_grouping/calorimeter/purity": 0.5173848271369934, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8299633860588074, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08147243410348892, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11090225726366043, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08050483465194702, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0422254242002964, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04135338217020035, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8237066268920898, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.21078456938266754, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10653325170278549, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.20839186012744904, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.20359782874584198, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2789876461029053, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8296464085578918, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4857342839241028, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.364666610956192, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4876641631126404, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.47497305274009705, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.41140615940093994, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7356988191604614, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41677531599998474, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4776255488395691, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4905143678188324, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5730841755867004, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7700350284576416, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.294153094291687, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4585080146789551, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.45484429597854614, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.43399637937545776, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6625781059265137, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4425097107887268, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4020048677921295, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.41056856513023376, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6757087111473083, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6929887533187866, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42364203929901123, "metrics/hit_grouping/tracker/all/object_completeness": 0.4294201731681824, "metrics/hit_grouping/tracker/all/object_purity": 0.43841004371643066, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6263134479522705, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0182576235383749, "metrics/hit_grouping/tracker/background_recall": 0.13119672238826752, "metrics/hit_grouping/tracker/completeness": 0.4303140640258789, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5008644461631775, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.705357551574707, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4130282402038574, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.47913238406181335, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.49229007959365845, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6152025461196899, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6881512999534607, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44018879532814026, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4183933436870575, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4241914451122284, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6336902976036072, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6723524332046509, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42903515696525574, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.33811813592910767, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3417057394981384, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6418395042419434, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.717708945274353, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45300161838531494, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.463856041431427, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4621758460998535, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.590557873249054, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.718925416469574, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.49938422441482544, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4462571442127228, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4050782024860382, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5806650519371033, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6562020778656006, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37189534306526184, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3816341459751129, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.41348904371261597, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6809802651405334, "metrics/hit_grouping/tracker/purity": 0.40891918540000916, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8792528510093689, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02828054316341877, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.05122637376189232, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.04983873292803764, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07126697152853012, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6672729849815369, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27155396342277527, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.30935975909233093, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3089107275009155, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5570470094680786, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6681351065635681, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4987969398498535, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4985560476779938, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5101089477539062, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7161882519721985, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5093857645988464, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5973602533340454, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6816853880882263, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6155321002006531, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.03976697847247124, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04105733707547188, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.65517807006836, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.27993321418762207, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.887499988079071, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18831004202365875, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8859137296676636, "metrics/particle/class_1/pid_accuracy": 0.9721648097038269, "metrics/particle/class_2/efficiency": 0.4693623185157776, "metrics/particle/class_2/pid_accuracy": 0.6551640629768372, "metrics/particle/class_3/efficiency": 0.7469558119773865, "metrics/particle/class_3/pid_accuracy": 0.9658452272415161, "metrics/particle/class_4/efficiency": 0.679358720779419, "metrics/particle/class_4/pid_accuracy": 0.3495575189590454, "metrics/particle/class_5/efficiency": 0.9233716726303101, "metrics/particle/class_5/pid_accuracy": 0.3879668116569519, "metrics/particle/count/bias_mean": -1.69921875, "metrics/particle/count/mae": 3.9700520038604736, "metrics/particle/count/prediction_mean": 68.95572662353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08115065097808838, "metrics/particle/event/energy_response_mean": 0.9733397364616394, "metrics/particle/event/met_abs_error": 7.908538818359375, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06179868057370186, "metrics/particle/event/scalar_pt_response_mean": 0.9879631400108337, "metrics/particle/event/vector_pt_closure": 0.06603432446718216, "metrics/particle/matched/delta_eta_abs_mean": 0.013993124477565289, "metrics/particle/matched/delta_phi_abs_mean": 0.01321255974471569, "metrics/particle/matched/delta_r_mean": 0.02155890315771103, "metrics/particle/matched/energy_relative_abs_mean": 0.14746062457561493, "metrics/particle/matched/pid_accuracy": 0.946173369884491, "metrics/particle/matched/pt_relative_abs_mean": 0.14672154188156128, "metrics/particle/matching/duplicate_fraction": 0.06280448287725449, "metrics/particle/matching/efficiency": 0.7917365431785583, "metrics/particle/matching/f1": 0.8013728857040405, "metrics/particle/matching/purity": 0.8112466335296631, "metrics/particle/matching/target_coverage_dr0p05": 0.7580026984214783, "metrics/particle/matching/target_coverage_dr0p10": 0.8437701463699341, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5900337100028992, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8273593187332153, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021300343796610832, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1479174643754959, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14810125529766083, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.030020833015441895, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1884460300207138, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009627803228795528, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7245504260063171, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0765402689576149, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.0175940990448, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8517.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2476789355278015, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.964812159538269, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1299585103988647, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9834421873092651, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13585419952869415, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.498703002929688, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.019881103187799454, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0869077444076538, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.001555061899125576, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.20938777923584, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10261.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7906609773635864, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.6135004162788391, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.202079772949219, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.937868356704712, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.4162976145744324, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.29951557517051697, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.803584575653076, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.4962320625782013, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.99060344696045, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023525172844529152, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15698492527008057, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16026458144187927, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.021236736327409744, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20117232203483582, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.029714979231357574, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3895114660263062, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.16298344731330872, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.440821886062622, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1649.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5888100862503052, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7688778638839722, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.383748531341553, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.201356887817383, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2306368350982666, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 56.46845245361328, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06497418135404587, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.649364948272705, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.15582990646362305, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.376884460449219, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2324.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5978970527648926, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.2364325523376465, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.623701095581055, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.18430757522583, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0859425961971283, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.07458695769309998, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.48238468170166, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.24813789129257202, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.028961181640625, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021686898544430733, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.146836519241333, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14531908929347992, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015029769390821457, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18498355150222778, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007382798357866704, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.23880349099636078, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.013638158328831196, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.803578794002533, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18963.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.21007996797561646, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9870017766952515, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7727017998695374, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6093944907188416, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12985698878765106, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.878859519958496, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003960396163165569, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.41869649291038513, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.013951891101896763, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.579336404800415, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22725.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8149469494819641, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.9974396228790283, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.410093307495117, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.886033058166504, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.7305092811584473, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.5116260051727295, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.524733066558838, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.275071382522583, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.461268424987793, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02167794108390808, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14772707223892212, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14697864651679993, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0197643730789423, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18695150315761566, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004977857228368521, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4459724426269531, "metrics/particle/momentum/all/accepted/pz_bias": -0.04048451781272888, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5102394819259644, "metrics/particle/momentum/all/accepted/samples": 29129.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24593307077884674, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0320143699645996, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1141880750656128, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9545041918754578, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13823279738426208, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 17.36118507385254, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012602661736309528, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7596930265426636, "metrics/particle/momentum/all/assigned/pz_bias": -0.001728929695673287, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0359556674957275, "metrics/particle/momentum/all/assigned/samples": 35310.0, "metrics/particle/momentum/all/efficiency": 0.7915704250335693, "metrics/particle/momentum/all/event/energy_bias": -4.625492572784424, "metrics/particle/momentum/all/event/p3_error_mean": 20.995338439941406, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.044763565063477, "metrics/particle/momentum/all/event/px_bias": 2.0608644485473633, "metrics/particle/momentum/all/event/py_bias": 2.7365548610687256, "metrics/particle/momentum/all/event/pz_abs_mean": 14.946920394897461, "metrics/particle/momentum/all/event/pz_bias": -0.02697739005088806, "metrics/particle/momentum/all/event/pz_residual_rms": 26.40325927734375, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021913930773735046, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13080348074436188, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.130999356508255, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02325410395860672, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.16877461969852448, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004261544439941645, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.44610318541526794, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.04413863644003868, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.374589204788208, "metrics/particle/momentum/class_1/accepted/samples": 15722.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13093477487564087, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.4099779725074768, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.468667596578598, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3397074043750763, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07235898077487946, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.6744160652160645, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008435186930000782, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6296900510787964, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02795691415667534, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.760075807571411, "metrics/particle/momentum/class_1/assigned/samples": 17427.0, "metrics/particle/momentum/class_1/efficiency": 0.8861958384513855, "metrics/particle/momentum/class_1/event/energy_bias": 7.621734619140625, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.318925857543945, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.13067626953125, "metrics/particle/momentum/class_1/event/px_bias": 0.40178489685058594, "metrics/particle/momentum/class_1/event/py_bias": 2.1699957847595215, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.892602920532227, "metrics/particle/momentum/class_1/event/pz_bias": 1.0062079429626465, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.435850143432617, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03039884753525257, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19486545026302338, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19591180980205536, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.00469261733815074, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23561379313468933, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03766879811882973, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6620824337005615, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11525782197713852, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9086897373199463, "metrics/particle/momentum/class_2/accepted/samples": 1407.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6862358450889587, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.702933132648468, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9361948370933533, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5583910942077637, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.31548118591308594, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.727441310882568, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.061924684792757034, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5237882137298584, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.07636312395334244, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.728194713592529, "metrics/particle/momentum/class_2/assigned/samples": 2390.0, "metrics/particle/momentum/class_2/efficiency": 0.4711989164352417, "metrics/particle/momentum/class_2/event/energy_bias": -11.485990524291992, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.342159271240234, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.734063148498535, "metrics/particle/momentum/class_2/event/px_bias": 0.8725526928901672, "metrics/particle/momentum/class_2/event/py_bias": 1.145979642868042, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.808923721313477, "metrics/particle/momentum/class_2/event/pz_bias": -0.5292985439300537, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.1192569732666, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02037705108523369, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1640663892030716, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16199758648872375, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.016954297199845314, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20226353406906128, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001724881469272077, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.27556830644607544, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.027981866151094437, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0189746618270874, "metrics/particle/momentum/class_3/accepted/samples": 11595.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3082523047924042, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.752413034439087, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8610200881958008, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6992254257202148, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18439337611198425, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 26.03909683227539, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009420095942914486, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6156570911407471, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.012613785453140736, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.710378646850586, "metrics/particle/momentum/class_3/assigned/samples": 14968.0, "metrics/particle/momentum/class_3/efficiency": 0.7464752197265625, "metrics/particle/momentum/class_3/event/energy_bias": 1.9324781894683838, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.150125503540039, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.556157112121582, "metrics/particle/momentum/class_3/event/px_bias": 0.5408459305763245, "metrics/particle/momentum/class_3/event/py_bias": -0.29295992851257324, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.651683807373047, "metrics/particle/momentum/class_3/event/pz_bias": -0.31215494871139526, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.638628005981445, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.023052243515849113, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18728013336658478, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17822957038879395, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08544124662876129, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22337308526039124, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003937007859349251, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.679304301738739, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.035469505935907364, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5154354572296143, "metrics/particle/momentum/class_4/accepted/samples": 254.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.37109363079071045, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.7252132892608643, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.880140542984009, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.7777976989746094, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2603878080844879, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.860302925109863, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008310249075293541, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7872462272644043, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.06529492139816284, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.8250802755355835, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.6773333549499512, "metrics/particle/momentum/class_4/event/energy_bias": -1.0426135063171387, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.567880392074585, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9797977209091187, "metrics/particle/momentum/class_4/event/px_bias": 0.12384850531816483, "metrics/particle/momentum/class_4/event/py_bias": -0.03267580270767212, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2673532962799072, "metrics/particle/momentum/class_4/event/pz_bias": -0.37190812826156616, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.304605484008789, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013427423313260078, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14937123656272888, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14893068373203278, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.01039237529039383, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18387921154499054, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.026490066200494766, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7933003902435303, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.0682167187333107, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.159881591796875, "metrics/particle/momentum/class_5/accepted/samples": 151.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0860137864947319, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2488166093826294, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.25323641300201416, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.07388296723365784, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.0731707289814949, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.741420328617096, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03658536449074745, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9509011507034302, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.19069620966911316, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.356925964355469, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9207317233085632, "metrics/particle/momentum/class_5/event/energy_bias": -1.6511032581329346, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.1928529739379883, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.6865097284317017, "metrics/particle/momentum/class_5/event/px_bias": 0.12183218449354172, "metrics/particle/momentum/class_5/event/py_bias": -0.25378504395484924, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0895750522613525, "metrics/particle/momentum/class_5/event/pz_bias": 0.18017619848251343, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.7158946990966797, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3553210198879242, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.35736697912216187, "metrics/set_slot/angular_recall_all_dr0p10": 0.9511361122131348, "metrics/set_slot/angular_recall_selected_dr0p10": 0.830206573009491, "metrics/set_slot/soft_count_bias_mean": 0.6435502171516418}, "step_train_time": 5.864131212234497, "step_valid_time": 71038.07378029823, "step_total_time": 71043.93791222572, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000964776013396, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15406928950928256, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3020, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2878, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2863, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9408303367113435, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9529801324503311, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.946866260898174, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04701986754966887, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9359267734553776, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9480132450331126, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9419312386905742, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9408303367113435, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9932817085646515, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14411120902354135, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2257, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2154, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2147, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9426695842450766, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9543642002658396, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9484808454425363, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04563579973416039, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9396061269146608, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.951262738147984, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9453985028621753, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9426695842450766, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9981592751283479, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15332322215924965, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1304, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1236, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1229, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9349470499243571, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9478527607361963, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9413556740289414, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05214723926380368, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9296520423600605, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9424846625766872, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9360243716679361, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9349470499243571} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_29000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..c1c3aa342b3c67d851cb52c5b1c93f242e46e46e --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 1.3736263736263736, "train": {"Classification_binary": 0.14389261603355408, "Cardinality": 0.10703958570957184, "Classification": 0.08612009137868881, "Regression_pt": 0.14143122732639313, "Regression_eta": 0.0013969269348308444, "Regression_sin_phi": 0.000771998951677233, "Regression_cos_phi": 0.000810566358268261, "Regression_energy": 0.13894978165626526, "HitGrouping": 0.19414910674095154, "Auxiliary": 0.21928732097148895, "Total": 1.0237526893615723, "Total_unweighted": 0.5133732086396776}, "train_group": {}, "valid": {"Classification_binary": 0.17220298449198404, "Cardinality": 0.140861044327418, "Classification": 0.09833088517189026, "Regression_pt": 0.13793096939722696, "Regression_eta": 0.0015239780768752098, "Regression_sin_phi": 0.0007667682754496733, "Regression_cos_phi": 0.0007629351069529852, "Regression_energy": 0.13641818364461264, "HitGrouping": 0.1979477604230245, "Auxiliary": 0.2435452938079834, "Total": 1.1173624992370605, "Total_unweighted": 0.5479367041649917, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8213547468185425, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.48919999599456787, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.36746475100517273, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4934263825416565, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4870137572288513, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.41563358902931213, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7684500813484192, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.43371275067329407, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3607828617095947, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.43741726875305176, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4380640685558319, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4804283678531647, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8405582904815674, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4611949622631073, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.33959731459617615, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4636279344558716, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4469197392463684, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.37991204857826233, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8291422128677368, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4675455689430237, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.34968462586402893, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4706205427646637, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4583788216114044, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.39853784441947937, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0030134732369333506, "metrics/hit_grouping/calorimeter/background_recall": 0.1299470067024231, "metrics/hit_grouping/calorimeter/completeness": 0.5220237374305725, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5301369428634644, "metrics/hit_grouping/calorimeter/energy_purity": 0.523147702217102, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4028385877609253, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.854175329208374, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4948185980319977, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2757737934589386, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4994592070579529, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4851042628288269, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.336595356464386, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.828463077545166, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4864181578159332, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.38116076588630676, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.48976296186447144, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4794585108757019, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.40292492508888245, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7787840962409973, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.39292359352111816, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.46867749094963074, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3925345540046692, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.38264986872673035, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5204176306724548, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8154335618019104, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.512481153011322, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.430719256401062, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5181987285614014, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5035383701324463, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4358912706375122, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7840953469276428, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5329181551933289, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5620175004005432, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5285820364952087, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5249974131584167, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4848172962665558, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8484856486320496, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4139428436756134, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24176834523677826, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4152064025402069, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.40439921617507935, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.35017475485801697, "metrics/hit_grouping/calorimeter/purity": 0.5143508911132812, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.841886043548584, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09770023077726364, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12781955301761627, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09645988792181015, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05482427403330803, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8148649334907532, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.20451971888542175, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10829900205135345, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.20253807306289673, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.19653195142745972, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.27604472637176514, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.829677402973175, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4872594177722931, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.36586111783981323, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4906678795814514, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4785805940628052, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4108854830265045, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7376152276992798, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41938287019729614, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4863313138484955, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.49979302287101746, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5726495981216431, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7705431580543518, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.26160338521003723, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.45557254552841187, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.45884549617767334, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.41711875796318054, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6664530038833618, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4340192973613739, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.406689316034317, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.41872498393058777, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6633328795433044, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6960130333900452, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4167853891849518, "metrics/hit_grouping/tracker/all/object_completeness": 0.43477770686149597, "metrics/hit_grouping/tracker/all/object_purity": 0.446604460477829, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6172751784324646, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.02115629054605961, "metrics/hit_grouping/tracker/background_recall": 0.1499447375535965, "metrics/hit_grouping/tracker/completeness": 0.4346866309642792, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.49105697870254517, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7094409465789795, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4125021994113922, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4870605766773224, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4963630139827728, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6086270213127136, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6908316016197205, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4296605587005615, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.42244523763656616, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4432012438774109, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6242512464523315, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.673539936542511, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41280433535575867, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3394848108291626, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.34763264656066895, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6281334757804871, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7171582579612732, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4349155128002167, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.46499258279800415, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.47501787543296814, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5860575437545776, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.72514408826828, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.46366995573043823, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4321780502796173, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3995383381843567, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5504925847053528, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6633013486862183, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3848106861114502, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3959634006023407, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.41810354590415955, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6698310971260071, "metrics/hit_grouping/tracker/purity": 0.41429761052131653, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.879658579826355, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02375565655529499, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.04182840883731842, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.042985863983631134, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06033182516694069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6651305556297302, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2612287104129791, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3031879663467407, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3016974627971649, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5554981827735901, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6721078753471375, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4919702410697937, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.507352888584137, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5222063064575195, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7066196799278259, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5343443155288696, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999761581421, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6302840709686279, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.723591148853302, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6106963753700256, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.014808444306254387, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.036221593618392944, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.27269172668457, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2662546932697296, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8999999165534973, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17463155090808868, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8806881308555603, "metrics/particle/class_1/pid_accuracy": 0.9712850451469421, "metrics/particle/class_2/efficiency": 0.4669837951660156, "metrics/particle/class_2/pid_accuracy": 0.6614115834236145, "metrics/particle/class_3/efficiency": 0.7459720373153687, "metrics/particle/class_3/pid_accuracy": 0.9704011678695679, "metrics/particle/class_4/efficiency": 0.6703406572341919, "metrics/particle/class_4/pid_accuracy": 0.3423019349575043, "metrics/particle/class_5/efficiency": 0.9099617004394531, "metrics/particle/class_5/pid_accuracy": 0.4252631664276123, "metrics/particle/count/bias_mean": -2.8157551288604736, "metrics/particle/count/mae": 4.517578125, "metrics/particle/count/prediction_mean": 67.83919525146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08334559202194214, "metrics/particle/event/energy_response_mean": 0.948459804058075, "metrics/particle/event/met_abs_error": 7.018259525299072, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06175335869193077, "metrics/particle/event/scalar_pt_response_mean": 0.9837586283683777, "metrics/particle/event/vector_pt_closure": 0.062294501811265945, "metrics/particle/matched/delta_eta_abs_mean": 0.014092468656599522, "metrics/particle/matched/delta_phi_abs_mean": 0.012755288742482662, "metrics/particle/matched/delta_r_mean": 0.02127744071185589, "metrics/particle/matched/energy_relative_abs_mean": 0.1466277688741684, "metrics/particle/matched/pid_accuracy": 0.9480530023574829, "metrics/particle/matched/pt_relative_abs_mean": 0.14847537875175476, "metrics/particle/matching/duplicate_fraction": 0.05467317998409271, "metrics/particle/matching/efficiency": 0.7884562015533447, "metrics/particle/matching/f1": 0.8044865131378174, "metrics/particle/matching/purity": 0.821182131767273, "metrics/particle/matching/target_coverage_dr0p05": 0.7569246292114258, "metrics/particle/matching/target_coverage_dr0p10": 0.8399462103843689, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5814366936683655, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8213423490524292, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020675163716077805, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14690595865249634, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15007877349853516, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03463182970881462, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19007204473018646, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00802075956016779, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7150581479072571, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.10353513807058334, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9218859672546387, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8478.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.22822226583957672, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7515397071838379, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8124731779098511, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6662116050720215, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1289016157388687, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.966179847717285, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014816278591752052, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0108977556228638, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.07900037616491318, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9423446655273438, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10124.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7870404720306396, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.253131866455078, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.228352546691895, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.857769727706909, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.1983337253332138, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.1799028068780899, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.941038131713867, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.5873634815216064, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.842738151550293, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02399921789765358, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15644681453704834, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1595003306865692, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02832384593784809, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20201951265335083, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.028676021844148636, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4237029552459717, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.3067147433757782, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.9990994930267334, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1639.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5065743923187256, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2968966960906982, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.370213270187378, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.1989829540252686, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2180781066417694, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 33.682044982910156, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07020623236894608, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.779629945755005, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1923857480287552, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.271795272827148, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2279.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.594271183013916, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.181512355804443, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.00455093383789, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2714887857437134, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.1761195957660675, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.10326055437326431, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.851201057434082, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.18880975246429443, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.47381591796875, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02152523770928383, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1465577930212021, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1468980312347412, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02823651023209095, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18685248494148254, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005832758615724742, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.23535901308059692, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.03003896027803421, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7389178276062012, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18859.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20945391058921814, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8737593293190002, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6811113357543945, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5229129791259766, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12884606420993805, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.589903354644775, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004173511639237404, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.42632609605789185, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0006340002291835845, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.0991413593292236, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22523.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8104774355888367, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.8889539241790771, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.229126930236816, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.662333488464355, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.452420949935913, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9384303092956543, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.493105888366699, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5027260780334473, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.925788879394531, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021416455507278442, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14721901714801788, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14854152500629425, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.03011263720691204, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1886863112449646, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004348426125943661, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.44293028116226196, "metrics/particle/momentum/all/accepted/pz_bias": -0.06719288975000381, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5299322605133057, "metrics/particle/momentum/all/accepted/samples": 28976.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23428206145763397, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8659422397613525, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8946591019630432, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.739070475101471, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.134684756398201, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.222323417663574, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.0115673141553998, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7493342757225037, "metrics/particle/momentum/all/assigned/pz_bias": -0.0350445918738842, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.310678482055664, "metrics/particle/momentum/all/assigned/samples": 34926.0, "metrics/particle/momentum/all/efficiency": 0.787412703037262, "metrics/particle/momentum/all/event/energy_bias": -12.32359790802002, "metrics/particle/momentum/all/event/p3_error_mean": 20.99626350402832, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.729427337646484, "metrics/particle/momentum/all/event/px_bias": 1.077967643737793, "metrics/particle/momentum/all/event/py_bias": -1.2215937376022339, "metrics/particle/momentum/all/event/pz_abs_mean": 15.275861740112305, "metrics/particle/momentum/all/event/pz_bias": -2.2788991928100586, "metrics/particle/momentum/all/event/pz_residual_rms": 27.872631072998047, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02167990803718567, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13113902509212494, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13534043729305267, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.04667266458272934, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17359861731529236, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0037060703616589308, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4464152753353119, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.08329525589942932, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.349287509918213, "metrics/particle/momentum/class_1/accepted/samples": 15650.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11990239471197128, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.34835085272789, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3865111172199249, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2735733985900879, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07229959964752197, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.548931360244751, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007570941466838121, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6182479858398438, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.07045803219079971, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7413408756256104, "metrics/particle/momentum/class_1/assigned/samples": 17303.0, "metrics/particle/momentum/class_1/efficiency": 0.8821374177932739, "metrics/particle/momentum/class_1/event/energy_bias": 2.374046802520752, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.498558044433594, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.362457275390625, "metrics/particle/momentum/class_1/event/px_bias": -1.4922738075256348, "metrics/particle/momentum/class_1/event/py_bias": -0.4602426588535309, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.52343463897705, "metrics/particle/momentum/class_1/event/pz_bias": -2.3991329669952393, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.027202606201172, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029624806717038155, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18826410174369812, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19252413511276245, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.004479618743062019, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23202084004878998, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03249630704522133, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6333098411560059, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.2636340260505676, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.952052593231201, "metrics/particle/momentum/class_2/accepted/samples": 1354.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6530073285102844, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6701856851577759, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9076898694038391, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5363941788673401, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30289727449417114, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.119574069976807, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05882352963089943, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.531024694442749, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0430535264313221, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.909513473510742, "metrics/particle/momentum/class_2/assigned/samples": 2278.0, "metrics/particle/momentum/class_2/efficiency": 0.45344942808151245, "metrics/particle/momentum/class_2/event/energy_bias": -13.724627494812012, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.53817367553711, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.520844459533691, "metrics/particle/momentum/class_2/event/px_bias": 1.227904200553894, "metrics/particle/momentum/class_2/event/py_bias": -0.08644232153892517, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.218252182006836, "metrics/particle/momentum/class_2/event/pz_bias": -1.6065516471862793, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.180879592895508, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020235227420926094, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16383473575115204, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16107012331485748, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.009920274838805199, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20181848108768463, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014676681021228433, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.27905741333961487, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.02465895749628544, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1673204898834229, "metrics/particle/momentum/class_3/accepted/samples": 11583.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30084171891212463, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4541699886322021, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.45316481590271, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2797205448150635, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17897363007068634, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.982349395751953, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00869917031377554, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6128063201904297, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.007388476748019457, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.062664031982422, "metrics/particle/momentum/class_3/assigned/samples": 14829.0, "metrics/particle/momentum/class_3/efficiency": 0.7457026839256287, "metrics/particle/momentum/class_3/event/energy_bias": 0.967322826385498, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.551486015319824, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.131707191467285, "metrics/particle/momentum/class_3/event/px_bias": 1.203906774520874, "metrics/particle/momentum/class_3/event/py_bias": -0.4432451128959656, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.733053207397461, "metrics/particle/momentum/class_3/event/pz_bias": 1.7518641948699951, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.53510284423828, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.019780032336711884, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16381625831127167, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15721730887889862, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06107605621218681, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2002187818288803, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004149377811700106, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.708199679851532, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.042132746428251266, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7402559518814087, "metrics/particle/momentum/class_4/accepted/samples": 241.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.41935017704963684, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.074260711669922, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.560047149658203, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.433126449584961, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.27966102957725525, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.054953575134277, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9527155756950378, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.09377904236316681, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.8622539043426514, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.6426666378974915, "metrics/particle/momentum/class_4/event/energy_bias": -1.2858039140701294, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.840003728866577, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.12156343460083, "metrics/particle/momentum/class_4/event/px_bias": 0.016108930110931396, "metrics/particle/momentum/class_4/event/py_bias": -0.120932936668396, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.483841896057129, "metrics/particle/momentum/class_4/event/pz_bias": -0.2750815749168396, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.387267589569092, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01357459556311369, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14462988078594208, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14742553234100342, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.043414246290922165, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19533994793891907, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04054053872823715, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5773608684539795, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.0630326196551323, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.3599941730499268, "metrics/particle/momentum/class_5/accepted/samples": 148.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06593520194292068, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.23163235187530518, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2228872925043106, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.11683829128742218, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.06172839552164078, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.538223385810852, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.043209876865148544, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7497705221176147, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.10418534278869629, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.538822650909424, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.9024389982223511, "metrics/particle/momentum/class_5/event/energy_bias": -0.654535710811615, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.3215112686157227, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.7887845039367676, "metrics/particle/momentum/class_5/event/px_bias": 0.12232141941785812, "metrics/particle/momentum/class_5/event/py_bias": -0.11073064804077148, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.13666832447052, "metrics/particle/momentum/class_5/event/pz_bias": 0.25000274181365967, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.6760213375091553, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3567616045475006, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3606155216693878, "metrics/set_slot/angular_recall_all_dr0p10": 0.954370379447937, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8277094960212708, "metrics/set_slot/soft_count_bias_mean": -0.5712478756904602}, "step_train_time": 4.702664375305176, "step_valid_time": 73532.16691160202, "step_total_time": 73536.86957621574, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011621052031243, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1488012068573228, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3043, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2907, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2877, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9503105590062112, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9553072625698324, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9528023598820059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0446927374301676, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9405034324942791, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9454485704896484, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9429695181907571, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9503105590062112, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9857116176906394, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14050157598265678, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2260, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2139, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2130, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9361050328227571, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9464601769911505, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9412541254125413, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05353982300884956, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9321663019693655, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9424778761061947, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9372937293729373, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9361050328227571, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9956993998212031, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14798538887499268, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1306, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1221, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1214, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9236006051437217, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9349157733537519, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9292237442922374, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06508422664624809, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9183055975794251, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9295558958652373, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.923896499238965, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9236006051437217} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_3000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..b365150a7654822c54c05a7dc79066780f62d14a --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 0.14209928003031452, "train": {"Classification_binary": 0.4204716980457306, "Cardinality": 0.28294146060943604, "Classification": 0.35243886709213257, "Regression_pt": 0.47327128052711487, "Regression_eta": 0.0061149257235229015, "Regression_sin_phi": 0.0032067974098026752, "Regression_cos_phi": 0.0032075128983706236, "Regression_energy": 0.4673052430152893, "HitGrouping": 0.30848559737205505, "Auxiliary": 0.5466107726097107, "Total": 2.933558464050293, "Total_unweighted": 1.7260163247119635}, "train_group": {}, "valid": {"Classification_binary": 0.3782234191894531, "Cardinality": 0.26526470979054767, "Classification": 0.3039420048395793, "Regression_pt": 0.47825050354003906, "Regression_eta": 0.005411230027675629, "Regression_sin_phi": 0.0027102973933021226, "Regression_cos_phi": 0.0026938014974196753, "Regression_energy": 0.47402111689249676, "HitGrouping": 0.3005926211675008, "Auxiliary": 0.5212651093800863, "Total": 2.713685671488444, "Total_unweighted": 1.6452523733799658, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7840169668197632, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.11560972034931183, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.26478660106658936, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.11278729885816574, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.08551201224327087, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5470805168151855, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7672141790390015, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.1222890093922615, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.22341211140155792, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.11782322078943253, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.08368735015392303, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5476366281509399, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.781180202960968, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.10169675946235657, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.24341587722301483, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.09978974610567093, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.07775697857141495, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5591298341751099, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7809553742408752, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.10751046240329742, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.24833714962005615, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.1051274836063385, "metrics/hit_grouping/calorimeter/all/object_purity": 0.08056680858135223, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.554587185382843, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.001146424561738968, "metrics/hit_grouping/calorimeter/background_recall": 0.014347728341817856, "metrics/hit_grouping/calorimeter/completeness": 0.20481280982494354, "metrics/hit_grouping/calorimeter/energy_completeness": 0.2401208132505417, "metrics/hit_grouping/calorimeter/energy_purity": 0.23682235181331635, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.6165507435798645, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7961568832397461, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.10271794348955154, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.20141100883483887, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.10112934559583664, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.09271655231714249, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5174670219421387, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7809978127479553, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.10551105439662933, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2523030936717987, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.10250928997993469, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.07104658335447311, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5570013523101807, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7499167919158936, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.11929656565189362, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3400231897830963, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.11591722071170807, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.065384641289711, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6278422474861145, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7568007707595825, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.13027575612068176, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3237912058830261, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.12607811391353607, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.09212127327919006, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6081910133361816, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7787626385688782, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3396153151988983, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6489964127540588, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3392847180366516, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.23542556166648865, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5681934952735901, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8058435916900635, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.05664769187569618, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.12367404252290726, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.05587035045027733, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.050336986780166626, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.4983138144016266, "metrics/hit_grouping/calorimeter/purity": 0.2014252096414566, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.903573751449585, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0271648857742548, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10902255773544312, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.027725564315915108, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.009193607605993748, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.043233081698417664, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8501935601234436, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.02719014696776867, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.07298410683870316, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.02716541290283203, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.015785589814186096, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3184226155281067, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.775354266166687, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.11299939453601837, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.25973230600357056, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.11044572293758392, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.08510091155767441, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5752090215682983, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6794484853744507, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.385049968957901, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.23788505792617798, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.18696123361587524, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6920179724693298, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7453116774559021, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28209763765335083, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.22923770546913147, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.16934654116630554, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5201928615570068, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6014496684074402, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.40638941526412964, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.19844897091388702, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.1593533456325531, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7696071267127991, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6360486149787903, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.39065003395080566, "metrics/hit_grouping/tracker/all/object_completeness": 0.21284371614456177, "metrics/hit_grouping/tracker/all/object_purity": 0.1685766726732254, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7273374795913696, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005686149932444096, "metrics/hit_grouping/tracker/background_recall": 0.027233975008130074, "metrics/hit_grouping/tracker/completeness": 0.21906566619873047, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6496102809906006, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6310312747955322, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39540591835975647, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.2586284875869751, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.2229103296995163, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7464492321014404, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6442067623138428, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.3775639832019806, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.18401598930358887, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.13168677687644958, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7161009311676025, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6382641196250916, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.3938683569431305, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.14730548858642578, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.09346339106559753, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6991884708404541, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6637413501739502, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4178483486175537, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.21704600751399994, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.1599004566669464, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6992442607879639, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6575658917427063, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.47413793206214905, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.23446743190288544, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.15601518750190735, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6865763664245605, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5961897969245911, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3403245508670807, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.2035040706396103, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.18210795521736145, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7711668014526367, "metrics/hit_grouping/tracker/purity": 0.20554189383983612, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.9003533124923706, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.015460030175745487, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.00826106034219265, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0035731110256165266, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.050904978066682816, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6817111968994141, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.17552916705608368, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.09387952834367752, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.05429800599813461, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5570470094680786, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5918773412704468, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.46964356303215027, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.25609737634658813, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.20544911921024323, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8461753726005554, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.24228093028068542, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.5750000476837158, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.567686140537262, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6449212431907654, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5439299941062927, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.30687183141708374, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.030544791370630264, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.652347564697266, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.11028489470481873, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.018661726266145706, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.4959186911582947, "metrics/particle/class_1/pid_accuracy": 0.8485943675041199, "metrics/particle/class_2/efficiency": 0.3580246865749359, "metrics/particle/class_2/pid_accuracy": 0.05346409231424332, "metrics/particle/class_3/efficiency": 0.43370571732521057, "metrics/particle/class_3/pid_accuracy": 0.8772619366645813, "metrics/particle/class_4/efficiency": 0.4559118151664734, "metrics/particle/class_4/pid_accuracy": 0.006593406666070223, "metrics/particle/class_5/efficiency": 0.561302661895752, "metrics/particle/class_5/pid_accuracy": 0.1535836160182953, "metrics/particle/count/bias_mean": -4.449869632720947, "metrics/particle/count/mae": 8.001953125, "metrics/particle/count/prediction_mean": 66.205078125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.159138023853302, "metrics/particle/event/energy_response_mean": 1.0349353551864624, "metrics/particle/event/met_abs_error": 12.896942138671875, "metrics/particle/event/scalar_pt_relative_abs_error": 0.1480722725391388, "metrics/particle/event/scalar_pt_response_mean": 1.0769072771072388, "metrics/particle/event/vector_pt_closure": 0.11056199669837952, "metrics/particle/matched/delta_eta_abs_mean": 0.022275468334555626, "metrics/particle/matched/delta_phi_abs_mean": 0.021701142191886902, "metrics/particle/matched/delta_r_mean": 0.03476904705166817, "metrics/particle/matched/energy_relative_abs_mean": 0.20679566264152527, "metrics/particle/matched/pid_accuracy": 0.7977126240730286, "metrics/particle/matched/pt_relative_abs_mean": 0.20515631139278412, "metrics/particle/matching/duplicate_fraction": 0.11599846929311752, "metrics/particle/matching/efficiency": 0.45842471718788147, "metrics/particle/matching/f1": 0.47332993149757385, "metrics/particle/matching/purity": 0.4892370104789734, "metrics/particle/matching/target_coverage_dr0p05": 0.40594881772994995, "metrics/particle/matching/target_coverage_dr0p10": 0.5631645917892456, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.27773988246917725, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5383502840995789, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.034115269780159, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20718979835510254, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.20631657540798187, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.021550482138991356, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24711421132087708, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01987447775900364, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.4217822551727295, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0491393506526947, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.093892812728882, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4780.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.7630874514579773, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.9549694061279297, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.9006881713867188, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.710408926010132, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.4125412404537201, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.66175651550293, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0385393388569355, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.208672523498535, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12988770008087158, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.682649612426758, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9393.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.4437430500984192, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 10.884401321411133, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 16.904315948486328, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.823474407196045, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.724413275718689, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.3641185760498047, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 14.20533275604248, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.5423169136047363, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 21.807119369506836, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03544292598962784, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.21615508198738098, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.20653700828552246, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0009331798064522445, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.24873577058315277, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04287515580654144, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.3196585178375244, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.23923559486865997, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.821506500244141, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 793.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.2888191938400269, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 3.0964066982269287, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10.439863204956055, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10.262580871582031, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.5195729732513428, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 96.37395477294922, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09100152552127838, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.115334987640381, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.4294380247592926, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.9688138961792, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1967.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.2875272035598755, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -11.581086158752441, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 16.064258575439453, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.411876678466797, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04898914694786072, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.20069369673728943, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 15.789854049682617, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.18254536390304565, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 35.79663848876953, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03487849608063698, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20631453394889832, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.20453296601772308, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.02169625461101532, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.24537689983844757, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00302087957970798, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.49210992455482483, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.039921026676893234, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.2513412237167358, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 11255.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.6697573065757751, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.0325517654418945, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.348158597946167, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.1514511108398438, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.399261474609375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.154154777526855, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.017878051847219467, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.2521814107894897, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.08868478238582611, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.4274988174438477, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22206.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.4836907386779785, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 13.411636352539062, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 18.372644424438477, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.987223625183105, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.8282485008239746, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -4.505646705627441, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.948869705200195, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.3465845584869385, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 13.737970352172852, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03468829393386841, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.207026869058609, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.20513403415679932, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.02067641168832779, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24603061378002167, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.009686237201094627, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8423047661781311, "metrics/particle/momentum/all/accepted/pz_bias": 0.02401585876941681, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.204937219619751, "metrics/particle/momentum/all/accepted/samples": 16828.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.7321520447731018, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.676145553588867, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.9183571338653564, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.7245869636535645, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.41002801060676575, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 27.499269485473633, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.02794494479894638, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.6876263618469238, "metrics/particle/momentum/all/assigned/pz_bias": 0.12018334865570068, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.578042507171631, "metrics/particle/momentum/all/assigned/samples": 33566.0, "metrics/particle/momentum/all/efficiency": 0.4572950303554535, "metrics/particle/momentum/all/event/energy_bias": 12.714949607849121, "metrics/particle/momentum/all/event/p3_error_mean": 37.90305709838867, "metrics/particle/momentum/all/event/pt_vector_error_mean": 18.403369903564453, "metrics/particle/momentum/all/event/px_bias": 2.5036723613739014, "metrics/particle/momentum/all/event/py_bias": -6.070458889007568, "metrics/particle/momentum/all/event/pz_abs_mean": 28.69943618774414, "metrics/particle/momentum/all/event/pz_bias": 5.071446418762207, "metrics/particle/momentum/all/event/pz_residual_rms": 44.138511657714844, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.042745981365442276, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.22432321310043335, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.22077541053295135, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.042074691504240036, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2600243091583252, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.010941417887806892, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.9670989513397217, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.01502617821097374, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.3134989738464355, "metrics/particle/momentum/class_1/accepted/samples": 8774.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.574862003326416, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 2.2739808559417725, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.495119333267212, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 2.260557174682617, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.3644259572029114, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 12.318246841430664, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.026362469419836998, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.619778037071228, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.1415773183107376, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.05643367767334, "metrics/particle/momentum/class_1/assigned/samples": 16349.0, "metrics/particle/momentum/class_1/efficiency": 0.4945606291294098, "metrics/particle/momentum/class_1/event/energy_bias": 31.595149993896484, "metrics/particle/momentum/class_1/event/p3_error_mean": 33.23530197143555, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 20.232128143310547, "metrics/particle/momentum/class_1/event/px_bias": 3.7862420082092285, "metrics/particle/momentum/class_1/event/py_bias": -4.1671013832092285, "metrics/particle/momentum/class_1/event/pz_abs_mean": 22.16550636291504, "metrics/particle/momentum/class_1/event/pz_bias": 2.3328640460968018, "metrics/particle/momentum/class_1/event/pz_residual_rms": 29.906858444213867, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.043575823307037354, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23441191017627716, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23028627038002014, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0797194391489029, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27156537771224976, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031865041702985764, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8329071998596191, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.09179161489009857, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.146117687225342, "metrics/particle/momentum/class_2/accepted/samples": 1067.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8236554265022278, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.0198431015014648, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.6109788417816162, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 1.2330079078674316, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3563685715198517, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 9.376652717590332, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0614272803068161, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.038017988204956, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.13762624561786652, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.18718147277832, "metrics/particle/momentum/class_2/assigned/samples": 2214.0, "metrics/particle/momentum/class_2/efficiency": 0.3573342263698578, "metrics/particle/momentum/class_2/event/energy_bias": -39.365928649902344, "metrics/particle/momentum/class_2/event/p3_error_mean": 28.502962112426758, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.70134449005127, "metrics/particle/momentum/class_2/event/px_bias": -2.1568121910095215, "metrics/particle/momentum/class_2/event/py_bias": -0.26274216175079346, "metrics/particle/momentum/class_2/event/pz_abs_mean": 20.820240020751953, "metrics/particle/momentum/class_2/event/pz_bias": -1.3140671253204346, "metrics/particle/momentum/class_2/event/pz_residual_rms": 34.09463882446289, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022628316655755043, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1784851849079132, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17944128811359406, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.019977618008852005, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.22042399644851685, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0029704440385103226, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.470133900642395, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.041944719851017, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3287389278411865, "metrics/particle/momentum/class_3/accepted/samples": 6733.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.8986033797264099, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.469265460968018, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.753548622131348, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.637587547302246, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4693342447280884, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 39.21474075317383, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.02345636487007141, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.5196021795272827, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.08449950069189072, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.577363967895508, "metrics/particle/momentum/class_3/assigned/samples": 14495.0, "metrics/particle/momentum/class_3/efficiency": 0.4334642291069031, "metrics/particle/momentum/class_3/event/energy_bias": 29.23740577697754, "metrics/particle/momentum/class_3/event/p3_error_mean": 28.164079666137695, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 16.096181869506836, "metrics/particle/momentum/class_3/event/px_bias": -0.28572607040405273, "metrics/particle/momentum/class_3/event/py_bias": -0.3663256764411926, "metrics/particle/momentum/class_3/event/pz_abs_mean": 19.586406707763672, "metrics/particle/momentum/class_3/event/pz_bias": 3.729583263397217, "metrics/particle/momentum/class_3/event/pz_residual_rms": 33.065895080566406, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03918891400098801, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.2567923069000244, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.24687829613685608, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.11489205807447433, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2839984893798828, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.04968944191932678, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 2.374875545501709, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.20559772849082947, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 4.4583587646484375, "metrics/particle/momentum/class_4/accepted/samples": 161.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.7833762764930725, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 12.646904945373535, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 10.071208000183105, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 9.849571228027344, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.46518105268478394, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 33.98643493652344, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.04735375940799713, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.419522523880005, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.3330977261066437, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.593698501586914, "metrics/particle/momentum/class_4/assigned/samples": 359.0, "metrics/particle/momentum/class_4/efficiency": 0.42933332920074463, "metrics/particle/momentum/class_4/event/energy_bias": -6.437422752380371, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.833254814147949, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.7506103515625, "metrics/particle/momentum/class_4/event/px_bias": 0.6173573136329651, "metrics/particle/momentum/class_4/event/py_bias": -0.4521946907043457, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.607815980911255, "metrics/particle/momentum/class_4/event/pz_bias": -0.0659131407737732, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.958776950836182, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.037850894033908844, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.24123266339302063, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.22872012853622437, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.10462771356105804, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2658859193325043, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05376344174146652, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9946366548538208, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.5884499549865723, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.9115943908691406, "metrics/particle/momentum/class_5/accepted/samples": 93.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.31502416729927063, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.946441650390625, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.0128968954086304, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.5297988057136536, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3087248206138611, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.071356773376465, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0939597338438034, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.6490063667297363, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.471944659948349, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 7.6985368728637695, "metrics/particle/momentum/class_5/assigned/samples": 149.0, "metrics/particle/momentum/class_5/efficiency": 0.5670731663703918, "metrics/particle/momentum/class_5/event/energy_bias": -2.3142502307891846, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.793108940124512, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.6263813972473145, "metrics/particle/momentum/class_5/event/px_bias": 0.5426113605499268, "metrics/particle/momentum/class_5/event/py_bias": -0.8220947980880737, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.464604377746582, "metrics/particle/momentum/class_5/event/pz_bias": 0.3889797329902649, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.217170715332031, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.7030417323112488, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.7079732418060303, "metrics/set_slot/angular_recall_all_dr0p10": 0.8283544778823853, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6019110679626465, "metrics/set_slot/soft_count_bias_mean": 0.5909695029258728}, "step_train_time": 4.821022987365723, "step_valid_time": 7184.024151563644, "step_total_time": 7188.845174789429, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1501594353874927, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3300025798848477, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2533, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2228, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8280483818241255, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.835973597359736, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8319921169321728, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.16402640264026402, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7283425956194834, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7353135313531353, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7318114632944654, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8280483818241255, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.107005740958825, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28197679033529854, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2248, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1900, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1770, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8315098468271335, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8451957295373665, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8382969335980587, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.15480427046263345, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.774617067833698, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7873665480427047, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7809397749834547, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8315098468271335, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0705159433887157, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27533850998795106, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1256, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1043, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 954, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7889561270801816, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8304140127388535, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8091543832428238, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1695859872611465, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7216338880484114, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7595541401273885, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7401086113266098, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7889561270801816} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_30000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..d340bd114a06c6936e09ad0f6134123e9dd1bba4 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 1.4209928003031451, "train": {"Classification_binary": 0.15405774116516113, "Cardinality": 0.13367383182048798, "Classification": 0.10004149377346039, "Regression_pt": 0.1518511176109314, "Regression_eta": 0.0014484357088804245, "Regression_sin_phi": 0.0006306662689894438, "Regression_cos_phi": 0.0006895075784996152, "Regression_energy": 0.14992137253284454, "HitGrouping": 0.19826388359069824, "Auxiliary": 0.24156007170677185, "Total": 1.0853697061538696, "Total_unweighted": 0.5586403346387669}, "train_group": {}, "valid": {"Classification_binary": 0.17323567469914755, "Cardinality": 0.1539241075515747, "Classification": 0.10533866286277771, "Regression_pt": 0.13893340031305948, "Regression_eta": 0.0015312366498013337, "Regression_sin_phi": 0.0008139134539912144, "Regression_cos_phi": 0.0007804580964148045, "Regression_energy": 0.1378829081853231, "HitGrouping": 0.19533689816792807, "Auxiliary": 0.24906784296035767, "Total": 1.1452925205230713, "Total_unweighted": 0.5585162542605152, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8157804012298584, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.48855170607566833, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.36822181940078735, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.48917925357818604, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4907591938972473, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4366423785686493, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7610362768173218, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4264804422855377, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.36299851536750793, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.42765435576438904, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4283011853694916, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.511447548866272, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8354874849319458, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.45805948972702026, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.34283730387687683, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.45916548371315, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4457302987575531, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.39379772543907166, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.823736846446991, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4648454785346985, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3520928919315338, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4658117890357971, "metrics/hit_grouping/calorimeter/all/object_purity": 0.45801877975463867, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4159117043018341, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0031656159553676844, "metrics/hit_grouping/calorimeter/background_recall": 0.13558462262153625, "metrics/hit_grouping/calorimeter/completeness": 0.524296224117279, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5374070405960083, "metrics/hit_grouping/calorimeter/energy_purity": 0.5303914546966553, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.402156800031662, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8477057218551636, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4924202561378479, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2799840569496155, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.49527010321617126, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4851205348968506, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.35639509558677673, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8221904039382935, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.48380985856056213, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3868033289909363, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4831320345401764, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4749005138874054, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.42745277285575867, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7764231562614441, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3895159959793091, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.46415314078330994, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.38829824328422546, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.38575178384780884, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5256380438804626, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8120494484901428, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5134184956550598, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4387178122997284, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5158798098564148, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5056635737419128, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.44791916012763977, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7964284420013428, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5546329617500305, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5630468130111694, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5514498353004456, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5278226137161255, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4678332507610321, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.838906466960907, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.40462541580200195, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23864124715328217, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.404561847448349, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4011259078979492, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3770923912525177, "metrics/hit_grouping/calorimeter/purity": 0.5166688561439514, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8307017683982849, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09264086186885834, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11466165632009506, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.089285708963871, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06031721830368042, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0357142873108387, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8145419955253601, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.20380903780460358, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10005886107683182, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.20239688456058502, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.19811655580997467, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2937021851539612, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8241019248962402, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4844943583011627, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3690771460533142, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4856548309326172, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.47802403569221497, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.42846640944480896, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.730473518371582, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.42155584692955017, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4905192255973816, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5099188089370728, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5767057538032532, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7655443549156189, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2881253659725189, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4627954959869385, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4646841585636139, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4291742146015167, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6552354693412781, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45042452216148376, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4070547819137573, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.42680951952934265, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6767160892486572, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6865075826644897, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4295636713504791, "metrics/hit_grouping/tracker/all/object_completeness": 0.43682458996772766, "metrics/hit_grouping/tracker/all/object_purity": 0.4551504850387573, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6276936531066895, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.024944409728050232, "metrics/hit_grouping/tracker/background_recall": 0.16624565422534943, "metrics/hit_grouping/tracker/completeness": 0.4354158639907837, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.48746252059936523, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7011842727661133, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.421883225440979, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.48931214213371277, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5061245560646057, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6150271892547607, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6844624280929565, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.442548543214798, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.42827796936035156, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.44828295707702637, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6306044459342957, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6583495736122131, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.432461678981781, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3373492956161499, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.35712024569511414, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.650856614112854, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7124179601669312, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45359599590301514, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.473861426115036, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.48444339632987976, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5912371277809143, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6942293643951416, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4766009747982025, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.42203691601753235, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.41476282477378845, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5991379022598267, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6514388918876648, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3898884952068329, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.39132630825042725, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4243089556694031, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6802075505256653, "metrics/hit_grouping/tracker/purity": 0.4166049063205719, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8747006058692932, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.028657617047429085, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.04718643054366112, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.04744047299027443, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06749623268842697, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6645594239234924, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.273102730512619, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3205485939979553, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3263465464115143, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5529168844223022, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6609591245651245, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5060153603553772, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.50724858045578, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5296141505241394, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7189301252365112, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5128889679908752, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6231294870376587, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7141984105110168, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.624634325504303, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.036263760179281235, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.050159551203250885, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.66980743408203, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2820059359073639, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.19038277864456177, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.879753589630127, "metrics/particle/class_1/pid_accuracy": 0.97320556640625, "metrics/particle/class_2/efficiency": 0.45928192138671875, "metrics/particle/class_2/pid_accuracy": 0.6478421688079834, "metrics/particle/class_3/efficiency": 0.7233456373214722, "metrics/particle/class_3/pid_accuracy": 0.9680246710777283, "metrics/particle/class_4/efficiency": 0.6743487119674683, "metrics/particle/class_4/pid_accuracy": 0.4041604697704315, "metrics/particle/class_5/efficiency": 0.9233716726303101, "metrics/particle/class_5/pid_accuracy": 0.4564315378665924, "metrics/particle/count/bias_mean": -4.100911617279053, "metrics/particle/count/mae": 5.104817867279053, "metrics/particle/count/prediction_mean": 66.55403900146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08287154883146286, "metrics/particle/event/energy_response_mean": 0.9498193264007568, "metrics/particle/event/met_abs_error": 7.156594753265381, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06271141022443771, "metrics/particle/event/scalar_pt_response_mean": 0.9749614596366882, "metrics/particle/event/vector_pt_closure": 0.06277935951948166, "metrics/particle/matched/delta_eta_abs_mean": 0.014104858040809631, "metrics/particle/matched/delta_phi_abs_mean": 0.013131254352629185, "metrics/particle/matched/delta_r_mean": 0.021590039134025574, "metrics/particle/matched/energy_relative_abs_mean": 0.14716319739818573, "metrics/particle/matched/pid_accuracy": 0.9480616450309753, "metrics/particle/matched/pt_relative_abs_mean": 0.148589089512825, "metrics/particle/matching/duplicate_fraction": 0.05194322392344475, "metrics/particle/matching/efficiency": 0.7779425978660583, "metrics/particle/matching/f1": 0.8011938333511353, "metrics/particle/matching/purity": 0.8258777260780334, "metrics/particle/matching/target_coverage_dr0p05": 0.7406888604164124, "metrics/particle/matching/target_coverage_dr0p10": 0.8283452987670898, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5743047595024109, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.811363160610199, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020904898643493652, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14667607843875885, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14803627133369446, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03313314542174339, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18749931454658508, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008875030092895031, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7274010181427002, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07055655121803284, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.103457450866699, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8338.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.21427218616008759, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9499599933624268, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9191086292266846, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7758461236953735, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12821802496910095, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.679276466369629, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017799677327275276, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0578231811523438, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.04351912811398506, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2097747325897217, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9944.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7740437984466553, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.8639532327651978, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.649746894836426, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.929145336151123, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.42779165506362915, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.3096584975719452, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.3864054679870605, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.019454941153526306, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.73602294921875, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02397499419748783, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15667825937271118, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15831118822097778, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018868371844291687, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19621609151363373, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02479851245880127, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3767035007476807, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2690695822238922, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.305363178253174, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1613.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4602290689945221, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.183366298675537, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.301666736602783, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.1245927810668945, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20816326141357422, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 19.824748992919922, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06167800351977348, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6127755641937256, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.195072740316391, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.466690063476562, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2205.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5848441123962402, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.4910173416137695, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.844176292419434, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.265342116355896, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.15285059809684753, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.11154746264219284, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.694002151489258, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.09046375751495361, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.04914665222168, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02184191346168518, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14650601148605347, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14820632338523865, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.030311545357108116, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18816839158535004, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009109908132813871, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2390778362751007, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00803548377007246, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7958247065544128, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18661.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20641769468784332, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9034194350242615, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7119734287261963, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5535150170326233, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12692998349666595, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.101292133331299, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0037253142800182104, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4080827534198761, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.015549307689070702, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5524320602416992, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22280.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8019682765007019, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -3.440965414047241, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.62330436706543, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.106351852416992, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.4641822576522827, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.36836257576942444, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.304206848144531, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.40934160351753235, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.682323932647705, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021689103916287422, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14712902903556824, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1487264186143875, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.030488699674606323, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18843695521354675, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004578498657792807, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4455167055130005, "metrics/particle/momentum/all/accepted/pz_bias": -0.04097092151641846, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5226191282272339, "metrics/particle/momentum/all/accepted/samples": 28612.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22494158148765564, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9347907900810242, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8736112117767334, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7183495759963989, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13250456750392914, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.212180137634277, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011501931585371494, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7369440793991089, "metrics/particle/momentum/all/assigned/pz_bias": -0.015000464394688606, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.020982027053833, "metrics/particle/momentum/all/assigned/samples": 34429.0, "metrics/particle/momentum/all/efficiency": 0.7775211334228516, "metrics/particle/momentum/all/event/energy_bias": -11.795936584472656, "metrics/particle/momentum/all/event/p3_error_mean": 20.5736141204834, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.288029670715332, "metrics/particle/momentum/all/event/px_bias": 1.7391233444213867, "metrics/particle/momentum/all/event/py_bias": -0.05284330993890762, "metrics/particle/momentum/all/event/pz_abs_mean": 15.268908500671387, "metrics/particle/momentum/all/event/pz_bias": -0.29942309856414795, "metrics/particle/momentum/all/event/pz_residual_rms": 26.19142723083496, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.022266289219260216, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.131393164396286, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13446062803268433, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.04132228344678879, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.172056183218956, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0042981780134141445, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4548565447330475, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.041696976870298386, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4624056816101074, "metrics/particle/momentum/class_1/accepted/samples": 15588.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11606637388467789, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.35899537801742554, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.39579662680625916, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.281270295381546, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06806434690952301, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.170963764190674, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008014402352273464, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.622872531414032, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.011378329247236252, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8027188777923584, "metrics/particle/momentum/class_1/assigned/samples": 17219.0, "metrics/particle/momentum/class_1/efficiency": 0.8786426782608032, "metrics/particle/momentum/class_1/event/energy_bias": 4.819782733917236, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.304157257080078, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.358503341674805, "metrics/particle/momentum/class_1/event/px_bias": 3.379436731338501, "metrics/particle/momentum/class_1/event/py_bias": -0.2609978914260864, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.349151611328125, "metrics/particle/momentum/class_1/event/pz_bias": 0.6177228689193726, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.667652130126953, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029466379433870316, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1886666715145111, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1895042061805725, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.011242268607020378, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22770078480243683, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02581121027469635, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5257267951965332, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.20384828746318817, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.59010648727417, "metrics/particle/momentum/class_2/accepted/samples": 1356.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5665979385375977, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6199582815170288, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8016706109046936, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4263777434825897, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2764568626880646, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.973019123077393, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04988345131278038, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.380045175552368, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.2985132932662964, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.510920524597168, "metrics/particle/momentum/class_2/assigned/samples": 2145.0, "metrics/particle/momentum/class_2/efficiency": 0.4541192352771759, "metrics/particle/momentum/class_2/event/energy_bias": -15.40446949005127, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.529829025268555, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.451841354370117, "metrics/particle/momentum/class_2/event/px_bias": -1.0872390270233154, "metrics/particle/momentum/class_2/event/py_bias": -0.7000406980514526, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.233903884887695, "metrics/particle/momentum/class_2/event/pz_bias": -1.2365188598632812, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.020462036132812, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020098108798265457, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16330572962760925, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1632048487663269, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.020047184079885483, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20375606417655945, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015081617748364806, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2786523997783661, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.026224525645375252, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0287317037582397, "metrics/particle/momentum/class_3/accepted/samples": 11272.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3015277087688446, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6375948190689087, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4262497425079346, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2543827295303345, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18471688032150269, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.213064193725586, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009208356030285358, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6128149032592773, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.018163835629820824, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7348592281341553, "metrics/particle/momentum/class_3/assigned/samples": 14552.0, "metrics/particle/momentum/class_3/efficiency": 0.7256808280944824, "metrics/particle/momentum/class_3/event/energy_bias": 0.7224138975143433, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.128348350524902, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.460121154785156, "metrics/particle/momentum/class_3/event/px_bias": -0.8001293540000916, "metrics/particle/momentum/class_3/event/py_bias": 0.9887811541557312, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.670970916748047, "metrics/particle/momentum/class_3/event/pz_bias": 0.17441648244857788, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.630206108093262, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02035200037062168, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16805551946163177, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16182096302509308, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07844524830579758, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20689068734645844, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.008163264952600002, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7078346610069275, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.04580412805080414, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6791306734085083, "metrics/particle/momentum/class_4/accepted/samples": 245.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3806098699569702, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.254976749420166, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.100459098815918, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9839997291564941, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2800000011920929, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.003484725952148, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011428571306169033, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8351662158966064, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07009395956993103, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9602015018463135, "metrics/particle/momentum/class_4/assigned/samples": 350.0, "metrics/particle/momentum/class_4/efficiency": 0.653333306312561, "metrics/particle/momentum/class_4/event/energy_bias": -0.7831078767776489, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.656872034072876, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.0022571086883545, "metrics/particle/momentum/class_4/event/px_bias": 0.19842208921909332, "metrics/particle/momentum/class_4/event/py_bias": 0.04628552496433258, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.351346492767334, "metrics/particle/momentum/class_4/event/pz_bias": -0.07267431169748306, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.904134273529053, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013200083747506142, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1570286899805069, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15317200124263763, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.011493762955069542, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1950620710849762, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.06622516363859177, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8115525245666504, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.4036799371242523, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.381698131561279, "metrics/particle/momentum/class_5/accepted/samples": 151.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.058711882680654526, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3253747820854187, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3239600658416748, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.16016259789466858, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.06748466193675995, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.2433369159698486, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.07975459843873978, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.0356903076171875, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.18975771963596344, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.713376045227051, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9207317233085632, "metrics/particle/momentum/class_5/event/energy_bias": -1.1505554914474487, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.401505470275879, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.7561390399932861, "metrics/particle/momentum/class_5/event/px_bias": 0.04863264411687851, "metrics/particle/momentum/class_5/event/py_bias": -0.12687142193317413, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2905328273773193, "metrics/particle/momentum/class_5/event/pz_bias": 0.21763069927692413, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.095958232879639, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3568851053714752, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.35903072357177734, "metrics/set_slot/angular_recall_all_dr0p10": 0.9508965611457825, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8178869485855103, "metrics/set_slot/soft_count_bias_mean": -1.7216764688491821}, "step_train_time": 7.794251918792725, "step_valid_time": 76169.01926136017, "step_total_time": 76176.8135137558, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9955061861725378, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15006502980651293, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3039, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2900, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2882, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9480222294867604, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9542612701546561, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9511315185306658, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04573872984534386, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9421379535796012, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9483382691674893, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9452279435880616, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9480222294867604, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9885052568487491, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14648420357293235, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2264, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2148, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2140, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9400437636761488, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9487632508833922, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.944383380962849, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05123674911660778, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.936542669584245, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9452296819787986, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9408661244229501, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9400437636761488, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9908601534065316, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15845678682566966, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1309, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1220, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1209, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9228441754916793, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.932009167303285, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.927404028886355, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06799083269671505, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9145234493192134, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9236058059587471, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9190421892816419, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9228441754916793} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_31000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..b02b685c03144cdf92bc4509fa18852b377119b6 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 1.4683592269799166, "train": {"Classification_binary": 0.16927263140678406, "Cardinality": 0.14462314546108246, "Classification": 0.08878561854362488, "Regression_pt": 0.1307753473520279, "Regression_eta": 0.0016100506763905287, "Regression_sin_phi": 0.0007938757189549506, "Regression_cos_phi": 0.0007913825684227049, "Regression_energy": 0.12978796660900116, "HitGrouping": 0.19667574763298035, "Auxiliary": 0.23741410672664642, "Total": 1.1053743362426758, "Total_unweighted": 0.5218168728752062}, "train_group": {}, "valid": {"Classification_binary": 0.16862197717030844, "Cardinality": 0.1300858755906423, "Classification": 0.09814895192782085, "Regression_pt": 0.1341456671555837, "Regression_eta": 0.001515138428658247, "Regression_sin_phi": 0.0007605934515595436, "Regression_cos_phi": 0.0007484414769957463, "Regression_energy": 0.13288177053133646, "HitGrouping": 0.1946800152460734, "Auxiliary": 0.23540892203648886, "Total": 1.0859278837839763, "Total_unweighted": 0.5368225401422629, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8190456628799438, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4927849769592285, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3711554706096649, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.49492019414901733, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4900378882884979, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4227311313152313, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7664220929145813, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4331651031970978, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.35228949785232544, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4362764060497284, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4421094059944153, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4922451972961426, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8394796252250671, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4611106514930725, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.34232816100120544, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.46330058574676514, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.44851523637771606, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.38523489236831665, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8276170492172241, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4685368835926056, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.35183486342430115, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.47078174352645874, "metrics/hit_grouping/calorimeter/all/object_purity": 0.46059733629226685, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.40490251779556274, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0026008698623627424, "metrics/hit_grouping/calorimeter/background_recall": 0.1177697628736496, "metrics/hit_grouping/calorimeter/completeness": 0.528660774230957, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5386249423027039, "metrics/hit_grouping/calorimeter/energy_purity": 0.5314589142799377, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.39842915534973145, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8543445467948914, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.49493134021759033, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2783909738063812, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4988177418708801, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.483313649892807, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3387005031108856, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8240531086921692, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4860350787639618, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.38380929827690125, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4876798987388611, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.48014578223228455, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.416973739862442, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7767108082771301, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.39709094166755676, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.46937355399131775, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3965934216976166, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3945856988430023, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5277262330055237, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8122981786727905, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5135727524757385, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4351094663143158, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.518028199672699, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5110452771186829, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.44340869784355164, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7898728251457214, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5495609045028687, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5666494965553284, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.548314094543457, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5317158102989197, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.47967061400413513, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8477323055267334, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4129672050476074, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24133913218975067, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.41337430477142334, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4006897807121277, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.35673555731773376, "metrics/hit_grouping/calorimeter/purity": 0.5206748247146606, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8214598894119263, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06960365176200867, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12406015396118164, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07108843326568604, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03999900445342064, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.05827067792415619, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8259414434432983, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.20624253153800964, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11477339267730713, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.20677441358566284, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.19047461450099945, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.29605650901794434, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8278046250343323, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.48868662118911743, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.367882639169693, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.49103304743766785, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4815075397491455, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.41621488332748413, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7403848171234131, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4199623465538025, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4953329563140869, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5035237669944763, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5645371675491333, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7743760943412781, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2706449627876282, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.46526312828063965, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4704309403896332, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.42374923825263977, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6714516878128052, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44380486011505127, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.41935303807258606, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4308282136917114, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6610303521156311, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7002405524253845, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42368656396865845, "metrics/hit_grouping/tracker/all/object_completeness": 0.4460963010787964, "metrics/hit_grouping/tracker/all/object_purity": 0.45609617233276367, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6138468384742737, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011348474770784378, "metrics/hit_grouping/tracker/background_recall": 0.1040022075176239, "metrics/hit_grouping/tracker/completeness": 0.44542086124420166, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.49316316843032837, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7128273844718933, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4110994338989258, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4977048635482788, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5054473280906677, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6036296486854553, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6943695545196533, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43946269154548645, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.43598130345344543, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4524472653865814, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6246142387390137, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6801822781562805, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43390440940856934, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.34998753666877747, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.35820648074150085, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6241658926010132, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7229781746864319, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4432368278503418, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.47766193747520447, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.48521688580513, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5764625668525696, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7174128890037537, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.46736451983451843, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4420108497142792, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4212266206741333, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5769704580307007, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6676023006439209, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.390440434217453, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4057922661304474, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.424489289522171, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6690583825111389, "metrics/hit_grouping/tracker/purity": 0.4203168749809265, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8830530643463135, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01923076994717121, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.0500890277326107, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0534340925514698, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06711915880441666, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6687133312225342, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.25916364789009094, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.31430625915527344, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.32411324977874756, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5549819469451904, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6765289306640625, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5015388131141663, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5191469788551331, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5301553010940552, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.701359748840332, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.535555362701416, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7562499642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6122220754623413, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7004262208938599, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6223181486129761, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.013597397133708, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04784340411424637, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.549518585205078, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.27896103262901306, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18733786046504974, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.891730546951294, "metrics/particle/class_1/pid_accuracy": 0.9694377183914185, "metrics/particle/class_2/efficiency": 0.490089476108551, "metrics/particle/class_2/pid_accuracy": 0.6711347103118896, "metrics/particle/class_3/efficiency": 0.7526615858078003, "metrics/particle/class_3/pid_accuracy": 0.9726393222808838, "metrics/particle/class_4/efficiency": 0.6833667159080505, "metrics/particle/class_4/pid_accuracy": 0.4164222776889801, "metrics/particle/class_5/efficiency": 0.9272030591964722, "metrics/particle/class_5/pid_accuracy": 0.4070248007774353, "metrics/particle/count/bias_mean": -2.24609375, "metrics/particle/count/mae": 4.098958492279053, "metrics/particle/count/prediction_mean": 68.40885162353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08194015175104141, "metrics/particle/event/energy_response_mean": 0.9536722302436829, "metrics/particle/event/met_abs_error": 9.50568675994873, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06288579851388931, "metrics/particle/event/scalar_pt_response_mean": 0.9845425486564636, "metrics/particle/event/vector_pt_closure": 0.07251652330160141, "metrics/particle/matched/delta_eta_abs_mean": 0.014330795034766197, "metrics/particle/matched/delta_phi_abs_mean": 0.012496452778577805, "metrics/particle/matched/delta_r_mean": 0.021313156932592392, "metrics/particle/matched/energy_relative_abs_mean": 0.14173340797424316, "metrics/particle/matched/pid_accuracy": 0.948326587677002, "metrics/particle/matched/pt_relative_abs_mean": 0.14419996738433838, "metrics/particle/matching/duplicate_fraction": 0.0555122010409832, "metrics/particle/matching/efficiency": 0.7986934185028076, "metrics/particle/matching/f1": 0.8115935325622559, "metrics/particle/matching/purity": 0.824917197227478, "metrics/particle/matching/target_coverage_dr0p05": 0.7673552632331848, "metrics/particle/matching/target_coverage_dr0p10": 0.8479258418083191, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6015424728393555, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8306028246879578, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021131617948412895, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1399262547492981, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14437465369701385, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016742084175348282, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1838957816362381, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007906977087259293, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7088658213615417, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07495636492967606, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.009613513946533, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8600.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.22193950414657593, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.039221167564392, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0734022855758667, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9207543730735779, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12964588403701782, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.334244728088379, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01794946752488613, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0671988725662231, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.08025438338518143, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2110190391540527, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10251.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.798366129398346, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.7183520793914795, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.304253578186035, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.529991626739502, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.2733006477355957, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.6642583012580872, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.762002468109131, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.932861566543579, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.004472732543945, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02420378290116787, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15591536462306976, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15820786356925964, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.004558914806693792, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19798554480075836, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022366097196936607, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3868846893310547, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2837418019771576, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.565181255340576, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1699.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5270910263061523, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4967625141143799, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.4962334632873535, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6.305186748504639, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21475054323673248, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 150.32574462890625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.058568328619003296, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5884037017822266, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.31696251034736633, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.580334663391113, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2305.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6160261034965515, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.32826042175293, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.298065185546875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3365435600280762, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.040763579308986664, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04526668041944504, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.135210990905762, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5443696975708008, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.646530151367188, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021346762776374817, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14090925455093384, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14257676899433136, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.006345045752823353, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18137088418006897, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009430996724404395, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.23815062642097473, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.003597876289859414, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7795987129211426, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19086.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.19892804324626923, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.051884412765503, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8354460000991821, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.668041467666626, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12520994246006012, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.653489112854004, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004508087877184153, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.41361016035079956, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.006550748832523823, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6391342878341675, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22626.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8202329277992249, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -3.428964614868164, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.033348083496094, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.3272066116333, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 4.455911636352539, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 3.6852707862854004, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.136499404907227, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.0053514242172241, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.57901668548584, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02144898846745491, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.141489177942276, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14400671422481537, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.00928463600575924, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1831112802028656, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004219840280711651, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4423312544822693, "metrics/particle/momentum/all/accepted/pz_bias": -0.04067963734269142, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5203977823257446, "metrics/particle/momentum/all/accepted/samples": 29385.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22713300585746765, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0773415565490723, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2756541967391968, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.111000418663025, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13236881792545319, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 39.57421875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011966346763074398, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7465314269065857, "metrics/particle/momentum/all/assigned/pz_bias": -0.04836289584636688, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0912086963653564, "metrics/particle/momentum/all/assigned/samples": 35182.0, "metrics/particle/momentum/all/efficiency": 0.7985271215438843, "metrics/particle/momentum/all/event/energy_bias": -10.475577354431152, "metrics/particle/momentum/all/event/p3_error_mean": 23.573867797851562, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.750421524047852, "metrics/particle/momentum/all/event/px_bias": 5.7699761390686035, "metrics/particle/momentum/all/event/py_bias": 4.3947954177856445, "metrics/particle/momentum/all/event/pz_abs_mean": 16.595745086669922, "metrics/particle/momentum/all/event/pz_bias": -4.482583045959473, "metrics/particle/momentum/all/event/pz_residual_rms": 28.000322341918945, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02120363712310791, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.12262887507677078, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12771469354629517, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011709271930158138, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.16375580430030823, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003852955996990204, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.44701558351516724, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.05581098422408104, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4279170036315918, "metrics/particle/momentum/class_1/accepted/samples": 15832.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11888816207647324, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.32944202423095703, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3900156021118164, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2512194514274597, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06442883610725403, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.138940811157227, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00823353324085474, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.624434232711792, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.05455620214343071, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.894477605819702, "metrics/particle/momentum/class_1/assigned/samples": 17368.0, "metrics/particle/momentum/class_1/efficiency": 0.8923961520195007, "metrics/particle/momentum/class_1/event/energy_bias": -0.12877404689788818, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.722570419311523, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.69761848449707, "metrics/particle/momentum/class_1/event/px_bias": 2.591763973236084, "metrics/particle/momentum/class_1/event/py_bias": 4.627038955688477, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.623504638671875, "metrics/particle/momentum/class_1/event/pz_bias": -2.6145827770233154, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.416879653930664, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02931741252541542, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18874667584896088, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19143971800804138, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.002249953104183078, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23238961398601532, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.026425590738654137, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.526519536972046, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1938929408788681, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7505199909210205, "metrics/particle/momentum/class_2/accepted/samples": 1438.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6286105513572693, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7099223136901855, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0229518413543701, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6694663166999817, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.29881155490875244, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.030905246734619, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05730050802230835, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4444401264190674, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.20242394506931305, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.701442718505859, "metrics/particle/momentum/class_2/assigned/samples": 2356.0, "metrics/particle/momentum/class_2/efficiency": 0.4815807044506073, "metrics/particle/momentum/class_2/event/energy_bias": -10.774023056030273, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.65349769592285, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.866243362426758, "metrics/particle/momentum/class_2/event/px_bias": 1.1792172193527222, "metrics/particle/momentum/class_2/event/py_bias": -0.9014332294464111, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.23462963104248, "metrics/particle/momentum/class_2/event/pz_bias": -1.4369516372680664, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.537931442260742, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02091098576784134, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16089199483394623, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16001512110233307, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0065561821684241295, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19985131919384003, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016211604233831167, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2828349769115448, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0021270313300192356, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.04580557346344, "metrics/particle/momentum/class_3/accepted/samples": 11720.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.28660985827445984, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.9542598724365234, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.3126766681671143, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.1469783782958984, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18203119933605194, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 60.395076751708984, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008703220635652542, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6041426062583923, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.019316472113132477, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7023415565490723, "metrics/particle/momentum/class_3/assigned/samples": 14937.0, "metrics/particle/momentum/class_3/efficiency": 0.7545226216316223, "metrics/particle/momentum/class_3/event/energy_bias": 2.9058830738067627, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.40446662902832, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.628673553466797, "metrics/particle/momentum/class_3/event/px_bias": 1.7426939010620117, "metrics/particle/momentum/class_3/event/py_bias": 0.7099287509918213, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.886795520782471, "metrics/particle/momentum/class_3/event/pz_bias": -0.2587655484676361, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.095888137817383, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.021417764946818352, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1603095680475235, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15753871202468872, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05331624671816826, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20383623242378235, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004048583097755909, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7479670643806458, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.050199199467897415, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7317266464233398, "metrics/particle/momentum/class_4/accepted/samples": 247.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.421332985162735, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.6010053157806396, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.128929853439331, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.9923617839813232, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.287709504365921, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.095083236694336, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011173184029757977, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8989537358283997, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.12003910541534424, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.2301459312438965, "metrics/particle/momentum/class_4/assigned/samples": 358.0, "metrics/particle/momentum/class_4/efficiency": 0.6586666703224182, "metrics/particle/momentum/class_4/event/energy_bias": -1.3161468505859375, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.7406811714172363, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.074725866317749, "metrics/particle/momentum/class_4/event/px_bias": 0.11452106386423111, "metrics/particle/momentum/class_4/event/py_bias": 0.17462855577468872, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.41984224319458, "metrics/particle/momentum/class_4/event/pz_bias": -0.1402364820241928, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.079293251037598, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01389943715184927, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1319691389799118, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13566511869430542, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004565620794892311, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1781281977891922, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03378378227353096, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5273394584655762, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.1379997581243515, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.5446548461914062, "metrics/particle/momentum/class_5/accepted/samples": 148.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08103299885988235, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.17647510766983032, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.19381457567214966, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.037361081689596176, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07361963391304016, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.37734344601631165, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05521472543478012, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9281635284423828, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.19327756762504578, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.451227188110352, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9024389982223511, "metrics/particle/momentum/class_5/event/energy_bias": -1.1625170707702637, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.0663928985595703, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5519241094589233, "metrics/particle/momentum/class_5/event/px_bias": 0.1417797952890396, "metrics/particle/momentum/class_5/event/py_bias": -0.21536704897880554, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0757594108581543, "metrics/particle/momentum/class_5/event/pz_bias": -0.03204652667045593, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4166314601898193, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3482268750667572, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.35128721594810486, "metrics/set_slot/angular_recall_all_dr0p10": 0.9537253975868225, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8341227173805237, "metrics/set_slot/soft_count_bias_mean": 0.06246122717857361}, "step_train_time": 5.403670310974121, "step_valid_time": 78852.5429391861, "step_total_time": 78857.94660973549, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9918445530545713, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16633623040696954, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2887, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2865, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9437724746649232, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9534346103038309, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9485789387218663, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046565389696169086, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9365805818895063, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9461690885072656, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9413504189255791, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9437724746649232, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9910543712922832, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16178186046594223, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2254, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2140, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2135, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.936542669584245, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9494232475598935, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9429389733421458, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050576752440106475, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9343544857768052, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9472049689440993, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9407358448997577, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.936542669584245, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.994548184797637, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15926727947658592, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1299, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1218, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1210, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9213313161875946, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9376443418013857, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9294162533384205, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06235565819861432, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9152798789712556, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9314857582755967, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9233117130866082, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9213313161875946} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_32000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..3b3e9e32729e7a34f9c65f5b2d457de939b7d894 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 1.515725653656688, "train": {"Classification_binary": 0.17484748363494873, "Cardinality": 0.14056982100009918, "Classification": 0.09540975838899612, "Regression_pt": 0.13119089603424072, "Regression_eta": 0.0018233568407595158, "Regression_sin_phi": 0.0008075980003923178, "Regression_cos_phi": 0.0007528946152888238, "Regression_energy": 0.13024774193763733, "HitGrouping": 0.19117024540901184, "Auxiliary": 0.23928067088127136, "Total": 1.1114978790283203, "Total_unweighted": 0.5350797294522636}, "train_group": {}, "valid": {"Classification_binary": 0.16546749075253805, "Cardinality": 0.1281822919845581, "Classification": 0.09866944948832194, "Regression_pt": 0.13278823097546896, "Regression_eta": 0.0014829433833559353, "Regression_sin_phi": 0.0007452207307020823, "Regression_cos_phi": 0.0007324392596880595, "Regression_energy": 0.13153794407844543, "HitGrouping": 0.19287735223770142, "Auxiliary": 0.23530749479929605, "Total": 1.0744415124257405, "Total_unweighted": 0.5314237186685205, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8217589855194092, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4923902750015259, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.36301693320274353, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4964854419231415, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4915819466114044, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.42083844542503357, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7695204019546509, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.43308091163635254, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34601181745529175, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.43319860100746155, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.43511033058166504, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4948301315307617, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8444324135780334, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.46563854813575745, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.33552417159080505, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4683458209037781, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.455066055059433, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3727840781211853, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8317474722862244, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4712153673171997, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3446674346923828, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.47414204478263855, "metrics/hit_grouping/calorimeter/all/object_purity": 0.464579313993454, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3968176543712616, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.00288498611189425, "metrics/hit_grouping/calorimeter/background_recall": 0.13065171241760254, "metrics/hit_grouping/calorimeter/completeness": 0.5320686101913452, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5425252318382263, "metrics/hit_grouping/calorimeter/energy_purity": 0.5353453159332275, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.39448055624961853, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.854490339756012, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5009498000144958, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.27287209033966064, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.504754364490509, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4916855990886688, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3398953080177307, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8328773975372314, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4870246648788452, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.37367573380470276, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.49008071422576904, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.48197898268699646, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.402003675699234, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7842366695404053, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.39466074109077454, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4618329405784607, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.39566728472709656, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3917812407016754, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5076566338539124, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8187492489814758, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5191801190376282, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4323430359363556, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5250809788703918, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5125371217727661, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.43041858077049255, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.802753746509552, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5602691769599915, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5620175004005432, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5572224259376526, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5284245014190674, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.447761207818985, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8484541773796082, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.41170281171798706, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22938254475593567, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.41230884194374084, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4080771207809448, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.35649028420448303, "metrics/hit_grouping/calorimeter/purity": 0.52418053150177, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8407066464424133, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09081361442804337, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12406015396118164, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.0919441506266594, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.050461821258068085, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04135338217020035, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8226126432418823, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2043672353029251, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11065332591533661, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2048424333333969, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.19376477599143982, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.29664507508277893, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.832076907157898, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.491300106048584, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.36043983697891235, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4943837523460388, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.485419899225235, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4078226089477539, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7374167442321777, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41518181562423706, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.49817126989364624, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.524308979511261, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5675793290138245, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7666380405426025, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2730560600757599, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4626913368701935, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4724593460559845, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4285714328289032, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6617775559425354, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44114261865615845, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.41319048404693604, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.43921637535095215, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6698805689811707, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6927704215049744, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4207479953765869, "metrics/hit_grouping/tracker/all/object_completeness": 0.44296538829803467, "metrics/hit_grouping/tracker/all/object_purity": 0.4678247570991516, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6206144094467163, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.014750106260180473, "metrics/hit_grouping/tracker/background_recall": 0.1339201182126999, "metrics/hit_grouping/tracker/completeness": 0.4408212900161743, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.46987560391426086, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7061415314674377, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4080308675765991, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.49487540125846863, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5176519155502319, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6135367155075073, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6857539415359497, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44309312105178833, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4317047595977783, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.46427932381629944, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6275185942649841, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6722372770309448, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42470693588256836, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3473746180534363, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3688531517982483, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6283137798309326, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7154224514961243, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43848177790641785, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.47378480434417725, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5018829107284546, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.584104597568512, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7186973094940186, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45997536182403564, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.44653013348579407, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.42603588104248047, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.557881772518158, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6586742401123047, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39066120982170105, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.40226006507873535, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4310394823551178, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6793244481086731, "metrics/hit_grouping/tracker/purity": 0.4174126982688904, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8781116008758545, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.030920060351490974, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.057385221123695374, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.056695882230997086, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06674208492040634, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6649418473243713, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27000516653060913, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3197482228279114, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.33954644203186035, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5534331202507019, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.668282687664032, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.49493592977523804, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5135395526885986, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5427387952804565, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7100889682769775, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5061398148536682, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6431412696838379, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7406570911407471, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6332554221153259, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.04301295801997185, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05878063291311264, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.486431121826172, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2698630392551422, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8624999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1782398819923401, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8905290365219116, "metrics/particle/class_1/pid_accuracy": 0.9731442928314209, "metrics/particle/class_2/efficiency": 0.4805753827095032, "metrics/particle/class_2/pid_accuracy": 0.6745227575302124, "metrics/particle/class_3/efficiency": 0.7563998699188232, "metrics/particle/class_3/pid_accuracy": 0.9700866937637329, "metrics/particle/class_4/efficiency": 0.6763526797294617, "metrics/particle/class_4/pid_accuracy": 0.36888888478279114, "metrics/particle/class_5/efficiency": 0.9252873659133911, "metrics/particle/class_5/pid_accuracy": 0.4037266969680786, "metrics/particle/count/bias_mean": -2.1477863788604736, "metrics/particle/count/mae": 4.113932132720947, "metrics/particle/count/prediction_mean": 68.50716400146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07819601148366928, "metrics/particle/event/energy_response_mean": 0.9550399780273438, "metrics/particle/event/met_abs_error": 6.38859748840332, "metrics/particle/event/scalar_pt_relative_abs_error": 0.059351760894060135, "metrics/particle/event/scalar_pt_response_mean": 0.9802977442741394, "metrics/particle/event/vector_pt_closure": 0.05862998962402344, "metrics/particle/matched/delta_eta_abs_mean": 0.01324255671352148, "metrics/particle/matched/delta_phi_abs_mean": 0.013091789558529854, "metrics/particle/matched/delta_r_mean": 0.020836474373936653, "metrics/particle/matched/energy_relative_abs_mean": 0.1442224234342575, "metrics/particle/matched/pid_accuracy": 0.9494318962097168, "metrics/particle/matched/pt_relative_abs_mean": 0.1466618925333023, "metrics/particle/matching/duplicate_fraction": 0.0553470104932785, "metrics/particle/matching/efficiency": 0.7988408207893372, "metrics/particle/matching/f1": 0.8111699223518372, "metrics/particle/matching/purity": 0.8238855004310608, "metrics/particle/matching/target_coverage_dr0p05": 0.770322322845459, "metrics/particle/matching/target_coverage_dr0p10": 0.8503676652908325, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5925584435462952, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8303171396255493, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02059389092028141, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14477111399173737, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1481449455022812, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0358528271317482, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18778246641159058, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007234538905322552, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6975042819976807, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.1141604483127594, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9725635051727295, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8570.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.21789714694023132, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7876314520835876, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8993498086929321, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7556208372116089, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1302516907453537, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.308613777160645, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016452845185995102, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0271480083465576, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.1481202393770218, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.0707294940948486, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10211.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7955811619758606, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.1130436658859253, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.81495189666748, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.046745300292969, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.03425029665231705, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.550003707408905, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.4930338859558105, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.6283247470855713, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.167679786682129, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0240798220038414, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16124388575553894, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16375862061977386, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0345025435090065, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20518720149993896, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022182254120707512, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3492186069488525, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.4042324721813202, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.237769365310669, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1668.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.54400235414505, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9112541079521179, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.379889488220215, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.1988794803619385, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22481626272201538, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.00086784362793, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05923043563961983, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.589176893234253, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.45065629482269287, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.280983924865723, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2313.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6047860980033875, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.229519367218018, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.784775733947754, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2025015354156494, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.009798979386687279, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.06822025030851364, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.651138305664062, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.232856035232544, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.7520694732666, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020860956981778145, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14264409244060516, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14542651176452637, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.026565834879875183, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18533116579055786, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005747727118432522, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2336827665567398, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.017820468172430992, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8441242575645447, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19138.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.194978728890419, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8948004245758057, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6885337829589844, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5326247811317444, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12670162320137024, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.299531936645508, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003568439045920968, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.39465993642807007, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.019492434337735176, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5167361497879028, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22699.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8224676847457886, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.8134992122650146, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.277799606323242, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.656118392944336, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.0883055180311203, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.0125843286514282, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.291790962219238, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -2.037792205810547, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.762609481811523, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0209658145904541, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14432074129581451, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14726050198078156, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.029725825414061546, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18722914159297943, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0037445533089339733, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4323368966579437, "metrics/particle/momentum/all/accepted/pz_bias": -0.06786707043647766, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4814176559448242, "metrics/particle/momentum/all/accepted/samples": 29376.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22454212605953217, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8648130297660828, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8607152700424194, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7066888213157654, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1341736912727356, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.189817428588867, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010958748869597912, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7221236228942871, "metrics/particle/momentum/all/assigned/pz_bias": -0.08509445935487747, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.952824592590332, "metrics/particle/momentum/all/assigned/samples": 35223.0, "metrics/particle/momentum/all/efficiency": 0.7982825636863708, "metrics/particle/momentum/all/event/energy_bias": -10.156061172485352, "metrics/particle/momentum/all/event/p3_error_mean": 20.038923263549805, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.442859649658203, "metrics/particle/momentum/all/event/px_bias": 0.1127568930387497, "metrics/particle/momentum/all/event/py_bias": -1.6308082342147827, "metrics/particle/momentum/all/event/pz_abs_mean": 15.276183128356934, "metrics/particle/momentum/all/event/pz_bias": -6.898972511291504, "metrics/particle/momentum/all/event/pz_residual_rms": 25.763858795166016, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021146360784769058, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.12680377066135406, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13244445621967316, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.043168485164642334, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17014041543006897, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0033001205883920193, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4355045557022095, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.07804245501756668, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3613933324813843, "metrics/particle/momentum/class_1/accepted/samples": 15757.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11475424468517303, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.31050819158554077, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3645874559879303, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.25057265162467957, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07060245424509048, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.8925414085388184, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006444559432566166, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.593338668346405, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.08479061722755432, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.661918878555298, "metrics/particle/momentum/class_1/assigned/samples": 17379.0, "metrics/particle/momentum/class_1/efficiency": 0.8881686329841614, "metrics/particle/momentum/class_1/event/energy_bias": 5.828447341918945, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.025802612304688, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.777751922607422, "metrics/particle/momentum/class_1/event/px_bias": 0.8368759155273438, "metrics/particle/momentum/class_1/event/py_bias": -0.4401853382587433, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.396578788757324, "metrics/particle/momentum/class_1/event/pz_bias": -3.3259055614471436, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.988107681274414, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029934728518128395, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18351081013679504, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18816933035850525, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.009355936199426651, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22922073304653168, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0263522882014513, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5206267833709717, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.23653846979141235, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.586426258087158, "metrics/particle/momentum/class_2/accepted/samples": 1442.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6403802633285522, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7530004978179932, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8682307600975037, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4871237576007843, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3045879304409027, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.6570258140563965, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05480033904314041, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4345264434814453, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.40193450450897217, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.516382694244385, "metrics/particle/momentum/class_2/assigned/samples": 2354.0, "metrics/particle/momentum/class_2/efficiency": 0.4829202890396118, "metrics/particle/momentum/class_2/event/energy_bias": -14.33614444732666, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.10343360900879, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.887598037719727, "metrics/particle/momentum/class_2/event/px_bias": -0.26085183024406433, "metrics/particle/momentum/class_2/event/py_bias": -0.13689731061458588, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.304081916809082, "metrics/particle/momentum/class_2/event/pz_bias": -3.0481619834899902, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.507122039794922, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01972947269678116, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16249126195907593, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1616540402173996, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01611766777932644, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2021917998790741, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013580037048086524, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.27668553590774536, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.03144403174519539, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1156009435653687, "metrics/particle/momentum/class_3/accepted/samples": 11782.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2841931879520416, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4586488008499146, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3739608526229858, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2085829973220825, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.177673801779747, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.80630874633789, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00909090880304575, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5864611864089966, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.036731235682964325, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.683655023574829, "metrics/particle/momentum/class_3/assigned/samples": 14960.0, "metrics/particle/momentum/class_3/efficiency": 0.7585141062736511, "metrics/particle/momentum/class_3/event/energy_bias": 1.0000814199447632, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.55350112915039, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.912112236022949, "metrics/particle/momentum/class_3/event/px_bias": -0.43931519985198975, "metrics/particle/momentum/class_3/event/py_bias": -0.5191535949707031, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.447010517120361, "metrics/particle/momentum/class_3/event/pz_bias": -0.2910481095314026, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.333558082580566, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.019664712250232697, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16906346380710602, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17109833657741547, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06439410150051117, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21639981865882874, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004098360426723957, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7428410053253174, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.19782818853855133, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6078810691833496, "metrics/particle/momentum/class_4/accepted/samples": 244.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.38719385862350464, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.9272477626800537, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.6789886951446533, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.550516366958618, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3060109317302704, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.853719711303711, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005464480724185705, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8545320630073547, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.010861866176128387, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.8660329580307007, "metrics/particle/momentum/class_4/assigned/samples": 366.0, "metrics/particle/momentum/class_4/efficiency": 0.6506666541099548, "metrics/particle/momentum/class_4/event/energy_bias": -1.3932636976242065, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.5836398601531982, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9484047889709473, "metrics/particle/momentum/class_4/event/px_bias": 0.017615750432014465, "metrics/particle/momentum/class_4/event/py_bias": -0.21366596221923828, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3476253747940063, "metrics/particle/momentum/class_4/event/pz_bias": -0.3091110587120056, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.044600486755371, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.015045341104269028, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1402163803577423, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14106681942939758, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005965664051473141, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17792735993862152, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.01986755058169365, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3521716594696045, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.027257515117526054, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.0694940090179443, "metrics/particle/momentum/class_5/accepted/samples": 151.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08557851612567902, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.20527072250843048, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2196919023990631, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.06354747712612152, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.0731707289814949, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.5310366749763489, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04268292710185051, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.8697185516357422, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.14681632816791534, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.500257968902588, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9207317233085632, "metrics/particle/momentum/class_5/event/energy_bias": -1.255183219909668, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.1190991401672363, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5946766138076782, "metrics/particle/momentum/class_5/event/px_bias": -0.04156777262687683, "metrics/particle/momentum/class_5/event/py_bias": -0.3209061324596405, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0881696939468384, "metrics/particle/momentum/class_5/event/pz_bias": 0.0752534419298172, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.7688138484954834, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.34592875838279724, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.34850528836250305, "metrics/set_slot/angular_recall_all_dr0p10": 0.9547573924064636, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8381586074829102, "metrics/set_slot/soft_count_bias_mean": 0.19501076638698578}, "step_train_time": 5.992389678955078, "step_valid_time": 81661.6691725254, "step_total_time": 81667.6615626812, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9993454969808387, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14495520217325852, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3034, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2899, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2879, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.947695325269696, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9555042847725774, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9515837846709339, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044495715227422544, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.941157240928408, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9489123269611075, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9450188741178401, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.947695325269696, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9885755143744128, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14154710353257127, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2264, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2172, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2164, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9505470459518599, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9593639575971732, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9549351505825456, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04063604240282685, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9470459518599562, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9558303886925795, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9514178940426468, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9505470459518599, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9879418352350644, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14160673927597234, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1306, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1234, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1229, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9334341906202723, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9448698315467075, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.939117199391172, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.055130168453292494, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9296520423600605, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.94104134762634, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9353120243531202, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9334341906202723} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_33000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..e5def5385a0033661962bd7857bc2c55dba05040 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 1.5630920803334596, "train": {"Classification_binary": 0.18436813354492188, "Cardinality": 0.10168150067329407, "Classification": 0.1098749041557312, "Regression_pt": 0.12054790556430817, "Regression_eta": 0.0014442962128669024, "Regression_sin_phi": 0.0007374826818704605, "Regression_cos_phi": 0.0007272235816344619, "Regression_energy": 0.11955959349870682, "HitGrouping": 0.18859562277793884, "Auxiliary": 0.23098021745681763, "Total": 1.0508451461791992, "Total_unweighted": 0.5372595392400399}, "train_group": {}, "valid": {"Classification_binary": 0.16307092706362405, "Cardinality": 0.12948453426361084, "Classification": 0.09579503536224365, "Regression_pt": 0.13205130894978842, "Regression_eta": 0.0014368739599982898, "Regression_sin_phi": 0.0007532036397606134, "Regression_cos_phi": 0.0007264750699202219, "Regression_energy": 0.13040276368459067, "HitGrouping": 0.192926287651062, "Auxiliary": 0.2325141429901123, "Total": 1.066846529642741, "Total_unweighted": 0.5242365877299259, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8201583623886108, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4966527223587036, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.35289108753204346, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.49981728196144104, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4986884295940399, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.41941893100738525, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7707276344299316, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.42355838418006897, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3338257074356079, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.42679068446159363, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4309765100479126, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.488552451133728, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8361411094665527, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.46664586663246155, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3290904760360718, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.46929746866226196, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.45815029740333557, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.38153204321861267, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8262205123901367, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4723913371562958, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.33666858077049255, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.47524353861808777, "metrics/hit_grouping/calorimeter/all/object_purity": 0.46832171082496643, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.40131881833076477, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0034176104236394167, "metrics/hit_grouping/calorimeter/background_recall": 0.14187055826187134, "metrics/hit_grouping/calorimeter/completeness": 0.5348655581474304, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5427619814872742, "metrics/hit_grouping/calorimeter/energy_purity": 0.535700798034668, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3850858509540558, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8476856350898743, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.499836266040802, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.26098087430000305, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5036998987197876, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.49568691849708557, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.34023669362068176, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8287765383720398, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4875379204750061, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.36803317070007324, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4897468686103821, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.48450011014938354, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.40856748819351196, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7798784971237183, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4011726677417755, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45939674973487854, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4026103615760803, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3962261974811554, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5185614824295044, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8169260025024414, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5213160514831543, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.41971373558044434, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.527262806892395, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5198295712471008, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.43246331810951233, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7894545793533325, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.555519700050354, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5676788687705994, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5544595718383789, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5374274253845215, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4868759512901306, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8400769233703613, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.41260606050491333, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22447727620601654, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.412769079208374, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.40757328271865845, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3593721389770508, "metrics/hit_grouping/calorimeter/purity": 0.5272176265716553, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8126880526542664, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07363435626029968, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10338345915079117, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07440923154354095, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05112958699464798, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.033834587782621384, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.784551739692688, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19397471845149994, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09947028011083603, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19426648318767548, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.19069461524486542, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2731018364429474, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8286094665527344, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4933772385120392, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3528132438659668, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4963964521884918, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4895669221878052, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4139789938926697, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7380457520484924, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4124293923377991, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.49404647946357727, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5162826776504517, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5732290148735046, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7666565775871277, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.283905953168869, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.45968979597091675, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.45860016345977783, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4309825301170349, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6672069430351257, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44107064604759216, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.42343345284461975, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.442362904548645, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6663548946380615, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6963248252868652, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4206589460372925, "metrics/hit_grouping/tracker/all/object_completeness": 0.4478141665458679, "metrics/hit_grouping/tracker/all/object_purity": 0.4662812352180481, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.620347261428833, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.014591274783015251, "metrics/hit_grouping/tracker/background_recall": 0.12259235978126526, "metrics/hit_grouping/tracker/completeness": 0.4463316798210144, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4742911458015442, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7096602320671082, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4112747609615326, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.49690210819244385, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5124586224555969, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6097667813301086, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6908282041549683, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4300236105918884, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4373486042022705, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4639464318752289, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6304229497909546, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6743549108505249, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4306582510471344, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.35723844170570374, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3736145794391632, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6321009993553162, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7188125252723694, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4400101900100708, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.48070478439331055, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4967782497406006, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.587585985660553, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7242392301559448, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4716748893260956, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.45182809233665466, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.42869266867637634, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5554187297821045, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.662085771560669, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3863561153411865, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4043356776237488, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4333726465702057, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6745777726173401, "metrics/hit_grouping/tracker/purity": 0.4225623905658722, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8831136226654053, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02677224762737751, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.05235935002565384, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0514487698674202, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06485671550035477, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6643407940864563, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2612287104129791, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3180890679359436, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3340790867805481, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5554981827735901, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6720726490020752, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.49639078974723816, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5205590724945068, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5421701669692993, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7098091840744019, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5313618779182434, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6375957131385803, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7316641807556152, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.632377564907074, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.01779083162546158, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05790277197957039, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.294641494750977, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3105836808681488, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8749999403953552, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.21896052360534668, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8980242013931274, "metrics/particle/class_1/pid_accuracy": 0.9762991666793823, "metrics/particle/class_2/efficiency": 0.4972250461578369, "metrics/particle/class_2/pid_accuracy": 0.7086560130119324, "metrics/particle/class_3/efficiency": 0.7524648308753967, "metrics/particle/class_3/pid_accuracy": 0.9684485793113708, "metrics/particle/class_4/efficiency": 0.7104208469390869, "metrics/particle/class_4/pid_accuracy": 0.3836389183998108, "metrics/particle/class_5/efficiency": 0.931034505367279, "metrics/particle/class_5/pid_accuracy": 0.42592594027519226, "metrics/particle/count/bias_mean": -3.2096354961395264, "metrics/particle/count/mae": 4.420572757720947, "metrics/particle/count/prediction_mean": 67.4453125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07760000973939896, "metrics/particle/event/energy_response_mean": 0.9715531468391418, "metrics/particle/event/met_abs_error": 7.510368347167969, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05959214270114899, "metrics/particle/event/scalar_pt_response_mean": 0.9941862225532532, "metrics/particle/event/vector_pt_closure": 0.06265035271644592, "metrics/particle/matched/delta_eta_abs_mean": 0.013591343536973, "metrics/particle/matched/delta_phi_abs_mean": 0.012612438760697842, "metrics/particle/matched/delta_r_mean": 0.020728420466184616, "metrics/particle/matched/energy_relative_abs_mean": 0.1404145359992981, "metrics/particle/matched/pid_accuracy": 0.9518095850944519, "metrics/particle/matched/pt_relative_abs_mean": 0.14259827136993408, "metrics/particle/matching/duplicate_fraction": 0.05333217605948448, "metrics/particle/matching/efficiency": 0.8024989366531372, "metrics/particle/matching/f1": 0.8211500644683838, "metrics/particle/matching/purity": 0.8406888246536255, "metrics/particle/matching/target_coverage_dr0p05": 0.7720915079116821, "metrics/particle/matching/target_coverage_dr0p10": 0.8479350805282593, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.605781078338623, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8320586681365967, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02026066929101944, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13936564326286316, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1423833966255188, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.024901043623685837, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18209807574748993, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007683352567255497, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6972969770431519, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.08116007596254349, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9380197525024414, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8590.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1973755657672882, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7634437084197998, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8238276243209839, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6840493679046631, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1119752749800682, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.5126261711120605, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015255758538842201, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9918721318244934, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12765678763389587, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9226067066192627, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10029.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7974377870559692, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.10791337490081787, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.504587173461914, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.866342306137085, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.808745265007019, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.7329007387161255, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.198137283325195, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.023273468017578, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.498442649841309, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02386624552309513, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15819843113422394, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16201655566692352, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04183036461472511, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20432943105697632, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03113553114235401, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4620325565338135, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.010943686589598656, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.009798526763916, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1638.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4866771697998047, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7535403966903687, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.943934440612793, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.794349193572998, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.19937275350093842, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 159.2436981201172, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06810035556554794, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7560489177703857, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.1538659632205963, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.90359878540039, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2232.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5939086079597473, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.3950324058532715, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.928061485290527, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.285267949104309, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.054453153163194656, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.1585756540298462, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.77639102935791, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.2096054553985596, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.411714553833008, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02063784748315811, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1385980248451233, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.140329971909523, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.005394499283283949, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18054750561714172, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007815349381417036, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2280738800764084, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.013922318816184998, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7238550186157227, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19193.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18575048446655273, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.177040934562683, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8870324492454529, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7247834205627441, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11707469075918198, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.01880931854248, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0041764783672988415, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.40483248233795166, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.019510990008711815, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6388510465621948, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22507.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8248313069343567, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.7924511432647705, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.50722885131836, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.11419677734375, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.917997360229492, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.1482683420181274, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.006250858306885, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.42130663990974426, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.515373706817627, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02070746012032032, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1399133801460266, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14213690161705017, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.013118340633809566, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18240386247634888, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004486591089516878, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4337722361087799, "metrics/particle/momentum/all/accepted/pz_bias": 0.03338778391480446, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5276075601577759, "metrics/particle/momentum/all/accepted/samples": 29421.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20842237770557404, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0947462320327759, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1934385299682617, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0384840965270996, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12088702619075775, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 41.1440315246582, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011476069688796997, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7251078486442566, "metrics/particle/momentum/all/assigned/pz_bias": 0.05933135375380516, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0482285022735596, "metrics/particle/momentum/all/assigned/samples": 34768.0, "metrics/particle/momentum/all/efficiency": 0.7995054125785828, "metrics/particle/momentum/all/event/energy_bias": -5.079570293426514, "metrics/particle/momentum/all/event/p3_error_mean": 20.502811431884766, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.209931373596191, "metrics/particle/momentum/all/event/px_bias": 3.6722896099090576, "metrics/particle/momentum/all/event/py_bias": -2.0397446155548096, "metrics/particle/momentum/all/event/pz_abs_mean": 15.03569507598877, "metrics/particle/momentum/all/event/pz_bias": 3.6541857719421387, "metrics/particle/momentum/all/event/pz_residual_rms": 26.22001838684082, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021083256229758263, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11922542750835419, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12435350567102432, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.020667510107159615, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1616012006998062, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0036489462945610285, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4298968017101288, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06127550080418587, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3697528839111328, "metrics/particle/momentum/class_1/accepted/samples": 15895.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09954677522182465, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.32663387060165405, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.40269941091537476, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.28300049901008606, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.060062311589717865, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.612053394317627, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006981306243687868, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5811630487442017, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.08516791462898254, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.678654909133911, "metrics/particle/momentum/class_1/assigned/samples": 17332.0, "metrics/particle/momentum/class_1/efficiency": 0.8959472179412842, "metrics/particle/momentum/class_1/event/energy_bias": 4.767350196838379, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.50168228149414, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.292858123779297, "metrics/particle/momentum/class_1/event/px_bias": 3.34747052192688, "metrics/particle/momentum/class_1/event/py_bias": -0.2508801221847534, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.527610778808594, "metrics/particle/momentum/class_1/event/pz_bias": 2.9636130332946777, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.179580688476562, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02864029072225094, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18150970339775085, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18100397288799286, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0003471624222584069, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22211357951164246, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029431896284222603, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5232874155044556, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.10472332686185837, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8852765560150146, "metrics/particle/momentum/class_2/accepted/samples": 1461.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.552786648273468, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7285208702087402, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8890883326530457, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5744885802268982, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.26625385880470276, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.949252605438232, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05616983771324158, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3822920322418213, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.061592113226652145, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.654415130615234, "metrics/particle/momentum/class_2/assigned/samples": 2261.0, "metrics/particle/momentum/class_2/efficiency": 0.48928332328796387, "metrics/particle/momentum/class_2/event/energy_bias": -6.535741806030273, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.533432006835938, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.623775482177734, "metrics/particle/momentum/class_2/event/px_bias": -1.1277508735656738, "metrics/particle/momentum/class_2/event/py_bias": -0.34797775745391846, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.995904922485352, "metrics/particle/momentum/class_2/event/pz_bias": 1.1480741500854492, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.940431594848633, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01926307938992977, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16210858523845673, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1608341634273529, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0034443724434822798, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20160353183746338, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0019757752306759357, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2797733247280121, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.006371207069605589, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0912013053894043, "metrics/particle/momentum/class_3/accepted/samples": 11641.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2831278443336487, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0266189575195312, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.166773796081543, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9932423830032349, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16899774968624115, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 63.105464935302734, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00961997639387846, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6278846859931946, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.041741471737623215, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.894310712814331, "metrics/particle/momentum/class_3/assigned/samples": 14657.0, "metrics/particle/momentum/class_3/efficiency": 0.7494366765022278, "metrics/particle/momentum/class_3/event/energy_bias": -0.7413941621780396, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.117206573486328, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.382804870605469, "metrics/particle/momentum/class_3/event/px_bias": 0.8685417771339417, "metrics/particle/momentum/class_3/event/py_bias": -1.2663624286651611, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.672300338745117, "metrics/particle/momentum/class_3/event/pz_bias": -0.30425935983657837, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.664580345153809, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02121097408235073, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16876022517681122, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16648338735103607, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06780941039323807, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21599090099334717, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0036900369450449944, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7118446230888367, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.18794646859169006, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8997797966003418, "metrics/particle/momentum/class_4/accepted/samples": 271.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.329548716545105, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.876213550567627, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.019914388656616, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9320214986801147, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.20621469616889954, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.508088111877441, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7479955554008484, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.24329020082950592, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9123514890670776, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.7226666808128357, "metrics/particle/momentum/class_4/event/energy_bias": -1.2903990745544434, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.61979079246521, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9760801792144775, "metrics/particle/momentum/class_4/event/px_bias": 0.3480806350708008, "metrics/particle/momentum/class_4/event/py_bias": -0.008558139204978943, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3705588579177856, "metrics/particle/momentum/class_4/event/pz_bias": -0.20790134370326996, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.0725812911987305, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014920017682015896, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15214398503303528, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15278738737106323, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.003401641035452485, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19665662944316864, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04575163498520851, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6570464372634888, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.2367839366197586, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.5610768795013428, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02907748892903328, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.191486656665802, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18413043022155762, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.019792547449469566, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.060975611209869385, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.26450642943382263, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04878048598766327, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7303258180618286, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.17093051970005035, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.766753911972046, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9329268336296082, "metrics/particle/momentum/class_5/event/energy_bias": -1.2793853282928467, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.052842140197754, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5685030221939087, "metrics/particle/momentum/class_5/event/px_bias": 0.23594726622104645, "metrics/particle/momentum/class_5/event/py_bias": -0.16596627235412598, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0133416652679443, "metrics/particle/momentum/class_5/event/pz_bias": 0.05465923249721527, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3494300842285156, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.34335899353027344, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.34700050950050354, "metrics/set_slot/angular_recall_all_dr0p10": 0.9556419849395752, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8349796533584595, "metrics/set_slot/soft_count_bias_mean": -0.9339526295661926}, "step_train_time": 5.0669097900390625, "step_valid_time": 84277.74128985405, "step_total_time": 84282.80820012093, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0146992657752487, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1493710708393956, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3048, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2906, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2885, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9499836547891468, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9534120734908137, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9516947764859996, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046587926509186355, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9431186662307943, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.946522309711286, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9448174226297691, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9499836547891468, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994255496519215, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14798477763355833, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2262, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2155, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2149, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9431072210065645, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526967285587975, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9478777215746647, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04730327144120248, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9404814004376367, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9500442086648984, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9452386188695844, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9431072210065645, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0060206792045778, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1507912593917181, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1306, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1234, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1227, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9334341906202723, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9448698315467075, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.939117199391172, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.055130168453292494, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9281391830559758, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9395099540581929, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9337899543378996, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9334341906202723} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_34000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..c3ab816261acb4c85514560fa012497712f8f204 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 1.610458507010231, "train": {"Classification_binary": 0.17072342336177826, "Cardinality": 0.13091912865638733, "Classification": 0.09851213544607162, "Regression_pt": 0.11850880086421967, "Regression_eta": 0.001392637612298131, "Regression_sin_phi": 0.0006919741281308234, "Regression_cos_phi": 0.0007799779414199293, "Regression_energy": 0.1170305609703064, "HitGrouping": 0.19136321544647217, "Auxiliary": 0.23685047030448914, "Total": 1.0653164386749268, "Total_unweighted": 0.5076395103242248}, "train_group": {}, "valid": {"Classification_binary": 0.16327988107999167, "Cardinality": 0.12873036662737528, "Classification": 0.09593404332796733, "Regression_pt": 0.1294869581858317, "Regression_eta": 0.0014669603357712428, "Regression_sin_phi": 0.0007724528356144825, "Regression_cos_phi": 0.0007252118084579706, "Regression_energy": 0.1285953919092814, "HitGrouping": 0.19627519448598227, "Auxiliary": 0.23409454027811685, "Total": 1.0714821815490723, "Total_unweighted": 0.5202608994829158, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8161678910255432, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4891906678676605, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.36377400159835815, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.49018609523773193, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.48606008291244507, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.42698967456817627, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7580958604812622, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4307933449745178, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3430576026439667, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4309779703617096, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4348644018173218, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5007385611534119, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8378591537475586, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4613034427165985, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.33413562178611755, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.46330466866493225, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4499816596508026, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3760703504085541, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8250951170921326, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4673832654953003, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3438073396682739, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4689387083053589, "metrics/hit_grouping/calorimeter/all/object_purity": 0.45973801612854004, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.40117546916007996, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003619554452598095, "metrics/hit_grouping/calorimeter/background_recall": 0.14213834702968597, "metrics/hit_grouping/calorimeter/completeness": 0.5225003361701965, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5316776037216187, "metrics/hit_grouping/calorimeter/energy_purity": 0.5248236656188965, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3965553045272827, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8483991026878357, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4958677291870117, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2683204412460327, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.49899494647979736, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.486601322889328, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.33255577087402344, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8248597979545593, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.483132541179657, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3699907958507538, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.48464348912239075, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.47544345259666443, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4188162088394165, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7778157591819763, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.39343777298927307, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4713456928730011, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3918333649635315, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.38914233446121216, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5233178734779358, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8127941489219666, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5137196779251099, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4316815137863159, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5174739360809326, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5094708204269409, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.44160452485084534, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7838285565376282, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5433265566825867, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5712815523147583, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5380612015724182, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5326261520385742, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4868759512901306, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8425531387329102, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4110928773880005, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22711385786533356, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4112190902233124, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.40034887194633484, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3497455418109894, "metrics/hit_grouping/calorimeter/purity": 0.5151336193084717, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8150510191917419, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07830486446619034, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12781955301761627, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08059882372617722, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.047704000025987625, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02255639061331749, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7807989120483398, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1871759593486786, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.0959387868642807, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1865329146385193, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.17063137888908386, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2636845111846924, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8275639414787292, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.48830464482307434, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3602254390716553, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4899624288082123, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.48149654269218445, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4144996702671051, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7433820962905884, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40735912322998047, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.49297961592674255, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.506781280040741, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5661306381225586, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7763436436653137, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2778782546520233, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4640347957611084, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4599907398223877, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.41470766067504883, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6724944710731506, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43099725246429443, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.41872793436050415, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4370786249637604, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6588717699050903, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.701952338218689, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4124220907688141, "metrics/hit_grouping/tracker/all/object_completeness": 0.44489553570747375, "metrics/hit_grouping/tracker/all/object_purity": 0.4601939022541046, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6123330593109131, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013603875413537025, "metrics/hit_grouping/tracker/background_recall": 0.11884275078773499, "metrics/hit_grouping/tracker/completeness": 0.4457840621471405, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.47926783561706543, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7145505547523499, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40426090359687805, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4995216429233551, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5147969126701355, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6031913161277771, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6966486573219299, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.423307329416275, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4293922483921051, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.449786901473999, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6237066388130188, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6813072562217712, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41839495301246643, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.34793272614479065, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3582155108451843, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6198377013206482, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7235399484634399, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4293113648891449, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4722296893596649, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4850257635116577, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5801137685775757, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7248403429985046, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.47413793206214905, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.444129079580307, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.42092981934547424, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5628078579902649, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6697846055030823, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37940171360969543, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.40949752926826477, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.43495041131973267, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6630975008010864, "metrics/hit_grouping/tracker/purity": 0.4216214418411255, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8831300139427185, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.021870287135243416, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.04864199832081795, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.04932229220867157, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0682503804564476, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6764091849327087, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2586473822593689, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.30685320496559143, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.31056109070777893, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5410428643226624, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.677834689617157, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4870460629463196, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5186603665351868, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5373843312263489, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7008001804351807, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5423834323883057, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6483765840530396, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7446497678756714, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6319047212600708, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.006769281346350908, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05742999166250229, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.440837860107422, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30703169107437134, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.2154085487127304, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8985009789466858, "metrics/particle/class_1/pid_accuracy": 0.9712812304496765, "metrics/particle/class_2/efficiency": 0.47932949662208557, "metrics/particle/class_2/pid_accuracy": 0.6791115403175354, "metrics/particle/class_3/efficiency": 0.7637234330177307, "metrics/particle/class_3/pid_accuracy": 0.9739516377449036, "metrics/particle/class_4/efficiency": 0.6863727569580078, "metrics/particle/class_4/pid_accuracy": 0.41605839133262634, "metrics/particle/class_5/efficiency": 0.9272030591964722, "metrics/particle/class_5/pid_accuracy": 0.42768594622612, "metrics/particle/count/bias_mean": -2.267578125, "metrics/particle/count/mae": 4.081380367279053, "metrics/particle/count/prediction_mean": 68.38736724853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07887350767850876, "metrics/particle/event/energy_response_mean": 0.9827634692192078, "metrics/particle/event/met_abs_error": 6.440959453582764, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06192171573638916, "metrics/particle/event/scalar_pt_response_mean": 0.9980459213256836, "metrics/particle/event/vector_pt_closure": 0.05876312777400017, "metrics/particle/matched/delta_eta_abs_mean": 0.013807435519993305, "metrics/particle/matched/delta_phi_abs_mean": 0.012323256582021713, "metrics/particle/matched/delta_r_mean": 0.020717021077871323, "metrics/particle/matched/energy_relative_abs_mean": 0.14165747165679932, "metrics/particle/matched/pid_accuracy": 0.950850784778595, "metrics/particle/matched/pt_relative_abs_mean": 0.1402607262134552, "metrics/particle/matching/duplicate_fraction": 0.05363517627120018, "metrics/particle/matching/efficiency": 0.8057792782783508, "metrics/particle/matching/f1": 0.818920373916626, "metrics/particle/matching/purity": 0.8324971795082092, "metrics/particle/matching/target_coverage_dr0p05": 0.7771040797233582, "metrics/particle/matching/target_coverage_dr0p10": 0.853574275970459, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6153824925422668, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8365184664726257, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020356502383947372, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1414748579263687, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14038097858428955, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015040055848658085, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17973420023918152, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007720672991126776, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6935188174247742, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.01571950502693653, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9615709781646729, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8678.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.20758138597011566, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.79460608959198, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.83404541015625, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6847512125968933, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11845973134040833, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.168091773986816, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01606897823512554, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.011062502861023, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0025870422832667828, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.086609125137329, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10206.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8056071400642395, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.2689911127090454, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.396954536437988, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.818363666534424, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.022782765328884125, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.27392271161079407, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.150099754333496, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.22048746049404144, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.587471961975098, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024962643161416054, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15757519006729126, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15320412814617157, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017181111499667168, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19353270530700684, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023837901651859283, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3542118072509766, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.18852545320987701, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4028091430664062, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1678.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5445593595504761, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.115199089050293, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.5832436084747314, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.4064254760742188, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21434707939624786, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 25.92856216430664, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06271477788686752, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6128547191619873, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1546200066804886, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.783079147338867, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2328.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6084119081497192, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.646705627441406, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.126853942871094, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2607622146606445, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.17128942906856537, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.033994160592556, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.968348503112793, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.15087905526161194, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.54081153869629, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020551951602101326, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13967862725257874, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13839928805828094, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.009087632410228252, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17745432257652283, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006226649857126176, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2220795750617981, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01659809798002243, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6962525248527527, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19272.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18913084268569946, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0023738145828247, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7970409989356995, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.635067880153656, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11911075562238693, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.842426776885986, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004021921660751104, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.39808183908462524, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.023479139432311058, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5686801671981812, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22626.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8282263875007629, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.3119499683380127, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.628744125366211, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.164815902709961, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.8973575830459595, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.21209481358528137, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.200839996337891, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.06170430779457092, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.588935852050781, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.020744504407048225, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14121834933757782, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1398182064294815, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011289468966424465, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17907088994979858, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004016471095383167, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4242824614048004, "metrics/particle/momentum/all/accepted/pz_bias": -0.004484981298446655, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4484882354736328, "metrics/particle/momentum/all/accepted/samples": 29628.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2180200219154358, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9495348334312439, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9260497093200684, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7667739987373352, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12522752583026886, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.463440895080566, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011405006051063538, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7226572632789612, "metrics/particle/momentum/all/assigned/pz_bias": 0.004120600875467062, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.891343355178833, "metrics/particle/momentum/all/assigned/samples": 35160.0, "metrics/particle/momentum/all/efficiency": 0.8051306009292603, "metrics/particle/momentum/all/event/energy_bias": -2.065764904022217, "metrics/particle/momentum/all/event/p3_error_mean": 19.896556854248047, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.945737838745117, "metrics/particle/momentum/all/event/px_bias": 0.7032853960990906, "metrics/particle/momentum/all/event/py_bias": -0.45202332735061646, "metrics/particle/momentum/all/event/pz_abs_mean": 14.395624160766602, "metrics/particle/momentum/all/event/pz_bias": -0.30966222286224365, "metrics/particle/momentum/all/event/pz_residual_rms": 25.46979522705078, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021113932132720947, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.12175001949071884, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12123629450798035, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009878239594399929, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15717710554599762, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0033930253703147173, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4260549247264862, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.009199630469083786, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3845884799957275, "metrics/particle/momentum/class_1/accepted/samples": 15915.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10941852629184723, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3542417883872986, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.37653231620788574, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2447373867034912, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.061467625200748444, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.9533462524414062, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007827337831258774, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5981001853942871, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.006066434551030397, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.538167715072632, "metrics/particle/momentum/class_1/assigned/samples": 17375.0, "metrics/particle/momentum/class_1/efficiency": 0.8970745801925659, "metrics/particle/momentum/class_1/event/energy_bias": 6.904965400695801, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.91386890411377, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.29612922668457, "metrics/particle/momentum/class_1/event/px_bias": 1.3560408353805542, "metrics/particle/momentum/class_1/event/py_bias": -1.8172354698181152, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.765291213989258, "metrics/particle/momentum/class_1/event/pz_bias": -0.11199721693992615, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.284687042236328, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029772039502859116, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18331436812877655, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18414093554019928, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0017232975224032998, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22445820271968842, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030345801264047623, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5596526861190796, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1020621508359909, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.647040367126465, "metrics/particle/momentum/class_2/accepted/samples": 1417.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6828660368919373, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6651704907417297, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8350411653518677, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4837295413017273, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2962166666984558, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.3797149658203125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05674978345632553, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.504054069519043, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.02350899577140808, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.6569342613220215, "metrics/particle/momentum/class_2/assigned/samples": 2326.0, "metrics/particle/momentum/class_2/efficiency": 0.4745478928089142, "metrics/particle/momentum/class_2/event/energy_bias": -11.584134101867676, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.687644958496094, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.52512264251709, "metrics/particle/momentum/class_2/event/px_bias": -1.5011156797409058, "metrics/particle/momentum/class_2/event/py_bias": 0.2982617914676666, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.245834350585938, "metrics/particle/momentum/class_2/event/pz_bias": -0.80100017786026, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.470375061035156, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01927356608211994, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16157373785972595, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1590198129415512, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013729755766689777, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1987910270690918, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012618827167898417, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.26645562052726746, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.014247155748307705, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9060727953910828, "metrics/particle/momentum/class_3/accepted/samples": 11887.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2698180377483368, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6313824653625488, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5531903505325317, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3905787467956543, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17042791843414307, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.738567352294922, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008169825188815594, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5750730633735657, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.001096520689316094, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6215758323669434, "metrics/particle/momentum/class_3/assigned/samples": 14933.0, "metrics/particle/momentum/class_3/efficiency": 0.765273928642273, "metrics/particle/momentum/class_3/event/energy_bias": 4.767029285430908, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.448647499084473, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.707261562347412, "metrics/particle/momentum/class_3/event/px_bias": 0.5689108371734619, "metrics/particle/momentum/class_3/event/py_bias": 1.2297003269195557, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.723481178283691, "metrics/particle/momentum/class_3/event/pz_bias": 0.7098726034164429, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.921778678894043, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.020700542256236076, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1713024377822876, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1589808613061905, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06320438534021378, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20366601645946503, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6167978048324585, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.006003856658935547, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3646563291549683, "metrics/particle/momentum/class_4/accepted/samples": 261.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3871524930000305, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.558251142501831, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.351130962371826, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.2355997562408447, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24309392273426056, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.416905403137207, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008287292905151844, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8270567059516907, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0992748960852623, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.3276095390319824, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.6959999799728394, "metrics/particle/momentum/class_4/event/energy_bias": -1.2864915132522583, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.5749876499176025, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9227864742279053, "metrics/particle/momentum/class_4/event/px_bias": 0.2435067892074585, "metrics/particle/momentum/class_4/event/py_bias": 0.1091151311993599, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3581252098083496, "metrics/particle/momentum/class_4/event/pz_bias": -0.3014150857925415, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.06308650970459, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012806063517928123, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14372685551643372, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13762237131595612, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 8.19890046841465e-05, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18237611651420593, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04729729890823364, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7000499963760376, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.24494485557079315, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.159296035766602, "metrics/particle/momentum/class_5/accepted/samples": 148.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0411478690803051, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2072267085313797, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18567794561386108, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.045788321644067764, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07926829159259796, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2892213761806488, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04878048598766327, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.8612793684005737, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.2551591694355011, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.25789737701416, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9024389982223511, "metrics/particle/momentum/class_5/event/energy_bias": -0.8671340346336365, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.103757381439209, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.554298758506775, "metrics/particle/momentum/class_5/event/px_bias": 0.035942625254392624, "metrics/particle/momentum/class_5/event/py_bias": -0.27186518907546997, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0963776111602783, "metrics/particle/momentum/class_5/event/pz_bias": 0.1948777288198471, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.726250648498535, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.34235015511512756, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3435923159122467, "metrics/set_slot/angular_recall_all_dr0p10": 0.9560289978981018, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8411440849304199, "metrics/set_slot/soft_count_bias_mean": 0.058188874274492264}, "step_train_time": 5.401463508605957, "step_valid_time": 86910.42202830315, "step_total_time": 86915.82349205017, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004758067359879, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14422349319160022, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3025, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2895, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2880, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9463877084014384, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9570247933884297, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9516765285996055, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04297520661157025, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9414841451454724, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9520661157024793, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9467455621301775, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9463877084014384, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9997262290605952, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1401918456656266, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2253, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2151, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2144, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9413566739606127, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547270306258322, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9479947113265755, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045272969374167776, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9382932166301969, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9516200621393698, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9449096518289996, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9413566739606127, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0070120438739802, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14243727575136755, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1303, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1232, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1225, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9319213313161876, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.945510360706063, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9386666666666666, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05448963929393707, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.926626323751891, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9401381427475057, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9333333333333333, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9319213313161876} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_35000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..c8e3f54407521b76a0af7e1d27c88c09622f1910 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 1.6578249336870026, "train": {"Classification_binary": 0.155486062169075, "Cardinality": 0.14005401730537415, "Classification": 0.0852101743221283, "Regression_pt": 0.11634032428264618, "Regression_eta": 0.001153054996393621, "Regression_sin_phi": 0.0006834602681919932, "Regression_cos_phi": 0.0007264668820425868, "Regression_energy": 0.11509627848863602, "HitGrouping": 0.1831817328929901, "Auxiliary": 0.22236064076423645, "Total": 1.0147299766540527, "Total_unweighted": 0.4746958214091137}, "train_group": {}, "valid": {"Classification_binary": 0.16230088472366333, "Cardinality": 0.1355862021446228, "Classification": 0.09462364514668782, "Regression_pt": 0.12578970193862915, "Regression_eta": 0.0014288965612649918, "Regression_sin_phi": 0.0007268871801594893, "Regression_cos_phi": 0.0007292888282487789, "Regression_energy": 0.12406143546104431, "HitGrouping": 0.1913652221361796, "Auxiliary": 0.2335314154624939, "Total": 1.0626002152760823, "Total_unweighted": 0.5096607398396978, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.822895884513855, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5025196075439453, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.36273303627967834, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5046296119689941, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.49950557947158813, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4154443144798279, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7657704949378967, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.438381552696228, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3530280590057373, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.43855106830596924, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.43335673213005066, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4933530390262604, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8427404761314392, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4706919491291046, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.33450591564178467, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.47253918647766113, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4574262499809265, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.37204352021217346, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8307527303695679, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.47782576084136963, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.34449541568756104, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.47962233424186707, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4683055877685547, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.39461007714271545, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002843152964487672, "metrics/hit_grouping/calorimeter/background_recall": 0.12597249448299408, "metrics/hit_grouping/calorimeter/completeness": 0.5382257103919983, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5493263602256775, "metrics/hit_grouping/calorimeter/energy_purity": 0.5420572757720947, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3900962471961975, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8563112020492554, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5041341185569763, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.27224624156951904, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5074697136878967, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4932365119457245, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.33562812209129333, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8297165632247925, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4943335950374603, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.36803317070007324, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.495469331741333, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4858853220939636, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.40108245611190796, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7796832919120789, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.407553106546402, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.468097448348999, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.406877338886261, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.39976179599761963, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5083526968955994, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8158358335494995, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5274760723114014, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4263290762901306, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5311024188995361, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5212940573692322, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4295766055583954, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7954484820365906, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5683592557907104, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5779722332954407, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.563676118850708, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5430871844291687, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4590838849544525, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.850167453289032, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4164184033870697, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23324544727802277, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.41712138056755066, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4053714871406555, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3512784242630005, "metrics/hit_grouping/calorimeter/purity": 0.5302241444587708, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.843884289264679, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08098745346069336, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.129699245095253, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08050035685300827, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05466369912028313, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03947368264198303, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8267053961753845, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1973397433757782, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1006474420428276, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19801554083824158, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.19025947153568268, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2884049415588379, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8307493925094604, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4988880753517151, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.360684871673584, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5007801651954651, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4895147383213043, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.40592360496520996, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7448695302009583, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.403158038854599, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5026673078536987, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5231945514678955, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5590323209762573, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7726128101348877, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.279083788394928, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.47173213958740234, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4727614223957062, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4261603355407715, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6757118701934814, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4310692250728607, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.43148621916770935, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4562123715877533, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6535472869873047, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7041248083114624, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41126447916030884, "metrics/hit_grouping/tracker/all/object_completeness": 0.45633625984191895, "metrics/hit_grouping/tracker/all/object_purity": 0.4780215322971344, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6077025532722473, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0142524354159832, "metrics/hit_grouping/tracker/background_recall": 0.11572466045618057, "metrics/hit_grouping/tracker/completeness": 0.45433059334754944, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4669262170791626, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.715942919254303, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4008416533470154, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5061355829238892, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5264714956283569, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6014378666877747, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6994951367378235, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4249410033226013, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4467771351337433, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.47305628657341003, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6159012317657471, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6844146847724915, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4191163182258606, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.36339637637138367, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.383293479681015, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6124436259269714, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7260132431983948, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4198862314224243, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4854743778705597, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5126957297325134, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5696697235107422, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7292003631591797, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4747537076473236, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46195927262306213, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4343515634536743, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5474137663841248, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6711739301681519, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.388674259185791, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.41744765639305115, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4407725930213928, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6679545044898987, "metrics/hit_grouping/tracker/purity": 0.42998743057250977, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8811126947402954, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.030165912583470345, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.052900709211826324, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.053464796394109726, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06598793715238571, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6696765422821045, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.273102730512619, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.32987886667251587, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3358323872089386, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5554981827735901, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6815942525863647, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4827933609485626, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5299112200737, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5564359426498413, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6937496662139893, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5507080554962158, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6384356021881104, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7356807589530945, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6218193769454956, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.0015552937984466553, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0473446287214756, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.685592651367188, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29368069767951965, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8999999761581421, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20205754041671753, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.905900776386261, "metrics/particle/class_1/pid_accuracy": 0.972947359085083, "metrics/particle/class_2/efficiency": 0.5035678148269653, "metrics/particle/class_2/pid_accuracy": 0.6911830902099609, "metrics/particle/class_3/efficiency": 0.7778457999229431, "metrics/particle/class_3/pid_accuracy": 0.9729912281036377, "metrics/particle/class_4/efficiency": 0.7014027833938599, "metrics/particle/class_4/pid_accuracy": 0.36714285612106323, "metrics/particle/class_5/efficiency": 0.9348658919334412, "metrics/particle/class_5/pid_accuracy": 0.42827868461608887, "metrics/particle/count/bias_mean": -0.7838541865348816, "metrics/particle/count/mae": 3.7213542461395264, "metrics/particle/count/prediction_mean": 69.87109375, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07991086691617966, "metrics/particle/event/energy_response_mean": 0.9597652554512024, "metrics/particle/event/met_abs_error": 6.334472179412842, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06147551164031029, "metrics/particle/event/scalar_pt_response_mean": 0.9736025929450989, "metrics/particle/event/vector_pt_closure": 0.058412808924913406, "metrics/particle/matched/delta_eta_abs_mean": 0.014476673677563667, "metrics/particle/matched/delta_phi_abs_mean": 0.012366457842290401, "metrics/particle/matched/delta_r_mean": 0.021254651248455048, "metrics/particle/matched/energy_relative_abs_mean": 0.1384781152009964, "metrics/particle/matched/pid_accuracy": 0.9510680437088013, "metrics/particle/matched/pt_relative_abs_mean": 0.13860966265201569, "metrics/particle/matching/duplicate_fraction": 0.05804029107093811, "metrics/particle/matching/efficiency": 0.8174538612365723, "metrics/particle/matching/f1": 0.8220136165618896, "metrics/particle/matching/purity": 0.8266245722770691, "metrics/particle/matching/target_coverage_dr0p05": 0.781683623790741, "metrics/particle/matching/target_coverage_dr0p10": 0.8635349869728088, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6291119456291199, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8483220338821411, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020694758743047714, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13756310939788818, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1386570930480957, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0017233585240319371, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1771681159734726, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007451565004885197, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6859919428825378, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07637592405080795, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9894068241119385, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8723.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2289915829896927, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8994386792182922, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0171691179275513, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8512963056564331, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12780506908893585, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.870674133300781, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016758162528276443, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0215568542480469, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.13243812322616577, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.047865390777588, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10383.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8097846508026123, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.243830442428589, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.396937370300293, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.7629408836364746, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.08576027303934097, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.36799630522727966, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.249473571777344, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.9456777572631836, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.531937599182129, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02435293234884739, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15546280145645142, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15861690044403076, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.006510984152555466, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19807255268096924, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02654867246747017, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4315099716186523, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.36517417430877686, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.026412487030029, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1695.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5601959824562073, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.854266881942749, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.442802429199219, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.245953559875488, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2145593911409378, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 71.78515625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06215410679578781, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.695664882659912, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.5666035413742065, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.891047477722168, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2349.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6145758032798767, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.121570587158203, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.732976913452148, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3545241355895996, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.08978405594825745, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.03654240816831589, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.579817771911621, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.7949864864349365, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.243738174438477, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021385209634900093, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13717682659626007, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13718131184577942, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.005268750246614218, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17693281173706055, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006634345627389848, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.22742433845996857, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004224235657602549, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7564049959182739, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19595.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1882370561361313, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9244707226753235, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7572413682937622, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.583938479423523, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1149631068110466, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.964090824127197, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003449329873546958, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3782508075237274, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.00032948754960671067, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.465808629989624, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22903.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8421075344085693, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.362151563167572, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.828161239624023, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.382308483123779, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.44404882192611694, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.6081088185310364, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.982595443725586, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.8871418833732605, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.282590389251709, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021352140232920647, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13832183182239532, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1388208270072937, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0035730486270040274, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17826145887374878, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004098223987966776, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4287041425704956, "metrics/particle/momentum/all/accepted/pz_bias": -0.04006342217326164, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5618603229522705, "metrics/particle/momentum/all/accepted/samples": 30013.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22463063895702362, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9784677624702454, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.141841173171997, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9691511988639832, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12527009844779968, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 19.98107147216797, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011196857318282127, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7184514999389648, "metrics/particle/momentum/all/assigned/pz_bias": -0.07614994049072266, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.049344539642334, "metrics/particle/momentum/all/assigned/samples": 35635.0, "metrics/particle/momentum/all/efficiency": 0.8155928254127502, "metrics/particle/momentum/all/event/energy_bias": -8.003249168395996, "metrics/particle/momentum/all/event/p3_error_mean": 20.47344970703125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.651355743408203, "metrics/particle/momentum/all/event/px_bias": -0.6195931434631348, "metrics/particle/momentum/all/event/py_bias": -0.9395627379417419, "metrics/particle/momentum/all/event/pz_abs_mean": 15.573326110839844, "metrics/particle/momentum/all/event/pz_bias": -5.627806663513184, "metrics/particle/momentum/all/event/pz_residual_rms": 27.472522735595703, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.022027652710676193, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11646060645580292, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11830475926399231, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.006955577060580254, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15442095696926117, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003486272878944874, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4254729449748993, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.02959381975233555, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4204111099243164, "metrics/particle/momentum/class_1/accepted/samples": 16063.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11563252657651901, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3875836431980133, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.49434417486190796, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.34976354241371155, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.059710875153541565, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.627373695373535, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007485286332666874, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5925560593605042, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02936970815062523, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8283684253692627, "metrics/particle/momentum/class_1/assigned/samples": 17501.0, "metrics/particle/momentum/class_1/efficiency": 0.9054168462753296, "metrics/particle/momentum/class_1/event/energy_bias": 2.8057994842529297, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.506858825683594, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.636794090270996, "metrics/particle/momentum/class_1/event/px_bias": -1.4331011772155762, "metrics/particle/momentum/class_1/event/py_bias": 0.2759461998939514, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.912784576416016, "metrics/particle/momentum/class_1/event/pz_bias": -1.602144479751587, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.698667526245117, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02926778979599476, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18504859507083893, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1863861382007599, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.030276672914624214, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2270735651254654, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03236682340502739, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5584312677383423, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.16791404783725739, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8973231315612793, "metrics/particle/momentum/class_2/accepted/samples": 1483.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6796827912330627, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.710311770439148, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9449001550674438, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5483740568161011, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3076923191547394, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.142672538757324, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05942206084728241, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4873883724212646, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.3540072739124298, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.859408378601074, "metrics/particle/momentum/class_2/assigned/samples": 2457.0, "metrics/particle/momentum/class_2/efficiency": 0.4966510236263275, "metrics/particle/momentum/class_2/event/energy_bias": -11.572235107421875, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.004352569580078, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.867702007293701, "metrics/particle/momentum/class_2/event/px_bias": 0.32803067564964294, "metrics/particle/momentum/class_2/event/py_bias": -0.7779199481010437, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.961250305175781, "metrics/particle/momentum/class_2/event/pz_bias": -2.2077088356018066, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.257217407226562, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019565211609005928, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16126559674739838, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16007520258426666, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0033797668293118477, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1995503455400467, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009951073443517089, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2742588222026825, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.03476567193865776, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1475281715393066, "metrics/particle/momentum/class_3/accepted/samples": 12059.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.27430370450019836, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6400507688522339, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.884257197380066, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.7141563892364502, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16858085989952087, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 29.68100929260254, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0073267328552901745, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5607475638389587, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.07711602747440338, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.603665828704834, "metrics/particle/momentum/class_3/assigned/samples": 15150.0, "metrics/particle/momentum/class_3/efficiency": 0.7763471603393555, "metrics/particle/momentum/class_3/event/energy_bias": 3.235715866088867, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.98433780670166, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.312478065490723, "metrics/particle/momentum/class_3/event/px_bias": 0.4418988525867462, "metrics/particle/momentum/class_3/event/py_bias": -0.4104112386703491, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.635671138763428, "metrics/particle/momentum/class_3/event/pz_bias": -1.8424735069274902, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.148926734924316, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.020685648545622826, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1596468687057495, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15129354596138, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04555812105536461, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19179606437683105, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003921568859368563, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7063811421394348, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.1962461918592453, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7816529273986816, "metrics/particle/momentum/class_4/accepted/samples": 255.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4071587324142456, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.024025917053223, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.132404088973999, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.0086419582366943, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2719780206680298, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.826451301574707, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008241758681833744, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.89909428358078, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.33075642585754395, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.2532477378845215, "metrics/particle/momentum/class_4/assigned/samples": 364.0, "metrics/particle/momentum/class_4/efficiency": 0.6800000071525574, "metrics/particle/momentum/class_4/event/energy_bias": -1.4831492900848389, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.4673080444335938, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.8337596654891968, "metrics/particle/momentum/class_4/event/px_bias": 0.16794365644454956, "metrics/particle/momentum/class_4/event/py_bias": 0.0833171084523201, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3003405332565308, "metrics/particle/momentum/class_4/event/pz_bias": -0.0417257696390152, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.98346471786499, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01565844565629959, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1366482973098755, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1357038915157318, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.019503923133015633, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17435291409492493, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03921568766236305, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.527823805809021, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0572507381439209, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.224997043609619, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04380999878048897, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.17083622515201569, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18208232522010803, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.015461756847798824, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.061349693685770035, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.33995503187179565, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04907975345849991, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.8256869316101074, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2521762549877167, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.8718016147613525, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9329268336296082, "metrics/particle/momentum/class_5/event/energy_bias": -0.9893788695335388, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9155030250549316, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4646785259246826, "metrics/particle/momentum/class_5/event/px_bias": -0.12436522543430328, "metrics/particle/momentum/class_5/event/py_bias": -0.11049487441778183, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.966393768787384, "metrics/particle/momentum/class_5/event/pz_bias": 0.0662456676363945, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.124624490737915, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3431318402290344, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3457303047180176, "metrics/set_slot/angular_recall_all_dr0p10": 0.9573558568954468, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8499622344970703, "metrics/set_slot/soft_count_bias_mean": 1.5721739530563354}, "step_train_time": 5.520889759063721, "step_valid_time": 89407.0801205635, "step_total_time": 89412.60101079941, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9940480927384987, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1467437618876546, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2918, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2901, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9539065053939196, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9598684210526316, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.956878176750287, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04013157894736842, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9483491337038248, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9542763157894737, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9513034923757994, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9539065053939196, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9807752070497883, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14004429141681096, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2257, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2156, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2148, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9435448577680525, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9552503322995126, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9493615147512109, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04474966770048737, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9400437636761488, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9517058041648205, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9458388375165125, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9435448577680525, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9743531216101698, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1409005380867192, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1299, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1224, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1221, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9258698940998488, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9422632794457275, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9339946585272797, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.057736720554272515, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9236006051437217, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9399538106235565, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.93170545593285, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9258698940998488} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_36000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_36000.json new file mode 100644 index 0000000000000000000000000000000000000000..023a05004dc89df30284a75f7831f6590fecce0c --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_36000.json @@ -0,0 +1 @@ +{"epoch": 1.705191360363774, "train": {"Classification_binary": 0.14341413974761963, "Cardinality": 0.12758585810661316, "Classification": 0.1030769944190979, "Regression_pt": 0.14619633555412292, "Regression_eta": 0.0016592313768342137, "Regression_sin_phi": 0.0007485996466130018, "Regression_cos_phi": 0.0008020735112950206, "Regression_energy": 0.1452457308769226, "HitGrouping": 0.20087303221225739, "Auxiliary": 0.2316906899213791, "Total": 1.0831453800201416, "Total_unweighted": 0.5411431051325053}, "train_group": {}, "valid": {"Classification_binary": 0.15783133109410605, "Cardinality": 0.12763447562853494, "Classification": 0.09411378701527913, "Regression_pt": 0.12301522493362427, "Regression_eta": 0.0013263816945254803, "Regression_sin_phi": 0.0007033619719247023, "Regression_cos_phi": 0.0006829408618311087, "Regression_energy": 0.12196041146914165, "HitGrouping": 0.19432085752487183, "Auxiliary": 0.23042172193527222, "Total": 1.0386323928833008, "Total_unweighted": 0.4996334390404324, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8128742575645447, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.48764732480049133, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3721964657306671, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4884566068649292, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.48268869519233704, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.44090092182159424, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7449935674667358, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.41364702582359314, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3541359007358551, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.41155436635017395, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.42146065831184387, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.550960123538971, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8398723006248474, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4622337222099304, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.33376532793045044, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4638011157512665, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.44823381304740906, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3810229003429413, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8243271112442017, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.46616068482398987, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3469896912574768, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4672141969203949, "metrics/hit_grouping/calorimeter/all/object_purity": 0.45659342408180237, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4123566448688507, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003022935474291444, "metrics/hit_grouping/calorimeter/background_recall": 0.1301725059747696, "metrics/hit_grouping/calorimeter/completeness": 0.5256967544555664, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5366699695587158, "metrics/hit_grouping/calorimeter/energy_purity": 0.5296153426170349, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.39781543612480164, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8486716151237488, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.49154725670814514, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.26906007528305054, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.49448254704475403, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.48374876379966736, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3522416949272156, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8238545656204224, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4844677150249481, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3834638297557831, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4851478636264801, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.47376495599746704, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4180101454257965, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.775165319442749, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3959551751613617, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.46914154291152954, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.39354807138442993, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3839251697063446, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5292343497276306, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.810272216796875, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5159308314323425, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.43378639221191406, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5183811783790588, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.50667804479599, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.44834014773368835, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7930297255516052, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5607062578201294, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5759135484695435, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5536223649978638, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5278730392456055, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.47195059061050415, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8423855900764465, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4041532576084137, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2312220185995102, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4047521650791168, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3970370888710022, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3685694932937622, "metrics/hit_grouping/calorimeter/purity": 0.5179747939109802, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8408409357070923, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08740010112524033, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0958646610379219, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08900375664234161, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04768388718366623, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8179351091384888, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1918686330318451, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10359034687280655, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19105906784534454, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18650192022323608, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2872277796268463, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.824390709400177, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4866061508655548, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3637477457523346, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4877476990222931, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.47731152176856995, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4251585006713867, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7464810013771057, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4132985770702362, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4978578984737396, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.509240984916687, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5494712591171265, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7726926207542419, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.283905953168869, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.46577900648117065, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4697645306587219, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4225437045097351, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6718382239341736, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4320765435695648, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.42229753732681274, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.44320282340049744, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6595913171768188, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7022290229797363, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41536062955856323, "metrics/hit_grouping/tracker/all/object_completeness": 0.4487324655056, "metrics/hit_grouping/tracker/all/object_purity": 0.4654613435268402, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6082368493080139, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.015406607650220394, "metrics/hit_grouping/tracker/background_recall": 0.12125039100646973, "metrics/hit_grouping/tracker/completeness": 0.4482782185077667, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4701732397079468, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7159918546676636, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4069787859916687, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5030730366706848, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5219047665596008, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5954760909080505, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6982126235961914, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4278453588485718, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.437544584274292, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4589191973209381, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6137229800224304, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6779094338417053, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42019838094711304, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3480699360370636, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3558575510978699, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6290351748466492, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7255429029464722, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43185871839523315, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.479670912027359, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.491728276014328, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5706886053085327, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7209354043006897, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4655172526836395, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.44805148243904114, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.42813584208488464, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.567118227481842, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6685668230056763, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3849210739135742, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4086335301399231, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4380047619342804, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6644220948219299, "metrics/hit_grouping/tracker/purity": 0.4247566759586334, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8849426507949829, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02375565655529499, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.056420616805553436, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.061775196343660355, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.060708899050951004, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6714146137237549, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.26690760254859924, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.32040414214134216, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.325373113155365, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5420753955841064, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6784547567367554, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.48956409096717834, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5208595395088196, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.540550947189331, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6966593861579895, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5388936400413513, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6373699307441711, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7326440811157227, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6136213541030884, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.010259131900966167, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.03914657607674599, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.421615600585938, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2959076762199402, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8999999761581421, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20428450405597687, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8987870216369629, "metrics/particle/class_1/pid_accuracy": 0.9744095802307129, "metrics/particle/class_2/efficiency": 0.49133536219596863, "metrics/particle/class_2/pid_accuracy": 0.7023974061012268, "metrics/particle/class_3/efficiency": 0.7566403746604919, "metrics/particle/class_3/pid_accuracy": 0.9738811254501343, "metrics/particle/class_4/efficiency": 0.7004008293151855, "metrics/particle/class_4/pid_accuracy": 0.4220314621925354, "metrics/particle/class_5/efficiency": 0.954023003578186, "metrics/particle/class_5/pid_accuracy": 0.4538152515888214, "metrics/particle/count/bias_mean": -2.8131511211395264, "metrics/particle/count/mae": 4.169921875, "metrics/particle/count/prediction_mean": 67.841796875, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07429436594247818, "metrics/particle/event/energy_response_mean": 0.9732158780097961, "metrics/particle/event/met_abs_error": 6.075338363647461, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05739468336105347, "metrics/particle/event/scalar_pt_response_mean": 0.9902164936065674, "metrics/particle/event/vector_pt_closure": 0.05669727921485901, "metrics/particle/matched/delta_eta_abs_mean": 0.012589875608682632, "metrics/particle/matched/delta_phi_abs_mean": 0.012038927525281906, "metrics/particle/matched/delta_r_mean": 0.019498318433761597, "metrics/particle/matched/energy_relative_abs_mean": 0.13697071373462677, "metrics/particle/matched/pid_accuracy": 0.953284502029419, "metrics/particle/matched/pt_relative_abs_mean": 0.13816499710083008, "metrics/particle/matching/duplicate_fraction": 0.050151143223047256, "metrics/particle/matching/efficiency": 0.8041667342185974, "metrics/particle/matching/f1": 0.8205010294914246, "metrics/particle/matching/purity": 0.8375126123428345, "metrics/particle/matching/target_coverage_dr0p05": 0.7802554368972778, "metrics/particle/matching/target_coverage_dr0p10": 0.8525606989860535, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6209940314292908, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8348230123519897, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019327959045767784, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13768793642520905, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1399080902338028, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02248772233724594, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17929551005363464, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00832466222345829, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6855625510215759, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.07028678804636002, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9445792436599731, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8649.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.19977128505706787, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8622857332229614, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8820635676383972, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7418168187141418, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1158963218331337, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.76673698425293, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016359515488147736, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9818240404129028, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04598625749349594, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.95416259765625, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10147.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8029149770736694, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.1340680867433548, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.770977020263672, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.5232434272766113, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2697468400001526, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.09045351296663284, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.623224258422852, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7779476642608643, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.851522445678711, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023480428382754326, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1522115170955658, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1548268049955368, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01898496225476265, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19437158107757568, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03139810264110565, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.393201231956482, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.045230258256196976, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5994768142700195, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1688.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5194913148880005, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.034195899963379, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.8397154808044434, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.6581835746765137, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2022569477558136, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 44.438880920410156, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0685763880610466, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.669057607650757, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.07859187573194504, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.660426139831543, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2304.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.612037718296051, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.175190448760986, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.2325439453125, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1877522468566895, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06045330688357353, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.0455998070538044, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.076078414916992, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.5072828531265259, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.379566192626953, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01932222582399845, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13489802181720734, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13588488101959229, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017774978652596474, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17564408481121063, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006251302547752857, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.21575911343097687, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01696842350065708, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6896389126777649, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19196.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18675853312015533, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8321746587753296, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6554462909698486, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5048893094062805, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11775907874107361, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.757986545562744, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003941541071981192, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3931659758090973, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.018438145518302917, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6758378744125366, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22580.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8249602317810059, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.6968260407447815, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.601034164428711, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.242011547088623, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.09632467478513718, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.42941099405288696, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.8264269828796387, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.1983148604631424, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.928398609161377, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01956157200038433, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13670465350151062, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13814577460289001, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01922430656850338, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1778383105993271, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004638878628611565, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4206435978412628, "metrics/particle/momentum/all/accepted/pz_bias": 0.02902812510728836, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.46869957447052, "metrics/particle/momentum/all/accepted/samples": 29533.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21241173148155212, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8541836142539978, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8647477626800537, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7151400446891785, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12277697026729584, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.912711143493652, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01178955752402544, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7133614420890808, "metrics/particle/momentum/all/assigned/pz_bias": 0.020035970956087112, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0448436737060547, "metrics/particle/momentum/all/assigned/samples": 35031.0, "metrics/particle/momentum/all/efficiency": 0.8025490045547485, "metrics/particle/momentum/all/event/energy_bias": -4.344296455383301, "metrics/particle/momentum/all/event/p3_error_mean": 18.554580688476562, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.208372116088867, "metrics/particle/momentum/all/event/px_bias": -0.23387543857097626, "metrics/particle/momentum/all/event/py_bias": 0.2933576703071594, "metrics/particle/momentum/all/event/pz_abs_mean": 13.699617385864258, "metrics/particle/momentum/all/event/pz_bias": 0.4689798057079315, "metrics/particle/momentum/all/event/pz_residual_rms": 25.02292823791504, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01940622739493847, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11615835130214691, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11915796250104904, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.015362081117928028, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1552457958459854, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004210393875837326, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4227006435394287, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06220569089055061, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4023323059082031, "metrics/particle/momentum/class_1/accepted/samples": 15913.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10838484019041061, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.32045409083366394, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.34180837869644165, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.21721626818180084, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05940023064613342, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.941229820251465, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008535178378224373, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5867817401885986, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.06753803789615631, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8460617065429688, "metrics/particle/momentum/class_1/assigned/samples": 17340.0, "metrics/particle/momentum/class_1/efficiency": 0.8969618678092957, "metrics/particle/momentum/class_1/event/energy_bias": 3.7282586097717285, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.91502857208252, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.123920440673828, "metrics/particle/momentum/class_1/event/px_bias": -0.23932044208049774, "metrics/particle/momentum/class_1/event/py_bias": 0.17369282245635986, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.719834327697754, "metrics/particle/momentum/class_1/event/pz_bias": 2.9533491134643555, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.289215087890625, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.028174323961138725, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17741534113883972, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17950932681560516, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.0008004688424989581, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2212950736284256, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028493894264101982, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4570883512496948, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.19025935232639313, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6373291015625, "metrics/particle/momentum/class_2/accepted/samples": 1474.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6169877052307129, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7511125802993774, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8325824737548828, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.49772441387176514, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.28082191944122314, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.5752005577087402, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05821917951107025, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.5456111431121826, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0995141938328743, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.22365665435791, "metrics/particle/momentum/class_2/assigned/samples": 2336.0, "metrics/particle/momentum/class_2/efficiency": 0.49363696575164795, "metrics/particle/momentum/class_2/event/energy_bias": -10.01008415222168, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.364808082580566, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.788501262664795, "metrics/particle/momentum/class_2/event/px_bias": 0.11293861269950867, "metrics/particle/momentum/class_2/event/py_bias": -0.15721572935581207, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.415971755981445, "metrics/particle/momentum/class_2/event/pz_bias": -2.048677444458008, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.542434692382812, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018789783120155334, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15927401185035706, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15867869555950165, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.026106514036655426, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19902566075325012, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016186743741855025, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.26814478635787964, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.00836890283972025, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9355796575546265, "metrics/particle/momentum/class_3/accepted/samples": 11738.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.26831525564193726, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4415109157562256, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.429951548576355, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2786803245544434, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16957606375217438, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.111331939697266, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007885690778493881, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5599620938301086, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.023049963638186455, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4642698764801025, "metrics/particle/momentum/class_3/assigned/samples": 14837.0, "metrics/particle/momentum/class_3/efficiency": 0.7556814551353455, "metrics/particle/momentum/class_3/event/energy_bias": 3.9503986835479736, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.477753639221191, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.018568515777588, "metrics/particle/momentum/class_3/event/px_bias": -0.16287365555763245, "metrics/particle/momentum/class_3/event/py_bias": 0.37489354610443115, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.358823299407959, "metrics/particle/momentum/class_3/event/pz_bias": -0.3897985816001892, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.210479736328125, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.019456708803772926, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15496797859668732, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1479850858449936, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06648384034633636, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1918657422065735, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003952569328248501, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6528433561325073, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1362716555595398, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.705169677734375, "metrics/particle/momentum/class_4/accepted/samples": 253.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.36087071895599365, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.3652546405792236, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.296872615814209, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.2135162353515625, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2542372941970825, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 18.608684539794922, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8089613318443298, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.19299755990505219, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.0641911029815674, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.6746666431427002, "metrics/particle/momentum/class_4/event/energy_bias": -0.8991804122924805, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2394561767578125, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.693224310874939, "metrics/particle/momentum/class_4/event/px_bias": 0.09855865687131882, "metrics/particle/momentum/class_4/event/py_bias": -0.008246518671512604, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1618295907974243, "metrics/particle/momentum/class_4/event/pz_bias": -0.22148174047470093, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.229970455169678, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012223446741700172, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.11996842920780182, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1231682300567627, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.007380418013781309, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16158832609653473, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05161290243268013, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.522768497467041, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.09767668694257736, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.501375913619995, "metrics/particle/momentum/class_5/accepted/samples": 155.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0705864354968071, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.19899903237819672, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.23065069317817688, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08226980268955231, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05487804859876633, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.7863291501998901, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05487804859876633, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6700767278671265, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.22504252195358276, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6870222091674805, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9451219439506531, "metrics/particle/momentum/class_5/event/energy_bias": -1.1136882305145264, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8410223722457886, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3853299617767334, "metrics/particle/momentum/class_5/event/px_bias": -0.043178681284189224, "metrics/particle/momentum/class_5/event/py_bias": -0.08976638317108154, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9378001689910889, "metrics/particle/momentum/class_5/event/pz_bias": 0.1755884885787964, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.0542800426483154, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.33034199476242065, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3324420750141144, "metrics/set_slot/angular_recall_all_dr0p10": 0.9573466181755066, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8420378565788269, "metrics/set_slot/soft_count_bias_mean": -0.6167230606079102}, "step_train_time": 3.886510133743286, "step_valid_time": 2268.979932308197, "step_total_time": 2272.8664429187775, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0035689941535753, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13976056725779729, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2921, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2912, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9548872180451128, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9624382207578254, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9586478503446012, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03756177924217463, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9519450800915332, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9594728171334431, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9556941253692156, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9548872180451128, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9912367997569562, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13303631193116516, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2266, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2170, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2162, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.949671772428884, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9576345984112974, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.953636563392661, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04236540158870256, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9461706783369803, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9541041482789056, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.950120852559877, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.949671772428884, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9996810059529526, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14156142493783497, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1305, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1227, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1220, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9281391830559758, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9402298850574713, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9341454130186525, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.059770114942528735, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9228441754916793, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9348659003831418, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9288161400837457, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9281391830559758} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_37000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_37000.json new file mode 100644 index 0000000000000000000000000000000000000000..2b67a4ce63e1b10cbb180bb95b3002d2aade5e7a --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_37000.json @@ -0,0 +1 @@ +{"epoch": 1.7525577870405458, "train": {"Classification_binary": 0.14021436870098114, "Cardinality": 0.12679310142993927, "Classification": 0.09264153242111206, "Regression_pt": 0.12506331503391266, "Regression_eta": 0.0014156282413750887, "Regression_sin_phi": 0.0006157057941891253, "Regression_cos_phi": 0.0007670671329833567, "Regression_energy": 0.12409298866987228, "HitGrouping": 0.19668148458003998, "Auxiliary": 0.21938848495483398, "Total": 1.015013337135315, "Total_unweighted": 0.4848106059944257}, "train_group": {}, "valid": {"Classification_binary": 0.1578224003314972, "Cardinality": 0.12763179341952005, "Classification": 0.09371846914291382, "Regression_pt": 0.1225770612557729, "Regression_eta": 0.0013267660203079383, "Regression_sin_phi": 0.0007094559259712696, "Regression_cos_phi": 0.0007054608625670274, "Regression_energy": 0.12125640114148457, "HitGrouping": 0.19177927573521933, "Auxiliary": 0.23112493753433228, "Total": 1.0383888880411785, "Total_unweighted": 0.49811601468051475, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8262251019477844, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5031682252883911, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3640579283237457, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5055252313613892, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5008057951927185, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.40777894854545593, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7633475661277771, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.43863096833229065, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3604135811328888, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4390493333339691, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.442209392786026, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.503323495388031, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8438550233840942, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.46893370151519775, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3265910744667053, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4714837074279785, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.45965564250946045, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3673686683177948, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8322635889053345, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.47695252299308777, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3405676484107971, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.479278564453125, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4707677364349365, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3901662826538086, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0031227869912981987, "metrics/hit_grouping/calorimeter/background_recall": 0.13727590441703796, "metrics/hit_grouping/calorimeter/completeness": 0.5405078530311584, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5514610409736633, "metrics/hit_grouping/calorimeter/energy_purity": 0.5442448854446411, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3859125077724457, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8556533455848694, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5022607445716858, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.26536184549331665, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5060940980911255, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.49570220708847046, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.33187299966812134, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8328807950019836, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4966822862625122, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.37551820278167725, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.49915963411331177, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.48871392011642456, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.39440348744392395, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.783950686454773, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.40547317266464233, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45870068669319153, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4045734703540802, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4018474817276001, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5047563910484314, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8185173869132996, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5277825593948364, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4256074130535126, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.532430112361908, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5235954523086548, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4297570288181305, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7916051149368286, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5750272274017334, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5671641826629639, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5684102177619934, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5604887008666992, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.46114256978034973, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8511226773262024, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4134438931941986, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22686859965324402, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4144684970378876, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4062177240848541, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.34134528040885925, "metrics/hit_grouping/calorimeter/purity": 0.5326216816902161, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8287818431854248, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09199035167694092, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11654135584831238, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08939759433269501, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.059446290135383606, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04887218028306961, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8260376453399658, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19301837682724, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10005886107683182, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19349056482315063, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18245473504066467, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.26957032084465027, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8326443433761597, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.49800071120262146, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3567337393760681, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.500503420829773, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4924733638763428, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.40200310945510864, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7411951422691345, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4198174774646759, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5074487924575806, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5231044292449951, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.555700421333313, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7694377303123474, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.26883664727211, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.48418229818344116, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.48494234681129456, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4159131944179535, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6723079085350037, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4413584768772125, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.43338051438331604, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.44732075929641724, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6513887047767639, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7006546854972839, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4219946563243866, "metrics/hit_grouping/tracker/all/object_completeness": 0.45989754796028137, "metrics/hit_grouping/tracker/all/object_purity": 0.47339150309562683, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6045859456062317, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.016491953283548355, "metrics/hit_grouping/tracker/background_recall": 0.1340779960155487, "metrics/hit_grouping/tracker/completeness": 0.4592876732349396, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.470323383808136, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7136920690536499, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4174995720386505, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5104562044143677, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5183724761009216, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5928458571434021, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6958416700363159, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4309312105178833, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45313507318496704, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.47496864199638367, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6175349354743958, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6786183714866638, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42236247658729553, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3626178205013275, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3792992830276489, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6158701777458191, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7270205020904541, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44230279326438904, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.49403759837150574, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5042985677719116, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5639806389808655, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7252579927444458, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.46490147709846497, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46256473660469055, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.44055843353271484, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5474137663841248, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6619675755500793, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38790154457092285, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.41503629088401794, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.43909719586372375, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6676233410835266, "metrics/hit_grouping/tracker/purity": 0.4356687068939209, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8781142830848694, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02526395209133625, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.055772606283426285, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0578417032957077, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06372549384832382, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6637114882469177, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2782653570175171, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.32481762766838074, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3336162567138672, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5616933107376099, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6783243417739868, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.49644675850868225, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5345094799995422, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5502076745033264, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6894969344139099, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5324850678443909, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6415207386016846, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7370737791061401, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6274793148040771, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.016667703166604042, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05300455540418625, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.1807861328125, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2937319576740265, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.875, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20210880041122437, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8970133662223816, "metrics/particle/class_1/pid_accuracy": 0.976272463798523, "metrics/particle/class_2/efficiency": 0.48929664492607117, "metrics/particle/class_2/pid_accuracy": 0.6923611164093018, "metrics/particle/class_3/efficiency": 0.7618870735168457, "metrics/particle/class_3/pid_accuracy": 0.9727697968482971, "metrics/particle/class_4/efficiency": 0.6893787384033203, "metrics/particle/class_4/pid_accuracy": 0.447674423456192, "metrics/particle/class_5/efficiency": 0.9444444179534912, "metrics/particle/class_5/pid_accuracy": 0.45233264565467834, "metrics/particle/count/bias_mean": -3.181640625, "metrics/particle/count/mae": 4.392578125, "metrics/particle/count/prediction_mean": 67.47330474853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07730192691087723, "metrics/particle/event/energy_response_mean": 0.9833933711051941, "metrics/particle/event/met_abs_error": 6.140207290649414, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06150679662823677, "metrics/particle/event/scalar_pt_response_mean": 1.0113846063613892, "metrics/particle/event/vector_pt_closure": 0.058744143694639206, "metrics/particle/matched/delta_eta_abs_mean": 0.012912604957818985, "metrics/particle/matched/delta_phi_abs_mean": 0.011927230283617973, "metrics/particle/matched/delta_r_mean": 0.01966072805225849, "metrics/particle/matched/energy_relative_abs_mean": 0.13840925693511963, "metrics/particle/matched/pid_accuracy": 0.9537225365638733, "metrics/particle/matched/pt_relative_abs_mean": 0.13988319039344788, "metrics/particle/matching/duplicate_fraction": 0.051003966480493546, "metrics/particle/matching/efficiency": 0.8052079677581787, "metrics/particle/matching/f1": 0.8237550854682922, "metrics/particle/matching/purity": 0.8431767821311951, "metrics/particle/matching/target_coverage_dr0p05": 0.783250093460083, "metrics/particle/matching/target_coverage_dr0p10": 0.8536203503608704, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6127840280532837, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8340029120445251, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019700653851032257, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1410289853811264, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14382785558700562, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.04213516041636467, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18386179208755493, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006337865721434355, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6735729575157166, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.06921170651912689, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.881957769393921, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8678.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18022093176841736, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8549619913101196, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9026498794555664, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7801598310470581, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11279311031103134, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.89499568939209, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013950726948678493, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9788573980331421, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0770048201084137, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9996092319488525, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10107.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8056071400642395, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.7160687446594238, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.187054634094238, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.779362201690674, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.0012876763939857483, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.27395007014274597, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.952061653137207, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.1451694965362549, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.318338394165039, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023396283388137817, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16255193948745728, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16190072894096375, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04887092113494873, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20207838714122772, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.027010804042220116, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3983005285263062, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2435416728258133, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.717639446258545, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1666.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.45376577973365784, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2466835975646973, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.500687599182129, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.348738431930542, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21101321280002594, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 36.84598159790039, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.058149781078100204, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.503002166748047, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.274493008852005, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.536213397979736, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2270.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6040608882904053, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.685197830200195, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.706829071044922, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1985583305358887, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06736847758293152, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0003044530749320984, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.5582275390625, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5051460266113281, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.97183609008789, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019578685984015465, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1358141452074051, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13707903027534485, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02604367956519127, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17729227244853973, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005179468425922096, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.21681943535804749, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.013953271321952343, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7137088179588318, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19307.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1808442622423172, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9264642000198364, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7074077129364014, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5659609436988831, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11388406902551651, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.858943462371826, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003469905350357294, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.38612788915634155, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.01266484335064888, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5575902462005615, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22479.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8297305703163147, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.474379062652588, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.349142074584961, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.749806880950928, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.1261242628097534, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.03729625046253204, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.398839473724365, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5056968927383423, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.888697624206543, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019828882068395615, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13884268701076508, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14044888317584991, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.03203578665852547, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18070770800113678, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003709824290126562, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4168819785118103, "metrics/particle/momentum/all/accepted/pz_bias": -0.04302572086453438, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4645147323608398, "metrics/particle/momentum/all/accepted/samples": 29651.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19843754172325134, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.92658531665802, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9459335803985596, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8092995882034302, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11989327520132065, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.040454864501953, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010070002637803555, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.695859968662262, "metrics/particle/momentum/all/assigned/pz_bias": -0.04837272688746452, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.805772066116333, "metrics/particle/momentum/all/assigned/samples": 34856.0, "metrics/particle/momentum/all/efficiency": 0.805755615234375, "metrics/particle/momentum/all/event/energy_bias": -0.4947498142719269, "metrics/particle/momentum/all/event/p3_error_mean": 20.00510597229004, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.6884765625, "metrics/particle/momentum/all/event/px_bias": -1.1947803497314453, "metrics/particle/momentum/all/event/py_bias": -0.23634931445121765, "metrics/particle/momentum/all/event/pz_abs_mean": 15.029206275939941, "metrics/particle/momentum/all/event/pz_bias": -3.156012535095215, "metrics/particle/momentum/all/event/pz_residual_rms": 27.075088500976562, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019828833639621735, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11812928318977356, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12124387174844742, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.03901125118136406, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15848365426063538, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003010536776855588, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4103640615940094, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.04719002917408943, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3220903873443604, "metrics/particle/momentum/class_1/accepted/samples": 15944.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10103245079517365, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3291309177875519, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4125606417655945, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3119369447231293, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.060184650123119354, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.954144477844238, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006462781224399805, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5658010244369507, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.03651413694024086, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.380250930786133, "metrics/particle/momentum/class_1/assigned/samples": 17330.0, "metrics/particle/momentum/class_1/efficiency": 0.8987091779708862, "metrics/particle/momentum/class_1/event/energy_bias": 8.720022201538086, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.482072830200195, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.647311210632324, "metrics/particle/momentum/class_1/event/px_bias": -1.2254810333251953, "metrics/particle/momentum/class_1/event/py_bias": -0.3022305965423584, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.77638053894043, "metrics/particle/momentum/class_1/event/pz_bias": -1.7529606819152832, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.04484748840332, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027972260490059853, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18038925528526306, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18221436440944672, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.014895322732627392, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22416433691978455, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029045643284916878, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5330982208251953, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1534983366727829, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.83571195602417, "metrics/particle/momentum/class_2/accepted/samples": 1446.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5538238286972046, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7630184888839722, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8607450723648071, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5619121193885803, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.28463587164878845, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.367251396179199, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.052397869527339935, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.471747875213623, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.27587637305259705, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.840586185455322, "metrics/particle/momentum/class_2/assigned/samples": 2252.0, "metrics/particle/momentum/class_2/efficiency": 0.48425987362861633, "metrics/particle/momentum/class_2/event/energy_bias": -10.030856132507324, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.247989654541016, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.112244606018066, "metrics/particle/momentum/class_2/event/px_bias": -0.04021567106246948, "metrics/particle/momentum/class_2/event/py_bias": 0.008788913488388062, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.11343002319336, "metrics/particle/momentum/class_2/event/pz_bias": -0.8380123376846313, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.629606246948242, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01891503855586052, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16097435355186462, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1606437861919403, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.023434096947312355, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20086444914340973, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010966762201860547, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.27006828784942627, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.02607591077685356, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.005744457244873, "metrics/particle/momentum/class_3/accepted/samples": 11854.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2565230429172516, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.607733130455017, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5441040992736816, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3888649940490723, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16247542202472687, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.553470611572266, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00759167643263936, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5661699771881104, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0323687382042408, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5541725158691406, "metrics/particle/momentum/class_3/assigned/samples": 14753.0, "metrics/particle/momentum/class_3/efficiency": 0.7631494402885437, "metrics/particle/momentum/class_3/event/energy_bias": 2.9447286128997803, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.533390998840332, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.847817897796631, "metrics/particle/momentum/class_3/event/px_bias": 0.08259943127632141, "metrics/particle/momentum/class_3/event/py_bias": 0.38242271542549133, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.509923934936523, "metrics/particle/momentum/class_3/event/pz_bias": -0.6039458513259888, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.32795524597168, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.019425908103585243, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16996647417545319, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16525468230247498, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.09569200873374939, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21306604146957397, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5858337879180908, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.05616438388824463, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1591376066207886, "metrics/particle/momentum/class_4/accepted/samples": 254.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3526138961315155, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.1404035091400146, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.9946727752685547, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.8977537155151367, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24929971992969513, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.046283721923828, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6718182563781738, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.10193459689617157, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.4137425422668457, "metrics/particle/momentum/class_4/assigned/samples": 357.0, "metrics/particle/momentum/class_4/efficiency": 0.6773333549499512, "metrics/particle/momentum/class_4/event/energy_bias": -1.079255223274231, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2424874305725098, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7080453634262085, "metrics/particle/momentum/class_4/event/px_bias": 0.09620489180088043, "metrics/particle/momentum/class_4/event/py_bias": -0.021756501868367195, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.129150152206421, "metrics/particle/momentum/class_4/event/pz_bias": -0.0015888996422290802, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.186378002166748, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014342105016112328, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13834761083126068, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1412360817193985, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.027879927307367325, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.186739981174469, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04575163498520851, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6409882307052612, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.04288241267204285, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.722639799118042, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.050414882600307465, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2127661406993866, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.20806993544101715, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08067265897989273, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05487804859876633, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.5274794101715088, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.05487804859876633, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.77219557762146, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.055664584040641785, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.85227632522583, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9329268336296082, "metrics/particle/momentum/class_5/event/energy_bias": -1.0493899583816528, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.033277750015259, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5561763048171997, "metrics/particle/momentum/class_5/event/px_bias": -0.10788805037736893, "metrics/particle/momentum/class_5/event/py_bias": -0.3035737872123718, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0138031244277954, "metrics/particle/momentum/class_5/event/pz_bias": 0.040495529770851135, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2219319343566895, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3288443684577942, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.33112525939941406, "metrics/set_slot/angular_recall_all_dr0p10": 0.958461582660675, "metrics/set_slot/angular_recall_selected_dr0p10": 0.843502938747406, "metrics/set_slot/soft_count_bias_mean": -0.7971566319465637}, "step_train_time": 4.21151065826416, "step_valid_time": 4739.368683576584, "step_total_time": 4743.580194711685, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0247874714119425, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13596815796250106, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3044, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2911, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2898, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9516181758744687, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9563074901445466, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9539570702932983, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04369250985545335, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9473684210526315, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9520367936925098, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9496968703916107, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9516181758744687, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0211455149805737, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13547673346278355, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2267, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2163, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2157, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9466083150984683, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9541243934715483, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9503514938488576, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0458756065284517, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9439824945295405, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9514777238641376, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9477152899824253, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9466083150984683, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0246875037285306, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13072291631784683, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1309, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1224, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1214, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9258698940998488, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.935064935064935, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9304446978335233, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06493506493506493, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9183055975794251, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9274255156608098, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9228430254656025, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9258698940998488} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_38000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_38000.json new file mode 100644 index 0000000000000000000000000000000000000000..71347f6be151889cb2b17d0647fab0ba25971bfe --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_38000.json @@ -0,0 +1 @@ +{"epoch": 1.7999242137173173, "train": {"Classification_binary": 0.14130301773548126, "Cardinality": 0.1213393360376358, "Classification": 0.07807335257530212, "Regression_pt": 0.1286361962556839, "Regression_eta": 0.001217644545249641, "Regression_sin_phi": 0.0005076839006505907, "Regression_cos_phi": 0.0007503475062549114, "Regression_energy": 0.12772737443447113, "HitGrouping": 0.18727000057697296, "Auxiliary": 0.22017741203308105, "Total": 0.981071949005127, "Total_unweighted": 0.47821561695309356}, "train_group": {}, "valid": {"Classification_binary": 0.1547132134437561, "Cardinality": 0.12727108597755432, "Classification": 0.09361283977826436, "Regression_pt": 0.1231313943862915, "Regression_eta": 0.0013449642186363537, "Regression_sin_phi": 0.0006793922899911801, "Regression_cos_phi": 0.0006835816893726587, "Regression_energy": 0.12208078304926555, "HitGrouping": 0.19017227490743002, "Auxiliary": 0.23044878244400024, "Total": 1.0294491449991863, "Total_unweighted": 0.4962461688555777, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8211102485656738, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.503946840763092, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3601779043674469, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5064273476600647, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5041777491569519, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.416579931974411, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.765028178691864, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4372303783893585, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.35044312477111816, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.44065797328948975, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4490726590156555, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5018463730812073, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8422238230705261, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4700467586517334, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3294607698917389, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4729721248149872, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4609486758708954, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3701457977294922, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8298341631889343, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.477769136428833, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.34039562940597534, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.48059865832328796, "metrics/hit_grouping/calorimeter/all/object_purity": 0.47312304377555847, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.39443808794021606, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003369303187355399, "metrics/hit_grouping/calorimeter/background_recall": 0.14739541709423065, "metrics/hit_grouping/calorimeter/completeness": 0.5408540964126587, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5511062145233154, "metrics/hit_grouping/calorimeter/energy_purity": 0.5439311861991882, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3870975971221924, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8525960445404053, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5036566257476807, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2657032310962677, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5082945823669434, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.49888375401496887, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3360264003276825, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8292670845985413, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.49524983763694763, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.37367573380470276, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.4976600706577301, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.49109381437301636, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.40522801876068115, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7839938998222351, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4073747992515564, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45916473865509033, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4069393277168274, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.40249305963516235, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5026682019233704, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8185304999351501, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.527854859828949, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4229612648487091, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5330521464347839, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5283606052398682, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.42945635318756104, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7937183976173401, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5733129382133484, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5640761852264404, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.569709300994873, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5493738055229187, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4657745659351349, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8456614017486572, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4153209626674652, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22956649959087372, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4165029227733612, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4077207148075104, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.35023605823516846, "metrics/hit_grouping/calorimeter/purity": 0.5330947637557983, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8204566240310669, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08389320224523544, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11654135584831238, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08383682370185852, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05134809762239456, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.043233081698417664, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7985434532165527, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1955471634864807, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1006474420428276, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19654124975204468, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18866372108459473, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2984108328819275, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8316152691841125, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4988735318183899, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.35651934146881104, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5018454790115356, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4947984218597412, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4051578938961029, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7388322949409485, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41286396980285645, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5018168687820435, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5227882862091064, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5668550133705139, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7740438580513, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.27245327830314636, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4705842137336731, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4772128164768219, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4285714328289032, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6747548580169678, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43373146653175354, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4321221113204956, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.45523276925086975, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6573607921600342, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7017827033996582, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41540515422821045, "metrics/hit_grouping/tracker/all/object_completeness": 0.4563835561275482, "metrics/hit_grouping/tracker/all/object_purity": 0.4776192903518677, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6126446723937988, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.014183608815073967, "metrics/hit_grouping/tracker/background_recall": 0.12121092528104782, "metrics/hit_grouping/tracker/completeness": 0.4524748623371124, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.46699967980384827, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7175105214118958, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40461161732673645, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.509594202041626, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5278450846672058, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5991583466529846, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6958476305007935, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4329279363155365, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4438709616661072, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.47093331813812256, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6213468909263611, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6753274202346802, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42019838094711304, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3593612611293793, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.38094794750213623, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6317403316497803, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7237750291824341, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4276980459690094, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.48585668206214905, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5109496712684631, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5785853862762451, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7123199105262756, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45935961604118347, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4514327347278595, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4390358626842499, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5701970458030701, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6713030934333801, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39154431223869324, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.41895508766174316, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.44120538234710693, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.664532482624054, "metrics/hit_grouping/tracker/purity": 0.42820125818252563, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8831168413162231, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02526395209133625, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.06170090287923813, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.060675621032714844, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06447964161634445, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6658357381820679, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2891068756580353, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3296853005886078, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.33808842301368713, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5632421374320984, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.678769588470459, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4869900941848755, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5286858081817627, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.55461585521698, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6993452906608582, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5159404277801514, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6204845309257507, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.714665949344635, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6267458200454712, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.03321230784058571, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0522710457444191, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.269834518432617, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30559682846069336, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8999999165534973, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.21397370100021362, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9016478061676025, "metrics/particle/class_1/pid_accuracy": 0.9737504720687866, "metrics/particle/class_2/efficiency": 0.47864991426467896, "metrics/particle/class_2/pid_accuracy": 0.6975863575935364, "metrics/particle/class_3/efficiency": 0.7750694155693054, "metrics/particle/class_3/pid_accuracy": 0.9757996201515198, "metrics/particle/class_4/efficiency": 0.7084168195724487, "metrics/particle/class_4/pid_accuracy": 0.45120227336883545, "metrics/particle/class_5/efficiency": 0.9444444179534912, "metrics/particle/class_5/pid_accuracy": 0.476673424243927, "metrics/particle/count/bias_mean": -2.7102863788604736, "metrics/particle/count/mae": 4.175130367279053, "metrics/particle/count/prediction_mean": 67.94466400146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07618796825408936, "metrics/particle/event/energy_response_mean": 0.9722256660461426, "metrics/particle/event/met_abs_error": 6.379177570343018, "metrics/particle/event/scalar_pt_relative_abs_error": 0.059688787907361984, "metrics/particle/event/scalar_pt_response_mean": 0.988147497177124, "metrics/particle/event/vector_pt_closure": 0.059907954186201096, "metrics/particle/matched/delta_eta_abs_mean": 0.012522412464022636, "metrics/particle/matched/delta_phi_abs_mean": 0.011804411187767982, "metrics/particle/matched/delta_r_mean": 0.019255293533205986, "metrics/particle/matched/energy_relative_abs_mean": 0.1353653073310852, "metrics/particle/matched/pid_accuracy": 0.9543654918670654, "metrics/particle/matched/pt_relative_abs_mean": 0.13556884229183197, "metrics/particle/matching/duplicate_fraction": 0.0471239797770977, "metrics/particle/matching/efficiency": 0.8123122453689575, "metrics/particle/matching/f1": 0.8281968832015991, "metrics/particle/matching/purity": 0.8447150588035583, "metrics/particle/matching/target_coverage_dr0p05": 0.786207914352417, "metrics/particle/matching/target_coverage_dr0p10": 0.8563938736915588, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6329266428947449, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.84097820520401, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019222846254706383, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1357061117887497, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1365426927804947, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.008178087882697582, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1769554167985916, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008937778882682323, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6777364015579224, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.10155915468931198, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9602211713790894, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8727.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.20121359825134277, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8497256636619568, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8809288740158081, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7259348630905151, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10873422771692276, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.24314022064209, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016462933272123337, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9626665115356445, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.09019434452056885, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.855569839477539, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10144.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.810155987739563, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.10527486354112625, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.419412612915039, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.7829205989837646, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.24632865190505981, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.28698813915252686, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.26977014541626, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.7332279682159424, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.631513595581055, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023278437554836273, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1513534039258957, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15223278105258942, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0021747148130089045, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19105516374111176, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02306327596306801, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2912747859954834, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.3407086431980133, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4511525630950928, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1691.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5004802346229553, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.005391001701355, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.4497132301330566, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.2541818618774414, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20484010875225067, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 18.702173233032227, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06266205757856369, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.694526433944702, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.39157751202583313, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.90837287902832, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2314.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6131254434585571, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.1106133460998535, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.037105560302734, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2715885639190674, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.14247266948223114, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.018383048474788666, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.877090454101562, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.7642134428024292, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.009382247924805, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0190923810005188, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13387998938560486, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13373790681362152, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.003443480934947729, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17398981750011444, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00030907124164514244, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2076137214899063, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005633300635963678, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.645210862159729, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19413.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17733906209468842, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8894073367118835, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6567767262458801, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.48933646082878113, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.111501045525074, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.08052396774292, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0036426635924726725, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3874874711036682, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.02067124843597412, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7315465211868286, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22511.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8342859745025635, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.5209896564483643, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.423419952392578, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.002705574035645, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.1438618898391724, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.5251319408416748, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.150798797607422, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.081546425819397, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.736438751220703, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01936783827841282, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13540472090244293, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1356068253517151, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.004756659269332886, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1758694350719452, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004123227670788765, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4065755605697632, "metrics/particle/momentum/all/accepted/pz_bias": -0.052690304815769196, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.438809871673584, "metrics/particle/momentum/all/accepted/samples": 29831.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20564790070056915, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8855712413787842, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8404439687728882, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6747550964355469, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11687494814395905, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.79673957824707, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011267121881246567, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7070021033287048, "metrics/particle/momentum/all/assigned/pz_bias": -0.06538282334804535, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.089815616607666, "metrics/particle/momentum/all/assigned/samples": 34969.0, "metrics/particle/momentum/all/efficiency": 0.8106470108032227, "metrics/particle/momentum/all/event/energy_bias": -5.526328086853027, "metrics/particle/momentum/all/event/p3_error_mean": 20.068153381347656, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.95452880859375, "metrics/particle/momentum/all/event/px_bias": -1.5326632261276245, "metrics/particle/momentum/all/event/py_bias": -0.8305031657218933, "metrics/particle/momentum/all/event/pz_abs_mean": 14.881636619567871, "metrics/particle/momentum/all/event/pz_bias": -4.578988075256348, "metrics/particle/momentum/all/event/pz_residual_rms": 26.244815826416016, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019384518265724182, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11245174705982208, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11352140456438065, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011430363170802593, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15040162205696106, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003507673041895032, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.40414735674858093, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.07383273541927338, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3300670385360718, "metrics/particle/momentum/class_1/accepted/samples": 15965.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1010589450597763, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2789413332939148, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30641642212867737, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1824234575033188, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05694299936294556, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.1613845825195312, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006937218364328146, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.557166576385498, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.08548568934202194, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7396163940429688, "metrics/particle/momentum/class_1/assigned/samples": 17298.0, "metrics/particle/momentum/class_1/efficiency": 0.8998929262161255, "metrics/particle/momentum/class_1/event/energy_bias": 2.8794994354248047, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.556838035583496, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.538581848144531, "metrics/particle/momentum/class_1/event/px_bias": -0.3166200518608093, "metrics/particle/momentum/class_1/event/py_bias": 0.8895432949066162, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.145917892456055, "metrics/particle/momentum/class_1/event/pz_bias": -2.8191230297088623, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.65300750732422, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027932796627283096, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1821008026599884, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18340426683425903, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.02515140175819397, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22441774606704712, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03055555559694767, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.536073923110962, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.060240015387535095, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.717625379562378, "metrics/particle/momentum/class_2/accepted/samples": 1440.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6045065522193909, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6755450963973999, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9320816993713379, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5980708599090576, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3009079098701477, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.491013050079346, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05923043563961983, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.495488405227661, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.18224354088306427, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.85746431350708, "metrics/particle/momentum/class_2/assigned/samples": 2313.0, "metrics/particle/momentum/class_2/efficiency": 0.48225051164627075, "metrics/particle/momentum/class_2/event/energy_bias": -7.91346549987793, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.580530166625977, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.633843421936035, "metrics/particle/momentum/class_2/event/px_bias": -0.9385218620300293, "metrics/particle/momentum/class_2/event/py_bias": -0.9070045948028564, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.048192024230957, "metrics/particle/momentum/class_2/event/pz_bias": -0.7830803394317627, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.081390380859375, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018400000408291817, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16015692055225372, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15930962562561035, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.008018259890377522, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1990426927804947, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013322231825441122, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.257219523191452, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.02637229487299919, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9685132503509521, "metrics/particle/momentum/class_3/accepted/samples": 12010.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2647116780281067, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5585970878601074, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4171968698501587, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.22640860080719, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15556754171848297, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.84471607208252, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008492855355143547, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5902170538902283, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.027880312874913216, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.848924160003662, "metrics/particle/momentum/class_3/assigned/samples": 14836.0, "metrics/particle/momentum/class_3/efficiency": 0.7731925845146179, "metrics/particle/momentum/class_3/event/energy_bias": 0.940505862236023, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.934438705444336, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.21685791015625, "metrics/particle/momentum/class_3/event/px_bias": -0.3676581084728241, "metrics/particle/momentum/class_3/event/py_bias": -0.5883092880249023, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.663453102111816, "metrics/particle/momentum/class_3/event/pz_bias": -0.8299095034599304, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.227622032165527, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.0193755105137825, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14346835017204285, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.13540881872177124, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06679599732160568, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18066410720348358, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003816793905571103, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5597933530807495, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.04500710219144821, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.402683138847351, "metrics/particle/momentum/class_4/accepted/samples": 262.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3011452853679657, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.9923949241638184, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.4519615173339844, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.381255865097046, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24581006169319153, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.39660930633545, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008379888720810413, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7667329907417297, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.11913497745990753, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.0057389736175537, "metrics/particle/momentum/class_4/assigned/samples": 358.0, "metrics/particle/momentum/class_4/efficiency": 0.6986666917800903, "metrics/particle/momentum/class_4/event/energy_bias": -0.780013382434845, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3158979415893555, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7119667530059814, "metrics/particle/momentum/class_4/event/px_bias": 0.04890318959951401, "metrics/particle/momentum/class_4/event/py_bias": -0.1510080099105835, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.22731614112854, "metrics/particle/momentum/class_4/event/pz_bias": -0.13445502519607544, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.696785926818848, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013016694225370884, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13420628011226654, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13006646931171417, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012927805073559284, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17375001311302185, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03896103799343109, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.483923316001892, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.1441810429096222, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.446328639984131, "metrics/particle/momentum/class_5/accepted/samples": 154.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06029209867119789, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.16620923578739166, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18200097978115082, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05564292520284653, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.060975611209869385, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3497905135154724, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04878048598766327, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.721172571182251, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.09225305914878845, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.134971618652344, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9390243887901306, "metrics/particle/momentum/class_5/event/energy_bias": -0.652854859828949, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8376140594482422, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.410360336303711, "metrics/particle/momentum/class_5/event/px_bias": 0.0412336103618145, "metrics/particle/momentum/class_5/event/py_bias": -0.07372453063726425, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.906720757484436, "metrics/particle/momentum/class_5/event/pz_bias": -0.012419922277331352, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.0931622982025146, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.32328298687934875, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.325167179107666, "metrics/set_slot/angular_recall_all_dr0p10": 0.9570794105529785, "metrics/set_slot/angular_recall_selected_dr0p10": 0.845566987991333, "metrics/set_slot/soft_count_bias_mean": -0.5541136860847473}, "step_train_time": 4.181879043579102, "step_valid_time": 7329.055753469467, "step_total_time": 7333.237633228302, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9987656398908695, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14004308025893253, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3029, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2911, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2900, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9516181758744687, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9610432485968967, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9563074901445466, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03895675140310333, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9480222294867604, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9574116870254209, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9526938239159002, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9516181758744687, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9879551895554631, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14137176803366724, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2253, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2154, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2150, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9426695842450766, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9560585885486018, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9493168796826796, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043941411451398134, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9409190371991247, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.954283177984909, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9475539885412075, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9426695842450766, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9987538948890655, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14090757257490316, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1310, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1223, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1216, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9251134644478064, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.933587786259542, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9293313069908815, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06641221374045801, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9198184568835098, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9282442748091603, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9240121580547113, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9251134644478064} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_39000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_39000.json new file mode 100644 index 0000000000000000000000000000000000000000..0f60927740dccc02e3cd75903b7245d5a411251f --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_39000.json @@ -0,0 +1 @@ +{"epoch": 1.8472906403940887, "train": {"Classification_binary": 0.15182501077651978, "Cardinality": 0.12274932861328125, "Classification": 0.0846894234418869, "Regression_pt": 0.11888427287340164, "Regression_eta": 0.0013057279866188765, "Regression_sin_phi": 0.0006227949052117765, "Regression_cos_phi": 0.0006313368212431669, "Regression_energy": 0.11786605417728424, "HitGrouping": 0.1852814108133316, "Auxiliary": 0.21859246492385864, "Total": 0.9857014417648315, "Total_unweighted": 0.4758246209821664}, "train_group": {}, "valid": {"Classification_binary": 0.15451860427856445, "Cardinality": 0.11790172259012859, "Classification": 0.09123218059539795, "Regression_pt": 0.11947773893674214, "Regression_eta": 0.00133167440071702, "Regression_sin_phi": 0.0006907532612482706, "Regression_cos_phi": 0.0006441191847746571, "Regression_energy": 0.11864260832468669, "HitGrouping": 0.189532200495402, "Auxiliary": 0.22576171159744263, "Total": 1.0069164435068767, "Total_unweighted": 0.4865376789821312, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.825476348400116, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5084564089775085, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3492003381252289, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.511663019657135, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5060480237007141, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.40777894854545593, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7695806622505188, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.44334766268730164, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.33419498801231384, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.44831371307373047, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4547024369239807, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4841211140155792, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8429425358772278, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4753056466579437, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.32186993956565857, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4780891537666321, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4675564169883728, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3645915389060974, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8319555521011353, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.48286759853363037, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.33110666275024414, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.48594874143600464, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4782196283340454, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.38695526123046875, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0032495311461389065, "metrics/hit_grouping/calorimeter/background_recall": 0.14081351459026337, "metrics/hit_grouping/calorimeter/completeness": 0.5460992455482483, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5544871091842651, "metrics/hit_grouping/calorimeter/energy_purity": 0.5472488403320312, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.37637490034103394, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8553327918052673, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5092467665672302, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.252617210149765, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5138474106788635, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5041818022727966, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.32629722356796265, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8315052390098572, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5016444325447083, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.36745738983154297, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5044328570365906, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.49339210987091064, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.39774298667907715, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7847431898117065, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.41016465425491333, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45452436804771423, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.41044262051582336, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4099981486797333, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4997679889202118, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8218051791191101, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5351274609565735, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4140004813671112, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5408368706703186, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5363339185714722, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.41911235451698303, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7932547926902771, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5678434371948242, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5723108649253845, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5655295252799988, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5546347498893738, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4575398862361908, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8469149470329285, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.41946175694465637, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21785517036914825, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.420506089925766, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4098648130893707, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3457600176334381, "metrics/hit_grouping/calorimeter/purity": 0.5381999015808105, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8234156370162964, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09262323379516602, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10902255773544312, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09333378076553345, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06286763399839401, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8019826412200928, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.18629318475723267, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1012360230088234, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.18570537865161896, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.17484791576862335, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.28546202182769775, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8336544036865234, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5046597123146057, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3466874957084656, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.50797039270401, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5007745027542114, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3981132507324219, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7484657764434814, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.403158038854599, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5071051716804504, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5231099128723145, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5507750511169434, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7738745808601379, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.27245327830314636, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.47711125016212463, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4766000509262085, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4141048789024353, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6755992770195007, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4339473247528076, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.43165987730026245, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4531756341457367, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6505972146987915, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.705253541469574, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41255566477775574, "metrics/hit_grouping/tracker/all/object_completeness": 0.4582049548625946, "metrics/hit_grouping/tracker/all/object_purity": 0.4763999283313751, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6024488210678101, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013826238922774792, "metrics/hit_grouping/tracker/background_recall": 0.10984370112419128, "metrics/hit_grouping/tracker/completeness": 0.4576674699783325, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.46359995007514954, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7163160443305969, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4063650667667389, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5090998411178589, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5259918570518494, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5946869850158691, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.699962854385376, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4309312105178833, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.44946524500846863, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4701569080352783, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6075512766838074, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6877545714378357, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.40703335404396057, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.36219775676727295, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.38059234619140625, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6133453845977783, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7313450574874878, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43007558584213257, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4904724061489105, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5062461495399475, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5595652461051941, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7289149165153503, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.44766008853912354, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.460439532995224, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.44448426365852356, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5535714030265808, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6670921444892883, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3834860324859619, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.41585561633110046, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4433203637599945, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6669610142707825, "metrics/hit_grouping/tracker/purity": 0.43295833468437195, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8820594549179077, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.027149321511387825, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.0595693401992321, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.06082082539796829, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06447964161634445, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.671737015247345, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.26019617915153503, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3241216242313385, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3352862000465393, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5493030548095703, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6826490163803101, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.48626264929771423, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5318942666053772, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5533655285835266, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6880421042442322, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5274993777275085, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6423611640930176, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7384607791900635, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6370812654495239, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.02165335789322853, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06260645389556885, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.506847381591797, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3266620635986328, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8999999761581421, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.23503893613815308, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9110119342803955, "metrics/particle/class_1/pid_accuracy": 0.9769092202186584, "metrics/particle/class_2/efficiency": 0.5122890472412109, "metrics/particle/class_2/pid_accuracy": 0.7017466425895691, "metrics/particle/class_3/efficiency": 0.7752224206924438, "metrics/particle/class_3/pid_accuracy": 0.9742816090583801, "metrics/particle/class_4/efficiency": 0.7154308557510376, "metrics/particle/class_4/pid_accuracy": 0.4215686321258545, "metrics/particle/class_5/efficiency": 0.9501915574073792, "metrics/particle/class_5/pid_accuracy": 0.46370968222618103, "metrics/particle/count/bias_mean": -1.8756510019302368, "metrics/particle/count/mae": 3.8561198711395264, "metrics/particle/count/prediction_mean": 68.779296875, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07609555870294571, "metrics/particle/event/energy_response_mean": 0.9766852259635925, "metrics/particle/event/met_abs_error": 6.5895256996154785, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05877009034156799, "metrics/particle/event/scalar_pt_response_mean": 0.9968730807304382, "metrics/particle/event/vector_pt_closure": 0.05723050236701965, "metrics/particle/matched/delta_eta_abs_mean": 0.012861228547990322, "metrics/particle/matched/delta_phi_abs_mean": 0.01178575586527586, "metrics/particle/matched/delta_r_mean": 0.019535645842552185, "metrics/particle/matched/energy_relative_abs_mean": 0.1341242790222168, "metrics/particle/matched/pid_accuracy": 0.9545536041259766, "metrics/particle/matched/pt_relative_abs_mean": 0.13420426845550537, "metrics/particle/matching/duplicate_fraction": 0.052648019045591354, "metrics/particle/matching/efficiency": 0.8197298049926758, "metrics/particle/matching/f1": 0.8307567238807678, "metrics/particle/matching/purity": 0.8420843482017517, "metrics/particle/matching/target_coverage_dr0p05": 0.7919300198554993, "metrics/particle/matching/target_coverage_dr0p10": 0.8646222949028015, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6407036185264587, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8476678133010864, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019328346475958824, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13566645979881287, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13532425463199615, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016742372885346413, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17532643675804138, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006912181153893471, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6616872549057007, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.029467839747667313, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9472155570983887, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8825.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.19232377409934998, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8343275785446167, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8589550852775574, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.719718873500824, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1124076247215271, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.863651752471924, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014877479523420334, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9672119617462158, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.019057944416999817, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.003216028213501, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10284.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8192536234855652, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.183671474456787, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.55341911315918, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.574760913848877, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.36264121532440186, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2862136662006378, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.309713840484619, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.25402992963790894, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.098560333251953, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023777280002832413, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1512698084115982, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15027250349521637, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02597997337579727, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1916525810956955, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.030885780230164528, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3566340208053589, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.04132251814007759, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.524020195007324, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1716.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4990273118019104, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5107942819595337, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.721715211868286, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.559451103210449, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1996527761220932, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 42.570735931396484, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0659722238779068, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.549839735031128, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.16704702377319336, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.838004112243652, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2304.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.622189998626709, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.0359673500061035, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.671070098876953, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1950688362121582, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.017130563035607338, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08254089951515198, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.53791618347168, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.990565299987793, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.348371505737305, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019474027678370476, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1328580230474472, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1326821893453598, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.00851910188794136, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17250759899616241, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000508750497829169, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20929782092571259, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.009020245634019375, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6819817423820496, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19656.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1700320839881897, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9388958811759949, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7343862652778625, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.582665741443634, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10843850672245026, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.781032562255859, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004133503418415785, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.37613412737846375, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.007247085217386484, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5774325132369995, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22741.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8447290658950806, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.0662084817886353, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.11977481842041, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.636623382568359, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.0195971727371216, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.5834788680076599, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.150201797485352, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.14548444747924805, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.003181457519531, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01967599242925644, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13472506403923035, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1344539225101471, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011914579197764397, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17447523772716522, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004106368403881788, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4067070186138153, "metrics/particle/momentum/all/accepted/pz_bias": 0.00039216812001541257, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4548406600952148, "metrics/particle/momentum/all/accepted/samples": 30197.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19797661900520325, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9457535743713379, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9654674530029297, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8166934251785278, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.115542471408844, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.708257675170898, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011293837800621986, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6899517774581909, "metrics/particle/momentum/all/assigned/pz_bias": -0.010011335834860802, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8694283962249756, "metrics/particle/momentum/all/assigned/samples": 35329.0, "metrics/particle/momentum/all/efficiency": 0.8205929398536682, "metrics/particle/momentum/all/event/energy_bias": -2.7860872745513916, "metrics/particle/momentum/all/event/p3_error_mean": 18.88270378112793, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.397806167602539, "metrics/particle/momentum/all/event/px_bias": 1.365107774734497, "metrics/particle/momentum/all/event/py_bias": -0.787151575088501, "metrics/particle/momentum/all/event/pz_abs_mean": 13.857446670532227, "metrics/particle/momentum/all/event/pz_bias": -1.0991108417510986, "metrics/particle/momentum/all/event/pz_residual_rms": 25.22327995300293, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019633905962109566, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11013215035200119, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1110956147313118, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004788597580045462, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14661362767219543, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0031584815587848425, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.39521220326423645, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.013867883943021297, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.338964581489563, "metrics/particle/momentum/class_1/accepted/samples": 16147.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09839077293872833, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3150636553764343, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.41834408044815063, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.29631829261779785, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05337771773338318, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.134079456329346, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006772656925022602, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5369264483451843, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01604205183684826, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.359675645828247, "metrics/particle/momentum/class_1/assigned/samples": 17423.0, "metrics/particle/momentum/class_1/efficiency": 0.9101516008377075, "metrics/particle/momentum/class_1/event/energy_bias": 3.73195743560791, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.510610580444336, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.45477294921875, "metrics/particle/momentum/class_1/event/px_bias": 1.0522266626358032, "metrics/particle/momentum/class_1/event/py_bias": 0.41645875573158264, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.823052406311035, "metrics/particle/momentum/class_1/event/pz_bias": 0.8966237306594849, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.842302322387695, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0275558028370142, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18076163530349731, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18118388950824738, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.0014773278962820768, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22249265015125275, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029527559876441956, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4857630729675293, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.03630659356713295, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7076940536499023, "metrics/particle/momentum/class_2/accepted/samples": 1524.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5901169776916504, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7704415321350098, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.947135865688324, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6343198418617249, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.27694860100746155, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.2446818351745605, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0555555559694767, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4481818675994873, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.028737155720591545, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.925606727600098, "metrics/particle/momentum/class_2/assigned/samples": 2412.0, "metrics/particle/momentum/class_2/efficiency": 0.5103817582130432, "metrics/particle/momentum/class_2/event/energy_bias": -8.734207153320312, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.354351043701172, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.973234176635742, "metrics/particle/momentum/class_2/event/px_bias": -0.09238598495721817, "metrics/particle/momentum/class_2/event/py_bias": -0.9671727418899536, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.340022087097168, "metrics/particle/momentum/class_2/event/pz_bias": -1.3935742378234863, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.905988693237305, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01880953460931778, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16092608869075775, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15916801989078522, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.021761560812592506, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1993083506822586, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018178813625127077, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.26603028178215027, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.016619466245174408, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9626502394676208, "metrics/particle/momentum/class_3/accepted/samples": 12102.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.24822859466075897, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6237266063690186, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5405552387237549, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3861007690429688, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15947066247463226, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.86937713623047, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009022857993841171, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5673456788063049, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.04024524986743927, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6453189849853516, "metrics/particle/momentum/class_3/assigned/samples": 14962.0, "metrics/particle/momentum/class_3/efficiency": 0.7791154384613037, "metrics/particle/momentum/class_3/event/energy_bias": 4.547060966491699, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.502495765686035, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.075930118560791, "metrics/particle/momentum/class_3/event/px_bias": 0.18849214911460876, "metrics/particle/momentum/class_3/event/py_bias": -0.09307491779327393, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.459330081939697, "metrics/particle/momentum/class_3/event/pz_bias": -0.7804245948791504, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.384249687194824, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.020638482645154, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1719125360250473, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16179916262626648, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.061517708003520966, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20618663728237152, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00374531839042902, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6536508798599243, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.03751493617892265, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4394779205322266, "metrics/particle/momentum/class_4/accepted/samples": 267.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3675452172756195, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.741929054260254, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.970837116241455, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.8615620136260986, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.25, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 24.94571876525879, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.013586956076323986, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9336394667625427, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.06890395283699036, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.220555543899536, "metrics/particle/momentum/class_4/assigned/samples": 368.0, "metrics/particle/momentum/class_4/efficiency": 0.7120000123977661, "metrics/particle/momentum/class_4/event/energy_bias": -1.3839319944381714, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2610862255096436, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.717526912689209, "metrics/particle/momentum/class_4/event/px_bias": 0.1985476315021515, "metrics/particle/momentum/class_4/event/py_bias": 0.024641476571559906, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1521618366241455, "metrics/particle/momentum/class_4/event/pz_bias": -0.07616035640239716, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.4467034339904785, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012667370028793812, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1342684030532837, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13164407014846802, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.002725764410570264, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17164000868797302, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03184713423252106, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5383126735687256, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.21886081993579865, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.78271484375, "metrics/particle/momentum/class_5/accepted/samples": 157.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04535762593150139, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1563519984483719, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.15029294788837433, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.002059035934507847, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03658536449074745, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.212729811668396, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04268292710185051, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7269402742385864, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.3879810869693756, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.032322406768799, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.957317054271698, "metrics/particle/momentum/class_5/event/energy_bias": -0.9469677805900574, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8399759531021118, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.362697958946228, "metrics/particle/momentum/class_5/event/px_bias": 0.018227258697152138, "metrics/particle/momentum/class_5/event/py_bias": -0.16800424456596375, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9434284567832947, "metrics/particle/momentum/class_5/event/pz_bias": 0.25442466139793396, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2342910766601562, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3229060769081116, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3243027925491333, "metrics/set_slot/angular_recall_all_dr0p10": 0.9598990082740784, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8524408936500549, "metrics/set_slot/soft_count_bias_mean": 0.28595346212387085}, "step_train_time": 4.471672534942627, "step_valid_time": 9881.351830005646, "step_total_time": 9885.823502540588, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0047796716951196, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13873473868917374, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3045, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2928, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2913, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9571755475645636, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.961576354679803, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9593709043250328, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03842364532019704, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9522719843085976, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9566502463054187, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9544560943643512, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9571755475645636, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9970910839841176, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13117469153596573, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2258, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2151, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2144, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9413566739606127, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526129317980514, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9469513537310148, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04738706820194863, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9382932166301969, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9495128432240921, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9438696896324015, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9413566739606127, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0046802547519194, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14203966931255374, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1311, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1221, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1212, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9236006051437217, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.931350114416476, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9274591720470946, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06864988558352403, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9167927382753404, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9244851258581236, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.920622863653627, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9236006051437217} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_4000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..8ad0089b7fb98d2f66c47869ba1be6a188b9aab9 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 0.189465706707086, "train": {"Classification_binary": 0.3811817765235901, "Cardinality": 0.2445000559091568, "Classification": 0.3069608807563782, "Regression_pt": 0.4491924047470093, "Regression_eta": 0.006183908320963383, "Regression_sin_phi": 0.0027909891214221716, "Regression_cos_phi": 0.002698557684198022, "Regression_energy": 0.44126492738723755, "HitGrouping": 0.2906467020511627, "Auxiliary": 0.5111600160598755, "Total": 2.675987958908081, "Total_unweighted": 1.5902734445407987}, "train_group": {}, "valid": {"Classification_binary": 0.36696310838063556, "Cardinality": 0.27791579564412433, "Classification": 0.27765554189682007, "Regression_pt": 0.4419814745585124, "Regression_eta": 0.00523729311923186, "Regression_sin_phi": 0.002643689513206482, "Regression_cos_phi": 0.002575976929316918, "Regression_energy": 0.43401233355204266, "HitGrouping": 0.29007307688395184, "Auxiliary": 0.49926217397054035, "Total": 2.609302838643392, "Total_unweighted": 1.531069417949766, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.774063766002655, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.14000454545021057, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3100217580795288, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.13642697036266327, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.11456673592329025, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5616542100906372, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7599502801895142, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.14406602084636688, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.27511078119277954, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.1404624879360199, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.11137817054986954, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5505908131599426, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7748639583587646, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.12256676703691483, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2751215100288391, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.1204514354467392, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.10162386298179626, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5660263895988464, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7734636664390564, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.1295187622308731, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.28569379448890686, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.1268448680639267, "metrics/hit_grouping/calorimeter/all/object_purity": 0.10630223900079727, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5635034441947937, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.000990297761745751, "metrics/hit_grouping/calorimeter/background_recall": 0.03732100501656532, "metrics/hit_grouping/calorimeter/completeness": 0.22767357528209686, "metrics/hit_grouping/calorimeter/energy_completeness": 0.2609092891216278, "metrics/hit_grouping/calorimeter/energy_purity": 0.25726401805877686, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.6199904084205627, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7890287041664124, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.12962627410888672, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2348657250404358, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.12756112217903137, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.12034961581230164, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5279927253723145, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7702274322509766, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.12998086214065552, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.30055272579193115, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.12667374312877655, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.10053298622369766, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5718562602996826, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7449870705604553, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.12883394956588745, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.37436196208000183, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.12555690109729767, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.08347203582525253, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.627494215965271, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7481135725975037, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.15387961268424988, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3731657564640045, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.1489458829164505, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.12306786328554153, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6187154054641724, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7724401354789734, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.360176146030426, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6942871809005737, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.35899481177330017, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.26430755853652954, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.559958815574646, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7994315028190613, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.07720161974430084, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1478324830532074, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.07665401697158813, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.07038445770740509, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5076338052749634, "metrics/hit_grouping/calorimeter/purity": 0.2238728553056717, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8958601355552673, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.02131516858935356, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10526315867900848, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.02095864713191986, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.003609951352700591, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8379351496696472, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.031344223767519, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.07533843070268631, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.030869195237755775, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.01921260915696621, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3490288257598877, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7681143283843994, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.13639071583747864, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2995803952217102, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.13356466591358185, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.11250755935907364, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5833256840705872, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6676729917526245, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43082717061042786, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.27382397651672363, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.24270325899124146, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6912936568260193, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7193584442138672, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3273056149482727, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.2837747037410736, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.2438185214996338, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5015069246292114, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.594963788986206, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.42394590377807617, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.2172907441854477, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.1861429065465927, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7580946683883667, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6264990568161011, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4189225435256958, "metrics/hit_grouping/tracker/all/object_completeness": 0.23957684636116028, "metrics/hit_grouping/tracker/all/object_purity": 0.2077867090702057, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7186108827590942, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.04425825923681259, "metrics/hit_grouping/tracker/background_recall": 0.15132617950439453, "metrics/hit_grouping/tracker/completeness": 0.24596792459487915, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6423215270042419, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6344614028930664, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4224969446659088, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.28876030445098877, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.2566813826560974, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7224267721176147, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.624815046787262, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.41132691502571106, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.21511217951774597, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.1828041970729828, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.719549834728241, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6117936968803406, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4191163182258606, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.1627129167318344, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.13203123211860657, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7098286747932434, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6445564031600952, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4483315050601959, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.2449684888124466, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.2059742510318756, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.706291913986206, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.636284351348877, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5264778137207031, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.26173973083496094, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.21219488978385925, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6902709603309631, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6012697815895081, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.361408531665802, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.22859440743923187, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.20935268700122833, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7397063970565796, "metrics/hit_grouping/tracker/purity": 0.24009738862514496, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.870460033416748, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.0214932132512331, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.010824265889823437, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.005765979178249836, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.05731523409485817, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.66745525598526, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.1832731068134308, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.10313976556062698, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.0659843236207962, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5549819469451904, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5858568549156189, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.503441333770752, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.28831109404563904, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.253135621547699, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8344804644584656, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.27146822214126587, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6499999761581421, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5552000403404236, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6298115849494934, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5679689049720764, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.2776845097541809, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.00650590006262064, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.870616912841797, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.17094123363494873, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.0793180763721466, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.49256208539009094, "metrics/particle/class_1/pid_accuracy": 0.8568939566612244, "metrics/particle/class_2/efficiency": 0.3482840657234192, "metrics/particle/class_2/pid_accuracy": 0.13105690479278564, "metrics/particle/class_3/efficiency": 0.448833703994751, "metrics/particle/class_3/pid_accuracy": 0.8837854862213135, "metrics/particle/class_4/efficiency": 0.46492984890937805, "metrics/particle/class_4/pid_accuracy": 0.11853448301553726, "metrics/particle/class_5/efficiency": 0.5498084425926208, "metrics/particle/class_5/pid_accuracy": 0.1324041783809662, "metrics/particle/count/bias_mean": -6.688150882720947, "metrics/particle/count/mae": 8.770182609558105, "metrics/particle/count/prediction_mean": 63.966796875, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.179209366440773, "metrics/particle/event/energy_response_mean": 1.089706540107727, "metrics/particle/event/met_abs_error": 13.665898323059082, "metrics/particle/event/scalar_pt_relative_abs_error": 0.18189851939678192, "metrics/particle/event/scalar_pt_response_mean": 1.140798568725586, "metrics/particle/event/vector_pt_closure": 0.11467745900154114, "metrics/particle/matched/delta_eta_abs_mean": 0.02289940044283867, "metrics/particle/matched/delta_phi_abs_mean": 0.021156996488571167, "metrics/particle/matched/delta_r_mean": 0.034803569316864014, "metrics/particle/matched/energy_relative_abs_mean": 0.20206306874752045, "metrics/particle/matched/pid_accuracy": 0.8124501705169678, "metrics/particle/matched/pt_relative_abs_mean": 0.1995311677455902, "metrics/particle/matching/duplicate_fraction": 0.12483079731464386, "metrics/particle/matching/efficiency": 0.46241453289985657, "metrics/particle/matching/f1": 0.48538777232170105, "metrics/particle/matching/purity": 0.5107630491256714, "metrics/particle/matching/target_coverage_dr0p05": 0.4113023579120636, "metrics/particle/matching/target_coverage_dr0p10": 0.5680942535400391, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.28632768988609314, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5367285013198853, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.034370057284832, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20096354186534882, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.19869446754455566, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.010003866627812386, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2396313101053238, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.017742933705449104, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.3798463344573975, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.08831281214952469, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.031543016433716, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4847.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.6949050426483154, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 3.0494656562805176, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 4.1782307624816895, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.9965858459472656, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.3823953866958618, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 21.963268280029297, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.039627037942409515, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.1980621814727783, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05111506208777428, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.75882625579834, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9009.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.449962854385376, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 14.248931884765625, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 17.748302459716797, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 8.103784561157227, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2494206428527832, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.6390913128852844, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 15.013251304626465, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.598738193511963, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 23.55516815185547, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.034342825412750244, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1973704844713211, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1935524195432663, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.006748853716999292, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2348398119211197, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04352806508541107, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.250943899154663, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.176679790019989, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.219811916351318, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 873.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.075282096862793, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.763230323791504, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 9.073992729187012, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 8.907334327697754, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.4612526595592499, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 57.34225082397461, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08598726242780685, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.834805727005005, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2124188393354416, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.725894927978516, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1884.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.3165337145328522, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.80526065826416, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 13.965295791625977, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.1013972759246826, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.1593252420425415, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.09481659531593323, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 13.71160888671875, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.006889268755912781, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 32.41579055786133, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03508735075592995, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20206251740455627, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1991013139486313, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.012388225644826889, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.24008090794086456, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0023289143573492765, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.47702834010124207, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.014248444698750973, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.1675617694854736, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 11164.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.6502833366394043, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.531773567199707, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.8579118251800537, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.677371025085449, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.4015408456325531, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 17.519302368164062, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.02127854712307453, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.3105888366699219, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.1165618970990181, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.610865592956543, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21806.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.4797799587249756, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 23.140439987182617, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 20.98587989807129, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 17.059764862060547, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.1402437686920166, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.6812429428100586, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 10.225774765014648, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.0261458158493042, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 15.80174446105957, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03484293818473816, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.2015044242143631, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.19869761168956757, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.011412144638597965, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.239683598279953, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00888415053486824, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8279278874397278, "metrics/particle/momentum/all/accepted/pz_bias": -0.025066573172807693, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.2245171070098877, "metrics/particle/momentum/all/accepted/samples": 16884.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.6870641112327576, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 4.021481513977051, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4.2466959953308105, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4.066650390625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.3997064232826233, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 22.957216262817383, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.030062081292271614, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.700535535812378, "metrics/particle/momentum/all/assigned/pz_bias": 0.07957573980093002, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.643570899963379, "metrics/particle/momentum/all/assigned/samples": 32699.0, "metrics/particle/momentum/all/efficiency": 0.4588168263435364, "metrics/particle/momentum/all/event/energy_bias": 29.584108352661133, "metrics/particle/momentum/all/event/p3_error_mean": 36.854705810546875, "metrics/particle/momentum/all/event/pt_vector_error_mean": 18.899744033813477, "metrics/particle/momentum/all/event/px_bias": -2.548989772796631, "metrics/particle/momentum/all/event/py_bias": -3.4151506423950195, "metrics/particle/momentum/all/event/pz_abs_mean": 27.042564392089844, "metrics/particle/momentum/all/event/pz_bias": 3.6317734718322754, "metrics/particle/momentum/all/event/pz_residual_rms": 41.43875503540039, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.04245667904615402, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2154964655637741, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21375177800655365, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.018621083348989487, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2540934085845947, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.009979113936424255, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.9750894904136658, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0019461206393316388, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.3881466388702393, "metrics/particle/momentum/class_1/accepted/samples": 8618.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.5445049405097961, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 2.4875054359436035, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.7936713695526123, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 2.590618133544922, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.36564958095550537, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 15.678791046142578, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02642340399324894, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.6430861949920654, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.11472242325544357, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.129322528839111, "metrics/particle/momentum/class_1/assigned/samples": 15895.0, "metrics/particle/momentum/class_1/efficiency": 0.4857674241065979, "metrics/particle/momentum/class_1/event/energy_bias": 41.08696365356445, "metrics/particle/momentum/class_1/event/p3_error_mean": 31.815948486328125, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 18.857892990112305, "metrics/particle/momentum/class_1/event/px_bias": -0.1474122703075409, "metrics/particle/momentum/class_1/event/py_bias": -3.196650266647339, "metrics/particle/momentum/class_1/event/pz_abs_mean": 21.884681701660156, "metrics/particle/momentum/class_1/event/pz_bias": 6.714630126953125, "metrics/particle/momentum/class_1/event/pz_residual_rms": 29.602067947387695, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04314339533448219, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22860568761825562, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2220844030380249, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06788615137338638, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2617592215538025, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03735632076859474, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9686931371688843, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1866692453622818, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.409790992736816, "metrics/particle/momentum/class_2/accepted/samples": 1044.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7835739850997925, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9971061944961548, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.3633489608764648, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 1.010472059249878, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3504560589790344, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.688126087188721, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06577052175998688, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.92287015914917, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.06468769162893295, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.2999467849731445, "metrics/particle/momentum/class_2/assigned/samples": 2083.0, "metrics/particle/momentum/class_2/efficiency": 0.3496316075325012, "metrics/particle/momentum/class_2/event/energy_bias": -21.312152862548828, "metrics/particle/momentum/class_2/event/p3_error_mean": 30.36615753173828, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 16.562273025512695, "metrics/particle/momentum/class_2/event/px_bias": -1.6023515462875366, "metrics/particle/momentum/class_2/event/py_bias": -1.5024046897888184, "metrics/particle/momentum/class_2/event/pz_abs_mean": 21.39443588256836, "metrics/particle/momentum/class_2/event/pz_bias": -4.075594902038574, "metrics/particle/momentum/class_2/event/pz_residual_rms": 33.910675048828125, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023928703740239143, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17863689363002777, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17528587579727173, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.008227834478020668, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21551834046840668, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002584721427410841, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4392298758029938, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.027472110465168953, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2993913888931274, "metrics/particle/momentum/class_3/accepted/samples": 6964.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.8334913849830627, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 6.067416667938232, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 6.235715389251709, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 6.1099042892456055, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.44465526938438416, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 30.243091583251953, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.02825216017663479, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.5593677759170532, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.035764507949352264, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.842864513397217, "metrics/particle/momentum/class_3/assigned/samples": 14229.0, "metrics/particle/momentum/class_3/efficiency": 0.44833579659461975, "metrics/particle/momentum/class_3/event/energy_bias": 16.5615177154541, "metrics/particle/momentum/class_3/event/p3_error_mean": 25.04684829711914, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 15.324199676513672, "metrics/particle/momentum/class_3/event/px_bias": -0.6575629711151123, "metrics/particle/momentum/class_3/event/py_bias": 3.9872169494628906, "metrics/particle/momentum/class_3/event/pz_abs_mean": 16.54628562927246, "metrics/particle/momentum/class_3/event/pz_bias": 0.4657055139541626, "metrics/particle/momentum/class_3/event/pz_residual_rms": 28.660472869873047, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04077349603176117, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.24396978318691254, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.230442076921463, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.06516016274690628, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2652154266834259, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.03680981695652008, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.8510692119598389, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.14565591514110565, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.200303077697754, "metrics/particle/momentum/class_4/accepted/samples": 163.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.7947423458099365, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 9.822061538696289, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 7.943267822265625, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 7.744481563568115, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.4571428596973419, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 26.34638786315918, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03999999910593033, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.9505661725997925, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.1928028017282486, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.5043392181396484, "metrics/particle/momentum/class_4/assigned/samples": 350.0, "metrics/particle/momentum/class_4/efficiency": 0.4346666634082794, "metrics/particle/momentum/class_4/event/energy_bias": -3.428847074508667, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.308685302734375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.300171852111816, "metrics/particle/momentum/class_4/event/px_bias": 0.1419096738100052, "metrics/particle/momentum/class_4/event/py_bias": -1.1997482776641846, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.4301583766937256, "metrics/particle/momentum/class_4/event/pz_bias": -0.2951160967350006, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.310887336730957, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.04283403605222702, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2378242313861847, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.23777517676353455, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08432131260633469, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2730894684791565, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.010526316240429878, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6797410249710083, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.03671612590551376, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.655730962753296, "metrics/particle/momentum/class_5/accepted/samples": 95.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.2909025549888611, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.7855509519577026, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.7698050141334534, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.3120676875114441, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.2887323796749115, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.0028791427612305, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.07042253762483597, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.7301406860351562, "metrics/particle/momentum/class_5/assigned/pz_bias": 1.4251941442489624, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 9.700237274169922, "metrics/particle/momentum/class_5/assigned/samples": 142.0, "metrics/particle/momentum/class_5/efficiency": 0.5792682766914368, "metrics/particle/momentum/class_5/event/energy_bias": -3.3233706951141357, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.943443775177002, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.8212790489196777, "metrics/particle/momentum/class_5/event/px_bias": -0.2835724949836731, "metrics/particle/momentum/class_5/event/py_bias": -1.503564715385437, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.53391695022583, "metrics/particle/momentum/class_5/event/pz_bias": 0.8221480846405029, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.704554557800293, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6957573890686035, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.7014105916023254, "metrics/set_slot/angular_recall_all_dr0p10": 0.8340121507644653, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6085546612739563, "metrics/set_slot/soft_count_bias_mean": -2.084226369857788}, "step_train_time": 4.663195371627808, "step_valid_time": 9648.166846036911, "step_total_time": 9652.830041646957, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1893354405717622, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3370379779010344, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2990, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2522, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2174, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8244524354364171, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8434782608695652, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8338568358406349, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1565217391304348, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7106897678980059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7270903010033445, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7187964952884774, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8244524354364171, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1613190472611203, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28930362030647205, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2250, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1926, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1780, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8428884026258205, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.856, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.849393605292172, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.144, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7789934354485777, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7911111111111111, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7850055126791621, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8428884026258205, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1564774074243165, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.30113536073473113, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1293, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1073, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 978, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8116490166414524, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8298530549110595, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8206500956022944, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17014694508894046, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7397881996974282, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7563805104408353, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7479923518164436, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8116490166414524} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_40000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..5445a2bab21fae656c14ab0c95212bf7aeebd020 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 1.8946570670708602, "train": {"Classification_binary": 0.1711706519126892, "Cardinality": 0.11001764982938766, "Classification": 0.09623183310031891, "Regression_pt": 0.12255798280239105, "Regression_eta": 0.0012501692399382591, "Regression_sin_phi": 0.0006137725431472063, "Regression_cos_phi": 0.000695593364071101, "Regression_energy": 0.1206265315413475, "HitGrouping": 0.18679377436637878, "Auxiliary": 0.2348106950521469, "Total": 1.0231568813323975, "Total_unweighted": 0.5131465345039032}, "train_group": {}, "valid": {"Classification_binary": 0.15188981095949808, "Cardinality": 0.12324438492457072, "Classification": 0.09013878305753072, "Regression_pt": 0.11980125308036804, "Regression_eta": 0.001311072614043951, "Regression_sin_phi": 0.0006924105497697989, "Regression_cos_phi": 0.00066208823894461, "Regression_energy": 0.11877142389615376, "HitGrouping": 0.19098536173502603, "Auxiliary": 0.22643991311391196, "Total": 1.0124013423919678, "Total_unweighted": 0.48326684239630896, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8265158534049988, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5080785155296326, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.35043057799339294, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5113950371742249, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.505801260471344, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4038989245891571, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7698512077331543, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4488721787929535, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3382570147514343, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4525720775127411, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4546966850757599, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4926144778728485, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8435286283493042, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.47234535217285156, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3254339396953583, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.47527801990509033, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4651874005794525, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3705160915851593, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8326544761657715, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4813483953475952, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.334002286195755, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.48445698618888855, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4766770601272583, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.39010894298553467, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002918601967394352, "metrics/hit_grouping/calorimeter/background_recall": 0.13190607726573944, "metrics/hit_grouping/calorimeter/completeness": 0.5450992584228516, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5569779872894287, "metrics/hit_grouping/calorimeter/energy_purity": 0.5496343374252319, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3792303204536438, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8589804172515869, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5116291642189026, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2606395184993744, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5161238312721252, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5041900873184204, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3240782916545868, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8322719931602478, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5005746483802795, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.360087513923645, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5039892196655273, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.49486202001571655, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.401197612285614, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7793616652488708, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4002375900745392, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4573085904121399, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.40021148324012756, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4022584855556488, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5135731101036072, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8197053670883179, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5325239896774292, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4149627089500427, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5378950238227844, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5314207673072815, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.42608851194381714, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7915133237838745, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5712820291519165, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5635614991188049, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5680091977119446, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5610018372535706, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.46320122480392456, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8507581949234009, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.41845738887786865, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22410938143730164, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4200195372104645, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4108162820339203, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3447176516056061, "metrics/hit_grouping/calorimeter/purity": 0.5370360612869263, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8269155621528625, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08249778300523758, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12030075490474701, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08227712661027908, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.051660045981407166, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.033834587782621384, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8191769123077393, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19981220364570618, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10829900205135345, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19944600760936737, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18029280006885529, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.28016480803489685, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8334493637084961, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5024981498718262, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.34922969341278076, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5058417916297913, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.49902579188346863, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4016355872154236, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7455335259437561, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40764886140823364, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5050187110900879, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5256356000900269, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5543966293334961, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7768903970718384, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28933092951774597, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.48079854249954224, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.47339171171188354, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.41109102964401245, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.673861563205719, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43639373779296875, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.43330755829811096, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4564729630947113, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6490861773490906, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7034998536109924, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41669633984565735, "metrics/hit_grouping/tracker/all/object_completeness": 0.45885562896728516, "metrics/hit_grouping/tracker/all/object_purity": 0.47897952795028687, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6024042963981628, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.0167910847812891, "metrics/hit_grouping/tracker/background_recall": 0.13774865865707397, "metrics/hit_grouping/tracker/completeness": 0.4593018591403961, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.46842890977859497, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.717978835105896, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41320356726646423, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5129513740539551, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5260986089706421, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5929335355758667, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7002391219139099, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.423851877450943, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45090392231941223, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4802756607532501, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.5990197658538818, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6769563555717468, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41677185893058777, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.35548147559165955, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3807685077190399, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6252479553222656, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7274335622787476, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.42616966366767883, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.48917245864868164, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5096823573112488, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5627918839454651, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7192505598068237, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4655172526836395, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.45344898104667664, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4470875561237335, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5646551847457886, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6695616245269775, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3956286609172821, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.42041194438934326, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4447821378707886, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6606689691543579, "metrics/hit_grouping/tracker/purity": 0.4358147382736206, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8818626403808594, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.028657617047429085, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.05690660700201988, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.059937767684459686, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06523378938436508, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6676216721534729, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.26587507128715515, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3235468864440918, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.33357688784599304, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5420753955841064, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6809200644493103, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.490627259016037, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5331693887710571, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5569239854812622, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6886575818061829, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.533615231513977, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6284977197647095, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7217204570770264, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6300297975540161, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.015537559054791927, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05555504187941551, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.59717559814453, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2989247143268585, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.887499988079071, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.2073015570640564, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9033070206642151, "metrics/particle/class_1/pid_accuracy": 0.9773034453392029, "metrics/particle/class_2/efficiency": 0.4986974596977234, "metrics/particle/class_2/pid_accuracy": 0.7031569480895996, "metrics/particle/class_3/efficiency": 0.7751568555831909, "metrics/particle/class_3/pid_accuracy": 0.9754639267921448, "metrics/particle/class_4/efficiency": 0.6873747706413269, "metrics/particle/class_4/pid_accuracy": 0.4446064233779907, "metrics/particle/class_5/efficiency": 0.931034505367279, "metrics/particle/class_5/pid_accuracy": 0.4485596716403961, "metrics/particle/count/bias_mean": -2.9713542461395264, "metrics/particle/count/mae": 4.178385257720947, "metrics/particle/count/prediction_mean": 67.68359375, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.0761890634894371, "metrics/particle/event/energy_response_mean": 0.9872215390205383, "metrics/particle/event/met_abs_error": 6.347585201263428, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05912577733397484, "metrics/particle/event/scalar_pt_response_mean": 1.0072526931762695, "metrics/particle/event/vector_pt_closure": 0.056876376271247864, "metrics/particle/matched/delta_eta_abs_mean": 0.012425770051777363, "metrics/particle/matched/delta_phi_abs_mean": 0.011630347929894924, "metrics/particle/matched/delta_r_mean": 0.019030224531888962, "metrics/particle/matched/energy_relative_abs_mean": 0.1358000636100769, "metrics/particle/matched/pid_accuracy": 0.9558695554733276, "metrics/particle/matched/pt_relative_abs_mean": 0.13624584674835205, "metrics/particle/matching/duplicate_fraction": 0.049469999969005585, "metrics/particle/matching/efficiency": 0.8145236968994141, "metrics/particle/matching/f1": 0.8320187330245972, "metrics/particle/matching/purity": 0.850281834602356, "metrics/particle/matching/target_coverage_dr0p05": 0.7927961945533752, "metrics/particle/matching/target_coverage_dr0p10": 0.8599137663841248, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6313786506652832, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8411993384361267, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01884489133954048, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13683104515075684, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13864342868328094, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02734382264316082, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17928636074066162, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0069333938881754875, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6480764150619507, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.041216179728507996, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.758888840675354, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8798.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.17199358344078064, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7935791015625, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9142777919769287, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7846672534942627, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10722725838422775, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.3140230178833, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01427727472037077, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9425253868103027, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.02389344945549965, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.002223491668701, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10156.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8167471289634705, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.6995246410369873, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.8300604820251465, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.6750800609588623, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.13305294513702393, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.25856730341911316, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.593997001647949, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.766293466091156, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.804059028625488, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023903226479887962, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1545216590166092, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15491971373558044, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01909056305885315, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19560669362545013, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.028471818193793297, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3885555267333984, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.15492746233940125, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.685370445251465, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1721.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4371866285800934, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.455224633216858, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.695948362350464, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.5300333499908447, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18683744966983795, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 47.82974624633789, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06360424309968948, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6052565574645996, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.29702258110046387, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.486461639404297, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2264.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6240028738975525, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.412346839904785, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.754608154296875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2405786514282227, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.09299492835998535, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.05229589343070984, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.607926368713379, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.0595812797546387, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.448078155517578, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01884140446782112, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13428393006324768, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1347370445728302, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02128324657678604, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17477034032344818, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00040983606595546007, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20807872712612152, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.015101197175681591, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6420573592185974, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19520.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1736900806427002, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9180726408958435, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7366206049919128, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5935434699058533, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10660933703184128, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.707898139953613, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004078194964677095, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3824463486671448, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.009752323850989342, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.569455623626709, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22559.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8388843536376953, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.864356279373169, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.655448913574219, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.2933735847473145, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.31468161940574646, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.3383030295372009, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.8927550315856934, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.0901800394058228, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.032201290130615, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019132427871227264, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13618941605091095, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1370374858379364, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.022932682186365128, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17735590040683746, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0039282264187932014, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4045799672603607, "metrics/particle/momentum/all/accepted/pz_bias": -0.030760861933231354, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.397182583808899, "metrics/particle/momentum/all/accepted/samples": 30039.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19025219976902008, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.916693389415741, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9797436594963074, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8390984535217285, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11198147386312485, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.761990547180176, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010892249643802643, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.688933253288269, "metrics/particle/momentum/all/assigned/pz_bias": -0.0324515737593174, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9777567386627197, "metrics/particle/momentum/all/assigned/samples": 34979.0, "metrics/particle/momentum/all/efficiency": 0.8162993788719177, "metrics/particle/momentum/all/event/energy_bias": 0.15153390169143677, "metrics/particle/momentum/all/event/p3_error_mean": 18.944517135620117, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.037057876586914, "metrics/particle/momentum/all/event/px_bias": 0.08863373100757599, "metrics/particle/momentum/all/event/py_bias": 0.027439847588539124, "metrics/particle/momentum/all/event/pz_abs_mean": 14.282262802124023, "metrics/particle/momentum/all/event/pz_bias": -2.9160547256469727, "metrics/particle/momentum/all/event/pz_residual_rms": 25.58837127685547, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019332747906446457, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11374817788600922, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11578237265348434, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.029572267085313797, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15288366377353668, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003434280399233103, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.40309610962867737, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.046099189668893814, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3045196533203125, "metrics/particle/momentum/class_1/accepted/samples": 16015.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1001560315489769, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.33263659477233887, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.434475839138031, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3308294713497162, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05502999573945999, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 9.895331382751465, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0070950621739029884, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5706349611282349, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.029255595058202744, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.875645160675049, "metrics/particle/momentum/class_1/assigned/samples": 17336.0, "metrics/particle/momentum/class_1/efficiency": 0.9027112126350403, "metrics/particle/momentum/class_1/event/energy_bias": 8.77856159210205, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.764130592346191, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.929023742675781, "metrics/particle/momentum/class_1/event/px_bias": 0.9809839725494385, "metrics/particle/momentum/class_1/event/py_bias": 0.829543948173523, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.936244010925293, "metrics/particle/momentum/class_1/event/pz_bias": -2.045149087905884, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.81212615966797, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.028157509863376617, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.180227592587471, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18037670850753784, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.009988626465201378, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2218458205461502, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028150133788585663, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4357787370681763, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.01120524387806654, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.546220541000366, "metrics/particle/momentum/class_2/accepted/samples": 1492.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5379105806350708, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8071947693824768, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0044195652008057, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7087497115135193, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2631118893623352, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.221388816833496, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0559440553188324, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3987643718719482, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.20897915959358215, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.709656715393066, "metrics/particle/momentum/class_2/assigned/samples": 2288.0, "metrics/particle/momentum/class_2/efficiency": 0.49966511130332947, "metrics/particle/momentum/class_2/event/energy_bias": -8.815370559692383, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.710880279541016, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.954057693481445, "metrics/particle/momentum/class_2/event/px_bias": -0.8859754800796509, "metrics/particle/momentum/class_2/event/py_bias": -0.43249937891960144, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.611867904663086, "metrics/particle/momentum/class_2/event/pz_bias": -1.0569038391113281, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.961750030517578, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017853250727057457, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15961013734340668, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1590755730867386, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.014002719894051552, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19916656613349915, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012370113981887698, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2595677077770233, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.009772171266376972, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9253823757171631, "metrics/particle/momentum/class_3/accepted/samples": 12126.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.24015668034553528, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5718109607696533, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5803613662719727, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4186426401138306, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1523527055978775, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.47629165649414, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008224349468946457, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5513572692871094, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.008258743211627007, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4531092643737793, "metrics/particle/momentum/class_3/assigned/samples": 14834.0, "metrics/particle/momentum/class_3/efficiency": 0.7806605100631714, "metrics/particle/momentum/class_3/event/energy_bias": 1.747236728668213, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.846808433532715, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.764359951019287, "metrics/particle/momentum/class_3/event/px_bias": -0.2252112627029419, "metrics/particle/momentum/class_3/event/py_bias": -0.4675638973712921, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.897125720977783, "metrics/particle/momentum/class_3/event/pz_bias": 0.18343941867351532, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.824246406555176, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01850009337067604, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17071570456027985, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16814391314983368, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.10847257077693939, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2122163623571396, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6880314946174622, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.15609382092952728, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4076108932495117, "metrics/particle/momentum/class_4/accepted/samples": 252.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3415253460407257, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.100203037261963, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.712676525115967, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.6407339572906494, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.25770309567451477, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.202339172363281, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.002801120514050126, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7625958323478699, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.08862757682800293, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.5766085386276245, "metrics/particle/momentum/class_4/assigned/samples": 357.0, "metrics/particle/momentum/class_4/efficiency": 0.671999990940094, "metrics/particle/momentum/class_4/event/energy_bias": -0.8517771363258362, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3171322345733643, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7652140855789185, "metrics/particle/momentum/class_4/event/px_bias": -0.00271611288189888, "metrics/particle/momentum/class_4/event/py_bias": 0.10429247468709946, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.155779242515564, "metrics/particle/momentum/class_4/event/pz_bias": -0.1729508340358734, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.470696449279785, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012619967572391033, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14262478053569794, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1413598656654358, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.021037762984633446, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17600178718566895, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03896103799343109, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5227851867675781, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.07270000874996185, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.3610923290252686, "metrics/particle/momentum/class_5/accepted/samples": 154.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.020603235810995102, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.17412389814853668, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.175376757979393, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.04304401949048042, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05487804859876633, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2455468624830246, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04268292710185051, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6232647895812988, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.02649877965450287, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.5432815551757812, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9390243887901306, "metrics/particle/momentum/class_5/event/energy_bias": -0.7071176767349243, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9107086658477783, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4835562705993652, "metrics/particle/momentum/class_5/event/px_bias": 0.22155264019966125, "metrics/particle/momentum/class_5/event/py_bias": -0.006333403289318085, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9375686049461365, "metrics/particle/momentum/class_5/event/pz_bias": 0.1755094975233078, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.081413507461548, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.323625385761261, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.32510125637054443, "metrics/set_slot/angular_recall_all_dr0p10": 0.9604519009590149, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8503584265708923, "metrics/set_slot/soft_count_bias_mean": -0.9903424382209778}, "step_train_time": 8.233779668807983, "step_valid_time": 12454.93537735939, "step_total_time": 12463.169157266617, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0197166266487439, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13211351058558796, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2917, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2900, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9535796011768551, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9636603898249092, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9585934932632271, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.036339610175090846, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9480222294867604, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9580442682523951, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9530069010844562, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9535796011768551, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.013409334910813, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13507519797504475, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2254, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2159, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2155, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9448577680525164, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9578527062999113, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.951310861423221, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04214729370008873, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9431072210065645, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.956078083407276, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9495483586693104, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9448577680525164, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0189098699615207, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1355891187388134, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1304, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1229, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1225, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9296520423600605, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9424846625766872, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9360243716679361, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.057515337423312884, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.926626323751891, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9394171779141104, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9329779131759329, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9296520423600605} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_41000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_41000.json new file mode 100644 index 0000000000000000000000000000000000000000..93619add8a2c570736842112bf984fa66719bf68 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_41000.json @@ -0,0 +1 @@ +{"epoch": 1.9420234937476317, "train": {"Classification_binary": 0.16175585985183716, "Cardinality": 0.10313668102025986, "Classification": 0.09295833855867386, "Regression_pt": 0.11732126772403717, "Regression_eta": 0.0014267859514802694, "Regression_sin_phi": 0.0006923245382495224, "Regression_cos_phi": 0.0007131530437618494, "Regression_energy": 0.11644928902387619, "HitGrouping": 0.1846400648355484, "Auxiliary": 0.23005938529968262, "Total": 1.0062634944915771, "Total_unweighted": 0.491317018691916}, "train_group": {}, "valid": {"Classification_binary": 0.15260722239812216, "Cardinality": 0.12027339140574138, "Classification": 0.08919620513916016, "Regression_pt": 0.11665179332097371, "Regression_eta": 0.0013006098257998626, "Regression_sin_phi": 0.0006642498386402925, "Regression_cos_phi": 0.0006549387859801451, "Regression_energy": 0.11572144428888957, "HitGrouping": 0.19053316116333008, "Auxiliary": 0.223762313524882, "Total": 1.0002044836680095, "Total_unweighted": 0.4767964635975659, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8261994123458862, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.512464165687561, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.34910571575164795, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5172674059867859, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5134636163711548, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4003974497318268, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7689710855484009, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4552675783634186, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3489660322666168, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4579361081123352, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4648723602294922, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.48338258266448975, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8416699767112732, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4741254150867462, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3258504867553711, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4772433042526245, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4670611321926117, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3704698085784912, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8313390016555786, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4842761754989624, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3346903622150421, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4878697395324707, "metrics/hit_grouping/calorimeter/all/object_purity": 0.48094895482063293, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.388302743434906, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003026670543476939, "metrics/hit_grouping/calorimeter/background_recall": 0.13187788426876068, "metrics/hit_grouping/calorimeter/completeness": 0.5469669103622437, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5533089637756348, "metrics/hit_grouping/calorimeter/energy_purity": 0.5460265278816223, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.37650126218795776, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8571537733078003, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.512404203414917, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2600705623626709, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5171413421630859, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5061574578285217, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.32441967725753784, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8289114236831665, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5025768280029297, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3648088574409485, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5063064694404602, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5012480020523071, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4015430808067322, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7811487913131714, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4084872305393219, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4564965069293976, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4096120297908783, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4090995788574219, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5052204132080078, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8198349475860596, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5357525944709778, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.41730815172195435, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5425348877906799, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5386532545089722, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4237430691719055, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7891859412193298, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5699142217636108, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5681934952735901, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5674747824668884, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5643558502197266, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4559958875179291, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8480899930000305, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.42159026861190796, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22263780236244202, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4226514995098114, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4121791124343872, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.34410446882247925, "metrics/hit_grouping/calorimeter/purity": 0.5389345288276672, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8060530424118042, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08040793240070343, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1184210553765297, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08073978871107101, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05096597969532013, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04135338217020035, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8048083782196045, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.20155678689479828, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10417892783880234, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.20015498995780945, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.19175489246845245, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2842848598957062, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8331316113471985, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5055692791938782, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3502098023891449, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5094759464263916, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5030044913291931, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3993690609931946, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7440488338470459, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4060553312301636, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5079230070114136, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5246160626411438, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5519339442253113, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7716497182846069, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.279083788394928, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.47766485810279846, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.46934404969215393, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4177215099334717, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.672025203704834, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4386242628097534, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.43279945850372314, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.45218440890312195, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6581522226333618, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7015201449394226, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.416829913854599, "metrics/hit_grouping/tracker/all/object_completeness": 0.45920243859291077, "metrics/hit_grouping/tracker/all/object_purity": 0.4757135212421417, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6077470779418945, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.015290130861103535, "metrics/hit_grouping/tracker/background_recall": 0.12949952483177185, "metrics/hit_grouping/tracker/completeness": 0.4571652412414551, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4659118950366974, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7137636542320251, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4061897099018097, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5089147090911865, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5252900719642639, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5996843576431274, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.697957456111908, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4254855811595917, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45038706064224243, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.47176024317741394, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6113632321357727, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6798750162124634, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4301172196865082, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3657028377056122, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.37766265869140625, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6207394003868103, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.726215660572052, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43814215064048767, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.49313122034072876, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5051702857017517, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5669525265693665, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7234340906143188, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.47598522901535034, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46227237582206726, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4338644742965698, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5652709603309631, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.665486752986908, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37851861119270325, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4145434498786926, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4449210464954376, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6683960556983948, "metrics/hit_grouping/tracker/purity": 0.43312615156173706, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.881668210029602, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03506787493824959, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.0617985837161541, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.06263777613639832, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06447964161634445, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6678019165992737, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2627774775028229, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.32673922181129456, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3390456438064575, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5379452705383301, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6784413456916809, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4901796281337738, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5325332880020142, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.551825761795044, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6959319710731506, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.552223265171051, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6531847715377808, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7538082003593445, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6235860586166382, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.003070471342653036, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.049111247062683105, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.22079849243164, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.27078694105148315, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.9124999046325684, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17916376888751984, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9140824675559998, "metrics/particle/class_1/pid_accuracy": 0.9755680561065674, "metrics/particle/class_2/efficiency": 0.5189715623855591, "metrics/particle/class_2/pid_accuracy": 0.7195547819137573, "metrics/particle/class_3/efficiency": 0.7846446633338928, "metrics/particle/class_3/pid_accuracy": 0.9738660454750061, "metrics/particle/class_4/efficiency": 0.7314629554748535, "metrics/particle/class_4/pid_accuracy": 0.44383561611175537, "metrics/particle/class_5/efficiency": 0.9444444179534912, "metrics/particle/class_5/pid_accuracy": 0.4503042697906494, "metrics/particle/count/bias_mean": -1.4615885019302368, "metrics/particle/count/mae": 3.642578125, "metrics/particle/count/prediction_mean": 69.193359375, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.0745161697268486, "metrics/particle/event/energy_response_mean": 0.9722490906715393, "metrics/particle/event/met_abs_error": 6.162404537200928, "metrics/particle/event/scalar_pt_relative_abs_error": 0.057000428438186646, "metrics/particle/event/scalar_pt_response_mean": 0.9993479251861572, "metrics/particle/event/vector_pt_closure": 0.05652514472603798, "metrics/particle/matched/delta_eta_abs_mean": 0.012684768997132778, "metrics/particle/matched/delta_phi_abs_mean": 0.011433368548750877, "metrics/particle/matched/delta_r_mean": 0.01911330781877041, "metrics/particle/matched/energy_relative_abs_mean": 0.13317511975765228, "metrics/particle/matched/pid_accuracy": 0.9545779228210449, "metrics/particle/matched/pt_relative_abs_mean": 0.13443590700626373, "metrics/particle/matching/duplicate_fraction": 0.054083045572042465, "metrics/particle/matching/efficiency": 0.8258481621742249, "metrics/particle/matching/f1": 0.8344793319702148, "metrics/particle/matching/purity": 0.843292772769928, "metrics/particle/matching/target_coverage_dr0p05": 0.8014392852783203, "metrics/particle/matching/target_coverage_dr0p10": 0.8701140880584717, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6443156599998474, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8536571860313416, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019147006794810295, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13387763500213623, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1357773095369339, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018607642501592636, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17509432137012482, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007389162667095661, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6579912304878235, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.05830838531255722, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.91854727268219, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8932.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.19506707787513733, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7033119797706604, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8052042126655579, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6678190231323242, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10893962532281876, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.883523941040039, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014415634796023369, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9212515950202942, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.031166106462478638, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7831313610076904, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10336.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8291867971420288, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.5349684357643127, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.784005641937256, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.4762585163116455, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.16651898622512817, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.20924696326255798, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.677546501159668, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.661570131778717, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.259016990661621, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023242807015776634, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14929096400737762, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1500251442193985, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.021814892068505287, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18943169713020325, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0251141544431448, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3289836645126343, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.17190098762512207, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.476651430130005, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1752.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4817017912864685, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1211777925491333, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.123910427093506, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.956860303878784, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.19557823240756989, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 42.83867263793945, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.061649661511182785, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.526777505874634, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.08376933634281158, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.733746528625488, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2352.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6352429389953613, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.997090816497803, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.747218132019043, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2276638746261597, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06948205083608627, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.0274907648563385, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.604732513427734, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.026725202798843384, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.009294509887695, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018843889236450195, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1318037062883377, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13299570977687836, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01425034087151289, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17349959909915924, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000557922525331378, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20807893574237823, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0025046702940016985, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6700475215911865, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19716.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1725764274597168, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9632074236869812, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7524967193603516, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.606519341468811, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10802509635686874, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.147071838378906, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0040366812609136105, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3711415231227875, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.012574377469718456, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.436926007270813, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22791.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.847307562828064, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.24228036403656, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.586496353149414, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.167232036590576, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.7405582666397095, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.35972779989242554, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.9578030109405518, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.0819912925362587, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.9456329345703125, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019186466932296753, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13342086970806122, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1347944140434265, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015966542065143585, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1749248206615448, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003980263136327267, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.40486979484558105, "metrics/particle/momentum/all/accepted/pz_bias": -0.02541443705558777, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4384920597076416, "metrics/particle/momentum/all/accepted/samples": 30400.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19962136447429657, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8979651927947998, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9250593185424805, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7801880836486816, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11409566551446915, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.996969223022461, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010879674926400185, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6743065714836121, "metrics/particle/momentum/all/assigned/pz_bias": -0.006555308122187853, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7473530769348145, "metrics/particle/momentum/all/assigned/samples": 35479.0, "metrics/particle/momentum/all/efficiency": 0.8261094093322754, "metrics/particle/momentum/all/event/energy_bias": -4.289778709411621, "metrics/particle/momentum/all/event/p3_error_mean": 17.97456169128418, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.716998100280762, "metrics/particle/momentum/all/event/px_bias": 0.5045573115348816, "metrics/particle/momentum/all/event/py_bias": 0.541483998298645, "metrics/particle/momentum/all/event/pz_abs_mean": 13.45935344696045, "metrics/particle/momentum/all/event/pz_bias": -0.7168360352516174, "metrics/particle/momentum/all/event/pz_residual_rms": 24.984975814819336, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019512807950377464, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10862646251916885, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11159659922122955, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.016547977924346924, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14726264774799347, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003577817464247346, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.39339616894721985, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.034678392112255096, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3284837007522583, "metrics/particle/momentum/class_1/accepted/samples": 16211.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10387758910655975, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.261077344417572, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3149579167366028, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.20289088785648346, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.051260024309158325, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.315781354904175, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006185567006468773, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5105301141738892, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.031631194055080414, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.194342613220215, "metrics/particle/momentum/class_1/assigned/samples": 17460.0, "metrics/particle/momentum/class_1/efficiency": 0.9137591123580933, "metrics/particle/momentum/class_1/event/energy_bias": 3.296180248260498, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.412437438964844, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.067030906677246, "metrics/particle/momentum/class_1/event/px_bias": 0.11161616444587708, "metrics/particle/momentum/class_1/event/py_bias": -0.4799167513847351, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.103280067443848, "metrics/particle/momentum/class_1/event/pz_bias": -0.9199048280715942, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.074325561523438, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027508972212672234, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1805497109889984, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18129464983940125, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.004684770945459604, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22248703241348267, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028442146256566048, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4915404319763184, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.09416055679321289, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7364978790283203, "metrics/particle/momentum/class_2/accepted/samples": 1547.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5858303904533386, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7724506258964539, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0347726345062256, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.719359278678894, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2700490951538086, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.854173183441162, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05728314071893692, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4481821060180664, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.030682427808642387, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.960682392120361, "metrics/particle/momentum/class_2/assigned/samples": 2444.0, "metrics/particle/momentum/class_2/efficiency": 0.5180844068527222, "metrics/particle/momentum/class_2/event/energy_bias": -8.325525283813477, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.982847213745117, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.696301460266113, "metrics/particle/momentum/class_2/event/px_bias": 0.39006856083869934, "metrics/particle/momentum/class_2/event/py_bias": 0.6764274835586548, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.12191104888916, "metrics/particle/momentum/class_2/event/pz_bias": -0.24593475461006165, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.711803436279297, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017759544774889946, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15980076789855957, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15930575132369995, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.016591522842645645, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19960227608680725, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012278978247195482, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.26461097598075867, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.007738051004707813, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9322146773338318, "metrics/particle/momentum/class_3/accepted/samples": 12216.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.24644498527050018, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6170488595962524, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5839873552322388, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.427217960357666, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15953488647937775, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.44670867919922, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008704318664968014, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5631228685379028, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.025742432102560997, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.489377737045288, "metrics/particle/momentum/class_3/assigned/samples": 15050.0, "metrics/particle/momentum/class_3/efficiency": 0.7864546179771423, "metrics/particle/momentum/class_3/event/energy_bias": 2.8381614685058594, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.955759048461914, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.647897720336914, "metrics/particle/momentum/class_3/event/px_bias": -0.12462647259235382, "metrics/particle/momentum/class_3/event/py_bias": 0.6087010502815247, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.050168991088867, "metrics/particle/momentum/class_3/event/pz_bias": 0.40106767416000366, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.402097702026367, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.020843178033828735, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1580348163843155, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15232180058956146, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07573339343070984, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19716261327266693, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0036900369450449944, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6000182628631592, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.04638209193944931, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4821704626083374, "metrics/particle/momentum/class_4/accepted/samples": 271.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3176427483558655, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.8223931789398193, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.4743456840515137, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.3897862434387207, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23545706272125244, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.88554573059082, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005540166050195694, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7336496710777283, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.018994824960827827, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7902709245681763, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.7226666808128357, "metrics/particle/momentum/class_4/event/energy_bias": -1.2097887992858887, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2946178913116455, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7259881496429443, "metrics/particle/momentum/class_4/event/px_bias": 0.09323357045650482, "metrics/particle/momentum/class_4/event/py_bias": -0.0850096046924591, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2019613981246948, "metrics/particle/momentum/class_4/event/pz_bias": -0.16531431674957275, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.418083190917969, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011554818600416183, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13410967588424683, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1344292014837265, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007517073769122362, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17658720910549164, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.019354838877916336, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4722015857696533, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.2731380760669708, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.0958335399627686, "metrics/particle/momentum/class_5/accepted/samples": 155.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08065423369407654, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.34850412607192993, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.36454108357429504, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.22782284021377563, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04268292710185051, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.569364309310913, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.030487805604934692, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7479462623596191, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.002512332983314991, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.876770496368408, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9451219439506531, "metrics/particle/momentum/class_5/event/energy_bias": -0.8888063430786133, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.004038095474243, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4562997817993164, "metrics/particle/momentum/class_5/event/px_bias": 0.03426539897918701, "metrics/particle/momentum/class_5/event/py_bias": -0.17871814966201782, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0628982782363892, "metrics/particle/momentum/class_5/event/pz_bias": 0.21325033903121948, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.6049883365631104, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3182666301727295, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3200782835483551, "metrics/set_slot/angular_recall_all_dr0p10": 0.9607743620872498, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8579234480857849, "metrics/set_slot/soft_count_bias_mean": 0.705195426940918}, "step_train_time": 5.192018508911133, "step_valid_time": 14999.037665128708, "step_total_time": 15004.229684114456, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0099342366064996, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12959625572384093, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2914, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2890, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.952598888525662, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9626693095474067, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9576076240552087, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.037330690452593324, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9447531873161163, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9547406673273868, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9497206703910615, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.952598888525662, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.008297566451182, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12858562149681807, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2260, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2163, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2160, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9466083150984683, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9570796460176991, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9518151815181518, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04292035398230089, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9452954048140044, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9557522123893806, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9504950495049505, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9466083150984683, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.012750199348567, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14521808354460775, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1244, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1240, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9409984871406959, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9438543247344461, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9424242424242424, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05614567526555387, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9379727685325264, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9408194233687405, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9393939393939394, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9409984871406959} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_42000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_42000.json new file mode 100644 index 0000000000000000000000000000000000000000..210eca7f67917c754362eac3ec8ba802cd01dd29 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_42000.json @@ -0,0 +1 @@ +{"epoch": 1.9893899204244032, "train": {"Classification_binary": 0.14127475023269653, "Cardinality": 0.10932309180498123, "Classification": 0.10149092972278595, "Regression_pt": 0.11357837915420532, "Regression_eta": 0.0013998575741425157, "Regression_sin_phi": 0.0006861276342533529, "Regression_cos_phi": 0.0006653812597505748, "Regression_energy": 0.11280472576618195, "HitGrouping": 0.18989163637161255, "Auxiliary": 0.22038374841213226, "Total": 0.9837199449539185, "Total_unweighted": 0.4719001513440162}, "train_group": {}, "valid": {"Classification_binary": 0.14905006686846414, "Cardinality": 0.12266212701797485, "Classification": 0.09001212318738301, "Regression_pt": 0.11997319261233012, "Regression_eta": 0.001329064058760802, "Regression_sin_phi": 0.0006424035333717862, "Regression_cos_phi": 0.0006774479988962412, "Regression_energy": 0.11909104386965434, "HitGrouping": 0.19038955370585123, "Auxiliary": 0.2232442299524943, "Total": 1.0030280749003093, "Total_unweighted": 0.4807753421288605, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.815712034702301, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5064899325370789, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3597993850708008, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5088310837745667, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.506956934928894, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4278413951396942, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7558615207672119, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4455678164958954, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3497045934200287, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4466196298599243, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4563275873661041, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5118168592453003, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8381748199462891, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4749823808670044, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3322378993034363, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.47723400592803955, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4662575423717499, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3791714906692505, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8249790668487549, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4822440445423126, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.34194380044937134, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.48442965745925903, "metrics/hit_grouping/calorimeter/all/object_purity": 0.477816641330719, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.40421444177627563, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0029938018415123224, "metrics/hit_grouping/calorimeter/background_recall": 0.13122956454753876, "metrics/hit_grouping/calorimeter/completeness": 0.5474138259887695, "metrics/hit_grouping/calorimeter/energy_completeness": 0.557890772819519, "metrics/hit_grouping/calorimeter/energy_purity": 0.550535261631012, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.38261178135871887, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8492387533187866, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5061372518539429, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.26462221145629883, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.509641170501709, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5015316605567932, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3390987813472748, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8251685500144958, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5019978880882263, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3743666410446167, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5043560266494751, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.49621158838272095, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.41559189558029175, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7753229737281799, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4136257469654083, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4669373631477356, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4129493832588196, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4109305143356323, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5255220532417297, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8137645721435547, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.534834086894989, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4290955066680908, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5390472412109375, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5334889888763428, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.43980032205581665, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7878848314285278, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5796225070953369, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5656201839447021, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5759919881820679, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5589446425437927, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.47967061400413513, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8408322334289551, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4170239567756653, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22643938660621643, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4178353250026703, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4113900065422058, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3589429259300232, "metrics/hit_grouping/calorimeter/purity": 0.5393571257591248, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8196406364440918, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08290833234786987, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.129699245095253, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08091210573911667, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05062316358089447, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.043233081698417664, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7952476143836975, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19323179125785828, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10300176590681076, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19271153211593628, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1845782995223999, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.29605650901794434, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8266133069992065, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5037907361984253, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.35783639550209045, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5061852931976318, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5000371336936951, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.41572484374046326, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7466464042663574, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4188034236431122, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5133556723594666, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5243386030197144, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5554106831550598, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7768009901046753, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2881253659725189, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4818864166736603, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4746728837490082, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4056660532951355, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6779342293739319, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4379047453403473, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.43973156809806824, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.45401298999786377, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6418909430503845, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7063554525375366, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.420970618724823, "metrics/hit_grouping/tracker/all/object_completeness": 0.4654734432697296, "metrics/hit_grouping/tracker/all/object_purity": 0.47715336084365845, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5978628396987915, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013873888179659843, "metrics/hit_grouping/tracker/background_recall": 0.1199873685836792, "metrics/hit_grouping/tracker/completeness": 0.46476972103118896, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4645961821079254, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7194836735725403, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.411976158618927, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5150356292724609, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5266339182853699, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5858319997787476, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7003955245018005, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4302051067352295, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4569087028503418, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4749424159526825, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6104556322097778, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6852721571922302, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4302975535392761, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3720337152481079, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.37756896018981934, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.610099196434021, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7319920063018799, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44128385186195374, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4980728328227997, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5041937232017517, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5558291673660278, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7195478677749634, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.467980295419693, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46639284491539, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4470328390598297, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5591133236885071, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.670661985874176, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3861353397369385, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4229282736778259, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4473995566368103, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6594547033309937, "metrics/hit_grouping/tracker/purity": 0.43969836831092834, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8795199990272522, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.0354449488222599, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.0634804517030716, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.060423821210861206, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06598793715238571, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6670910120010376, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2694889008998871, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3283759355545044, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3394567370414734, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5585957765579224, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6849141120910645, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.49460017681121826, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5399876236915588, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5539193153381348, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6810474991798401, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5390366911888123, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6305081248283386, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7252833247184753, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6436425447463989, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.010116064921021461, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06916775554418564, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.63998794555664, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.32286837697029114, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.887499988079071, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.231245219707489, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9051951169967651, "metrics/particle/class_1/pid_accuracy": 0.9766976237297058, "metrics/particle/class_2/efficiency": 0.5015290379524231, "metrics/particle/class_2/pid_accuracy": 0.716576337814331, "metrics/particle/class_3/efficiency": 0.7797258496284485, "metrics/particle/class_3/pid_accuracy": 0.9770095348358154, "metrics/particle/class_4/efficiency": 0.7004008293151855, "metrics/particle/class_4/pid_accuracy": 0.4391988515853882, "metrics/particle/class_5/efficiency": 0.9578543901443481, "metrics/particle/class_5/pid_accuracy": 0.44999998807907104, "metrics/particle/count/bias_mean": -2.94921875, "metrics/particle/count/mae": 4.154947757720947, "metrics/particle/count/prediction_mean": 67.70572662353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.0771988183259964, "metrics/particle/event/energy_response_mean": 0.9591119289398193, "metrics/particle/event/met_abs_error": 6.301904201507568, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05777445808053017, "metrics/particle/event/scalar_pt_response_mean": 0.980867326259613, "metrics/particle/event/vector_pt_closure": 0.057164449244737625, "metrics/particle/matched/delta_eta_abs_mean": 0.012636041268706322, "metrics/particle/matched/delta_phi_abs_mean": 0.01126827858388424, "metrics/particle/matched/delta_r_mean": 0.018939966335892677, "metrics/particle/matched/energy_relative_abs_mean": 0.13694213330745697, "metrics/particle/matched/pid_accuracy": 0.956645667552948, "metrics/particle/matched/pt_relative_abs_mean": 0.13774651288986206, "metrics/particle/matching/duplicate_fraction": 0.044520944356918335, "metrics/particle/matching/efficiency": 0.8178408741950989, "metrics/particle/matching/f1": 0.8352735042572021, "metrics/particle/matching/purity": 0.8534654974937439, "metrics/particle/matching/target_coverage_dr0p05": 0.7966846823692322, "metrics/particle/matching/target_coverage_dr0p10": 0.8617658615112305, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6299504041671753, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8441939949989319, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019135793671011925, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13730143010616302, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13997432589530945, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015492278151214123, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17929315567016602, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007372122257947922, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6596498489379883, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07579614222049713, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8599727153778076, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8817.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.17892128229141235, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7273947596549988, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7771087884902954, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6308541893959045, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10371318459510803, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.596674919128418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014281493611633778, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.930440366268158, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.11395979672670364, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.889683246612549, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10153.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8185109496116638, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.4180288314819336, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.294269561767578, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.6910157203674316, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.007314644753932953, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.0014443807303905487, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.148652076721191, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.850700616836548, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.123504638671875, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02358241379261017, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1482410430908203, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.150077223777771, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.00024209587718360126, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18887034058570862, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023795705288648605, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2684646844863892, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.25155603885650635, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.333369731903076, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1723.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4407196342945099, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2546793222427368, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.980586051940918, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.790780544281006, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18739053606987, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 30.558767318725586, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.062171630561351776, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4081649780273438, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.6312404274940491, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.432600975036621, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2284.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6247280836105347, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.124365329742432, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.739301681518555, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.222644329071045, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.00364656001329422, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.007606374099850655, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.596711158752441, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.5060553550720215, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.47671890258789, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01858770288527012, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13588309288024902, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13613846898078918, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.021629581227898598, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1753409504890442, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00041040373616851866, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.21066468954086304, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0009034552494995296, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6865259408950806, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19493.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17332933843135834, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.885351836681366, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6416904926300049, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4949013292789459, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10972689837217331, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.194296836853027, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0033693916629999876, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.37150508165359497, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0055604297667741776, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5452964305877686, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22556.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8377240300178528, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.9116913676261902, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.184011459350586, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.695311069488525, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.7118284702301025, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.3693965673446655, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.205438613891602, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.9125083088874817, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.150851726531982, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019035156816244125, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13700847327709198, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13806426525115967, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01860080659389496, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17730939388275146, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0037958244793117046, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4031626880168915, "metrics/particle/momentum/all/accepted/pz_bias": -0.0360974445939064, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3996429443359375, "metrics/particle/momentum/all/accepted/samples": 30033.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19240443408489227, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8636276125907898, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8336414694786072, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6841996908187866, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11305118352174759, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.751269340515137, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010373502969741821, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6666098237037659, "metrics/particle/momentum/all/assigned/pz_bias": -0.07785008102655411, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.75095796585083, "metrics/particle/momentum/all/assigned/samples": 34993.0, "metrics/particle/momentum/all/efficiency": 0.8161363005638123, "metrics/particle/momentum/all/event/energy_bias": -8.454086303710938, "metrics/particle/momentum/all/event/p3_error_mean": 19.32256507873535, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.425371170043945, "metrics/particle/momentum/all/event/px_bias": 0.7008672952651978, "metrics/particle/momentum/all/event/py_bias": 0.3784473240375519, "metrics/particle/momentum/all/event/pz_abs_mean": 14.424171447753906, "metrics/particle/momentum/all/event/pz_bias": -6.26926326751709, "metrics/particle/momentum/all/event/pz_residual_rms": 25.308895111083984, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019073886796832085, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1169167309999466, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11868760734796524, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.03109663538634777, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15436452627182007, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003246753243729472, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.39823171496391296, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.04111606627702713, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2947900295257568, "metrics/particle/momentum/class_1/accepted/samples": 16016.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10171093791723251, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2906563878059387, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.34092891216278076, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23259428143501282, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.056555863469839096, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.527026176452637, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006636657286435366, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5422545671463013, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.07462549954652786, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3345656394958496, "metrics/particle/momentum/class_1/assigned/samples": 17328.0, "metrics/particle/momentum/class_1/efficiency": 0.9027675986289978, "metrics/particle/momentum/class_1/event/energy_bias": 4.390416622161865, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.202165603637695, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.217879295349121, "metrics/particle/momentum/class_1/event/px_bias": 1.9816725254058838, "metrics/particle/momentum/class_1/event/py_bias": 0.6724657416343689, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.695901870727539, "metrics/particle/momentum/class_1/event/pz_bias": -1.5587384700775146, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.691555976867676, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.028280923143029213, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1820475459098816, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18188594281673431, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0018147989176213741, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22385936975479126, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028704939410090446, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4785940647125244, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.16345715522766113, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.592634439468384, "metrics/particle/momentum/class_2/accepted/samples": 1498.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5334829092025757, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5448775887489319, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7954806089401245, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4722321033477783, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2632966935634613, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.551987886428833, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0531868115067482, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.2766048908233643, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.3042186200618744, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.225555896759033, "metrics/particle/momentum/class_2/assigned/samples": 2275.0, "metrics/particle/momentum/class_2/efficiency": 0.501674473285675, "metrics/particle/momentum/class_2/event/energy_bias": -10.136019706726074, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.405824661254883, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.173788070678711, "metrics/particle/momentum/class_2/event/px_bias": -1.0789363384246826, "metrics/particle/momentum/class_2/event/py_bias": -0.41960418224334717, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.179351806640625, "metrics/particle/momentum/class_2/event/pz_bias": -3.9147353172302246, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.64923095703125, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01787799410521984, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1572660505771637, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15767702460289001, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0027950070798397064, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19722352921962738, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011566424509510398, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2597963809967041, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0123190488666296, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.952811062335968, "metrics/particle/momentum/class_3/accepted/samples": 12104.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.24480699002742767, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5345752239227295, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.356781244277954, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.183908224105835, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15352027118206024, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.222627639770508, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007800417020916939, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5518670082092285, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.05006412789225578, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6530489921569824, "metrics/particle/momentum/class_3/assigned/samples": 14871.0, "metrics/particle/momentum/class_3/efficiency": 0.7792441844940186, "metrics/particle/momentum/class_3/event/energy_bias": -0.7615980505943298, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.159387588500977, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.818418502807617, "metrics/particle/momentum/class_3/event/px_bias": -0.4319356679916382, "metrics/particle/momentum/class_3/event/py_bias": 0.04629091918468475, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.10568380355835, "metrics/particle/momentum/class_3/event/pz_bias": -0.7012753486633301, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.409040451049805, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.020148595795035362, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1600380390882492, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1559906303882599, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08214960992336273, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2017407864332199, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0038910505827516317, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6241921782493591, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.01659824326634407, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.351983666419983, "metrics/particle/momentum/class_4/accepted/samples": 257.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.31239500641822815, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.050478935241699, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.4831275939941406, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.398979425430298, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24788732826709747, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.833147048950195, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014084506779909134, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7911268472671509, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.20109601318836212, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9979476928710938, "metrics/particle/momentum/class_4/assigned/samples": 355.0, "metrics/particle/momentum/class_4/efficiency": 0.6853333115577698, "metrics/particle/momentum/class_4/event/energy_bias": -1.319056510925293, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2616045475006104, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7168831825256348, "metrics/particle/momentum/class_4/event/px_bias": 0.13748204708099365, "metrics/particle/momentum/class_4/event/py_bias": 0.09054792672395706, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.160643219947815, "metrics/particle/momentum/class_4/event/pz_bias": -0.07563827931880951, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.3892951011657715, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014286202378571033, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15728668868541718, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15510033071041107, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.05297243222594261, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19841913878917694, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025316456332802773, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.330241322517395, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.17319808900356293, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.745598793029785, "metrics/particle/momentum/class_5/accepted/samples": 158.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03208785504102707, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2514723241329193, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.25047600269317627, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.15208448469638824, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03658536449074745, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.8239049911499023, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03658536449074745, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6070188283920288, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4017391800880432, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6347947120666504, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9634146094322205, "metrics/particle/momentum/class_5/event/energy_bias": -0.6278282999992371, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.860795259475708, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.416388750076294, "metrics/particle/momentum/class_5/event/px_bias": 0.09258484095335007, "metrics/particle/momentum/class_5/event/py_bias": -0.011253111064434052, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9184824824333191, "metrics/particle/momentum/class_5/event/pz_bias": -0.018876729533076286, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.0873682498931885, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3233622908592224, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3253166675567627, "metrics/set_slot/angular_recall_all_dr0p10": 0.9599082469940186, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8500083088874817, "metrics/set_slot/soft_count_bias_mean": -0.8849605917930603}, "step_train_time": 4.339835166931152, "step_valid_time": 17468.57215166092, "step_total_time": 17472.911987304688, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974471037414978, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1306418045600073, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2924, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2904, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9558679306963059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9634266886326195, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9596324253363965, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03657331136738056, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9493298463550179, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.956836902800659, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9530685920577617, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9558679306963059, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9868501547311732, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13470567846840598, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2261, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2160, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2157, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9452954048140044, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.955329500221141, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9502859656841179, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04467049977885891, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9439824945295405, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540026536930561, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9489661240651122, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9452954048140044, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.995881229857971, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13730608417406498, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1303, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1236, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1233, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9349470499243571, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9485801995395242, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9417142857142857, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.051419800460475826, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.93267776096823, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9462778204144282, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9394285714285714, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9349470499243571} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_43000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_43000.json new file mode 100644 index 0000000000000000000000000000000000000000..0250ffe383bed105232c31e2995ecca556fb7260 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_43000.json @@ -0,0 +1 @@ +{"epoch": 2.0367563471011745, "train": {"Classification_binary": 0.1606326401233673, "Cardinality": 0.10964961349964142, "Classification": 0.09871118515729904, "Regression_pt": 0.1063016727566719, "Regression_eta": 0.001224398030899465, "Regression_sin_phi": 0.0006533467094413936, "Regression_cos_phi": 0.0005740288179367781, "Regression_energy": 0.10555938631296158, "HitGrouping": 0.18715369701385498, "Auxiliary": 0.22376149892807007, "Total": 0.9799979329109192, "Total_unweighted": 0.47365665790857747}, "train_group": {}, "valid": {"Classification_binary": 0.14915943145751953, "Cardinality": 0.1172386606534322, "Classification": 0.08959242701530457, "Regression_pt": 0.1153971254825592, "Regression_eta": 0.0012451271371295054, "Regression_sin_phi": 0.0006508886193235716, "Regression_cos_phi": 0.0006406016570205489, "Regression_energy": 0.11442714929580688, "HitGrouping": 0.1865556240081787, "Auxiliary": 0.22140695651372275, "Total": 0.9825191497802734, "Total_unweighted": 0.4711127506646638, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8278389573097229, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5128211975097656, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.35355350375175476, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5168510675430298, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5073294043540955, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3964228332042694, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7712323069572449, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4512377381324768, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.33124077320098877, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.45402222871780396, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.46400657296180725, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4763663113117218, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8465653657913208, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.47705167531967163, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.32566535472869873, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.48076021671295166, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4663209915161133, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3583892583847046, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8350435495376587, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.48588401079177856, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3345470130443573, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4896181523799896, "metrics/hit_grouping/calorimeter/all/object_purity": 0.47856494784355164, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.37907108664512634, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0034250805620104074, "metrics/hit_grouping/calorimeter/background_recall": 0.1457323282957077, "metrics/hit_grouping/calorimeter/completeness": 0.5538094639778137, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5628881454467773, "metrics/hit_grouping/calorimeter/energy_purity": 0.5555817484855652, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3785015344619751, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8576579093933105, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5106183886528015, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.26098087430000305, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5150086283683777, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5027575492858887, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.31929904222488403, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8353948593139648, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5049309730529785, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.36285123229026794, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5092490911483765, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4958362877368927, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3868033289909363, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7885791659355164, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4162628650665283, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45603248476982117, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.41807082295417786, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4118371307849884, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.49315544962882996, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.825389564037323, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5405850410461426, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4156242609024048, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5472434759140015, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5382076501846313, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4104522466659546, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7932477593421936, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.581272542476654, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5620175004005432, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5765015482902527, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5702877044677734, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4580545425415039, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8498657941818237, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4187488257884979, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22478386759757996, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4205147325992584, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4068281054496765, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3376663327217102, "metrics/hit_grouping/calorimeter/purity": 0.5458948016166687, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7957124710083008, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08234520256519318, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12593984603881836, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08345864713191986, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05175727605819702, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03947368264198303, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8060401678085327, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19646409153938293, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09829311072826385, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19589346647262573, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.17980675399303436, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2683931589126587, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8371936678886414, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5075204372406006, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.35024043917655945, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5115212798118591, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5010664463043213, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3903641700744629, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7508183121681213, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40359264612197876, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5173035264015198, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5392523407936096, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5514993667602539, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.776444673538208, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2591922879219055, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4763057827949524, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4834027886390686, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.40928271412849426, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6821472644805908, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4254569113254547, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4423089027404785, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4669647514820099, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.641171395778656, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7102183103561401, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.406455934047699, "metrics/hit_grouping/tracker/all/object_completeness": 0.4678694009780884, "metrics/hit_grouping/tracker/all/object_purity": 0.49039629101753235, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5964826345443726, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.020052414387464523, "metrics/hit_grouping/tracker/background_recall": 0.1470239907503128, "metrics/hit_grouping/tracker/completeness": 0.4685208797454834, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4588521420955658, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7223281860351562, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3971593976020813, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5182915329933167, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5381656289100647, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5881991982460022, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7056565284729004, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4127790927886963, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45695242285728455, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4889434278011322, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.5990197658538818, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6898406744003296, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41929665207862854, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3749977648258209, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.39357873797416687, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6110008955001831, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7316266894340515, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.41631993651390076, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.496107816696167, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.524118185043335, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5601596236228943, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7357266545295715, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45073890686035156, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.473002165555954, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.45376503467559814, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5363300442695618, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6778138875961304, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3856937885284424, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.43023836612701416, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.453123539686203, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6544872522354126, "metrics/hit_grouping/tracker/purity": 0.44604185223579407, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8801220655441284, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02677224762737751, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.05708402767777443, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.06053784117102623, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06485671550035477, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6700373291969299, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.25916364789009094, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.33442583680152893, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.35658788681030273, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5565307140350342, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6893602013587952, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.47876447439193726, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5432922840118408, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5686891078948975, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6797045469284058, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5331018567085266, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6543841361999512, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7540503144264221, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6254971623420715, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.016050927340984344, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05102238059043884, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.58355140686035, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2644224464893341, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.9249998927116394, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17279930412769318, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9162375330924988, "metrics/particle/class_1/pid_accuracy": 0.9772490859031677, "metrics/particle/class_2/efficiency": 0.5047004222869873, "metrics/particle/class_2/pid_accuracy": 0.7060143351554871, "metrics/particle/class_3/efficiency": 0.7954222559928894, "metrics/particle/class_3/pid_accuracy": 0.9752095937728882, "metrics/particle/class_4/efficiency": 0.7354709506034851, "metrics/particle/class_4/pid_accuracy": 0.46457764506340027, "metrics/particle/class_5/efficiency": 0.938697338104248, "metrics/particle/class_5/pid_accuracy": 0.45510202646255493, "metrics/particle/count/bias_mean": -1.6608072519302368, "metrics/particle/count/mae": 3.6334636211395264, "metrics/particle/count/prediction_mean": 68.994140625, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.0735526829957962, "metrics/particle/event/energy_response_mean": 0.975942075252533, "metrics/particle/event/met_abs_error": 5.968992233276367, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05549571290612221, "metrics/particle/event/scalar_pt_response_mean": 0.9929583668708801, "metrics/particle/event/vector_pt_closure": 0.05473954603075981, "metrics/particle/matched/delta_eta_abs_mean": 0.011982637457549572, "metrics/particle/matched/delta_phi_abs_mean": 0.011402391828596592, "metrics/particle/matched/delta_r_mean": 0.01851959340274334, "metrics/particle/matched/energy_relative_abs_mean": 0.1327926367521286, "metrics/particle/matched/pid_accuracy": 0.9559967517852783, "metrics/particle/matched/pt_relative_abs_mean": 0.13266220688819885, "metrics/particle/matching/duplicate_fraction": 0.05037980526685715, "metrics/particle/matching/efficiency": 0.8302803039550781, "metrics/particle/matching/f1": 0.8401545882225037, "metrics/particle/matching/purity": 0.8502665758132935, "metrics/particle/matching/target_coverage_dr0p05": 0.8049038648605347, "metrics/particle/matching/target_coverage_dr0p10": 0.8711368441581726, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6531891226768494, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8568545579910278, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018389685079455376, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13275548815727234, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1322403848171234, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009371425956487656, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1723625212907791, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00737842358648777, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6427558660507202, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.04536214843392372, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.863842248916626, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8945.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1849031299352646, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7320756912231445, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8234670162200928, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6765488982200623, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10619126260280609, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.509857177734375, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014630365185439587, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9168797731399536, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06572536379098892, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.759439468383789, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10321.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8303936123847961, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.263523817062378, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.486555099487305, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.3278536796569824, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.17673644423484802, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.036098137497901917, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.374579906463623, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.4845714569091797, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.912190437316895, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022699683904647827, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1495390087366104, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14883290231227875, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.009294730611145496, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1889895647764206, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02440408617258072, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3141878843307495, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.014422090724110603, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.453392505645752, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1762.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5047203302383423, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.201351284980774, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.808837652206421, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.6368322372436523, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.19542178511619568, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 77.1773681640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.065705806016922, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6691360473632812, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.08794359117746353, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.541356086730957, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2359.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6388687491416931, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.808398246765137, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.511558532714844, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.229922890663147, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.08132864534854889, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.13661488890647888, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.350691795349121, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.7924129962921143, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.781417846679688, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018287135288119316, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1304614394903183, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1306966096162796, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.003840198740363121, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1701260507106781, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004537662607617676, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20427513122558594, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.013422484509646893, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6539910435676575, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19834.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1732487827539444, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8945412635803223, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6759076714515686, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5201159119606018, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10378145426511765, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.970223426818848, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0032939522061496973, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.367314875125885, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.012109613046050072, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5298408269882202, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22769.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8523787260055542, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.24351723492145538, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.976561069488525, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.740394592285156, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.38834285736083984, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3284063935279846, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.486943244934082, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.4629305601119995, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.644261360168457, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01857174187898636, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13223397731781006, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1321951001882553, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.005774900782853365, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17192450165748596, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003863658756017685, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.39673373103141785, "metrics/particle/momentum/all/accepted/pz_bias": 0.02283480018377304, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.408281683921814, "metrics/particle/momentum/all/accepted/samples": 30541.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.1987001746892929, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8676563501358032, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9273545742034912, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7730674147605896, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1105813980102539, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 20.873119354248047, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010747835040092468, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6804987788200378, "metrics/particle/momentum/all/assigned/pz_bias": 0.032766345888376236, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.92832612991333, "metrics/particle/momentum/all/assigned/samples": 35449.0, "metrics/particle/momentum/all/efficiency": 0.8299410343170166, "metrics/particle/momentum/all/event/energy_bias": -3.3013572692871094, "metrics/particle/momentum/all/event/p3_error_mean": 18.403486251831055, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.808273315429688, "metrics/particle/momentum/all/event/px_bias": 0.13027778267860413, "metrics/particle/momentum/all/event/py_bias": -0.5011194944381714, "metrics/particle/momentum/all/event/pz_abs_mean": 13.851346969604492, "metrics/particle/momentum/all/event/pz_bias": 1.155089020729065, "metrics/particle/momentum/all/event/pz_residual_rms": 25.028493881225586, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.018621856346726418, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10699419677257538, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10836853086948395, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009656225331127644, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14371894299983978, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0030793866608291864, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.38580602407455444, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.03111797571182251, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3151068687438965, "metrics/particle/momentum/class_1/accepted/samples": 16237.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10133887827396393, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2954433560371399, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3404736816883087, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.22386309504508972, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05186627060174942, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.3806099891662598, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006640714593231678, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.543724536895752, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.052362069487571716, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6495137214660645, "metrics/particle/momentum/class_1/assigned/samples": 17468.0, "metrics/particle/momentum/class_1/efficiency": 0.9152246117591858, "metrics/particle/momentum/class_1/event/energy_bias": 4.334659099578857, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.732540130615234, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.262807846069336, "metrics/particle/momentum/class_1/event/px_bias": 1.0197921991348267, "metrics/particle/momentum/class_1/event/py_bias": -0.353628933429718, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.310340881347656, "metrics/particle/momentum/class_1/event/pz_bias": 1.9027611017227173, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.883771896362305, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027658158913254738, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17912152409553528, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18003003299236298, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.009490150958299637, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2209285944700241, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03030303120613098, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5202505588531494, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0029870544094592333, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6983325481414795, "metrics/particle/momentum/class_2/accepted/samples": 1485.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6411082148551941, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.769253671169281, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0200891494750977, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6710872054100037, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2924292981624603, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.007906913757324, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.056156404316425323, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4782800674438477, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05639294162392616, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.72158145904541, "metrics/particle/momentum/class_2/assigned/samples": 2404.0, "metrics/particle/momentum/class_2/efficiency": 0.49732083082199097, "metrics/particle/momentum/class_2/event/energy_bias": -10.80311107635498, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.07532024383545, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.752784729003906, "metrics/particle/momentum/class_2/event/px_bias": -1.052231788635254, "metrics/particle/momentum/class_2/event/py_bias": -0.05033241957426071, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.033466339111328, "metrics/particle/momentum/class_2/event/pz_bias": -1.243240237236023, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.907482147216797, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01752776838839054, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15884257853031158, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15700778365135193, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0007097428315319121, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1963856816291809, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012103606713935733, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2569694221019745, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.012591809034347534, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8898718953132629, "metrics/particle/momentum/class_3/accepted/samples": 12393.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.23955057561397552, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5116760730743408, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.568599820137024, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3997231721878052, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14782088994979858, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 31.70611572265625, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008038799278438091, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.540842592716217, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.003809247398748994, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.573472499847412, "metrics/particle/momentum/class_3/assigned/samples": 15052.0, "metrics/particle/momentum/class_3/efficiency": 0.797849714756012, "metrics/particle/momentum/class_3/event/energy_bias": 4.6818318367004395, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.08609390258789, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.706182956695557, "metrics/particle/momentum/class_3/event/px_bias": -0.09106209129095078, "metrics/particle/momentum/class_3/event/py_bias": 0.14116215705871582, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.098973274230957, "metrics/particle/momentum/class_3/event/pz_bias": 0.3798729479312897, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.697566032409668, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.018026454374194145, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17407475411891937, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1666136384010315, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07597517967224121, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21761874854564667, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0036231884732842445, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6075648665428162, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0176311656832695, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.438844919204712, "metrics/particle/momentum/class_4/accepted/samples": 276.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.33040982484817505, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.6616477966308594, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.3065829277038574, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.2052385807037354, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.20718231797218323, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.133299827575684, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6454779505729675, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0063581946305930614, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.457457423210144, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.7360000014305115, "metrics/particle/momentum/class_4/event/energy_bias": -1.0603500604629517, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2276158332824707, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6555334329605103, "metrics/particle/momentum/class_4/event/px_bias": 0.12854896485805511, "metrics/particle/momentum/class_4/event/py_bias": -0.023406609892845154, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1735458374023438, "metrics/particle/momentum/class_4/event/pz_bias": -0.1784270703792572, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.774456977844238, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.010448053479194641, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.12477975338697433, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12441939115524292, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.026073234155774117, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.15993282198905945, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.046666666865348816, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6161918640136719, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.2376958280801773, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.456662893295288, "metrics/particle/momentum/class_5/accepted/samples": 150.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0428660586476326, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.18524612486362457, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.17519740760326385, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08469479531049728, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.06748466193675995, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2839165925979614, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04907975345849991, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7976100444793701, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.31697216629981995, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.674807071685791, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9146341681480408, "metrics/particle/momentum/class_5/event/energy_bias": -0.45438748598098755, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.0546538829803467, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5351587533950806, "metrics/particle/momentum/class_5/event/px_bias": 0.1252303570508957, "metrics/particle/momentum/class_5/event/py_bias": -0.2149137407541275, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0721935033798218, "metrics/particle/momentum/class_5/event/pz_bias": 0.29412224888801575, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.494556188583374, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3102014362812042, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3117961883544922, "metrics/set_slot/angular_recall_all_dr0p10": 0.9607375264167786, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8590107560157776, "metrics/set_slot/soft_count_bias_mean": 0.2782825529575348}, "step_train_time": 4.656005859375, "step_valid_time": 20016.66392159462, "step_total_time": 20021.319927930832, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0003640808169016, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.129263408882331, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3034, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2914, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2893, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.952598888525662, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.96044825313118, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9565074675857541, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03955174686882004, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9457338999673096, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9535266974291364, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9496143115050057, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.952598888525662, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9938521808592543, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1301858400160275, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2265, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2160, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2154, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9452954048140044, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9536423841059603, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9494505494505494, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046357615894039736, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9426695842450766, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9509933774834437, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9468131868131868, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9452954048140044, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036007518584165, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12556663297911796, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1308, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1237, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1236, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9357034795763994, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9457186544342507, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9406844106463879, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054281345565749234, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9349470499243571, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.944954128440367, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9399239543726235, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9357034795763994} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_44000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_44000.json new file mode 100644 index 0000000000000000000000000000000000000000..6b7a9ed8438279239ad1f4d2be599eb00a3b52b1 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_44000.json @@ -0,0 +1 @@ +{"epoch": 2.0841227737779464, "train": {"Classification_binary": 0.14213693141937256, "Cardinality": 0.10778286308050156, "Classification": 0.09557124972343445, "Regression_pt": 0.10896731913089752, "Regression_eta": 0.001104469527490437, "Regression_sin_phi": 0.0006177483592182398, "Regression_cos_phi": 0.0005771814030595124, "Regression_energy": 0.1082983911037445, "HitGrouping": 0.18083971738815308, "Auxiliary": 0.21476443111896515, "Total": 0.9390872716903687, "Total_unweighted": 0.4572732906672172}, "train_group": {}, "valid": {"Classification_binary": 0.14592576026916504, "Cardinality": 0.11946831146876018, "Classification": 0.08828842639923096, "Regression_pt": 0.11528791983922322, "Regression_eta": 0.001227060177673896, "Regression_sin_phi": 0.0006511831500877937, "Regression_cos_phi": 0.0006168135053788623, "Regression_energy": 0.11422664920488994, "HitGrouping": 0.1873767375946045, "Auxiliary": 0.22040992975234985, "Total": 0.9779377778371176, "Total_unweighted": 0.4662238125456497, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8260103464126587, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5093353390693665, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.34645596146583557, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5137486457824707, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5063741207122803, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.40777894854545593, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7735563516616821, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4477708637714386, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3382570147514343, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4503440260887146, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.447115421295166, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.47267356514930725, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.846062183380127, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.47446951270103455, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3211756646633148, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.47812625765800476, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4633352756500244, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.36255496740341187, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8343581557273865, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.48295941948890686, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.33016055822372437, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4867612421512604, "metrics/hit_grouping/calorimeter/all/object_purity": 0.47511473298072815, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3848050534725189, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0032353377901017666, "metrics/hit_grouping/calorimeter/background_recall": 0.14316721260547638, "metrics/hit_grouping/calorimeter/completeness": 0.549888551235199, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5600455403327942, "metrics/hit_grouping/calorimeter/energy_purity": 0.5527181625366211, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.37386277318000793, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8586465716362, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5082806348800659, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2580223083496094, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.513201892375946, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.500815749168396, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3233955502510071, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8338848352432251, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.502860963344574, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3572086691856384, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5065503716468811, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4941721260547638, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.39774298667907715, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7853116989135742, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4112806022167206, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.44999998807907104, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4129132330417633, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.40351200103759766, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.49698376655578613, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.822504997253418, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5367434620857239, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.41153475642204285, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5427950024604797, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5338273048400879, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4164060652256012, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7968566417694092, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5803970694541931, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5620175004005432, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.577749490737915, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5599246621131897, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.45187854766845703, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8509111404418945, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.41651496291160583, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2195720225572586, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.41879141330718994, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.40514981746673584, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3445950150489807, "metrics/hit_grouping/calorimeter/purity": 0.5419267416000366, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8265955448150635, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08757030963897705, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12218045443296432, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.0869271382689476, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04810812696814537, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.043233081698417664, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8094227313995361, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19501745700836182, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10359034687280655, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.198217511177063, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1875424087047577, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.29546791315078735, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8357823491096497, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5043860673904419, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.345339834690094, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5082916617393494, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4970373809337616, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3950197696685791, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7532446980476379, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3964942693710327, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5138911604881287, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.529299259185791, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5459945201873779, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7780473828315735, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2694394290447235, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4733561873435974, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4759688079357147, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.40687161684036255, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.68537837266922, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4249532222747803, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4439951479434967, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4669713079929352, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6401640772819519, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7130817770957947, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4047195017337799, "metrics/hit_grouping/tracker/all/object_completeness": 0.46764618158340454, "metrics/hit_grouping/tracker/all/object_purity": 0.4867921769618988, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5939893126487732, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01799819990992546, "metrics/hit_grouping/tracker/background_recall": 0.1418929547071457, "metrics/hit_grouping/tracker/completeness": 0.4663388431072235, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.45568081736564636, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7254490256309509, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39426618814468384, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5177387595176697, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5358708500862122, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5864457488059998, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7079687714576721, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4194953739643097, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45648255944252014, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4830097258090973, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6002904176712036, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6927223205566406, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4115419387817383, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.37569740414619446, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3895958364009857, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6032461524009705, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7360900044441223, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.41954657435417175, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.49908024072647095, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5199534296989441, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5559989809989929, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.735545814037323, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4568965435028076, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4708670675754547, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.44220104813575745, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5400246381759644, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6791431903839111, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37609007954597473, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.42620348930358887, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4516752362251282, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6530522108078003, "metrics/hit_grouping/tracker/purity": 0.4430357813835144, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8799971342086792, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02526395209133625, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.05357585847377777, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.05687113106250763, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06598793715238571, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6710377931594849, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2586473822593689, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.32880324125289917, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3442199230194092, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5493030548095703, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6928691267967224, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.47686195373535156, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5441417694091797, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5660441517829895, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6771864891052246, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5408363342285156, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6205877065658569, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7140229344367981, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6321526169776917, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.00831648986786604, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05767781659960747, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.649160385131836, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3263373374938965, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.9124999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.23471419513225555, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9174581170082092, "metrics/particle/class_1/pid_accuracy": 0.9776119589805603, "metrics/particle/class_2/efficiency": 0.5026617050170898, "metrics/particle/class_2/pid_accuracy": 0.7174403071403503, "metrics/particle/class_3/efficiency": 0.7845572233200073, "metrics/particle/class_3/pid_accuracy": 0.9765938520431519, "metrics/particle/class_4/efficiency": 0.7244489192962646, "metrics/particle/class_4/pid_accuracy": 0.48962655663490295, "metrics/particle/class_5/efficiency": 0.961685836315155, "metrics/particle/class_5/pid_accuracy": 0.4621514081954956, "metrics/particle/count/bias_mean": -2.919921875, "metrics/particle/count/mae": 4.110025882720947, "metrics/particle/count/prediction_mean": 67.73502349853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07369960844516754, "metrics/particle/event/energy_response_mean": 0.9632623195648193, "metrics/particle/event/met_abs_error": 5.783754825592041, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05448569729924202, "metrics/particle/event/scalar_pt_response_mean": 0.9859825968742371, "metrics/particle/event/vector_pt_closure": 0.05428577587008476, "metrics/particle/matched/delta_eta_abs_mean": 0.011908028274774551, "metrics/particle/matched/delta_phi_abs_mean": 0.01124771498143673, "metrics/particle/matched/delta_r_mean": 0.01833057403564453, "metrics/particle/matched/energy_relative_abs_mean": 0.13123676180839539, "metrics/particle/matched/pid_accuracy": 0.9575046896934509, "metrics/particle/matched/pt_relative_abs_mean": 0.13214147090911865, "metrics/particle/matching/duplicate_fraction": 0.04218529164791107, "metrics/particle/matching/efficiency": 0.8261338472366333, "metrics/particle/matching/f1": 0.8435646295547485, "metrics/particle/matching/purity": 0.8617467880249023, "metrics/particle/matching/target_coverage_dr0p05": 0.8034572601318359, "metrics/particle/matching/target_coverage_dr0p10": 0.8661426901817322, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6491439938545227, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8516300320625305, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01840437762439251, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13272631168365479, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13435068726539612, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017095936462283134, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17444823682308197, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006829377729445696, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6281945109367371, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.026121582835912704, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.813909649848938, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8932.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.16891291737556458, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.57088303565979, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.5861068367958069, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.44586220383644104, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09649208933115005, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 4.299422264099121, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013658248819410801, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8814876079559326, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03154613822698593, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7008185386657715, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10177.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8291867971420288, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.6257728338241577, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.365979194641113, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.4549784660339355, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.14049497246742249, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.000493224710226059, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.220944404602051, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.8067981004714966, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.535993576049805, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023023555055260658, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1497184932231903, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15301206707954407, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.020933782681822777, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1934690773487091, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01808931678533554, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2568120956420898, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.009557397104799747, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.3771626949310303, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1769.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4296859800815582, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3481675386428833, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.359193801879883, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.1932108402252197, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17029616236686707, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 38.47757339477539, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.053135886788368225, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3319437503814697, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.12013673782348633, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.806850910186768, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2296.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6414068341255188, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.442629814147949, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.054059982299805, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0714073181152344, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.08525247871875763, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11420030891895294, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.92310905456543, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.8193913698196411, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.7737979888916, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018099240958690643, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1286800503730774, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1297290176153183, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011696616187691689, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16954466700553894, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005075111403129995, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2066722810268402, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.012423708103597164, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6835936307907104, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19704.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16097712516784668, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7963656783103943, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6147549748420715, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.46489769220352173, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10205257683992386, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.815078258514404, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0030145852360874414, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.34081029891967773, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.028045497834682465, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2417991161346436, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22557.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8467918634414673, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.5604193210601807, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.121594429016113, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.8539886474609375, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.3433454632759094, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.09119843691587448, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.63840913772583, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.05757895112037659, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.897563457489014, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018475383520126343, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13109274208545685, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13244135677814484, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.013820190913975239, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17247092723846436, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003387600649148226, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.39160025119781494, "metrics/particle/momentum/all/accepted/pz_bias": 0.015168811194598675, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3903160095214844, "metrics/particle/momentum/all/accepted/samples": 30405.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.1808948665857315, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7670251727104187, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.786312997341156, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6381914615631104, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10491007566452026, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.502920150756836, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009391949512064457, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6283955574035645, "metrics/particle/momentum/all/assigned/pz_bias": 0.0193500816822052, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.4797251224517822, "metrics/particle/momentum/all/assigned/samples": 35030.0, "metrics/particle/momentum/all/efficiency": 0.8262453079223633, "metrics/particle/momentum/all/event/energy_bias": -6.628821849822998, "metrics/particle/momentum/all/event/p3_error_mean": 17.7690372467041, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.567338943481445, "metrics/particle/momentum/all/event/px_bias": 0.11759798973798752, "metrics/particle/momentum/all/event/py_bias": 0.022508643567562103, "metrics/particle/momentum/all/event/pz_abs_mean": 13.3447847366333, "metrics/particle/momentum/all/event/pz_bias": 0.04498571157455444, "metrics/particle/momentum/all/event/pz_residual_rms": 23.609933853149414, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.018489785492420197, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10646118223667145, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10912813991308212, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.020986855030059814, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14500753581523895, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0024595707654953003, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3802758455276489, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.03005388379096985, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2644785642623901, "metrics/particle/momentum/class_1/accepted/samples": 16263.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09195221215486526, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2721022665500641, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3084526062011719, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2018422931432724, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04754239693284035, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.4173693656921387, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005921241827309132, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5041161179542542, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.034641943871974945, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.8537824153900146, "metrics/particle/momentum/class_1/assigned/samples": 17395.0, "metrics/particle/momentum/class_1/efficiency": 0.9166901707649231, "metrics/particle/momentum/class_1/event/energy_bias": 6.2524094581604, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.478072166442871, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.082755088806152, "metrics/particle/momentum/class_1/event/px_bias": 0.5463789105415344, "metrics/particle/momentum/class_1/event/py_bias": 1.1160081624984741, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.195941925048828, "metrics/particle/momentum/class_1/event/pz_bias": 0.39195990562438965, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.41046905517578, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02749365195631981, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17893792688846588, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18114051222801208, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.0033903508447110653, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22326524555683136, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02679169550538063, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.463746190071106, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.10942960530519485, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.665907382965088, "metrics/particle/momentum/class_2/accepted/samples": 1493.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5349944233894348, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6496436595916748, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7914583086967468, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4632816016674042, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2707786560058594, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.895373821258545, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05074365809559822, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.2406423091888428, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.026865631341934204, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.179941654205322, "metrics/particle/momentum/class_2/assigned/samples": 2286.0, "metrics/particle/momentum/class_2/efficiency": 0.5, "metrics/particle/momentum/class_2/event/energy_bias": -11.864748001098633, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.184022903442383, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.701287269592285, "metrics/particle/momentum/class_2/event/px_bias": -0.23801887035369873, "metrics/particle/momentum/class_2/event/py_bias": -1.0773311853408813, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.196172714233398, "metrics/particle/momentum/class_2/event/pz_bias": -0.4468756914138794, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.14145278930664, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017385847866535187, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1572839915752411, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1568806916475296, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0036363776307553053, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19644272327423096, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011455691419541836, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2558212876319885, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.007261090911924839, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.913354218006134, "metrics/particle/momentum/class_3/accepted/samples": 12221.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.23058269917964935, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3074514865875244, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2987037897109985, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1273858547210693, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14488942921161652, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.852943420410156, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006674757227301598, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5137537121772766, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.006619660183787346, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.479499101638794, "metrics/particle/momentum/class_3/assigned/samples": 14832.0, "metrics/particle/momentum/class_3/efficiency": 0.7867765426635742, "metrics/particle/momentum/class_3/event/energy_bias": 0.9261244535446167, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.881848335266113, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.566875457763672, "metrics/particle/momentum/class_3/event/px_bias": -0.29611778259277344, "metrics/particle/momentum/class_3/event/py_bias": -0.06304661929607391, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.004688739776611, "metrics/particle/momentum/class_3/event/pz_bias": 0.059862494468688965, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.28190803527832, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.020030278712511063, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15698416531085968, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1548364758491516, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08695372939109802, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20174339413642883, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5556175112724304, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.022529011592268944, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1931896209716797, "metrics/particle/momentum/class_4/accepted/samples": 268.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.259317547082901, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.477069616317749, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.051443576812744, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.9777886867523193, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2197183072566986, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 22.80475616455078, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.002816901309415698, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6313686966896057, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0374029278755188, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.514547348022461, "metrics/particle/momentum/class_4/assigned/samples": 355.0, "metrics/particle/momentum/class_4/efficiency": 0.7146666646003723, "metrics/particle/momentum/class_4/event/energy_bias": -1.1973037719726562, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2660727500915527, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6770273447036743, "metrics/particle/momentum/class_4/event/px_bias": 0.1234552189707756, "metrics/particle/momentum/class_4/event/py_bias": 0.0961710587143898, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2170666456222534, "metrics/particle/momentum/class_4/event/pz_bias": -0.16143333911895752, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.793476581573486, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013475358486175537, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14439421892166138, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1434411108493805, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03805040940642357, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1859700083732605, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05624999850988388, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6344201564788818, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.2565256953239441, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.647305965423584, "metrics/particle/momentum/class_5/accepted/samples": 160.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.013461125083267689, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.14880599081516266, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.14877372980117798, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.04468415305018425, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.012345679104328156, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.19562982022762299, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06172839552164078, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7121020555496216, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.15549886226654053, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.7556393146514893, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.9756097793579102, "metrics/particle/momentum/class_5/event/energy_bias": -0.7453023195266724, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.018393039703369, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5407116413116455, "metrics/particle/momentum/class_5/event/px_bias": -0.01809951290488243, "metrics/particle/momentum/class_5/event/py_bias": -0.04929286986589432, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.014892339706421, "metrics/particle/momentum/class_5/event/pz_bias": 0.2014724314212799, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.441041946411133, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.30709850788116455, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.30906954407691956, "metrics/set_slot/angular_recall_all_dr0p10": 0.9622763395309448, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8567255735397339, "metrics/set_slot/soft_count_bias_mean": -0.8996592164039612}, "step_train_time": 4.740143775939941, "step_valid_time": 22517.49365758896, "step_total_time": 22522.233801841736, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0060242218192, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1223685463581492, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2930, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2918, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9578293559986923, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9638157894736842, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9608132480734547, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03618421052631579, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9539065053939196, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9598684210526316, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.956878176750287, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9578293559986923, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991828411850893, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12080152672208255, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2261, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2167, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2158, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9483588621444201, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9584254754533392, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9533655961284646, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04157452454666077, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9444201312910284, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9544449358690845, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9494060712714474, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9483588621444201, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038704648082961, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12890341804051897, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1305, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1233, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1231, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.93267776096823, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9448275862068966, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9387133612485725, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05517241379310345, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9311649016641452, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9432950191570881, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9371907118385991, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.93267776096823} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_45000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_45000.json new file mode 100644 index 0000000000000000000000000000000000000000..9b74b5e8e4c13fa6e27a4dbd154e3667467cbda1 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_45000.json @@ -0,0 +1 @@ +{"epoch": 2.131489200454718, "train": {"Classification_binary": 0.13109268248081207, "Cardinality": 0.10526225715875626, "Classification": 0.08316823840141296, "Regression_pt": 0.11387530714273453, "Regression_eta": 0.0013531980803236365, "Regression_sin_phi": 0.0006826048484072089, "Regression_cos_phi": 0.000587338290642947, "Regression_energy": 0.11357495933771133, "HitGrouping": 0.19329209625720978, "Auxiliary": 0.20509301126003265, "Total": 0.9390829205513, "Total_unweighted": 0.4443343285820447}, "train_group": {}, "valid": {"Classification_binary": 0.14507425824801126, "Cardinality": 0.1163129210472107, "Classification": 0.08761494358380635, "Regression_pt": 0.11113246281941731, "Regression_eta": 0.001236633552859227, "Regression_sin_phi": 0.0006464772547284762, "Regression_cos_phi": 0.0006023629102855921, "Regression_energy": 0.11011292537053426, "HitGrouping": 0.18624438842137656, "Auxiliary": 0.2179211974143982, "Total": 0.9645204544067383, "Total_unweighted": 0.4564200637396425, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8263576030731201, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5124378800392151, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.34740230441093445, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5159837603569031, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5122821927070618, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3985047936439514, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.775709331035614, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.45295774936676025, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3401033878326416, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4560800790786743, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4558583199977875, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.46381092071533203, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8450667858123779, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.47853314876556396, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3173339366912842, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4820273518562317, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.47077399492263794, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.35774126648902893, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8340139985084534, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4868190288543701, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3282110095024109, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.49030011892318726, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4821910560131073, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3783257007598877, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.00364071992225945, "metrics/hit_grouping/calorimeter/background_recall": 0.1534135788679123, "metrics/hit_grouping/calorimeter/completeness": 0.5564467310905457, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5644127726554871, "metrics/hit_grouping/calorimeter/energy_purity": 0.5571220517158508, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.36902958154678345, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8557217717170715, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.509650468826294, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2557464838027954, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5145961046218872, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5026045441627502, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3191283643245697, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8351010084152222, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5080545544624329, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3572086691856384, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5110343098640442, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5060616731643677, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3876093924045563, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7886573672294617, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.418873131275177, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.44675174355506897, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4198727607727051, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4165205955505371, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4896751642227173, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8254647254943848, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5429725050926208, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4078060984611511, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5493305325508118, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5432597398757935, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.41069281101226807, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8045224547386169, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5826492309570312, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5558415055274963, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.580820620059967, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.559483528137207, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.43540915846824646, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8462440967559814, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4181503653526306, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2199399173259735, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4193307161331177, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.41071948409080505, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.338524729013443, "metrics/hit_grouping/calorimeter/purity": 0.5486130714416504, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7966054081916809, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08148306608200073, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10526315867900848, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08189447224140167, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.047930099070072174, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03947368264198303, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7847803235054016, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19328364729881287, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09652736783027649, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19303013384342194, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18497534096240997, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2919364273548126, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8371856212615967, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5086988806724548, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.34390026330947876, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5124242901802063, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.504733681678772, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3883426785469055, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.747136652469635, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40489640831947327, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5130204558372498, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5311696529388428, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5533825755119324, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7740796208381653, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28450873494148254, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4742645025253296, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.47812792658805847, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.41711875796318054, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6815661191940308, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4254569113254547, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.44596394896507263, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4715617895126343, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6427543759346008, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7085524201393127, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40872663259506226, "metrics/hit_grouping/tracker/all/object_completeness": 0.46866390109062195, "metrics/hit_grouping/tracker/all/object_purity": 0.49036702513694763, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.5986197590827942, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.016428420320153236, "metrics/hit_grouping/tracker/background_recall": 0.13538049161434174, "metrics/hit_grouping/tracker/completeness": 0.4670879542827606, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4568929970264435, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7248613238334656, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4018060564994812, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5206221342086792, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5349273085594177, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5832895040512085, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7019587159156799, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4209475517272949, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45812347531318665, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.48980242013931274, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.609729528427124, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6815561056137085, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4108205735683441, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.37225857377052307, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3992680609226227, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6191163063049316, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7301539182662964, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.41419717669487, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4996102750301361, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5291932225227356, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5630466341972351, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7204376459121704, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.44211822748184204, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4685199558734894, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4658312201499939, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5584975481033325, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6783391237258911, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3956286609172821, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4284585416316986, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4442902207374573, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6520587205886841, "metrics/hit_grouping/tracker/purity": 0.4430392384529114, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8846978545188904, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.027149321511387825, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.05981407314538956, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.06352394074201584, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06108597293496132, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6638360619544983, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2519359886646271, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.32038620114326477, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3446413576602936, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5518843531608582, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6872597932815552, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.48234570026397705, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5454075336456299, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5695041418075562, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6834536194801331, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5535003542900085, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6516078114509583, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7504854798316956, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6316574215888977, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.004347641486674547, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.057182617485523224, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.668415069580078, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.32162779569625854, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.9124999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.23000463843345642, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9186024069786072, "metrics/particle/class_1/pid_accuracy": 0.9784079790115356, "metrics/particle/class_2/efficiency": 0.5031147599220276, "metrics/particle/class_2/pid_accuracy": 0.7122917771339417, "metrics/particle/class_3/efficiency": 0.7913997769355774, "metrics/particle/class_3/pid_accuracy": 0.9762161374092102, "metrics/particle/class_4/efficiency": 0.7384769320487976, "metrics/particle/class_4/pid_accuracy": 0.48575305938720703, "metrics/particle/class_5/efficiency": 0.9559386968612671, "metrics/particle/class_5/pid_accuracy": 0.472945898771286, "metrics/particle/count/bias_mean": -2.6328125, "metrics/particle/count/mae": 3.89453125, "metrics/particle/count/prediction_mean": 68.02213287353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07508192211389542, "metrics/particle/event/energy_response_mean": 0.954246997833252, "metrics/particle/event/met_abs_error": 5.876617431640625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05441146716475487, "metrics/particle/event/scalar_pt_response_mean": 0.9793923497200012, "metrics/particle/event/vector_pt_closure": 0.05349399149417877, "metrics/particle/matched/delta_eta_abs_mean": 0.01248341053724289, "metrics/particle/matched/delta_phi_abs_mean": 0.010983607731759548, "metrics/particle/matched/delta_r_mean": 0.018587639555335045, "metrics/particle/matched/energy_relative_abs_mean": 0.129956915974617, "metrics/particle/matched/pid_accuracy": 0.9575656652450562, "metrics/particle/matched/pt_relative_abs_mean": 0.1308189183473587, "metrics/particle/matching/duplicate_fraction": 0.04216994345188141, "metrics/particle/matching/efficiency": 0.829708993434906, "metrics/particle/matching/f1": 0.8454611897468567, "metrics/particle/matching/purity": 0.8618230819702148, "metrics/particle/matching/target_coverage_dr0p05": 0.8068389296531677, "metrics/particle/matching/target_coverage_dr0p10": 0.8692663311958313, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6560455560684204, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8553158044815063, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01858828403055668, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1291932463645935, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13053730130195618, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018597964197397232, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16987480223178864, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006137707736343145, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6220743656158447, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.05396754667162895, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8195817470550537, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8961.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.17024342715740204, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.6825882196426392, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.6875537633895874, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.54982990026474, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09643974900245667, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.421671867370605, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012617371045053005, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8821930885314941, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06841649115085602, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.870192050933838, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10224.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8318789601325989, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.1690242290496826, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.424030780792236, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.2870330810546875, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.11785788834095001, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.006459305062890053, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.403820991516113, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.384907841682434, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.802342414855957, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023644236847758293, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1473212093114853, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14894086122512817, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01891939900815487, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18985223770141602, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02491506189107895, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3221981525421143, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1857619434595108, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.7530674934387207, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1766.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4301009476184845, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.001168131828308, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.09635329246521, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9259145259857178, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18012958765029907, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 25.697734832763672, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.059611231088638306, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4382495880126953, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.36114412546157837, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.564643859863281, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2315.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6403190493583679, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.583089351654053, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.685970306396484, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.162138819694519, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.12034707516431808, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.009196348488330841, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.5516939163208, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.410771131515503, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.142852783203125, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01804449036717415, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1287952959537506, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12942202389240265, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.009005964733660221, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16907654702663422, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003531429683789611, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20086625218391418, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.011330452747642994, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6444000005722046, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19822.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16085456311702728, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.729315459728241, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5227158665657043, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3690946400165558, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09874413907527924, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.16550874710083, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003183868480846286, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3488882780075073, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0019541271030902863, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4783635139465332, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22614.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8518629670143127, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.589733123779297, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.235658645629883, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.911447525024414, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.21267658472061157, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.2422546148300171, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.709580421447754, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.4935532212257385, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.940289497375488, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018527716398239136, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12998299300670624, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1308775395154953, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012392690405249596, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17057937383651733, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0034698352683335543, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.38924285769462585, "metrics/particle/momentum/all/accepted/pz_bias": -0.033920951187610626, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4334896802902222, "metrics/particle/momentum/all/accepted/samples": 30549.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18131645023822784, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7336280345916748, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6742896437644958, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5241845846176147, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10343356430530548, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.676348686218262, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009643558412790298, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6415910124778748, "metrics/particle/momentum/all/assigned/pz_bias": -0.04493868350982666, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7514383792877197, "metrics/particle/momentum/all/assigned/samples": 35153.0, "metrics/particle/momentum/all/efficiency": 0.8301584124565125, "metrics/particle/momentum/all/event/energy_bias": -9.341846466064453, "metrics/particle/momentum/all/event/p3_error_mean": 18.34705352783203, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.704427719116211, "metrics/particle/momentum/all/event/px_bias": -0.02552834153175354, "metrics/particle/momentum/all/event/py_bias": -0.2579103112220764, "metrics/particle/momentum/all/event/pz_abs_mean": 13.9053316116333, "metrics/particle/momentum/all/event/pz_bias": -4.28923225402832, "metrics/particle/momentum/all/event/pz_residual_rms": 24.795251846313477, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.018734581768512726, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10457593947649002, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10730133205652237, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01742868684232235, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1428842544555664, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0025724261067807674, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.37531229853630066, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03206115961074829, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3051931858062744, "metrics/particle/momentum/class_1/accepted/samples": 16327.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08886712789535522, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2340886890888214, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2539081871509552, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.14513365924358368, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.045642439275979996, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.0916507244110107, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005913422908633947, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5083965063095093, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.04557254537940025, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3377983570098877, "metrics/particle/momentum/class_1/assigned/samples": 17418.0, "metrics/particle/momentum/class_1/efficiency": 0.9202976226806641, "metrics/particle/momentum/class_1/event/energy_bias": 3.9134693145751953, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.985275268554688, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.719213008880615, "metrics/particle/momentum/class_1/event/px_bias": 0.6437289714813232, "metrics/particle/momentum/class_1/event/py_bias": 0.1586727350950241, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.87076187133789, "metrics/particle/momentum/class_1/event/pz_bias": -1.3296934366226196, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.676935195922852, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02764131687581539, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.178059384226799, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17872199416160583, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.0005736903985962272, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21894793212413788, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02684563770890236, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4955884218215942, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.13584648072719574, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7013802528381348, "metrics/particle/momentum/class_2/accepted/samples": 1490.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5682981014251709, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5994258522987366, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7896087169647217, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4495817720890045, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2776823937892914, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.822285175323486, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04978540912270546, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.270569324493408, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.11052381247282028, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.275137901306152, "metrics/particle/momentum/class_2/assigned/samples": 2330.0, "metrics/particle/momentum/class_2/efficiency": 0.498995304107666, "metrics/particle/momentum/class_2/event/energy_bias": -11.916525840759277, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.548759460449219, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.66577672958374, "metrics/particle/momentum/class_2/event/px_bias": -0.745690107345581, "metrics/particle/momentum/class_2/event/py_bias": -0.46510645747184753, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.744856834411621, "metrics/particle/momentum/class_2/event/pz_bias": -1.7866885662078857, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.497188568115234, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017212174832820892, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15711098909378052, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15573237836360931, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.005360250826925039, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19500714540481567, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015440877759829164, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.25618746876716614, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.023064136505126953, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0038882493972778, "metrics/particle/momentum/class_3/accepted/samples": 12305.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.22658351063728333, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3181374073028564, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1311763525009155, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9603395462036133, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14179155230522156, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.938101768493652, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007459176238626242, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.530437707901001, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03259028121829033, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6277992725372314, "metrics/particle/momentum/class_3/assigned/samples": 14881.0, "metrics/particle/momentum/class_3/efficiency": 0.7921843528747559, "metrics/particle/momentum/class_3/event/energy_bias": 0.4219471216201782, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.46694278717041, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.271916389465332, "metrics/particle/momentum/class_3/event/px_bias": -0.09284283220767975, "metrics/particle/momentum/class_3/event/py_bias": 0.21807505190372467, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.721693515777588, "metrics/particle/momentum/class_3/event/pz_bias": -1.1811885833740234, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.176875114440918, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01949898526072502, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1611814796924591, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15875887870788574, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08167792111635208, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20714299380779266, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0036630036775022745, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6370877027511597, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.12307750433683395, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4862971305847168, "metrics/particle/momentum/class_4/accepted/samples": 273.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3255058228969574, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8595030307769775, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6090843677520752, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5256702899932861, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21329639852046967, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.303211212158203, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008310249075293541, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6962891817092896, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.13704028725624084, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6904191970825195, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.7279999852180481, "metrics/particle/momentum/class_4/event/energy_bias": -1.2547677755355835, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.248037576675415, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6903793811798096, "metrics/particle/momentum/class_4/event/px_bias": 0.01056792214512825, "metrics/particle/momentum/class_4/event/py_bias": -0.04392518848180771, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1841133832931519, "metrics/particle/momentum/class_4/event/pz_bias": -0.17212191224098206, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.421203136444092, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011811968870460987, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13556836545467377, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13210991024971008, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03191804140806198, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1707468181848526, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025974025949835777, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3540111780166626, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.045630574226379395, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.166496515274048, "metrics/particle/momentum/class_5/accepted/samples": 154.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07667568325996399, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.17622116208076477, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.16582585871219635, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05900662764906883, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.042944785207509995, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.25809940695762634, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03680981695652008, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6157608032226562, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.0369393415749073, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.7574191093444824, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9390243887901306, "metrics/particle/momentum/class_5/event/energy_bias": -0.5059696435928345, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.012890100479126, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4897783994674683, "metrics/particle/momentum/class_5/event/px_bias": 0.15870775282382965, "metrics/particle/momentum/class_5/event/py_bias": -0.12562638521194458, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0521007776260376, "metrics/particle/momentum/class_5/event/pz_bias": 0.18046018481254578, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4490115642547607, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.30855995416641235, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3104008138179779, "metrics/set_slot/angular_recall_all_dr0p10": 0.9625988006591797, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8571494221687317, "metrics/set_slot/soft_count_bias_mean": -0.3545009195804596}, "step_train_time": 4.864952564239502, "step_valid_time": 25045.21447944641, "step_total_time": 25050.07943224907, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9970938922929662, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12516417189153817, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2930, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2914, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9578293559986923, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9676354029062088, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9627074092327912, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03236459709379128, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.952598888525662, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9623513870541611, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9574503039264005, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9578293559986923, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9897193278112018, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12016754832875387, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2264, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2171, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2166, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.950109409190372, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.958922261484099, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9544954935150582, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04107773851590106, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9479212253829322, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9567137809187279, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9522972081776214, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.950109409190372, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9922252502450544, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12533214575590712, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1305, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1228, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1225, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9288956127080181, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9409961685823754, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9349067377236391, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05900383141762452, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.926626323751891, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9386973180076629, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9326227636086791, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9288956127080181} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_46000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_46000.json new file mode 100644 index 0000000000000000000000000000000000000000..c051f93ec911a8f24c51f3b6d922d7040400bd14 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_46000.json @@ -0,0 +1 @@ +{"epoch": 2.1788556271314894, "train": {"Classification_binary": 0.11098668724298477, "Cardinality": 0.09383514523506165, "Classification": 0.08170009404420853, "Regression_pt": 0.10226535052061081, "Regression_eta": 0.0009820744162425399, "Regression_sin_phi": 0.0006501986645162106, "Regression_cos_phi": 0.000567447510547936, "Regression_energy": 0.10255774855613708, "HitGrouping": 0.1819293200969696, "Auxiliary": 0.18460427224636078, "Total": 0.8489903211593628, "Total_unweighted": 0.3997096009552479}, "train_group": {}, "valid": {"Classification_binary": 0.14673155546188354, "Cardinality": 0.1122493048508962, "Classification": 0.0875508685906728, "Regression_pt": 0.10898093382517497, "Regression_eta": 0.0012301288079470396, "Regression_sin_phi": 0.0006238004037489494, "Regression_cos_phi": 0.0006449302503218254, "Regression_energy": 0.10837467511494954, "HitGrouping": 0.1861539085706075, "Auxiliary": 0.21779996156692505, "Total": 0.9614710013071696, "Total_unweighted": 0.4541368924546987, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8285098075866699, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.517726480960846, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3403993546962738, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5227012038230896, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.521776556968689, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.39651745557785034, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7717559933662415, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.45669424533843994, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3356720805168152, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4616895616054535, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4679495096206665, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.48449039459228516, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8441634774208069, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.47513478994369507, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.312381386756897, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4788763225078583, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4705185294151306, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3642212450504303, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8337995409965515, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4866063594818115, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.32267776131629944, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.49081891775131226, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4858478009700775, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3833428919315338, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003163374960422516, "metrics/hit_grouping/calorimeter/background_recall": 0.13889671862125397, "metrics/hit_grouping/calorimeter/completeness": 0.5583747029304504, "metrics/hit_grouping/calorimeter/energy_completeness": 0.56765216588974, "metrics/hit_grouping/calorimeter/energy_purity": 0.5602102875709534, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3657299876213074, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8572705984115601, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5114895701408386, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.25017067790031433, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5168462991714478, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5080715417861938, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.32015249133110046, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8329953551292419, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5059860944747925, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.34891754388809204, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5098441243171692, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.507670521736145, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3970520496368408, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7867528796195984, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.41634654998779297, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4440835118293762, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4185831546783447, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.418549507856369, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.49837586283683777, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8224472403526306, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5433364510536194, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3995068669319153, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5503239631652832, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5499057769775391, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.41706758737564087, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7977938055992126, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5864781737327576, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5630468130111694, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5840774178504944, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5694869756698608, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4565105438232422, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.849663496017456, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.41686826944351196, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2157091200351715, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.41903936862945557, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4105724096298218, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.34024158120155334, "metrics/hit_grouping/calorimeter/purity": 0.5502502918243408, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.81009441614151, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09752720594406128, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11090225726366043, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09740646183490753, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.061544083058834076, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0357142873108387, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.805010974407196, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.18857111036777496, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10535608977079391, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1885596215724945, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18125048279762268, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.281930536031723, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8356840014457703, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5084555149078369, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.33743759989738464, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5129584670066833, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5086124539375305, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3942846655845642, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7455862760543823, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3995364308357239, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5094207525253296, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5345291495323181, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5572939515113831, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7741209864616394, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28511151671409607, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4797857999801636, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4828774034976959, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4243520200252533, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6696807146072388, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.42423370480537415, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.433444082736969, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.46788573265075684, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.659447431564331, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7007244825363159, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40636688470840454, "metrics/hit_grouping/tracker/all/object_completeness": 0.46021825075149536, "metrics/hit_grouping/tracker/all/object_purity": 0.48947569727897644, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6106856465339661, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.019755929708480835, "metrics/hit_grouping/tracker/background_recall": 0.13048626482486725, "metrics/hit_grouping/tracker/completeness": 0.4585641026496887, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.45488905906677246, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7170794606208801, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3971593976020813, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5127171277999878, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5371745228767395, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5967034697532654, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6928931474685669, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.41640952229499817, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4492480754852295, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4864271581172943, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6202577352523804, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6748629808425903, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41532912850379944, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3631276786327362, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.39438846707344055, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6299368739128113, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7216137647628784, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4129234850406647, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.48737144470214844, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5219988226890564, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5794345140457153, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7141076922416687, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45997536182403564, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46399375796318054, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4638494551181793, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.564039409160614, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6711684465408325, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3882327079772949, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.42424142360687256, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.45178860425949097, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6596754789352417, "metrics/hit_grouping/tracker/purity": 0.43643075227737427, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.881471574306488, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.028657617047429085, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.057049620896577835, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.058594945818185806, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07051282376050949, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6685329079627991, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.25916364789009094, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3317453861236572, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3554828464984894, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5518843531608582, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6773912906646729, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4783727824687958, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5339721441268921, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5679402351379395, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6972189545631409, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5288770794868469, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6471846103668213, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7446672916412354, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.634530782699585, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.02027573436498642, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0600559376180172, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.631244659423828, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29400956630706787, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8999999165534973, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20238640904426575, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9181637763977051, "metrics/particle/class_1/pid_accuracy": 0.9772137403488159, "metrics/particle/class_2/efficiency": 0.5232755541801453, "metrics/particle/class_2/pid_accuracy": 0.723809540271759, "metrics/particle/class_3/efficiency": 0.7953129410743713, "metrics/particle/class_3/pid_accuracy": 0.9772952198982239, "metrics/particle/class_4/efficiency": 0.7274549007415771, "metrics/particle/class_4/pid_accuracy": 0.45179063081741333, "metrics/particle/class_5/efficiency": 0.969348669052124, "metrics/particle/class_5/pid_accuracy": 0.47826087474823, "metrics/particle/count/bias_mean": -1.9029947519302368, "metrics/particle/count/mae": 3.6803386211395264, "metrics/particle/count/prediction_mean": 68.751953125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07180958241224289, "metrics/particle/event/energy_response_mean": 0.9661867022514343, "metrics/particle/event/met_abs_error": 5.875105381011963, "metrics/particle/event/scalar_pt_relative_abs_error": 0.053673699498176575, "metrics/particle/event/scalar_pt_response_mean": 0.9875783324241638, "metrics/particle/event/vector_pt_closure": 0.05393027141690254, "metrics/particle/matched/delta_eta_abs_mean": 0.012299065478146076, "metrics/particle/matched/delta_phi_abs_mean": 0.011072173714637756, "metrics/particle/matched/delta_r_mean": 0.01850295253098011, "metrics/particle/matched/energy_relative_abs_mean": 0.12842638790607452, "metrics/particle/matched/pid_accuracy": 0.9572780132293701, "metrics/particle/matched/pt_relative_abs_mean": 0.12897872924804688, "metrics/particle/matching/duplicate_fraction": 0.047403957694768906, "metrics/particle/matching/efficiency": 0.8327497839927673, "metrics/particle/matching/f1": 0.8441173434257507, "metrics/particle/matching/purity": 0.8557995557785034, "metrics/particle/matching/target_coverage_dr0p05": 0.8072720170021057, "metrics/particle/matching/target_coverage_dr0p10": 0.8722794651985168, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6636289954185486, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8589646816253662, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018672704696655273, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12962602078914642, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1309598684310913, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012161655351519585, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17149662971496582, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007119020912796259, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6265677213668823, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.02324739471077919, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8218427896499634, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8990.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18144923448562622, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7837781310081482, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8917441368103027, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7506420612335205, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09948458522558212, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.085416793823242, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0139064472168684, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9002591967582703, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02779090777039528, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2075881958007812, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10283.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.834571123123169, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.0081918239593506, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.5628180503845215, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.335137367248535, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.06610536575317383, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2968829572200775, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.488005638122559, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.2558184266090393, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.571467399597168, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022748775780200958, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1474347561597824, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14717428386211395, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.006066657602787018, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18980465829372406, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.029279278591275215, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3631521463394165, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.12582910060882568, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.825115919113159, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1776.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4782871901988983, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2596718072891235, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.2203545570373535, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.0395348072052, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17828132212162018, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 39.60554122924805, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05985463783144951, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.461300849914551, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3734939396381378, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.649383544921875, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2339.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6439448595046997, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.2930426597595215, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.848751544952393, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0885034799575806, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.058275654911994934, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04747242480516434, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.713924407958984, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.170345425605774, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.450908660888672, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018105389550328255, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12626323103904724, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1271088421344757, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.007112720515578985, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16708478331565857, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004537891363725066, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.199361652135849, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.009623750112950802, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6669184565544128, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19833.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16681836545467377, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8269375562667847, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6462906002998352, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.49823105335235596, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10039664804935455, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.2420854568481445, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0033054209779947996, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3408515453338623, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0016423584893345833, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3283129930496216, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22690.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8523357510566711, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.10734067857265472, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.435769081115723, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.101949691772461, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.8426588177680969, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.6076030731201172, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.7961080074310303, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.005385085940361023, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.199091911315918, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018541574478149414, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12848004698753357, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12940490245819092, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.008535385131835938, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16978463530540466, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004085100721567869, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.39242273569107056, "metrics/particle/momentum/all/accepted/pz_bias": -0.020371071994304657, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.453503966331482, "metrics/particle/momentum/all/accepted/samples": 30599.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19171005487442017, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8430328369140625, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8882688283920288, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.740065336227417, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.105289988219738, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.372334480285645, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010138196870684624, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6442078948020935, "metrics/particle/momentum/all/assigned/pz_bias": -0.01559139508754015, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.829421043395996, "metrics/particle/momentum/all/assigned/samples": 35312.0, "metrics/particle/momentum/all/efficiency": 0.8315171599388123, "metrics/particle/momentum/all/event/energy_bias": -6.408575534820557, "metrics/particle/momentum/all/event/p3_error_mean": 17.942819595336914, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.689957618713379, "metrics/particle/momentum/all/event/px_bias": 0.850488543510437, "metrics/particle/momentum/all/event/py_bias": 0.358192503452301, "metrics/particle/momentum/all/event/pz_abs_mean": 13.478347778320312, "metrics/particle/momentum/all/event/pz_bias": -0.909142017364502, "metrics/particle/momentum/all/event/pz_residual_rms": 23.60730743408203, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01890622265636921, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10304617136716843, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10476028174161911, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011185125447809696, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14057660102844238, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003320420626550913, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.37892287969589233, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.02288997359573841, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3618119955062866, "metrics/particle/momentum/class_1/accepted/samples": 16263.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09689858555793762, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.32535314559936523, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.35683751106262207, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24520505964756012, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04705814644694328, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.332132339477539, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005918179638683796, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5081758499145508, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02271641604602337, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.5597105026245117, "metrics/particle/momentum/class_1/assigned/samples": 17404.0, "metrics/particle/momentum/class_1/efficiency": 0.9166901707649231, "metrics/particle/momentum/class_1/event/energy_bias": 4.239584445953369, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.921107292175293, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.953024864196777, "metrics/particle/momentum/class_1/event/px_bias": 1.05612051486969, "metrics/particle/momentum/class_1/event/py_bias": 1.1962932348251343, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.644842147827148, "metrics/particle/momentum/class_1/event/pz_bias": -0.10902422666549683, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.716069221496582, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.026621727272868156, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17930594086647034, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18016253411769867, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.0019167319405823946, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22272582352161407, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03300970792770386, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5286641120910645, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.2232615053653717, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.779135227203369, "metrics/particle/momentum/class_2/accepted/samples": 1545.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5570425391197205, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6764752268791199, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8230690360069275, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5059797763824463, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2685690224170685, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.72195291519165, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05497272312641144, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3538966178894043, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.154922753572464, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.645028114318848, "metrics/particle/momentum/class_2/assigned/samples": 2383.0, "metrics/particle/momentum/class_2/efficiency": 0.5174146294593811, "metrics/particle/momentum/class_2/event/energy_bias": -9.880518913269043, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.28551959991455, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.61871862411499, "metrics/particle/momentum/class_2/event/px_bias": 0.017401844263076782, "metrics/particle/momentum/class_2/event/py_bias": -1.081735372543335, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.432132720947266, "metrics/particle/momentum/class_2/event/pz_bias": -1.681321620941162, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.312532424926758, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017137810587882996, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15479925274848938, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1549016535282135, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.004262847825884819, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1946069896221161, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.00129428890068084, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.25170791149139404, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0008160921861417592, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9103794693946838, "metrics/particle/momentum/class_3/accepted/samples": 12362.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.24159680306911469, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4520095586776733, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.500624418258667, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3350634574890137, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14465701580047607, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.837818145751953, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007932804524898529, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5230417251586914, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0054208808578550816, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4508674144744873, "metrics/particle/momentum/class_3/assigned/samples": 15001.0, "metrics/particle/momentum/class_3/efficiency": 0.7958539724349976, "metrics/particle/momentum/class_3/event/energy_bias": 1.190239429473877, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.668938636779785, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.712516784667969, "metrics/particle/momentum/class_3/event/px_bias": -0.36760324239730835, "metrics/particle/momentum/class_3/event/py_bias": 0.24111323058605194, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.773531913757324, "metrics/particle/momentum/class_3/event/pz_bias": 0.6529938578605652, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.585046768188477, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.018537184223532677, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16292893886566162, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1563950926065445, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07949380576610565, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20595328509807587, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003703703638166189, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5895921587944031, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08070283383131027, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.543208360671997, "metrics/particle/momentum/class_4/accepted/samples": 270.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.35329681634902954, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8968379497528076, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8101651668548584, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.71880304813385, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23545706272125244, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.04451847076416, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005540166050195694, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6423147916793823, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.18499428033828735, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7266151905059814, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.7200000286102295, "metrics/particle/momentum/class_4/event/energy_bias": -1.0738794803619385, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3169422149658203, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7955965995788574, "metrics/particle/momentum/class_4/event/px_bias": 0.04728161171078682, "metrics/particle/momentum/class_4/event/py_bias": 0.09806039184331894, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1456751823425293, "metrics/particle/momentum/class_4/event/pz_bias": -0.04641030728816986, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.3943891525268555, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011877586133778095, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13128364086151123, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1287565976381302, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.013512842357158661, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16670824587345123, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.01886792480945587, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3379253149032593, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.3898487091064453, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.1035168170928955, "metrics/particle/momentum/class_5/accepted/samples": 159.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.025006139650940895, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1738417148590088, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18673567473888397, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.07431997358798981, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.024539876729249954, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.500918447971344, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.01840490847826004, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.3289374113082886, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.4041265845298767, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.07426381111145, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9695122241973877, "metrics/particle/momentum/class_5/event/energy_bias": -0.8840011358261108, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7468096017837524, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3033958673477173, "metrics/particle/momentum/class_5/event/px_bias": 0.0972878560423851, "metrics/particle/momentum/class_5/event/py_bias": -0.09553896635770798, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9147157669067383, "metrics/particle/momentum/class_5/event/pz_bias": 0.2746203541755676, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.283038377761841, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.30827581882476807, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.30957961082458496, "metrics/set_slot/angular_recall_all_dr0p10": 0.961548388004303, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8593608736991882, "metrics/set_slot/soft_count_bias_mean": 0.3380497395992279}, "step_train_time": 4.258474111557007, "step_valid_time": 27738.225236415863, "step_total_time": 27742.48371076584, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023020672215597, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12537698959235266, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2924, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2915, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9558679306963059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9650165016501651, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9604204302841189, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03498349834983498, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9529257927427264, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9620462046204621, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9574642798489079, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9558679306963059, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9919664977612526, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12364106996563784, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2171, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2165, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.950109409190372, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547053649956024, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9524018425093222, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04529463500439754, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9474835886214442, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9520668425681619, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9497696863347225, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.950109409190372, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9970299462629058, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12411886423569285, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1307, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1230, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1227, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9304084720121029, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9410864575363428, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9357170026626094, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05891354246365723, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9281391830559758, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9387911247130833, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9334347660707494, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9304084720121029} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_47000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_47000.json new file mode 100644 index 0000000000000000000000000000000000000000..692d05f8df5f3f6927c24824e4ee3bda6a1807a2 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_47000.json @@ -0,0 +1 @@ +{"epoch": 2.226222053808261, "train": {"Classification_binary": 0.1325177699327469, "Cardinality": 0.09532766789197922, "Classification": 0.08017668128013611, "Regression_pt": 0.11773905158042908, "Regression_eta": 0.001008463092148304, "Regression_sin_phi": 0.0004927905974909663, "Regression_cos_phi": 0.0005441289395093918, "Regression_energy": 0.11753461509943008, "HitGrouping": 0.18756334483623505, "Auxiliary": 0.20933860540390015, "Total": 0.9022561311721802, "Total_unweighted": 0.4500135005218908}, "train_group": {}, "valid": {"Classification_binary": 0.14412648479143778, "Cardinality": 0.11357580622037251, "Classification": 0.08698894580205281, "Regression_pt": 0.11198910077412923, "Regression_eta": 0.0011862995258222024, "Regression_sin_phi": 0.0006433388528724512, "Regression_cos_phi": 0.0005913380688677231, "Regression_energy": 0.11126217246055603, "HitGrouping": 0.18683356046676636, "Auxiliary": 0.21765496333440146, "Total": 0.9591770172119141, "Total_unweighted": 0.45678768027573824, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8260362148284912, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5151435136795044, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.35279643535614014, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5194928646087646, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.514940619468689, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4041828215122223, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7685438394546509, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.44908270239830017, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34194979071617126, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4534043073654175, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.46525096893310547, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4870753288269043, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8455523252487183, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.47612595558166504, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3239065110683441, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4799445569515228, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.46810734272003174, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.36306411027908325, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8336610794067383, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4858468472957611, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.33405962586402893, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4898653030395508, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4820738732814789, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3851490914821625, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003170098178088665, "metrics/hit_grouping/calorimeter/background_recall": 0.13868530094623566, "metrics/hit_grouping/calorimeter/completeness": 0.5542114973068237, "metrics/hit_grouping/calorimeter/energy_completeness": 0.560529887676239, "metrics/hit_grouping/calorimeter/energy_purity": 0.5531903505325317, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.37413108348846436, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8593025803565979, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5112870335578918, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2620618939399719, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5162439346313477, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5035534501075745, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.32185935974121094, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8324268460273743, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5042033791542053, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3606632947921753, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5082870125770569, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5029963254928589, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.39359742403030396, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7826220393180847, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4154820442199707, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45406031608581543, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4175214171409607, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4171997606754303, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5056844353675842, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8212447762489319, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5414935946464539, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4158647954463959, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5481500625610352, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5427089929580688, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.41953331232070923, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7906777262687683, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5772261023521423, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5635614991188049, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5756510496139526, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5690188407897949, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.459598571062088, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8514412045478821, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4182250201702118, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22331228852272034, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.42022034525871277, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4098944664001465, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3412226438522339, "metrics/hit_grouping/calorimeter/purity": 0.546151340007782, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8177698254585266, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09717500954866409, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10150375962257385, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09736841171979904, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.062153954058885574, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04135338217020035, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.818789005279541, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1949094533920288, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10535608977079391, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19506889581680298, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.17613545060157776, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2872277796268463, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8346938490867615, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5073199272155762, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3497503697872162, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5116016268730164, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5048368573188782, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.39584672451019287, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.748109757900238, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3917137384414673, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5133991837501526, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5377593636512756, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5509198904037476, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7730287313461304, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.26582279801368713, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.48026716709136963, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.48822909593582153, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4141048789024353, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6769750118255615, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4219312071800232, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.44283193349838257, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4727714955806732, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6499496102333069, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7059329152107239, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4011130928993225, "metrics/hit_grouping/tracker/all/object_completeness": 0.467285692691803, "metrics/hit_grouping/tracker/all/object_purity": 0.4938870072364807, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6020926237106323, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01688903011381626, "metrics/hit_grouping/tracker/background_recall": 0.12768392264842987, "metrics/hit_grouping/tracker/completeness": 0.4655784070491791, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4501646161079407, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7207890152931213, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.391811341047287, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5171477794647217, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5371719002723694, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5899526476860046, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6980389952659607, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4124160408973694, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4574885070323944, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.49755942821502686, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6126338839530945, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.68321692943573, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4090171456336975, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.37445321679115295, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.40120208263397217, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6165915131568909, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7279385924339294, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.41029125452041626, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.49816280603408813, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5284577012062073, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5645750164985657, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7152976989746094, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4378078877925873, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46364423632621765, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4688853323459625, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5621921420097351, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6756460070610046, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3826029300689697, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4277971684932709, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4534260928630829, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6580196619033813, "metrics/hit_grouping/tracker/purity": 0.4418143033981323, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8799120187759399, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02978883869946003, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.06273260712623596, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.06628375500440598, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0656108632683754, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6693079471588135, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.25606608390808105, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3324893116950989, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3520299196243286, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5513681173324585, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6840846538543701, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.47193777561187744, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5419303178787231, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5727176666259766, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6872027516365051, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5427433252334595, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6399006843566895, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7331941723823547, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6256319880485535, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0064094518311321735, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05115722864866257, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.647994995117188, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2970179319381714, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8874999284744263, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20539475977420807, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9188694357872009, "metrics/particle/class_1/pid_accuracy": 0.9759443998336792, "metrics/particle/class_2/efficiency": 0.5161399841308594, "metrics/particle/class_2/pid_accuracy": 0.7254772782325745, "metrics/particle/class_3/efficiency": 0.7868307828903198, "metrics/particle/class_3/pid_accuracy": 0.9771615862846375, "metrics/particle/class_4/efficiency": 0.7334669232368469, "metrics/particle/class_4/pid_accuracy": 0.49180328845977783, "metrics/particle/class_5/efficiency": 0.9444444179534912, "metrics/particle/class_5/pid_accuracy": 0.47058823704719543, "metrics/particle/count/bias_mean": -2.0436198711395264, "metrics/particle/count/mae": 3.7141926288604736, "metrics/particle/count/prediction_mean": 68.611328125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07286755740642548, "metrics/particle/event/energy_response_mean": 0.9703288078308105, "metrics/particle/event/met_abs_error": 6.142223834991455, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05467803403735161, "metrics/particle/event/scalar_pt_response_mean": 0.9921126365661621, "metrics/particle/event/vector_pt_closure": 0.05516387149691582, "metrics/particle/matched/delta_eta_abs_mean": 0.011760924011468887, "metrics/particle/matched/delta_phi_abs_mean": 0.010901124216616154, "metrics/particle/matched/delta_r_mean": 0.01794855110347271, "metrics/particle/matched/energy_relative_abs_mean": 0.12975461781024933, "metrics/particle/matched/pid_accuracy": 0.9570335745811462, "metrics/particle/matched/pt_relative_abs_mean": 0.1306125670671463, "metrics/particle/matching/duplicate_fraction": 0.04632449895143509, "metrics/particle/matching/efficiency": 0.8288704752922058, "metrics/particle/matching/f1": 0.8410335183143616, "metrics/particle/matching/purity": 0.8535587787628174, "metrics/particle/matching/target_coverage_dr0p05": 0.8115842938423157, "metrics/particle/matching/target_coverage_dr0p10": 0.8730995059013367, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.655566394329071, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8543482422828674, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017910398542881012, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1292896419763565, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1312050223350525, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.023224249482154846, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17082762718200684, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00661509158089757, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6144096851348877, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.00885139312595129, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7959811687469482, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8919.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1820649355649948, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7638790607452393, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8209699988365173, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.689304530620575, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10564202070236206, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.21030044555664, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014785991981625557, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8940144777297974, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.010303317569196224, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8034934997558594, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10280.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8279799222946167, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.12274469435214996, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.428309917449951, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.4577016830444336, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.139156773686409, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2982991337776184, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.269457817077637, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.718741238117218, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.321783065795898, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02312738075852394, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1468067467212677, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14843878149986267, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.014442874118685722, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18855918943881989, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03028603456914425, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3510016202926636, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.08540266007184982, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.681168556213379, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1783.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4568425416946411, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9474020004272461, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.4179158210754395, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.243166446685791, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18108223378658295, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 31.233726501464844, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06646782904863358, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.467017650604248, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.15830302238464355, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.577121257781982, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2347.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6464829444885254, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.568968296051025, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.175117492675781, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.080981731414795, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.12342463433742523, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.0002883598208427429, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.047098159790039, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.061112165451049805, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.413652420043945, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01754305325448513, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12753640115261078, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1281290054321289, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01973630115389824, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1682269126176834, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003531786205712706, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19900065660476685, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.010625330731272697, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6789983510971069, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19820.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16147008538246155, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8385141491889954, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6126155853271484, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4720132350921631, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10355602204799652, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.586273670196533, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003300765762105584, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.34558984637260437, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03552805259823799, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3239049911499023, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22722.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8517770171165466, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.467616468667984, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.30710506439209, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.987864971160889, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6943199634552002, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.5571660995483398, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.730424404144287, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.43162786960601807, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.775475025177002, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01797661744058132, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12917445600032806, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13021430373191833, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.020446309819817543, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17024022340774536, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0039315903559327126, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3876858949661255, "metrics/particle/momentum/all/accepted/pz_bias": 0.004497303161770105, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4260245561599731, "metrics/particle/momentum/all/accepted/samples": 30522.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18707065284252167, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8240387439727783, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7930710911750793, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6528005003929138, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10931002348661423, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.219304084777832, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010834818705916405, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6459320187568665, "metrics/particle/momentum/all/assigned/pz_bias": 0.015322905965149403, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6877944469451904, "metrics/particle/momentum/all/assigned/samples": 35349.0, "metrics/particle/momentum/all/efficiency": 0.8294247388839722, "metrics/particle/momentum/all/event/energy_bias": -4.224096775054932, "metrics/particle/momentum/all/event/p3_error_mean": 17.82875633239746, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.76990795135498, "metrics/particle/momentum/all/event/px_bias": 0.7100521326065063, "metrics/particle/momentum/all/event/py_bias": 0.25857865810394287, "metrics/particle/momentum/all/event/pz_abs_mean": 13.18506145477295, "metrics/particle/momentum/all/event/pz_bias": 1.2114813327789307, "metrics/particle/momentum/all/event/pz_residual_rms": 24.25434112548828, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.017886251211166382, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10567913204431534, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10718908160924911, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.023827025666832924, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14302852749824524, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0033082154113799334, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3808508813381195, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005096929147839546, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.394848346710205, "metrics/particle/momentum/class_1/accepted/samples": 16323.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09180913120508194, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2397676557302475, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.25408294796943665, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.15229366719722748, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0472969114780426, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.9071861505508423, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006650232244282961, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5111095309257507, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.00835365615785122, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3013288974761963, "metrics/particle/momentum/class_1/assigned/samples": 17443.0, "metrics/particle/momentum/class_1/efficiency": 0.9200721383094788, "metrics/particle/momentum/class_1/event/energy_bias": 3.6528122425079346, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.447547912597656, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.30557632446289, "metrics/particle/momentum/class_1/event/px_bias": 0.36382168531417847, "metrics/particle/momentum/class_1/event/py_bias": 0.8043550848960876, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.903009414672852, "metrics/particle/momentum/class_1/event/pz_bias": 0.21799413859844208, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.17949104309082, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02720049023628235, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17315375804901123, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17526233196258545, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.002066640183329582, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21669620275497437, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028871390968561172, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4056018590927124, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08807987719774246, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.493051052093506, "metrics/particle/momentum/class_2/accepted/samples": 1524.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5673800706863403, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6807529926300049, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8230581283569336, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5081918835639954, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.27551448345184326, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.2945008277893066, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05627887323498726, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3097753524780273, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.008303402923047543, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.394906520843506, "metrics/particle/momentum/class_2/assigned/samples": 2381.0, "metrics/particle/momentum/class_2/efficiency": 0.5103817582130432, "metrics/particle/momentum/class_2/event/energy_bias": -8.017728805541992, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.529563903808594, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.854188919067383, "metrics/particle/momentum/class_2/event/px_bias": 0.31475937366485596, "metrics/particle/momentum/class_2/event/py_bias": -0.22614210844039917, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.60000228881836, "metrics/particle/momentum/class_2/event/pz_bias": -0.38842377066612244, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.863155364990234, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017040926963090897, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15445679426193237, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15495778620243073, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.016651887446641922, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1947726607322693, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013871889095753431, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.25583183765411377, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.012175425887107849, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9005889892578125, "metrics/particle/momentum/class_3/accepted/samples": 12255.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.23618687689304352, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4819252490997314, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3718777894973755, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.212456464767456, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15254350006580353, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.330262184143066, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008267218247056007, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.527273416519165, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.024208160117268562, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.457479238510132, "metrics/particle/momentum/class_3/assigned/samples": 14999.0, "metrics/particle/momentum/class_3/efficiency": 0.7889654040336609, "metrics/particle/momentum/class_3/event/energy_bias": 1.3823373317718506, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.598036766052246, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.221287250518799, "metrics/particle/momentum/class_3/event/px_bias": -0.13352802395820618, "metrics/particle/momentum/class_3/event/py_bias": -0.17397886514663696, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.934757709503174, "metrics/particle/momentum/class_3/event/pz_bias": 1.082324504852295, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.504033088684082, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.017704349011182785, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1540783792734146, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14925172924995422, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.080874964594841, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1952790468931198, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0037313431967049837, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5492662787437439, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.04530718922615051, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3036948442459106, "metrics/particle/momentum/class_4/accepted/samples": 268.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.31206247210502625, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.9595322608947754, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.869415760040283, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.7879176139831543, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23756906390190125, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.61469841003418, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008287292905151844, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7382470369338989, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.043671902269124985, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.8735620975494385, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.7146666646003723, "metrics/particle/momentum/class_4/event/energy_bias": -0.8940683603286743, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.149639129638672, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6239616870880127, "metrics/particle/momentum/class_4/event/px_bias": 0.08778263628482819, "metrics/particle/momentum/class_4/event/py_bias": -0.09320878237485886, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0989021062850952, "metrics/particle/momentum/class_4/event/pz_bias": 0.018043816089630127, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.084779739379883, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011119448579847813, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.12904365360736847, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12267492711544037, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.04105761647224426, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16151191294193268, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02631578966975212, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2615840435028076, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.33707743883132935, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.9467225074768066, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.029683168977499008, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.16500042378902435, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.165108785033226, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08855041861534119, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05487804859876633, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.31385013461112976, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03658536449074745, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.4778800010681152, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.1760806143283844, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.3474440574645996, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9268292784690857, "metrics/particle/momentum/class_5/event/energy_bias": -0.34744906425476074, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9568589925765991, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5248980522155762, "metrics/particle/momentum/class_5/event/px_bias": 0.07721638679504395, "metrics/particle/momentum/class_5/event/py_bias": -0.05244666710495949, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9286811947822571, "metrics/particle/momentum/class_5/event/pz_bias": 0.2815425395965576, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.1507554054260254, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3052791953086853, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3068472743034363, "metrics/set_slot/angular_recall_all_dr0p10": 0.9621104598045349, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8594990968704224, "metrics/set_slot/soft_count_bias_mean": 0.10294448584318161}, "step_train_time": 4.181291103363037, "step_valid_time": 30378.195496320724, "step_total_time": 30382.376787900925, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002868479625997, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1267154933247604, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3039, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2932, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2911, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9584831644328212, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9647910496873972, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9616267628730731, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03520895031260283, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9516181758744687, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9578808818690359, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9547392587733683, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9584831644328212, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015323954627835, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1218307856097618, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2265, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2164, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2158, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9470459518599562, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9554083885209713, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9512087912087912, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044591611479028695, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9444201312910284, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9527593818984548, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9485714285714286, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9470459518599562, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00096732052957, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12729982138205054, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1234, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1231, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9334341906202723, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.936267071320182, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9348484848484848, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0637329286798179, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9311649016641452, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9339908952959028, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9325757575757576, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9334341906202723} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_48000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_48000.json new file mode 100644 index 0000000000000000000000000000000000000000..506cd9d752d0b3e7d5efaf614008a1eaddcc5540 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_48000.json @@ -0,0 +1 @@ +{"epoch": 2.2735884804850324, "train": {"Classification_binary": 0.1524886190891266, "Cardinality": 0.12207775563001633, "Classification": 0.08229000121355057, "Regression_pt": 0.10731889307498932, "Regression_eta": 0.001728505128994584, "Regression_sin_phi": 0.0006355666555464268, "Regression_cos_phi": 0.0006188153638504446, "Regression_energy": 0.10790969431400299, "HitGrouping": 0.18367576599121094, "Auxiliary": 0.21929198503494263, "Total": 0.9846245050430298, "Total_unweighted": 0.4529900948400609}, "train_group": {}, "valid": {"Classification_binary": 0.14536577463150024, "Cardinality": 0.11407884955406189, "Classification": 0.08559746543566386, "Regression_pt": 0.10914167761802673, "Regression_eta": 0.0011926558800041676, "Regression_sin_phi": 0.0006164619699120522, "Regression_cos_phi": 0.0006274225500722727, "Regression_energy": 0.10817221800486247, "HitGrouping": 0.18674492835998535, "Auxiliary": 0.21707038084665933, "Total": 0.9574975967407227, "Total_unweighted": 0.4507136760900418, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8242704272270203, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.512782096862793, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3455096185207367, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.517497718334198, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5170851945877075, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4032364785671234, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7709060907363892, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4597797691822052, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.33973410725593567, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.46331560611724854, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.46637198328971863, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4837518334388733, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8434186577796936, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4771825969219208, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3199722170829773, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4807794988155365, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4693748652935028, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3606572449207306, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8319879174232483, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.48661643266677856, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.32924312353134155, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4905475080013275, "metrics/hit_grouping/calorimeter/all/object_purity": 0.48359569907188416, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3831135332584381, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0034997824113816023, "metrics/hit_grouping/calorimeter/background_recall": 0.14984779059886932, "metrics/hit_grouping/calorimeter/completeness": 0.5572142601013184, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5662122964859009, "metrics/hit_grouping/calorimeter/energy_purity": 0.558868408203125, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3677425682544708, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8566058278083801, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5115754008293152, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.25421029329299927, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.516686737537384, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5056842565536499, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.31867319345474243, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8325578570365906, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5055184960365295, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.35974204540252686, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5099716782569885, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5026084184646606, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.39440348744392395, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7812184691429138, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4166833162307739, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45150813460350037, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4176819622516632, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.41940364241600037, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5031322240829468, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8215872645378113, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5436018705368042, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.40876835584640503, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5506945848464966, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5499784350395203, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4167669117450714, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7951657772064209, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5859344601631165, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5620175004005432, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5843862891197205, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5742793679237366, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4487905204296112, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8469789028167725, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.41668400168418884, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2204304337501526, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.41804438829421997, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.40511080622673035, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.34097737073898315, "metrics/hit_grouping/calorimeter/purity": 0.5492920875549316, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7971267700195312, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09125127643346786, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11466165632009506, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08860543370246887, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06326239556074142, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0357142873108387, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7927095293998718, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.18942564725875854, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10241318494081497, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1895664483308792, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1762067824602127, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2672159969806671, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.834600031375885, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5085240006446838, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3445434868335724, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5127595067024231, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5064408183097839, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.39480534195899963, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7473191618919373, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40098509192466736, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5172054767608643, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5397526025772095, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5533825755119324, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7767927646636963, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.27667269110679626, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4789882004261017, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4817297160625458, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.416515976190567, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6744000911712646, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4314289689064026, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4403361678123474, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.46458467841148376, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6497337818145752, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7043747305870056, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4106411337852478, "metrics/hit_grouping/tracker/all/object_completeness": 0.4668166935443878, "metrics/hit_grouping/tracker/all/object_purity": 0.4889536201953888, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6028940081596375, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.017384054139256477, "metrics/hit_grouping/tracker/background_recall": 0.13739342987537384, "metrics/hit_grouping/tracker/completeness": 0.46483781933784485, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4570936858654022, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7188775539398193, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40066632628440857, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.516456663608551, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5357242822647095, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5933719277381897, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7006680965423584, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42040297389030457, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4578304886817932, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4856360852718353, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6093664765357971, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.678225040435791, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4214607775211334, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3736357092857361, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.39604315161705017, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6160504817962646, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7281500101089478, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4241317808628082, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5003684759140015, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5227876305580139, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5622824430465698, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7152309417724609, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4568965435028076, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46241822838783264, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.452593058347702, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5695812702178955, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.671519935131073, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3848106861114502, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4239867925643921, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4514867663383484, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6616624593734741, "metrics/hit_grouping/tracker/purity": 0.44133374094963074, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8814940452575684, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.0290346909314394, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.062651127576828, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.06566209346055984, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06485671550035477, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.672522246837616, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27361899614334106, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3364261984825134, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3512403070926666, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5539494156837463, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6815431118011475, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.48212188482284546, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5409263372421265, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5666952729225159, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6880421042442322, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5724578499794006, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999165534973, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6448392868041992, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7411465048789978, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6388916373252869, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.023305121809244156, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06441687792539597, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.375354766845703, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29982084035873413, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.9124999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20819765329360962, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9211961627006531, "metrics/particle/class_1/pid_accuracy": 0.9770196080207825, "metrics/particle/class_2/efficiency": 0.5172726511955261, "metrics/particle/class_2/pid_accuracy": 0.7166630029678345, "metrics/particle/class_3/efficiency": 0.7929300665855408, "metrics/particle/class_3/pid_accuracy": 0.9786606431007385, "metrics/particle/class_4/efficiency": 0.7444889545440674, "metrics/particle/class_4/pid_accuracy": 0.511440098285675, "metrics/particle/class_5/efficiency": 0.954023003578186, "metrics/particle/class_5/pid_accuracy": 0.4819277226924896, "metrics/particle/count/bias_mean": -1.6985677480697632, "metrics/particle/count/mae": 3.560546875, "metrics/particle/count/prediction_mean": 68.95638275146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07383684068918228, "metrics/particle/event/energy_response_mean": 0.9712817072868347, "metrics/particle/event/met_abs_error": 5.74584436416626, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0551399402320385, "metrics/particle/event/scalar_pt_response_mean": 0.9890878796577454, "metrics/particle/event/vector_pt_closure": 0.05280191823840141, "metrics/particle/matched/delta_eta_abs_mean": 0.011730825528502464, "metrics/particle/matched/delta_phi_abs_mean": 0.010997606441378593, "metrics/particle/matched/delta_r_mean": 0.018009411171078682, "metrics/particle/matched/energy_relative_abs_mean": 0.12980371713638306, "metrics/particle/matched/pid_accuracy": 0.9579668045043945, "metrics/particle/matched/pt_relative_abs_mean": 0.13030806183815002, "metrics/particle/matching/duplicate_fraction": 0.04770716652274132, "metrics/particle/matching/efficiency": 0.8328050374984741, "metrics/particle/matching/f1": 0.8429372906684875, "metrics/particle/matching/purity": 0.8533191084861755, "metrics/particle/matching/target_coverage_dr0p05": 0.8138418197631836, "metrics/particle/matching/target_coverage_dr0p10": 0.8755505681037903, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6613253951072693, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8592226505279541, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01806597411632538, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13055454194545746, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1319989562034607, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02127368003129959, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1715247929096222, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008129175752401352, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6341388821601868, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.01313159428536892, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8270535469055176, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8980.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18108820915222168, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.715958833694458, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7803630828857422, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.646203339099884, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09975716471672058, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.027849197387695, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014764448627829552, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8802167177200317, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.037259869277477264, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7525012493133545, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10295.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8336427807807922, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.1801934689283371, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.977517127990723, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.0936782360076904, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.3488713800907135, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.3912632465362549, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.9648284912109375, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.43022266030311584, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.533998489379883, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022345075383782387, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1456841230392456, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14767126739025116, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01850789226591587, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1882610321044922, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.027840908616781235, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3281192779541016, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.16194361448287964, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5499489307403564, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1760.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4555495083332062, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.300143837928772, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.9853498935699463, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.8183465003967285, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1922421157360077, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 67.5434799194336, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06138107553124428, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.363759994506836, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.22791136801242828, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.579553604125977, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2346.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6381435990333557, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.93922233581543, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.355572700500488, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.134496808052063, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07384548336267471, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.05320744588971138, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.2177095413208, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.1868207454681396, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.123735427856445, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017589803785085678, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1272093653678894, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1276237815618515, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015353170223534107, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16786938905715942, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00030187159427441657, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2006879448890686, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0007639987161383033, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6944504380226135, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19876.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16468283534049988, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8422365784645081, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6473974585533142, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5046120882034302, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10240091383457184, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.933860778808594, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003204143373295665, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.35712072253227234, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0010633906349539757, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6501764059066772, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22783.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8541836738586426, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.16472077369689941, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.521878242492676, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.182687282562256, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.0641316175460815, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.268564373254776, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.8330726623535156, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.08477044105529785, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.025631904602051, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01800283044576645, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1292525678873062, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13005951046943665, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017271071672439575, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17018075287342072, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004180820658802986, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.39263561367988586, "metrics/particle/momentum/all/accepted/pz_bias": -0.0136571628972888, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4200855493545532, "metrics/particle/momentum/all/accepted/samples": 30616.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18871362507343292, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.835862934589386, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9071004390716553, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7652179598808289, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10758242756128311, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.667203903198242, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010416666977107525, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6420362591743469, "metrics/particle/momentum/all/assigned/pz_bias": -0.026606183499097824, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6115427017211914, "metrics/particle/momentum/all/assigned/samples": 35424.0, "metrics/particle/momentum/all/efficiency": 0.8319791555404663, "metrics/particle/momentum/all/event/energy_bias": -4.954695224761963, "metrics/particle/momentum/all/event/p3_error_mean": 18.14749526977539, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.742351531982422, "metrics/particle/momentum/all/event/px_bias": 0.6414147615432739, "metrics/particle/momentum/all/event/py_bias": -0.606620192527771, "metrics/particle/momentum/all/event/pz_abs_mean": 13.467385292053223, "metrics/particle/momentum/all/event/pz_bias": -1.7018136978149414, "metrics/particle/momentum/all/event/pz_residual_rms": 24.05103302001953, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.017697157338261604, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10389001667499542, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10604853928089142, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.021109264343976974, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14130738377571106, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003675119485706091, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3802092373371124, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005167938768863678, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.316405177116394, "metrics/particle/momentum/class_1/accepted/samples": 16326.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09059768170118332, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.24854089319705963, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2649760842323303, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.16196772456169128, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.046678122133016586, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.295600414276123, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006586483214050531, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.49725058674812317, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.007545133121311665, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.8653218746185303, "metrics/particle/momentum/class_1/assigned/samples": 17460.0, "metrics/particle/momentum/class_1/efficiency": 0.9202412366867065, "metrics/particle/momentum/class_1/event/energy_bias": 3.644181728363037, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.839913368225098, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.867490768432617, "metrics/particle/momentum/class_1/event/px_bias": 0.8268271684646606, "metrics/particle/momentum/class_1/event/py_bias": -0.1306181699037552, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.664517402648926, "metrics/particle/momentum/class_1/event/pz_bias": 0.06864094734191895, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.988109588623047, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027001431211829185, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17436976730823517, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1763526201248169, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.00528355035930872, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2182113379240036, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029411764815449715, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4457926750183105, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.15207688510417938, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.590972661972046, "metrics/particle/momentum/class_2/accepted/samples": 1530.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5770556926727295, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6947147250175476, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8636434078216553, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5271081924438477, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.26846474409103394, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.694249629974365, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0593360997736454, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3474020957946777, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.18865394592285156, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.483238697052002, "metrics/particle/momentum/class_2/assigned/samples": 2410.0, "metrics/particle/momentum/class_2/efficiency": 0.5123911499977112, "metrics/particle/momentum/class_2/event/energy_bias": -10.597160339355469, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.590211868286133, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.434935569763184, "metrics/particle/momentum/class_2/event/px_bias": -0.1648358404636383, "metrics/particle/momentum/class_2/event/py_bias": -0.6798990368843079, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.69412899017334, "metrics/particle/momentum/class_2/event/pz_bias": -0.7741203308105469, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.022624969482422, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017348267138004303, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15656524896621704, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15560224652290344, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013322841376066208, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1957598179578781, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014599724672734737, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.26118236780166626, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.021261529996991158, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9741131067276001, "metrics/particle/momentum/class_3/accepted/samples": 12329.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.23951490223407745, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5215425491333008, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6530957221984863, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.495123028755188, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1507551074028015, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 28.43311309814453, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006719446275383234, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.524940550327301, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.04443785920739174, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6448442935943604, "metrics/particle/momentum/class_3/assigned/samples": 15031.0, "metrics/particle/momentum/class_3/efficiency": 0.7937294840812683, "metrics/particle/momentum/class_3/event/energy_bias": 3.3891043663024902, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.84475326538086, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.66008996963501, "metrics/particle/momentum/class_3/event/px_bias": -0.14704012870788574, "metrics/particle/momentum/class_3/event/py_bias": 0.24505387246608734, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.841087341308594, "metrics/particle/momentum/class_3/event/pz_bias": -1.1682672500610352, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.85152816772461, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01861334964632988, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15928076207637787, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15451276302337646, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08292409777641296, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20091690123081207, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003636363660916686, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.608441174030304, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.09395694732666016, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4524617195129395, "metrics/particle/momentum/class_4/accepted/samples": 275.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.2871544361114502, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.9504724740982056, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.534326195716858, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4682875871658325, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21169915795326233, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 7.8546881675720215, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008356546051800251, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6955445408821106, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.05177563056349754, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.832311749458313, "metrics/particle/momentum/class_4/assigned/samples": 359.0, "metrics/particle/momentum/class_4/efficiency": 0.7333333492279053, "metrics/particle/momentum/class_4/event/energy_bias": -0.7849621176719666, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0947964191436768, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5604897737503052, "metrics/particle/momentum/class_4/event/px_bias": 0.06902724504470825, "metrics/particle/momentum/class_4/event/py_bias": 0.03579201549291611, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.096084475517273, "metrics/particle/momentum/class_4/event/pz_bias": 0.040335603058338165, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.07405424118042, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012392540462315083, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.12953612208366394, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12707196176052094, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03310038894414902, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.16537541151046753, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025641025975346565, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3726508617401123, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.11634443700313568, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.039569854736328, "metrics/particle/momentum/class_5/accepted/samples": 156.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05618522688746452, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.15424844622612, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.16313330829143524, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05175720155239105, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04268292710185051, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2950645089149475, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04268292710185051, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6108357906341553, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.1815750002861023, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.5246572494506836, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9512194991111755, "metrics/particle/momentum/class_5/event/energy_bias": -0.6058589816093445, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9143505096435547, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4601995944976807, "metrics/particle/momentum/class_5/event/px_bias": 0.057436347007751465, "metrics/particle/momentum/class_5/event/py_bias": -0.07694883644580841, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9703069925308228, "metrics/particle/momentum/class_5/event/pz_bias": 0.13159729540348053, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.15535044670105, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3060348331928253, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3078557252883911, "metrics/set_slot/angular_recall_all_dr0p10": 0.9625158905982971, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8637377023696899, "metrics/set_slot/soft_count_bias_mean": 0.5044324398040771}, "step_train_time": 4.810326099395752, "step_valid_time": 33124.265265226364, "step_total_time": 33129.07559156418, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000666609302972, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12408615652479216, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3045, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2919, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2905, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.954233409610984, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9586206896551724, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9564220183486238, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041379310344827586, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9496567505720824, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540229885057471, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9518348623853211, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.954233409610984, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9933031176299101, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12211666820013922, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2174, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2170, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9514223194748359, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9560246262093228, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9537179205966221, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04397537379067722, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.949671772428884, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9542656112576957, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9519631498135556, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9514223194748359, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9996797786054085, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12397425905578963, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1311, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1237, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1232, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9357034795763994, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9435545385202135, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9396126091910368, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05644546147978642, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9319213313161876, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9397406559877955, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9358146600835549, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9357034795763994} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_49000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_49000.json new file mode 100644 index 0000000000000000000000000000000000000000..63b36768360cb6725af567cbee4b90d2c8bcfa31 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_49000.json @@ -0,0 +1 @@ +{"epoch": 2.320954907161804, "train": {"Classification_binary": 0.1586858183145523, "Cardinality": 0.1353997141122818, "Classification": 0.08928680419921875, "Regression_pt": 0.10810403525829315, "Regression_eta": 0.0010022098431363702, "Regression_sin_phi": 0.0006048957002349198, "Regression_cos_phi": 0.0006447239429689944, "Regression_energy": 0.1074005663394928, "HitGrouping": 0.18376609683036804, "Auxiliary": 0.23336301743984222, "Total": 1.0019199848175049, "Total_unweighted": 0.4657290535978973}, "train_group": {}, "valid": {"Classification_binary": 0.1446483333905538, "Cardinality": 0.11247056722640991, "Classification": 0.08639771739641826, "Regression_pt": 0.11069102088610332, "Regression_eta": 0.0012040795603146155, "Regression_sin_phi": 0.0006273497516910235, "Regression_cos_phi": 0.0006233495660126209, "Regression_energy": 0.11002877354621887, "HitGrouping": 0.18652337789535522, "Auxiliary": 0.21832656860351562, "Total": 0.9592256546020508, "Total_unweighted": 0.4542206240973125, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8270525336265564, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5158802270889282, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.34295448660850525, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5199609994888306, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5188085436820984, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3986940383911133, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7745704650878906, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4511915147304535, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3423190414905548, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4545958340167999, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.45976725220680237, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4700886309146881, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8449056148529053, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4800029695034027, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3168248236179352, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.48372769355773926, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.472106009721756, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3647766709327698, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8340363502502441, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.48863524198532104, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3267201781272888, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4924429655075073, "metrics/hit_grouping/calorimeter/all/object_purity": 0.48529675602912903, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.38322821259498596, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003512979717925191, "metrics/hit_grouping/calorimeter/background_recall": 0.15117262303829193, "metrics/hit_grouping/calorimeter/completeness": 0.5584903359413147, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5674400329589844, "metrics/hit_grouping/calorimeter/energy_purity": 0.5600912570953369, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.36823827028274536, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8587104678153992, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5159964561462402, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2587619423866272, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5210700631141663, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5096691846847534, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3202662765979767, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8347093462944031, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5076547861099243, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3505297005176544, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5106917023658752, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.505437433719635, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3885306417942047, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7830479741096497, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.41368579864501953, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.44129928946495056, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4156886041164398, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.41531166434288025, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5062645077705383, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8234745860099792, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5448509454727173, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.405701220035553, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5512760877609253, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5467289090156555, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4158647954463959, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7994198203086853, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5913799405097961, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5501801371574402, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.589541494846344, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5731734037399292, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4379824995994568, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.848928689956665, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.41907933354377747, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2195720225572586, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.42089104652404785, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.41219377517700195, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3434300124645233, "metrics/hit_grouping/calorimeter/purity": 0.5505573153495789, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8157671093940735, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0882953330874443, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11278195679187775, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08857186883687973, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.053197409957647324, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03947368264198303, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8083484172821045, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19738346338272095, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09770452976226807, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19611357152462006, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.18415628373622894, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2872277796268463, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8356708288192749, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5103148818016052, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.342123806476593, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5144442915916443, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5080084800720215, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3938252329826355, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7468639016151428, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3956250846385956, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5164079666137695, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5441392064094543, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5509198904037476, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7701249718666077, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2778782546520233, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4807194173336029, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4890514314174652, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4249548017978668, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6787478923797607, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43078139424324036, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.441271036863327, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.46167901158332825, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6457043886184692, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7064326405525208, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40868210792541504, "metrics/hit_grouping/tracker/all/object_completeness": 0.4672778844833374, "metrics/hit_grouping/tracker/all/object_purity": 0.48904475569725037, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6002671122550964, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.019192079082131386, "metrics/hit_grouping/tracker/background_recall": 0.14461635053157806, "metrics/hit_grouping/tracker/completeness": 0.4648207724094391, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.45562779903411865, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7224203944206238, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4025951325893402, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5209462642669678, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5358317494392395, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5878484845161438, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6993704438209534, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4171355962753296, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4554773271083832, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.484839528799057, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6068251729011536, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6805623769760132, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41280433535575867, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3686068654060364, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.39698225259780884, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.619296669960022, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7306368947029114, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4198012948036194, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.49854978919029236, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5231930613517761, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5594803690910339, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7211742401123047, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4513546824455261, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4702001214027405, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.46617430448532104, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5591133236885071, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6723235845565796, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38657689094543457, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.42609959840774536, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4487508237361908, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6606689691543579, "metrics/hit_grouping/tracker/purity": 0.4421311020851135, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8811309337615967, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.0290346909314394, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.06471896171569824, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.06874009966850281, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06787330657243729, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6700483560562134, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2586473822593689, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3336675465106964, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.34524092078208923, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5456892251968384, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6844515204429626, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4812825322151184, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5414981245994568, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5670031905174255, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6851882934570312, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5231965780258179, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6399457454681396, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7342446446418762, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6390528082847595, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.025956159457564354, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06457800418138504, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.471824645996094, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.32027319073677063, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8999999165534973, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.2286500334739685, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9213487505912781, "metrics/particle/class_1/pid_accuracy": 0.978141188621521, "metrics/particle/class_2/efficiency": 0.5222561955451965, "metrics/particle/class_2/pid_accuracy": 0.7187160849571228, "metrics/particle/class_3/efficiency": 0.7931923866271973, "metrics/particle/class_3/pid_accuracy": 0.9761596322059631, "metrics/particle/class_4/efficiency": 0.7204408645629883, "metrics/particle/class_4/pid_accuracy": 0.4993045926094055, "metrics/particle/class_5/efficiency": 0.954023003578186, "metrics/particle/class_5/pid_accuracy": 0.4879518151283264, "metrics/particle/count/bias_mean": -2.0266926288604736, "metrics/particle/count/mae": 3.541015625, "metrics/particle/count/prediction_mean": 68.62825775146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07169836759567261, "metrics/particle/event/energy_response_mean": 0.9588289260864258, "metrics/particle/event/met_abs_error": 5.1861138343811035, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05326414108276367, "metrics/particle/event/scalar_pt_response_mean": 0.9789511561393738, "metrics/particle/event/vector_pt_closure": 0.051239773631095886, "metrics/particle/matched/delta_eta_abs_mean": 0.011824697256088257, "metrics/particle/matched/delta_phi_abs_mean": 0.010934260673820972, "metrics/particle/matched/delta_r_mean": 0.018030542880296707, "metrics/particle/matched/energy_relative_abs_mean": 0.12843911349773407, "metrics/particle/matched/pid_accuracy": 0.9576094150543213, "metrics/particle/matched/pt_relative_abs_mean": 0.12906324863433838, "metrics/particle/matching/duplicate_fraction": 0.04509880021214485, "metrics/particle/matching/efficiency": 0.8331736326217651, "metrics/particle/matching/f1": 0.8452970385551453, "metrics/particle/matching/purity": 0.8577784299850464, "metrics/particle/matching/target_coverage_dr0p05": 0.8133350610733032, "metrics/particle/matching/target_coverage_dr0p10": 0.8741223216056824, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6628642082214355, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8587803840637207, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017999568954110146, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12845011055469513, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12999625504016876, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.011712257750332355, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1704186201095581, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007004669867455959, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6254815459251404, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.03730311617255211, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8647102117538452, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8994.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.17420054972171783, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7760547399520874, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8270515203475952, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6897031664848328, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09563267976045609, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.876773834228516, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014525248669087887, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.883269190788269, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.04772447049617767, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8570556640625, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10258.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8349424600601196, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.4725968837738037, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.510587215423584, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.31248140335083, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.5878637433052063, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.15544649958610535, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.434288024902344, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.0471200942993164, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.83654499053955, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022767486050724983, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14072689414024353, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14283622801303864, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.007085224147886038, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1828731894493103, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022701475769281387, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2950198650360107, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1523781269788742, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5088694095611572, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1762.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4595586955547333, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9894620776176453, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.2296407222747803, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.0560097694396973, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18961818516254425, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 19.66123390197754, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.056199055165052414, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3717050552368164, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.26241686940193176, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.909601211547852, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2331.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6388687491416931, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.562946319580078, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.413509368896484, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.183916449546814, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.12522664666175842, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.02210119739174843, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.271952629089355, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.360140323638916, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.820878982543945, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017691725865006447, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12666258215904236, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12711670994758606, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.006364602595567703, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16741345822811127, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005529027548618615, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20183396339416504, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00992614310234785, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.708254337310791, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19895.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1586739867925644, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8124887943267822, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5985334515571594, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4483420252799988, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09893012791872025, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.108007907867432, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028177695348858833, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.34145915508270264, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0009165088413283229, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4176486730575562, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22713.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8550001978874207, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.5431249141693115, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.148710250854492, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.8441948890686035, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5840795636177063, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.47583872079849243, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.6426897048950195, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.2603403925895691, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.016871929168701, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01807384006679058, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12799561023712158, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1288653165102005, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.007975203916430473, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1692231446504593, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003719291416928172, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3889887034893036, "metrics/particle/momentum/all/accepted/pz_bias": -0.026148416101932526, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4330660104751587, "metrics/particle/momentum/all/accepted/samples": 30651.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.1830531656742096, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8135874271392822, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7726383209228516, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6246311068534851, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10396011918783188, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.405845642089844, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009744490496814251, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6329550743103027, "metrics/particle/momentum/all/assigned/pz_bias": -0.03178482875227928, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6110191345214844, "metrics/particle/momentum/all/assigned/samples": 35302.0, "metrics/particle/momentum/all/efficiency": 0.8329302668571472, "metrics/particle/momentum/all/event/energy_bias": -9.578667640686035, "metrics/particle/momentum/all/event/p3_error_mean": 17.697477340698242, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.512206077575684, "metrics/particle/momentum/all/event/px_bias": -1.2971699237823486, "metrics/particle/momentum/all/event/py_bias": -0.6533864140510559, "metrics/particle/momentum/all/event/pz_abs_mean": 13.277278900146484, "metrics/particle/momentum/all/event/pz_bias": -2.6676011085510254, "metrics/particle/momentum/all/event/pz_residual_rms": 23.596363067626953, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01819239929318428, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10203782469034195, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10411740094423294, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.013398117385804653, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.139752596616745, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0032417885959148407, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3721359074115753, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03940216824412346, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2951138019561768, "metrics/particle/momentum/class_1/accepted/samples": 16349.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.09150179475545883, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.23890730738639832, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3056536912918091, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1968025267124176, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04567858949303627, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.7399489879608154, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005788628943264484, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.47575461864471436, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.04574063792824745, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7543619871139526, "metrics/particle/momentum/class_1/assigned/samples": 17448.0, "metrics/particle/momentum/class_1/efficiency": 0.9215376973152161, "metrics/particle/momentum/class_1/event/energy_bias": 4.02046537399292, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.602312088012695, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.694927215576172, "metrics/particle/momentum/class_1/event/px_bias": -0.12885165214538574, "metrics/particle/momentum/class_1/event/py_bias": 0.06966091692447662, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.553228378295898, "metrics/particle/momentum/class_1/event/pz_bias": -1.4742473363876343, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.223522186279297, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02739744447171688, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1786804348230362, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1795971691608429, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.005061966832727194, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22275306284427643, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02462734840810299, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4674912691116333, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08324923366308212, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7347114086151123, "metrics/particle/momentum/class_2/accepted/samples": 1543.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5542223453521729, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6406679749488831, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9002598524093628, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5740790367126465, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2616940438747406, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.195791721343994, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05351875349879265, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3873302936553955, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1560114175081253, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.8716559410095215, "metrics/particle/momentum/class_2/assigned/samples": 2373.0, "metrics/particle/momentum/class_2/efficiency": 0.5167447924613953, "metrics/particle/momentum/class_2/event/energy_bias": -11.476739883422852, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.285873413085938, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.187931060791016, "metrics/particle/momentum/class_2/event/px_bias": -0.16137662529945374, "metrics/particle/momentum/class_2/event/py_bias": -0.452793151140213, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.531968116760254, "metrics/particle/momentum/class_2/event/pz_bias": -1.7768747806549072, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.1329288482666, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.016793960705399513, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15527746081352234, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15476340055465698, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0008821959490887821, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19440734386444092, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012965964851900935, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2566407322883606, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0044513968750834465, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9600781202316284, "metrics/particle/momentum/class_3/accepted/samples": 12340.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.22973082959651947, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.483932614326477, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2773265838623047, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.11021888256073, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14391760528087616, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.020099639892578, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007222631014883518, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5255765318870544, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0027328028809279203, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6122071743011475, "metrics/particle/momentum/class_3/assigned/samples": 14953.0, "metrics/particle/momentum/class_3/efficiency": 0.7944376468658447, "metrics/particle/momentum/class_3/event/energy_bias": -0.19897596538066864, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.976099014282227, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.6961565017700195, "metrics/particle/momentum/class_3/event/px_bias": -1.0646376609802246, "metrics/particle/momentum/class_3/event/py_bias": -0.0831136703491211, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.012184143066406, "metrics/particle/momentum/class_3/event/pz_bias": 0.3858119249343872, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.059469223022461, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01965823955833912, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15445782244205475, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15027159452438354, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07563310861587524, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19944126904010773, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003816793905571103, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.594482958316803, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.09988574683666229, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3291431665420532, "metrics/particle/momentum/class_4/accepted/samples": 262.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3056401014328003, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.2257723808288574, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8558721542358398, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7719488143920898, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.25824177265167236, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.415413856506348, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6853412389755249, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.17375677824020386, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.5227826833724976, "metrics/particle/momentum/class_4/assigned/samples": 364.0, "metrics/particle/momentum/class_4/efficiency": 0.6986666917800903, "metrics/particle/momentum/class_4/event/energy_bias": -1.162241816520691, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.023341655731201, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5059401988983154, "metrics/particle/momentum/class_4/event/px_bias": 0.06595298647880554, "metrics/particle/momentum/class_4/event/py_bias": -0.042204536497592926, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.078180193901062, "metrics/particle/momentum/class_4/event/pz_bias": -0.09502580761909485, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.278778076171875, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012048212811350822, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14446772634983063, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13608616590499878, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.015991007909178734, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17958718538284302, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03821656107902527, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6038305759429932, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.3328966200351715, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.714031457901001, "metrics/particle/momentum/class_5/accepted/samples": 157.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02458318881690502, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.20169714093208313, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18849723041057587, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0520331934094429, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03658536449074745, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.47368666529655457, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03658536449074745, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6467649936676025, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.41837525367736816, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.726363182067871, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.957317054271698, "metrics/particle/momentum/class_5/event/energy_bias": -0.7611759901046753, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9974383115768433, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5587716102600098, "metrics/particle/momentum/class_5/event/px_bias": -0.008257059380412102, "metrics/particle/momentum/class_5/event/py_bias": -0.14493593573570251, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9665017127990723, "metrics/particle/momentum/class_5/event/pz_bias": 0.29273492097854614, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3206264972686768, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.30247417092323303, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.30399012565612793, "metrics/set_slot/angular_recall_all_dr0p10": 0.9623408317565918, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8624200820922852, "metrics/set_slot/soft_count_bias_mean": 0.06212872266769409}, "step_train_time": 5.267922401428223, "step_valid_time": 35683.5274541378, "step_total_time": 35688.79537701607, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9921403412741724, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12107986205741761, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3037, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2930, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2922, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9578293559986923, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9647678630227198, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9612860892388452, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03523213697728021, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9552141222621772, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9621336845571288, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9586614173228346, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9578293559986923, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9849584776103852, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11779386017560478, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2265, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2156, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2151, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9435448577680525, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9518763796909492, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9476923076923077, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04812362030905077, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9413566739606127, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9496688741721855, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9454945054945055, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9435448577680525, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9890735153107224, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12069002852431643, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1308, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1241, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1237, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9387291981845688, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9487767584097859, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9437262357414449, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05122324159021407, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9357034795763994, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9457186544342507, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9406844106463879, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9387291981845688} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_5000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..5939051b40a3c1d301b9314aa2776ad9ca902b7f --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 0.23683213338385753, "train": {"Classification_binary": 0.31975337862968445, "Cardinality": 0.2573046088218689, "Classification": 0.274806946516037, "Regression_pt": 0.4073961079120636, "Regression_eta": 0.004285749047994614, "Regression_sin_phi": 0.002324742265045643, "Regression_cos_phi": 0.0024415075313299894, "Regression_energy": 0.4019608795642853, "HitGrouping": 0.27682608366012573, "Auxiliary": 0.47555869817733765, "Total": 2.4059500694274902, "Total_unweighted": 1.4129693114664406}, "train_group": {}, "valid": {"Classification_binary": 0.357954740524292, "Cardinality": 0.2525951862335205, "Classification": 0.2694165309270223, "Regression_pt": 0.41344745953877765, "Regression_eta": 0.005124608054757118, "Regression_sin_phi": 0.002563723362982273, "Regression_cos_phi": 0.002485026139765978, "Regression_energy": 0.4075009822845459, "HitGrouping": 0.28348128000895184, "Auxiliary": 0.481210192044576, "Total": 2.5010425249735513, "Total_unweighted": 1.458493070832143, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.764616072177887, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.15996131300926208, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3452256917953491, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.1578209102153778, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.14114728569984436, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.575754702091217, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7436789274215698, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.16775882244110107, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3002215623855591, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.16198813915252686, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.13518568873405457, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5812407732009888, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7616233825683594, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.14488941431045532, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3194630742073059, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.14239555597305298, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.13116231560707092, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.59259432554245, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7611368298530579, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.15123103559017181, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.32577407360076904, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.1485898345708847, "metrics/hit_grouping/calorimeter/all/object_purity": 0.134499654173851, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5866112112998962, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0006115592550486326, "metrics/hit_grouping/calorimeter/background_recall": 0.02555248700082302, "metrics/hit_grouping/calorimeter/completeness": 0.24529126286506653, "metrics/hit_grouping/calorimeter/energy_completeness": 0.2770846486091614, "metrics/hit_grouping/calorimeter/energy_purity": 0.2731742858886719, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.6197683215141296, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7822515368461609, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.154089093208313, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2662721872329712, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.15218579769134521, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.14759895205497742, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5373805165290833, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7552971839904785, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.1555226594209671, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.35122063755989075, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.15165546536445618, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.13492627441883087, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.60628741979599, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7239674925804138, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.14107999205589294, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.42146173119544983, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.13816937804222107, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.10736067593097687, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6671693921089172, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7311960458755493, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.1812676638364792, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.42885494232177734, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.1761717051267624, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.15753057599067688, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6496872901916504, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7430570721626282, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3592435419559479, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.7133299112319946, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.35907915234565735, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.29030337929725647, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6103962659835815, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7938172221183777, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.0958249643445015, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.17450487613677979, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.09539148956537247, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.09245632588863373, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5194677710533142, "metrics/hit_grouping/calorimeter/purity": 0.24110503494739532, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9008973240852356, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.03173444792628288, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12593984603881836, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.030607769265770912, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.009181142784655094, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.033834587782621384, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8403229713439941, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.03569920361042023, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.08711006492376328, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.035852380096912384, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.023206880316138268, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3431430160999298, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7547388076782227, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.15919028222560883, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3414499759674072, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.15637896955013275, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.14233314990997314, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6082881689071655, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6730310320854187, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43473851680755615, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.28417152166366577, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.2584571838378906, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6933217644691467, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.741161048412323, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.32007232308387756, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.28379595279693604, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.23424288630485535, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.523809552192688, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5922878980636597, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4387681782245636, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.2267838716506958, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.2049664556980133, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7732767462730408, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.628100574016571, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4287622570991516, "metrics/hit_grouping/tracker/all/object_completeness": 0.24863293766975403, "metrics/hit_grouping/tracker/all/object_purity": 0.2235691398382187, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7302760481834412, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.014720986597239971, "metrics/hit_grouping/tracker/background_recall": 0.0759788453578949, "metrics/hit_grouping/tracker/completeness": 0.25424206256866455, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6571778059005737, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6426881551742554, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41434332728385925, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.2961757481098175, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.2719937264919281, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7279502153396606, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6284555792808533, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42639318108558655, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.22671069204807281, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.19872786104679108, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7335269451141357, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.5977410674095154, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4607754647731781, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.17261788249015808, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.1486402004957199, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7318304777145386, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6446831822395325, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4759276509284973, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.26103976368904114, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.22217638790607452, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7140188217163086, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6143272519111633, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5406404137611389, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.2608099579811096, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.22970718145370483, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7309113144874573, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.609011709690094, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3473893404006958, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.2303207665681839, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.22427937388420105, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7512970566749573, "metrics/hit_grouping/tracker/purity": 0.24073413014411926, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.893547773361206, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.015837104991078377, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.009488795883953571, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.003039389615878463, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.060708899050951004, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6720268726348877, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.19463087618350983, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.11416047811508179, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.06883904337882996, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5663396716117859, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5839478969573975, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5154160261154175, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.2986963987350464, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.2730659544467926, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8474063873291016, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.32746919989585876, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5740224719047546, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.648067057132721, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5691554546356201, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.2216835469007492, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.005319324322044849, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.68845558166504, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.189273402094841, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.09765023738145828, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5409467220306396, "metrics/particle/class_1/pid_accuracy": 0.8742420077323914, "metrics/particle/class_2/efficiency": 0.3426209092140198, "metrics/particle/class_2/pid_accuracy": 0.0895867794752121, "metrics/particle/class_3/efficiency": 0.4638524055480957, "metrics/particle/class_3/pid_accuracy": 0.8817042112350464, "metrics/particle/class_4/efficiency": 0.495991975069046, "metrics/particle/class_4/pid_accuracy": 0.13535353541374207, "metrics/particle/class_5/efficiency": 0.5670498013496399, "metrics/particle/class_5/pid_accuracy": 0.1756756752729416, "metrics/particle/count/bias_mean": -5.76953125, "metrics/particle/count/mae": 8.423176765441895, "metrics/particle/count/prediction_mean": 64.88541412353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.14101813733577728, "metrics/particle/event/energy_response_mean": 1.0178238153457642, "metrics/particle/event/met_abs_error": 12.808049201965332, "metrics/particle/event/scalar_pt_relative_abs_error": 0.12092828005552292, "metrics/particle/event/scalar_pt_response_mean": 1.0454710721969604, "metrics/particle/event/vector_pt_closure": 0.10481400042772293, "metrics/particle/matched/delta_eta_abs_mean": 0.02441537380218506, "metrics/particle/matched/delta_phi_abs_mean": 0.022492405027151108, "metrics/particle/matched/delta_r_mean": 0.03703177720308304, "metrics/particle/matched/energy_relative_abs_mean": 0.20450378954410553, "metrics/particle/matched/pid_accuracy": 0.8220345377922058, "metrics/particle/matched/pt_relative_abs_mean": 0.202074334025383, "metrics/particle/matching/duplicate_fraction": 0.1171737015247345, "metrics/particle/matching/efficiency": 0.4920295476913452, "metrics/particle/matching/f1": 0.5129737257957458, "metrics/particle/matching/purity": 0.5357802510261536, "metrics/particle/matching/target_coverage_dr0p05": 0.41789984703063965, "metrics/particle/matching/target_coverage_dr0p10": 0.5893334150314331, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.2997991144657135, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.569762110710144, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03701706975698471, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20257286727428436, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.20111976563930511, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.014872751198709011, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24162353575229645, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.018442220985889435, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.3529120683670044, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.06987051665782928, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.2299232482910156, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5097.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.6755004525184631, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.4800074100494385, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.356405258178711, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.156158208847046, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.3611966669559479, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 15.788633346557617, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03618565946817398, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.0295498371124268, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.17849776148796082, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.482705116271973, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9092.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.4731711745262146, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 5.360074996948242, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 16.999895095825195, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 8.117260932922363, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.918874979019165, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.10954251885414124, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 14.230279922485352, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.435981512069702, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 22.931001663208008, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03654537349939346, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.20200008153915405, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.2008989304304123, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.006850475445389748, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.24203065037727356, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03915662690997124, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.0994296073913574, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.13583189249038696, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.694413185119629, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 996.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.0093095302581787, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.6358649730682373, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 8.81389331817627, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 8.625699996948242, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.4241361916065216, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 120.7875747680664, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0846269428730011, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.8284683227539062, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.15456344187259674, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.031839370727539, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1997.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.3611312508583069, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -9.292557716369629, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 13.522043228149414, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.089167833328247, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.018667157739400864, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.0011654123663902283, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 13.270450592041016, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.5759172439575195, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 31.688159942626953, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.036975618451833725, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20662930607795715, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.20296543836593628, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.022820541635155678, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.24375362694263458, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.002011566422879696, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.46481087803840637, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03364652767777443, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.1641342639923096, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 11931.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.6271411776542664, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.699205160140991, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.009739398956299, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.7995293140411377, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.3683612644672394, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 14.249756813049316, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.016037436202168465, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.133985996246338, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.06753462553024292, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.3022522926330566, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22011.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.512742280960083, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 12.727641105651855, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 17.465848922729492, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.44357681274414, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.5243351459503174, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.0303438901901245, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.194887161254883, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.14368344843387604, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 12.988621711730957, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0369635671377182, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20522639155387878, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.20232929289340973, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.019690491259098053, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2430580109357834, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.008710607886314392, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8062851428985596, "metrics/particle/momentum/all/accepted/pz_bias": -0.004992337431758642, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.2505648136138916, "metrics/particle/momentum/all/accepted/samples": 18024.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.6634817123413086, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.3001585006713867, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.4551405906677246, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.248995780944824, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.3697583079338074, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 32.92003631591797, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.0257099699229002, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.5425461530685425, "metrics/particle/momentum/all/assigned/pz_bias": 0.08461460471153259, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.484837055206299, "metrics/particle/momentum/all/assigned/samples": 33100.0, "metrics/particle/momentum/all/efficiency": 0.4897959232330322, "metrics/particle/momentum/all/event/energy_bias": 8.795158386230469, "metrics/particle/momentum/all/event/p3_error_mean": 33.960628509521484, "metrics/particle/momentum/all/event/pt_vector_error_mean": 17.495471954345703, "metrics/particle/momentum/all/event/px_bias": 5.461877346038818, "metrics/particle/momentum/all/event/py_bias": 0.9196360111236572, "metrics/particle/momentum/all/event/pz_abs_mean": 24.91765785217285, "metrics/particle/momentum/all/event/pz_bias": 4.868215084075928, "metrics/particle/momentum/all/event/pz_residual_rms": 39.258201599121094, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.04357288032770157, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.21998468041419983, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21770033240318298, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.037293314933776855, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2566291093826294, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.00966589618474245, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.9282690286636353, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.056935373693704605, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.3420932292938232, "metrics/particle/momentum/class_1/accepted/samples": 9518.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.5101004838943481, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.932882308959961, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.1387386322021484, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.9000837802886963, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.32700344920158386, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 12.567472457885742, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.02261553518474102, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.4816102981567383, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.11969222128391266, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.049299716949463, "metrics/particle/momentum/class_1/assigned/samples": 16272.0, "metrics/particle/momentum/class_1/efficiency": 0.5364973545074463, "metrics/particle/momentum/class_1/event/energy_bias": 35.156734466552734, "metrics/particle/momentum/class_1/event/p3_error_mean": 32.49176788330078, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 19.074729919433594, "metrics/particle/momentum/class_1/event/px_bias": 7.05335807800293, "metrics/particle/momentum/class_1/event/py_bias": 1.1482208967208862, "metrics/particle/momentum/class_1/event/pz_abs_mean": 22.16619300842285, "metrics/particle/momentum/class_1/event/pz_bias": 7.70407772064209, "metrics/particle/momentum/class_1/event/pz_residual_rms": 31.14737319946289, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04605325683951378, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.230241060256958, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22966362535953522, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06965523213148117, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.268827348947525, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.033718690276145935, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8791407346725464, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.262276291847229, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.555355548858643, "metrics/particle/momentum/class_2/accepted/samples": 1038.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8074727654457092, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8461301326751709, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.2131803035736084, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7950275540351868, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35474729537963867, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.499532699584961, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06376948207616806, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.837395668029785, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.0632428452372551, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.26895809173584, "metrics/particle/momentum/class_2/assigned/samples": 2117.0, "metrics/particle/momentum/class_2/efficiency": 0.34762224555015564, "metrics/particle/momentum/class_2/event/energy_bias": -32.70158386230469, "metrics/particle/momentum/class_2/event/p3_error_mean": 27.503793716430664, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.764642715454102, "metrics/particle/momentum/class_2/event/px_bias": 0.3113163113594055, "metrics/particle/momentum/class_2/event/py_bias": -0.6627373695373535, "metrics/particle/momentum/class_2/event/pz_abs_mean": 19.544322967529297, "metrics/particle/momentum/class_2/event/pz_bias": -2.95106840133667, "metrics/particle/momentum/class_2/event/pz_residual_rms": 31.059715270996094, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.026766972616314888, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.18141277134418488, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17714281380176544, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013186977244913578, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21852000057697296, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0031971086282283068, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4489997327327728, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.061749666929244995, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3317207098007202, "metrics/particle/momentum/class_3/accepted/samples": 7194.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.8205626606941223, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.031769275665283, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.187117576599121, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.052093029022217, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.41953417658805847, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 47.976192474365234, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.022939976304769516, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.3867743015289307, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04813915491104126, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.59097957611084, "metrics/particle/momentum/class_3/assigned/samples": 14211.0, "metrics/particle/momentum/class_3/efficiency": 0.463142991065979, "metrics/particle/momentum/class_3/event/energy_bias": 12.621255874633789, "metrics/particle/momentum/class_3/event/p3_error_mean": 22.732616424560547, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 13.816556930541992, "metrics/particle/momentum/class_3/event/px_bias": -2.810088872909546, "metrics/particle/momentum/class_3/event/py_bias": 0.9229075312614441, "metrics/particle/momentum/class_3/event/pz_abs_mean": 15.261126518249512, "metrics/particle/momentum/class_3/event/pz_bias": 0.2000299096107483, "metrics/particle/momentum/class_3/event/pz_residual_rms": 27.0721492767334, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04038553312420845, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.21605749428272247, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.21887384355068207, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.08312641829252243, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2571274936199188, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.6281802654266357, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.08257082849740982, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.1028571128845215, "metrics/particle/momentum/class_4/accepted/samples": 177.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.7027117013931274, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 12.397420883178711, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 8.958086967468262, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 8.765711784362793, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.44602271914482117, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 39.409881591796875, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.017045455053448677, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.8069950342178345, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.08979173749685287, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.7192511558532715, "metrics/particle/momentum/class_4/assigned/samples": 352.0, "metrics/particle/momentum/class_4/efficiency": 0.47200000286102295, "metrics/particle/momentum/class_4/event/energy_bias": -3.639355182647705, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.541962623596191, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.5108203887939453, "metrics/particle/momentum/class_4/event/px_bias": 0.36563336849212646, "metrics/particle/momentum/class_4/event/py_bias": -0.20909249782562256, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.2971277236938477, "metrics/particle/momentum/class_4/event/pz_bias": 0.03910266235470772, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.917516708374023, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0411495603621006, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.23577482998371124, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.23933015763759613, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08036316186189651, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2822779715061188, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05154639109969139, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.3544209003448486, "metrics/particle/momentum/class_5/accepted/pz_bias": 1.0226002931594849, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.634887218475342, "metrics/particle/momentum/class_5/accepted/samples": 97.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.29120489954948425, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.8227011561393738, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.8643348217010498, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4033525586128235, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3243243098258972, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.8265304565429688, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.10810811072587967, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.048918724060059, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.4508613348007202, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 8.762173652648926, "metrics/particle/momentum/class_5/assigned/samples": 148.0, "metrics/particle/momentum/class_5/efficiency": 0.5914633870124817, "metrics/particle/momentum/class_5/event/energy_bias": -2.641892194747925, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.0764994621276855, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.1867802143096924, "metrics/particle/momentum/class_5/event/px_bias": 0.5416585803031921, "metrics/particle/momentum/class_5/event/py_bias": -0.2796623706817627, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.0603253841400146, "metrics/particle/momentum/class_5/event/pz_bias": -0.12392708659172058, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.587377548217773, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6978179812431335, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.7049060463905334, "metrics/set_slot/angular_recall_all_dr0p10": 0.8324088454246521, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6200357675552368, "metrics/set_slot/soft_count_bias_mean": -0.08877599984407425}, "step_train_time": 4.4991068840026855, "step_valid_time": 12165.23874592781, "step_total_time": 12169.73785328865, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1043127338646046, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2866479342505106, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2981, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2554, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2378, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8349133703824779, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.856759476685676, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8456953642384106, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.14324052331432405, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7773782281791435, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7977188862797719, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7874172185430464, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8349133703824779, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0744328828137943, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24839115269279355, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2216, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1913, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1839, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8371991247264771, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8632671480144405, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8500333259275716, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13673285198555957, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8048140043763676, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8298736462093863, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8171517440568763, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8371991247264771, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0540096211832712, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25270880888017166, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1247, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1058, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1013, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.800302571860817, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8484362469927826, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.82366679641884, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.15156375300721733, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7662632375189108, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8123496391339214, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7886337096146361, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.800302571860817} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_50000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..45b4f6617e85a1ef71961a7f26c37ef7ffecf192 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 2.3683213338385753, "train": {"Classification_binary": 0.1583002656698227, "Cardinality": 0.11956053972244263, "Classification": 0.09019576758146286, "Regression_pt": 0.11047506332397461, "Regression_eta": 0.0013884601648896933, "Regression_sin_phi": 0.0006562697817571461, "Regression_cos_phi": 0.0006931284442543983, "Regression_energy": 0.10949410498142242, "HitGrouping": 0.1834360510110855, "Auxiliary": 0.22971706092357635, "Total": 1.0035524368286133, "Total_unweighted": 0.4712030599475838}, "train_group": {}, "valid": {"Classification_binary": 0.14361808697382608, "Cardinality": 0.11147816975911458, "Classification": 0.08609896898269653, "Regression_pt": 0.10857922832171123, "Regression_eta": 0.00118991918861866, "Regression_sin_phi": 0.0006292453035712242, "Regression_cos_phi": 0.0006212607646981875, "Regression_energy": 0.10796856880187988, "HitGrouping": 0.1849674383799235, "Auxiliary": 0.21689939498901367, "Total": 0.9515787760416666, "Total_unweighted": 0.4487052783370018, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8295422792434692, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5163463950157166, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.34153497219085693, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.521473228931427, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5179433822631836, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3991672098636627, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7807071805000305, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4608118236064911, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.33013293147087097, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4657863676548004, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.46920451521873474, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.46713441610336304, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8478807210922241, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4805143475532532, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3127516806125641, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4845183789730072, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.46995827555656433, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.35315898060798645, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8371098637580872, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4898401200771332, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.32282111048698425, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.49425968527793884, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4844369888305664, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3759461045265198, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003073234809562564, "metrics/hit_grouping/calorimeter/background_recall": 0.1348094493150711, "metrics/hit_grouping/calorimeter/completeness": 0.5616315007209778, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5689783692359924, "metrics/hit_grouping/calorimeter/energy_purity": 0.5614989399909973, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3663199245929718, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8606874346733093, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.513213574886322, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2506258487701416, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.518769383430481, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5043496489524841, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.31383705139160156, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8381841778755188, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5111982226371765, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3534085750579834, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5155569911003113, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5064949989318848, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.38403961062431335, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7879531383514404, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4206653833389282, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4392111301422119, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.42282941937446594, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.42161381244659424, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4944315552711487, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8272653222084045, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5484799146652222, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4036564826965332, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5559089779853821, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5487360954284668, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4062424898147583, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8005338311195374, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5892772674560547, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5496654510498047, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5873311161994934, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5715857744216919, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.43592381477355957, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8515045046806335, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.41820675134658813, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2133791148662567, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.42031630873680115, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.40849754214286804, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.33791157603263855, "metrics/hit_grouping/calorimeter/purity": 0.5534096360206604, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8114169836044312, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08756040781736374, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.09398496150970459, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08670112490653992, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.04902176558971405, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0357142873108387, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8092514276504517, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.18943467736244202, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09417304396629333, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19041039049625397, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.17259597778320312, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.27133607864379883, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8389781713485718, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5120276808738708, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.33844834566116333, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5167124271392822, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5077595710754395, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.38693374395370483, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7455956339836121, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40141966938972473, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5149648785591125, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5416553616523743, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5545415282249451, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7684316635131836, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2754671573638916, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.47982701659202576, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4947558343410492, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4327908456325531, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6799938678741455, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4331558644771576, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4456488788127899, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.46621543169021606, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6432580351829529, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.7066887021064758, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41175422072410583, "metrics/hit_grouping/tracker/all/object_completeness": 0.46947741508483887, "metrics/hit_grouping/tracker/all/object_purity": 0.49150973558425903, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6004452109336853, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.012973845936357975, "metrics/hit_grouping/tracker/background_recall": 0.114066943526268, "metrics/hit_grouping/tracker/completeness": 0.4675759971141815, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.46041151881217957, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7206393480300903, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40504997968673706, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5197666883468628, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5354957580566406, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5925828218460083, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.7020785212516785, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42040297389030457, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4596676528453827, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.48992663621902466, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6048284769058228, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6825729012489319, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4169522225856781, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3757792115211487, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4026038646697998, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6122633218765259, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.7308143377304077, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.426509290933609, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.5015465021133423, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5222211480140686, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.5590558052062988, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7162125110626221, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.435960590839386, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.46571028232574463, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.47440972924232483, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5603448152542114, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6736172437667847, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3882327079772949, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.42846208810806274, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4546494483947754, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6614416837692261, "metrics/hit_grouping/tracker/purity": 0.44194990396499634, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8815503716468811, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02752639539539814, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.06501612067222595, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.06738203763961792, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06975867599248886, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6636929512023926, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2627774775028229, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.33437711000442505, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.35605406761169434, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5668559670448303, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6854000687599182, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4849196970462799, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5441414713859558, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5691307187080383, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.6828381419181824, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5448068380355835, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6464522480964661, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7415992617607117, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6373893022537231, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.004345910158008337, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06291446834802628, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.396465301513672, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3066052496433258, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.9249999523162842, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.2149820625782013, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9212152361869812, "metrics/particle/class_1/pid_accuracy": 0.9770200848579407, "metrics/particle/class_2/efficiency": 0.5193113684654236, "metrics/particle/class_2/pid_accuracy": 0.7214831113815308, "metrics/particle/class_3/efficiency": 0.792405366897583, "metrics/particle/class_3/pid_accuracy": 0.9777361154556274, "metrics/particle/class_4/efficiency": 0.7374749779701233, "metrics/particle/class_4/pid_accuracy": 0.49320653080940247, "metrics/particle/class_5/efficiency": 0.9712643623352051, "metrics/particle/class_5/pid_accuracy": 0.4911242723464966, "metrics/particle/count/bias_mean": -2.16796875, "metrics/particle/count/mae": 3.71875, "metrics/particle/count/prediction_mean": 68.48697662353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07351379096508026, "metrics/particle/event/energy_response_mean": 0.9751690030097961, "metrics/particle/event/met_abs_error": 5.562965393066406, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0546671561896801, "metrics/particle/event/scalar_pt_response_mean": 0.9961664080619812, "metrics/particle/event/vector_pt_closure": 0.053244542330503464, "metrics/particle/matched/delta_eta_abs_mean": 0.011381357908248901, "metrics/particle/matched/delta_phi_abs_mean": 0.010918485932052135, "metrics/particle/matched/delta_r_mean": 0.017662784084677696, "metrics/particle/matched/energy_relative_abs_mean": 0.1292753666639328, "metrics/particle/matched/pid_accuracy": 0.9576777815818787, "metrics/particle/matched/pt_relative_abs_mean": 0.12981022894382477, "metrics/particle/matching/duplicate_fraction": 0.0457431823015213, "metrics/particle/matching/efficiency": 0.8327773809432983, "metrics/particle/matching/f1": 0.8457528948783875, "metrics/particle/matching/purity": 0.8591391444206238, "metrics/particle/matching/target_coverage_dr0p05": 0.8148001432418823, "metrics/particle/matching/target_coverage_dr0p10": 0.874140739440918, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6606711745262146, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8570941686630249, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017589684575796127, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13045577704906464, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13102221488952637, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02276383526623249, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17072942852973938, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006788337603211403, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6241467595100403, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.07805907726287842, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8347469568252563, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8986.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1649400293827057, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7199938893318176, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7565522193908691, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6275873780250549, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09961051493883133, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.3510260581970215, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013729308731853962, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8979869484901428, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.09029551595449448, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8592329025268555, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10270.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8341997861862183, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.5079500675201416, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.540544509887695, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.4570746421813965, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.28659558296203613, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.10254893451929092, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.383844375610352, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.9288722276687622, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.149632453918457, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02327938750386238, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14654044806957245, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1475847214460373, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.011365669779479504, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18730297684669495, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023782558739185333, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.265324354171753, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1831762194633484, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1874308586120605, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1766.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4407881200313568, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9181366562843323, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.413626194000244, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.246180534362793, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18542474508285522, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.213144302368164, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05476498603820801, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.311424970626831, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.34635159373283386, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.262278079986572, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2319.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6403190493583679, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.415369987487793, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.767375946044922, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1998353004455566, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.17589646577835083, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.010785238817334175, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.620665550231934, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3331762552261353, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.416683197021484, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017293646931648254, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12637385725975037, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12714457511901855, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01439877413213253, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16703684628009796, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00045239771134220064, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2004595249891281, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.003538511460646987, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6765270829200745, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19894.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16285820305347443, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8281100392341614, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6189254522323608, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.47805896401405334, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09892624616622925, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.571256160736084, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002904418157413602, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.343830943107605, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.005082114599645138, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4276660680770874, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22724.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8549572229385376, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.51796293258667, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.357473373413086, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.060486793518066, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.21091067790985107, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.19508448243141174, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.698965072631836, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5550241470336914, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.787221908569336, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0177253857254982, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12873287498950958, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1294594407081604, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01667678728699684, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1693541705608368, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0036546369083225727, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.386056512594223, "metrics/particle/momentum/all/accepted/pz_bias": -0.03574114292860031, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3673464059829712, "metrics/particle/momentum/all/accepted/samples": 30646.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18171527981758118, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8025789260864258, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7768088579177856, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6376583576202393, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1048055961728096, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 7.662491321563721, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009458272717893124, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6342065334320068, "metrics/particle/momentum/all/assigned/pz_bias": -0.0457349494099617, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.674445390701294, "metrics/particle/momentum/all/assigned/samples": 35313.0, "metrics/particle/momentum/all/efficiency": 0.8327943682670593, "metrics/particle/momentum/all/event/energy_bias": -3.3894574642181396, "metrics/particle/momentum/all/event/p3_error_mean": 18.238574981689453, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.824028015136719, "metrics/particle/momentum/all/event/px_bias": -0.6734027862548828, "metrics/particle/momentum/all/event/py_bias": -0.3084186911582947, "metrics/particle/momentum/all/event/pz_abs_mean": 13.57231616973877, "metrics/particle/momentum/all/event/pz_bias": -3.817072868347168, "metrics/particle/momentum/all/event/pz_residual_rms": 24.172534942626953, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01773432269692421, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10321827977895737, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10448992997407913, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.014136693440377712, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.13893473148345947, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002818109467625618, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.37057459354400635, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.05147378891706467, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2587454319000244, "metrics/particle/momentum/class_1/accepted/samples": 16323.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.08931257575750351, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.25885507464408875, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3085915446281433, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2002900093793869, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.044953495264053345, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.680433988571167, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005741187371313572, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.49485400319099426, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.06860119104385376, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.195080518722534, "metrics/particle/momentum/class_1/assigned/samples": 17418.0, "metrics/particle/momentum/class_1/efficiency": 0.9200721383094788, "metrics/particle/momentum/class_1/event/energy_bias": 4.081209182739258, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.854984283447266, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.91089391708374, "metrics/particle/momentum/class_1/event/px_bias": 0.07753618061542511, "metrics/particle/momentum/class_1/event/py_bias": 0.030612193048000336, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.64284610748291, "metrics/particle/momentum/class_1/event/pz_bias": -2.6417150497436523, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.403106689453125, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02677191235125065, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17770221829414368, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17900101840496063, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.0035508209839463234, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22079521417617798, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.026640675961971283, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4388668537139893, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.13217706978321075, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4461162090301514, "metrics/particle/momentum/class_2/accepted/samples": 1539.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5412396192550659, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.719610333442688, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9314432740211487, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6233004331588745, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2718120813369751, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.16832971572876, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05285234749317169, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.347386121749878, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.025586452335119247, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.515668869018555, "metrics/particle/momentum/class_2/assigned/samples": 2384.0, "metrics/particle/momentum/class_2/efficiency": 0.5154052376747131, "metrics/particle/momentum/class_2/event/energy_bias": -9.471697807312012, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.640054702758789, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.368122100830078, "metrics/particle/momentum/class_2/event/px_bias": -1.0732423067092896, "metrics/particle/momentum/class_2/event/py_bias": -0.26396170258522034, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.836777687072754, "metrics/particle/momentum/class_2/event/pz_bias": -0.8307064771652222, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.508010864257812, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0166430976241827, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15561436116695404, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15573051571846008, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.020024776458740234, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19582460820674896, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015380879631265998, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2586245834827423, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.00774212833493948, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.933030903339386, "metrics/particle/momentum/class_3/accepted/samples": 12353.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.23098841309547424, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4206608533859253, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2738945484161377, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1239285469055176, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14620311558246613, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.78528881072998, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006672894582152367, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5144478678703308, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.024717548862099648, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4941787719726562, "metrics/particle/momentum/class_3/assigned/samples": 14986.0, "metrics/particle/momentum/class_3/efficiency": 0.795274555683136, "metrics/particle/momentum/class_3/event/energy_bias": 3.8005318641662598, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.689870834350586, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.658440589904785, "metrics/particle/momentum/class_3/event/px_bias": 0.3778034746646881, "metrics/particle/momentum/class_3/event/py_bias": -0.05189199000597, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.701438903808594, "metrics/particle/momentum/class_3/event/pz_bias": -0.35025525093078613, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.284982681274414, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.018589399755001068, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16082556545734406, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15624448657035828, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08672093600034714, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20019365847110748, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0036900369450449944, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5929079055786133, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.021596228703856468, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4734092950820923, "metrics/particle/momentum/class_4/accepted/samples": 271.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.28480565547943115, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.228076457977295, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.004300117492676, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9253963232040405, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.21329639852046967, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.19601058959961, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005540166050195694, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6628909111022949, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.022881684824824333, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.5815743207931519, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.7226666808128357, "metrics/particle/momentum/class_4/event/energy_bias": -1.171586513519287, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.254642963409424, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7126812934875488, "metrics/particle/momentum/class_4/event/px_bias": 0.05025477707386017, "metrics/particle/momentum/class_4/event/py_bias": 0.06953249126672745, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1740156412124634, "metrics/particle/momentum/class_4/event/pz_bias": -0.09656910598278046, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.580923080444336, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011893317103385925, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13089951872825623, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12663032114505768, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02494613453745842, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.163560152053833, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03125, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3269679546356201, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.23806241154670715, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.368173599243164, "metrics/particle/momentum/class_5/accepted/samples": 160.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03989315778017044, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.13916485011577606, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.13234953582286835, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.02906196378171444, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.012195121496915817, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.17391853034496307, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03658536449074745, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.4108411073684692, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.11910189688205719, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.49043607711792, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9756097793579102, "metrics/particle/momentum/class_5/event/energy_bias": -0.6279146075248718, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.750986099243164, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3110072612762451, "metrics/particle/momentum/class_5/event/px_bias": -0.1057548001408577, "metrics/particle/momentum/class_5/event/py_bias": -0.09270971268415451, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9223455786705017, "metrics/particle/momentum/class_5/event/pz_bias": 0.10217328369617462, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.039609432220459, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3028225898742676, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3038299083709717, "metrics/set_slot/angular_recall_all_dr0p10": 0.9620736241340637, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8628991842269897, "metrics/set_slot/soft_count_bias_mean": -0.18570178747177124}, "step_train_time": 4.957149982452393, "step_valid_time": 38226.41109967232, "step_total_time": 38231.36824989319, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0089143224618362, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12604339995519698, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2916, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2905, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9532526969597908, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9592105263157895, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.956222331529759, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04078947368421053, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9496567505720824, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9555921052631579, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9526151828168552, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9532526969597908, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991088114473379, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11889581878781597, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2263, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2160, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2155, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9452954048140044, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9544851966416261, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9498680738786279, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04551480335837384, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9431072210065645, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9522757401679187, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9476693051890941, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9452954048140044, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0135616273027641, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12810749882050143, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1311, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1240, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1234, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9379727685325264, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9458428680396643, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.941891378655526, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05415713196033562, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9334341906202723, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9412662090007627, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9373338397265477, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9379727685325264} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_6000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..689328e793ba3132d6cb1080d12c1e7622033c4b --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 0.28419856006062905, "train": {"Classification_binary": 0.2847760021686554, "Cardinality": 0.191626638174057, "Classification": 0.20910578966140747, "Regression_pt": 0.4028857350349426, "Regression_eta": 0.004112076945602894, "Regression_sin_phi": 0.002180269453674555, "Regression_cos_phi": 0.0018734581535682082, "Regression_energy": 0.3965788781642914, "HitGrouping": 0.2745651602745056, "Auxiliary": 0.41794973611831665, "Total": 2.137178421020508, "Total_unweighted": 1.3015122095821425}, "train_group": {}, "valid": {"Classification_binary": 0.3390121857325236, "Cardinality": 0.2470248540242513, "Classification": 0.24837015072504678, "Regression_pt": 0.39946409066518146, "Regression_eta": 0.004605073171357314, "Regression_sin_phi": 0.0024222773499786854, "Regression_cos_phi": 0.002405394489566485, "Regression_energy": 0.39356176058451336, "HitGrouping": 0.2742604414621989, "Auxiliary": 0.46011122067769367, "Total": 2.3871386845906577, "Total_unweighted": 1.3898409327181678, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7500571012496948, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.22058993577957153, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4072111248970032, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.20995086431503296, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.20121049880981445, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6034825444221497, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.729945957660675, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.20695438981056213, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.35450518131256104, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.19806036353111267, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.1791626214981079, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6074593663215637, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.751306414604187, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.2065601646900177, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.37491321563720703, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.19624106585979462, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.19443486630916595, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6107382774353027, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7492695450782776, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.2108411341905594, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3831135332584381, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.20053574442863464, "metrics/hit_grouping/calorimeter/all/object_purity": 0.1953018605709076, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6082855463027954, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0004208204918541014, "metrics/hit_grouping/calorimeter/background_recall": 0.01835043355822563, "metrics/hit_grouping/calorimeter/completeness": 0.28049370646476746, "metrics/hit_grouping/calorimeter/energy_completeness": 0.30554139614105225, "metrics/hit_grouping/calorimeter/energy_purity": 0.30119895935058594, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5888403654098511, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7729777693748474, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.2192016839981079, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.322712779045105, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.21108834445476532, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.21083970367908478, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5583750605583191, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7344785928726196, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2194652408361435, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.413173645734787, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.20699083805084229, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.20317970216274261, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6471672058105469, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7158296704292297, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.1851060688495636, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4759860932826996, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.17251616716384888, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.1556841880083084, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6708816885948181, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7159063220024109, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.23639701306819916, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.47798892855644226, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.22286079823970795, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.21576552093029022, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6736227869987488, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7440241575241089, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3616478741168976, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.7195059061050415, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.35126832127571106, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3305414617061615, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6078229546546936, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7839102745056152, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.1668187826871872, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24630571901798248, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.15981623530387878, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.1583259105682373, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5417254567146301, "metrics/hit_grouping/calorimeter/purity": 0.2756540775299072, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8844075202941895, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.04108108952641487, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.129699245095253, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.03900375962257385, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.012718391604721546, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0357142873108387, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8361218571662903, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.05955275520682335, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11418481171131134, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.05799058452248573, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.038473308086395264, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.3472630977630615, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7425479292869568, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.2214800864458084, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4012373983860016, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.21058565378189087, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.2064380645751953, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.631198525428772, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6938456892967224, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.439374178647995, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.30476096272468567, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.26685070991516113, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6749239563941956, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7351316809654236, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3291139304637909, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3041220009326935, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.265004426240921, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5207956433296204, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6270380020141602, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43272411823272705, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.2449161857366562, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.20291128754615784, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7445675730705261, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6555554270744324, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.427114874124527, "metrics/hit_grouping/tracker/all/object_completeness": 0.2676824629306793, "metrics/hit_grouping/tracker/all/object_purity": 0.22714930772781372, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7066340446472168, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009686044417321682, "metrics/hit_grouping/tracker/background_recall": 0.06220397725701332, "metrics/hit_grouping/tracker/completeness": 0.27715763449668884, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.650905191898346, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6613867878913879, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42416271567344666, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.3196142911911011, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.28426042199134827, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7152376174926758, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6579777598381042, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42203667759895325, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.24409684538841248, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.19526126980781555, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6981303095817566, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6411537528038025, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4382326304912567, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.18429182469844818, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.14135348796844482, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6973850131034851, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6734190583229065, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4650590121746063, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.2757343649864197, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.21892927587032318, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6885454654693604, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6788967251777649, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.517241358757019, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.2851429879665375, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.23320496082305908, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6607142686843872, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6281476616859436, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.36162930727005005, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.25408461689949036, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.23675008118152618, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7383817434310913, "metrics/hit_grouping/tracker/purity": 0.2610979378223419, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8979797959327698, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.016591252759099007, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.008151081390678883, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0043573640286922455, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.059954751282930374, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6725953221321106, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.18069179356098175, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.10009261220693588, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.06420911848545074, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5751161575317383, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6177335381507874, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5147445797920227, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.32436078786849976, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.27787166833877563, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8168541193008423, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.38148927688598633, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5573204159736633, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6259918212890625, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5992190837860107, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.16766345500946045, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.02474426105618477, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.751285552978516, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.22300033271312714, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.13137714564800262, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5607430338859558, "metrics/particle/class_1/pid_accuracy": 0.8838854432106018, "metrics/particle/class_2/efficiency": 0.3420546054840088, "metrics/particle/class_2/pid_accuracy": 0.1940397322177887, "metrics/particle/class_3/efficiency": 0.494851678609848, "metrics/particle/class_3/pid_accuracy": 0.8995405435562134, "metrics/particle/class_4/efficiency": 0.5060120224952698, "metrics/particle/class_4/pid_accuracy": 0.18217821419239044, "metrics/particle/class_5/efficiency": 0.6149425506591797, "metrics/particle/class_5/pid_accuracy": 0.2554517090320587, "metrics/particle/count/bias_mean": -6.881510257720947, "metrics/particle/count/mae": 8.838541984558105, "metrics/particle/count/prediction_mean": 63.7734375, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.11847403645515442, "metrics/particle/event/energy_response_mean": 0.992428719997406, "metrics/particle/event/met_abs_error": 10.865757942199707, "metrics/particle/event/scalar_pt_relative_abs_error": 0.10299541801214218, "metrics/particle/event/scalar_pt_response_mean": 1.011954665184021, "metrics/particle/event/vector_pt_closure": 0.09794216603040695, "metrics/particle/matched/delta_eta_abs_mean": 0.020308595150709152, "metrics/particle/matched/delta_phi_abs_mean": 0.020618174225091934, "metrics/particle/matched/delta_r_mean": 0.03241092339158058, "metrics/particle/matched/energy_relative_abs_mean": 0.2016424834728241, "metrics/particle/matched/pid_accuracy": 0.8429961800575256, "metrics/particle/matched/pt_relative_abs_mean": 0.19847308099269867, "metrics/particle/matching/duplicate_fraction": 0.10840581357479095, "metrics/particle/matching/efficiency": 0.5149365067481995, "metrics/particle/matching/f1": 0.5412966012954712, "metrics/particle/matching/purity": 0.5705010294914246, "metrics/particle/matching/target_coverage_dr0p05": 0.4655013680458069, "metrics/particle/matching/target_coverage_dr0p10": 0.6114479303359985, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.31623756885528564, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5867165327072144, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03091992251574993, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20183677971363068, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.19923467934131622, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.032254453748464584, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24090589582920074, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.015849338844418526, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.2393416166305542, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.06396661698818207, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.9170408248901367, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5363.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.5982722640037537, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.2925617694854736, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.747880458831787, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.519622564315796, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.33130261301994324, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.333080291748047, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.036552947014570236, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.984901785850525, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06228304281830788, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.79539155960083, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9028.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.49786484241485596, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.004056692123413, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 14.314382553100586, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.486363410949707, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.4721294343471527, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.158854603767395, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 12.171689987182617, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.245591402053833, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 20.030488967895508, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030811022967100143, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.20456983149051666, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.19649763405323029, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.033156536519527435, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2384650856256485, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04633920267224312, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.0321292877197266, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.17575706541538239, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.692924976348877, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1079.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.9126277565956116, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8020048141479492, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.954907417297363, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.708456516265869, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3836978077888489, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 53.091407775878906, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0834990069270134, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.6574337482452393, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.37144771218299866, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.407783508300781, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2012.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.39122551679611206, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.218022346496582, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.202836990356445, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.8785383701324463, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.20328865945339203, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.005637705326080322, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.979843139648438, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.8001608848571777, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 29.378223419189453, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03310919180512428, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20199033617973328, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1986587643623352, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.029129035770893097, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23938292264938354, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0022569724824279547, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.4442206025123596, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00040799827547743917, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.1930310726165771, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 12406.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.5663446187973022, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.163405418395996, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.5445780754089355, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.3266046047210693, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.34587088227272034, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.91819953918457, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.012047084979712963, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.9899837970733643, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.026192989200353622, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.1596128940582275, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21748.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5331556797027588, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.089260101318359, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 15.561238288879395, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 13.251174926757812, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.0041112899780273, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.942202091217041, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.682810306549072, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.2905232906341553, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.415788650512695, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0323546938598156, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20209431648254395, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.19869890809059143, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.03024890460073948, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23976489901542664, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.008648131974041462, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.761367678642273, "metrics/particle/momentum/all/accepted/pz_bias": -0.02853122167289257, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.1491451263427734, "metrics/particle/momentum/all/accepted/samples": 18848.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.59638512134552, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.8400826454162598, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.8098273277282715, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.5872747898101807, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.34418079257011414, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.445117950439453, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.02317921258509159, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.4276143312454224, "metrics/particle/momentum/all/assigned/pz_bias": -0.0573163740336895, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.427252292633057, "metrics/particle/momentum/all/assigned/samples": 32788.0, "metrics/particle/momentum/all/efficiency": 0.5121878385543823, "metrics/particle/momentum/all/event/energy_bias": 0.8752939105033875, "metrics/particle/momentum/all/event/p3_error_mean": 30.450315475463867, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.938573837280273, "metrics/particle/momentum/all/event/px_bias": -2.6795291900634766, "metrics/particle/momentum/all/event/py_bias": -3.106694459915161, "metrics/particle/momentum/all/event/pz_abs_mean": 22.106426239013672, "metrics/particle/momentum/all/event/pz_bias": -7.336275100708008, "metrics/particle/momentum/all/event/pz_residual_rms": 34.85316848754883, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03842996433377266, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2204810529947281, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21898537874221802, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.04860450327396393, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2581638991832733, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.010291420854628086, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.9158892631530762, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.07378891855478287, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.325186014175415, "metrics/particle/momentum/class_1/accepted/samples": 9814.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.44830718636512756, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.609250545501709, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.8564497232437134, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.6020971536636353, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.3110092282295227, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 12.476910591125488, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.019701382145285606, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.3601760864257812, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.07578300684690475, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.6898670196533203, "metrics/particle/momentum/class_1/assigned/samples": 16141.0, "metrics/particle/momentum/class_1/efficiency": 0.5531818866729736, "metrics/particle/momentum/class_1/event/energy_bias": 13.37393569946289, "metrics/particle/momentum/class_1/event/p3_error_mean": 26.408241271972656, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.932819366455078, "metrics/particle/momentum/class_1/event/px_bias": -2.847727060317993, "metrics/particle/momentum/class_1/event/py_bias": 0.08615484833717346, "metrics/particle/momentum/class_1/event/pz_abs_mean": 17.522586822509766, "metrics/particle/momentum/class_1/event/pz_bias": -3.7942893505096436, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.814390182495117, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04304236173629761, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22857603430747986, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22557500004768372, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06627465039491653, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2647908926010132, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.032977692782878876, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8351287841796875, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.10866484045982361, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.121700286865234, "metrics/particle/momentum/class_2/accepted/samples": 1031.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7556054592132568, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7803825736045837, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0764957666397095, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6610156297683716, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34382566809654236, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.2910356521606445, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06198547035455704, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.84028697013855, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.013972073793411255, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.6416401863098145, "metrics/particle/momentum/class_2/assigned/samples": 2065.0, "metrics/particle/momentum/class_2/efficiency": 0.34527796506881714, "metrics/particle/momentum/class_2/event/energy_bias": -16.11117172241211, "metrics/particle/momentum/class_2/event/p3_error_mean": 28.089885711669922, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 15.873356819152832, "metrics/particle/momentum/class_2/event/px_bias": -0.9595957398414612, "metrics/particle/momentum/class_2/event/py_bias": -1.165971279144287, "metrics/particle/momentum/class_2/event/pz_abs_mean": 19.346965789794922, "metrics/particle/momentum/class_2/event/pz_bias": -1.7259128093719482, "metrics/particle/momentum/class_2/event/pz_residual_rms": 30.861528396606445, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023039430379867554, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1743817925453186, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1687544882297516, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.0009517577127553523, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20930279791355133, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002591680735349655, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3851141929626465, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.025995265692472458, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2465825080871582, "metrics/particle/momentum/class_3/accepted/samples": 7717.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.7444048523902893, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.389135837554932, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.050939559936523, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.893707752227783, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.38245514035224915, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 23.49138641357422, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.02063683420419693, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.2731980085372925, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.04786797985434532, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.76510763168335, "metrics/particle/momentum/class_3/assigned/samples": 14101.0, "metrics/particle/momentum/class_3/efficiency": 0.49681323766708374, "metrics/particle/momentum/class_3/event/energy_bias": 8.298812866210938, "metrics/particle/momentum/class_3/event/p3_error_mean": 21.70838165283203, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 13.067129135131836, "metrics/particle/momentum/class_3/event/px_bias": 0.5808985233306885, "metrics/particle/momentum/class_3/event/py_bias": -1.673520803451538, "metrics/particle/momentum/class_3/event/pz_abs_mean": 14.4243803024292, "metrics/particle/momentum/class_3/event/pz_bias": -1.7842597961425781, "metrics/particle/momentum/class_3/event/pz_residual_rms": 26.335763931274414, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03881293907761574, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.20845729112625122, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.196563258767128, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.04781221225857735, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.24100947380065918, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.011173184029757977, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.3788679838180542, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.10579968988895416, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6567790508270264, "metrics/particle/momentum/class_4/accepted/samples": 179.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.7048996090888977, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 10.378686904907227, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 7.858658313751221, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 7.684969902038574, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.38643068075180054, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 49.127628326416016, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.029498524963855743, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.6848187446594238, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.045979101210832596, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.603628396987915, "metrics/particle/momentum/class_4/assigned/samples": 339.0, "metrics/particle/momentum/class_4/efficiency": 0.47733333706855774, "metrics/particle/momentum/class_4/event/energy_bias": -3.157512664794922, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.026008605957031, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.27860164642334, "metrics/particle/momentum/class_4/event/px_bias": 0.3816586136817932, "metrics/particle/momentum/class_4/event/py_bias": 0.03998500108718872, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8367503881454468, "metrics/particle/momentum/class_4/event/pz_bias": 0.09009990096092224, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.763404369354248, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03317840397357941, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.24852703511714935, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.24227602779865265, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08312578499317169, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.28482741117477417, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05607476457953453, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.345412015914917, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.7373624444007874, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.490708351135254, "metrics/particle/momentum/class_5/accepted/samples": 107.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.15498290956020355, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.8777825832366943, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.0867598056793213, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6812392473220825, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.21830986440181732, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.759942054748535, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09154929220676422, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.2697513103485107, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.4461260139942169, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 7.13479471206665, "metrics/particle/momentum/class_5/assigned/samples": 142.0, "metrics/particle/momentum/class_5/efficiency": 0.6524389982223511, "metrics/particle/momentum/class_5/event/energy_bias": -1.5287714004516602, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.9498791694641113, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.974836826324463, "metrics/particle/momentum/class_5/event/px_bias": 0.16523641347885132, "metrics/particle/momentum/class_5/event/py_bias": -0.3933417499065399, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.03782320022583, "metrics/particle/momentum/class_5/event/pz_bias": -0.12191331386566162, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.504781723022461, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6604992151260376, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6657230854034424, "metrics/set_slot/angular_recall_all_dr0p10": 0.8441571593284607, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6407681107521057, "metrics/set_slot/soft_count_bias_mean": 1.399422287940979}, "step_train_time": 4.444360256195068, "step_valid_time": 14751.808403253555, "step_total_time": 14756.252763748169, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0522774283052647, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26556620244056417, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3003, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2618, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2489, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8558352402745996, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8717948717948718, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8637413394919169, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1282051282051282, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8136645962732919, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8288378288378289, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8211811283404817, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8558352402745996, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0203856552346873, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24405203519515872, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2243, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1984, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1940, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8682713347921225, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8845296477931341, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8763250883392226, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1154703522068658, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.849015317286652, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8649130628622381, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8568904593639576, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8682713347921225, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0065726010413407, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2576597562162206, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1268, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1098, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1058, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8305597579425114, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8659305993690851, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8478764478764479, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13406940063091483, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.800302571860817, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8343848580441641, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.816988416988417, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8305597579425114} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_7000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..ecc54326e28fb26a2190da3def2b80a006052972 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 0.33156498673740054, "train": {"Classification_binary": 0.3002185821533203, "Cardinality": 0.27458456158638, "Classification": 0.1970474123954773, "Regression_pt": 0.3851776123046875, "Regression_eta": 0.004490057937800884, "Regression_sin_phi": 0.0022108822595328093, "Regression_cos_phi": 0.002007860690355301, "Regression_energy": 0.37953588366508484, "HitGrouping": 0.2655137777328491, "Auxiliary": 0.43224191665649414, "Total": 2.2409613132476807, "Total_unweighted": 1.270688291406259}, "train_group": {}, "valid": {"Classification_binary": 0.3335030476252238, "Cardinality": 0.25917484362920123, "Classification": 0.21732630332310995, "Regression_pt": 0.37484057744344074, "Regression_eta": 0.004450843048592408, "Regression_sin_phi": 0.0023219315335154533, "Regression_cos_phi": 0.0023231420976420245, "Regression_energy": 0.36768146355946857, "HitGrouping": 0.26631975173950195, "Auxiliary": 0.4375809033711751, "Total": 2.3006165822347007, "Total_unweighted": 1.302447308630993, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7354575395584106, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.24165014922618866, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.42727360129356384, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.22969380021095276, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.2280699759721756, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6297908425331116, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.694376528263092, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.22041355073451996, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.36410635709762573, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.2097882181406021, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.20399528741836548, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6713441610336304, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7511096000671387, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.22577384114265442, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.38602176308631897, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.2156393975019455, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.21973365545272827, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6133302450180054, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7419630885124207, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.2301674336194992, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3968176543712616, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.2194429188966751, "metrics/hit_grouping/calorimeter/all/object_purity": 0.22103729844093323, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6228210926055908, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.001099860412068665, "metrics/hit_grouping/calorimeter/background_recall": 0.04841301217675209, "metrics/hit_grouping/calorimeter/completeness": 0.299167662858963, "metrics/hit_grouping/calorimeter/energy_completeness": 0.3349277377128601, "metrics/hit_grouping/calorimeter/energy_purity": 0.3302539885044098, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5813093781471252, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7690865993499756, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.23946984112262726, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.33756259083747864, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.23143641650676727, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.23651033639907837, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5668525099754333, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7265674471855164, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.24074174463748932, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.43493780493736267, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.2269863784313202, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.2320065200328827, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6594887375831604, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7021687626838684, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2005472481250763, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.47923433780670166, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.18738903105258942, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.17843739688396454, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.699999988079071, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7089411616325378, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.261378675699234, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5037887692451477, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.24754032492637634, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2473974972963333, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6888982653617859, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7139312028884888, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.40379858016967773, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6829645037651062, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.38825124502182007, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3599376380443573, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6644364595413208, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7789705395698547, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.17765989899635315, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2536636292934418, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.17068469524383545, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.17761336266994476, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.550493597984314, "metrics/hit_grouping/calorimeter/purity": 0.2942056953907013, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9008362293243408, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.04754957556724548, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.19360902905464172, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.04570802301168442, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.00846678577363491, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03759398311376572, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8475773334503174, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.05475778132677078, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11712772399187088, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.05313553288578987, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.03765754774212837, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.31606826186180115, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7338783740997314, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.24227115511894226, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.41468346118927, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.2309282124042511, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.23404379189014435, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6483200192451477, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6960219740867615, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4615384638309479, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.32084721326828003, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.27190935611724854, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6621758937835693, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7543398141860962, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3477998673915863, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.31733983755111694, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.2584677040576935, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.47739601135253906, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.627296507358551, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4329400062561035, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.25464433431625366, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.21244937181472778, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7399625778198242, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6578029990196228, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4354407787322998, "metrics/hit_grouping/tracker/all/object_completeness": 0.27962252497673035, "metrics/hit_grouping/tracker/all/object_purity": 0.2341233193874359, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6966607570648193, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.017868487164378166, "metrics/hit_grouping/tracker/background_recall": 0.10881749540567398, "metrics/hit_grouping/tracker/completeness": 0.2878464460372925, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6580555438995361, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6638240814208984, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4327546954154968, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.3297427296638489, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.2851051986217499, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6991057395935059, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6612547636032104, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43873661756515503, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.25888702273368835, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.208796888589859, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6975857615470886, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6419883966445923, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4376916289329529, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.19712677597999573, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.15441614389419556, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6907123327255249, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6815547347068787, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.47397470474243164, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.2943922281265259, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.2353624701499939, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6755540370941162, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6894547939300537, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5387930870056152, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.31517302989959717, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.2427741438150406, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6416256427764893, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6212509274482727, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.36681753396987915, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.2540483772754669, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.23096156120300293, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7339662313461304, "metrics/hit_grouping/tracker/purity": 0.2734265923500061, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8966118693351746, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.013197586871683598, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.009953035973012447, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.005004382226616144, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06033182516694069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6640937328338623, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.18688693642616272, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.11982180923223495, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.08124639838933945, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5787299871444702, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6216827034950256, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5250405669212341, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.33696112036705017, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.2846938371658325, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8038721680641174, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.37004294991493225, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5451005697250366, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6135058403015137, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5916640758514404, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.17910981178283691, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.017189249396324158, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.88892936706543, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.24126408994197845, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.14964091777801514, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.583724319934845, "metrics/particle/class_1/pid_accuracy": 0.8982259035110474, "metrics/particle/class_2/efficiency": 0.33571186661720276, "metrics/particle/class_2/pid_accuracy": 0.13866396248340607, "metrics/particle/class_3/efficiency": 0.5259602665901184, "metrics/particle/class_3/pid_accuracy": 0.9065630435943604, "metrics/particle/class_4/efficiency": 0.5280560851097107, "metrics/particle/class_4/pid_accuracy": 0.14611005783081055, "metrics/particle/class_5/efficiency": 0.6302682161331177, "metrics/particle/class_5/pid_accuracy": 0.19452887773513794, "metrics/particle/count/bias_mean": -4.802083492279053, "metrics/particle/count/mae": 7.759114742279053, "metrics/particle/count/prediction_mean": 65.85286712646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.1208985224366188, "metrics/particle/event/energy_response_mean": 0.9620296359062195, "metrics/particle/event/met_abs_error": 13.4424409866333, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0961267277598381, "metrics/particle/event/scalar_pt_response_mean": 0.9857058525085449, "metrics/particle/event/vector_pt_closure": 0.1019357368350029, "metrics/particle/matched/delta_eta_abs_mean": 0.021000627428293228, "metrics/particle/matched/delta_phi_abs_mean": 0.022895541042089462, "metrics/particle/matched/delta_r_mean": 0.034744568169116974, "metrics/particle/matched/energy_relative_abs_mean": 0.2005275934934616, "metrics/particle/matched/pid_accuracy": 0.8524262309074402, "metrics/particle/matched/pt_relative_abs_mean": 0.1981390416622162, "metrics/particle/matching/duplicate_fraction": 0.11078596115112305, "metrics/particle/matching/efficiency": 0.5389123558998108, "metrics/particle/matching/f1": 0.5578702092170715, "metrics/particle/matching/purity": 0.5782105922698975, "metrics/particle/matching/target_coverage_dr0p05": 0.4820411801338196, "metrics/particle/matching/target_coverage_dr0p10": 0.6344931125640869, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3289073705673218, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6105725765228271, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03385639190673828, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1997077763080597, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.20007745921611786, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.025185689330101013, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2418086975812912, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.01650165021419525, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.221726655960083, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.1914481669664383, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.8654537200927734, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5757.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.5896618366241455, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.9969762563705444, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.605076313018799, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.383737325668335, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.3134169578552246, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.650225639343262, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.030087178573012352, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.7992079257965088, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.10240176320075989, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.042215347290039, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9406.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5344411730766296, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.361272931098938, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 13.684741973876953, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.466691493988037, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.6778905391693115, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.7533569931983948, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.41238784790039, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.323251962661743, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 18.891042709350586, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.035769540816545486, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.20182909071445465, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.19889676570892334, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.03438989073038101, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.24234409630298615, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.041020967066287994, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.068052291870117, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.19385817646980286, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.044515609741211, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1097.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.9229815006256104, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.70868182182312, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.509824752807617, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6.271373748779297, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3806006908416748, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 37.058265686035156, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08911865949630737, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.9716811180114746, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.23243191838264465, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 12.16718578338623, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2031.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.39775198698043823, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -11.62952995300293, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 13.18700885772705, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.9497178792953491, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03910265862941742, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.06095544993877411, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.964043617248535, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.075844407081604, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 31.108165740966797, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03506184741854668, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20166106522083282, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.19775602221488953, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.023655911907553673, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23907138407230377, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0019316952675580978, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.42620614171028137, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.04563320800662041, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0954022407531738, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 12942.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.5512922406196594, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.0432329177856445, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.47906231880188, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.26452898979187, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.33660513162612915, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.873419761657715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.011706438846886158, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.9604759812355042, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.024662550538778305, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.0184214115142822, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22210.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5561906695365906, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.885965824127197, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 17.39987564086914, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.590246200561523, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 5.908928871154785, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.6848115921020508, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.9015583992004395, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -2.957749843597412, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 13.189255714416504, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03475049510598183, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.2011023461818695, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1984943449497223, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0246956218034029, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24005259573459625, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00833501759916544, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7485398650169373, "metrics/particle/momentum/all/accepted/pz_bias": -0.09625249356031418, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.1406686305999756, "metrics/particle/momentum/all/accepted/samples": 19796.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.5844542980194092, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.730558156967163, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.757594585418701, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.539715051651001, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.33277854323387146, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.341371536254883, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.021517520770430565, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.376705288887024, "metrics/particle/momentum/all/assigned/pz_bias": -0.058935873210430145, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.417883396148682, "metrics/particle/momentum/all/assigned/samples": 33647.0, "metrics/particle/momentum/all/efficiency": 0.5379493832588196, "metrics/particle/momentum/all/event/energy_bias": -5.382290363311768, "metrics/particle/momentum/all/event/p3_error_mean": 33.11547088623047, "metrics/particle/momentum/all/event/pt_vector_error_mean": 17.63132095336914, "metrics/particle/momentum/all/event/px_bias": 7.625921249389648, "metrics/particle/momentum/all/event/py_bias": 2.3772130012512207, "metrics/particle/momentum/all/event/pz_abs_mean": 23.720094680786133, "metrics/particle/momentum/all/event/pz_bias": -5.205157279968262, "metrics/particle/momentum/all/event/pz_residual_rms": 37.3038215637207, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03825639560818672, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.21890754997730255, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21722128987312317, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.048053622245788574, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.25708481669425964, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.009866171516478062, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.8911411762237549, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.11262181401252747, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.300398588180542, "metrics/particle/momentum/class_1/accepted/samples": 10237.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.4406552016735077, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.5393073558807373, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.6662003993988037, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.4122533798217773, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.2981520891189575, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.580629348754883, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01890336349606514, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.322933316230774, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.13279566168785095, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.68102765083313, "metrics/particle/momentum/class_1/assigned/samples": 16505.0, "metrics/particle/momentum/class_1/efficiency": 0.5770249962806702, "metrics/particle/momentum/class_1/event/energy_bias": 19.783946990966797, "metrics/particle/momentum/class_1/event/p3_error_mean": 27.759695053100586, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.96352767944336, "metrics/particle/momentum/class_1/event/px_bias": 5.888355731964111, "metrics/particle/momentum/class_1/event/py_bias": 2.142540216445923, "metrics/particle/momentum/class_1/event/pz_abs_mean": 18.528247833251953, "metrics/particle/momentum/class_1/event/pz_bias": -7.100159645080566, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.450639724731445, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.046701595187187195, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2284191995859146, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22993764281272888, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.09356213361024857, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26835060119628906, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04360464960336685, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9832556247711182, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.3588549792766571, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.658786773681641, "metrics/particle/momentum/class_2/accepted/samples": 1032.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8202187418937683, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8277896642684937, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.170793056488037, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7106344699859619, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3655813932418823, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.362517356872559, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06976744532585144, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9745113849639893, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.14573226869106293, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.068665504455566, "metrics/particle/momentum/class_2/assigned/samples": 2150.0, "metrics/particle/momentum/class_2/efficiency": 0.34561285376548767, "metrics/particle/momentum/class_2/event/energy_bias": -29.39499855041504, "metrics/particle/momentum/class_2/event/p3_error_mean": 27.696239471435547, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.81500244140625, "metrics/particle/momentum/class_2/event/px_bias": 1.1253293752670288, "metrics/particle/momentum/class_2/event/py_bias": -0.3017134666442871, "metrics/particle/momentum/class_2/event/pz_abs_mean": 19.737436294555664, "metrics/particle/momentum/class_2/event/pz_bias": 1.5230604410171509, "metrics/particle/momentum/class_2/event/pz_residual_rms": 30.766395568847656, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02880263328552246, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17488771677017212, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17065493762493134, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.014773979783058167, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21214930713176727, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015807392774149776, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.38955995440483093, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.04045674204826355, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1729661226272583, "metrics/particle/momentum/class_3/accepted/samples": 8224.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.7156099081039429, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.23942756652832, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.1553850173950195, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.016150951385498, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.36705541610717773, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.3424015045166, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.016907045617699623, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.1836191415786743, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04960452765226364, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.677597522735596, "metrics/particle/momentum/class_3/assigned/samples": 14491.0, "metrics/particle/momentum/class_3/efficiency": 0.5294533967971802, "metrics/particle/momentum/class_3/event/energy_bias": 9.975868225097656, "metrics/particle/momentum/class_3/event/p3_error_mean": 20.6337833404541, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 11.895271301269531, "metrics/particle/momentum/class_3/event/px_bias": 0.37812599539756775, "metrics/particle/momentum/class_3/event/py_bias": 1.130324363708496, "metrics/particle/momentum/class_3/event/pz_abs_mean": 14.319552421569824, "metrics/particle/momentum/class_3/event/pz_bias": 0.6875105500221252, "metrics/particle/momentum/class_3/event/pz_residual_rms": 26.830978393554688, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03809019923210144, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.20511884987354279, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.20092281699180603, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.05631210282444954, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.24290069937705994, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01587301678955555, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.3348522186279297, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.44311150908470154, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.761268377304077, "metrics/particle/momentum/class_4/accepted/samples": 189.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.6519421935081482, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 9.010359764099121, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 6.90902042388916, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 6.717499256134033, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3868194818496704, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 24.89572525024414, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.020057305693626404, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.4651336669921875, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.4300934076309204, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.992976665496826, "metrics/particle/momentum/class_4/assigned/samples": 349.0, "metrics/particle/momentum/class_4/efficiency": 0.5040000081062317, "metrics/particle/momentum/class_4/event/energy_bias": -3.4276516437530518, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.8954904079437256, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.119020462036133, "metrics/particle/momentum/class_4/event/px_bias": -0.03816106170415878, "metrics/particle/momentum/class_4/event/py_bias": -0.39435070753097534, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7992424964904785, "metrics/particle/momentum/class_4/event/pz_bias": -0.1403569132089615, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.872617721557617, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03528202325105667, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.23940633237361908, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.23652061820030212, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0986994057893753, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.27388572692871094, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02631578966975212, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6906380653381348, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.30085721611976624, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.4881362915039062, "metrics/particle/momentum/class_5/accepted/samples": 114.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.20539160072803497, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.729740560054779, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9210803508758545, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4882935881614685, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.2368421107530594, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 6.013530254364014, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06578947603702545, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.819988965988159, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.30667752027511597, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.990408897399902, "metrics/particle/momentum/class_5/assigned/samples": 152.0, "metrics/particle/momentum/class_5/efficiency": 0.6951219439506531, "metrics/particle/momentum/class_5/event/energy_bias": -2.3194541931152344, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.392343521118164, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.561588764190674, "metrics/particle/momentum/class_5/event/px_bias": 0.27227216958999634, "metrics/particle/momentum/class_5/event/py_bias": -0.19958744943141937, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.7671903371810913, "metrics/particle/momentum/class_5/event/pz_bias": -0.17521148920059204, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.219476699829102, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6527324318885803, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6575788259506226, "metrics/set_slot/angular_recall_all_dr0p10": 0.8442861437797546, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6559441685676575, "metrics/set_slot/soft_count_bias_mean": 2.236361265182495}, "step_train_time": 6.551457405090332, "step_valid_time": 17484.273574590683, "step_total_time": 17490.825032234192, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.041015847964743, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.272725665589469, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3008, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2645, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2536, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8646616541353384, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8793218085106383, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8719301137300148, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12067819148936171, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8290290944753187, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8430851063829787, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8359980220866985, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8646616541353384, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9980503898914523, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23987908839694283, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2211, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1951, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1903, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8538293216630197, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8824061510628675, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8678825622775801, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11759384893713253, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8328227571115974, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8606965174129353, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8465302491103203, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8538293216630197, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9939916407214631, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24961376605768026, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1264, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1090, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1054, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8245083207261724, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8623417721518988, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.843000773395205, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13765822784810128, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7972768532526475, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8338607594936709, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8151585460170147, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8245083207261724} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_8000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..8ab5a652623de793d406e4dea59d416ad50e8f3d --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 0.378931413414172, "train": {"Classification_binary": 0.3087299168109894, "Cardinality": 0.24514952301979065, "Classification": 0.20306190848350525, "Regression_pt": 0.3437533378601074, "Regression_eta": 0.004054515156894922, "Regression_sin_phi": 0.002071032067760825, "Regression_cos_phi": 0.00204527098685503, "Regression_energy": 0.33822691440582275, "HitGrouping": 0.2587125301361084, "Auxiliary": 0.4069470763206482, "Total": 2.1303389072418213, "Total_unweighted": 1.2019428957719356}, "train_group": {}, "valid": {"Classification_binary": 0.31562215089797974, "Cardinality": 0.2691027522087097, "Classification": 0.21132612228393555, "Regression_pt": 0.3524692455927531, "Regression_eta": 0.004169893451035023, "Regression_sin_phi": 0.002131478705753883, "Regression_cos_phi": 0.002103677640358607, "Regression_energy": 0.3452832301457723, "HitGrouping": 0.26045119762420654, "Auxiliary": 0.4249926805496216, "Total": 2.2067343393961587, "Total_unweighted": 1.2331057987175882, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7425591945648193, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.26704317331314087, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4350335896015167, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.25726041197776794, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.25075405836105347, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6187186241149902, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7098349928855896, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.24433152377605438, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3799852430820465, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.23328538239002228, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.22731764614582062, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6477105021476746, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7669922113418579, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.2386128455400467, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3846331834793091, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.2299189269542694, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.2228042334318161, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.5748669505119324, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7551525831222534, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.24766989052295685, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.39954128861427307, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.23846349120140076, "metrics/hit_grouping/calorimeter/all/object_purity": 0.2316221296787262, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.5938073396682739, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0014175924006849527, "metrics/hit_grouping/calorimeter/background_recall": 0.05936407670378685, "metrics/hit_grouping/calorimeter/completeness": 0.31297117471694946, "metrics/hit_grouping/calorimeter/energy_completeness": 0.3419034779071808, "metrics/hit_grouping/calorimeter/energy_purity": 0.3371972441673279, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5881950855255127, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7836813926696777, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.2627801299095154, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.34256941080093384, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.2558578550815582, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.25221526622772217, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5349339842796326, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7368330955505371, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2554223835468292, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.44092583656311035, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.24368354678153992, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.2422444373369217, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.6343850493431091, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7154383659362793, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.20905034244060516, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.47401392459869385, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.19773796200752258, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.17893196642398834, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.6729698181152344, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7223929762840271, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.27442416548728943, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.49669232964515686, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.2634392976760864, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2526913285255432, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6577459573745728, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.721321165561676, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3985408544540405, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.708183228969574, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.3799915611743927, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.3614184260368347, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6459084153175354, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7925834059715271, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2024180293083191, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2637194097042084, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.19613797962665558, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.19467732310295105, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.5224109292030334, "metrics/hit_grouping/calorimeter/purity": 0.30787819623947144, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9031589031219482, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.04405180737376213, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.18045112490653992, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.04522019252181053, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.008569168858230114, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04135338217020035, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8553871512413025, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.06214364618062973, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11712772399187088, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.06074654310941696, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.04461212828755379, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2972336709499359, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7475247979164124, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.26064223051071167, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.41780757904052734, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.25086039304733276, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.2449883669614792, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6182425022125244, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.706559956073761, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4612487256526947, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.34508639574050903, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.29566285014152527, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6465305089950562, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7686638236045837, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.34478601813316345, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3398379981517792, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.27223846316337585, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.45871007442474365, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6365042328834534, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4379047453403473, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.2731589674949646, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.23203198611736298, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7326953411102295, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6677975058555603, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43820124864578247, "metrics/hit_grouping/tracker/all/object_completeness": 0.30019083619117737, "metrics/hit_grouping/tracker/all/object_purity": 0.2545585036277771, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6859750747680664, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01538807712495327, "metrics/hit_grouping/tracker/background_recall": 0.11079096794128418, "metrics/hit_grouping/tracker/completeness": 0.3082253336906433, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6597077250480652, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6715534925460815, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4456426501274109, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.35755470395088196, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3119487762451172, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6884973049163818, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6637373566627502, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4361953139305115, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.2744954824447632, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.23170119524002075, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.695407509803772, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6641053557395935, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4248872995376587, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.20772258937358856, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.15921640396118164, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6714156866073608, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6900693774223328, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.46837055683135986, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.31193655729293823, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.25529560446739197, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6664685606956482, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.7165619134902954, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5332512259483337, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3384789228439331, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.2416081577539444, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.5991379022598267, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6301014423370361, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3819406032562256, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.27805712819099426, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.25592195987701416, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7269014120101929, "metrics/hit_grouping/tracker/purity": 0.29204699397087097, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8983868956565857, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.013951734639704227, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.010031879879534245, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0041055153124034405, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.055429864674806595, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6755144596099854, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.1899845153093338, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1262139230966568, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.08118599653244019, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5875064730644226, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6327424049377441, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5280622243881226, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.3621065020561218, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.31051644682884216, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7902187705039978, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4416496753692627, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5112360119819641, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5705971121788025, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5856106281280518, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.10750310122966766, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.011135798878967762, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.004409790039062, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2399943470954895, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.14837118983268738, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5788801312446594, "metrics/particle/class_1/pid_accuracy": 0.9060719013214111, "metrics/particle/class_2/efficiency": 0.335258811712265, "metrics/particle/class_2/pid_accuracy": 0.17499999701976776, "metrics/particle/class_3/efficiency": 0.49279671907424927, "metrics/particle/class_3/pid_accuracy": 0.9145151376724243, "metrics/particle/class_4/efficiency": 0.5160320401191711, "metrics/particle/class_4/pid_accuracy": 0.20970873534679413, "metrics/particle/class_5/efficiency": 0.6858237385749817, "metrics/particle/class_5/pid_accuracy": 0.2569832503795624, "metrics/particle/count/bias_mean": -10.296223640441895, "metrics/particle/count/mae": 11.185546875, "metrics/particle/count/prediction_mean": 60.35872268676758, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.12225750088691711, "metrics/particle/event/energy_response_mean": 1.0086256265640259, "metrics/particle/event/met_abs_error": 15.852112770080566, "metrics/particle/event/scalar_pt_relative_abs_error": 0.11985304206609726, "metrics/particle/event/scalar_pt_response_mean": 1.0595225095748901, "metrics/particle/event/vector_pt_closure": 0.1124105229973793, "metrics/particle/matched/delta_eta_abs_mean": 0.020812131464481354, "metrics/particle/matched/delta_phi_abs_mean": 0.01895606331527233, "metrics/particle/matched/delta_r_mean": 0.031494200229644775, "metrics/particle/matched/energy_relative_abs_mean": 0.19191773235797882, "metrics/particle/matched/pid_accuracy": 0.8608623743057251, "metrics/particle/matched/pt_relative_abs_mean": 0.19242261350154877, "metrics/particle/matching/duplicate_fraction": 0.1113891527056694, "metrics/particle/matching/efficiency": 0.5227134227752686, "metrics/particle/matching/f1": 0.5637929439544678, "metrics/particle/matching/purity": 0.6118799448013306, "metrics/particle/matching/target_coverage_dr0p05": 0.48649170994758606, "metrics/particle/matching/target_coverage_dr0p10": 0.6258960962295532, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.32249414920806885, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5781195163726807, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.031105954200029373, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.19068767130374908, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.19312016665935516, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015129645355045795, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.23472794890403748, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.014059120789170265, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.191158652305603, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.1346573531627655, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.7957231998443604, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5548.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4314197897911072, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.863656759262085, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.3996052742004395, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.2274723052978516, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.28613603115081787, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 15.804723739624023, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03330184146761894, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8586204051971436, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0864143893122673, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.340281009674072, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8468.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5150389671325684, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 6.632296562194824, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 15.09626579284668, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.265872955322266, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 2.9960410594940186, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.5279425382614136, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 12.605266571044922, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.3433589935302734, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 20.578937530517578, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03132891654968262, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1959177851676941, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.19929374754428864, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.007121216040104628, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.24010814726352692, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0398104265332222, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.0667881965637207, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.17258206009864807, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.770498275756836, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1055.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6829590201377869, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.1800389289855957, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.629695892333984, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6.442732810974121, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3466068506240845, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 49.95680618286133, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08973640203475952, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.9771056175231934, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.6619652509689331, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.378631591796875, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1783.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.3825235664844513, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -9.215043067932129, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.751407623291016, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.9679927825927734, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.24017682671546936, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.009267747402191162, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.529287338256836, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.929290533065796, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 30.158798217773438, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03203121945261955, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1925957053899765, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.19114260375499725, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0030811005271971226, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23256447911262512, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0016746411565691233, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.4101329743862152, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.013226800598204136, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0525017976760864, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 12540.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.5052704811096191, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.4429268836975098, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.8362600803375244, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.6619372367858887, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.3315020203590393, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 15.424484252929688, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.015048729255795479, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0204695463180542, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.015541022643446922, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.2553069591522217, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20932.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5389144420623779, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 8.128629684448242, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 19.185819625854492, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 16.169675827026367, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 6.655689716339111, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.5117790699005127, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.51673698425293, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.9072035551071167, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 14.505839347839355, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03172435611486435, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.19222579896450043, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.19216498732566833, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0067956517450511456, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23361453413963318, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007365616504102945, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.727789580821991, "metrics/particle/momentum/all/accepted/pz_bias": -0.03987300395965576, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.0603668689727783, "metrics/particle/momentum/all/accepted/samples": 19143.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.49537572264671326, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.941852569580078, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.9345860481262207, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.7601354122161865, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.32004618644714355, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 19.241498947143555, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.024276047945022583, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.4171326160430908, "metrics/particle/momentum/all/assigned/pz_bias": -0.07174889743328094, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.430814266204834, "metrics/particle/momentum/all/assigned/samples": 31183.0, "metrics/particle/momentum/all/efficiency": 0.5202043652534485, "metrics/particle/momentum/all/event/energy_bias": 5.545882701873779, "metrics/particle/momentum/all/event/p3_error_mean": 35.08369445800781, "metrics/particle/momentum/all/event/pt_vector_error_mean": 19.68178367614746, "metrics/particle/momentum/all/event/px_bias": 9.89190673828125, "metrics/particle/momentum/all/event/py_bias": -3.0489895343780518, "metrics/particle/momentum/all/event/pz_abs_mean": 24.410907745361328, "metrics/particle/momentum/all/event/pz_bias": -8.179853439331055, "metrics/particle/momentum/all/event/pz_residual_rms": 37.296844482421875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.037801917642354965, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2061084806919098, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.2066299319267273, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0015038792043924332, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.24693021178245544, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007683215197175741, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.8318113684654236, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03018070012331009, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.0925796031951904, "metrics/particle/momentum/class_1/accepted/samples": 10152.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.35572749376296997, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.5548709630966187, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.7673894166946411, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.5795711278915405, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.2909669280052185, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 11.464844703674316, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.018575062975287437, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.2872390747070312, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.06690403819084167, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.5604498386383057, "metrics/particle/momentum/class_1/assigned/samples": 15720.0, "metrics/particle/momentum/class_1/efficiency": 0.5722337961196899, "metrics/particle/momentum/class_1/event/energy_bias": 27.23843765258789, "metrics/particle/momentum/class_1/event/p3_error_mean": 29.861303329467773, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 18.07976722717285, "metrics/particle/momentum/class_1/event/px_bias": 9.13369083404541, "metrics/particle/momentum/class_1/event/py_bias": -0.8112016916275024, "metrics/particle/momentum/class_1/event/pz_abs_mean": 19.780445098876953, "metrics/particle/momentum/class_1/event/pz_bias": -2.085285186767578, "metrics/particle/momentum/class_1/event/pz_residual_rms": 28.94608497619629, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.045007769018411636, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2221781611442566, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2175111025571823, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05754171684384346, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2571655511856079, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03431372717022896, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8781009912490845, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.20121236145496368, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.466142654418945, "metrics/particle/momentum/class_2/accepted/samples": 1020.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5954420566558838, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8486950993537903, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.5221446752548218, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 1.1559503078460693, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30420535802841187, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 10.964787483215332, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06881485879421234, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.030365467071533, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.33218127489089966, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.264455318450928, "metrics/particle/momentum/class_2/assigned/samples": 1831.0, "metrics/particle/momentum/class_2/efficiency": 0.3415940999984741, "metrics/particle/momentum/class_2/event/energy_bias": -27.043176651000977, "metrics/particle/momentum/class_2/event/p3_error_mean": 28.244821548461914, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.758599281311035, "metrics/particle/momentum/class_2/event/px_bias": -0.7073585391044617, "metrics/particle/momentum/class_2/event/py_bias": 0.17556381225585938, "metrics/particle/momentum/class_2/event/pz_abs_mean": 20.347030639648438, "metrics/particle/momentum/class_2/event/pz_bias": -4.703643321990967, "metrics/particle/momentum/class_2/event/pz_residual_rms": 32.434059143066406, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021750349551439285, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1694825142621994, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16952495276927948, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.027376960963010788, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2110741287469864, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0026041667442768812, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.40580809116363525, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.03998636454343796, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3134429454803467, "metrics/particle/momentum/class_3/accepted/samples": 7680.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.6505138278007507, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.6387553215026855, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.335383415222168, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.203901767730713, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3565785586833954, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.60930633544922, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.024232756346464157, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.3219858407974243, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.040164362639188766, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.816627025604248, "metrics/particle/momentum/class_3/assigned/samples": 13164.0, "metrics/particle/momentum/class_3/efficiency": 0.49443119764328003, "metrics/particle/momentum/class_3/event/energy_bias": 7.807353973388672, "metrics/particle/momentum/class_3/event/p3_error_mean": 21.521190643310547, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 12.722979545593262, "metrics/particle/momentum/class_3/event/px_bias": 0.39503252506256104, "metrics/particle/momentum/class_3/event/py_bias": -1.6121745109558105, "metrics/particle/momentum/class_3/event/pz_abs_mean": 14.745996475219727, "metrics/particle/momentum/class_3/event/pz_bias": -1.0257071256637573, "metrics/particle/momentum/class_3/event/pz_residual_rms": 27.20183753967285, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03738957270979881, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17682141065597534, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17347751557826996, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0057336874306201935, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20962461829185486, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.2095649242401123, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.18549023568630219, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2852556705474854, "metrics/particle/momentum/class_4/accepted/samples": 177.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5583464503288269, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 14.244383811950684, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 11.694499015808105, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 11.563658714294434, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3956386148929596, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 44.06971740722656, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.021806852892041206, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.7238744497299194, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.1877865195274353, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.6578209400177, "metrics/particle/momentum/class_4/assigned/samples": 321.0, "metrics/particle/momentum/class_4/efficiency": 0.47200000286102295, "metrics/particle/momentum/class_4/event/energy_bias": -2.1561472415924072, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.6535804271698, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.8381683826446533, "metrics/particle/momentum/class_4/event/px_bias": 0.1647089719772339, "metrics/particle/momentum/class_4/event/py_bias": -0.3542657494544983, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8328750133514404, "metrics/particle/momentum/class_4/event/pz_bias": -0.1284904181957245, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.074023246765137, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03478702902793884, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.24403779208660126, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.23147714138031006, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.06334061175584793, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2703767716884613, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.061403509229421616, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.115494728088379, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.774291455745697, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.482861042022705, "metrics/particle/momentum/class_5/accepted/samples": 114.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.15248657763004303, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.6954624056816101, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.7746425271034241, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4748326539993286, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.190476194024086, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.474453926086426, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08843537420034409, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.0643532276153564, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.0789978876709938, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.692732810974121, "metrics/particle/momentum/class_5/assigned/samples": 147.0, "metrics/particle/momentum/class_5/efficiency": 0.6951219439506531, "metrics/particle/momentum/class_5/event/energy_bias": -0.30058756470680237, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.342043876647949, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.3777616024017334, "metrics/particle/momentum/class_5/event/px_bias": 0.9058333039283752, "metrics/particle/momentum/class_5/event/py_bias": -0.4469112157821655, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.098586082458496, "metrics/particle/momentum/class_5/event/pz_bias": -0.2367270439863205, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.245339870452881, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6291374564170837, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6343101859092712, "metrics/set_slot/angular_recall_all_dr0p10": 0.8502202033996582, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6573355793952942, "metrics/set_slot/soft_count_bias_mean": -2.9599006175994873}, "step_train_time": 4.568055152893066, "step_valid_time": 20088.178925037384, "step_total_time": 20092.746980428696, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0824065397980014, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28974399646616655, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3009, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2663, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2480, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8705459300424976, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8850116317713526, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8777191825972314, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11498836822864739, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8107224583197123, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8241940844134263, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8174027686222808, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8705459300424976, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0725540066359756, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2828373083620771, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2252, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1990, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1901, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8708971553610503, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8836589698046181, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8772316508706194, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11634103019538189, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8319474835886215, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8441385435168739, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8379986775402248, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8708971553610503, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0757404840069802, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2685330877331473, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1280, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1119, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1066, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.846444780635401, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.87421875, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8601076095311299, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12578125, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8063540090771558, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8328125, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.819369715603382, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.846444780635401} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_9000.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..6a3831c1c650e133284806ca6134699e2e2e7693 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 0.4262978400909435, "train": {"Classification_binary": 0.3467126190662384, "Cardinality": 0.23514717817306519, "Classification": 0.2003478705883026, "Regression_pt": 0.3186103403568268, "Regression_eta": 0.004122914746403694, "Regression_sin_phi": 0.0021857251413166523, "Regression_cos_phi": 0.0020191529765725136, "Regression_energy": 0.31426915526390076, "HitGrouping": 0.25315555930137634, "Auxiliary": 0.40644124150276184, "Total": 2.1357009410858154, "Total_unweighted": 1.1882677781395614}, "train_group": {}, "valid": {"Classification_binary": 0.30716802676518756, "Cardinality": 0.2461634079615275, "Classification": 0.19303512573242188, "Regression_pt": 0.3196107546488444, "Regression_eta": 0.004036712149779002, "Regression_sin_phi": 0.0019825585186481476, "Regression_cos_phi": 0.001981269412984451, "Regression_energy": 0.3159725069999695, "HitGrouping": 0.2582081953684489, "Auxiliary": 0.39474356174468994, "Total": 2.0751943588256836, "Total_unweighted": 1.143786954227835, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7316458821296692, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.2814851999282837, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.4517838656902313, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.2704870104789734, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.26291602849960327, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.6449323296546936, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.6958766579627991, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.25065580010414124, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.39992615580558777, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.23895667493343353, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.2288271188735962, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.6683899760246277, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.7501590251922607, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.2666650116443634, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.40907198190689087, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.25841575860977173, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.25137510895729065, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.6092571020126343, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7403360605239868, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.26991191506385803, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.42130160331726074, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.260562002658844, "metrics/hit_grouping/calorimeter/all/object_purity": 0.25312089920043945, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.6246559619903564, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0012390549527481198, "metrics/hit_grouping/calorimeter/background_recall": 0.04979422688484192, "metrics/hit_grouping/calorimeter/completeness": 0.3268720209598541, "metrics/hit_grouping/calorimeter/energy_completeness": 0.3591797351837158, "metrics/hit_grouping/calorimeter/energy_purity": 0.35421085357666016, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5673129558563232, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.7683725357055664, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.28865760564804077, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3643604815006256, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.28199487924575806, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.2760761082172394, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.5668525099754333, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7262127995491028, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.27911698818206787, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4609626829624176, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.26712796092033386, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.26403072476387024, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.654882550239563, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.6973984837532043, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.22241626679897308, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.49744778871536255, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.21024611592292786, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.19532479345798492, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.7120649814605713, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7070909142494202, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.2884424924850464, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.5224320292472839, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.27806955575942993, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2683148682117462, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.6848087310791016, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7135477066040039, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.42239874601364136, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6819351315498352, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4041817784309387, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.37888103723526, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6695831418037415, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.7774229049682617, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.23285208642482758, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2871420681476593, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.22560161352157593, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.222647026181221, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.557974100112915, "metrics/hit_grouping/calorimeter/purity": 0.3214953541755676, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9019266963005066, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0502229779958725, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1654135286808014, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.048308271914720535, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.017229238525032997, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.043233081698417664, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8353057503700256, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.0847194567322731, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11712772399187088, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.08201954513788223, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.06673765927553177, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.354326069355011, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.732761025428772, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.2831287086009979, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4412998855113983, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.27331164479255676, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.26666370034217834, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.6481974720954895, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6966716051101685, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.46834710240364075, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.3515477180480957, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.3067413866519928, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6640591025352478, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7628577947616577, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.35684147477149963, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3500860631465912, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.28654107451438904, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.47498494386672974, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6303580403327942, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45344653725624084, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.2860339879989624, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.24738144874572754, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7384516000747681, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6605263352394104, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4508904814720154, "metrics/hit_grouping/tracker/all/object_completeness": 0.31090059876441956, "metrics/hit_grouping/tracker/all/object_purity": 0.26851800084114075, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6961264610290527, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009651631116867065, "metrics/hit_grouping/tracker/background_recall": 0.07629460096359253, "metrics/hit_grouping/tracker/completeness": 0.31750959157943726, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6476334929466248, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6668840646743774, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.44897422194480896, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.3657442331314087, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3259056806564331, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6970892548561096, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6593549847602844, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.46233436465263367, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.28879594802856445, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.24312087893486023, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7035759687423706, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6486124396324158, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4434625804424286, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2200489640235901, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.17570455372333527, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6867448091506958, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6842874884605408, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.48815488815307617, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3283185362815857, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.268911749124527, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6729217767715454, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.686336100101471, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.549876868724823, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.34051573276519775, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.2661639451980591, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6477832794189453, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.625009298324585, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38470029830932617, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.2829475700855255, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.26842814683914185, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7349597215652466, "metrics/hit_grouping/tracker/purity": 0.2991013526916504, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.896416962146759, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.013574660755693913, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.011837481521070004, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.007124741096049547, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6717034578323364, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.19514714181423187, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1360609382390976, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.10083339363336563, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5787299871444702, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6243094205856323, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5435062646865845, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.37423110008239746, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.32548293471336365, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8028090000152588, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4246523976325989, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5277552008628845, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5916961431503296, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5822664499282837, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.12450035661458969, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0077916476875543594, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.886768341064453, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2605924606323242, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1689692735671997, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6644924879074097, "metrics/particle/class_1/pid_accuracy": 0.9126628637313843, "metrics/particle/class_2/efficiency": 0.3690112233161926, "metrics/particle/class_2/pid_accuracy": 0.2599754333496094, "metrics/particle/class_3/efficiency": 0.5336991548538208, "metrics/particle/class_3/pid_accuracy": 0.9135706424713135, "metrics/particle/class_4/efficiency": 0.5661322474479675, "metrics/particle/class_4/pid_accuracy": 0.18053098022937775, "metrics/particle/class_5/efficiency": 0.7375478744506836, "metrics/particle/class_5/pid_accuracy": 0.26753246784210205, "metrics/particle/count/bias_mean": -5.434895992279053, "metrics/particle/count/mae": 7.84765625, "metrics/particle/count/prediction_mean": 65.22005462646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.11561582237482071, "metrics/particle/event/energy_response_mean": 0.9982352256774902, "metrics/particle/event/met_abs_error": 11.533524513244629, "metrics/particle/event/scalar_pt_relative_abs_error": 0.10346253961324692, "metrics/particle/event/scalar_pt_response_mean": 1.0295097827911377, "metrics/particle/event/vector_pt_closure": 0.09013864398002625, "metrics/particle/matched/delta_eta_abs_mean": 0.021574635058641434, "metrics/particle/matched/delta_phi_abs_mean": 0.018565664067864418, "metrics/particle/matched/delta_r_mean": 0.03186146542429924, "metrics/particle/matched/energy_relative_abs_mean": 0.19370707869529724, "metrics/particle/matched/pid_accuracy": 0.8690733313560486, "metrics/particle/matched/pt_relative_abs_mean": 0.19274559617042542, "metrics/particle/matching/duplicate_fraction": 0.1168120726943016, "metrics/particle/matching/efficiency": 0.5847722887992859, "metrics/particle/matching/f1": 0.6081627607345581, "metrics/particle/matching/purity": 0.6335023641586304, "metrics/particle/matching/target_coverage_dr0p05": 0.5134161114692688, "metrics/particle/matching/target_coverage_dr0p10": 0.66650390625, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3595912456512451, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6455503702163696, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.030923493206501007, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.19331318140029907, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.19222329556941986, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.024371759966015816, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.23330280184745789, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.013515662401914597, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.1376234292984009, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.04930482432246208, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.778669834136963, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6289.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4674689471721649, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.0619242191314697, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.651578903198242, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.444265127182007, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.2700503170490265, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.189298629760742, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.031052574515342712, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.7452722787857056, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.20636625587940216, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.2604193687438965, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9339.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5838284492492676, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.250779867172241, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 13.805022239685059, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.045627593994141, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.42976614832878113, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.4377663731575012, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.865498542785645, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 4.932267189025879, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 19.468969345092773, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03106040693819523, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.20230808854103088, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.19766609370708466, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.01923433318734169, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2408350706100464, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.043902438133955, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.0487966537475586, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.02329728752374649, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.868946552276611, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1230.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7738175392150879, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.4125590324401855, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.1393280029296875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.931760787963867, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.33949416875839233, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 40.62456512451172, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08852139860391617, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.6348977088928223, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.12098518013954163, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.813375473022461, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2056.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.44597533345222473, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.942047595977783, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.04944896697998, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.7076539993286133, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07850530743598938, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.18258190155029297, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.848554611206055, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.13612623512744904, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.446922302246094, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03250766173005104, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19345416128635406, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1923661231994629, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.02487102895975113, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23323270678520203, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0013560773804783821, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.398670494556427, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.040756162256002426, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0076828002929688, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 14011.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.4875650405883789, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.925776243209839, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.4633493423461914, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.255479097366333, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.2928454875946045, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.441512107849121, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.011017410084605217, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.8892201781272888, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.13021525740623474, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.699744701385498, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22056.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6021316051483154, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 9.142312049865723, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 15.959883689880371, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 13.160972595214844, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.4791967868804932, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.1284915208816528, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.55325174331665, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.3338539600372314, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.976834297180176, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03196223825216293, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.19391882419586182, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1926272064447403, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.024403167888522148, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23369412124156952, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007338597439229488, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7087926864624023, "metrics/particle/momentum/all/accepted/pz_bias": 0.03959391266107559, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.066516876220703, "metrics/particle/momentum/all/accepted/samples": 21530.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.4995484948158264, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.6530582904815674, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.7418370246887207, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.5341408252716064, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2893486022949219, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 16.633407592773438, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.021374547854065895, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.2969743013381958, "metrics/particle/momentum/all/assigned/pz_bias": 0.13603588938713074, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.97395658493042, "metrics/particle/momentum/all/assigned/samples": 33451.0, "metrics/particle/momentum/all/efficiency": 0.5850702524185181, "metrics/particle/momentum/all/event/energy_bias": 5.451045036315918, "metrics/particle/momentum/all/event/p3_error_mean": 30.254283905029297, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.655076026916504, "metrics/particle/momentum/all/event/px_bias": 1.8304575681686401, "metrics/particle/momentum/all/event/py_bias": 0.5081431865692139, "metrics/particle/momentum/all/event/pz_abs_mean": 22.368825912475586, "metrics/particle/momentum/all/event/pz_bias": 7.1299943923950195, "metrics/particle/momentum/all/event/pz_residual_rms": 32.42143249511719, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.037624746561050415, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.20859862864017487, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.2072659283876419, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.05285164713859558, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.24658282101154327, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007958240807056427, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.8067777752876282, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06433046609163284, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.1702613830566406, "metrics/particle/momentum/class_1/accepted/samples": 11686.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.34578317403793335, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.3051077127456665, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.540071964263916, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.287856101989746, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.2293047308921814, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.583222389221191, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.017291240394115448, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1942894458770752, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.1624993085861206, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.534262180328369, "metrics/particle/momentum/class_1/assigned/samples": 16598.0, "metrics/particle/momentum/class_1/efficiency": 0.658700168132782, "metrics/particle/momentum/class_1/event/energy_bias": 16.27052879333496, "metrics/particle/momentum/class_1/event/p3_error_mean": 27.078737258911133, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.362689971923828, "metrics/particle/momentum/class_1/event/px_bias": -1.2315698862075806, "metrics/particle/momentum/class_1/event/py_bias": 2.2965168952941895, "metrics/particle/momentum/class_1/event/pz_abs_mean": 19.122697830200195, "metrics/particle/momentum/class_1/event/pz_bias": 9.479177474975586, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.895774841308594, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0426783487200737, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22968696057796478, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22786030173301697, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08073031157255173, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2657966613769531, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03539822995662689, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9104613065719604, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.09716349095106125, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.388468265533447, "metrics/particle/momentum/class_2/accepted/samples": 1130.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.7183775901794434, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9855393171310425, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.4876950979232788, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 1.099616527557373, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34183672070503235, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 8.513482093811035, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06725417077541351, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.984358549118042, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05381665378808975, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.010477542877197, "metrics/particle/momentum/class_2/assigned/samples": 2156.0, "metrics/particle/momentum/class_2/efficiency": 0.3784326910972595, "metrics/particle/momentum/class_2/event/energy_bias": -21.11707305908203, "metrics/particle/momentum/class_2/event/p3_error_mean": 25.278324127197266, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 13.87530517578125, "metrics/particle/momentum/class_2/event/px_bias": -0.5444800853729248, "metrics/particle/momentum/class_2/event/py_bias": -1.5564087629318237, "metrics/particle/momentum/class_2/event/pz_abs_mean": 17.7967586517334, "metrics/particle/momentum/class_2/event/pz_bias": -1.71404230594635, "metrics/particle/momentum/class_2/event/pz_residual_rms": 27.776466369628906, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02256888896226883, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.168233722448349, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1673060804605484, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.023700285702943802, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20897817611694336, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0021469464991241693, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3762487471103668, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01068548671901226, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.123591661453247, "metrics/particle/momentum/class_3/accepted/samples": 8384.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.6487565040588379, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.3653645515441895, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.254337787628174, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.1265482902526855, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3516692519187927, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.088109970092773, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01880546472966671, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.1400327682495117, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.11612146347761154, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.8019566535949707, "metrics/particle/momentum/class_3/assigned/samples": 14198.0, "metrics/particle/momentum/class_3/efficiency": 0.5397540926933289, "metrics/particle/momentum/class_3/event/energy_bias": 14.472967147827148, "metrics/particle/momentum/class_3/event/p3_error_mean": 19.51019859313965, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 12.327919006347656, "metrics/particle/momentum/class_3/event/px_bias": 3.408491611480713, "metrics/particle/momentum/class_3/event/py_bias": 0.21108727157115936, "metrics/particle/momentum/class_3/event/pz_abs_mean": 12.643592834472656, "metrics/particle/momentum/class_3/event/pz_bias": -0.6615609526634216, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.771211624145508, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03427700325846672, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.19675126671791077, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.18713724613189697, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.013694144785404205, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.23300568759441376, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004950494971126318, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1348519325256348, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.1198582574725151, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1170554161071777, "metrics/particle/momentum/class_4/accepted/samples": 202.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.557631254196167, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 8.348690032958984, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 7.02579927444458, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 6.860686302185059, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3469387888908386, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 30.019271850585938, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0058309040032327175, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3006829023361206, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.08986640721559525, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.75720477104187, "metrics/particle/momentum/class_4/assigned/samples": 343.0, "metrics/particle/momentum/class_4/efficiency": 0.5386666655540466, "metrics/particle/momentum/class_4/event/energy_bias": -2.9524295330047607, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.344346761703491, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.4859466552734375, "metrics/particle/momentum/class_4/event/px_bias": 0.04552175849676132, "metrics/particle/momentum/class_4/event/py_bias": -0.13697603344917297, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7922322750091553, "metrics/particle/momentum/class_4/event/pz_bias": -0.2292896956205368, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.4935431480407715, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03200078383088112, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2158345878124237, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.21231073141098022, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.09755957871675491, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2550699710845947, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.046875, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.263838291168213, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.8179285526275635, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.3401055335998535, "metrics/particle/momentum/class_5/accepted/samples": 128.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.12790466845035553, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.7525802850723267, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.8632716536521912, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.5191704630851746, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1538461595773697, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.7338409423828125, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08974359184503555, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.177391767501831, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.7658709287643433, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.767733573913574, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.7804877758026123, "metrics/particle/momentum/class_5/event/energy_bias": -1.2229503393173218, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.406414747238159, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.6007277965545654, "metrics/particle/momentum/class_5/event/px_bias": 0.15249420702457428, "metrics/particle/momentum/class_5/event/py_bias": -0.30607619881629944, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.7681636810302734, "metrics/particle/momentum/class_5/event/pz_bias": 0.2557108402252197, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.5425190925598145, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5924496054649353, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5949634313583374, "metrics/set_slot/angular_recall_all_dr0p10": 0.8531872630119324, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6801688075065613, "metrics/set_slot/soft_count_bias_mean": 2.6043577194213867}, "step_train_time": 5.536674976348877, "step_valid_time": 22694.364741563797, "step_total_time": 22699.901417016983, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0685270029012952, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24208301662441578, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2991, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2708, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2601, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8852566198103956, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9053828151120027, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.895206611570248, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09461718488799732, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8502778685845047, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8696088264794383, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8598347107438017, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8852566198103956, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.031458477048561, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2301529932525035, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2226, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2023, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1991, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8853391684901531, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9088050314465409, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8969186433163379, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09119496855345911, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8713347921225383, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.894429469901168, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8827311017512747, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8853391684901531, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0303667714408036, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2625712701697893, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1252, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1122, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1090, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.848714069591528, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8961661341853036, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8717948717948718, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10383386581469649, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8245083207261724, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8706070287539937, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.846930846930847, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.848714069591528} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/hyperparameters.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..edbb7eac4f45dddf37b248f2ce06608dde63df32 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 7260244, "dataset": "cld_hits", "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": false, "output_mode": "set", "binary_classification_focal_gamma": 2.0, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": true, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": {"num_slots": 256, "num_layers": 4, "num_heads": 8, "ffn_multiplier": 4.0, "dropout": 0.0, "query_init": "input-conditioned", "local_attention_radius": 0.4, "attention_mode": "global-flash", "num_sectors": 32, "sector_neighbors": 1, "tracker_query_fraction": 0.6, "proposal_mode": "grid-diverse", "proposal_grid_size": 0.1, "use_aggregate_anchors": true, "aggregate_grid_size": 0.1, "presence_threshold": 0.5, "no_object_weight": 1.0, "cardinality_loss_weight": 0.05, "auxiliary_loss_weight": 0.25, "momentum_loss": "mse", "momentum_huber_delta": 0.2, "hit_grouping": true, "hit_grouping_dim": 32, "hit_grouping_chunk_size": 1024, "hit_grouping_loss_weight": 0.1, "hit_grouping_background_weight": 0.1, "hit_grouping_detector_balanced": false, "hit_grouping_tracker_particle_balanced": false, "tracker_overlap_loss_weight": 0.0, "hit_grouping_sparse": false, "hit_grouping_topk": 4, "hit_grouping_temperature": 1.0, "tracker_momentum_readout": true, "tracker_momentum_max_log_correction": 0.4054651081081644, "tracker_geometry_loss_weight": 0.0, "tracker_curvature_loss_weight": 0.0, "tracker_loss_max_hits": 64, "tracker_group_readout": false, "tracker_fit_correction": false, "tracker_fit_max_log_correction": 0.4054651081081644, "tracker_group_magnetic_field": 2.0, "event_consistency_loss_weight": 0.0, "matcher": {"presence": 1.0, "pid": 1.0, "geometry": 2.0, "pt": 1.0, "energy": 0.0, "dr_scale": 0.1, "log_pt_scale": 0.6931471805599453, "log_energy_scale": 0.6931471805599453}}, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": true, "use_flash_attn_varlen": true, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0001}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 15, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 50000, "patience": 50000, "checkpoint_freq": 1000, "val_freq": 1000, "num_workers": 2, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.001, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "cosinedecay", "lr_schedule_config": {}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 4, "validation_linear_probe": true, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth", "relaxed_load": false, "load_weights_only": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 100, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld_hits": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld_hits": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cld_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cld_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_hits", "cld_edm_ww_fullhad_hits", "cld_edm_qq_hits"], "ntrain": null, "nvalid": 512, "ntest": 512, "data_config": null, "gpus": 8} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/model_kwargs.pkl b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1f926541d1a2918fd6be6ae705c6428cd3f84496 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72c3356bd75892977dcf280e91b3c88edba3d42e5979931e6d5452199c3f7382 +size 6881 diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/particleflow_spec.yaml b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..764890baecded6715d37cf35f42c4079f6e00e2c --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/particleflow_spec.yaml @@ -0,0 +1,2551 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.0 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.0 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id019 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id020 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id019 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id020 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_eta.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..595cc7766beb72eff1383a9176e6654e5d0b6eef Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_eta.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_eta.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..bdc906bdc9ebbb2aae4bf13c023205e599cd0bac Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_eta.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..52726a95257e95ee042f36361958a5e4da7da82d Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..7a83ebf0c3851e5f377ee53fc6e8dd62f194c210 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt_log.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f7daf1076e451bb3b16558c71b7d35635c0081d0 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt_log.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt_log.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..b394be97dde92b7fb34a7d945f38386c1b67ae2f Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_pt_log.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7117555b40b7d3ccca03544ed62e7023c567addc Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..901642128a4c8a7921a3be7decb2cb3785a03273 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..82ce970580d8e31c345079325427969e4262272c Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..e7b5e3029a2ce4e9836b9f1710573fed709a229c Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/num_elements.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0a855115fe9375ab8c054ac7fc29a4b56bfd3c10 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/num_elements.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/num_elements.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..20c0094a268d5faa4f266572aecfbfd9e400f625 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_qq_hits/num_elements.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_eta.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..90efbbeda6492223d1fe6f5e7ed63eb452ec2256 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_eta.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_eta.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..33fcbdd6a5150b2e1f174e5d689ef370530fc6de Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_eta.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0df9f2666fc4e35bf483a9539172f6e9ae5b75c3 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..006db662bbb8671e03c949ea08f3c898466ab067 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..08da00c8a6364b7fc5b2db0001d06041de85fdfb Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..c4a040541de63ad4ff163aef3f3c9c88894eca12 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5f2c98fcda2c4b29b5dc2d8dc4564a02e2d0900e Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..40001379a444bf39e9b2248bbaac0c941c6cefbb Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ab450d5a09ce38e75599edc949ba553ea2de8ca6 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..cbd90eb162fee404ebacc502407cf40f46c8ff7a Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/num_elements.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..52ced215821c406202ae1d2c1d85165cb69a48d3 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/num_elements.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/num_elements.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..d7bd97d4d55dee442dcf6317d1adb5a85aa29a7c Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ttbar_hits/num_elements.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4c59f318d1b13f90d6567c9e909d0b201d183654 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..a2f5ae3e17f5d41599652eeb6c143aa308985f5d Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..70310803981d2244c5bf02674e7b7cf6d0ffbdcd Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..d503c8bb14363abd7bb1e5b19e4302c9adf724bc Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b554c7cc172bd9d052eaf240edc07ac32d7bdd97 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..b5d1e988300b80b48bb2dd519a219a2f719a077f Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..99e4bb82eccee28ce576bb333aa9b78c52855503 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..047a4ec9f8e8041b2822e957a3d6e994f91e82cc Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..80452972178fb69ea060817e92fb8819f78dbbe6 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..9a8964e0327172f5a002804e159503569cb367c4 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.pdf b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1c5ac8480380630e8ea734b93a7e27a198bdc232 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.png b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..cae4c4be74c24304d10c88941171169e157af474 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.png differ diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/train/events.out.tfevents.1793178801.nid006207.95745.0 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/train/events.out.tfevents.1793178801.nid006207.95745.0 new file mode 100644 index 0000000000000000000000000000000000000000..f8c0dcc4013adab54d2532b496935cc13ce08493 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/train/events.out.tfevents.1793178801.nid006207.95745.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6611e6819aa90e72376b199874b3c2c2e9e7cce7a1fedf3c2130e512550d140f +size 33710319 diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/train/events.out.tfevents.1793278150.nid005379.42183.0 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/train/events.out.tfevents.1793278150.nid005379.42183.0 new file mode 100644 index 0000000000000000000000000000000000000000..7e1dccc96a2a6efe68b8f7088eca7ec228df012e --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/train/events.out.tfevents.1793278150.nid005379.42183.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb7b2a3d26a80285ac27dd86de4eb0622c12dd40cd848d69ec168d89fabc2b8b +size 14192383 diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/valid/events.out.tfevents.1793178801.nid006207.95745.1 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/valid/events.out.tfevents.1793178801.nid006207.95745.1 new file mode 100644 index 0000000000000000000000000000000000000000..11b904dff2e3a9cb64e9fa3872998c6b7049711b --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/valid/events.out.tfevents.1793178801.nid006207.95745.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44f9e14fcef344f9156f797267b20d24c7aa892cec3f809ea2c6758af6fbfd78 +size 2204063 diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/valid/events.out.tfevents.1793278150.nid005379.42183.1 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/valid/events.out.tfevents.1793278150.nid005379.42183.1 new file mode 100644 index 0000000000000000000000000000000000000000..1553ff8323be03568118e71b3bb1a89ddd506d4f --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/runs/valid/events.out.tfevents.1793278150.nid005379.42183.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6b67c7f5067e538b2f5c96216eb325ac55c9f03ebe263ebcb4d10039226e6a9 +size 949003 diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/scenario-manifest.json b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..c0fa814ce27b69f122031bc0ae695dd6e37dcf01 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/scenario-manifest.json @@ -0,0 +1,1176 @@ +{ + "scenario": { + "name": "clic_cld_backbone_benchmark", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld_pf_attention": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": {} + }, + "cld_pf_heptv2": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "cld_hits_attention": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "cld_hits_heptv2": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_pf_attention": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": {} + }, + "clic_pf_heptv2": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "clic_hits_attention": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_hits_heptv2": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 128, + "parameters": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null + } + }, + "common_overrides": { + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true + }, + "allowed_variant_differences": [ + "conv_type", + "data_dir", + "dataset", + "elemtypes_nonzero", + "enabled_test_datasets", + "hyperparameters.lr", + "input_dim", + "model.type", + "model.attention", + "model.heptv2", + "model.binary_classification_focal_gamma", + "model.hit_feature_engineering.tracker_neighborhood", + "model.output_mode", + "model.set_decoder", + "train_dataset", + "valid_dataset", + "test_dataset" + ] + }, + "platform": { + "name": "lumi_mi250x", + "gpus": 8, + "python_command": [ + "python3" + ], + "data_dir": { + "cld": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "cms_run3": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/" + }, + "experiments_dir": "/scratch/project_465001293/patajoos/particleflow/experiments", + "environment": { + "MIOPEN_USER_DB_PATH": "/tmp/patajoos-23239273-miopen-cache", + "MIOPEN_CUSTOM_CACHE_DIR": "/tmp/patajoos-23239273-miopen-cache", + "ROCM_PATH": "/opt/rocm", + "KERAS_BACKEND": "torch", + "OMP_NUM_THREADS": "4", + "PYTHONUNBUFFERED": "1", + "NCCL_SOCKET_IFNAME": "hsn", + "NCCL_NET_GDR_LEVEL": "3" + }, + "runtime_overrides": { + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true + }, + "slurm": { + "partition": "standard-g", + "constraint": null, + "account": "project_465001293", + "time": "2-00:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 32, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "480G", + "memory_per_gpu": null, + "no_requeue": true + } + }, + "job": { + "scenario_name": "clic_cld_backbone_benchmark", + "platform_name": "lumi_mi250x", + "variant_name": "cld_hits_attention", + "model_name": "pyg-cld-hits-set-v1", + "production_name": "cld", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "seed": 12345, + "global_batch_size": 128, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null, + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true, + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld_hits", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": false, + "output_mode": "set", + "binary_classification_focal_gamma": 2.0, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": true, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": { + "num_slots": 256, + "num_layers": 4, + "num_heads": 8, + "ffn_multiplier": 4.0, + "dropout": 0.0, + "query_init": "input-conditioned", + "local_attention_radius": 0.4, + "attention_mode": "global-flash", + "num_sectors": 32, + "sector_neighbors": 1, + "tracker_query_fraction": 0.6, + "proposal_mode": "grid-diverse", + "proposal_grid_size": 0.1, + "use_aggregate_anchors": true, + "aggregate_grid_size": 0.1, + "presence_threshold": 0.5, + "no_object_weight": 1.0, + "cardinality_loss_weight": 0.05, + "auxiliary_loss_weight": 0.25, + "momentum_loss": "mse", + "momentum_huber_delta": 0.2, + "hit_grouping": true, + "hit_grouping_dim": 32, + "hit_grouping_chunk_size": 1024, + "hit_grouping_loss_weight": 0.1, + "hit_grouping_background_weight": 0.1, + "hit_grouping_detector_balanced": false, + "hit_grouping_tracker_particle_balanced": false, + "tracker_overlap_loss_weight": 0.0, + "hit_grouping_sparse": false, + "hit_grouping_topk": 4, + "hit_grouping_temperature": 1.0, + "tracker_momentum_readout": true, + "tracker_momentum_max_log_correction": 0.4054651081081644, + "tracker_geometry_loss_weight": 0.0, + "tracker_curvature_loss_weight": 0.0, + "tracker_loss_max_hits": 64, + "tracker_group_readout": false, + "tracker_fit_correction": false, + "tracker_fit_max_log_correction": 0.4054651081081644, + "tracker_group_magnetic_field": 2.0, + "event_consistency_loss_weight": 0.0, + "matcher": { + "presence": 1.0, + "pid": 1.0, + "geometry": 2.0, + "pt": 1.0, + "energy": 0.0, + "dr_scale": 0.1, + "log_pt_scale": 0.6931471805599453, + "log_energy_scale": 0.6931471805599453 + } + }, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": true, + "use_flash_attn_varlen": true, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0001 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 15, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 50000, + "patience": 50000, + "checkpoint_freq": 1000, + "val_freq": 1000, + "num_workers": 2, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.001, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "cosinedecay", + "lr_schedule_config": {}, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 100, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld_hits": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld_hits": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cld_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cld_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_hits", + "cld_edm_ww_fullhad_hits", + "cld_edm_qq_hits" + ], + "ntrain": null, + "nvalid": 512, + "ntest": 512, + "data_config": null, + "gpus": 8 + }, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-hits-set-v1", + "--production-name", + "cld", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187", + "train", + "--gpus", + "8", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.set_decoder.num_slots", + "256", + "--model.set_decoder.num_layers", + "4", + "--model.set_decoder.num_heads", + "8", + "--model.set_decoder.attention_mode", + "global-flash", + "--model.set_decoder.query_init", + "input-conditioned", + "--model.set_decoder.local_attention_radius", + "0.4", + "--model.set_decoder.tracker_query_fraction", + "0.6", + "--model.set_decoder.proposal_mode", + "grid-diverse", + "--model.set_decoder.proposal_grid_size", + "0.1", + "--model.set_decoder.use_aggregate_anchors", + "true", + "--model.set_decoder.aggregate_grid_size", + "0.1", + "--model.set_decoder.presence_threshold", + "0.5", + "--model.set_decoder.no_object_weight", + "1.0", + "--model.set_decoder.cardinality_loss_weight", + "0.05", + "--model.set_decoder.auxiliary_loss_weight", + "0.25", + "--model.set_decoder.matcher.presence", + "1.0", + "--model.set_decoder.matcher.pid", + "1.0", + "--model.set_decoder.matcher.geometry", + "2.0", + "--model.set_decoder.matcher.pt", + "1.0", + "--model.set_decoder.matcher.energy", + "0.0", + "--model.set_decoder.matcher.dr_scale", + "0.1", + "--model.set_decoder.matcher.log_pt_scale", + "0.6931471805599453", + "--model.set_decoder.matcher.log_energy_scale", + "0.6931471805599453", + "--model.set_decoder.hit_grouping", + "true", + "--model.set_decoder.hit_grouping_dim", + "32", + "--model.set_decoder.hit_grouping_chunk_size", + "1024", + "--model.set_decoder.hit_grouping_loss_weight", + "0.1", + "--model.set_decoder.hit_grouping_background_weight", + "0.1", + "--model.set_decoder.hit_grouping_detector_balanced", + "false", + "--model.set_decoder.hit_grouping_sparse", + "false", + "--model.set_decoder.hit_grouping_tracker_particle_balanced", + "false", + "--model.set_decoder.tracker_momentum_readout", + "true", + "--model.set_decoder.tracker_group_readout", + "false", + "--model.set_decoder.tracker_fit_correction", + "false", + "--model.set_decoder.tracker_overlap_loss_weight", + "0.0", + "--model.set_decoder.tracker_geometry_loss_weight", + "0.0", + "--model.set_decoder.tracker_curvature_loss_weight", + "0.0", + "--model.set_decoder.event_consistency_loss_weight", + "0.0", + "--model.hit_feature_engineering.enabled", + "true", + "--model.hit_feature_engineering.geometry", + "true", + "--model.hit_feature_engineering.calorimeter_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_momentum", + "false", + "--model.hit_feature_engineering.tracker_doublets", + "false", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "continuations": [ + { + "checkpoint": "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth", + "step": 35000, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-hits-set-v1", + "--production-name", + "cld", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187", + "train", + "--gpus", + "8", + "--dtype", + "bfloat16", + "--load", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.set_decoder.num_slots", + "256", + "--model.set_decoder.num_layers", + "4", + "--model.set_decoder.num_heads", + "8", + "--model.set_decoder.attention_mode", + "global-flash", + "--model.set_decoder.query_init", + "input-conditioned", + "--model.set_decoder.local_attention_radius", + "0.4", + "--model.set_decoder.tracker_query_fraction", + "0.6", + "--model.set_decoder.proposal_mode", + "grid-diverse", + "--model.set_decoder.proposal_grid_size", + "0.1", + "--model.set_decoder.use_aggregate_anchors", + "true", + "--model.set_decoder.aggregate_grid_size", + "0.1", + "--model.set_decoder.presence_threshold", + "0.5", + "--model.set_decoder.no_object_weight", + "1.0", + "--model.set_decoder.cardinality_loss_weight", + "0.05", + "--model.set_decoder.auxiliary_loss_weight", + "0.25", + "--model.set_decoder.matcher.presence", + "1.0", + "--model.set_decoder.matcher.pid", + "1.0", + "--model.set_decoder.matcher.geometry", + "2.0", + "--model.set_decoder.matcher.pt", + "1.0", + "--model.set_decoder.matcher.energy", + "0.0", + "--model.set_decoder.matcher.dr_scale", + "0.1", + "--model.set_decoder.matcher.log_pt_scale", + "0.6931471805599453", + "--model.set_decoder.matcher.log_energy_scale", + "0.6931471805599453", + "--model.set_decoder.hit_grouping", + "true", + "--model.set_decoder.hit_grouping_dim", + "32", + "--model.set_decoder.hit_grouping_chunk_size", + "1024", + "--model.set_decoder.hit_grouping_loss_weight", + "0.1", + "--model.set_decoder.hit_grouping_background_weight", + "0.1", + "--model.set_decoder.hit_grouping_detector_balanced", + "false", + "--model.set_decoder.hit_grouping_sparse", + "false", + "--model.set_decoder.hit_grouping_tracker_particle_balanced", + "false", + "--model.set_decoder.tracker_momentum_readout", + "true", + "--model.set_decoder.tracker_group_readout", + "false", + "--model.set_decoder.tracker_fit_correction", + "false", + "--model.set_decoder.tracker_overlap_loss_weight", + "0.0", + "--model.set_decoder.tracker_geometry_loss_weight", + "0.0", + "--model.set_decoder.tracker_curvature_loss_weight", + "0.0", + "--model.set_decoder.event_consistency_loss_weight", + "0.0", + "--model.hit_feature_engineering.enabled", + "true", + "--model.hit_feature_engineering.geometry", + "true", + "--model.hit_feature_engineering.calorimeter_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_momentum", + "false", + "--model.hit_feature_engineering.tracker_doublets", + "false", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "submitted_at": "2026-10-29T14:47:51.203192" + } + ] +} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train-config.yaml b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d1a8431bd72a2aeaf842fca5a8aec9ba5b6c422 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train-config.yaml @@ -0,0 +1,329 @@ +backend: pytorch +batch_size: 1 +checkpoint_freq: 1000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds +dataset: cld_hits +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_hits +- cld_edm_ww_fullhad_hits +- cld_edm_qq_hits +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 8 +hyperparameters: + batch_size: 1 + lr: 0.0001 +input_dim: 15 +load: /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth +load_weights_only: false +lr: 0.001 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: cosinedecay +lr_schedule_config: {} +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: true + use_jagged_attention: true + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: 2.0 + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: true + input_encoding: split + learned_representation_mode: last + litept: null + output_mode: set + pt_mode: direct-elemtype-split + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + dropout: 0.0 + event_consistency_loss_weight: 0.0 + ffn_multiplier: 4.0 + hit_grouping: true + hit_grouping_background_weight: 0.1 + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + hit_grouping_sparse: false + hit_grouping_temperature: 1.0 + hit_grouping_topk: 4 + hit_grouping_tracker_particle_balanced: false + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + momentum_huber_delta: 0.2 + momentum_loss: mse + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_curvature_loss_weight: 0.0 + tracker_fit_correction: false + tracker_fit_max_log_correction: 0.4054651081081644 + tracker_geometry_loss_weight: 0.0 + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_loss_max_hits: 64 + tracker_momentum_max_log_correction: 0.4054651081081644 + tracker_momentum_readout: true + tracker_overlap_loss_weight: 0.0 + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + sin_phi_mode: linear + task_queries: false + trainable: all + type: attention +ntest: 512 +ntrain: null +num_classes: 6 +num_steps: 50000 +num_workers: 2 +nvalid: 512 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 50000 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 100 +test: true +test_dataset: + cld_edm_qq_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ttbar_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ww_fullhad_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 +threads: 16 +train: true +train_dataset: + cld_hits: + physical: + batch_size: 1 + samples: + cld_edm_qq_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ttbar_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ww_fullhad_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + weight: null +val_freq: 1000 +valid_dataset: + cld_hits: + physical: + batch_size: 1 + samples: + cld_edm_qq_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ttbar_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ww_fullhad_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + weight: null +validation_diagnostics_batches: 4 +validation_linear_probe: true +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a6d983a42c16def47ae615f3c16b8b7a47b721b9 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log @@ -0,0 +1,20 @@ +rank0 [2026-10-28 11:12:24,694] INFO : Will use torch.nn.parallel.DistributedDataParallel() and 8 gpus +rank0 [2026-10-28 11:12:24,891] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,892] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,892] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,892] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,892] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,892] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,892] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,892] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,892] INFO : Spawning DDP processes +rank0 [2026-10-29 14:48:14,443] INFO : Will use torch.nn.parallel.DistributedDataParallel() and 8 gpus +rank0 [2026-10-29 14:48:14,648] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:14,648] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:14,648] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:14,648] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:14,648] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:14,648] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:14,648] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:14,648] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:14,648] INFO : Spawning DDP processes diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.0 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..b0ea235adf6fbebebe031dd689d41cd1bcc19761 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.0 @@ -0,0 +1,15004 @@ +rank0 [2026-10-28 11:12:33,102] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:33,103] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:54,339] INFO : Instantiating model +rank0 [2026-10-28 11:12:54,340] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank0 [2026-10-28 11:12:54,340] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:54,358] INFO : Input encoding initialization took 0.02s +rank0 [2026-10-28 11:12:54,359] INFO : nn0 parameters: 171776 +rank0 [2026-10-28 11:12:54,359] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:54,359] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:54,359] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:54,364] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:54,365] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:54,370] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:54,370] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:54,375] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:54,375] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:54,380] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:54,380] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:54,385] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:54,386] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:54,391] INFO : Backbone initialization took 0.03s +rank0 [2026-10-28 11:12:54,391] INFO : backbone parameters: 2374656 +rank0 [2026-10-28 11:12:54,391] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:54,426] INFO : Output DNNs initialization took 0.04s +rank0 [2026-10-28 11:12:54,426] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:54,426] INFO : jagged_attention enabled=True +rank0 [2026-10-28 11:12:54,427] INFO : flash_attn_varlen enabled=True +rank0 [2026-10-28 11:12:54,427] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:54,427] INFO : classification_norm parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : regression_norm parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : output_mode=set +rank0 [2026-10-28 11:12:54,427] INFO : set_decoder parameters: 4713812 +rank0 [2026-10-28 11:12:54,427] INFO : nn_binary_particle parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : nn_pid parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : nn_pt parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : nn_eta parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : nn_sin_phi parameters: 0 +rank0 [2026-10-28 11:12:54,427] INFO : nn_cos_phi parameters: 0 +rank0 [2026-10-28 11:12:54,428] INFO : nn_energy parameters: 0 +rank0 [2026-10-28 11:12:54,428] INFO : Total MLPF parameters: 7260244 +rank0 [2026-10-28 11:12:54,428] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:54,430] INFO : Instantiated model +rank0 [2026-10-28 11:12:54,430] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:54,450] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:54,451] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:54,452] INFO : Configured model for SyncBatchNorm rank=0 +rank0 [2026-10-28 11:12:54,645] INFO : Configured model for DistributedDataParallel rank=0 +rank0 [2026-10-28 11:12:54,763] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank0 [2026-10-28 11:12:54,763] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank0 [2026-10-28 11:12:54,764] INFO : Trainable parameters: 7260244 +rank0 [2026-10-28 11:12:54,764] INFO : Non-trainable parameters: 0 +rank0 [2026-10-28 11:12:54,764] INFO : Total parameters: 7260244 +rank0 [2026-10-28 11:12:54,767] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank0 [2026-10-28 11:12:54,770] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187 +rank0 [2026-10-28 11:12:54,770] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187 +rank0 [2026-10-28 11:12:54,770] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:54,771] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:02,560] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,711] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,788] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,876] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,978] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,091] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,478] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,555] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,633] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,741] INFO : train_dataset: cld_edm_ttbar_hits, 90900 +rank0 [2026-10-28 11:13:03,741] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:03,861] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:03,965] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,060] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,156] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,254] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,348] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,470] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,572] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,692] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,810] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90900 +rank0 [2026-10-28 11:13:04,810] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:05,191] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,271] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-28 11:13:05,351] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,481] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,589] INFO : train_dataset: cld_edm_qq_hits, 89998 +rank0 [2026-10-28 11:13:05,673] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,846] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-28 11:13:05,948] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-28 11:13:06,039] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:06,146] INFO : train_dataset: cld_edm_qq_hits, 90599 +rank0 [2026-10-28 11:13:06,146] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:13:06,146] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:13:06,146] INFO : train mix: physical: weight=21112 step_share=1.000 batches_per_epoch=21112 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:13:06,146] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,172] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,199] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,225] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,252] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,280] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,306] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,334] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,361] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,387] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,414] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,414] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,440] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,466] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,493] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,518] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,545] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,569] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,595] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,629] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,659] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,693] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,693] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,717] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,742] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,767] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,793] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,819] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,844] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,870] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,895] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,920] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,945] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,945] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:13:06,945] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:13:21,056] INFO : Got dataloaders +rank0 [2026-10-28 11:13:21,056] INFO : loader split=train rank=0 len=21112 +rank0 [2026-10-28 11:13:21,056] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:13:21,056] INFO : Starting training from step 1 to 50000 on rank 0 of 8 +rank0 [2026-10-28 11:13:21,062] INFO : Creating train_iterator +rank0 [2026-10-28 11:13:21,062] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:13:35,644] INFO : Created train_iterator +rank0 [2026-10-28 11:17:15,588] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5434697668626904, 'Classification': 0.9147019968181849, 'Regression_pt': 1.0888657218962907, 'Regression_eta': 0.019410275079426356, 'Regression_sin_phi': 0.00660347732307855, 'Regression_cos_phi': 0.006484371119877323, 'Regression_energy': 1.1263945941627025}, weights={'Classification_binary': 1.0, 'Classification': 0.5941495299339294, 'Regression_pt': 0.4991154968738556, 'Regression_eta': 27.99907684326172, 'Regression_sin_phi': 82.30054473876953, 'Regression_cos_phi': 83.812255859375, 'Regression_energy': 0.48248612880706787} +rank0 [2026-10-28 11:17:18,263] INFO : Step 100/50000 rank0 | Train Loss: 5.4941 | Train Loss Unweighted: 3.3573 | Train Loss Components: Auxiliary: 1.0155 | Cardinality: 0.6417 | Classification: 0.8373 | Classification_binary: 0.4757 | HitGrouping: 0.4797 | Regression_cos_phi: 0.0063 | Regression_energy: 1.0524 | Regression_eta: 0.0130 | Regression_pt: 0.9668 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8490 | Classification_binary: 0.4983 | Regression_cos_phi: 0.0065 | Regression_energy: 0.9918 | Regression_eta: 0.0125 | Regression_pt: 0.9042 | Regression_sin_phi: 0.0063 | LR: 1.00e-03 | DataLoad Time: 0.0146s | Model Forward Time: 3.6135s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.5597 | max_loss: 0.2615 | max_backward: 1.0157 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.4654 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4647.0 | uss_MB: 3401.7 | child_rss_MB: 5567.8 | allocated_MB: 341.0 | reserved_MB: 22914.0 | inactive_split_MB: 237.2 +rank0 [2026-10-28 11:17:18,502] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 195.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 49.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 197.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 193.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:20:50,873] INFO : Step 200/50000 rank0 | Train Loss: 4.4651 | Train Loss Unweighted: 2.7232 | Train Loss Components: Auxiliary: 0.8610 | Cardinality: 0.5153 | Classification: 0.6425 | Classification_binary: 0.5162 | HitGrouping: 0.4582 | Regression_cos_phi: 0.0043 | Regression_energy: 0.7694 | Regression_eta: 0.0086 | Regression_pt: 0.7776 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3878 | Classification_binary: 0.5365 | Regression_cos_phi: 0.3960 | Regression_energy: 0.3308 | Regression_eta: 0.2731 | Regression_pt: 0.3432 | Regression_sin_phi: 0.4012 | LR: 1.00e-03 | DataLoad Time: 0.0155s | Model Forward Time: 4.3618s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7368 | max_loss: 0.2463 | max_backward: 1.7032 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5468 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4651.2 | uss_MB: 3409.7 | child_rss_MB: 5571.3 | allocated_MB: 343.8 | reserved_MB: 23264.0 | inactive_split_MB: 284.1 +rank0 [2026-10-28 11:20:51,110] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 186.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:24:24,758] INFO : Step 300/50000 rank0 | Train Loss: 3.6383 | Train Loss Unweighted: 2.1819 | Train Loss Components: Auxiliary: 0.6931 | Cardinality: 0.3447 | Classification: 0.5089 | Classification_binary: 0.4824 | HitGrouping: 0.4305 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5908 | Regression_eta: 0.0074 | Regression_pt: 0.5852 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2935 | Classification_binary: 0.4003 | Regression_cos_phi: 0.3367 | Regression_energy: 0.2951 | Regression_eta: 0.1881 | Regression_pt: 0.2959 | Regression_sin_phi: 0.3044 | LR: 1.00e-03 | DataLoad Time: 0.0149s | Model Forward Time: 4.4672s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0315 | max_forward: 0.6855 | max_loss: 0.1568 | max_backward: 1.3632 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 1.0182 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4652.7 | uss_MB: 3415.2 | child_rss_MB: 5569.6 | allocated_MB: 341.4 | reserved_MB: 23916.0 | inactive_split_MB: 289.5 +rank0 [2026-10-28 11:24:24,994] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:27:58,977] INFO : Step 400/50000 rank0 | Train Loss: 3.5064 | Train Loss Unweighted: 1.9938 | Train Loss Components: Auxiliary: 0.6582 | Cardinality: 0.3738 | Classification: 0.4591 | Classification_binary: 0.4306 | HitGrouping: 0.4041 | Regression_cos_phi: 0.0039 | Regression_energy: 0.5433 | Regression_eta: 0.0080 | Regression_pt: 0.5456 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2885 | Classification_binary: 0.4147 | Regression_cos_phi: 0.2794 | Regression_energy: 0.2465 | Regression_eta: 0.2057 | Regression_pt: 0.2553 | Regression_sin_phi: 0.2901 | LR: 1.00e-03 | DataLoad Time: 0.0156s | Model Forward Time: 4.3217s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.6100 | max_loss: 0.1486 | max_backward: 1.2396 | max_optimizer_step: 0.1474 | max_tensorboard_logging: 1.0157 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4650.3 | uss_MB: 3416.8 | child_rss_MB: 5584.9 | allocated_MB: 342.9 | reserved_MB: 23982.0 | inactive_split_MB: 297.5 +rank0 [2026-10-28 11:27:59,215] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:31:36,013] INFO : Step 500/50000 rank0 | Train Loss: 3.8899 | Train Loss Unweighted: 2.1856 | Train Loss Components: Auxiliary: 0.6982 | Cardinality: 0.3682 | Classification: 0.4707 | Classification_binary: 0.4836 | HitGrouping: 0.4071 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6048 | Regression_eta: 0.0103 | Regression_pt: 0.6070 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2944 | Classification_binary: 0.5266 | Regression_cos_phi: 0.4844 | Regression_energy: 0.2801 | Regression_eta: 0.3333 | Regression_pt: 0.2935 | Regression_sin_phi: 0.4361 | LR: 1.00e-03 | DataLoad Time: 0.0163s | Model Forward Time: 4.7750s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.7460 | max_loss: 0.2622 | max_backward: 1.7372 | max_optimizer_step: 0.1478 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 4651.3 | uss_MB: 3418.2 | child_rss_MB: 5567.0 | allocated_MB: 344.2 | reserved_MB: 24988.0 | inactive_split_MB: 288.4 +rank0 [2026-10-28 11:31:36,263] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:35:13,537] INFO : Step 600/50000 rank0 | Train Loss: 3.5648 | Train Loss Unweighted: 2.0654 | Train Loss Components: Auxiliary: 0.6756 | Cardinality: 0.3628 | Classification: 0.4159 | Classification_binary: 0.5246 | HitGrouping: 0.3632 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5539 | Regression_eta: 0.0077 | Regression_pt: 0.5557 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2221 | Classification_binary: 0.5136 | Regression_cos_phi: 0.2494 | Regression_energy: 0.2386 | Regression_eta: 0.1851 | Regression_pt: 0.2501 | Regression_sin_phi: 0.2667 | LR: 1.00e-03 | DataLoad Time: 0.0152s | Model Forward Time: 4.8022s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0071 | max_forward: 0.7483 | max_loss: 0.2574 | max_backward: 1.6305 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.9709 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 4647.1 | uss_MB: 3418.2 | child_rss_MB: 5567.3 | allocated_MB: 344.6 | reserved_MB: 24988.0 | inactive_split_MB: 289.5 +rank0 [2026-10-28 11:35:13,788] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:38:47,702] INFO : Step 700/50000 rank0 | Train Loss: 3.0543 | Train Loss Unweighted: 1.8237 | Train Loss Components: Auxiliary: 0.6036 | Cardinality: 0.3094 | Classification: 0.3407 | Classification_binary: 0.4142 | HitGrouping: 0.3367 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5254 | Regression_eta: 0.0060 | Regression_pt: 0.5314 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2051 | Classification_binary: 0.4439 | Regression_cos_phi: 0.2785 | Regression_energy: 0.2451 | Regression_eta: 0.1385 | Regression_pt: 0.2579 | Regression_sin_phi: 0.2346 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 4.1314s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.6445 | max_loss: 0.2136 | max_backward: 1.2179 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.7893 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4647.0 | uss_MB: 3416.3 | child_rss_MB: 5577.3 | allocated_MB: 341.4 | reserved_MB: 24988.0 | inactive_split_MB: 289.7 +rank0 [2026-10-28 11:38:47,938] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:42:25,658] INFO : Step 800/50000 rank0 | Train Loss: 3.1854 | Train Loss Unweighted: 1.9126 | Train Loss Components: Auxiliary: 0.6154 | Cardinality: 0.2934 | Classification: 0.4077 | Classification_binary: 0.4086 | HitGrouping: 0.3317 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5404 | Regression_eta: 0.0067 | Regression_pt: 0.5423 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2476 | Classification_binary: 0.4196 | Regression_cos_phi: 0.2567 | Regression_energy: 0.2613 | Regression_eta: 0.2246 | Regression_pt: 0.2760 | Regression_sin_phi: 0.2944 | LR: 9.99e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9188s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.6526 | max_loss: 0.2001 | max_backward: 1.2539 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5713 | max_tensorboard_flush: 0.0138 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 4642.6 | uss_MB: 3416.3 | child_rss_MB: 5579.0 | allocated_MB: 346.1 | reserved_MB: 24988.0 | inactive_split_MB: 222.7 +rank0 [2026-10-28 11:42:25,895] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:46:08,313] INFO : Step 900/50000 rank0 | Train Loss: 3.4950 | Train Loss Unweighted: 2.0678 | Train Loss Components: Auxiliary: 0.6566 | Cardinality: 0.3469 | Classification: 0.4160 | Classification_binary: 0.4953 | HitGrouping: 0.3295 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5663 | Regression_eta: 0.0083 | Regression_pt: 0.5742 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2407 | Classification_binary: 0.5159 | Regression_cos_phi: 0.3562 | Regression_energy: 0.2572 | Regression_eta: 0.2404 | Regression_pt: 0.2715 | Regression_sin_phi: 0.3192 | LR: 9.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5572s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8131 | max_loss: 0.2548 | max_backward: 1.8396 | max_optimizer_step: 0.1492 | max_tensorboard_logging: 0.4938 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 4643.4 | uss_MB: 3417.5 | child_rss_MB: 5588.2 | allocated_MB: 344.2 | reserved_MB: 26006.0 | inactive_split_MB: 273.7 +rank0 [2026-10-28 11:46:08,550] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 46.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:49:59,258] INFO : Step 1000/50000 rank0 | Train Loss: 3.0581 | Train Loss Unweighted: 1.8801 | Train Loss Components: Auxiliary: 0.5942 | Cardinality: 0.2760 | Classification: 0.3603 | Classification_binary: 0.4232 | HitGrouping: 0.3177 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5411 | Regression_eta: 0.0066 | Regression_pt: 0.5426 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1908 | Classification_binary: 0.3646 | Regression_cos_phi: 0.1681 | Regression_energy: 0.2783 | Regression_eta: 0.1276 | Regression_pt: 0.2887 | Regression_sin_phi: 0.1801 | LR: 9.99e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.0602s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1006 | max_forward: 0.7513 | max_loss: 0.1459 | max_backward: 1.3658 | max_optimizer_step: 0.1511 | max_tensorboard_logging: 1.2506 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4639.1 | uss_MB: 3415.3 | child_rss_MB: 5584.2 | allocated_MB: 344.4 | reserved_MB: 26006.0 | inactive_split_MB: 271.1 +rank0 [2026-10-28 11:49:59,497] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 46.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:49:59,525] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:49:59,731] INFO : Running validation on rank0 +rank0 [2026-10-28 11:49:59,731] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:49:59,731] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:50:53,280] INFO : VALIDATION | Step=1000/50000 | Train Loss=3.0581 | Valid Loss=3.1192 | Train Loss Unweighted=1.8801 | Valid Loss Unweighted=1.8921 | Stale=0 | ETA=1839.3m +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_fit_success_fraction=0.25 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 11:50:53,281] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/ari=0.256071 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/ari_threshold=0.5 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.594399 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/top1_competing=0.679915 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.522282 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.267225 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.0521932 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 11:50:53,282] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/ari=0.523296 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/seconds_per_rank=21.334 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding/ari=0.0940398 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding/ari_threshold=0.775 +rank0 [2026-10-28 11:50:53,283] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding_minus_raw/ari=0.00241667 +rank0 [2026-10-28 11:50:53,284] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 11:50:53,284] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 11:50:53,284] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 11:50:53,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:50:53,620] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 11:50:53,631] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 11:51:19,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 11:51:20,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 11:51:21,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 11:51:22,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 11:51:32,761] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 11:51:32,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:51:33,321] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 11:51:33,325] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 11:51:49,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 11:51:50,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 11:51:51,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 11:51:52,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 11:52:02,901] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 11:52:02,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:03,475] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 11:52:03,478] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 11:52:19,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 11:52:19,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 11:52:20,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 11:52:21,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 11:52:31,461] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 11:52:31,479] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits +rank0 [2026-10-28 11:52:34,694] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits +rank0 [2026-10-28 11:52:35,953] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:37,589] INFO : Plotted jets +rank0 [2026-10-28 11:52:38,737] INFO : Plotted jet ratio +rank0 [2026-10-28 11:52:38,811] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 11:52:41,966] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 11:52:43,233] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:44,700] INFO : Plotted jets +rank0 [2026-10-28 11:52:45,821] INFO : Plotted jet ratio +rank0 [2026-10-28 11:52:45,872] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits +rank0 [2026-10-28 11:52:48,171] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits +rank0 [2026-10-28 11:52:48,982] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:50,444] INFO : Plotted jets +rank0 [2026-10-28 11:52:51,594] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:36,145] INFO : Step 1100/50000 rank0 | Train Loss: 2.9181 | Train Loss Unweighted: 1.8556 | Train Loss Components: Auxiliary: 0.5873 | Cardinality: 0.2540 | Classification: 0.3370 | Classification_binary: 0.3789 | HitGrouping: 0.3247 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5584 | Regression_eta: 0.0064 | Regression_pt: 0.5695 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2348 | Classification_binary: 0.3884 | Regression_cos_phi: 0.2138 | Regression_energy: 0.2436 | Regression_eta: 0.1921 | Regression_pt: 0.2574 | Regression_sin_phi: 0.2496 | LR: 9.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6361s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8528 | max_loss: 0.1390 | max_backward: 1.4220 | max_optimizer_step: 0.2689 | max_tensorboard_logging: 0.9135 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 6058.3 | uss_MB: 4835.8 | child_rss_MB: 5619.2 | allocated_MB: 344.1 | reserved_MB: 21080.0 | inactive_split_MB: 227.9 +rank0 [2026-10-28 11:56:36,386] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:00:25,878] INFO : Step 1200/50000 rank0 | Train Loss: 3.1885 | Train Loss Unweighted: 1.9202 | Train Loss Components: Auxiliary: 0.6265 | Cardinality: 0.3516 | Classification: 0.3718 | Classification_binary: 0.4294 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5507 | Regression_eta: 0.0065 | Regression_pt: 0.5556 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2088 | Classification_binary: 0.4504 | Regression_cos_phi: 0.3227 | Regression_energy: 0.2607 | Regression_eta: 0.2017 | Regression_pt: 0.2746 | Regression_sin_phi: 0.2492 | LR: 9.99e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.0655s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0076 | max_forward: 0.7370 | max_loss: 0.3921 | max_backward: 1.7945 | max_optimizer_step: 0.1639 | max_tensorboard_logging: 1.0479 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 6061.1 | uss_MB: 4838.4 | child_rss_MB: 5619.8 | allocated_MB: 344.9 | reserved_MB: 23662.0 | inactive_split_MB: 241.9 +rank0 [2026-10-28 12:00:26,117] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:04:16,802] INFO : Step 1300/50000 rank0 | Train Loss: 3.2694 | Train Loss Unweighted: 1.9280 | Train Loss Components: Auxiliary: 0.6193 | Cardinality: 0.3461 | Classification: 0.3601 | Classification_binary: 0.4731 | HitGrouping: 0.3156 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5359 | Regression_eta: 0.0069 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2173 | Classification_binary: 0.4628 | Regression_cos_phi: 0.3356 | Regression_energy: 0.2246 | Regression_eta: 0.2507 | Regression_pt: 0.2363 | Regression_sin_phi: 0.2694 | LR: 9.98e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.1211s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0540 | max_forward: 0.9909 | max_loss: 0.3234 | max_backward: 2.2964 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 1.1252 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5498.9 | uss_MB: 4193.7 | child_rss_MB: 5645.6 | allocated_MB: 344.9 | reserved_MB: 24052.0 | inactive_split_MB: 299.1 +rank0 [2026-10-28 12:04:17,041] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +2 6 0x7408, 15961 49.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 46.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:07:56,069] INFO : Step 1400/50000 rank0 | Train Loss: 2.8491 | Train Loss Unweighted: 1.7889 | Train Loss Components: Auxiliary: 0.5645 | Cardinality: 0.2711 | Classification: 0.3226 | Classification_binary: 0.3768 | HitGrouping: 0.3109 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5376 | Regression_eta: 0.0061 | Regression_pt: 0.5407 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1987 | Classification_binary: 0.3662 | Regression_cos_phi: 0.2079 | Regression_energy: 0.2666 | Regression_eta: 0.1471 | Regression_pt: 0.2718 | Regression_sin_phi: 0.1777 | LR: 9.98e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3535s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0094 | max_forward: 0.6378 | max_loss: 0.4388 | max_backward: 1.2886 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 0.9631 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 5501.2 | uss_MB: 4277.0 | child_rss_MB: 5652.8 | allocated_MB: 350.4 | reserved_MB: 24916.0 | inactive_split_MB: 216.1 +rank0 [2026-10-28 12:07:56,306] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 46.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:11:42,691] INFO : Step 1500/50000 rank0 | Train Loss: 2.8928 | Train Loss Unweighted: 1.7749 | Train Loss Components: Auxiliary: 0.5690 | Cardinality: 0.2892 | Classification: 0.3566 | Classification_binary: 0.3704 | HitGrouping: 0.3078 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5147 | Regression_eta: 0.0058 | Regression_pt: 0.5216 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2275 | Classification_binary: 0.3759 | Regression_cos_phi: 0.2490 | Regression_energy: 0.2315 | Regression_eta: 0.1410 | Regression_pt: 0.2436 | Regression_sin_phi: 0.2055 | LR: 9.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1606s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.6094 | max_loss: 0.4244 | max_backward: 1.4479 | max_optimizer_step: 0.1602 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5488.8 | uss_MB: 4270.8 | child_rss_MB: 5621.5 | allocated_MB: 342.2 | reserved_MB: 24916.0 | inactive_split_MB: 182.9 +rank0 [2026-10-28 12:11:42,928] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:15:28,274] INFO : Step 1600/50000 rank0 | Train Loss: 2.9805 | Train Loss Unweighted: 1.8512 | Train Loss Components: Auxiliary: 0.5725 | Cardinality: 0.2336 | Classification: 0.3605 | Classification_binary: 0.4404 | HitGrouping: 0.3159 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5142 | Regression_eta: 0.0065 | Regression_pt: 0.5235 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2172 | Classification_binary: 0.4601 | Regression_cos_phi: 0.2547 | Regression_energy: 0.2443 | Regression_eta: 0.1523 | Regression_pt: 0.2601 | Regression_sin_phi: 0.2275 | LR: 9.98e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3234s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7382 | max_loss: 0.1451 | max_backward: 1.5359 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.4679 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5499.3 | uss_MB: 4281.1 | child_rss_MB: 5637.2 | allocated_MB: 343.2 | reserved_MB: 25046.0 | inactive_split_MB: 260.8 +rank0 [2026-10-28 12:15:28,531] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:19:20,426] INFO : Step 1700/50000 rank0 | Train Loss: 2.9423 | Train Loss Unweighted: 1.7603 | Train Loss Components: Auxiliary: 0.5618 | Cardinality: 0.2834 | Classification: 0.3350 | Classification_binary: 0.4141 | HitGrouping: 0.3155 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4976 | Regression_eta: 0.0064 | Regression_pt: 0.5011 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1985 | Classification_binary: 0.4038 | Regression_cos_phi: 0.2238 | Regression_energy: 0.2609 | Regression_eta: 0.1314 | Regression_pt: 0.2721 | Regression_sin_phi: 0.1718 | LR: 9.97e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.9871s | Phase Times: max_pre_step_barrier: 0.0070 | max_batch_to_device: 0.0065 | max_forward: 0.8351 | max_loss: 0.2466 | max_backward: 1.7292 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0857 | Memory: rss_MB: 5499.7 | uss_MB: 4254.7 | child_rss_MB: 5622.9 | allocated_MB: 342.0 | reserved_MB: 25046.0 | inactive_split_MB: 198.9 +rank0 [2026-10-28 12:19:20,665] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 1% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 49% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 24% +6 10 0x7408, 62036 46.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 98% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:23:12,574] INFO : Step 1800/50000 rank0 | Train Loss: 2.7182 | Train Loss Unweighted: 1.6765 | Train Loss Components: Auxiliary: 0.5305 | Cardinality: 0.2556 | Classification: 0.3337 | Classification_binary: 0.3613 | HitGrouping: 0.3103 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4855 | Regression_eta: 0.0054 | Regression_pt: 0.4854 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2177 | Classification_binary: 0.3746 | Regression_cos_phi: 0.2324 | Regression_energy: 0.2319 | Regression_eta: 0.1631 | Regression_pt: 0.2421 | Regression_sin_phi: 0.2275 | LR: 9.97e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8499s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6573 | max_loss: 0.2402 | max_backward: 1.4111 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 1.0385 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0959 | Memory: rss_MB: 5501.3 | uss_MB: 4283.5 | child_rss_MB: 5636.1 | allocated_MB: 340.6 | reserved_MB: 25046.0 | inactive_split_MB: 181.6 +rank0 [2026-10-28 12:23:12,811] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 45.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:26:54,387] INFO : Step 1900/50000 rank0 | Train Loss: 2.9893 | Train Loss Unweighted: 1.8606 | Train Loss Components: Auxiliary: 0.5839 | Cardinality: 0.2900 | Classification: 0.3568 | Classification_binary: 0.4041 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5399 | Regression_eta: 0.0063 | Regression_pt: 0.5477 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1934 | Classification_binary: 0.3833 | Regression_cos_phi: 0.2340 | Regression_energy: 0.2782 | Regression_eta: 0.1480 | Regression_pt: 0.2937 | Regression_sin_phi: 0.2050 | LR: 9.97e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5156s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.6296 | max_loss: 0.1482 | max_backward: 1.2931 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 1.0537 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 5503.8 | uss_MB: 4286.6 | child_rss_MB: 5619.9 | allocated_MB: 344.8 | reserved_MB: 25048.0 | inactive_split_MB: 225.2 +rank0 [2026-10-28 12:26:54,624] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 46.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:30:54,158] INFO : Step 2000/50000 rank0 | Train Loss: 3.0007 | Train Loss Unweighted: 1.8040 | Train Loss Components: Auxiliary: 0.5673 | Cardinality: 0.3091 | Classification: 0.3519 | Classification_binary: 0.4482 | HitGrouping: 0.3137 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4922 | Regression_eta: 0.0062 | Regression_pt: 0.4996 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1812 | Classification_binary: 0.4370 | Regression_cos_phi: 0.1979 | Regression_energy: 0.2423 | Regression_eta: 0.1494 | Regression_pt: 0.2591 | Regression_sin_phi: 0.2168 | LR: 9.96e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8066s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6813 | max_loss: 0.2416 | max_backward: 1.5165 | max_optimizer_step: 0.1635 | max_tensorboard_logging: 0.9842 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5502.9 | uss_MB: 4287.2 | child_rss_MB: 5669.7 | allocated_MB: 346.7 | reserved_MB: 25696.0 | inactive_split_MB: 287.8 +rank0 [2026-10-28 12:30:54,411] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 0% +6 10 0x7408, 62036 46.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:30:54,433] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 12:30:54,645] INFO : Running validation on rank0 +rank0 [2026-10-28 12:30:54,646] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:30:54,646] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:31:46,621] INFO : VALIDATION | Step=2000/50000 | Train Loss=3.0007 | Valid Loss=2.8807 | Train Loss Unweighted=1.8040 | Valid Loss Unweighted=1.7490 | Stale=0 | ETA=1882.2m +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_fit_success_fraction=0.5 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 12:31:46,622] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/ari=0.253398 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/ari_threshold=0.55 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.61512 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/top1_competing=0.701136 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.525983 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.271897 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.0484916 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 12:31:46,623] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/ari=0.525295 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/seconds_per_rank=21.0813 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 12:31:46,624] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding/ari=0.0736783 +rank0 [2026-10-28 12:31:46,625] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding/ari_threshold=0.83125 +rank0 [2026-10-28 12:31:46,625] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding_minus_raw/ari=-0.0179448 +rank0 [2026-10-28 12:31:46,625] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 12:31:46,625] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 12:31:46,625] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 12:31:46,630] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:31:46,877] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 12:31:46,885] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 12:32:13,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 12:32:14,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 12:32:16,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 12:32:17,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 12:32:27,135] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 12:32:27,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:27,706] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 12:32:27,709] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 12:32:43,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 12:32:44,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 12:32:46,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 12:32:47,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 12:32:57,113] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 12:32:57,126] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:57,714] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 12:32:57,717] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 12:33:13,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 12:33:14,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 12:33:15,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 12:33:16,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 12:33:26,442] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 12:33:26,457] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits +rank0 [2026-10-28 12:33:29,672] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits +rank0 [2026-10-28 12:33:30,668] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:32,507] INFO : Plotted jets +rank0 [2026-10-28 12:33:33,633] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:33,703] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 12:33:36,928] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 12:33:37,886] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:39,383] INFO : Plotted jets +rank0 [2026-10-28 12:33:40,527] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:40,578] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits +rank0 [2026-10-28 12:33:43,014] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits +rank0 [2026-10-28 12:33:43,848] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:45,702] INFO : Plotted jets +rank0 [2026-10-28 12:33:46,824] INFO : Plotted jet ratio +rank0 [2026-10-28 12:37:38,263] INFO : Step 2100/50000 rank0 | Train Loss: 2.6299 | Train Loss Unweighted: 1.6429 | Train Loss Components: Auxiliary: 0.5172 | Cardinality: 0.2489 | Classification: 0.3119 | Classification_binary: 0.3591 | HitGrouping: 0.3047 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4785 | Regression_eta: 0.0053 | Regression_pt: 0.4834 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1826 | Classification_binary: 0.3540 | Regression_cos_phi: 0.1916 | Regression_energy: 0.2211 | Regression_eta: 0.1533 | Regression_pt: 0.2286 | Regression_sin_phi: 0.1618 | LR: 9.96e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6865s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0685 | max_forward: 0.7343 | max_loss: 0.1422 | max_backward: 1.3363 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.0015 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6447.9 | uss_MB: 5231.1 | child_rss_MB: 5629.3 | allocated_MB: 341.1 | reserved_MB: 22546.0 | inactive_split_MB: 269.0 +rank0 [2026-10-28 12:37:38,500] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 46.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:41:25,533] INFO : Step 2200/50000 rank0 | Train Loss: 2.8483 | Train Loss Unweighted: 1.7068 | Train Loss Components: Auxiliary: 0.5372 | Cardinality: 0.2547 | Classification: 0.3411 | Classification_binary: 0.3679 | HitGrouping: 0.3030 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4876 | Regression_eta: 0.0061 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2056 | Classification_binary: 0.3921 | Regression_cos_phi: 0.3303 | Regression_energy: 0.2324 | Regression_eta: 0.2027 | Regression_pt: 0.2477 | Regression_sin_phi: 0.3096 | LR: 9.96e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1222s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6352 | max_loss: 0.1414 | max_backward: 1.3119 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.4703 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6448.9 | uss_MB: 5233.9 | child_rss_MB: 5636.2 | allocated_MB: 341.4 | reserved_MB: 22598.0 | inactive_split_MB: 313.4 +rank0 [2026-10-28 12:41:25,784] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 46.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:45:28,455] INFO : Step 2300/50000 rank0 | Train Loss: 3.1410 | Train Loss Unweighted: 1.8301 | Train Loss Components: Auxiliary: 0.5918 | Cardinality: 0.3583 | Classification: 0.3517 | Classification_binary: 0.4348 | HitGrouping: 0.3083 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5121 | Regression_eta: 0.0065 | Regression_pt: 0.5184 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2346 | Classification_binary: 0.4474 | Regression_cos_phi: 0.2953 | Regression_energy: 0.2436 | Regression_eta: 0.1789 | Regression_pt: 0.2543 | Regression_sin_phi: 0.2457 | LR: 9.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4528s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.7258 | max_loss: 0.1520 | max_backward: 1.6288 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.6027 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0166 | Memory: rss_MB: 5633.9 | uss_MB: 4422.4 | child_rss_MB: 5644.1 | allocated_MB: 346.5 | reserved_MB: 23596.0 | inactive_split_MB: 273.5 +rank0 [2026-10-28 12:45:28,689] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:49:23,148] INFO : Step 2400/50000 rank0 | Train Loss: 2.8063 | Train Loss Unweighted: 1.7109 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.2683 | Classification: 0.3276 | Classification_binary: 0.3929 | HitGrouping: 0.3025 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4873 | Regression_eta: 0.0054 | Regression_pt: 0.4919 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2107 | Classification_binary: 0.3717 | Regression_cos_phi: 0.2029 | Regression_energy: 0.2220 | Regression_eta: 0.1553 | Regression_pt: 0.2337 | Regression_sin_phi: 0.1998 | LR: 9.95e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3231s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0314 | max_forward: 0.7331 | max_loss: 0.1502 | max_backward: 1.4174 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.5071 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0157 | Memory: rss_MB: 5635.8 | uss_MB: 4426.4 | child_rss_MB: 5648.9 | allocated_MB: 344.4 | reserved_MB: 23596.0 | inactive_split_MB: 276.8 +rank0 [2026-10-28 12:49:23,390] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 65% +2 6 0x7408, 15961 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 47.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:53:11,758] INFO : Step 2500/50000 rank0 | Train Loss: 2.5300 | Train Loss Unweighted: 1.6034 | Train Loss Components: Auxiliary: 0.5018 | Cardinality: 0.2387 | Classification: 0.2872 | Classification_binary: 0.3313 | HitGrouping: 0.3000 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4857 | Regression_eta: 0.0042 | Regression_pt: 0.4904 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1871 | Classification_binary: 0.3457 | Regression_cos_phi: 0.2135 | Regression_energy: 0.2467 | Regression_eta: 0.1217 | Regression_pt: 0.2538 | Regression_sin_phi: 0.2071 | LR: 9.94e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.8692s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0177 | max_forward: 0.6104 | max_loss: 0.1394 | max_backward: 1.2397 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 1.1029 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5639.0 | uss_MB: 4422.1 | child_rss_MB: 5639.4 | allocated_MB: 342.5 | reserved_MB: 23596.0 | inactive_split_MB: 302.8 +rank0 [2026-10-28 12:53:11,997] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 1% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 49.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 46.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:56:56,578] INFO : Step 2600/50000 rank0 | Train Loss: 2.8992 | Train Loss Unweighted: 1.7075 | Train Loss Components: Auxiliary: 0.5471 | Cardinality: 0.3219 | Classification: 0.3420 | Classification_binary: 0.4005 | HitGrouping: 0.3027 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4737 | Regression_eta: 0.0057 | Regression_pt: 0.4796 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2203 | Classification_binary: 0.4618 | Regression_cos_phi: 0.3563 | Regression_energy: 0.2710 | Regression_eta: 0.1805 | Regression_pt: 0.2845 | Regression_sin_phi: 0.2742 | LR: 9.94e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.3647s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8849 | max_loss: 0.1495 | max_backward: 1.9136 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1254 | Memory: rss_MB: 5638.3 | uss_MB: 4434.4 | child_rss_MB: 5644.3 | allocated_MB: 343.4 | reserved_MB: 23596.0 | inactive_split_MB: 290.6 +rank0 [2026-10-28 12:56:56,818] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 5% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 2% +6 10 0x7408, 62036 46.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:00:47,227] INFO : Step 2700/50000 rank0 | Train Loss: 2.8481 | Train Loss Unweighted: 1.7200 | Train Loss Components: Auxiliary: 0.5468 | Cardinality: 0.2918 | Classification: 0.3199 | Classification_binary: 0.4344 | HitGrouping: 0.3029 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4756 | Regression_eta: 0.0057 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1819 | Classification_binary: 0.4219 | Regression_cos_phi: 0.2380 | Regression_energy: 0.2068 | Regression_eta: 0.1417 | Regression_pt: 0.2169 | Regression_sin_phi: 0.1887 | LR: 9.94e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.0235s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0064 | max_forward: 0.7554 | max_loss: 0.2027 | max_backward: 1.6333 | max_optimizer_step: 0.1633 | max_tensorboard_logging: 0.9978 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5637.9 | uss_MB: 4434.7 | child_rss_MB: 5643.6 | allocated_MB: 343.8 | reserved_MB: 23598.0 | inactive_split_MB: 284.2 +rank0 [2026-10-28 13:00:47,468] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 58% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:04:40,934] INFO : Step 2800/50000 rank0 | Train Loss: 2.6381 | Train Loss Unweighted: 1.5955 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2387 | Classification: 0.2848 | Classification_binary: 0.3420 | HitGrouping: 0.2974 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4746 | Regression_eta: 0.0056 | Regression_pt: 0.4829 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1834 | Classification_binary: 0.3276 | Regression_cos_phi: 0.2515 | Regression_energy: 0.2066 | Regression_eta: 0.1738 | Regression_pt: 0.2179 | Regression_sin_phi: 0.2881 | LR: 9.93e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3757s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6805 | max_loss: 0.2368 | max_backward: 1.1707 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5635.7 | uss_MB: 4432.9 | child_rss_MB: 5638.4 | allocated_MB: 341.2 | reserved_MB: 24554.0 | inactive_split_MB: 308.0 +rank0 [2026-10-28 13:04:41,176] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:08:37,209] INFO : Step 2900/50000 rank0 | Train Loss: 2.7831 | Train Loss Unweighted: 1.6528 | Train Loss Components: Auxiliary: 0.5323 | Cardinality: 0.3031 | Classification: 0.3069 | Classification_binary: 0.3681 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4798 | Regression_eta: 0.0056 | Regression_pt: 0.4869 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2181 | Classification_binary: 0.3831 | Regression_cos_phi: 0.2273 | Regression_energy: 0.2334 | Regression_eta: 0.1874 | Regression_pt: 0.2450 | Regression_sin_phi: 0.2317 | LR: 9.93e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5351s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6242 | max_loss: 0.1475 | max_backward: 1.3573 | max_optimizer_step: 0.2831 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5637.0 | uss_MB: 4433.8 | child_rss_MB: 5645.5 | allocated_MB: 341.4 | reserved_MB: 25570.0 | inactive_split_MB: 292.0 +rank0 [2026-10-28 13:08:37,449] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 45% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 3% +6 10 0x7408, 62036 46.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:12:16,575] INFO : Step 3000/50000 rank0 | Train Loss: 2.9336 | Train Loss Unweighted: 1.7260 | Train Loss Components: Auxiliary: 0.5466 | Cardinality: 0.2829 | Classification: 0.3524 | Classification_binary: 0.4205 | HitGrouping: 0.3085 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4673 | Regression_eta: 0.0061 | Regression_pt: 0.4733 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2266 | Classification_binary: 0.4380 | Regression_cos_phi: 0.2746 | Regression_energy: 0.2208 | Regression_eta: 0.1798 | Regression_pt: 0.2317 | Regression_sin_phi: 0.2400 | LR: 9.92e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8046s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0386 | max_forward: 0.8057 | max_loss: 0.1488 | max_backward: 1.9376 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5635.9 | uss_MB: 4440.1 | child_rss_MB: 5640.1 | allocated_MB: 346.5 | reserved_MB: 25570.0 | inactive_split_MB: 281.4 +rank0 [2026-10-28 13:12:16,812] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:12:16,933] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 13:12:17,102] INFO : Running validation on rank0 +rank0 [2026-10-28 13:12:17,103] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:12:17,103] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:13:10,440] INFO : VALIDATION | Step=3000/50000 | Train Loss=2.9336 | Valid Loss=2.7137 | Train Loss Unweighted=1.7260 | Valid Loss Unweighted=1.6453 | Stale=0 | ETA=1877.2m +rank0 [2026-10-28 13:13:10,440] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:13:10,440] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 13:13:10,440] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 13:13:10,440] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:13:10,440] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 13:13:10,440] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 13:13:10,440] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 13:13:10,440] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/ari=0.242281 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/ari_threshold=0.575 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.567686 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/top1_competing=0.644921 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.54393 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.306872 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.0305448 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 13:13:10,441] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/seconds_per_rank=21.6523 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:13:10,442] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 13:13:10,443] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 13:13:10,443] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:13:10,443] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding/ari=0.110285 +rank0 [2026-10-28 13:13:10,443] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding/ari_threshold=0.83125 +rank0 [2026-10-28 13:13:10,443] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding_minus_raw/ari=0.0186617 +rank0 [2026-10-28 13:13:10,443] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 13:13:10,443] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 13:13:10,443] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 13:13:10,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:13:10,724] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 13:13:10,737] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 13:13:30,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 13:13:31,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 13:13:32,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 13:13:33,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 13:13:43,472] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 13:13:43,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:13:44,016] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 13:13:44,020] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 13:14:00,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 13:14:01,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 13:14:02,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 13:14:03,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 13:14:13,741] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 13:14:13,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:14:14,218] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 13:14:14,221] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 13:14:29,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 13:14:30,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 13:14:31,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 13:14:31,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 13:14:41,914] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 13:14:41,930] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits +rank0 [2026-10-28 13:14:45,105] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits +rank0 [2026-10-28 13:14:46,142] INFO : Plotted number of elements +rank0 [2026-10-28 13:14:48,038] INFO : Plotted jets +rank0 [2026-10-28 13:14:49,193] INFO : Plotted jet ratio +rank0 [2026-10-28 13:14:49,258] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 13:14:52,549] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 13:14:53,553] INFO : Plotted number of elements +rank0 [2026-10-28 13:14:55,034] INFO : Plotted jets +rank0 [2026-10-28 13:14:56,119] INFO : Plotted jet ratio +rank0 [2026-10-28 13:14:56,172] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits +rank0 [2026-10-28 13:14:58,638] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits +rank0 [2026-10-28 13:14:59,477] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:01,353] INFO : Plotted jets +rank0 [2026-10-28 13:15:02,470] INFO : Plotted jet ratio +rank0 [2026-10-28 13:18:53,628] INFO : Step 3100/50000 rank0 | Train Loss: 2.7478 | Train Loss Unweighted: 1.6734 | Train Loss Components: Auxiliary: 0.5216 | Cardinality: 0.2531 | Classification: 0.2981 | Classification_binary: 0.3781 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4881 | Regression_eta: 0.0057 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1536 | Classification_binary: 0.2956 | Regression_cos_phi: 0.2559 | Regression_energy: 0.2769 | Regression_eta: 0.1534 | Regression_pt: 0.2944 | Regression_sin_phi: 0.2423 | LR: 9.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2360s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.6511 | max_loss: 0.3988 | max_backward: 1.4306 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 1.2631 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5749.3 | uss_MB: 4552.2 | child_rss_MB: 5658.3 | allocated_MB: 342.5 | reserved_MB: 23120.0 | inactive_split_MB: 194.6 +rank0 [2026-10-28 13:18:53,870] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 45.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:22:39,092] INFO : Step 3200/50000 rank0 | Train Loss: 2.5254 | Train Loss Unweighted: 1.5710 | Train Loss Components: Auxiliary: 0.4888 | Cardinality: 0.2129 | Classification: 0.2845 | Classification_binary: 0.3271 | HitGrouping: 0.2964 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4770 | Regression_eta: 0.0050 | Regression_pt: 0.4723 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2060 | Classification_binary: 0.3522 | Regression_cos_phi: 0.2249 | Regression_energy: 0.2234 | Regression_eta: 0.1601 | Regression_pt: 0.2308 | Regression_sin_phi: 0.2455 | LR: 9.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1473s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6134 | max_loss: 0.2441 | max_backward: 1.2833 | max_optimizer_step: 0.2753 | max_tensorboard_logging: 0.5629 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5733.9 | uss_MB: 4558.8 | child_rss_MB: 5628.8 | allocated_MB: 343.6 | reserved_MB: 23340.0 | inactive_split_MB: 191.9 +rank0 [2026-10-28 13:22:39,333] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:26:31,314] INFO : Step 3300/50000 rank0 | Train Loss: 2.8037 | Train Loss Unweighted: 1.6635 | Train Loss Components: Auxiliary: 0.5345 | Cardinality: 0.2841 | Classification: 0.3310 | Classification_binary: 0.3857 | HitGrouping: 0.3002 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4633 | Regression_eta: 0.0058 | Regression_pt: 0.4718 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2246 | Classification_binary: 0.4578 | Regression_cos_phi: 0.3423 | Regression_energy: 0.2257 | Regression_eta: 0.2082 | Regression_pt: 0.2347 | Regression_sin_phi: 0.3165 | LR: 9.90e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.0424s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9147 | max_loss: 0.1535 | max_backward: 1.9100 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.6631 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5735.1 | uss_MB: 4562.9 | child_rss_MB: 5661.7 | allocated_MB: 344.0 | reserved_MB: 23618.0 | inactive_split_MB: 250.0 +rank0 [2026-10-28 13:26:31,552] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:30:25,130] INFO : Step 3400/50000 rank0 | Train Loss: 2.8855 | Train Loss Unweighted: 1.7067 | Train Loss Components: Auxiliary: 0.5454 | Cardinality: 0.2859 | Classification: 0.3252 | Classification_binary: 0.4155 | HitGrouping: 0.2965 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4731 | Regression_eta: 0.0067 | Regression_pt: 0.4802 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2037 | Classification_binary: 0.4132 | Regression_cos_phi: 0.2899 | Regression_energy: 0.2429 | Regression_eta: 0.2095 | Regression_pt: 0.2555 | Regression_sin_phi: 0.2560 | LR: 9.90e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2495s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0438 | max_forward: 0.7271 | max_loss: 0.1506 | max_backward: 1.3643 | max_optimizer_step: 0.1716 | max_tensorboard_logging: 0.4827 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5729.2 | uss_MB: 4562.1 | child_rss_MB: 5658.8 | allocated_MB: 346.0 | reserved_MB: 23618.0 | inactive_split_MB: 235.6 +rank0 [2026-10-28 13:30:25,370] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 194.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:34:15,048] INFO : Step 3500/50000 rank0 | Train Loss: 2.4422 | Train Loss Unweighted: 1.4809 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.2120 | Classification: 0.2615 | Classification_binary: 0.3296 | HitGrouping: 0.2898 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4369 | Regression_eta: 0.0054 | Regression_pt: 0.4427 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1312 | Classification_binary: 0.2872 | Regression_cos_phi: 0.1790 | Regression_energy: 0.2031 | Regression_eta: 0.1113 | Regression_pt: 0.2144 | Regression_sin_phi: 0.1323 | LR: 9.89e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.5690s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0058 | max_forward: 0.7056 | max_loss: 0.1971 | max_backward: 1.3359 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.6307 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5720.3 | uss_MB: 4563.4 | child_rss_MB: 5630.0 | allocated_MB: 339.0 | reserved_MB: 24610.0 | inactive_split_MB: 280.8 +rank0 [2026-10-28 13:34:15,299] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 3% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +2 6 0x7408, 15961 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 47% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:38:03,373] INFO : Step 3600/50000 rank0 | Train Loss: 2.5780 | Train Loss Unweighted: 1.5250 | Train Loss Components: Auxiliary: 0.4892 | Cardinality: 0.2463 | Classification: 0.3079 | Classification_binary: 0.3446 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4295 | Regression_eta: 0.0052 | Regression_pt: 0.4324 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2004 | Classification_binary: 0.3823 | Regression_cos_phi: 0.2116 | Regression_energy: 0.2091 | Regression_eta: 0.1471 | Regression_pt: 0.2173 | Regression_sin_phi: 0.2053 | LR: 9.89e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1142s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1453 | max_backward: 1.4106 | max_optimizer_step: 0.1681 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0126 | Memory: rss_MB: 5726.7 | uss_MB: 4567.6 | child_rss_MB: 5631.8 | allocated_MB: 343.9 | reserved_MB: 24610.0 | inactive_split_MB: 244.5 +rank0 [2026-10-28 13:38:03,610] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:41:51,143] INFO : Step 3700/50000 rank0 | Train Loss: 2.9213 | Train Loss Unweighted: 1.6952 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.3153 | Classification: 0.3418 | Classification_binary: 0.4227 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4570 | Regression_eta: 0.0063 | Regression_pt: 0.4612 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1867 | Classification_binary: 0.4671 | Regression_cos_phi: 0.2403 | Regression_energy: 0.2210 | Regression_eta: 0.1444 | Regression_pt: 0.2272 | Regression_sin_phi: 0.2028 | LR: 9.88e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9776s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7710 | max_loss: 0.1507 | max_backward: 1.6430 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.8888 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0145 | Memory: rss_MB: 5726.2 | uss_MB: 4569.4 | child_rss_MB: 5661.5 | allocated_MB: 345.4 | reserved_MB: 24610.0 | inactive_split_MB: 268.9 +rank0 [2026-10-28 13:41:51,388] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:45:46,932] INFO : Step 3800/50000 rank0 | Train Loss: 2.6063 | Train Loss Unweighted: 1.5289 | Train Loss Components: Auxiliary: 0.4933 | Cardinality: 0.2661 | Classification: 0.2839 | Classification_binary: 0.3652 | HitGrouping: 0.2879 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4319 | Regression_eta: 0.0058 | Regression_pt: 0.4369 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1611 | Classification_binary: 0.3261 | Regression_cos_phi: 0.1488 | Regression_energy: 0.2179 | Regression_eta: 0.1340 | Regression_pt: 0.2288 | Regression_sin_phi: 0.2000 | LR: 9.87e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8035s | Phase Times: max_pre_step_barrier: 0.1134 | max_batch_to_device: 0.0063 | max_forward: 0.6784 | max_loss: 0.1451 | max_backward: 1.4269 | max_optimizer_step: 0.2795 | max_tensorboard_logging: 0.9060 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5726.3 | uss_MB: 4569.5 | child_rss_MB: 5664.0 | allocated_MB: 343.8 | reserved_MB: 24610.0 | inactive_split_MB: 202.2 +rank0 [2026-10-28 13:45:47,171] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:49:33,685] INFO : Step 3900/50000 rank0 | Train Loss: 2.2761 | Train Loss Unweighted: 1.4165 | Train Loss Components: Auxiliary: 0.4517 | Cardinality: 0.1938 | Classification: 0.2553 | Classification_binary: 0.2930 | HitGrouping: 0.2882 | Regression_cos_phi: 0.0020 | Regression_energy: 0.4264 | Regression_eta: 0.0045 | Regression_pt: 0.4331 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1875 | Classification_binary: 0.3496 | Regression_cos_phi: 0.1937 | Regression_energy: 0.1995 | Regression_eta: 0.1296 | Regression_pt: 0.2076 | Regression_sin_phi: 0.1785 | LR: 9.87e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6506s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6197 | max_loss: 0.1501 | max_backward: 1.2810 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.0075 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 5727.5 | uss_MB: 3764.5 | child_rss_MB: 5661.9 | allocated_MB: 343.5 | reserved_MB: 24610.0 | inactive_split_MB: 286.6 +rank0 [2026-10-28 13:49:33,924] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 91% +2 6 0x7408, 15961 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:53:24,315] INFO : Step 4000/50000 rank0 | Train Loss: 2.6760 | Train Loss Unweighted: 1.5903 | Train Loss Components: Auxiliary: 0.5112 | Cardinality: 0.2445 | Classification: 0.3070 | Classification_binary: 0.3812 | HitGrouping: 0.2906 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4413 | Regression_eta: 0.0062 | Regression_pt: 0.4492 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1767 | Classification_binary: 0.4128 | Regression_cos_phi: 0.2686 | Regression_energy: 0.1988 | Regression_eta: 0.1773 | Regression_pt: 0.2114 | Regression_sin_phi: 0.2684 | LR: 9.86e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.6444s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0492 | max_forward: 0.7834 | max_loss: 0.1586 | max_backward: 1.6903 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.5589 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5731.5 | uss_MB: 4577.2 | child_rss_MB: 5661.6 | allocated_MB: 345.5 | reserved_MB: 24610.0 | inactive_split_MB: 252.6 +rank0 [2026-10-28 13:53:24,555] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:53:24,575] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 13:53:24,758] INFO : Running validation on rank0 +rank0 [2026-10-28 13:53:24,758] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:53:24,758] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION | Step=4000/50000 | Train Loss=2.6760 | Valid Loss=2.6093 | Train Loss Unweighted=1.5903 | Valid Loss Unweighted=1.5311 | Stale=0 | ETA=1850.2m +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 13:54:14,404] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/ari=0.271468 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/ari_threshold=0.65 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.5552 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/top1_competing=0.629812 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.567969 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.277685 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.0065059 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 13:54:14,405] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/seconds_per_rank=21.8706 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 13:54:14,406] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:54:14,407] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding/ari=0.170941 +rank0 [2026-10-28 13:54:14,407] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 13:54:14,407] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding_minus_raw/ari=0.0793181 +rank0 [2026-10-28 13:54:14,407] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 13:54:14,407] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 13:54:14,407] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 13:54:14,412] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:54:14,664] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 13:54:14,674] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 13:54:39,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 13:54:41,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 13:54:42,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 13:54:43,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 13:54:53,300] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 13:54:53,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:54:53,946] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 13:54:53,950] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 13:55:10,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 13:55:11,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 13:55:12,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 13:55:13,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 13:55:23,226] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 13:55:23,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:55:23,753] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 13:55:23,770] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 13:55:39,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 13:55:40,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 13:55:40,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 13:55:41,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 13:55:51,472] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 13:55:51,488] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits +rank0 [2026-10-28 13:55:54,659] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits +rank0 [2026-10-28 13:55:55,758] INFO : Plotted number of elements +rank0 [2026-10-28 13:55:57,664] INFO : Plotted jets +rank0 [2026-10-28 13:55:58,859] INFO : Plotted jet ratio +rank0 [2026-10-28 13:55:58,934] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 13:56:02,189] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 13:56:03,203] INFO : Plotted number of elements +rank0 [2026-10-28 13:56:04,691] INFO : Plotted jets +rank0 [2026-10-28 13:56:05,783] INFO : Plotted jet ratio +rank0 [2026-10-28 13:56:05,845] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits +rank0 [2026-10-28 13:56:08,302] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits +rank0 [2026-10-28 13:56:09,157] INFO : Plotted number of elements +rank0 [2026-10-28 13:56:11,013] INFO : Plotted jets +rank0 [2026-10-28 13:56:12,139] INFO : Plotted jet ratio +rank0 [2026-10-28 13:59:55,870] INFO : Step 4100/50000 rank0 | Train Loss: 2.6923 | Train Loss Unweighted: 1.5640 | Train Loss Components: Auxiliary: 0.5058 | Cardinality: 0.2671 | Classification: 0.2981 | Classification_binary: 0.4073 | HitGrouping: 0.2911 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4210 | Regression_eta: 0.0055 | Regression_pt: 0.4265 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1459 | Classification_binary: 0.3764 | Regression_cos_phi: 0.2082 | Regression_energy: 0.1963 | Regression_eta: 0.1566 | Regression_pt: 0.2039 | Regression_sin_phi: 0.1877 | LR: 9.85e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9689s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0141 | max_forward: 0.7422 | max_loss: 0.3919 | max_backward: 1.3629 | max_optimizer_step: 0.1890 | max_tensorboard_logging: 0.9903 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6680.3 | uss_MB: 5525.0 | child_rss_MB: 5670.9 | allocated_MB: 345.2 | reserved_MB: 23430.0 | inactive_split_MB: 294.4 +rank0 [2026-10-28 13:59:56,107] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:03:48,547] INFO : Step 4200/50000 rank0 | Train Loss: 2.3415 | Train Loss Unweighted: 1.4424 | Train Loss Components: Auxiliary: 0.4542 | Cardinality: 0.2058 | Classification: 0.2540 | Classification_binary: 0.3143 | HitGrouping: 0.2846 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4310 | Regression_eta: 0.0050 | Regression_pt: 0.4337 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1411 | Classification_binary: 0.2796 | Regression_cos_phi: 0.1401 | Regression_energy: 0.2184 | Regression_eta: 0.1359 | Regression_pt: 0.2266 | Regression_sin_phi: 0.1710 | LR: 9.84e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.3865s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0168 | max_forward: 0.8335 | max_loss: 0.2425 | max_backward: 1.5586 | max_optimizer_step: 0.1744 | max_tensorboard_logging: 0.4781 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5904.9 | uss_MB: 4751.1 | child_rss_MB: 5672.4 | allocated_MB: 341.2 | reserved_MB: 24406.0 | inactive_split_MB: 273.9 +rank0 [2026-10-28 14:03:48,785] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:07:35,913] INFO : Step 4300/50000 rank0 | Train Loss: 2.5102 | Train Loss Unweighted: 1.4857 | Train Loss Components: Auxiliary: 0.4757 | Cardinality: 0.2309 | Classification: 0.2786 | Classification_binary: 0.3543 | HitGrouping: 0.2938 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4177 | Regression_eta: 0.0054 | Regression_pt: 0.4246 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1552 | Classification_binary: 0.3895 | Regression_cos_phi: 0.2189 | Regression_energy: 0.1943 | Regression_eta: 0.1643 | Regression_pt: 0.2045 | Regression_sin_phi: 0.1752 | LR: 9.84e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3368s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6520 | max_loss: 0.2453 | max_backward: 1.4084 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.5540 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5910.3 | uss_MB: 4757.7 | child_rss_MB: 5668.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 239.6 +rank0 [2026-10-28 14:07:36,151] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:11:24,593] INFO : Step 4400/50000 rank0 | Train Loss: 2.8258 | Train Loss Unweighted: 1.5691 | Train Loss Components: Auxiliary: 0.5223 | Cardinality: 0.3313 | Classification: 0.2877 | Classification_binary: 0.4120 | HitGrouping: 0.2902 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4255 | Regression_eta: 0.0063 | Regression_pt: 0.4316 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1621 | Classification_binary: 0.4459 | Regression_cos_phi: 0.2384 | Regression_energy: 0.2051 | Regression_eta: 0.1469 | Regression_pt: 0.2149 | Regression_sin_phi: 0.2545 | LR: 9.83e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.6648s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.7635 | max_loss: 0.2580 | max_backward: 1.7456 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.5221 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 5909.1 | uss_MB: 4757.0 | child_rss_MB: 5674.0 | allocated_MB: 344.6 | reserved_MB: 24406.0 | inactive_split_MB: 261.0 +rank0 [2026-10-28 14:11:24,852] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:15:17,855] INFO : Step 4500/50000 rank0 | Train Loss: 2.5622 | Train Loss Unweighted: 1.4664 | Train Loss Components: Auxiliary: 0.4942 | Cardinality: 0.3132 | Classification: 0.2685 | Classification_binary: 0.3695 | HitGrouping: 0.2876 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4070 | Regression_eta: 0.0048 | Regression_pt: 0.4117 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1468 | Classification_binary: 0.3410 | Regression_cos_phi: 0.1515 | Regression_energy: 0.1730 | Regression_eta: 0.1168 | Regression_pt: 0.1772 | Regression_sin_phi: 0.1818 | LR: 9.82e-04 | DataLoad Time: 0.0202s | Model Forward Time: 6.0159s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0462 | max_forward: 1.1330 | max_loss: 0.3526 | max_backward: 2.1744 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5571 | max_tensorboard_flush: 0.0016 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 5912.8 | uss_MB: 4736.1 | child_rss_MB: 5674.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 288.8 +rank0 [2026-10-28 14:15:18,092] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:19:10,088] INFO : Step 4600/50000 rank0 | Train Loss: 2.2516 | Train Loss Unweighted: 1.3548 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2151 | Classification: 0.2428 | Classification_binary: 0.2886 | HitGrouping: 0.2811 | Regression_cos_phi: 0.0019 | Regression_energy: 0.4059 | Regression_eta: 0.0042 | Regression_pt: 0.4088 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1331 | Classification_binary: 0.2846 | Regression_cos_phi: 0.1334 | Regression_energy: 0.2028 | Regression_eta: 0.1180 | Regression_pt: 0.2147 | Regression_sin_phi: 0.1712 | LR: 9.81e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.3430s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0445 | max_forward: 0.8897 | max_loss: 0.2214 | max_backward: 1.8748 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5913.2 | uss_MB: 4741.7 | child_rss_MB: 5653.3 | allocated_MB: 341.6 | reserved_MB: 25392.0 | inactive_split_MB: 298.4 +rank0 [2026-10-28 14:19:10,330] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 33% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 98% +6 10 0x7408, 62036 46.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:23:11,976] INFO : Step 4700/50000 rank0 | Train Loss: 2.5991 | Train Loss Unweighted: 1.5142 | Train Loss Components: Auxiliary: 0.4914 | Cardinality: 0.2679 | Classification: 0.2997 | Classification_binary: 0.3514 | HitGrouping: 0.2867 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4228 | Regression_eta: 0.0061 | Regression_pt: 0.4290 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1758 | Classification_binary: 0.4058 | Regression_cos_phi: 0.2416 | Regression_energy: 0.1939 | Regression_eta: 0.1656 | Regression_pt: 0.2043 | Regression_sin_phi: 0.1983 | LR: 9.81e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.6424s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7362 | max_loss: 0.2547 | max_backward: 1.8331 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5918.9 | uss_MB: 4764.8 | child_rss_MB: 5675.1 | allocated_MB: 343.1 | reserved_MB: 26384.0 | inactive_split_MB: 292.4 +rank0 [2026-10-28 14:23:12,214] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:27:13,727] INFO : Step 4800/50000 rank0 | Train Loss: 2.7077 | Train Loss Unweighted: 1.5200 | Train Loss Components: Auxiliary: 0.5118 | Cardinality: 0.3360 | Classification: 0.2606 | Classification_binary: 0.3918 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4264 | Regression_eta: 0.0053 | Regression_pt: 0.4305 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1134 | Classification_binary: 0.3323 | Regression_cos_phi: 0.1885 | Regression_energy: 0.2048 | Regression_eta: 0.1043 | Regression_pt: 0.2109 | Regression_sin_phi: 0.1645 | LR: 9.80e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.9278s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7154 | max_loss: 0.1552 | max_backward: 1.4398 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.8979 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0854 | Memory: rss_MB: 5917.3 | uss_MB: 4226.4 | child_rss_MB: 5678.8 | allocated_MB: 346.1 | reserved_MB: 26384.0 | inactive_split_MB: 284.0 +rank0 [2026-10-28 14:27:13,973] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 45.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:31:10,049] INFO : Step 4900/50000 rank0 | Train Loss: 2.3682 | Train Loss Unweighted: 1.4242 | Train Loss Components: Auxiliary: 0.4616 | Cardinality: 0.2176 | Classification: 0.2570 | Classification_binary: 0.3149 | HitGrouping: 0.2900 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4191 | Regression_eta: 0.0047 | Regression_pt: 0.4239 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1520 | Classification_binary: 0.2834 | Regression_cos_phi: 0.1783 | Regression_energy: 0.2080 | Regression_eta: 0.1086 | Regression_pt: 0.2200 | Regression_sin_phi: 0.1568 | LR: 9.79e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0972s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0516 | max_forward: 0.6608 | max_loss: 0.1430 | max_backward: 1.2657 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5916.2 | uss_MB: 4765.4 | child_rss_MB: 5677.6 | allocated_MB: 343.8 | reserved_MB: 26384.0 | inactive_split_MB: 201.9 +rank0 [2026-10-28 14:31:10,288] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 5% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 52.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 46.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:35:19,721] INFO : Step 5000/50000 rank0 | Train Loss: 2.4060 | Train Loss Unweighted: 1.4130 | Train Loss Components: Auxiliary: 0.4756 | Cardinality: 0.2573 | Classification: 0.2748 | Classification_binary: 0.3198 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4020 | Regression_eta: 0.0043 | Regression_pt: 0.4074 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1786 | Classification_binary: 0.3593 | Regression_cos_phi: 0.2976 | Regression_energy: 0.2024 | Regression_eta: 0.1566 | Regression_pt: 0.2133 | Regression_sin_phi: 0.2230 | LR: 9.78e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4825s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6153 | max_loss: 0.1441 | max_backward: 1.2119 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.9741 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5914.8 | uss_MB: 4767.1 | child_rss_MB: 5669.5 | allocated_MB: 342.5 | reserved_MB: 27388.0 | inactive_split_MB: 218.4 +rank0 [2026-10-28 14:35:19,960] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 46.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:35:19,983] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 14:35:20,278] INFO : Running validation on rank0 +rank0 [2026-10-28 14:35:20,278] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:35:20,278] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:36:11,318] INFO : VALIDATION | Step=5000/50000 | Train Loss=2.4060 | Valid Loss=2.5010 | Train Loss Unweighted=1.4130 | Valid Loss Unweighted=1.4585 | Stale=0 | ETA=1825.5m +rank0 [2026-10-28 14:36:11,318] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 14:36:11,318] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/ari=0.327469 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 14:36:11,319] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.574022 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/top1_competing=0.648067 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.569155 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.221684 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.00531932 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 14:36:11,320] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/seconds_per_rank=21.6885 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding/ari=0.189273 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding_minus_raw/ari=0.0976502 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 14:36:11,321] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 14:36:11,326] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:36:11,580] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 14:36:11,589] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 14:36:36,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 14:36:38,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 14:36:39,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 14:36:40,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 14:36:50,244] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 14:36:50,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:36:50,780] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 14:36:50,784] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 14:37:06,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 14:37:07,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 14:37:09,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 14:37:10,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 14:37:20,064] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 14:37:20,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:37:20,582] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 14:37:20,586] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 14:37:36,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 14:37:36,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 14:37:37,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 14:37:38,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 14:37:48,441] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 14:37:48,456] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits +rank0 [2026-10-28 14:37:51,585] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits +rank0 [2026-10-28 14:37:52,610] INFO : Plotted number of elements +rank0 [2026-10-28 14:37:54,150] INFO : Plotted jets +rank0 [2026-10-28 14:37:55,610] INFO : Plotted jet ratio +rank0 [2026-10-28 14:37:55,686] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 14:37:58,864] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 14:37:59,848] INFO : Plotted number of elements +rank0 [2026-10-28 14:38:01,349] INFO : Plotted jets +rank0 [2026-10-28 14:38:02,464] INFO : Plotted jet ratio +rank0 [2026-10-28 14:38:02,518] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits +rank0 [2026-10-28 14:38:04,983] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits +rank0 [2026-10-28 14:38:05,810] INFO : Plotted number of elements +rank0 [2026-10-28 14:38:07,318] INFO : Plotted jets +rank0 [2026-10-28 14:38:08,856] INFO : Plotted jet ratio +rank0 [2026-10-28 14:42:03,500] INFO : Step 5100/50000 rank0 | Train Loss: 2.7059 | Train Loss Unweighted: 1.5631 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2507 | Classification: 0.3068 | Classification_binary: 0.4110 | HitGrouping: 0.2884 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4148 | Regression_eta: 0.0060 | Regression_pt: 0.4186 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2103 | Classification_binary: 0.4081 | Regression_cos_phi: 0.2644 | Regression_energy: 0.1943 | Regression_eta: 0.1707 | Regression_pt: 0.2033 | Regression_sin_phi: 0.2637 | LR: 9.77e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2817s | Phase Times: max_pre_step_barrier: 0.1071 | max_batch_to_device: 0.0070 | max_forward: 0.7192 | max_loss: 0.2509 | max_backward: 1.6066 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.8859 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7032.2 | uss_MB: 5882.9 | child_rss_MB: 5662.0 | allocated_MB: 343.2 | reserved_MB: 23542.0 | inactive_split_MB: 236.2 +rank0 [2026-10-28 14:42:03,738] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:45:56,943] INFO : Step 5200/50000 rank0 | Train Loss: 2.4117 | Train Loss Unweighted: 1.4393 | Train Loss Components: Auxiliary: 0.4667 | Cardinality: 0.2215 | Classification: 0.2426 | Classification_binary: 0.3524 | HitGrouping: 0.2783 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4149 | Regression_eta: 0.0050 | Regression_pt: 0.4196 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1280 | Classification_binary: 0.3015 | Regression_cos_phi: 0.1969 | Regression_energy: 0.2287 | Regression_eta: 0.1381 | Regression_pt: 0.2360 | Regression_sin_phi: 0.1659 | LR: 9.76e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.5831s | Phase Times: max_pre_step_barrier: 0.1151 | max_batch_to_device: 0.0079 | max_forward: 0.8909 | max_loss: 0.1450 | max_backward: 1.3654 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.5791 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 6020.7 | uss_MB: 4875.4 | child_rss_MB: 5666.4 | allocated_MB: 346.1 | reserved_MB: 23690.0 | inactive_split_MB: 197.9 +rank0 [2026-10-28 14:45:57,180] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:50:04,649] INFO : Step 5300/50000 rank0 | Train Loss: 2.2417 | Train Loss Unweighted: 1.3238 | Train Loss Components: Auxiliary: 0.4353 | Cardinality: 0.2226 | Classification: 0.2387 | Classification_binary: 0.3027 | HitGrouping: 0.2808 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3844 | Regression_eta: 0.0041 | Regression_pt: 0.3895 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1192 | Classification_binary: 0.2520 | Regression_cos_phi: 0.1531 | Regression_energy: 0.1880 | Regression_eta: 0.0897 | Regression_pt: 0.1979 | Regression_sin_phi: 0.0940 | LR: 9.75e-04 | DataLoad Time: 0.0235s | Model Forward Time: 4.9947s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0155 | max_forward: 1.3549 | max_loss: 0.4272 | max_backward: 1.7517 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 6024.3 | uss_MB: 4878.1 | child_rss_MB: 5664.8 | allocated_MB: 341.7 | reserved_MB: 25496.0 | inactive_split_MB: 195.4 +rank0 [2026-10-28 14:50:04,888] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:54:27,318] INFO : Step 5400/50000 rank0 | Train Loss: 2.5857 | Train Loss Unweighted: 1.4820 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2715 | Classification: 0.2898 | Classification_binary: 0.3613 | HitGrouping: 0.2829 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4076 | Regression_eta: 0.0053 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1544 | Classification_binary: 0.3909 | Regression_cos_phi: 0.2223 | Regression_energy: 0.1971 | Regression_eta: 0.1656 | Regression_pt: 0.2035 | Regression_sin_phi: 0.1843 | LR: 9.74e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.5619s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7472 | max_loss: 0.1664 | max_backward: 1.6216 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4956 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6028.8 | uss_MB: 4880.2 | child_rss_MB: 5668.3 | allocated_MB: 345.8 | reserved_MB: 25496.0 | inactive_split_MB: 226.8 +rank0 [2026-10-28 14:54:27,560] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 6% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 53.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 46.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:58:30,596] INFO : Step 5500/50000 rank0 | Train Loss: 2.4383 | Train Loss Unweighted: 1.4361 | Train Loss Components: Auxiliary: 0.4636 | Cardinality: 0.2531 | Classification: 0.2547 | Classification_binary: 0.3846 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3900 | Regression_eta: 0.0045 | Regression_pt: 0.3974 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1292 | Classification_binary: 0.3295 | Regression_cos_phi: 0.2568 | Regression_energy: 0.1913 | Regression_eta: 0.1083 | Regression_pt: 0.2036 | Regression_sin_phi: 0.1860 | LR: 9.73e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1663s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7547 | max_loss: 0.2915 | max_backward: 1.8143 | max_optimizer_step: 0.2819 | max_tensorboard_logging: 0.5958 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6029.1 | uss_MB: 4882.8 | child_rss_MB: 5667.9 | allocated_MB: 342.5 | reserved_MB: 26496.0 | inactive_split_MB: 471.5 +rank0 [2026-10-28 14:58:30,837] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 52.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 3% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 4% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:02:38,494] INFO : Step 5600/50000 rank0 | Train Loss: 2.2622 | Train Loss Unweighted: 1.3177 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.1990 | Classification: 0.2257 | Classification_binary: 0.3268 | HitGrouping: 0.2800 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3758 | Regression_eta: 0.0051 | Regression_pt: 0.3799 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1162 | Classification_binary: 0.2765 | Regression_cos_phi: 0.1983 | Regression_energy: 0.1873 | Regression_eta: 0.1373 | Regression_pt: 0.1931 | Regression_sin_phi: 0.1617 | LR: 9.72e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8150s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0347 | max_forward: 1.1910 | max_loss: 0.2555 | max_backward: 1.5431 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 6024.4 | uss_MB: 4885.9 | child_rss_MB: 5669.8 | allocated_MB: 340.8 | reserved_MB: 26496.0 | inactive_split_MB: 191.2 +rank0 [2026-10-28 15:02:38,743] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:06:24,753] INFO : Step 5700/50000 rank0 | Train Loss: 2.4588 | Train Loss Unweighted: 1.4026 | Train Loss Components: Auxiliary: 0.4666 | Cardinality: 0.2348 | Classification: 0.2768 | Classification_binary: 0.3365 | HitGrouping: 0.2916 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3864 | Regression_eta: 0.0055 | Regression_pt: 0.3923 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1653 | Classification_binary: 0.3435 | Regression_cos_phi: 0.2368 | Regression_energy: 0.1859 | Regression_eta: 0.1714 | Regression_pt: 0.1961 | Regression_sin_phi: 0.1826 | LR: 9.71e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1696s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6547 | max_loss: 0.1874 | max_backward: 1.2748 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6027.1 | uss_MB: 4889.2 | child_rss_MB: 5674.0 | allocated_MB: 344.4 | reserved_MB: 26496.0 | inactive_split_MB: 224.5 +rank0 [2026-10-28 15:06:24,990] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 191.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:10:15,555] INFO : Step 5800/50000 rank0 | Train Loss: 2.4947 | Train Loss Unweighted: 1.4616 | Train Loss Components: Auxiliary: 0.4752 | Cardinality: 0.2715 | Classification: 0.2623 | Classification_binary: 0.3729 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4032 | Regression_eta: 0.0047 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1635 | Classification_binary: 0.4024 | Regression_cos_phi: 0.2610 | Regression_energy: 0.1947 | Regression_eta: 0.1180 | Regression_pt: 0.2068 | Regression_sin_phi: 0.2158 | LR: 9.70e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.4402s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.9768 | max_loss: 0.2512 | max_backward: 2.0573 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.6978 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6026.0 | uss_MB: 4048.8 | child_rss_MB: 5650.8 | allocated_MB: 343.4 | reserved_MB: 27494.0 | inactive_split_MB: 198.3 +rank0 [2026-10-28 15:10:15,793] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 1% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 46.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:14:12,995] INFO : Step 5900/50000 rank0 | Train Loss: 2.3800 | Train Loss Unweighted: 1.3838 | Train Loss Components: Auxiliary: 0.4549 | Cardinality: 0.2203 | Classification: 0.2438 | Classification_binary: 0.3435 | HitGrouping: 0.2757 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3900 | Regression_eta: 0.0050 | Regression_pt: 0.3965 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1053 | Classification_binary: 0.2794 | Regression_cos_phi: 0.1414 | Regression_energy: 0.2019 | Regression_eta: 0.1262 | Regression_pt: 0.2094 | Regression_sin_phi: 0.2428 | LR: 9.69e-04 | DataLoad Time: 0.0138s | Model Forward Time: 5.4101s | Phase Times: max_pre_step_barrier: 0.0158 | max_batch_to_device: 0.0069 | max_forward: 1.0355 | max_loss: 0.2311 | max_backward: 2.2749 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5131 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 6029.8 | uss_MB: 3868.6 | child_rss_MB: 5658.5 | allocated_MB: 342.1 | reserved_MB: 27494.0 | inactive_split_MB: 281.2 +rank0 [2026-10-28 15:14:13,236] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 3% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 17% +2 6 0x7408, 15961 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 46.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:18:14,665] INFO : Step 6000/50000 rank0 | Train Loss: 2.1372 | Train Loss Unweighted: 1.3015 | Train Loss Components: Auxiliary: 0.4179 | Cardinality: 0.1916 | Classification: 0.2091 | Classification_binary: 0.2848 | HitGrouping: 0.2746 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3966 | Regression_eta: 0.0041 | Regression_pt: 0.4029 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1393 | Classification_binary: 0.3189 | Regression_cos_phi: 0.1791 | Regression_energy: 0.2014 | Regression_eta: 0.1625 | Regression_pt: 0.2127 | Regression_sin_phi: 0.2187 | LR: 9.68e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4277s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6457 | max_loss: 0.2157 | max_backward: 1.4984 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.5590 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6033.8 | uss_MB: 4891.5 | child_rss_MB: 5658.2 | allocated_MB: 343.1 | reserved_MB: 27494.0 | inactive_split_MB: 260.7 +rank0 [2026-10-28 15:18:14,908] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 98% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 46.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:18:15,242] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 15:18:20,502] INFO : Running validation on rank0 +rank0 [2026-10-28 15:18:20,502] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:18:20,502] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:19:17,859] INFO : VALIDATION | Step=6000/50000 | Train Loss=2.1372 | Valid Loss=2.3871 | Train Loss Unweighted=1.3015 | Valid Loss Unweighted=1.3898 | Stale=0 | ETA=1803.6m +rank0 [2026-10-28 15:19:17,859] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 15:19:17,859] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 15:19:17,859] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/ari=0.381489 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.55732 +rank0 [2026-10-28 15:19:17,860] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/top1_competing=0.625992 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.599219 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.167663 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0247443 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 15:19:17,861] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/seconds_per_rank=21.7513 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding/ari=0.223 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding_minus_raw/ari=0.131377 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 15:19:17,862] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 15:19:17,867] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:19:18,123] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 15:19:18,129] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 15:19:44,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 15:19:45,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 15:19:46,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 15:19:47,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 15:19:57,581] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 15:19:57,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:19:58,136] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 15:19:58,140] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 15:20:14,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 15:20:15,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 15:20:16,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 15:20:17,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 15:20:27,582] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 15:20:27,596] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:20:28,077] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 15:20:28,081] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 15:20:43,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 15:20:44,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 15:20:45,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 15:20:46,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 15:20:56,210] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 15:20:56,229] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits +rank0 [2026-10-28 15:20:59,382] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits +rank0 [2026-10-28 15:21:00,463] INFO : Plotted number of elements +rank0 [2026-10-28 15:21:02,018] INFO : Plotted jets +rank0 [2026-10-28 15:21:03,490] INFO : Plotted jet ratio +rank0 [2026-10-28 15:21:03,573] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 15:21:06,865] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 15:21:07,942] INFO : Plotted number of elements +rank0 [2026-10-28 15:21:09,497] INFO : Plotted jets +rank0 [2026-10-28 15:21:10,616] INFO : Plotted jet ratio +rank0 [2026-10-28 15:21:10,667] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits +rank0 [2026-10-28 15:21:13,136] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits +rank0 [2026-10-28 15:21:14,008] INFO : Plotted number of elements +rank0 [2026-10-28 15:21:15,490] INFO : Plotted jets +rank0 [2026-10-28 15:21:16,632] INFO : Plotted jet ratio +rank0 [2026-10-28 15:25:30,854] INFO : Step 6100/50000 rank0 | Train Loss: 2.4040 | Train Loss Unweighted: 1.3907 | Train Loss Components: Auxiliary: 0.4661 | Cardinality: 0.2610 | Classification: 0.2427 | Classification_binary: 0.3401 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3968 | Regression_eta: 0.0047 | Regression_pt: 0.4016 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1359 | Classification_binary: 0.3732 | Regression_cos_phi: 0.2036 | Regression_energy: 0.1698 | Regression_eta: 0.1378 | Regression_pt: 0.1782 | Regression_sin_phi: 0.1889 | LR: 9.67e-04 | DataLoad Time: 0.0191s | Model Forward Time: 7.4112s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0073 | max_forward: 1.0567 | max_loss: 0.2452 | max_backward: 2.4106 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.9836 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 7214.6 | uss_MB: 5292.3 | child_rss_MB: 5654.8 | allocated_MB: 344.4 | reserved_MB: 23648.0 | inactive_split_MB: 194.8 +rank0 [2026-10-28 15:25:31,094] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 1% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 48.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 70% +4 8 0x7408, 13395 51.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 93% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:29:39,294] INFO : Step 6200/50000 rank0 | Train Loss: 2.4942 | Train Loss Unweighted: 1.4287 | Train Loss Components: Auxiliary: 0.4744 | Cardinality: 0.2537 | Classification: 0.2580 | Classification_binary: 0.3817 | HitGrouping: 0.2748 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3867 | Regression_eta: 0.0050 | Regression_pt: 0.3920 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1680 | Classification_binary: 0.3590 | Regression_cos_phi: 0.2067 | Regression_energy: 0.1953 | Regression_eta: 0.1527 | Regression_pt: 0.2071 | Regression_sin_phi: 0.1844 | LR: 9.66e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7464s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7838 | max_loss: 0.1532 | max_backward: 1.6314 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6093.8 | uss_MB: 4537.3 | child_rss_MB: 5656.7 | allocated_MB: 343.8 | reserved_MB: 23830.0 | inactive_split_MB: 215.7 +rank0 [2026-10-28 15:29:39,534] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 1% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 73% +2 6 0x7408, 15961 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +6 10 0x7408, 62036 46.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:34:14,939] INFO : Step 6300/50000 rank0 | Train Loss: 2.2204 | Train Loss Unweighted: 1.2763 | Train Loss Components: Auxiliary: 0.4285 | Cardinality: 0.2506 | Classification: 0.2258 | Classification_binary: 0.2983 | HitGrouping: 0.2710 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3704 | Regression_eta: 0.0045 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1466 | Classification_binary: 0.2833 | Regression_cos_phi: 0.1404 | Regression_energy: 0.1682 | Regression_eta: 0.1157 | Regression_pt: 0.1786 | Regression_sin_phi: 0.1451 | LR: 9.65e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3593s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.5854 | max_loss: 0.1923 | max_backward: 1.0848 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.8586 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6096.2 | uss_MB: 4957.2 | child_rss_MB: 5651.4 | allocated_MB: 339.7 | reserved_MB: 24826.0 | inactive_split_MB: 251.6 +rank0 [2026-10-28 15:34:15,178] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 44.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:38:40,303] INFO : Step 6400/50000 rank0 | Train Loss: 2.2882 | Train Loss Unweighted: 1.3014 | Train Loss Components: Auxiliary: 0.4348 | Cardinality: 0.2271 | Classification: 0.2431 | Classification_binary: 0.3058 | HitGrouping: 0.2702 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3679 | Regression_eta: 0.0046 | Regression_pt: 0.3751 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1695 | Classification_binary: 0.3622 | Regression_cos_phi: 0.2525 | Regression_energy: 0.1845 | Regression_eta: 0.1373 | Regression_pt: 0.1926 | Regression_sin_phi: 0.2002 | LR: 9.64e-04 | DataLoad Time: 0.0201s | Model Forward Time: 6.5490s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7489 | max_loss: 0.3608 | max_backward: 2.5327 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6095.6 | uss_MB: 4594.4 | child_rss_MB: 5649.8 | allocated_MB: 345.8 | reserved_MB: 24826.0 | inactive_split_MB: 253.0 +rank0 [2026-10-28 15:38:40,542] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 48.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 50.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +6 10 0x7408, 62036 45.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:42:46,231] INFO : Step 6500/50000 rank0 | Train Loss: 2.4626 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.3022 | Classification: 0.2393 | Classification_binary: 0.3805 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3783 | Regression_eta: 0.0049 | Regression_pt: 0.3831 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0989 | Classification_binary: 0.3687 | Regression_cos_phi: 0.1839 | Regression_energy: 0.1780 | Regression_eta: 0.1047 | Regression_pt: 0.1866 | Regression_sin_phi: 0.1276 | LR: 9.63e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8189s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7600 | max_loss: 0.1555 | max_backward: 1.3910 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6094.6 | uss_MB: 4960.0 | child_rss_MB: 5656.1 | allocated_MB: 344.4 | reserved_MB: 24826.0 | inactive_split_MB: 269.0 +rank0 [2026-10-28 15:42:46,468] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:46:48,418] INFO : Step 6600/50000 rank0 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3617 | Train Loss Components: Auxiliary: 0.4493 | Cardinality: 0.2405 | Classification: 0.2289 | Classification_binary: 0.3388 | HitGrouping: 0.2650 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3914 | Regression_eta: 0.0045 | Regression_pt: 0.3935 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1207 | Classification_binary: 0.3022 | Regression_cos_phi: 0.1576 | Regression_energy: 0.1912 | Regression_eta: 0.1208 | Regression_pt: 0.2002 | Regression_sin_phi: 0.1731 | LR: 9.62e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.4343s | Phase Times: max_pre_step_barrier: 0.1106 | max_batch_to_device: 0.0061 | max_forward: 0.7587 | max_loss: 0.2869 | max_backward: 2.2641 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1321 | Memory: rss_MB: 6092.2 | uss_MB: 4956.2 | child_rss_MB: 5651.6 | allocated_MB: 343.8 | reserved_MB: 25846.0 | inactive_split_MB: 267.9 +rank0 [2026-10-28 15:46:48,668] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:51:25,224] INFO : Step 6700/50000 rank0 | Train Loss: 2.1225 | Train Loss Unweighted: 1.2385 | Train Loss Components: Auxiliary: 0.4092 | Cardinality: 0.2100 | Classification: 0.2079 | Classification_binary: 0.2916 | HitGrouping: 0.2751 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3623 | Regression_eta: 0.0041 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1296 | Classification_binary: 0.2943 | Regression_cos_phi: 0.1698 | Regression_energy: 0.1929 | Regression_eta: 0.1558 | Regression_pt: 0.2073 | Regression_sin_phi: 0.2541 | LR: 9.61e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7744s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.6452 | max_loss: 0.1453 | max_backward: 1.2984 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.0269 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6098.1 | uss_MB: 4965.1 | child_rss_MB: 5657.8 | allocated_MB: 345.0 | reserved_MB: 25846.0 | inactive_split_MB: 258.8 +rank0 [2026-10-28 15:51:25,471] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 6% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 0% +2 6 0x7408, 15961 48.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 95% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 45.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:55:33,655] INFO : Step 6800/50000 rank0 | Train Loss: 2.4601 | Train Loss Unweighted: 1.3772 | Train Loss Components: Auxiliary: 0.4589 | Cardinality: 0.2427 | Classification: 0.2631 | Classification_binary: 0.3490 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3747 | Regression_eta: 0.0054 | Regression_pt: 0.3793 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1707 | Classification_binary: 0.3932 | Regression_cos_phi: 0.2490 | Regression_energy: 0.1786 | Regression_eta: 0.1628 | Regression_pt: 0.1884 | Regression_sin_phi: 0.2207 | LR: 9.60e-04 | DataLoad Time: 0.0240s | Model Forward Time: 6.2960s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0662 | max_forward: 1.6558 | max_loss: 0.3855 | max_backward: 2.1309 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 1.0709 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6092.6 | uss_MB: 4961.3 | child_rss_MB: 5647.9 | allocated_MB: 345.8 | reserved_MB: 26622.0 | inactive_split_MB: 194.2 +rank0 [2026-10-28 15:55:33,897] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:59:35,878] INFO : Step 6900/50000 rank0 | Train Loss: 2.3793 | Train Loss Unweighted: 1.3337 | Train Loss Components: Auxiliary: 0.4495 | Cardinality: 0.2757 | Classification: 0.2300 | Classification_binary: 0.3711 | HitGrouping: 0.2647 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3605 | Regression_eta: 0.0045 | Regression_pt: 0.3628 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1267 | Classification_binary: 0.3487 | Regression_cos_phi: 0.1764 | Regression_energy: 0.1716 | Regression_eta: 0.1085 | Regression_pt: 0.1788 | Regression_sin_phi: 0.1669 | LR: 9.58e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9632s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1120 | max_forward: 0.8190 | max_loss: 0.1496 | max_backward: 1.4356 | max_optimizer_step: 0.2847 | max_tensorboard_logging: 0.8440 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6096.9 | uss_MB: 4962.9 | child_rss_MB: 5656.8 | allocated_MB: 345.4 | reserved_MB: 26622.0 | inactive_split_MB: 300.5 +rank0 [2026-10-28 15:59:36,118] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:03:58,097] INFO : Step 7000/50000 rank0 | Train Loss: 2.2410 | Train Loss Unweighted: 1.2707 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2746 | Classification: 0.1970 | Classification_binary: 0.3002 | HitGrouping: 0.2655 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3795 | Regression_eta: 0.0045 | Regression_pt: 0.3852 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2533 | Regression_cos_phi: 0.1156 | Regression_energy: 0.1823 | Regression_eta: 0.0793 | Regression_pt: 0.1908 | Regression_sin_phi: 0.1530 | LR: 9.57e-04 | DataLoad Time: 0.0084s | Model Forward Time: 6.5431s | Phase Times: max_pre_step_barrier: 0.0229 | max_batch_to_device: 0.0154 | max_forward: 1.0393 | max_loss: 0.2251 | max_backward: 2.0451 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.9425 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6097.4 | uss_MB: 4965.6 | child_rss_MB: 5656.9 | allocated_MB: 345.5 | reserved_MB: 26622.0 | inactive_split_MB: 296.5 +rank0 [2026-10-28 16:03:58,342] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +4 8 0x7408, 13395 50.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 98% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 45.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 0% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 97% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:03:58,363] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 16:03:58,556] INFO : Running validation on rank0 +rank0 [2026-10-28 16:03:58,557] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:03:58,557] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:04:52,411] INFO : VALIDATION | Step=7000/50000 | Train Loss=2.2410 | Valid Loss=2.3006 | Train Loss Unweighted=1.2707 | Valid Loss Unweighted=1.3024 | Stale=0 | ETA=1790.8m +rank0 [2026-10-28 16:04:52,411] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:04:52,411] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 16:04:52,411] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 16:04:52,411] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:04:52,411] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 16:04:52,411] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 16:04:52,411] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/ari=0.370043 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.545101 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/top1_competing=0.613506 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.591664 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.17911 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0171892 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 16:04:52,412] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/seconds_per_rank=21.8889 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 16:04:52,413] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 16:04:52,414] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:04:52,414] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding/ari=0.241264 +rank0 [2026-10-28 16:04:52,414] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 16:04:52,414] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding_minus_raw/ari=0.149641 +rank0 [2026-10-28 16:04:52,414] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 16:04:52,414] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 16:04:52,414] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 16:04:52,418] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:04:52,686] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 16:04:52,701] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 16:05:18,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 16:05:19,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 16:05:20,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 16:05:22,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 16:05:32,014] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 16:05:32,028] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:05:32,550] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 16:05:32,553] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 16:05:48,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 16:05:49,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 16:05:51,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 16:05:52,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 16:06:02,129] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 16:06:02,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:06:02,705] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 16:06:02,709] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 16:06:18,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 16:06:19,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 16:06:20,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 16:06:20,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 16:06:30,988] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 16:06:31,003] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits +rank0 [2026-10-28 16:06:34,121] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits +rank0 [2026-10-28 16:06:35,165] INFO : Plotted number of elements +rank0 [2026-10-28 16:06:36,707] INFO : Plotted jets +rank0 [2026-10-28 16:06:37,865] INFO : Plotted jet ratio +rank0 [2026-10-28 16:06:37,940] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 16:06:41,325] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 16:06:42,799] INFO : Plotted number of elements +rank0 [2026-10-28 16:06:44,291] INFO : Plotted jets +rank0 [2026-10-28 16:06:45,392] INFO : Plotted jet ratio +rank0 [2026-10-28 16:06:45,444] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits +rank0 [2026-10-28 16:06:47,786] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits +rank0 [2026-10-28 16:06:48,627] INFO : Plotted number of elements +rank0 [2026-10-28 16:06:50,101] INFO : Plotted jets +rank0 [2026-10-28 16:06:51,239] INFO : Plotted jet ratio +rank0 [2026-10-28 16:10:49,834] INFO : Step 7100/50000 rank0 | Train Loss: 2.1086 | Train Loss Unweighted: 1.2525 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.1821 | Classification: 0.2186 | Classification_binary: 0.2903 | HitGrouping: 0.2613 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3655 | Regression_eta: 0.0041 | Regression_pt: 0.3696 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1228 | Classification_binary: 0.2829 | Regression_cos_phi: 0.1529 | Regression_energy: 0.1934 | Regression_eta: 0.1181 | Regression_pt: 0.2029 | Regression_sin_phi: 0.1521 | LR: 9.56e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.7308s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6376 | max_loss: 0.1424 | max_backward: 1.3436 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8454 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 7191.1 | uss_MB: 6057.4 | child_rss_MB: 5686.3 | allocated_MB: 345.6 | reserved_MB: 22618.0 | inactive_split_MB: 206.0 +rank0 [2026-10-28 16:10:50,077] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:15:06,331] INFO : Step 7200/50000 rank0 | Train Loss: 2.4589 | Train Loss Unweighted: 1.3873 | Train Loss Components: Auxiliary: 0.4658 | Cardinality: 0.2721 | Classification: 0.2648 | Classification_binary: 0.3806 | HitGrouping: 0.2633 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3642 | Regression_eta: 0.0051 | Regression_pt: 0.3675 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1451 | Classification_binary: 0.3899 | Regression_cos_phi: 0.2293 | Regression_energy: 0.1686 | Regression_eta: 0.1213 | Regression_pt: 0.1768 | Regression_sin_phi: 0.1674 | LR: 9.55e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.9302s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 1.0191 | max_loss: 0.1674 | max_backward: 1.7813 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6183.3 | uss_MB: 5050.5 | child_rss_MB: 5662.0 | allocated_MB: 343.5 | reserved_MB: 22618.0 | inactive_split_MB: 315.3 +rank0 [2026-10-28 16:15:06,572] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 29% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 45.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:19:18,412] INFO : Step 7300/50000 rank0 | Train Loss: 2.1617 | Train Loss Unweighted: 1.2404 | Train Loss Components: Auxiliary: 0.4141 | Cardinality: 0.2258 | Classification: 0.1979 | Classification_binary: 0.3107 | HitGrouping: 0.2671 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3613 | Regression_eta: 0.0041 | Regression_pt: 0.3621 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1047 | Classification_binary: 0.2884 | Regression_cos_phi: 0.1810 | Regression_energy: 0.1630 | Regression_eta: 0.1124 | Regression_pt: 0.1682 | Regression_sin_phi: 0.1548 | LR: 9.53e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5369s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0084 | max_forward: 0.7227 | max_loss: 0.1491 | max_backward: 1.4232 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.5902 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6180.9 | uss_MB: 5051.2 | child_rss_MB: 5696.9 | allocated_MB: 345.0 | reserved_MB: 23608.0 | inactive_split_MB: 295.0 +rank0 [2026-10-28 16:19:18,652] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:23:26,731] INFO : Step 7400/50000 rank0 | Train Loss: 2.0741 | Train Loss Unweighted: 1.2014 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2287 | Classification: 0.2035 | Classification_binary: 0.2781 | HitGrouping: 0.2646 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3549 | Regression_eta: 0.0040 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1144 | Classification_binary: 0.2517 | Regression_cos_phi: 0.1056 | Regression_energy: 0.1759 | Regression_eta: 0.0961 | Regression_pt: 0.1862 | Regression_sin_phi: 0.1364 | LR: 9.52e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.3930s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6129 | max_loss: 0.2127 | max_backward: 1.2944 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6450 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6180.9 | uss_MB: 5049.5 | child_rss_MB: 5695.6 | allocated_MB: 343.8 | reserved_MB: 23608.0 | inactive_split_MB: 229.5 +rank0 [2026-10-28 16:23:26,970] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:27:19,195] INFO : Step 7500/50000 rank0 | Train Loss: 2.4640 | Train Loss Unweighted: 1.3686 | Train Loss Components: Auxiliary: 0.4637 | Cardinality: 0.2640 | Classification: 0.2570 | Classification_binary: 0.3594 | HitGrouping: 0.2687 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3684 | Regression_eta: 0.0054 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1166 | Classification_binary: 0.3246 | Regression_cos_phi: 0.1568 | Regression_energy: 0.1806 | Regression_eta: 0.1079 | Regression_pt: 0.1886 | Regression_sin_phi: 0.1461 | LR: 9.51e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8085s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0887 | max_forward: 0.7717 | max_loss: 0.1508 | max_backward: 1.3590 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.9634 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6179.4 | uss_MB: 5053.3 | child_rss_MB: 5684.8 | allocated_MB: 348.6 | reserved_MB: 23608.0 | inactive_split_MB: 283.7 +rank0 [2026-10-28 16:27:19,435] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:31:22,985] INFO : Step 7600/50000 rank0 | Train Loss: 2.3745 | Train Loss Unweighted: 1.3447 | Train Loss Components: Auxiliary: 0.4509 | Cardinality: 0.2616 | Classification: 0.2519 | Classification_binary: 0.3743 | HitGrouping: 0.2627 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3532 | Regression_eta: 0.0050 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1350 | Classification_binary: 0.3266 | Regression_cos_phi: 0.2013 | Regression_energy: 0.1623 | Regression_eta: 0.1372 | Regression_pt: 0.1678 | Regression_sin_phi: 0.1615 | LR: 9.50e-04 | DataLoad Time: 0.0165s | Model Forward Time: 16.6252s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.7345 | max_loss: 0.1617 | max_backward: 1.4897 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 12.6867 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6179.5 | uss_MB: 5051.2 | child_rss_MB: 5695.8 | allocated_MB: 345.3 | reserved_MB: 23608.0 | inactive_split_MB: 284.8 +rank0 [2026-10-28 16:31:23,242] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 47.0°C 116.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 47.0°C 122.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:35:23,718] INFO : Step 7700/50000 rank0 | Train Loss: 2.0023 | Train Loss Unweighted: 1.1475 | Train Loss Components: Auxiliary: 0.3866 | Cardinality: 0.2005 | Classification: 0.1968 | Classification_binary: 0.2784 | HitGrouping: 0.2562 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3315 | Regression_eta: 0.0040 | Regression_pt: 0.3329 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1100 | Classification_binary: 0.2665 | Regression_cos_phi: 0.1734 | Regression_energy: 0.1578 | Regression_eta: 0.0894 | Regression_pt: 0.1706 | Regression_sin_phi: 0.1233 | LR: 9.48e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.6747s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7891 | max_loss: 0.1927 | max_backward: 1.4291 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6182.9 | uss_MB: 5055.0 | child_rss_MB: 5702.0 | allocated_MB: 343.9 | reserved_MB: 23608.0 | inactive_split_MB: 279.7 +rank0 [2026-10-28 16:35:23,976] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:39:23,781] INFO : Step 7800/50000 rank0 | Train Loss: 2.0473 | Train Loss Unweighted: 1.2037 | Train Loss Components: Auxiliary: 0.4013 | Cardinality: 0.1891 | Classification: 0.2102 | Classification_binary: 0.2760 | HitGrouping: 0.2604 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3526 | Regression_eta: 0.0038 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1243 | Classification_binary: 0.3302 | Regression_cos_phi: 0.2149 | Regression_energy: 0.1672 | Regression_eta: 0.1172 | Regression_pt: 0.1761 | Regression_sin_phi: 0.1816 | LR: 9.47e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.1717s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.7326 | max_loss: 0.2464 | max_backward: 1.8124 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9395 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6182.4 | uss_MB: 5059.4 | child_rss_MB: 5675.6 | allocated_MB: 343.5 | reserved_MB: 24612.0 | inactive_split_MB: 224.8 +rank0 [2026-10-28 16:39:24,023] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:43:22,258] INFO : Step 7900/50000 rank0 | Train Loss: 2.3872 | Train Loss Unweighted: 1.3279 | Train Loss Components: Auxiliary: 0.4525 | Cardinality: 0.2722 | Classification: 0.2401 | Classification_binary: 0.3580 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3573 | Regression_eta: 0.0051 | Regression_pt: 0.3627 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1465 | Classification_binary: 0.3568 | Regression_cos_phi: 0.1904 | Regression_energy: 0.1670 | Regression_eta: 0.1714 | Regression_pt: 0.1766 | Regression_sin_phi: 0.1955 | LR: 9.46e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4052s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7032 | max_loss: 0.2052 | max_backward: 1.5183 | max_optimizer_step: 0.1922 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6185.2 | uss_MB: 5060.7 | child_rss_MB: 5695.7 | allocated_MB: 347.1 | reserved_MB: 24612.0 | inactive_split_MB: 279.0 +rank0 [2026-10-28 16:43:22,496] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:47:22,503] INFO : Step 8000/50000 rank0 | Train Loss: 2.1303 | Train Loss Unweighted: 1.2019 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2451 | Classification: 0.2031 | Classification_binary: 0.3087 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3382 | Regression_eta: 0.0041 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1118 | Classification_binary: 0.2569 | Regression_cos_phi: 0.1227 | Regression_energy: 0.1394 | Regression_eta: 0.0922 | Regression_pt: 0.1483 | Regression_sin_phi: 0.1392 | LR: 9.44e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.5467s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0554 | max_forward: 0.7608 | max_loss: 0.1617 | max_backward: 1.3810 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 6186.5 | uss_MB: 4730.8 | child_rss_MB: 5692.5 | allocated_MB: 344.2 | reserved_MB: 24612.0 | inactive_split_MB: 224.1 +rank0 [2026-10-28 16:47:22,742] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:47:22,762] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 16:47:22,951] INFO : Running validation on rank0 +rank0 [2026-10-28 16:47:22,951] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:47:22,952] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:48:14,338] INFO : VALIDATION | Step=8000/50000 | Train Loss=2.1303 | Valid Loss=2.2067 | Train Loss Unweighted=1.2019 | Valid Loss Unweighted=1.2331 | Stale=0 | ETA=1758.2m +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 16:48:14,339] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/ari=0.44165 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.511236 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/top1_competing=0.570597 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.585611 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.107503 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0111358 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 16:48:14,340] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/seconds_per_rank=22.0044 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding/ari=0.239994 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 16:48:14,341] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding_minus_raw/ari=0.148371 +rank0 [2026-10-28 16:48:14,342] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 16:48:14,342] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 16:48:14,342] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 16:48:14,347] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:48:14,602] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 16:48:14,608] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 16:48:40,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 16:48:41,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 16:48:42,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 16:48:43,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 16:48:53,933] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 16:48:53,945] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:48:54,468] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 16:48:54,471] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 16:49:10,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 16:49:11,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 16:49:13,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 16:49:14,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 16:49:24,420] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 16:49:24,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:49:24,963] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 16:49:24,967] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 16:49:40,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 16:49:41,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 16:49:42,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 16:49:42,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 16:49:52,929] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 16:49:52,945] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits +rank0 [2026-10-28 16:49:56,253] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits +rank0 [2026-10-28 16:49:57,367] INFO : Plotted number of elements +rank0 [2026-10-28 16:49:58,986] INFO : Plotted jets +rank0 [2026-10-28 16:50:00,212] INFO : Plotted jet ratio +rank0 [2026-10-28 16:50:00,280] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 16:50:03,609] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 16:50:05,053] INFO : Plotted number of elements +rank0 [2026-10-28 16:50:06,546] INFO : Plotted jets +rank0 [2026-10-28 16:50:07,635] INFO : Plotted jet ratio +rank0 [2026-10-28 16:50:07,688] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits +rank0 [2026-10-28 16:50:10,073] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits +rank0 [2026-10-28 16:50:10,919] INFO : Plotted number of elements +rank0 [2026-10-28 16:50:12,397] INFO : Plotted jets +rank0 [2026-10-28 16:50:13,520] INFO : Plotted jet ratio +rank0 [2026-10-28 16:54:10,054] INFO : Step 8100/50000 rank0 | Train Loss: 2.0320 | Train Loss Unweighted: 1.1744 | Train Loss Components: Auxiliary: 0.3893 | Cardinality: 0.1986 | Classification: 0.1938 | Classification_binary: 0.2691 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3510 | Regression_eta: 0.0039 | Regression_pt: 0.3525 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1401 | Classification_binary: 0.2897 | Regression_cos_phi: 0.1572 | Regression_energy: 0.1668 | Regression_eta: 0.1325 | Regression_pt: 0.1736 | Regression_sin_phi: 0.1895 | LR: 9.43e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9214s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6251 | max_loss: 0.2465 | max_backward: 1.3564 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8507 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7244.9 | uss_MB: 6118.4 | child_rss_MB: 5660.8 | allocated_MB: 342.9 | reserved_MB: 23510.0 | inactive_split_MB: 258.1 +rank0 [2026-10-28 16:54:10,306] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:58:01,438] INFO : Step 8200/50000 rank0 | Train Loss: 2.1917 | Train Loss Unweighted: 1.2329 | Train Loss Components: Auxiliary: 0.4200 | Cardinality: 0.2405 | Classification: 0.2359 | Classification_binary: 0.3328 | HitGrouping: 0.2582 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3251 | Regression_eta: 0.0043 | Regression_pt: 0.3305 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1281 | Classification_binary: 0.3516 | Regression_cos_phi: 0.1570 | Regression_energy: 0.1588 | Regression_eta: 0.1126 | Regression_pt: 0.1651 | Regression_sin_phi: 0.1660 | LR: 9.42e-04 | DataLoad Time: 0.0141s | Model Forward Time: 5.0077s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0064 | max_forward: 0.7679 | max_loss: 0.2484 | max_backward: 1.6620 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.8049 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1289 | Memory: rss_MB: 6232.6 | uss_MB: 5106.1 | child_rss_MB: 5673.2 | allocated_MB: 343.2 | reserved_MB: 23510.0 | inactive_split_MB: 292.4 +rank0 [2026-10-28 16:58:01,679] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:02:04,330] INFO : Step 8300/50000 rank0 | Train Loss: 2.1924 | Train Loss Unweighted: 1.2399 | Train Loss Components: Auxiliary: 0.4122 | Cardinality: 0.2039 | Classification: 0.2083 | Classification_binary: 0.3661 | HitGrouping: 0.2599 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3243 | Regression_eta: 0.0043 | Regression_pt: 0.3323 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1338 | Classification_binary: 0.3571 | Regression_cos_phi: 0.1966 | Regression_energy: 0.1499 | Regression_eta: 0.1227 | Regression_pt: 0.1587 | Regression_sin_phi: 0.2072 | LR: 9.40e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.5215s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0061 | max_forward: 0.9407 | max_loss: 0.2296 | max_backward: 1.9622 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6228.1 | uss_MB: 5055.3 | child_rss_MB: 5669.2 | allocated_MB: 344.6 | reserved_MB: 24580.0 | inactive_split_MB: 267.5 +rank0 [2026-10-28 17:02:04,568] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:06:08,370] INFO : Step 8400/50000 rank0 | Train Loss: 1.9542 | Train Loss Unweighted: 1.1189 | Train Loss Components: Auxiliary: 0.3756 | Cardinality: 0.1979 | Classification: 0.1926 | Classification_binary: 0.2783 | HitGrouping: 0.2585 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3198 | Regression_eta: 0.0035 | Regression_pt: 0.3209 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0892 | Classification_binary: 0.2550 | Regression_cos_phi: 0.1567 | Regression_energy: 0.1715 | Regression_eta: 0.0747 | Regression_pt: 0.1777 | Regression_sin_phi: 0.1138 | LR: 9.39e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8582s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6939 | max_loss: 0.1474 | max_backward: 1.5357 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 0.9892 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 6216.9 | uss_MB: 5103.3 | child_rss_MB: 5675.6 | allocated_MB: 341.1 | reserved_MB: 24580.0 | inactive_split_MB: 178.9 +rank0 [2026-10-28 17:06:08,611] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:10:18,242] INFO : Step 8500/50000 rank0 | Train Loss: 2.0642 | Train Loss Unweighted: 1.1724 | Train Loss Components: Auxiliary: 0.3964 | Cardinality: 0.2023 | Classification: 0.2152 | Classification_binary: 0.2897 | HitGrouping: 0.2607 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3289 | Regression_eta: 0.0041 | Regression_pt: 0.3302 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0989 | Classification_binary: 0.2657 | Regression_cos_phi: 0.1277 | Regression_energy: 0.1570 | Regression_eta: 0.0881 | Regression_pt: 0.1641 | Regression_sin_phi: 0.1408 | LR: 9.37e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3821s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0069 | max_forward: 0.6424 | max_loss: 0.1972 | max_backward: 1.2711 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.7327 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6218.9 | uss_MB: 5108.3 | child_rss_MB: 5667.6 | allocated_MB: 344.4 | reserved_MB: 25586.0 | inactive_split_MB: 317.0 +rank0 [2026-10-28 17:10:18,482] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 5% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:14:25,396] INFO : Step 8600/50000 rank0 | Train Loss: 2.2820 | Train Loss Unweighted: 1.2960 | Train Loss Components: Auxiliary: 0.4378 | Cardinality: 0.2546 | Classification: 0.2294 | Classification_binary: 0.3478 | HitGrouping: 0.2584 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3521 | Regression_eta: 0.0046 | Regression_pt: 0.3576 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1520 | Classification_binary: 0.3773 | Regression_cos_phi: 0.2075 | Regression_energy: 0.1695 | Regression_eta: 0.1421 | Regression_pt: 0.1769 | Regression_sin_phi: 0.1962 | LR: 9.36e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7199s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0069 | max_forward: 0.7396 | max_loss: 0.1527 | max_backward: 1.6088 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5007 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6218.9 | uss_MB: 5112.5 | child_rss_MB: 5664.7 | allocated_MB: 344.8 | reserved_MB: 25586.0 | inactive_split_MB: 283.2 +rank0 [2026-10-28 17:14:25,637] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 1% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 49.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 5% +6 10 0x7408, 62036 46.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:18:21,646] INFO : Step 8700/50000 rank0 | Train Loss: 2.0581 | Train Loss Unweighted: 1.1705 | Train Loss Components: Auxiliary: 0.4038 | Cardinality: 0.2463 | Classification: 0.2066 | Classification_binary: 0.3017 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3267 | Regression_eta: 0.0035 | Regression_pt: 0.3282 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0992 | Classification_binary: 0.2692 | Regression_cos_phi: 0.1483 | Regression_energy: 0.1488 | Regression_eta: 0.0943 | Regression_pt: 0.1517 | Regression_sin_phi: 0.1472 | LR: 9.34e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.3301s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6873 | max_loss: 0.2444 | max_backward: 1.2985 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6214.6 | uss_MB: 5113.2 | child_rss_MB: 5650.1 | allocated_MB: 343.7 | reserved_MB: 25586.0 | inactive_split_MB: 260.4 +rank0 [2026-10-28 17:18:21,886] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 52.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:22:22,503] INFO : Step 8800/50000 rank0 | Train Loss: 1.8415 | Train Loss Unweighted: 1.0591 | Train Loss Components: Auxiliary: 0.3597 | Cardinality: 0.1811 | Classification: 0.1734 | Classification_binary: 0.2490 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3144 | Regression_eta: 0.0034 | Regression_pt: 0.3153 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1183 | Classification_binary: 0.2353 | Regression_cos_phi: 0.1925 | Regression_energy: 0.1499 | Regression_eta: 0.0899 | Regression_pt: 0.1535 | Regression_sin_phi: 0.1897 | LR: 9.33e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3609s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6737 | max_loss: 0.2385 | max_backward: 1.3073 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.5819 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6215.6 | uss_MB: 5114.6 | child_rss_MB: 5649.0 | allocated_MB: 344.1 | reserved_MB: 25586.0 | inactive_split_MB: 205.9 +rank0 [2026-10-28 17:22:22,740] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:26:44,702] INFO : Step 8900/50000 rank0 | Train Loss: 2.1706 | Train Loss Unweighted: 1.1768 | Train Loss Components: Auxiliary: 0.4167 | Cardinality: 0.3000 | Classification: 0.2064 | Classification_binary: 0.3037 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3284 | Regression_eta: 0.0039 | Regression_pt: 0.3304 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1440 | Classification_binary: 0.3470 | Regression_cos_phi: 0.2715 | Regression_energy: 0.1542 | Regression_eta: 0.1275 | Regression_pt: 0.1606 | Regression_sin_phi: 0.2287 | LR: 9.31e-04 | DataLoad Time: 0.0177s | Model Forward Time: 6.6512s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0073 | max_forward: 0.9973 | max_loss: 0.4040 | max_backward: 3.4126 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6194.6 | uss_MB: 5119.9 | child_rss_MB: 5654.4 | allocated_MB: 347.1 | reserved_MB: 25586.0 | inactive_split_MB: 275.1 +rank0 [2026-10-28 17:26:44,955] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 5% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 49.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 50.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 46.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:30:47,677] INFO : Step 9000/50000 rank0 | Train Loss: 2.1357 | Train Loss Unweighted: 1.1883 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.2351 | Classification: 0.2003 | Classification_binary: 0.3467 | HitGrouping: 0.2532 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3143 | Regression_eta: 0.0041 | Regression_pt: 0.3186 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0818 | Classification_binary: 0.2780 | Regression_cos_phi: 0.1246 | Regression_energy: 0.1432 | Regression_eta: 0.0908 | Regression_pt: 0.1510 | Regression_sin_phi: 0.1378 | LR: 9.30e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.5153s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.7873 | max_loss: 0.2178 | max_backward: 1.5407 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.3125 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 6192.6 | uss_MB: 5121.2 | child_rss_MB: 5660.4 | allocated_MB: 344.6 | reserved_MB: 25586.0 | inactive_split_MB: 289.9 +rank0 [2026-10-28 17:30:47,947] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 49.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 86% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 46.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:30:47,967] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 17:30:48,156] INFO : Running validation on rank0 +rank0 [2026-10-28 17:30:48,156] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:30:48,156] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:31:41,495] INFO : VALIDATION | Step=9000/50000 | Train Loss=2.1357 | Valid Loss=2.0752 | Train Loss Unweighted=1.1883 | Valid Loss Unweighted=1.1438 | Stale=0 | ETA=1723.6m +rank0 [2026-10-28 17:31:41,495] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 17:31:41,495] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 17:31:41,495] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 17:31:41,495] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 17:31:41,496] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 17:31:41,496] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 17:31:41,496] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 17:31:41,496] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 17:31:41,496] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 17:31:41,496] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 17:31:41,496] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 17:31:41,496] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 17:31:41,496] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 17:31:41,496] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/ari=0.424652 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.527755 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/top1_competing=0.591696 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.582266 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.1245 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.00779165 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 17:31:41,497] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/seconds_per_rank=21.8868 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 17:31:41,498] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 17:31:41,499] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 17:31:41,499] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 17:31:41,499] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 17:31:41,499] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding/ari=0.260592 +rank0 [2026-10-28 17:31:41,499] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 17:31:41,499] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding_minus_raw/ari=0.168969 +rank0 [2026-10-28 17:31:41,499] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 17:31:41,499] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 17:31:41,499] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 17:31:41,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:31:41,757] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 17:31:41,765] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 17:32:08,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 17:32:10,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 17:32:11,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 17:32:12,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 17:32:22,644] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-28 17:32:22,656] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:32:23,201] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 17:32:23,205] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 17:32:39,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 17:32:40,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 17:32:41,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 17:32:42,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 17:32:52,234] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 17:32:52,551] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:32:53,105] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 17:32:53,109] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 17:33:08,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 17:33:09,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 17:33:10,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 17:33:11,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 17:33:21,140] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 17:33:21,157] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits +rank0 [2026-10-28 17:33:24,396] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits +rank0 [2026-10-28 17:33:25,432] INFO : Plotted number of elements +rank0 [2026-10-28 17:33:26,971] INFO : Plotted jets +rank0 [2026-10-28 17:33:28,152] INFO : Plotted jet ratio +rank0 [2026-10-28 17:33:28,206] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 17:33:31,518] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 17:33:32,614] INFO : Plotted number of elements +rank0 [2026-10-28 17:33:34,176] INFO : Plotted jets +rank0 [2026-10-28 17:33:35,731] INFO : Plotted jet ratio +rank0 [2026-10-28 17:33:35,783] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits +rank0 [2026-10-28 17:33:38,206] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits +rank0 [2026-10-28 17:33:39,090] INFO : Plotted number of elements +rank0 [2026-10-28 17:33:40,565] INFO : Plotted jets +rank0 [2026-10-28 17:33:41,700] INFO : Plotted jet ratio +rank0 [2026-10-28 17:37:41,843] INFO : Step 9100/50000 rank0 | Train Loss: 1.8448 | Train Loss Unweighted: 1.0695 | Train Loss Components: Auxiliary: 0.3538 | Cardinality: 0.1722 | Classification: 0.1878 | Classification_binary: 0.2661 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3012 | Regression_eta: 0.0033 | Regression_pt: 0.3075 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0946 | Classification_binary: 0.2199 | Regression_cos_phi: 0.1227 | Regression_energy: 0.1383 | Regression_eta: 0.0610 | Regression_pt: 0.1458 | Regression_sin_phi: 0.1380 | LR: 9.28e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.0770s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0153 | max_forward: 1.2776 | max_loss: 0.3881 | max_backward: 1.3802 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9203 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7188.5 | uss_MB: 6128.3 | child_rss_MB: 5636.7 | allocated_MB: 345.3 | reserved_MB: 22582.0 | inactive_split_MB: 275.2 +rank0 [2026-10-28 17:37:42,082] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +6 10 0x7408, 62036 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:41:50,888] INFO : Step 9200/50000 rank0 | Train Loss: 1.9883 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3829 | Cardinality: 0.2086 | Classification: 0.1812 | Classification_binary: 0.2802 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3207 | Regression_eta: 0.0035 | Regression_pt: 0.3257 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1067 | Classification_binary: 0.3248 | Regression_cos_phi: 0.2151 | Regression_energy: 0.1472 | Regression_eta: 0.1103 | Regression_pt: 0.1549 | Regression_sin_phi: 0.1801 | LR: 9.27e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1736s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0073 | max_forward: 0.6839 | max_loss: 0.2414 | max_backward: 1.6336 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 1.0636 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6214.5 | uss_MB: 5161.4 | child_rss_MB: 5636.3 | allocated_MB: 345.6 | reserved_MB: 23224.0 | inactive_split_MB: 190.1 +rank0 [2026-10-28 17:41:51,125] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +6 10 0x7408, 62036 45.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:45:53,045] INFO : Step 9300/50000 rank0 | Train Loss: 2.0749 | Train Loss Unweighted: 1.1591 | Train Loss Components: Auxiliary: 0.4049 | Cardinality: 0.2573 | Classification: 0.2065 | Classification_binary: 0.3180 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3111 | Regression_eta: 0.0036 | Regression_pt: 0.3160 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1423 | Classification_binary: 0.3271 | Regression_cos_phi: 0.2075 | Regression_energy: 0.1511 | Regression_eta: 0.1278 | Regression_pt: 0.1587 | Regression_sin_phi: 0.1638 | LR: 9.25e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8326s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8329 | max_loss: 0.1499 | max_backward: 1.4059 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.8766 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6214.5 | uss_MB: 5164.3 | child_rss_MB: 5654.0 | allocated_MB: 344.8 | reserved_MB: 24586.0 | inactive_split_MB: 287.2 +rank0 [2026-10-28 17:45:53,283] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +6 10 0x7408, 62036 45.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:50:17,691] INFO : Step 9400/50000 rank0 | Train Loss: 1.8467 | Train Loss Unweighted: 1.0417 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1520 | Classification: 0.1837 | Classification_binary: 0.2977 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2742 | Regression_eta: 0.0040 | Regression_pt: 0.2784 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0944 | Classification_binary: 0.2593 | Regression_cos_phi: 0.1399 | Regression_energy: 0.1201 | Regression_eta: 0.0904 | Regression_pt: 0.1256 | Regression_sin_phi: 0.1399 | LR: 9.24e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.3507s | Phase Times: max_pre_step_barrier: 0.0180 | max_batch_to_device: 0.0062 | max_forward: 1.5076 | max_loss: 0.3748 | max_backward: 2.6247 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.5247 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6206.4 | uss_MB: 4649.5 | child_rss_MB: 5650.0 | allocated_MB: 343.9 | reserved_MB: 24586.0 | inactive_split_MB: 267.3 +rank0 [2026-10-28 17:50:17,958] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 99% +6 10 0x7408, 62036 45.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 78% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:54:18,877] INFO : Step 9500/50000 rank0 | Train Loss: 1.8199 | Train Loss Unweighted: 0.9995 | Train Loss Components: Auxiliary: 0.3464 | Cardinality: 0.1875 | Classification: 0.1651 | Classification_binary: 0.2558 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2851 | Regression_eta: 0.0035 | Regression_pt: 0.2864 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1032 | Classification_binary: 0.2315 | Regression_cos_phi: 0.1501 | Regression_energy: 0.1317 | Regression_eta: 0.0725 | Regression_pt: 0.1370 | Regression_sin_phi: 0.1288 | LR: 9.22e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.5179s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6532 | max_loss: 0.2385 | max_backward: 1.0043 | max_optimizer_step: 0.3001 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6205.2 | uss_MB: 5165.5 | child_rss_MB: 5654.5 | allocated_MB: 341.2 | reserved_MB: 24586.0 | inactive_split_MB: 180.8 +rank0 [2026-10-28 17:54:19,117] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 45% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 0% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 99% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 65% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:58:24,372] INFO : Step 9600/50000 rank0 | Train Loss: 2.0900 | Train Loss Unweighted: 1.1491 | Train Loss Components: Auxiliary: 0.4030 | Cardinality: 0.2464 | Classification: 0.2089 | Classification_binary: 0.3189 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3034 | Regression_eta: 0.0042 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0957 | Classification_binary: 0.3366 | Regression_cos_phi: 0.1546 | Regression_energy: 0.1341 | Regression_eta: 0.1121 | Regression_pt: 0.1442 | Regression_sin_phi: 0.1642 | LR: 9.21e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6222s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1083 | max_forward: 0.7381 | max_loss: 0.1457 | max_backward: 1.4890 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4970 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6206.2 | uss_MB: 5167.2 | child_rss_MB: 5637.2 | allocated_MB: 345.2 | reserved_MB: 25594.0 | inactive_split_MB: 190.6 +rank0 [2026-10-28 17:58:24,609] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +6 10 0x7408, 62036 46.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:02:35,790] INFO : Step 9700/50000 rank0 | Train Loss: 2.1397 | Train Loss Unweighted: 1.1700 | Train Loss Components: Auxiliary: 0.4082 | Cardinality: 0.2377 | Classification: 0.1942 | Classification_binary: 0.3460 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3095 | Regression_eta: 0.0043 | Regression_pt: 0.3118 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0976 | Classification_binary: 0.3237 | Regression_cos_phi: 0.1533 | Regression_energy: 0.1516 | Regression_eta: 0.0942 | Regression_pt: 0.1576 | Regression_sin_phi: 0.1380 | LR: 9.19e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7484s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7744 | max_loss: 0.1497 | max_backward: 1.6846 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4710 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 6209.3 | uss_MB: 5170.1 | child_rss_MB: 5650.1 | allocated_MB: 345.9 | reserved_MB: 25594.0 | inactive_split_MB: 288.1 +rank0 [2026-10-28 18:02:36,041] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +6 10 0x7408, 62036 46.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:06:42,951] INFO : Step 9800/50000 rank0 | Train Loss: 1.8589 | Train Loss Unweighted: 1.0245 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2213 | Classification: 0.1590 | Classification_binary: 0.2751 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2908 | Regression_eta: 0.0031 | Regression_pt: 0.2931 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1019 | Classification_binary: 0.2484 | Regression_cos_phi: 0.1332 | Regression_energy: 0.1366 | Regression_eta: 0.0972 | Regression_pt: 0.1433 | Regression_sin_phi: 0.1108 | LR: 9.17e-04 | DataLoad Time: 0.0238s | Model Forward Time: 4.8423s | Phase Times: max_pre_step_barrier: 0.1099 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.1713 | max_backward: 1.7200 | max_optimizer_step: 0.1898 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6188.3 | uss_MB: 5169.7 | child_rss_MB: 5642.5 | allocated_MB: 342.5 | reserved_MB: 26610.0 | inactive_split_MB: 264.5 +rank0 [2026-10-28 18:06:43,191] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 1% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 99% +6 10 0x7408, 62036 46.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:10:59,027] INFO : Step 9900/50000 rank0 | Train Loss: 1.8366 | Train Loss Unweighted: 1.0077 | Train Loss Components: Auxiliary: 0.3557 | Cardinality: 0.2091 | Classification: 0.1810 | Classification_binary: 0.2609 | HitGrouping: 0.2462 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2775 | Regression_eta: 0.0035 | Regression_pt: 0.2814 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1145 | Classification_binary: 0.2938 | Regression_cos_phi: 0.1713 | Regression_energy: 0.1464 | Regression_eta: 0.0925 | Regression_pt: 0.1523 | Regression_sin_phi: 0.1506 | LR: 9.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2917s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6346 | max_loss: 0.1692 | max_backward: 1.3542 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6188.4 | uss_MB: 5170.9 | child_rss_MB: 5645.5 | allocated_MB: 343.7 | reserved_MB: 26610.0 | inactive_split_MB: 205.7 +rank0 [2026-10-28 18:10:59,267] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 99% +6 10 0x7408, 62036 46.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:15:07,384] INFO : Step 10000/50000 rank0 | Train Loss: 2.0706 | Train Loss Unweighted: 1.1295 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2648 | Classification: 0.1819 | Classification_binary: 0.3225 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3077 | Regression_eta: 0.0037 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0952 | Classification_binary: 0.3313 | Regression_cos_phi: 0.1464 | Regression_energy: 0.1510 | Regression_eta: 0.1010 | Regression_pt: 0.1598 | Regression_sin_phi: 0.1394 | LR: 9.14e-04 | DataLoad Time: 0.0195s | Model Forward Time: 6.4575s | Phase Times: max_pre_step_barrier: 0.0136 | max_batch_to_device: 0.0708 | max_forward: 1.8842 | max_loss: 0.5484 | max_backward: 2.6620 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.5314 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6190.4 | uss_MB: 5174.2 | child_rss_MB: 5642.5 | allocated_MB: 345.2 | reserved_MB: 26610.0 | inactive_split_MB: 288.8 +rank0 [2026-10-28 18:15:07,625] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 28% 100% +6 10 0x7408, 62036 46.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:15:07,646] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 18:15:07,822] INFO : Running validation on rank0 +rank0 [2026-10-28 18:15:07,822] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 18:15:07,822] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 18:15:56,453] INFO : VALIDATION | Step=10000/50000 | Train Loss=2.0706 | Valid Loss=1.9469 | Train Loss Unweighted=1.1295 | Valid Loss Unweighted=1.0681 | Stale=0 | ETA=1690.4m +rank0 [2026-10-28 18:15:56,453] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:15:56,453] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 18:15:56,453] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 18:15:56,453] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:15:56,453] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 18:15:56,453] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 18:15:56,453] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 18:15:56,453] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/ari=0.408092 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.509479 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/top1_competing=0.571447 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.593263 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.14106 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0187882 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 18:15:56,454] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/seconds_per_rank=21.8031 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:15:56,455] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 18:15:56,456] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 18:15:56,456] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:15:56,456] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding/ari=0.251654 +rank0 [2026-10-28 18:15:56,456] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 18:15:56,456] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding_minus_raw/ari=0.160031 +rank0 [2026-10-28 18:15:56,456] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 18:15:56,456] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 18:15:56,456] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 18:15:56,462] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:15:56,715] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 18:15:56,732] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 18:16:22,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 18:16:23,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 18:16:25,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 18:16:26,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 18:16:36,548] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 18:16:36,560] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:16:37,046] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 18:16:37,049] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 18:16:52,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 18:16:53,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 18:16:55,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 18:16:57,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 18:17:07,065] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 18:17:09,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:17:10,081] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 18:17:10,086] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 18:17:25,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 18:17:26,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 18:17:27,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 18:17:28,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 18:17:38,093] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 18:17:38,273] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits +rank0 [2026-10-28 18:17:41,477] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits +rank0 [2026-10-28 18:17:42,492] INFO : Plotted number of elements +rank0 [2026-10-28 18:17:44,024] INFO : Plotted jets +rank0 [2026-10-28 18:17:45,191] INFO : Plotted jet ratio +rank0 [2026-10-28 18:17:45,241] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 18:17:48,485] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 18:17:49,497] INFO : Plotted number of elements +rank0 [2026-10-28 18:17:50,977] INFO : Plotted jets +rank0 [2026-10-28 18:17:52,476] INFO : Plotted jet ratio +rank0 [2026-10-28 18:17:52,525] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits +rank0 [2026-10-28 18:17:54,954] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits +rank0 [2026-10-28 18:17:55,815] INFO : Plotted number of elements +rank0 [2026-10-28 18:17:57,312] INFO : Plotted jets +rank0 [2026-10-28 18:17:58,477] INFO : Plotted jet ratio +rank0 [2026-10-28 18:22:02,242] INFO : Step 10100/50000 rank0 | Train Loss: 1.8115 | Train Loss Unweighted: 1.0221 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.1999 | Classification: 0.1589 | Classification_binary: 0.2815 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2868 | Regression_eta: 0.0032 | Regression_pt: 0.2886 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0838 | Classification_binary: 0.2348 | Regression_cos_phi: 0.1451 | Regression_energy: 0.1284 | Regression_eta: 0.1140 | Regression_pt: 0.1341 | Regression_sin_phi: 0.1289 | LR: 9.12e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6197s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7090 | max_loss: 0.1507 | max_backward: 1.2179 | max_optimizer_step: 0.3146 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 7166.8 | uss_MB: 6150.2 | child_rss_MB: 5645.8 | allocated_MB: 343.1 | reserved_MB: 21794.0 | inactive_split_MB: 243.8 +rank0 [2026-10-28 18:22:02,480] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 27% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:26:07,692] INFO : Step 10200/50000 rank0 | Train Loss: 1.6749 | Train Loss Unweighted: 0.9660 | Train Loss Components: Auxiliary: 0.3310 | Cardinality: 0.1773 | Classification: 0.1544 | Classification_binary: 0.2377 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2827 | Regression_eta: 0.0027 | Regression_pt: 0.2857 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1087 | Classification_binary: 0.2576 | Regression_cos_phi: 0.1059 | Regression_energy: 0.1535 | Regression_eta: 0.0694 | Regression_pt: 0.1620 | Regression_sin_phi: 0.1092 | LR: 9.11e-04 | DataLoad Time: 0.0113s | Model Forward Time: 4.8412s | Phase Times: max_pre_step_barrier: 0.0352 | max_batch_to_device: 0.0241 | max_forward: 0.6625 | max_loss: 0.1442 | max_backward: 1.3835 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7009 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1256 | Memory: rss_MB: 7164.8 | uss_MB: 6150.1 | child_rss_MB: 5645.0 | allocated_MB: 345.9 | reserved_MB: 21794.0 | inactive_split_MB: 292.4 +rank0 [2026-10-28 18:26:07,966] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 27% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:30:14,897] INFO : Step 10300/50000 rank0 | Train Loss: 1.9113 | Train Loss Unweighted: 1.0360 | Train Loss Components: Auxiliary: 0.3660 | Cardinality: 0.2108 | Classification: 0.1830 | Classification_binary: 0.2931 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0033 | Regression_pt: 0.2786 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1194 | Classification_binary: 0.3146 | Regression_cos_phi: 0.1664 | Regression_energy: 0.1295 | Regression_eta: 0.0927 | Regression_pt: 0.1361 | Regression_sin_phi: 0.1297 | LR: 9.09e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5986s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7205 | max_loss: 0.1496 | max_backward: 1.6083 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 6287.2 | uss_MB: 5273.5 | child_rss_MB: 5650.9 | allocated_MB: 343.8 | reserved_MB: 22764.0 | inactive_split_MB: 257.9 +rank0 [2026-10-28 18:30:15,144] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 27% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:34:20,631] INFO : Step 10400/50000 rank0 | Train Loss: 1.9303 | Train Loss Unweighted: 1.0569 | Train Loss Components: Auxiliary: 0.3739 | Cardinality: 0.2318 | Classification: 0.1704 | Classification_binary: 0.3342 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2718 | Regression_eta: 0.0033 | Regression_pt: 0.2738 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1023 | Classification_binary: 0.2947 | Regression_cos_phi: 0.1254 | Regression_energy: 0.1219 | Regression_eta: 0.0915 | Regression_pt: 0.1275 | Regression_sin_phi: 0.1141 | LR: 9.07e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4875s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7465 | max_loss: 0.1499 | max_backward: 1.4212 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.4712 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6287.2 | uss_MB: 5272.0 | child_rss_MB: 5660.4 | allocated_MB: 345.6 | reserved_MB: 22792.0 | inactive_split_MB: 259.8 +rank0 [2026-10-28 18:34:20,873] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 27% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:38:24,923] INFO : Step 10500/50000 rank0 | Train Loss: 1.6601 | Train Loss Unweighted: 0.9268 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1892 | Classification: 0.1410 | Classification_binary: 0.2342 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2693 | Regression_eta: 0.0029 | Regression_pt: 0.2766 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2229 | Regression_cos_phi: 0.1130 | Regression_energy: 0.1338 | Regression_eta: 0.0951 | Regression_pt: 0.1428 | Regression_sin_phi: 0.1195 | LR: 9.06e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.2392s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 0.6364 | max_loss: 0.2260 | max_backward: 1.1629 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6452 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6284.6 | uss_MB: 5269.3 | child_rss_MB: 5635.7 | allocated_MB: 340.4 | reserved_MB: 22792.0 | inactive_split_MB: 230.3 +rank0 [2026-10-28 18:38:25,163] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 27% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:42:23,281] INFO : Step 10600/50000 rank0 | Train Loss: 1.9388 | Train Loss Unweighted: 1.0719 | Train Loss Components: Auxiliary: 0.3737 | Cardinality: 0.2008 | Classification: 0.1863 | Classification_binary: 0.2709 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3047 | Regression_eta: 0.0042 | Regression_pt: 0.3019 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0991 | Classification_binary: 0.2791 | Regression_cos_phi: 0.1080 | Regression_energy: 0.1347 | Regression_eta: 0.0932 | Regression_pt: 0.1372 | Regression_sin_phi: 0.1233 | LR: 9.04e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6933s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6880 | max_loss: 0.2458 | max_backward: 1.4354 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7481 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 6285.6 | uss_MB: 5273.7 | child_rss_MB: 5653.3 | allocated_MB: 344.6 | reserved_MB: 22798.0 | inactive_split_MB: 286.4 +rank0 [2026-10-28 18:42:23,526] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 27% 10% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 95% +6 10 0x7408, 62036 46.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:46:29,552] INFO : Step 10700/50000 rank0 | Train Loss: 1.8955 | Train Loss Unweighted: 1.0230 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.2139 | Classification: 0.1752 | Classification_binary: 0.3186 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2598 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1026 | Classification_binary: 0.3293 | Regression_cos_phi: 0.1468 | Regression_energy: 0.1206 | Regression_eta: 0.0759 | Regression_pt: 0.1230 | Regression_sin_phi: 0.1294 | LR: 9.02e-04 | DataLoad Time: 0.0121s | Model Forward Time: 5.8444s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0072 | max_forward: 1.0363 | max_loss: 0.1757 | max_backward: 2.3255 | max_optimizer_step: 0.1834 | max_tensorboard_logging: 0.7232 | max_tensorboard_flush: 0.0051 | max_post_step_barrier: 0.1576 | Memory: rss_MB: 6287.7 | uss_MB: 5274.9 | child_rss_MB: 5649.7 | allocated_MB: 342.1 | reserved_MB: 22798.0 | inactive_split_MB: 282.9 +rank0 [2026-10-28 18:46:29,792] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 6% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 6% +4 8 0x7408, 13395 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 27% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 35% +6 10 0x7408, 62036 46.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 98% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:50:29,159] INFO : Step 10800/50000 rank0 | Train Loss: 1.7679 | Train Loss Unweighted: 0.9563 | Train Loss Components: Auxiliary: 0.3446 | Cardinality: 0.2074 | Classification: 0.1518 | Classification_binary: 0.2766 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2590 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.2339 | Regression_cos_phi: 0.1011 | Regression_energy: 0.1163 | Regression_eta: 0.0885 | Regression_pt: 0.1240 | Regression_sin_phi: 0.1280 | LR: 9.00e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.6131s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0995 | max_forward: 0.7884 | max_loss: 0.2225 | max_backward: 2.0144 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.5230 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6288.7 | uss_MB: 5271.1 | child_rss_MB: 5654.7 | allocated_MB: 343.6 | reserved_MB: 23784.0 | inactive_split_MB: 286.4 +rank0 [2026-10-28 18:50:29,397] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 2% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 27% 8% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:54:32,354] INFO : Step 10900/50000 rank0 | Train Loss: 1.6025 | Train Loss Unweighted: 0.8934 | Train Loss Components: Auxiliary: 0.3208 | Cardinality: 0.1729 | Classification: 0.1446 | Classification_binary: 0.2284 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2574 | Regression_eta: 0.0027 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2215 | Regression_cos_phi: 0.0798 | Regression_energy: 0.1236 | Regression_eta: 0.0948 | Regression_pt: 0.1276 | Regression_sin_phi: 0.1490 | LR: 8.99e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0614s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.5765 | max_loss: 0.1490 | max_backward: 1.1882 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6288.0 | uss_MB: 5275.5 | child_rss_MB: 5639.7 | allocated_MB: 342.1 | reserved_MB: 23796.0 | inactive_split_MB: 272.7 +rank0 [2026-10-28 18:54:32,593] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:58:46,117] INFO : Step 11000/50000 rank0 | Train Loss: 1.9315 | Train Loss Unweighted: 1.0466 | Train Loss Components: Auxiliary: 0.3721 | Cardinality: 0.2078 | Classification: 0.1806 | Classification_binary: 0.3087 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0037 | Regression_pt: 0.2758 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1135 | Classification_binary: 0.3271 | Regression_cos_phi: 0.1454 | Regression_energy: 0.1199 | Regression_eta: 0.0957 | Regression_pt: 0.1267 | Regression_sin_phi: 0.1131 | LR: 8.97e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.8686s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7424 | max_loss: 0.1704 | max_backward: 1.5866 | max_optimizer_step: 0.2906 | max_tensorboard_logging: 0.6287 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6277.0 | uss_MB: 5281.8 | child_rss_MB: 5663.8 | allocated_MB: 347.8 | reserved_MB: 23796.0 | inactive_split_MB: 262.8 +rank0 [2026-10-28 18:58:46,358] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:58:46,379] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 18:58:46,654] INFO : Running validation on rank0 +rank0 [2026-10-28 18:58:46,654] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 18:58:46,654] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 18:59:41,040] INFO : VALIDATION | Step=11000/50000 | Train Loss=1.9315 | Valid Loss=1.7688 | Train Loss Unweighted=1.0466 | Valid Loss Unweighted=0.9751 | Stale=0 | ETA=1653.4m +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 18:59:41,041] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/ari=0.39434 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/ari_threshold=0.75625 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.516616 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/top1_competing=0.582301 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.579318 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.154813 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.00484301 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 18:59:41,042] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/seconds_per_rank=21.8554 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding/ari=0.250083 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding_minus_raw/ari=0.15846 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 18:59:41,043] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 18:59:41,044] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 18:59:41,050] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:59:41,310] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 18:59:41,315] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 19:00:07,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 19:00:09,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 19:00:10,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 19:00:11,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 19:00:21,665] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 19:00:21,677] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:00:22,258] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 19:00:22,261] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 19:00:38,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 19:00:40,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 19:00:41,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 19:00:42,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 19:00:52,154] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 19:00:52,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:00:52,895] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 19:00:52,899] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 19:01:08,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 19:01:09,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 19:01:10,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 19:01:11,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 19:01:21,044] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 19:01:21,060] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits +rank0 [2026-10-28 19:01:24,302] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits +rank0 [2026-10-28 19:01:25,339] INFO : Plotted number of elements +rank0 [2026-10-28 19:01:26,897] INFO : Plotted jets +rank0 [2026-10-28 19:01:28,037] INFO : Plotted jet ratio +rank0 [2026-10-28 19:01:28,088] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 19:01:31,322] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 19:01:32,368] INFO : Plotted number of elements +rank0 [2026-10-28 19:01:33,882] INFO : Plotted jets +rank0 [2026-10-28 19:01:35,399] INFO : Plotted jet ratio +rank0 [2026-10-28 19:01:35,450] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits +rank0 [2026-10-28 19:01:37,791] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits +rank0 [2026-10-28 19:01:38,627] INFO : Plotted number of elements +rank0 [2026-10-28 19:01:40,086] INFO : Plotted jets +rank0 [2026-10-28 19:01:41,239] INFO : Plotted jet ratio +rank0 [2026-10-28 19:05:42,062] INFO : Step 11100/50000 rank0 | Train Loss: 1.9118 | Train Loss Unweighted: 1.0064 | Train Loss Components: Auxiliary: 0.3723 | Cardinality: 0.2793 | Classification: 0.1563 | Classification_binary: 0.3082 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2660 | Regression_eta: 0.0034 | Regression_pt: 0.2693 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.2812 | Regression_cos_phi: 0.1063 | Regression_energy: 0.1376 | Regression_eta: 0.0873 | Regression_pt: 0.1468 | Regression_sin_phi: 0.1009 | LR: 8.95e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7102s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.9217 | max_loss: 0.2304 | max_backward: 1.5570 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 7169.9 | uss_MB: 6179.3 | child_rss_MB: 5664.2 | allocated_MB: 343.2 | reserved_MB: 22146.0 | inactive_split_MB: 281.7 +rank0 [2026-10-28 19:05:42,305] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:09:39,964] INFO : Step 11200/50000 rank0 | Train Loss: 1.7366 | Train Loss Unweighted: 0.9397 | Train Loss Components: Auxiliary: 0.3529 | Cardinality: 0.2466 | Classification: 0.1384 | Classification_binary: 0.2548 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2689 | Regression_eta: 0.0027 | Regression_pt: 0.2722 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 0.2476 | Regression_cos_phi: 0.1011 | Regression_energy: 0.1123 | Regression_eta: 0.0691 | Regression_pt: 0.1169 | Regression_sin_phi: 0.1120 | LR: 8.93e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5407s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7793 | max_loss: 0.1825 | max_backward: 1.3743 | max_optimizer_step: 0.2128 | max_tensorboard_logging: 0.4676 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7167.9 | uss_MB: 6179.2 | child_rss_MB: 5641.7 | allocated_MB: 344.6 | reserved_MB: 22972.0 | inactive_split_MB: 273.4 +rank0 [2026-10-28 19:09:40,205] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:13:42,503] INFO : Step 11300/50000 rank0 | Train Loss: 1.6876 | Train Loss Unweighted: 0.9423 | Train Loss Components: Auxiliary: 0.3334 | Cardinality: 0.1862 | Classification: 0.1581 | Classification_binary: 0.2713 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2524 | Regression_eta: 0.0027 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0910 | Classification_binary: 0.2637 | Regression_cos_phi: 0.1139 | Regression_energy: 0.1202 | Regression_eta: 0.0710 | Regression_pt: 0.1249 | Regression_sin_phi: 0.1475 | LR: 8.91e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.3858s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0223 | max_forward: 0.9315 | max_loss: 0.2308 | max_backward: 1.9859 | max_optimizer_step: 0.1928 | max_tensorboard_logging: 0.5681 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6388.7 | uss_MB: 5273.7 | child_rss_MB: 5658.4 | allocated_MB: 345.5 | reserved_MB: 24368.0 | inactive_split_MB: 269.6 +rank0 [2026-10-28 19:13:42,744] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 8% +6 10 0x7408, 62036 46.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 93% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:17:46,105] INFO : Step 11400/50000 rank0 | Train Loss: 1.8563 | Train Loss Unweighted: 0.9980 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.2376 | Classification: 0.1627 | Classification_binary: 0.3164 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2551 | Regression_eta: 0.0029 | Regression_pt: 0.2577 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1288 | Classification_binary: 0.3421 | Regression_cos_phi: 0.1362 | Regression_energy: 0.1269 | Regression_eta: 0.0872 | Regression_pt: 0.1343 | Regression_sin_phi: 0.1514 | LR: 8.89e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.8600s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.7684 | max_loss: 0.1511 | max_backward: 1.6966 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5034 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 6391.7 | uss_MB: 5402.6 | child_rss_MB: 5664.5 | allocated_MB: 343.2 | reserved_MB: 25642.0 | inactive_split_MB: 251.6 +rank0 [2026-10-28 19:17:46,348] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:21:55,759] INFO : Step 11500/50000 rank0 | Train Loss: 1.6323 | Train Loss Unweighted: 0.8976 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1755 | Classification: 0.1375 | Classification_binary: 0.2717 | HitGrouping: 0.2346 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2408 | Regression_eta: 0.0028 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.2406 | Regression_cos_phi: 0.1006 | Regression_energy: 0.1209 | Regression_eta: 0.0708 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0973 | LR: 8.88e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4098s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7232 | max_loss: 0.1557 | max_backward: 1.4770 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6390.5 | uss_MB: 5401.3 | child_rss_MB: 5644.3 | allocated_MB: 345.3 | reserved_MB: 25642.0 | inactive_split_MB: 274.7 +rank0 [2026-10-28 19:21:56,000] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 46.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:25:55,346] INFO : Step 11600/50000 rank0 | Train Loss: 1.6133 | Train Loss Unweighted: 0.9062 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1718 | Classification: 0.1502 | Classification_binary: 0.2356 | HitGrouping: 0.2377 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2569 | Regression_eta: 0.0025 | Regression_pt: 0.2581 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1149 | Classification_binary: 0.2435 | Regression_cos_phi: 0.1364 | Regression_energy: 0.1249 | Regression_eta: 0.0706 | Regression_pt: 0.1302 | Regression_sin_phi: 0.1202 | LR: 8.86e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4013s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6646 | max_loss: 0.1710 | max_backward: 1.3208 | max_optimizer_step: 0.1963 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6391.5 | uss_MB: 5404.2 | child_rss_MB: 5665.3 | allocated_MB: 344.1 | reserved_MB: 26630.0 | inactive_split_MB: 209.9 +rank0 [2026-10-28 19:25:55,586] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 46.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:29:51,476] INFO : Step 11700/50000 rank0 | Train Loss: 1.7739 | Train Loss Unweighted: 0.9706 | Train Loss Components: Auxiliary: 0.3531 | Cardinality: 0.2418 | Classification: 0.1518 | Classification_binary: 0.2821 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2638 | Regression_eta: 0.0028 | Regression_pt: 0.2671 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0761 | Classification_binary: 0.3088 | Regression_cos_phi: 0.0993 | Regression_energy: 0.1069 | Regression_eta: 0.0628 | Regression_pt: 0.1134 | Regression_sin_phi: 0.1008 | LR: 8.84e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.2032s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8789 | max_loss: 0.2057 | max_backward: 2.3846 | max_optimizer_step: 0.1875 | max_tensorboard_logging: 1.0403 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6389.5 | uss_MB: 4921.9 | child_rss_MB: 5640.9 | allocated_MB: 343.8 | reserved_MB: 27624.0 | inactive_split_MB: 186.6 +rank0 [2026-10-28 19:29:51,723] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 49.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 84% +4 8 0x7408, 13395 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 19% +6 10 0x7408, 62036 46.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:33:54,810] INFO : Step 11800/50000 rank0 | Train Loss: 1.8223 | Train Loss Unweighted: 0.9745 | Train Loss Components: Auxiliary: 0.3609 | Cardinality: 0.2329 | Classification: 0.1589 | Classification_binary: 0.3100 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2473 | Regression_eta: 0.0028 | Regression_pt: 0.2523 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.2491 | Regression_cos_phi: 0.1283 | Regression_energy: 0.1182 | Regression_eta: 0.0645 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0891 | LR: 8.82e-04 | DataLoad Time: 0.0184s | Model Forward Time: 6.1821s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0070 | max_forward: 1.2302 | max_loss: 0.2767 | max_backward: 2.3745 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6390.5 | uss_MB: 5402.7 | child_rss_MB: 5661.5 | allocated_MB: 346.0 | reserved_MB: 27624.0 | inactive_split_MB: 280.5 +rank0 [2026-10-28 19:33:55,050] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 8% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 48.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 98% +4 8 0x7408, 13395 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 96% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 31% +6 10 0x7408, 62036 46.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:37:49,687] INFO : Step 11900/50000 rank0 | Train Loss: 1.5797 | Train Loss Unweighted: 0.8638 | Train Loss Components: Auxiliary: 0.3225 | Cardinality: 0.1935 | Classification: 0.1358 | Classification_binary: 0.2351 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2427 | Regression_eta: 0.0025 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0966 | Classification_binary: 0.2387 | Regression_cos_phi: 0.0954 | Regression_energy: 0.1198 | Regression_eta: 0.0844 | Regression_pt: 0.1239 | Regression_sin_phi: 0.1279 | LR: 8.80e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.8565s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0787 | max_forward: 0.7011 | max_loss: 0.1484 | max_backward: 1.2002 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.9463 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 6390.3 | uss_MB: 5402.5 | child_rss_MB: 5642.2 | allocated_MB: 340.1 | reserved_MB: 28620.0 | inactive_split_MB: 264.1 +rank0 [2026-10-28 19:37:49,929] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 1% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 46% 0% +6 10 0x7408, 62036 46.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:41:43,876] INFO : Step 12000/50000 rank0 | Train Loss: 1.6036 | Train Loss Unweighted: 0.8706 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1972 | Classification: 0.1481 | Classification_binary: 0.2459 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2330 | Regression_eta: 0.0025 | Regression_pt: 0.2385 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0952 | Classification_binary: 0.2805 | Regression_cos_phi: 0.1619 | Regression_energy: 0.1133 | Regression_eta: 0.1032 | Regression_pt: 0.1205 | Regression_sin_phi: 0.1272 | LR: 8.78e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.4056s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0072 | max_forward: 0.6194 | max_loss: 0.1546 | max_backward: 1.4628 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5143 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6383.3 | uss_MB: 5409.4 | child_rss_MB: 5646.2 | allocated_MB: 342.9 | reserved_MB: 28620.0 | inactive_split_MB: 215.8 +rank0 [2026-10-28 19:41:44,121] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 46.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:41:44,142] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 19:41:44,355] INFO : Running validation on rank0 +rank0 [2026-10-28 19:41:44,355] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 19:41:44,355] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 19:42:35,853] INFO : VALIDATION | Step=12000/50000 | Train Loss=1.6036 | Valid Loss=1.6480 | Train Loss Unweighted=0.8706 | Valid Loss Unweighted=0.8942 | Stale=0 | ETA=1612.6m +rank0 [2026-10-28 19:42:35,853] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/ari=0.449342 +rank0 [2026-10-28 19:42:35,854] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.49099 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/top1_competing=0.547909 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.59239 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0998104 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0179148 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 19:42:35,855] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/seconds_per_rank=21.5437 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding/ari=0.266388 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding_minus_raw/ari=0.174764 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 19:42:35,856] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 19:42:35,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:42:36,158] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 19:42:36,162] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 19:43:02,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 19:43:04,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 19:43:05,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 19:43:07,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 19:43:17,081] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-28 19:43:17,093] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:43:17,650] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 19:43:17,653] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 19:43:34,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 19:43:35,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 19:43:36,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 19:43:38,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 19:43:48,073] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 19:43:48,085] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:43:48,652] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 19:43:48,656] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 19:44:04,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 19:44:04,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 19:44:05,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 19:44:06,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 19:44:16,279] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 19:44:16,426] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits +rank0 [2026-10-28 19:44:19,797] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits +rank0 [2026-10-28 19:44:20,869] INFO : Plotted number of elements +rank0 [2026-10-28 19:44:22,457] INFO : Plotted jets +rank0 [2026-10-28 19:44:23,667] INFO : Plotted jet ratio +rank0 [2026-10-28 19:44:23,737] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 19:44:27,037] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 19:44:28,067] INFO : Plotted number of elements +rank0 [2026-10-28 19:44:29,551] INFO : Plotted jets +rank0 [2026-10-28 19:44:31,034] INFO : Plotted jet ratio +rank0 [2026-10-28 19:44:31,086] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits +rank0 [2026-10-28 19:44:33,475] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits +rank0 [2026-10-28 19:44:34,312] INFO : Plotted number of elements +rank0 [2026-10-28 19:44:35,784] INFO : Plotted jets +rank0 [2026-10-28 19:44:36,933] INFO : Plotted jet ratio +rank0 [2026-10-28 19:48:34,775] INFO : Step 12100/50000 rank0 | Train Loss: 1.8149 | Train Loss Unweighted: 0.9793 | Train Loss Components: Auxiliary: 0.3632 | Cardinality: 0.2324 | Classification: 0.1506 | Classification_binary: 0.3149 | HitGrouping: 0.2337 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2518 | Regression_eta: 0.0029 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0798 | Classification_binary: 0.3076 | Regression_cos_phi: 0.0923 | Regression_energy: 0.1294 | Regression_eta: 0.0788 | Regression_pt: 0.1377 | Regression_sin_phi: 0.1022 | LR: 8.76e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2180s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8305 | max_loss: 0.1678 | max_backward: 1.6498 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8610 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7131.3 | uss_MB: 6150.6 | child_rss_MB: 5646.8 | allocated_MB: 345.8 | reserved_MB: 23512.0 | inactive_split_MB: 299.6 +rank0 [2026-10-28 19:48:35,017] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:52:55,590] INFO : Step 12200/50000 rank0 | Train Loss: 1.6619 | Train Loss Unweighted: 0.9043 | Train Loss Components: Auxiliary: 0.3290 | Cardinality: 0.1906 | Classification: 0.1445 | Classification_binary: 0.2674 | HitGrouping: 0.2303 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2435 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2573 | Regression_cos_phi: 0.0926 | Regression_energy: 0.1304 | Regression_eta: 0.0490 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0948 | LR: 8.74e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.7519s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.7862 | max_loss: 0.1438 | max_backward: 1.5991 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.8843 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7133.3 | uss_MB: 5953.1 | child_rss_MB: 5653.1 | allocated_MB: 341.2 | reserved_MB: 23512.0 | inactive_split_MB: 206.8 +rank0 [2026-10-28 19:52:55,831] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 49.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 46.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 96% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:56:48,019] INFO : Step 12300/50000 rank0 | Train Loss: 1.4699 | Train Loss Unweighted: 0.8181 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1474 | Classification: 0.1404 | Classification_binary: 0.2280 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2222 | Regression_eta: 0.0028 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0734 | Classification_binary: 0.2150 | Regression_cos_phi: 0.1039 | Regression_energy: 0.1161 | Regression_eta: 0.0603 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0906 | LR: 8.72e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4041s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6177 | max_loss: 0.2232 | max_backward: 1.1179 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 0.7974 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6497.9 | uss_MB: 5527.4 | child_rss_MB: 5654.1 | allocated_MB: 343.2 | reserved_MB: 24286.0 | inactive_split_MB: 290.2 +rank0 [2026-10-28 19:56:48,259] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:00:36,922] INFO : Step 12400/50000 rank0 | Train Loss: 1.6135 | Train Loss Unweighted: 0.8782 | Train Loss Components: Auxiliary: 0.3265 | Cardinality: 0.2056 | Classification: 0.1546 | Classification_binary: 0.2540 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2308 | Regression_eta: 0.0025 | Regression_pt: 0.2337 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1043 | Classification_binary: 0.2374 | Regression_cos_phi: 0.1181 | Regression_energy: 0.1234 | Regression_eta: 0.0724 | Regression_pt: 0.1275 | Regression_sin_phi: 0.1160 | LR: 8.70e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.7245s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6928 | max_loss: 0.1486 | max_backward: 1.3860 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6497.9 | uss_MB: 5528.1 | child_rss_MB: 5671.7 | allocated_MB: 344.8 | reserved_MB: 24510.0 | inactive_split_MB: 295.5 +rank0 [2026-10-28 20:00:37,165] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 46.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:04:30,681] INFO : Step 12500/50000 rank0 | Train Loss: 1.6527 | Train Loss Unweighted: 0.8797 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1890 | Classification: 0.1391 | Classification_binary: 0.2779 | HitGrouping: 0.2315 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2273 | Regression_eta: 0.0030 | Regression_pt: 0.2295 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2405 | Regression_cos_phi: 0.0826 | Regression_energy: 0.1101 | Regression_eta: 0.0713 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0709 | LR: 8.68e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.7874s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0535 | max_forward: 0.7747 | max_loss: 0.1534 | max_backward: 1.3339 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6494.1 | uss_MB: 5528.8 | child_rss_MB: 5649.7 | allocated_MB: 347.2 | reserved_MB: 24510.0 | inactive_split_MB: 456.8 +rank0 [2026-10-28 20:04:30,923] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 4% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 97% +2 6 0x7408, 15961 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 52.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 46.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:08:31,202] INFO : Step 12600/50000 rank0 | Train Loss: 1.4969 | Train Loss Unweighted: 0.8185 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1595 | Classification: 0.1210 | Classification_binary: 0.2357 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2279 | Regression_eta: 0.0024 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2395 | Regression_cos_phi: 0.0874 | Regression_energy: 0.1017 | Regression_eta: 0.0571 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0682 | LR: 8.66e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4386s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0062 | max_forward: 0.6571 | max_loss: 0.1429 | max_backward: 1.4364 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6482.1 | uss_MB: 5531.3 | child_rss_MB: 5638.2 | allocated_MB: 342.9 | reserved_MB: 25526.0 | inactive_split_MB: 234.6 +rank0 [2026-10-28 20:08:31,455] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:12:35,224] INFO : Step 12700/50000 rank0 | Train Loss: 1.6234 | Train Loss Unweighted: 0.8661 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.2004 | Classification: 0.1459 | Classification_binary: 0.2387 | HitGrouping: 0.2403 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2360 | Regression_eta: 0.0027 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0905 | Classification_binary: 0.2495 | Regression_cos_phi: 0.0916 | Regression_energy: 0.1062 | Regression_eta: 0.0525 | Regression_pt: 0.1112 | Regression_sin_phi: 0.1014 | LR: 8.64e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.1192s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6221 | max_loss: 0.1846 | max_backward: 1.1554 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6469.4 | uss_MB: 5531.3 | child_rss_MB: 5648.7 | allocated_MB: 345.9 | reserved_MB: 26286.0 | inactive_split_MB: 241.7 +rank0 [2026-10-28 20:12:35,466] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 46.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:16:23,583] INFO : Step 12800/50000 rank0 | Train Loss: 1.7877 | Train Loss Unweighted: 0.9398 | Train Loss Components: Auxiliary: 0.3651 | Cardinality: 0.2327 | Classification: 0.1495 | Classification_binary: 0.3094 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2357 | Regression_eta: 0.0027 | Regression_pt: 0.2394 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.3132 | Regression_cos_phi: 0.1329 | Regression_energy: 0.1169 | Regression_eta: 0.0674 | Regression_pt: 0.1234 | Regression_sin_phi: 0.1022 | LR: 8.62e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8979s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.8468 | max_loss: 0.1534 | max_backward: 1.7420 | max_optimizer_step: 0.3013 | max_tensorboard_logging: 0.5725 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6458.4 | uss_MB: 5535.9 | child_rss_MB: 5651.5 | allocated_MB: 345.6 | reserved_MB: 26286.0 | inactive_split_MB: 284.4 +rank0 [2026-10-28 20:16:23,824] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 46.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:20:14,512] INFO : Step 12900/50000 rank0 | Train Loss: 1.5881 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.3191 | Cardinality: 0.2029 | Classification: 0.1304 | Classification_binary: 0.2564 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2291 | Regression_eta: 0.0023 | Regression_pt: 0.2312 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2205 | Regression_cos_phi: 0.0838 | Regression_energy: 0.1172 | Regression_eta: 0.0539 | Regression_pt: 0.1213 | Regression_sin_phi: 0.1166 | LR: 8.60e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6849s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7180 | max_loss: 0.2380 | max_backward: 1.4488 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6932 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6444.6 | uss_MB: 5531.5 | child_rss_MB: 5646.1 | allocated_MB: 341.8 | reserved_MB: 26286.0 | inactive_split_MB: 252.2 +rank0 [2026-10-28 20:20:14,753] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 49.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 46.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:24:07,204] INFO : Step 13000/50000 rank0 | Train Loss: 1.5217 | Train Loss Unweighted: 0.8547 | Train Loss Components: Auxiliary: 0.3141 | Cardinality: 0.1693 | Classification: 0.1382 | Classification_binary: 0.2135 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2470 | Regression_eta: 0.0024 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0876 | Regression_energy: 0.1278 | Regression_eta: 0.0705 | Regression_pt: 0.1336 | Regression_sin_phi: 0.1232 | LR: 8.58e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4826s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7101 | max_loss: 0.1763 | max_backward: 1.1845 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 0.8282 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6429.6 | uss_MB: 5536.9 | child_rss_MB: 5653.9 | allocated_MB: 339.8 | reserved_MB: 26286.0 | inactive_split_MB: 213.4 +rank0 [2026-10-28 20:24:07,448] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 46.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:24:07,468] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 20:24:07,656] INFO : Running validation on rank0 +rank0 [2026-10-28 20:24:07,656] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 20:24:07,656] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 20:24:59,120] INFO : VALIDATION | Step=13000/50000 | Train Loss=1.5217 | Valid Loss=1.5631 | Train Loss Unweighted=0.8547 | Valid Loss Unweighted=0.8449 | Stale=0 | ETA=1570.0m +rank0 [2026-10-28 20:24:59,120] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 20:24:59,120] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 20:24:59,120] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 20:24:59,120] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/ari=0.457138 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.520114 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/top1_competing=0.584166 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.604351 +rank0 [2026-10-28 20:24:59,121] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0920149 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0298758 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/seconds_per_rank=21.2678 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 20:24:59,122] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 20:24:59,123] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 20:24:59,123] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 20:24:59,123] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 20:24:59,123] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 20:24:59,123] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding/ari=0.271038 +rank0 [2026-10-28 20:24:59,123] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 20:24:59,123] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding_minus_raw/ari=0.179415 +rank0 [2026-10-28 20:24:59,123] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 20:24:59,123] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 20:24:59,123] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 20:24:59,128] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:24:59,394] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 20:24:59,406] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 20:25:24,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 20:25:25,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 20:25:26,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 20:25:28,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 20:25:38,101] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 20:25:38,113] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:25:38,651] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 20:25:38,654] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 20:25:55,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 20:25:56,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 20:25:57,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 20:25:58,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 20:26:08,380] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 20:26:08,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:26:09,007] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 20:26:09,011] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 20:26:24,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 20:26:25,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 20:26:26,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 20:26:27,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 20:26:37,066] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 20:26:37,082] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits +rank0 [2026-10-28 20:26:40,377] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits +rank0 [2026-10-28 20:26:41,411] INFO : Plotted number of elements +rank0 [2026-10-28 20:26:42,948] INFO : Plotted jets +rank0 [2026-10-28 20:26:44,065] INFO : Plotted jet ratio +rank0 [2026-10-28 20:26:44,115] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 20:26:47,396] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 20:26:48,419] INFO : Plotted number of elements +rank0 [2026-10-28 20:26:49,909] INFO : Plotted jets +rank0 [2026-10-28 20:26:51,426] INFO : Plotted jet ratio +rank0 [2026-10-28 20:26:51,480] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits +rank0 [2026-10-28 20:26:53,860] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits +rank0 [2026-10-28 20:26:54,708] INFO : Plotted number of elements +rank0 [2026-10-28 20:26:56,186] INFO : Plotted jets +rank0 [2026-10-28 20:26:57,323] INFO : Plotted jet ratio +rank0 [2026-10-28 20:30:48,019] INFO : Step 13100/50000 rank0 | Train Loss: 1.6168 | Train Loss Unweighted: 0.8649 | Train Loss Components: Auxiliary: 0.3213 | Cardinality: 0.1869 | Classification: 0.1467 | Classification_binary: 0.2696 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2210 | Regression_eta: 0.0027 | Regression_pt: 0.2221 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0826 | Classification_binary: 0.3189 | Regression_cos_phi: 0.1073 | Regression_energy: 0.1007 | Regression_eta: 0.0894 | Regression_pt: 0.1068 | Regression_sin_phi: 0.1492 | LR: 8.56e-04 | DataLoad Time: 0.0090s | Model Forward Time: 7.0785s | Phase Times: max_pre_step_barrier: 0.0337 | max_batch_to_device: 0.0136 | max_forward: 1.2842 | max_loss: 0.2583 | max_backward: 3.1281 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.9677 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7302.9 | uss_MB: 6406.9 | child_rss_MB: 5646.9 | allocated_MB: 341.8 | reserved_MB: 22940.0 | inactive_split_MB: 270.2 +rank0 [2026-10-28 20:30:48,261] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +4 8 0x7408, 13395 51.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 45.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 25% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:34:57,050] INFO : Step 13200/50000 rank0 | Train Loss: 1.6996 | Train Loss Unweighted: 0.8855 | Train Loss Components: Auxiliary: 0.3386 | Cardinality: 0.2474 | Classification: 0.1410 | Classification_binary: 0.2769 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2299 | Regression_eta: 0.0027 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1122 | Classification_binary: 0.2866 | Regression_cos_phi: 0.1319 | Regression_energy: 0.1138 | Regression_eta: 0.0960 | Regression_pt: 0.1194 | Regression_sin_phi: 0.1562 | LR: 8.54e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.9103s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0072 | max_forward: 0.7543 | max_loss: 0.1532 | max_backward: 1.3317 | max_optimizer_step: 0.3180 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7300.0 | uss_MB: 6407.2 | child_rss_MB: 5649.5 | allocated_MB: 343.4 | reserved_MB: 23942.0 | inactive_split_MB: 249.6 +rank0 [2026-10-28 20:34:57,293] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 25% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:38:54,082] INFO : Step 13300/50000 rank0 | Train Loss: 1.4876 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2970 | Cardinality: 0.2018 | Classification: 0.1100 | Classification_binary: 0.2192 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2205 | Regression_eta: 0.0026 | Regression_pt: 0.2230 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1976 | Regression_cos_phi: 0.1133 | Regression_energy: 0.1061 | Regression_eta: 0.1348 | Regression_pt: 0.1115 | Regression_sin_phi: 0.1318 | LR: 8.52e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.3461s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0259 | max_forward: 0.9208 | max_loss: 0.2245 | max_backward: 1.9523 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6473.0 | uss_MB: 4937.1 | child_rss_MB: 5535.0 | allocated_MB: 341.7 | reserved_MB: 24474.0 | inactive_split_MB: 265.4 +rank0 [2026-10-28 20:38:54,322] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 7% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 97% +2 6 0x7408, 15961 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 0% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 45.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 25% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:42:53,629] INFO : Step 13400/50000 rank0 | Train Loss: 1.4253 | Train Loss Unweighted: 0.7944 | Train Loss Components: Auxiliary: 0.2889 | Cardinality: 0.1585 | Classification: 0.1406 | Classification_binary: 0.2158 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2158 | Regression_eta: 0.0022 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2214 | Regression_cos_phi: 0.0758 | Regression_energy: 0.1083 | Regression_eta: 0.0614 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0839 | LR: 8.50e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4225s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6533 | max_loss: 0.2500 | max_backward: 1.4458 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.5454 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6474.0 | uss_MB: 5610.2 | child_rss_MB: 5538.3 | allocated_MB: 341.6 | reserved_MB: 25334.0 | inactive_split_MB: 266.6 +rank0 [2026-10-28 20:42:53,868] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 46.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 25% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:47:06,389] INFO : Step 13500/50000 rank0 | Train Loss: 1.6021 | Train Loss Unweighted: 0.8749 | Train Loss Components: Auxiliary: 0.3241 | Cardinality: 0.1832 | Classification: 0.1473 | Classification_binary: 0.2801 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2200 | Regression_eta: 0.0024 | Regression_pt: 0.2225 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2671 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0922 | Regression_eta: 0.0626 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0955 | LR: 8.48e-04 | DataLoad Time: 0.0157s | Model Forward Time: 6.8413s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.7859 | max_loss: 0.3303 | max_backward: 2.5111 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4974 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6412.7 | uss_MB: 5395.8 | child_rss_MB: 5343.5 | allocated_MB: 344.3 | reserved_MB: 25574.0 | inactive_split_MB: 284.1 +rank0 [2026-10-28 20:47:06,631] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 1% +2 6 0x7408, 15961 48.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 88% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 97% +4 8 0x7408, 13395 51.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 79% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 46% +6 10 0x7408, 62036 46.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 25% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:51:04,925] INFO : Step 13600/50000 rank0 | Train Loss: 1.5296 | Train Loss Unweighted: 0.8114 | Train Loss Components: Auxiliary: 0.3050 | Cardinality: 0.1872 | Classification: 0.1291 | Classification_binary: 0.2450 | HitGrouping: 0.2218 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2158 | Regression_eta: 0.0025 | Regression_pt: 0.2165 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0940 | Classification_binary: 0.2387 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0915 | Regression_eta: 0.0722 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0913 | LR: 8.45e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.3768s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.7508 | max_loss: 0.1507 | max_backward: 1.2891 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6404.7 | uss_MB: 5608.6 | child_rss_MB: 5283.3 | allocated_MB: 345.0 | reserved_MB: 26608.0 | inactive_split_MB: 257.3 +rank0 [2026-10-28 20:51:05,164] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 46.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 25% 45% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:55:12,926] INFO : Step 13700/50000 rank0 | Train Loss: 1.4515 | Train Loss Unweighted: 0.7895 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1575 | Classification: 0.1261 | Classification_binary: 0.2121 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2219 | Regression_eta: 0.0023 | Regression_pt: 0.2245 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.2137 | Regression_cos_phi: 0.0915 | Regression_energy: 0.0937 | Regression_eta: 0.0464 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0801 | LR: 8.43e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.4420s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.6804 | max_loss: 0.2128 | max_backward: 1.1331 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6398.1 | uss_MB: 5609.6 | child_rss_MB: 5245.6 | allocated_MB: 342.5 | reserved_MB: 26608.0 | inactive_split_MB: 241.7 +rank0 [2026-10-28 20:55:13,168] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 46.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:59:19,785] INFO : Step 13800/50000 rank0 | Train Loss: 1.5060 | Train Loss Unweighted: 0.8320 | Train Loss Components: Auxiliary: 0.3084 | Cardinality: 0.1762 | Classification: 0.1368 | Classification_binary: 0.2567 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2156 | Regression_eta: 0.0023 | Regression_pt: 0.2184 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2647 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0935 | Regression_eta: 0.0643 | Regression_pt: 0.1002 | Regression_sin_phi: 0.1123 | LR: 8.41e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0946s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7967 | max_loss: 0.1491 | max_backward: 1.5976 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6392.1 | uss_MB: 5613.3 | child_rss_MB: 5216.1 | allocated_MB: 342.1 | reserved_MB: 26608.0 | inactive_split_MB: 241.3 +rank0 [2026-10-28 20:59:20,028] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 41% 3% +2 6 0x7408, 15961 49.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 3% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 31% +6 10 0x7408, 62036 46.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:03:22,556] INFO : Step 13900/50000 rank0 | Train Loss: 1.6108 | Train Loss Unweighted: 0.8469 | Train Loss Components: Auxiliary: 0.3204 | Cardinality: 0.1942 | Classification: 0.1303 | Classification_binary: 0.2751 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2168 | Regression_eta: 0.0028 | Regression_pt: 0.2192 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.2151 | Regression_cos_phi: 0.1164 | Regression_energy: 0.0959 | Regression_eta: 0.0585 | Regression_pt: 0.0999 | Regression_sin_phi: 0.1163 | LR: 8.39e-04 | DataLoad Time: 0.0200s | Model Forward Time: 7.5128s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8010 | max_loss: 0.1552 | max_backward: 2.5526 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 1.2862 | max_tensorboard_flush: 0.0041 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 6389.1 | uss_MB: 5603.5 | child_rss_MB: 5217.8 | allocated_MB: 345.6 | reserved_MB: 26608.0 | inactive_split_MB: 292.8 +rank0 [2026-10-28 21:03:22,797] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 42% 0% +2 6 0x7408, 15961 48.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 41% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 92% +4 8 0x7408, 13395 51.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 45.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 97% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:07:30,105] INFO : Step 14000/50000 rank0 | Train Loss: 1.3659 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1707 | Classification: 0.0974 | Classification_binary: 0.1997 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2149 | Regression_eta: 0.0023 | Regression_pt: 0.2181 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.2012 | Regression_cos_phi: 0.0789 | Regression_energy: 0.1162 | Regression_eta: 0.0415 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0689 | LR: 8.37e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.1767s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7574 | max_loss: 0.1908 | max_backward: 1.5870 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6393.2 | uss_MB: 4632.1 | child_rss_MB: 5217.4 | allocated_MB: 342.3 | reserved_MB: 26608.0 | inactive_split_MB: 238.4 +rank0 [2026-10-28 21:07:30,349] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 2% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 46.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 32% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:07:30,369] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 21:07:30,550] INFO : Running validation on rank0 +rank0 [2026-10-28 21:07:30,550] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 21:07:30,550] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 21:08:21,824] INFO : VALIDATION | Step=14000/50000 | Train Loss=1.3659 | Valid Loss=1.5004 | Train Loss Unweighted=0.7342 | Valid Loss Unweighted=0.7884 | Stale=0 | ETA=1530.0m +rank0 [2026-10-28 21:08:21,824] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:08:21,824] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 21:08:21,824] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 21:08:21,824] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:08:21,824] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 21:08:21,824] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 21:08:21,824] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/ari=0.43881 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.529795 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/top1_competing=0.595706 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.600684 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.110343 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0262087 +rank0 [2026-10-28 21:08:21,825] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/seconds_per_rank=21.4958 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 21:08:21,826] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:08:21,827] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 21:08:21,827] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 21:08:21,827] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:08:21,827] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding/ari=0.271324 +rank0 [2026-10-28 21:08:21,827] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 21:08:21,827] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding_minus_raw/ari=0.179701 +rank0 [2026-10-28 21:08:21,827] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 21:08:21,827] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 21:08:21,827] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 21:08:21,832] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:08:22,085] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 21:08:22,090] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 21:08:50,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 21:08:53,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 21:08:54,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 21:08:55,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 21:09:05,921] INFO : Time taken to make predictions on device 0 is: 0.32 min +rank0 [2026-10-28 21:09:05,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:09:06,443] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 21:09:06,447] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 21:09:22,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 21:09:23,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 21:09:24,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 21:09:26,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 21:09:36,289] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 21:09:36,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:09:37,169] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 21:09:37,173] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 21:09:52,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 21:09:53,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 21:09:54,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 21:09:55,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 21:10:05,087] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 21:10:05,104] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits +rank0 [2026-10-28 21:10:08,426] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits +rank0 [2026-10-28 21:10:09,626] INFO : Plotted number of elements +rank0 [2026-10-28 21:10:11,319] INFO : Plotted jets +rank0 [2026-10-28 21:10:12,513] INFO : Plotted jet ratio +rank0 [2026-10-28 21:10:12,604] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 21:10:15,896] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 21:10:16,928] INFO : Plotted number of elements +rank0 [2026-10-28 21:10:18,424] INFO : Plotted jets +rank0 [2026-10-28 21:10:19,532] INFO : Plotted jet ratio +rank0 [2026-10-28 21:10:19,584] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits +rank0 [2026-10-28 21:10:22,069] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits +rank0 [2026-10-28 21:10:23,372] INFO : Plotted number of elements +rank0 [2026-10-28 21:10:24,840] INFO : Plotted jets +rank0 [2026-10-28 21:10:25,984] INFO : Plotted jet ratio +rank0 [2026-10-28 21:14:17,570] INFO : Step 14100/50000 rank0 | Train Loss: 1.5416 | Train Loss Unweighted: 0.8176 | Train Loss Components: Auxiliary: 0.3012 | Cardinality: 0.1751 | Classification: 0.1363 | Classification_binary: 0.2297 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2177 | Regression_eta: 0.0028 | Regression_pt: 0.2283 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1017 | Classification_binary: 0.2427 | Regression_cos_phi: 0.1927 | Regression_energy: 0.1089 | Regression_eta: 0.0942 | Regression_pt: 0.1155 | Regression_sin_phi: 0.1368 | LR: 8.35e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.0237s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7718 | max_loss: 0.1710 | max_backward: 1.6376 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 7412.7 | uss_MB: 6170.5 | child_rss_MB: 5197.6 | allocated_MB: 347.1 | reserved_MB: 23970.0 | inactive_split_MB: 244.9 +rank0 [2026-10-28 21:14:17,827] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 23% 7% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 46.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:18:18,803] INFO : Step 14200/50000 rank0 | Train Loss: 1.6164 | Train Loss Unweighted: 0.8542 | Train Loss Components: Auxiliary: 0.3190 | Cardinality: 0.1950 | Classification: 0.1478 | Classification_binary: 0.2794 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2094 | Regression_eta: 0.0027 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0905 | Classification_binary: 0.2893 | Regression_cos_phi: 0.1419 | Regression_energy: 0.1026 | Regression_eta: 0.0703 | Regression_pt: 0.1062 | Regression_sin_phi: 0.1017 | LR: 8.32e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.2232s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1153 | max_forward: 0.8788 | max_loss: 0.1615 | max_backward: 1.5286 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7413.4 | uss_MB: 6634.1 | child_rss_MB: 5195.0 | allocated_MB: 344.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.9 +rank0 [2026-10-28 21:18:19,057] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 23% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 31% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 46% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 46.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:22:22,400] INFO : Step 14300/50000 rank0 | Train Loss: 1.4246 | Train Loss Unweighted: 0.7744 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1685 | Classification: 0.1198 | Classification_binary: 0.2357 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2084 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2037 | Regression_cos_phi: 0.0715 | Regression_energy: 0.1159 | Regression_eta: 0.0463 | Regression_pt: 0.1201 | Regression_sin_phi: 0.0805 | LR: 8.30e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.6048s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7183 | max_loss: 0.1494 | max_backward: 1.3371 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6454.4 | uss_MB: 5677.1 | child_rss_MB: 5184.2 | allocated_MB: 341.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.1 +rank0 [2026-10-28 21:22:22,645] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 23% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:26:25,946] INFO : Step 14400/50000 rank0 | Train Loss: 1.3539 | Train Loss Unweighted: 0.7267 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1402 | Classification: 0.1303 | Classification_binary: 0.2015 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1957 | Regression_eta: 0.0022 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0918 | Classification_binary: 0.1997 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0952 | Regression_eta: 0.0475 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0791 | LR: 8.28e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2771s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6350 | max_loss: 0.2071 | max_backward: 1.2597 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6458.7 | uss_MB: 5681.9 | child_rss_MB: 5188.8 | allocated_MB: 346.1 | reserved_MB: 24962.0 | inactive_split_MB: 231.6 +rank0 [2026-10-28 21:26:26,187] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:30:24,450] INFO : Step 14500/50000 rank0 | Train Loss: 1.5651 | Train Loss Unweighted: 0.7967 | Train Loss Components: Auxiliary: 0.3070 | Cardinality: 0.2312 | Classification: 0.1364 | Classification_binary: 0.2565 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1984 | Regression_eta: 0.0026 | Regression_pt: 0.2005 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0929 | Classification_binary: 0.2836 | Regression_cos_phi: 0.1472 | Regression_energy: 0.0945 | Regression_eta: 0.0890 | Regression_pt: 0.0989 | Regression_sin_phi: 0.1337 | LR: 8.26e-04 | DataLoad Time: 0.0183s | Model Forward Time: 6.1011s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0069 | max_forward: 0.8745 | max_loss: 0.1717 | max_backward: 1.8485 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 1.4922 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6443.5 | uss_MB: 5682.8 | child_rss_MB: 5113.1 | allocated_MB: 344.2 | reserved_MB: 25374.0 | inactive_split_MB: 206.9 +rank0 [2026-10-28 21:30:24,706] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:34:35,599] INFO : Step 14600/50000 rank0 | Train Loss: 1.6485 | Train Loss Unweighted: 0.8413 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.2375 | Classification: 0.1249 | Classification_binary: 0.2902 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2104 | Regression_eta: 0.0023 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0892 | Classification_binary: 0.2860 | Regression_cos_phi: 0.1285 | Regression_energy: 0.1098 | Regression_eta: 0.0807 | Regression_pt: 0.1141 | Regression_sin_phi: 0.1092 | LR: 8.24e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.0175s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0068 | max_forward: 0.7866 | max_loss: 0.1515 | max_backward: 1.5469 | max_optimizer_step: 0.3128 | max_tensorboard_logging: 0.7882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6427.1 | uss_MB: 5683.8 | child_rss_MB: 5049.1 | allocated_MB: 345.7 | reserved_MB: 25374.0 | inactive_split_MB: 292.3 +rank0 [2026-10-28 21:34:35,842] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 10% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 15% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 11% +6 10 0x7408, 62036 46.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 97% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:38:40,429] INFO : Step 14700/50000 rank0 | Train Loss: 1.3645 | Train Loss Unweighted: 0.7326 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1507 | Classification: 0.1080 | Classification_binary: 0.2034 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2130 | Regression_cos_phi: 0.0984 | Regression_energy: 0.1061 | Regression_eta: 0.0696 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0816 | LR: 8.21e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5910s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6571 | max_loss: 0.1494 | max_backward: 1.2978 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7751 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0922 | Memory: rss_MB: 6423.6 | uss_MB: 5684.9 | child_rss_MB: 5040.9 | allocated_MB: 341.1 | reserved_MB: 25374.0 | inactive_split_MB: 264.9 +rank0 [2026-10-28 21:38:40,683] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:42:54,565] INFO : Step 14800/50000 rank0 | Train Loss: 1.3987 | Train Loss Unweighted: 0.7335 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1827 | Classification: 0.1136 | Classification_binary: 0.2244 | HitGrouping: 0.2204 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1949 | Regression_eta: 0.0022 | Regression_pt: 0.1963 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2489 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0884 | Regression_eta: 0.0601 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0875 | LR: 8.19e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.9286s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.1679 | max_loss: 0.2171 | max_backward: 2.8051 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 6420.4 | uss_MB: 4434.9 | child_rss_MB: 5026.4 | allocated_MB: 345.2 | reserved_MB: 25374.0 | inactive_split_MB: 222.5 +rank0 [2026-10-28 21:42:54,819] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 65% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 45.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:47:06,780] INFO : Step 14900/50000 rank0 | Train Loss: 1.5046 | Train Loss Unweighted: 0.7991 | Train Loss Components: Auxiliary: 0.3055 | Cardinality: 0.1834 | Classification: 0.1311 | Classification_binary: 0.2643 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1982 | Regression_eta: 0.0023 | Regression_pt: 0.2008 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0833 | Classification_binary: 0.2810 | Regression_cos_phi: 0.0993 | Regression_energy: 0.0871 | Regression_eta: 0.0630 | Regression_pt: 0.0920 | Regression_sin_phi: 0.1155 | LR: 8.17e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.7916s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0071 | max_forward: 0.7642 | max_loss: 0.1561 | max_backward: 1.5975 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6421.6 | uss_MB: 5685.9 | child_rss_MB: 5030.2 | allocated_MB: 345.4 | reserved_MB: 25374.0 | inactive_split_MB: 289.7 +rank0 [2026-10-28 21:47:07,021] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 46.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:51:17,324] INFO : Step 15000/50000 rank0 | Train Loss: 1.3836 | Train Loss Unweighted: 0.7341 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1614 | Classification: 0.1182 | Classification_binary: 0.2252 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1930 | Regression_eta: 0.0023 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1940 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0893 | Regression_eta: 0.0575 | Regression_pt: 0.0920 | Regression_sin_phi: 0.0795 | LR: 8.15e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.4119s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0066 | max_forward: 0.6828 | max_loss: 0.1453 | max_backward: 1.3064 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1064 | Memory: rss_MB: 6423.8 | uss_MB: 5686.3 | child_rss_MB: 5021.1 | allocated_MB: 342.6 | reserved_MB: 25374.0 | inactive_split_MB: 262.7 +rank0 [2026-10-28 21:51:17,567] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 45.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:51:17,590] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 21:51:17,767] INFO : Running validation on rank0 +rank0 [2026-10-28 21:51:17,767] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 21:51:17,767] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 21:52:07,049] INFO : VALIDATION | Step=15000/50000 | Train Loss=1.3836 | Valid Loss=1.4323 | Train Loss Unweighted=0.7341 | Valid Loss Unweighted=0.7579 | Stale=0 | ETA=1490.5m +rank0 [2026-10-28 21:52:07,049] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:52:07,049] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 21:52:07,049] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 21:52:07,049] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:52:07,049] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 21:52:07,049] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 21:52:07,049] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 21:52:07,049] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 21:52:07,049] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/ari=0.486819 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.534224 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/top1_competing=0.601582 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.622678 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0623339 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0482033 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 21:52:07,050] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/seconds_per_rank=21.5774 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 21:52:07,051] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:52:07,052] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding/ari=0.284474 +rank0 [2026-10-28 21:52:07,052] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 21:52:07,052] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding_minus_raw/ari=0.192851 +rank0 [2026-10-28 21:52:07,052] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 21:52:07,052] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 21:52:07,052] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 21:52:07,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:52:07,316] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 21:52:07,320] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 21:52:33,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 21:52:34,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 21:52:35,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 21:52:36,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 21:52:46,732] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 21:52:46,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:52:47,325] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 21:52:47,328] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 21:53:03,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 21:53:04,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 21:53:05,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 21:53:06,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 21:53:16,802] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 21:53:16,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:53:17,339] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 21:53:17,351] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 21:53:33,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 21:53:33,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 21:53:34,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 21:53:35,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 21:53:45,171] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 21:53:45,209] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits +rank0 [2026-10-28 21:53:48,782] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits +rank0 [2026-10-28 21:53:49,794] INFO : Plotted number of elements +rank0 [2026-10-28 21:53:51,339] INFO : Plotted jets +rank0 [2026-10-28 21:53:52,460] INFO : Plotted jet ratio +rank0 [2026-10-28 21:53:52,510] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 21:53:55,777] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 21:53:56,799] INFO : Plotted number of elements +rank0 [2026-10-28 21:53:58,296] INFO : Plotted jets +rank0 [2026-10-28 21:53:59,424] INFO : Plotted jet ratio +rank0 [2026-10-28 21:53:59,478] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits +rank0 [2026-10-28 21:54:01,926] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits +rank0 [2026-10-28 21:54:03,220] INFO : Plotted number of elements +rank0 [2026-10-28 21:54:04,780] INFO : Plotted jets +rank0 [2026-10-28 21:54:05,924] INFO : Plotted jet ratio +rank0 [2026-10-28 21:57:59,186] INFO : Step 15100/50000 rank0 | Train Loss: 1.3706 | Train Loss Unweighted: 0.7159 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1535 | Classification: 0.1271 | Classification_binary: 0.1954 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1940 | Regression_eta: 0.0022 | Regression_pt: 0.1948 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.1977 | Regression_cos_phi: 0.1417 | Regression_energy: 0.1009 | Regression_eta: 0.1026 | Regression_pt: 0.1061 | Regression_sin_phi: 0.1696 | LR: 8.12e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.6837s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.6658 | max_loss: 0.2401 | max_backward: 1.3105 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8277 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7654.5 | uss_MB: 6910.5 | child_rss_MB: 5047.2 | allocated_MB: 341.1 | reserved_MB: 22096.0 | inactive_split_MB: 215.5 +rank0 [2026-10-28 21:57:59,425] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:01:59,757] INFO : Step 15200/50000 rank0 | Train Loss: 1.5024 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3072 | Cardinality: 0.2080 | Classification: 0.1273 | Classification_binary: 0.2385 | HitGrouping: 0.2164 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1989 | Regression_eta: 0.0022 | Regression_pt: 0.2007 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0847 | Classification_binary: 0.2668 | Regression_cos_phi: 0.1077 | Regression_energy: 0.1006 | Regression_eta: 0.0842 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0836 | LR: 8.10e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.1704s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7502 | max_loss: 0.1534 | max_backward: 1.8002 | max_optimizer_step: 0.2270 | max_tensorboard_logging: 0.8682 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 7649.1 | uss_MB: 6910.3 | child_rss_MB: 5045.3 | allocated_MB: 345.6 | reserved_MB: 22420.0 | inactive_split_MB: 257.7 +rank0 [2026-10-28 22:02:00,000] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 3% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 45.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:05:42,903] INFO : Step 15300/50000 rank0 | Train Loss: 1.4920 | Train Loss Unweighted: 0.7880 | Train Loss Components: Auxiliary: 0.3065 | Cardinality: 0.1970 | Classification: 0.1161 | Classification_binary: 0.2589 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2026 | Regression_eta: 0.0022 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2498 | Regression_cos_phi: 0.1092 | Regression_energy: 0.1054 | Regression_eta: 0.0577 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0869 | LR: 8.08e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2477s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0192 | max_forward: 0.7642 | max_loss: 0.1530 | max_backward: 1.6292 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 0.9125 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6524.0 | uss_MB: 5797.3 | child_rss_MB: 5044.3 | allocated_MB: 341.2 | reserved_MB: 23096.0 | inactive_split_MB: 271.7 +rank0 [2026-10-28 22:05:43,147] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 1% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 87% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 2% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:09:28,809] INFO : Step 15400/50000 rank0 | Train Loss: 1.3549 | Train Loss Unweighted: 0.7210 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1572 | Classification: 0.1148 | Classification_binary: 0.2052 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1983 | Regression_eta: 0.0022 | Regression_pt: 0.1984 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0960 | Regression_eta: 0.0723 | Regression_pt: 0.0982 | Regression_sin_phi: 0.1013 | LR: 8.05e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.3610s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6594 | max_loss: 0.2185 | max_backward: 1.2416 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6511.0 | uss_MB: 5790.6 | child_rss_MB: 5047.2 | allocated_MB: 340.8 | reserved_MB: 23394.0 | inactive_split_MB: 166.6 +rank0 [2026-10-28 22:09:29,048] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:13:13,231] INFO : Step 15500/50000 rank0 | Train Loss: 1.4313 | Train Loss Unweighted: 0.7527 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1666 | Classification: 0.1321 | Classification_binary: 0.2182 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1979 | Regression_eta: 0.0025 | Regression_pt: 0.1996 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0800 | Classification_binary: 0.2891 | Regression_cos_phi: 0.0981 | Regression_energy: 0.1018 | Regression_eta: 0.0662 | Regression_pt: 0.1077 | Regression_sin_phi: 0.1261 | LR: 8.03e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9954s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0792 | max_forward: 0.7874 | max_loss: 0.1514 | max_backward: 1.8676 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5380 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6511.0 | uss_MB: 5805.3 | child_rss_MB: 5052.2 | allocated_MB: 345.6 | reserved_MB: 24092.0 | inactive_split_MB: 260.5 +rank0 [2026-10-28 22:13:13,470] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:17:02,001] INFO : Step 15600/50000 rank0 | Train Loss: 1.5305 | Train Loss Unweighted: 0.8109 | Train Loss Components: Auxiliary: 0.3137 | Cardinality: 0.1790 | Classification: 0.1304 | Classification_binary: 0.2721 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2011 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2516 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0953 | Regression_eta: 0.0545 | Regression_pt: 0.0983 | Regression_sin_phi: 0.0617 | LR: 8.01e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9615s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7645 | max_loss: 0.2230 | max_backward: 1.7015 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1398 | Memory: rss_MB: 6508.8 | uss_MB: 5803.4 | child_rss_MB: 5052.3 | allocated_MB: 344.1 | reserved_MB: 24402.0 | inactive_split_MB: 287.9 +rank0 [2026-10-28 22:17:02,240] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:20:47,293] INFO : Step 15700/50000 rank0 | Train Loss: 1.4083 | Train Loss Unweighted: 0.7205 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1775 | Classification: 0.1238 | Classification_binary: 0.2322 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1785 | Regression_eta: 0.0024 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1859 | Regression_cos_phi: 0.0876 | Regression_energy: 0.0845 | Regression_eta: 0.0669 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0734 | LR: 7.98e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.7764s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2392 | max_backward: 1.6606 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6506.8 | uss_MB: 5802.2 | child_rss_MB: 5055.5 | allocated_MB: 344.4 | reserved_MB: 25380.0 | inactive_split_MB: 228.2 +rank0 [2026-10-28 22:20:47,534] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:24:39,750] INFO : Step 15800/50000 rank0 | Train Loss: 1.2988 | Train Loss Unweighted: 0.6968 | Train Loss Components: Auxiliary: 0.2671 | Cardinality: 0.1436 | Classification: 0.1112 | Classification_binary: 0.1916 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1933 | Regression_eta: 0.0020 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.1878 | Regression_cos_phi: 0.0960 | Regression_energy: 0.1397 | Regression_eta: 0.0589 | Regression_pt: 0.1489 | Regression_sin_phi: 0.1442 | LR: 7.96e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4313s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6109 | max_loss: 0.2015 | max_backward: 1.1291 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6506.5 | uss_MB: 5803.6 | child_rss_MB: 5051.3 | allocated_MB: 341.6 | reserved_MB: 25380.0 | inactive_split_MB: 263.9 +rank0 [2026-10-28 22:24:39,993] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:28:35,770] INFO : Step 15900/50000 rank0 | Train Loss: 1.4020 | Train Loss Unweighted: 0.7322 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1800 | Classification: 0.1295 | Classification_binary: 0.2346 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1816 | Regression_eta: 0.0023 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2351 | Regression_cos_phi: 0.0935 | Regression_energy: 0.0728 | Regression_eta: 0.0607 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0816 | LR: 7.94e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9723s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0933 | max_forward: 0.7823 | max_loss: 0.1445 | max_backward: 1.4740 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6508.5 | uss_MB: 5807.0 | child_rss_MB: 5047.2 | allocated_MB: 342.9 | reserved_MB: 25380.0 | inactive_split_MB: 283.5 +rank0 [2026-10-28 22:28:36,011] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:32:41,364] INFO : Step 16000/50000 rank0 | Train Loss: 1.4978 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3126 | Cardinality: 0.2093 | Classification: 0.1190 | Classification_binary: 0.2580 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1920 | Regression_eta: 0.0021 | Regression_pt: 0.1969 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2465 | Regression_cos_phi: 0.1030 | Regression_energy: 0.0798 | Regression_eta: 0.0746 | Regression_pt: 0.0903 | Regression_sin_phi: 0.1313 | LR: 7.91e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1703s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.7474 | max_loss: 0.1892 | max_backward: 1.4114 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.7981 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6509.3 | uss_MB: 5675.5 | child_rss_MB: 5048.3 | allocated_MB: 342.7 | reserved_MB: 25380.0 | inactive_split_MB: 292.9 +rank0 [2026-10-28 22:32:41,608] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 3% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 97% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 86% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 45.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:32:41,723] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-16000.pth +rank0 [2026-10-28 22:32:42,004] INFO : Running validation on rank0 +rank0 [2026-10-28 22:32:42,005] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 22:32:42,005] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 22:33:35,141] INFO : VALIDATION | Step=16000/50000 | Train Loss=1.4978 | Valid Loss=1.3949 | Train Loss Unweighted=0.7701 | Valid Loss Unweighted=0.7314 | Stale=0 | ETA=1445.5m +rank0 [2026-10-28 22:33:35,141] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 22:33:35,141] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 22:33:35,141] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 22:33:35,141] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 22:33:35,141] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 22:33:35,141] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 22:33:35,141] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/ari=0.465895 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.57022 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/top1_competing=0.648746 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.61074 +rank0 [2026-10-28 22:33:35,142] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0832577 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0362656 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 22:33:35,143] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/seconds_per_rank=21.2599 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding/ari=0.290525 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding_minus_raw/ari=0.198902 +rank0 [2026-10-28 22:33:35,144] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 22:33:35,145] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 22:33:35,145] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 22:33:35,150] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:33:35,407] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 22:33:35,412] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 22:34:01,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 22:34:02,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 22:34:03,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 22:34:04,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 22:34:14,433] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 22:34:14,445] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:34:14,938] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 22:34:14,941] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 22:34:30,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 22:34:32,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 22:34:33,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 22:34:34,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 22:34:44,316] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 22:34:44,327] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:34:44,855] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 22:34:44,858] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 22:35:00,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 22:35:00,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 22:35:01,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 22:35:02,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 22:35:12,417] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 22:35:12,539] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits +rank0 [2026-10-28 22:35:16,040] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits +rank0 [2026-10-28 22:35:17,064] INFO : Plotted number of elements +rank0 [2026-10-28 22:35:18,639] INFO : Plotted jets +rank0 [2026-10-28 22:35:19,784] INFO : Plotted jet ratio +rank0 [2026-10-28 22:35:19,834] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 22:35:23,092] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 22:35:24,094] INFO : Plotted number of elements +rank0 [2026-10-28 22:35:25,596] INFO : Plotted jets +rank0 [2026-10-28 22:35:26,728] INFO : Plotted jet ratio +rank0 [2026-10-28 22:35:26,778] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits +rank0 [2026-10-28 22:35:29,247] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits +rank0 [2026-10-28 22:35:30,080] INFO : Plotted number of elements +rank0 [2026-10-28 22:35:32,000] INFO : Plotted jets +rank0 [2026-10-28 22:35:33,120] INFO : Plotted jet ratio +rank0 [2026-10-28 22:39:22,798] INFO : Step 16100/50000 rank0 | Train Loss: 1.3441 | Train Loss Unweighted: 0.7192 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1631 | Classification: 0.1197 | Classification_binary: 0.2058 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1931 | Regression_eta: 0.0020 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0753 | Regression_eta: 0.0578 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0755 | LR: 7.89e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8136s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0584 | max_forward: 0.7257 | max_loss: 0.1421 | max_backward: 1.2590 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.0197 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6577.1 | uss_MB: 5871.7 | child_rss_MB: 5053.6 | allocated_MB: 341.3 | reserved_MB: 23342.0 | inactive_split_MB: 221.6 +rank0 [2026-10-28 22:39:23,038] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:43:09,892] INFO : Step 16200/50000 rank0 | Train Loss: 1.3737 | Train Loss Unweighted: 0.7229 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1695 | Classification: 0.1213 | Classification_binary: 0.2167 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0834 | Classification_binary: 0.2380 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0855 | Regression_eta: 0.0649 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0978 | LR: 7.86e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6808s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.1062 | max_forward: 0.7366 | max_loss: 0.1469 | max_backward: 1.4999 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.6015 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6580.7 | uss_MB: 5882.6 | child_rss_MB: 5053.8 | allocated_MB: 342.5 | reserved_MB: 24304.0 | inactive_split_MB: 205.9 +rank0 [2026-10-28 22:43:10,134] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:46:55,304] INFO : Step 16300/50000 rank0 | Train Loss: 1.4671 | Train Loss Unweighted: 0.7592 | Train Loss Components: Auxiliary: 0.3008 | Cardinality: 0.1775 | Classification: 0.1342 | Classification_binary: 0.2702 | HitGrouping: 0.2148 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0023 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0829 | Classification_binary: 0.2677 | Regression_cos_phi: 0.1120 | Regression_energy: 0.0809 | Regression_eta: 0.0680 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0719 | LR: 7.84e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.0999s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7718 | max_loss: 0.1628 | max_backward: 1.7017 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.8513 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6574.7 | uss_MB: 5880.8 | child_rss_MB: 5053.1 | allocated_MB: 345.8 | reserved_MB: 24304.0 | inactive_split_MB: 281.0 +rank0 [2026-10-28 22:46:55,546] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:50:40,306] INFO : Step 16400/50000 rank0 | Train Loss: 1.4200 | Train Loss Unweighted: 0.7161 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.2086 | Classification: 0.1206 | Classification_binary: 0.2158 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0024 | Regression_pt: 0.1885 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1868 | Regression_cos_phi: 0.0976 | Regression_energy: 0.0902 | Regression_eta: 0.0604 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0661 | LR: 7.81e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5603s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7570 | max_loss: 0.1546 | max_backward: 1.2568 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5199 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1436 | Memory: rss_MB: 6571.7 | uss_MB: 5879.8 | child_rss_MB: 5049.7 | allocated_MB: 342.7 | reserved_MB: 25268.0 | inactive_split_MB: 213.6 +rank0 [2026-10-28 22:50:40,544] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +4 8 0x7408, 13395 52.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 20% +6 10 0x7408, 62036 46.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:54:25,787] INFO : Step 16500/50000 rank0 | Train Loss: 1.2201 | Train Loss Unweighted: 0.6592 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1195 | Classification: 0.1029 | Classification_binary: 0.1835 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1836 | Regression_eta: 0.0021 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0827 | Classification_binary: 0.2654 | Regression_cos_phi: 0.1010 | Regression_energy: 0.0910 | Regression_eta: 0.0816 | Regression_pt: 0.0954 | Regression_sin_phi: 0.1323 | LR: 7.79e-04 | DataLoad Time: 0.0120s | Model Forward Time: 5.0431s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0128 | max_forward: 0.7866 | max_loss: 0.2200 | max_backward: 1.9590 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.4913 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6572.2 | uss_MB: 5885.7 | child_rss_MB: 5046.8 | allocated_MB: 343.6 | reserved_MB: 26238.0 | inactive_split_MB: 248.8 +rank0 [2026-10-28 22:54:26,026] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:58:26,003] INFO : Step 16600/50000 rank0 | Train Loss: 1.4335 | Train Loss Unweighted: 0.7657 | Train Loss Components: Auxiliary: 0.2949 | Cardinality: 0.1698 | Classification: 0.1386 | Classification_binary: 0.2312 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1941 | Regression_eta: 0.0021 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2398 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0806 | Regression_eta: 0.0481 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0919 | LR: 7.77e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.7136s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0139 | max_forward: 1.0329 | max_loss: 0.3279 | max_backward: 2.5489 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.5244 | max_tensorboard_flush: 0.0014 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6575.3 | uss_MB: 4925.2 | child_rss_MB: 5045.0 | allocated_MB: 344.2 | reserved_MB: 27228.0 | inactive_split_MB: 219.6 +rank0 [2026-10-28 22:58:26,245] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 48.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:02:39,537] INFO : Step 16700/50000 rank0 | Train Loss: 1.4062 | Train Loss Unweighted: 0.7378 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1840 | Classification: 0.1203 | Classification_binary: 0.2439 | HitGrouping: 0.2108 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1839 | Regression_eta: 0.0020 | Regression_pt: 0.1855 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.2147 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0847 | Regression_eta: 0.0510 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0959 | LR: 7.74e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8206s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7336 | max_loss: 0.2460 | max_backward: 1.6747 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6577.4 | uss_MB: 5888.0 | child_rss_MB: 5049.7 | allocated_MB: 344.6 | reserved_MB: 27228.0 | inactive_split_MB: 279.4 +rank0 [2026-10-28 23:02:39,782] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 45.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:06:47,690] INFO : Step 16800/50000 rank0 | Train Loss: 1.3172 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1533 | Classification: 0.1173 | Classification_binary: 0.1949 | HitGrouping: 0.2175 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1819 | Regression_eta: 0.0027 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0929 | Classification_binary: 0.1997 | Regression_cos_phi: 0.1185 | Regression_energy: 0.0827 | Regression_eta: 0.1019 | Regression_pt: 0.0866 | Regression_sin_phi: 0.1100 | LR: 7.72e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.3234s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0962 | max_forward: 0.7772 | max_loss: 0.1423 | max_backward: 1.3121 | max_optimizer_step: 0.2117 | max_tensorboard_logging: 0.4865 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6580.1 | uss_MB: 5886.7 | child_rss_MB: 5043.1 | allocated_MB: 342.1 | reserved_MB: 27250.0 | inactive_split_MB: 200.8 +rank0 [2026-10-28 23:06:47,929] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 45.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:11:05,846] INFO : Step 16900/50000 rank0 | Train Loss: 1.3132 | Train Loss Unweighted: 0.6868 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1474 | Classification: 0.1260 | Classification_binary: 0.1988 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1779 | Regression_eta: 0.0021 | Regression_pt: 0.1798 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.2145 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0863 | Regression_eta: 0.0605 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0993 | LR: 7.69e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.4898s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7203 | max_loss: 0.1660 | max_backward: 1.1458 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6581.3 | uss_MB: 5889.6 | child_rss_MB: 5044.4 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 193.9 +rank0 [2026-10-28 23:11:06,088] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 45.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:15:09,799] INFO : Step 17000/50000 rank0 | Train Loss: 1.3575 | Train Loss Unweighted: 0.7115 | Train Loss Components: Auxiliary: 0.2906 | Cardinality: 0.1781 | Classification: 0.1179 | Classification_binary: 0.2325 | HitGrouping: 0.2089 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0018 | Regression_pt: 0.1792 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.2479 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0817 | Regression_eta: 0.0708 | Regression_pt: 0.0870 | Regression_sin_phi: 0.0950 | LR: 7.67e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0117s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.9008 | max_loss: 0.1474 | max_backward: 1.9229 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6575.6 | uss_MB: 5894.0 | child_rss_MB: 5025.6 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 278.9 +rank0 [2026-10-28 23:15:10,039] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:15:10,060] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-17000.pth +rank0 [2026-10-28 23:15:10,340] INFO : Running validation on rank0 +rank0 [2026-10-28 23:15:10,340] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 23:15:10,340] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 23:16:01,006] INFO : VALIDATION | Step=17000/50000 | Train Loss=1.3575 | Valid Loss=1.3345 | Train Loss Unweighted=0.7115 | Valid Loss Unweighted=0.6928 | Stale=0 | ETA=1402.8m +rank0 [2026-10-28 23:16:01,006] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 23:16:01,006] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 23:16:01,006] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 23:16:01,006] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 23:16:01,006] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 23:16:01,006] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 23:16:01,006] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 23:16:01,006] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/ari=0.478312 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.565871 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/top1_competing=0.641317 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.605727 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0708403 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0312525 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 23:16:01,007] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/seconds_per_rank=21.1853 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 23:16:01,008] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 23:16:01,009] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 23:16:01,009] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 23:16:01,009] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding/ari=0.272206 +rank0 [2026-10-28 23:16:01,009] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-28 23:16:01,009] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding_minus_raw/ari=0.180583 +rank0 [2026-10-28 23:16:01,009] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 23:16:01,009] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 23:16:01,009] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 23:16:01,014] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:16:01,267] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 23:16:01,271] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 23:16:27,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 23:16:28,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 23:16:30,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 23:16:31,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 23:16:41,998] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-28 23:16:42,017] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:16:42,521] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 23:16:42,525] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 23:16:58,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 23:16:59,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 23:17:00,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 23:17:01,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 23:17:11,512] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 23:17:11,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:17:12,023] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 23:17:12,026] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 23:17:28,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 23:17:28,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 23:17:29,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 23:17:30,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 23:17:40,434] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-28 23:17:40,451] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits +rank0 [2026-10-28 23:17:43,680] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits +rank0 [2026-10-28 23:17:44,700] INFO : Plotted number of elements +rank0 [2026-10-28 23:17:46,599] INFO : Plotted jets +rank0 [2026-10-28 23:17:47,753] INFO : Plotted jet ratio +rank0 [2026-10-28 23:17:47,835] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 23:17:51,102] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 23:17:52,117] INFO : Plotted number of elements +rank0 [2026-10-28 23:17:53,614] INFO : Plotted jets +rank0 [2026-10-28 23:17:54,753] INFO : Plotted jet ratio +rank0 [2026-10-28 23:17:54,815] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits +rank0 [2026-10-28 23:17:57,265] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits +rank0 [2026-10-28 23:17:58,115] INFO : Plotted number of elements +rank0 [2026-10-28 23:18:00,013] INFO : Plotted jets +rank0 [2026-10-28 23:18:01,119] INFO : Plotted jet ratio +rank0 [2026-10-28 23:22:00,208] INFO : Step 17100/50000 rank0 | Train Loss: 1.3790 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1704 | Classification: 0.1078 | Classification_binary: 0.2095 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1968 | Regression_eta: 0.0020 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.2017 | Regression_cos_phi: 0.0904 | Regression_energy: 0.1100 | Regression_eta: 0.0652 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0981 | LR: 7.64e-04 | DataLoad Time: 0.0081s | Model Forward Time: 5.6189s | Phase Times: max_pre_step_barrier: 0.0234 | max_batch_to_device: 0.0065 | max_forward: 0.8292 | max_loss: 0.1455 | max_backward: 1.4833 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6666.8 | uss_MB: 5307.9 | child_rss_MB: 5009.6 | allocated_MB: 343.8 | reserved_MB: 22688.0 | inactive_split_MB: 254.6 +rank0 [2026-10-28 23:22:00,447] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 99% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:26:07,693] INFO : Step 17200/50000 rank0 | Train Loss: 1.2786 | Train Loss Unweighted: 0.6598 | Train Loss Components: Auxiliary: 0.2576 | Cardinality: 0.1378 | Classification: 0.1161 | Classification_binary: 0.1910 | HitGrouping: 0.2191 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0025 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.2022 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0822 | Regression_eta: 0.0818 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0839 | LR: 7.62e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6931s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6470 | max_loss: 0.2505 | max_backward: 1.4380 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6320 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6664.1 | uss_MB: 5982.7 | child_rss_MB: 5028.0 | allocated_MB: 344.1 | reserved_MB: 22688.0 | inactive_split_MB: 295.9 +rank0 [2026-10-28 23:26:07,936] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +4 8 0x7408, 13395 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:30:06,470] INFO : Step 17300/50000 rank0 | Train Loss: 1.3571 | Train Loss Unweighted: 0.6976 | Train Loss Components: Auxiliary: 0.2789 | Cardinality: 0.1665 | Classification: 0.1193 | Classification_binary: 0.2287 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1724 | Regression_eta: 0.0021 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0769 | Classification_binary: 0.2665 | Regression_cos_phi: 0.0722 | Regression_energy: 0.1076 | Regression_eta: 0.0587 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0788 | LR: 7.59e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.3110s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7276 | max_loss: 0.2520 | max_backward: 1.8579 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8525 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6666.1 | uss_MB: 5121.5 | child_rss_MB: 5026.9 | allocated_MB: 344.2 | reserved_MB: 23616.0 | inactive_split_MB: 266.5 +rank0 [2026-10-28 23:30:06,716] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 99% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 45.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:34:26,128] INFO : Step 17400/50000 rank0 | Train Loss: 1.3668 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2811 | Cardinality: 0.1760 | Classification: 0.1195 | Classification_binary: 0.2351 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1732 | Regression_eta: 0.0019 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2279 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0812 | Regression_eta: 0.0588 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0781 | LR: 7.57e-04 | DataLoad Time: 0.0127s | Model Forward Time: 5.2029s | Phase Times: max_pre_step_barrier: 0.1038 | max_batch_to_device: 0.0306 | max_forward: 0.7782 | max_loss: 0.2429 | max_backward: 1.3743 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.8348 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0757 | Memory: rss_MB: 6667.1 | uss_MB: 5984.6 | child_rss_MB: 5026.4 | allocated_MB: 343.3 | reserved_MB: 24604.0 | inactive_split_MB: 262.4 +rank0 [2026-10-28 23:34:26,367] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 3% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 2% +2 6 0x7408, 15961 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:38:52,137] INFO : Step 17500/50000 rank0 | Train Loss: 1.3114 | Train Loss Unweighted: 0.6526 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.1481 | Classification: 0.1145 | Classification_binary: 0.1884 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1714 | Regression_eta: 0.0024 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1802 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0778 | Regression_eta: 0.0507 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0686 | LR: 7.54e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.6503s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.6394 | max_loss: 0.1427 | max_backward: 1.2079 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.8375 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 6667.1 | uss_MB: 5986.2 | child_rss_MB: 5023.0 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 240.0 +rank0 [2026-10-28 23:38:52,379] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:43:03,830] INFO : Step 17600/50000 rank0 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2688 | Cardinality: 0.1362 | Classification: 0.1273 | Classification_binary: 0.2065 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0019 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2252 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0718 | Regression_eta: 0.0459 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0823 | LR: 7.52e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.2630s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.1120 | max_forward: 1.2883 | max_loss: 0.2624 | max_backward: 1.5380 | max_optimizer_step: 0.2147 | max_tensorboard_logging: 0.7644 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 6665.9 | uss_MB: 5985.5 | child_rss_MB: 5028.5 | allocated_MB: 344.2 | reserved_MB: 25596.0 | inactive_split_MB: 237.9 +rank0 [2026-10-28 23:43:04,086] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:47:02,895] INFO : Step 17700/50000 rank0 | Train Loss: 1.3491 | Train Loss Unweighted: 0.6939 | Train Loss Components: Auxiliary: 0.2835 | Cardinality: 0.1622 | Classification: 0.1186 | Classification_binary: 0.2310 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0022 | Regression_pt: 0.1701 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2012 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0984 | Regression_eta: 0.0481 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0641 | LR: 7.49e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.4017s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0704 | max_forward: 0.9841 | max_loss: 0.1994 | max_backward: 1.8911 | max_optimizer_step: 0.2227 | max_tensorboard_logging: 0.5849 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6667.9 | uss_MB: 5989.0 | child_rss_MB: 5031.2 | allocated_MB: 345.5 | reserved_MB: 25596.0 | inactive_split_MB: 287.6 +rank0 [2026-10-28 23:47:03,138] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 4% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:51:18,159] INFO : Step 17800/50000 rank0 | Train Loss: 1.3063 | Train Loss Unweighted: 0.6692 | Train Loss Components: Auxiliary: 0.2664 | Cardinality: 0.1579 | Classification: 0.1042 | Classification_binary: 0.2130 | HitGrouping: 0.2150 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1741 | Regression_eta: 0.0019 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2005 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0846 | Regression_eta: 0.0811 | Regression_pt: 0.0871 | Regression_sin_phi: 0.1191 | LR: 7.47e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4963s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.7351 | max_loss: 0.1686 | max_backward: 1.4616 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5023 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 6666.0 | uss_MB: 5988.8 | child_rss_MB: 5032.9 | allocated_MB: 341.0 | reserved_MB: 25596.0 | inactive_split_MB: 197.8 +rank0 [2026-10-28 23:51:18,402] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 52.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 46.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:55:23,423] INFO : Step 17900/50000 rank0 | Train Loss: 1.2213 | Train Loss Unweighted: 0.6433 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1427 | Classification: 0.1147 | Classification_binary: 0.1725 | HitGrouping: 0.2070 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0843 | Classification_binary: 0.1914 | Regression_cos_phi: 0.1191 | Regression_energy: 0.0968 | Regression_eta: 0.0487 | Regression_pt: 0.1018 | Regression_sin_phi: 0.1156 | LR: 7.44e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4487s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7249 | max_loss: 0.1579 | max_backward: 1.1981 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6654.3 | uss_MB: 5990.9 | child_rss_MB: 5033.3 | allocated_MB: 341.6 | reserved_MB: 25596.0 | inactive_split_MB: 297.9 +rank0 [2026-10-28 23:55:23,689] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 45.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:59:20,971] INFO : Step 18000/50000 rank0 | Train Loss: 1.3597 | Train Loss Unweighted: 0.7083 | Train Loss Components: Auxiliary: 0.2781 | Cardinality: 0.1599 | Classification: 0.1247 | Classification_binary: 0.2275 | HitGrouping: 0.2122 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0021 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0800 | Classification_binary: 0.2558 | Regression_cos_phi: 0.1285 | Regression_energy: 0.0827 | Regression_eta: 0.0738 | Regression_pt: 0.0883 | Regression_sin_phi: 0.1021 | LR: 7.42e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.3308s | Phase Times: max_pre_step_barrier: 0.0816 | max_batch_to_device: 0.0068 | max_forward: 0.8980 | max_loss: 0.1586 | max_backward: 1.9185 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.7546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6654.3 | uss_MB: 5995.0 | child_rss_MB: 5035.3 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 254.8 +rank0 [2026-10-28 23:59:21,232] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 45.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:59:21,259] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-18000.pth +rank0 [2026-10-28 23:59:21,435] INFO : Running validation on rank0 +rank0 [2026-10-28 23:59:21,435] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 23:59:21,435] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 00:00:12,733] INFO : VALIDATION | Step=18000/50000 | Train Loss=1.3597 | Valid Loss=1.3002 | Train Loss Unweighted=0.7083 | Valid Loss Unweighted=0.6709 | Stale=0 | ETA=1363.3m +rank0 [2026-10-29 00:00:12,733] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:00:12,733] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 00:00:12,733] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/ari=0.509436 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.542975 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/top1_competing=0.614676 +rank0 [2026-10-29 00:00:12,734] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.614991 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0397169 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0405166 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 00:00:12,735] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/seconds_per_rank=21.0886 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding/ari=0.286524 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding_minus_raw/ari=0.194901 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 00:00:12,736] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 00:00:12,743] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:00:12,990] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 00:00:13,002] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 00:00:39,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 00:00:40,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 00:00:41,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 00:00:42,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 00:00:52,790] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 00:00:52,802] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:00:53,358] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 00:00:53,362] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 00:01:09,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 00:01:10,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 00:01:11,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 00:01:12,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 00:01:22,733] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 00:01:22,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:01:23,290] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 00:01:23,294] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 00:01:39,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 00:01:39,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 00:01:40,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 00:01:41,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 00:01:51,478] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 00:01:51,495] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits +rank0 [2026-10-29 00:01:54,706] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits +rank0 [2026-10-29 00:01:55,768] INFO : Plotted number of elements +rank0 [2026-10-29 00:01:57,621] INFO : Plotted jets +rank0 [2026-10-29 00:01:58,734] INFO : Plotted jet ratio +rank0 [2026-10-29 00:01:58,786] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 00:02:02,166] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 00:02:03,266] INFO : Plotted number of elements +rank0 [2026-10-29 00:02:04,748] INFO : Plotted jets +rank0 [2026-10-29 00:02:05,863] INFO : Plotted jet ratio +rank0 [2026-10-29 00:02:05,912] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits +rank0 [2026-10-29 00:02:08,381] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits +rank0 [2026-10-29 00:02:09,214] INFO : Plotted number of elements +rank0 [2026-10-29 00:02:11,100] INFO : Plotted jets +rank0 [2026-10-29 00:02:12,196] INFO : Plotted jet ratio +rank0 [2026-10-29 00:05:58,076] INFO : Step 18100/50000 rank0 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7216 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1666 | Classification: 0.1199 | Classification_binary: 0.2340 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1807 | Regression_eta: 0.0024 | Regression_pt: 0.1824 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0901 | Classification_binary: 0.2217 | Regression_cos_phi: 0.1170 | Regression_energy: 0.0909 | Regression_eta: 0.0617 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0862 | LR: 7.39e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.3478s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.9127 | max_loss: 0.4546 | max_backward: 1.7406 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.7463 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7789.9 | uss_MB: 7130.3 | child_rss_MB: 5031.5 | allocated_MB: 344.6 | reserved_MB: 22256.0 | inactive_split_MB: 291.4 +rank0 [2026-10-29 00:05:58,315] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:09:48,146] INFO : Step 18200/50000 rank0 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6468 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1359 | Classification: 0.1208 | Classification_binary: 0.1803 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1700 | Regression_eta: 0.0020 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0776 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0850 | Regression_eta: 0.0492 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0995 | LR: 7.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3772s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7114 | max_loss: 0.2369 | max_backward: 1.1051 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6784.1 | uss_MB: 6119.0 | child_rss_MB: 5031.9 | allocated_MB: 341.2 | reserved_MB: 23772.0 | inactive_split_MB: 170.7 +rank0 [2026-10-29 00:09:48,403] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:13:33,477] INFO : Step 18300/50000 rank0 | Train Loss: 1.3100 | Train Loss Unweighted: 0.6891 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1522 | Classification: 0.1230 | Classification_binary: 0.1938 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1834 | Regression_eta: 0.0019 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0802 | Classification_binary: 0.2326 | Regression_cos_phi: 0.1127 | Regression_energy: 0.1009 | Regression_eta: 0.0627 | Regression_pt: 0.1054 | Regression_sin_phi: 0.1183 | LR: 7.34e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8850s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7219 | max_loss: 0.1491 | max_backward: 1.5825 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.7390 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 6786.1 | uss_MB: 6124.1 | child_rss_MB: 5031.6 | allocated_MB: 344.4 | reserved_MB: 24778.0 | inactive_split_MB: 234.4 +rank0 [2026-10-29 00:13:33,744] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:17:17,470] INFO : Step 18400/50000 rank0 | Train Loss: 1.3667 | Train Loss Unweighted: 0.7337 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1448 | Classification: 0.1277 | Classification_binary: 0.2393 | HitGrouping: 0.2095 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1805 | Regression_eta: 0.0022 | Regression_pt: 0.1818 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.2676 | Regression_cos_phi: 0.0908 | Regression_energy: 0.0915 | Regression_eta: 0.0677 | Regression_pt: 0.0953 | Regression_sin_phi: 0.1022 | LR: 7.31e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.1100s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1744 | max_backward: 1.7308 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6782.0 | uss_MB: 6123.4 | child_rss_MB: 5027.3 | allocated_MB: 344.6 | reserved_MB: 24778.0 | inactive_split_MB: 281.4 +rank0 [2026-10-29 00:17:17,727] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 45.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:21:04,579] INFO : Step 18500/50000 rank0 | Train Loss: 1.2528 | Train Loss Unweighted: 0.6816 | Train Loss Components: Auxiliary: 0.2631 | Cardinality: 0.1350 | Classification: 0.1140 | Classification_binary: 0.1973 | HitGrouping: 0.2066 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1821 | Regression_eta: 0.0017 | Regression_pt: 0.1847 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1538 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0915 | Regression_eta: 0.0563 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0523 | LR: 7.29e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.0657s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0113 | max_forward: 0.8281 | max_loss: 0.2418 | max_backward: 1.4683 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7561 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1431 | Memory: rss_MB: 6783.0 | uss_MB: 6122.8 | child_rss_MB: 5026.7 | allocated_MB: 343.3 | reserved_MB: 24778.0 | inactive_split_MB: 226.5 +rank0 [2026-10-29 00:21:04,839] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 45.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:24:45,374] INFO : Step 18600/50000 rank0 | Train Loss: 1.1881 | Train Loss Unweighted: 0.6335 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1331 | Classification: 0.1084 | Classification_binary: 0.1723 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1746 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1647 | Regression_cos_phi: 0.1322 | Regression_energy: 0.0893 | Regression_eta: 0.0441 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0983 | LR: 7.26e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5342s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7299 | max_loss: 0.2279 | max_backward: 1.0799 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8080 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6782.4 | uss_MB: 6120.8 | child_rss_MB: 5022.4 | allocated_MB: 341.0 | reserved_MB: 24778.0 | inactive_split_MB: 229.3 +rank0 [2026-10-29 00:24:45,632] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 46.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:28:25,673] INFO : Step 18700/50000 rank0 | Train Loss: 1.3122 | Train Loss Unweighted: 0.6534 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1756 | Classification: 0.1225 | Classification_binary: 0.2098 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1571 | Regression_eta: 0.0020 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.2320 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0778 | Regression_eta: 0.0502 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0657 | LR: 7.24e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.3692s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9129 | max_loss: 0.1606 | max_backward: 1.9397 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6785.3 | uss_MB: 6128.9 | child_rss_MB: 5021.8 | allocated_MB: 344.8 | reserved_MB: 24778.0 | inactive_split_MB: 239.8 +rank0 [2026-10-29 00:28:25,945] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 45.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:32:06,224] INFO : Step 18800/50000 rank0 | Train Loss: 1.2696 | Train Loss Unweighted: 0.6643 | Train Loss Components: Auxiliary: 0.2674 | Cardinality: 0.1602 | Classification: 0.1070 | Classification_binary: 0.2200 | HitGrouping: 0.2058 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1652 | Regression_eta: 0.0018 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.2416 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0793 | Regression_eta: 0.0479 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0738 | LR: 7.21e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.3430s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7444 | max_loss: 0.2527 | max_backward: 1.7167 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.9324 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 6785.1 | uss_MB: 6124.1 | child_rss_MB: 5021.8 | allocated_MB: 345.4 | reserved_MB: 25044.0 | inactive_split_MB: 284.4 +rank0 [2026-10-29 00:32:06,483] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 0% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:35:50,185] INFO : Step 18900/50000 rank0 | Train Loss: 1.1718 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1382 | Classification: 0.0996 | Classification_binary: 0.1741 | HitGrouping: 0.2036 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1652 | Regression_eta: 0.0017 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0624 | Regression_eta: 0.0533 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0640 | LR: 7.18e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.7151s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0056 | max_forward: 0.6830 | max_loss: 0.1421 | max_backward: 1.2812 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.8931 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6786.1 | uss_MB: 6126.4 | child_rss_MB: 5006.2 | allocated_MB: 338.7 | reserved_MB: 25044.0 | inactive_split_MB: 232.6 +rank0 [2026-10-29 00:35:50,442] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 45.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:39:31,914] INFO : Step 19000/50000 rank0 | Train Loss: 1.2397 | Train Loss Unweighted: 0.6317 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1277 | Classification: 0.1188 | Classification_binary: 0.1866 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1600 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0833 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0932 | Regression_energy: 0.0753 | Regression_eta: 0.0516 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0784 | LR: 7.16e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.8297s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6593 | max_loss: 0.2467 | max_backward: 1.6203 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7732 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6787.7 | uss_MB: 6128.3 | child_rss_MB: 5026.8 | allocated_MB: 342.2 | reserved_MB: 25790.0 | inactive_split_MB: 229.6 +rank0 [2026-10-29 00:39:32,172] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:39:32,196] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-19000.pth +rank0 [2026-10-29 00:39:32,370] INFO : Running validation on rank0 +rank0 [2026-10-29 00:39:32,371] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 00:39:32,371] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION | Step=19000/50000 | Train Loss=1.2397 | Valid Loss=1.2812 | Train Loss Unweighted=0.6317 | Valid Loss Unweighted=0.6548 | Stale=0 | ETA=1316.8m +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 00:40:24,350] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/ari=0.489183 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.554687 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/top1_competing=0.631758 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.615871 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0599699 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0413959 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 00:40:24,351] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/seconds_per_rank=21.3336 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 00:40:24,352] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:40:24,353] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding/ari=0.280461 +rank0 [2026-10-29 00:40:24,353] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 00:40:24,353] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding_minus_raw/ari=0.188837 +rank0 [2026-10-29 00:40:24,353] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 00:40:24,353] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 00:40:24,353] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 00:40:24,358] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:40:24,613] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 00:40:24,617] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 00:40:50,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 00:40:51,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 00:40:52,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 00:40:53,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 00:41:03,510] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 00:41:03,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:41:04,088] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 00:41:04,092] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 00:41:20,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 00:41:21,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 00:41:22,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 00:41:23,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 00:41:33,280] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 00:41:33,291] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:41:34,030] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 00:41:34,033] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 00:41:49,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 00:41:50,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 00:41:50,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 00:41:51,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 00:42:01,683] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 00:42:01,698] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits +rank0 [2026-10-29 00:42:04,891] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits +rank0 [2026-10-29 00:42:05,963] INFO : Plotted number of elements +rank0 [2026-10-29 00:42:07,553] INFO : Plotted jets +rank0 [2026-10-29 00:42:08,700] INFO : Plotted jet ratio +rank0 [2026-10-29 00:42:08,789] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 00:42:12,011] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 00:42:13,003] INFO : Plotted number of elements +rank0 [2026-10-29 00:42:14,629] INFO : Plotted jets +rank0 [2026-10-29 00:42:15,837] INFO : Plotted jet ratio +rank0 [2026-10-29 00:42:15,919] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits +rank0 [2026-10-29 00:42:18,459] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits +rank0 [2026-10-29 00:42:19,762] INFO : Plotted number of elements +rank0 [2026-10-29 00:42:21,276] INFO : Plotted jets +rank0 [2026-10-29 00:42:22,374] INFO : Plotted jet ratio +rank0 [2026-10-29 00:46:02,837] INFO : Step 19100/50000 rank0 | Train Loss: 1.3562 | Train Loss Unweighted: 0.6933 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1644 | Classification: 0.1204 | Classification_binary: 0.2426 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1627 | Regression_eta: 0.0023 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2447 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0799 | Regression_eta: 0.0728 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0737 | LR: 7.13e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.4322s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7796 | max_loss: 0.1634 | max_backward: 1.6942 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.8232 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 8027.6 | uss_MB: 7365.9 | child_rss_MB: 5027.2 | allocated_MB: 345.2 | reserved_MB: 23900.0 | inactive_split_MB: 240.8 +rank0 [2026-10-29 00:46:03,082] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 45.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:49:49,396] INFO : Step 19200/50000 rank0 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5946 | Train Loss Components: Auxiliary: 0.2384 | Cardinality: 0.1391 | Classification: 0.0965 | Classification_binary: 0.1984 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1472 | Regression_eta: 0.0018 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1974 | Regression_cos_phi: 0.1073 | Regression_energy: 0.0749 | Regression_eta: 0.0466 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0870 | LR: 7.10e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6304s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7138 | max_loss: 0.2391 | max_backward: 1.2379 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.7395 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6876.1 | uss_MB: 6216.1 | child_rss_MB: 5025.0 | allocated_MB: 341.3 | reserved_MB: 23900.0 | inactive_split_MB: 206.8 +rank0 [2026-10-29 00:49:49,655] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:53:30,864] INFO : Step 19300/50000 rank0 | Train Loss: 1.1768 | Train Loss Unweighted: 0.6083 | Train Loss Components: Auxiliary: 0.2422 | Cardinality: 0.1345 | Classification: 0.1035 | Classification_binary: 0.1709 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0018 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.1918 | Regression_cos_phi: 0.1162 | Regression_energy: 0.0744 | Regression_eta: 0.0551 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0712 | LR: 7.08e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.6991s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0503 | max_forward: 0.7163 | max_loss: 0.1442 | max_backward: 1.3745 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6878.1 | uss_MB: 6217.4 | child_rss_MB: 5020.2 | allocated_MB: 341.0 | reserved_MB: 23900.0 | inactive_split_MB: 218.1 +rank0 [2026-10-29 00:53:31,106] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:57:11,800] INFO : Step 19400/50000 rank0 | Train Loss: 1.4590 | Train Loss Unweighted: 0.7109 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.2328 | Classification: 0.1237 | Classification_binary: 0.2265 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1770 | Regression_eta: 0.0021 | Regression_pt: 0.1794 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2195 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0889 | Regression_eta: 0.0489 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0751 | LR: 7.05e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.1020s | Phase Times: max_pre_step_barrier: 0.1118 | max_batch_to_device: 0.0070 | max_forward: 0.7578 | max_loss: 0.2459 | max_backward: 1.7975 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6876.3 | uss_MB: 6217.1 | child_rss_MB: 5016.1 | allocated_MB: 347.7 | reserved_MB: 24186.0 | inactive_split_MB: 276.3 +rank0 [2026-10-29 00:57:12,051] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:00:49,319] INFO : Step 19500/50000 rank0 | Train Loss: 1.3753 | Train Loss Unweighted: 0.6794 | Train Loss Components: Auxiliary: 0.2810 | Cardinality: 0.1865 | Classification: 0.1153 | Classification_binary: 0.2310 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1637 | Regression_eta: 0.0023 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2124 | Regression_cos_phi: 0.1054 | Regression_energy: 0.0784 | Regression_eta: 0.0613 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0783 | LR: 7.02e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6665s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7197 | max_loss: 0.2164 | max_backward: 1.3937 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.4987 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6876.3 | uss_MB: 6219.5 | child_rss_MB: 5017.7 | allocated_MB: 345.2 | reserved_MB: 24186.0 | inactive_split_MB: 275.2 +rank0 [2026-10-29 01:00:49,563] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:04:42,855] INFO : Step 19600/50000 rank0 | Train Loss: 1.2086 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1557 | Classification: 0.0942 | Classification_binary: 0.1768 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1735 | Regression_eta: 0.0015 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1499 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0831 | Regression_eta: 0.0378 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0637 | LR: 7.00e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3882s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6937 | max_loss: 0.1942 | max_backward: 1.2727 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6877.2 | uss_MB: 6219.0 | child_rss_MB: 5004.9 | allocated_MB: 340.6 | reserved_MB: 24186.0 | inactive_split_MB: 277.9 +rank0 [2026-10-29 01:04:43,098] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:08:29,854] INFO : Step 19700/50000 rank0 | Train Loss: 1.2115 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1478 | Classification: 0.1094 | Classification_binary: 0.1919 | HitGrouping: 0.2027 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0929 | Regression_energy: 0.0751 | Regression_eta: 0.0596 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0671 | LR: 6.97e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5213s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0914 | max_forward: 0.7091 | max_loss: 0.1429 | max_backward: 1.4681 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0652 | Memory: rss_MB: 6879.2 | uss_MB: 6222.3 | child_rss_MB: 5016.1 | allocated_MB: 344.0 | reserved_MB: 24630.0 | inactive_split_MB: 143.8 +rank0 [2026-10-29 01:08:30,109] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:12:37,571] INFO : Step 19800/50000 rank0 | Train Loss: 1.2919 | Train Loss Unweighted: 0.6584 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1453 | Classification: 0.1132 | Classification_binary: 0.2375 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0020 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2434 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0773 | Regression_eta: 0.0534 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0902 | LR: 6.94e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.1897s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0074 | max_forward: 0.7828 | max_loss: 0.2508 | max_backward: 1.8604 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6879.1 | uss_MB: 6223.6 | child_rss_MB: 5015.7 | allocated_MB: 348.0 | reserved_MB: 25634.0 | inactive_split_MB: 277.9 +rank0 [2026-10-29 01:12:37,813] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:16:28,215] INFO : Step 19900/50000 rank0 | Train Loss: 1.2010 | Train Loss Unweighted: 0.6167 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1346 | Classification: 0.0885 | Classification_binary: 0.1961 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1633 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1635 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0937 | Regression_eta: 0.0397 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0855 | LR: 6.92e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6903s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6671 | max_loss: 0.2548 | max_backward: 1.5862 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6881.1 | uss_MB: 6224.6 | child_rss_MB: 5016.9 | allocated_MB: 340.3 | reserved_MB: 25634.0 | inactive_split_MB: 201.7 +rank0 [2026-10-29 01:16:28,458] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:20:17,593] INFO : Step 20000/50000 rank0 | Train Loss: 1.2077 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1487 | Classification: 0.1030 | Classification_binary: 0.1729 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1639 | Regression_eta: 0.0019 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0780 | Classification_binary: 0.1879 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0797 | Regression_eta: 0.0670 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0978 | LR: 6.89e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5706s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.6570 | max_loss: 0.1550 | max_backward: 1.4848 | max_optimizer_step: 0.2205 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6884.1 | uss_MB: 6226.8 | child_rss_MB: 5009.1 | allocated_MB: 341.8 | reserved_MB: 26642.0 | inactive_split_MB: 237.7 +rank0 [2026-10-29 01:20:17,833] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:20:17,856] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-20000.pth +rank0 [2026-10-29 01:20:18,031] INFO : Running validation on rank0 +rank0 [2026-10-29 01:20:18,031] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 01:20:18,031] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 01:21:08,151] INFO : VALIDATION | Step=20000/50000 | Train Loss=1.2077 | Valid Loss=1.2858 | Train Loss Unweighted=0.6086 | Valid Loss Unweighted=0.6469 | Stale=1 | ETA=1271.7m +rank0 [2026-10-29 01:21:08,152] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 01:21:08,152] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 01:21:08,152] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 01:21:08,152] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 01:21:08,152] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 01:21:08,153] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 01:21:08,153] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 01:21:08,153] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 01:21:08,153] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 01:21:08,153] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 01:21:08,153] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 01:21:08,153] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 01:21:08,153] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 01:21:08,154] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/ari=0.506825 +rank0 [2026-10-29 01:21:08,154] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 01:21:08,154] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.596396 +rank0 [2026-10-29 01:21:08,154] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/top1_competing=0.681879 +rank0 [2026-10-29 01:21:08,154] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.602808 +rank0 [2026-10-29 01:21:08,154] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0423275 +rank0 [2026-10-29 01:21:08,154] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0283329 +rank0 [2026-10-29 01:21:08,154] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 01:21:08,154] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 01:21:08,155] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 01:21:08,155] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 01:21:08,155] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 01:21:08,155] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 01:21:08,155] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 01:21:08,155] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 01:21:08,155] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 01:21:08,155] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 01:21:08,155] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 01:21:08,155] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/seconds_per_rank=21.2812 +rank0 [2026-10-29 01:21:08,156] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 01:21:08,156] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 01:21:08,156] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 01:21:08,156] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 01:21:08,156] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 01:21:08,156] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 01:21:08,156] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 01:21:08,156] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding/ari=0.289823 +rank0 [2026-10-29 01:21:08,156] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 01:21:08,156] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding_minus_raw/ari=0.198199 +rank0 [2026-10-29 01:21:08,157] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 01:21:08,157] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 01:21:08,157] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 01:21:08,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:21:08,422] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 01:21:08,426] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 01:21:34,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 01:21:35,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 01:21:37,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 01:21:38,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 01:21:48,189] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 01:21:48,205] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:21:48,734] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 01:21:48,738] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 01:22:04,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 01:22:06,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 01:22:07,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 01:22:08,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 01:22:18,340] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 01:22:18,351] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:22:18,871] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 01:22:18,875] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 01:22:34,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 01:22:35,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 01:22:36,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 01:22:37,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 01:22:47,063] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 01:22:47,078] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits +rank0 [2026-10-29 01:22:50,311] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits +rank0 [2026-10-29 01:22:51,364] INFO : Plotted number of elements +rank0 [2026-10-29 01:22:52,904] INFO : Plotted jets +rank0 [2026-10-29 01:22:54,384] INFO : Plotted jet ratio +rank0 [2026-10-29 01:22:54,463] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 01:22:57,720] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 01:22:58,729] INFO : Plotted number of elements +rank0 [2026-10-29 01:23:00,223] INFO : Plotted jets +rank0 [2026-10-29 01:23:01,310] INFO : Plotted jet ratio +rank0 [2026-10-29 01:23:01,369] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits +rank0 [2026-10-29 01:23:03,857] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits +rank0 [2026-10-29 01:23:04,697] INFO : Plotted number of elements +rank0 [2026-10-29 01:23:06,180] INFO : Plotted jets +rank0 [2026-10-29 01:23:07,297] INFO : Plotted jet ratio +rank0 [2026-10-29 01:26:57,906] INFO : Step 20100/50000 rank0 | Train Loss: 1.3825 | Train Loss Unweighted: 0.6739 | Train Loss Components: Auxiliary: 0.2830 | Cardinality: 0.1854 | Classification: 0.1189 | Classification_binary: 0.2349 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1561 | Regression_eta: 0.0022 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.2432 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0644 | Regression_eta: 0.0456 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0701 | LR: 6.86e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1116s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.9496 | max_loss: 0.1965 | max_backward: 1.6425 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.5117 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1294 | Memory: rss_MB: 8120.6 | uss_MB: 7458.3 | child_rss_MB: 5010.7 | allocated_MB: 344.8 | reserved_MB: 21328.0 | inactive_split_MB: 198.7 +rank0 [2026-10-29 01:26:58,146] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 45.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:30:57,899] INFO : Step 20200/50000 rank0 | Train Loss: 1.3220 | Train Loss Unweighted: 0.6624 | Train Loss Components: Auxiliary: 0.2728 | Cardinality: 0.1545 | Classification: 0.1109 | Classification_binary: 0.2232 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1618 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1747 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0727 | Regression_eta: 0.0492 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0809 | LR: 6.84e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5297s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.9337 | max_loss: 0.2287 | max_backward: 1.4619 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6969.7 | uss_MB: 6305.7 | child_rss_MB: 5014.0 | allocated_MB: 344.4 | reserved_MB: 21328.0 | inactive_split_MB: 287.4 +rank0 [2026-10-29 01:30:58,139] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 51.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +6 10 0x7408, 62036 45.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:34:46,962] INFO : Step 20300/50000 rank0 | Train Loss: 1.2742 | Train Loss Unweighted: 0.6629 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1615 | Classification: 0.1132 | Classification_binary: 0.1937 | HitGrouping: 0.2129 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1761 | Regression_eta: 0.0018 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0920 | Regression_eta: 0.0503 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0783 | LR: 6.81e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5860s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6857 | max_loss: 0.2423 | max_backward: 1.2822 | max_optimizer_step: 0.2837 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0419 | Memory: rss_MB: 6967.2 | uss_MB: 6303.7 | child_rss_MB: 5019.2 | allocated_MB: 340.8 | reserved_MB: 21826.0 | inactive_split_MB: 239.4 +rank0 [2026-10-29 01:34:47,202] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:38:44,291] INFO : Step 20400/50000 rank0 | Train Loss: 1.2743 | Train Loss Unweighted: 0.6521 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1507 | Classification: 0.1279 | Classification_binary: 0.1990 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1604 | Regression_eta: 0.0019 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0791 | Regression_eta: 0.0446 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0927 | LR: 6.78e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.4359s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0283 | max_forward: 1.3648 | max_loss: 0.1816 | max_backward: 1.7875 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 6969.3 | uss_MB: 6306.4 | child_rss_MB: 5010.2 | allocated_MB: 343.0 | reserved_MB: 22154.0 | inactive_split_MB: 243.5 +rank0 [2026-10-29 01:38:44,532] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 45.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:42:50,346] INFO : Step 20500/50000 rank0 | Train Loss: 1.2712 | Train Loss Unweighted: 0.6509 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1409 | Classification: 0.1147 | Classification_binary: 0.2202 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1558 | Regression_eta: 0.0019 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2098 | Regression_cos_phi: 0.1082 | Regression_energy: 0.0829 | Regression_eta: 0.0385 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0982 | LR: 6.76e-04 | DataLoad Time: 0.0148s | Model Forward Time: 6.2273s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7205 | max_loss: 0.2283 | max_backward: 1.6623 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.6254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0704 | Memory: rss_MB: 6969.0 | uss_MB: 6306.9 | child_rss_MB: 5000.2 | allocated_MB: 345.0 | reserved_MB: 22154.0 | inactive_split_MB: 208.2 +rank0 [2026-10-29 01:42:50,590] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:46:46,324] INFO : Step 20600/50000 rank0 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6338 | Train Loss Components: Auxiliary: 0.2625 | Cardinality: 0.1623 | Classification: 0.1101 | Classification_binary: 0.1900 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0018 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1593 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0920 | Regression_eta: 0.0502 | Regression_pt: 0.0982 | Regression_sin_phi: 0.1175 | LR: 6.73e-04 | DataLoad Time: 0.0222s | Model Forward Time: 5.6333s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 1.1520 | max_loss: 0.2849 | max_backward: 2.1471 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.6141 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6963.6 | uss_MB: 6301.3 | child_rss_MB: 5002.5 | allocated_MB: 341.4 | reserved_MB: 22154.0 | inactive_split_MB: 183.5 +rank0 [2026-10-29 01:46:46,565] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:50:54,903] INFO : Step 20700/50000 rank0 | Train Loss: 1.1613 | Train Loss Unweighted: 0.6066 | Train Loss Components: Auxiliary: 0.2395 | Cardinality: 0.1327 | Classification: 0.1061 | Classification_binary: 0.1677 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0016 | Regression_pt: 0.1666 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0788 | Regression_eta: 0.0371 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0719 | LR: 6.70e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2829s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6276 | max_loss: 0.2330 | max_backward: 1.4954 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6965.5 | uss_MB: 5936.8 | child_rss_MB: 5004.9 | allocated_MB: 341.0 | reserved_MB: 22154.0 | inactive_split_MB: 285.0 +rank0 [2026-10-29 01:50:55,143] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 48.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 83% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 46.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:55:02,502] INFO : Step 20800/50000 rank0 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1493 | Classification: 0.1205 | Classification_binary: 0.2020 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0831 | Classification_binary: 0.2203 | Regression_cos_phi: 0.1213 | Regression_energy: 0.0768 | Regression_eta: 0.0599 | Regression_pt: 0.0797 | Regression_sin_phi: 0.1011 | LR: 6.67e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.9280s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.7679 | max_loss: 0.1604 | max_backward: 1.7694 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.4914 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6972.0 | uss_MB: 6311.7 | child_rss_MB: 5027.1 | allocated_MB: 344.1 | reserved_MB: 22154.0 | inactive_split_MB: 258.7 +rank0 [2026-10-29 01:55:02,756] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:59:07,078] INFO : Step 20900/50000 rank0 | Train Loss: 1.2934 | Train Loss Unweighted: 0.6525 | Train Loss Components: Auxiliary: 0.2719 | Cardinality: 0.1620 | Classification: 0.1167 | Classification_binary: 0.2121 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1591 | Regression_eta: 0.0021 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2086 | Regression_cos_phi: 0.1199 | Regression_energy: 0.0751 | Regression_eta: 0.0948 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0871 | LR: 6.65e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7554s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7787 | max_loss: 0.1747 | max_backward: 1.6678 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 6974.0 | uss_MB: 5772.6 | child_rss_MB: 5037.2 | allocated_MB: 343.6 | reserved_MB: 23208.0 | inactive_split_MB: 296.4 +rank0 [2026-10-29 01:59:07,331] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 1% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 78% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:03:09,659] INFO : Step 21000/50000 rank0 | Train Loss: 1.1402 | Train Loss Unweighted: 0.5874 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1236 | Classification: 0.0967 | Classification_binary: 0.1701 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1583 | Regression_eta: 0.0017 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0707 | Regression_eta: 0.0365 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0556 | LR: 6.62e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6793s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6629 | max_loss: 0.1833 | max_backward: 1.4689 | max_optimizer_step: 0.2206 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6973.2 | uss_MB: 6313.8 | child_rss_MB: 5014.5 | allocated_MB: 340.4 | reserved_MB: 23208.0 | inactive_split_MB: 264.1 +rank0 [2026-10-29 02:03:09,905] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:03:09,924] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-21000.pth +rank0 [2026-10-29 02:03:10,117] INFO : Running validation on rank0 +rank0 [2026-10-29 02:03:10,117] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 02:03:10,117] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 02:04:01,034] INFO : VALIDATION | Step=21000/50000 | Train Loss=1.1402 | Valid Loss=1.2321 | Train Loss Unweighted=0.5874 | Valid Loss Unweighted=0.6297 | Stale=0 | ETA=1230.0m +rank0 [2026-10-29 02:04:01,034] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:04:01,034] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 02:04:01,034] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 02:04:01,034] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:04:01,034] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 02:04:01,034] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/ari=0.474518 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.590556 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/top1_competing=0.677076 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.625037 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0746347 +rank0 [2026-10-29 02:04:01,035] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0505619 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/seconds_per_rank=21.5639 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 02:04:01,036] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 02:04:01,037] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 02:04:01,037] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:04:01,037] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 02:04:01,037] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 02:04:01,037] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:04:01,037] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding/ari=0.296551 +rank0 [2026-10-29 02:04:01,037] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 02:04:01,037] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding_minus_raw/ari=0.204928 +rank0 [2026-10-29 02:04:01,037] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 02:04:01,038] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 02:04:01,038] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 02:04:01,043] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:04:01,304] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 02:04:01,308] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 02:04:27,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 02:04:28,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 02:04:29,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 02:04:30,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 02:04:40,790] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 02:04:40,804] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:04:41,352] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 02:04:41,356] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 02:04:57,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 02:04:58,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 02:04:59,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 02:05:00,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 02:05:10,611] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 02:05:10,622] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:05:11,134] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 02:05:11,137] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 02:05:26,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 02:05:27,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 02:05:28,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 02:05:29,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 02:05:39,898] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 02:05:42,885] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits +rank0 [2026-10-29 02:05:46,122] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits +rank0 [2026-10-29 02:05:47,188] INFO : Plotted number of elements +rank0 [2026-10-29 02:05:48,738] INFO : Plotted jets +rank0 [2026-10-29 02:05:50,220] INFO : Plotted jet ratio +rank0 [2026-10-29 02:05:50,273] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 02:05:53,528] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 02:05:54,548] INFO : Plotted number of elements +rank0 [2026-10-29 02:05:56,044] INFO : Plotted jets +rank0 [2026-10-29 02:05:57,177] INFO : Plotted jet ratio +rank0 [2026-10-29 02:05:57,228] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits +rank0 [2026-10-29 02:05:59,691] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits +rank0 [2026-10-29 02:06:00,578] INFO : Plotted number of elements +rank0 [2026-10-29 02:06:02,081] INFO : Plotted jets +rank0 [2026-10-29 02:06:03,208] INFO : Plotted jet ratio +rank0 [2026-10-29 02:10:06,975] INFO : Step 21100/50000 rank0 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6632 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1574 | Classification: 0.1277 | Classification_binary: 0.2026 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0024 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0885 | Regression_energy: 0.0773 | Regression_eta: 0.0681 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0721 | LR: 6.59e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.0592s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7159 | max_loss: 0.1411 | max_backward: 1.5805 | max_optimizer_step: 0.3396 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8247.8 | uss_MB: 7584.2 | child_rss_MB: 5046.5 | allocated_MB: 343.8 | reserved_MB: 21372.0 | inactive_split_MB: 255.4 +rank0 [2026-10-29 02:10:07,216] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 30% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 45.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:10:38,255] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-29 02:10:38,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-29 02:14:16,162] INFO : Step 21200/50000 rank0 | Train Loss: 1.2354 | Train Loss Unweighted: 0.6366 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1422 | Classification: 0.1056 | Classification_binary: 0.1876 | HitGrouping: 0.2030 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0017 | Regression_pt: 0.1709 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0864 | Regression_eta: 0.0498 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0964 | LR: 6.56e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8166s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7385 | max_loss: 0.2183 | max_backward: 1.4276 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.7483 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7137.9 | uss_MB: 6469.9 | child_rss_MB: 5019.5 | allocated_MB: 342.5 | reserved_MB: 23286.0 | inactive_split_MB: 273.0 +rank0 [2026-10-29 02:14:16,419] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 30% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 45.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 33% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:18:18,619] INFO : Step 21300/50000 rank0 | Train Loss: 1.1511 | Train Loss Unweighted: 0.5782 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1240 | Classification: 0.1021 | Classification_binary: 0.1706 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1505 | Regression_eta: 0.0017 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.2001 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0684 | Regression_eta: 0.0397 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0724 | LR: 6.54e-04 | DataLoad Time: 0.0229s | Model Forward Time: 5.1187s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0290 | max_forward: 1.3238 | max_loss: 0.2105 | max_backward: 1.6672 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.6878 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7140.4 | uss_MB: 6475.6 | child_rss_MB: 5012.2 | allocated_MB: 343.8 | reserved_MB: 23286.0 | inactive_split_MB: 272.8 +rank0 [2026-10-29 02:18:18,863] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 45.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:22:15,238] INFO : Step 21400/50000 rank0 | Train Loss: 1.2993 | Train Loss Unweighted: 0.6458 | Train Loss Components: Auxiliary: 0.2724 | Cardinality: 0.1796 | Classification: 0.1204 | Classification_binary: 0.2158 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1518 | Regression_eta: 0.0018 | Regression_pt: 0.1542 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2226 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0720 | Regression_eta: 0.0527 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0759 | LR: 6.51e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.1935s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8696 | max_loss: 0.1572 | max_backward: 1.8002 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 0.6992 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7114.7 | uss_MB: 6081.0 | child_rss_MB: 5015.7 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 201.9 +rank0 [2026-10-29 02:22:15,480] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 7% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 46.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:26:15,121] INFO : Step 21500/50000 rank0 | Train Loss: 1.2839 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2693 | Cardinality: 0.1478 | Classification: 0.1058 | Classification_binary: 0.2162 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.2346 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0794 | Regression_eta: 0.0593 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0953 | LR: 6.48e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.4029s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.8684 | max_loss: 0.1577 | max_backward: 1.9355 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.7593 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0648 | Memory: rss_MB: 7121.8 | uss_MB: 6458.0 | child_rss_MB: 5014.4 | allocated_MB: 341.9 | reserved_MB: 25248.0 | inactive_split_MB: 284.1 +rank0 [2026-10-29 02:26:15,381] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 46.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:30:01,512] INFO : Step 21600/50000 rank0 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1290 | Classification: 0.1062 | Classification_binary: 0.1618 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1486 | Regression_eta: 0.0018 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0928 | Regression_energy: 0.0759 | Regression_eta: 0.0536 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0841 | LR: 6.45e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.7626s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0061 | max_forward: 0.7663 | max_loss: 0.1883 | max_backward: 1.4257 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.4997 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7120.6 | uss_MB: 6456.9 | child_rss_MB: 5032.2 | allocated_MB: 340.5 | reserved_MB: 25248.0 | inactive_split_MB: 234.8 +rank0 [2026-10-29 02:30:01,757] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 46.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:33:58,056] INFO : Step 21700/50000 rank0 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6283 | Train Loss Components: Auxiliary: 0.2577 | Cardinality: 0.1575 | Classification: 0.1186 | Classification_binary: 0.1853 | HitGrouping: 0.2068 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1594 | Regression_eta: 0.0022 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2013 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0686 | Regression_eta: 0.0533 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0762 | LR: 6.43e-04 | DataLoad Time: 0.0139s | Model Forward Time: 4.3629s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.6796 | max_loss: 0.1417 | max_backward: 1.3881 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7122.2 | uss_MB: 6458.0 | child_rss_MB: 5034.2 | allocated_MB: 343.8 | reserved_MB: 25248.0 | inactive_split_MB: 244.5 +rank0 [2026-10-29 02:33:58,296] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 46.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:37:55,232] INFO : Step 21800/50000 rank0 | Train Loss: 1.2260 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2691 | Cardinality: 0.1439 | Classification: 0.1105 | Classification_binary: 0.2043 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1576 | Regression_eta: 0.0016 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.2306 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0778 | Regression_eta: 0.0505 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0605 | LR: 6.40e-04 | DataLoad Time: 0.0188s | Model Forward Time: 6.7158s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 0.8043 | max_loss: 0.1895 | max_backward: 2.4027 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.9442 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7124.3 | uss_MB: 5827.3 | child_rss_MB: 5037.0 | allocated_MB: 344.5 | reserved_MB: 25248.0 | inactive_split_MB: 277.8 +rank0 [2026-10-29 02:37:55,472] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 46.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:41:43,146] INFO : Step 21900/50000 rank0 | Train Loss: 1.2119 | Train Loss Unweighted: 0.5952 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1861 | Classification: 0.1033 | Classification_binary: 0.1865 | HitGrouping: 0.1989 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0015 | Regression_pt: 0.1511 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1007 | Classification_binary: 0.1961 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0714 | Regression_eta: 0.0578 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0733 | LR: 6.37e-04 | DataLoad Time: 0.0132s | Model Forward Time: 5.1968s | Phase Times: max_pre_step_barrier: 0.0174 | max_batch_to_device: 0.0163 | max_forward: 0.9684 | max_loss: 0.1448 | max_backward: 1.6354 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.8990 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 7124.6 | uss_MB: 6460.4 | child_rss_MB: 5033.8 | allocated_MB: 342.7 | reserved_MB: 25248.0 | inactive_split_MB: 192.3 +rank0 [2026-10-29 02:41:43,388] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:45:42,466] INFO : Step 22000/50000 rank0 | Train Loss: 1.1094 | Train Loss Unweighted: 0.5651 | Train Loss Components: Auxiliary: 0.2344 | Cardinality: 0.1248 | Classification: 0.1037 | Classification_binary: 0.1579 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1497 | Regression_eta: 0.0017 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1879 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0762 | Regression_eta: 0.0581 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0672 | LR: 6.34e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.1552s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.6825 | max_loss: 0.1653 | max_backward: 1.4396 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.0256 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 7124.1 | uss_MB: 6463.7 | child_rss_MB: 5009.1 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 247.3 +rank0 [2026-10-29 02:45:42,708] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 45.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:45:42,828] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-22000.pth +rank0 [2026-10-29 02:45:43,018] INFO : Running validation on rank0 +rank0 [2026-10-29 02:45:43,018] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 02:45:43,018] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 02:46:38,052] INFO : VALIDATION | Step=22000/50000 | Train Loss=1.1094 | Valid Loss=1.2289 | Train Loss Unweighted=0.5651 | Valid Loss Unweighted=0.6279 | Stale=0 | ETA=1187.8m +rank0 [2026-10-29 02:46:38,052] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 02:46:38,053] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/ari=0.488066 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.598381 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/top1_competing=0.686295 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.60952 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0610871 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0350457 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 02:46:38,054] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/seconds_per_rank=21.4815 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding/ari=0.260685 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding_minus_raw/ari=0.169062 +rank0 [2026-10-29 02:46:38,055] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 02:46:38,056] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 02:46:38,056] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 02:46:38,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:46:38,327] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 02:46:38,331] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 02:46:57,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 02:46:58,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 02:47:00,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 02:47:01,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 02:47:11,403] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 02:47:11,416] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:47:11,959] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 02:47:11,963] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 02:47:28,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 02:47:29,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 02:47:30,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 02:47:31,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 02:47:41,448] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 02:47:41,459] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:47:42,029] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 02:47:42,033] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 02:47:57,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 02:47:58,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 02:47:59,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 02:48:00,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 02:48:10,028] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 02:48:10,044] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits +rank0 [2026-10-29 02:48:13,237] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits +rank0 [2026-10-29 02:48:14,268] INFO : Plotted number of elements +rank0 [2026-10-29 02:48:15,810] INFO : Plotted jets +rank0 [2026-10-29 02:48:16,929] INFO : Plotted jet ratio +rank0 [2026-10-29 02:48:16,980] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 02:48:20,428] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 02:48:21,413] INFO : Plotted number of elements +rank0 [2026-10-29 02:48:22,924] INFO : Plotted jets +rank0 [2026-10-29 02:48:24,116] INFO : Plotted jet ratio +rank0 [2026-10-29 02:48:24,172] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits +rank0 [2026-10-29 02:48:26,610] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits +rank0 [2026-10-29 02:48:27,499] INFO : Plotted number of elements +rank0 [2026-10-29 02:48:29,039] INFO : Plotted jets +rank0 [2026-10-29 02:48:30,186] INFO : Plotted jet ratio +rank0 [2026-10-29 02:52:25,467] INFO : Step 22100/50000 rank0 | Train Loss: 1.2514 | Train Loss Unweighted: 0.6219 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1530 | Classification: 0.1140 | Classification_binary: 0.2076 | HitGrouping: 0.2031 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1470 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.2071 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0826 | Regression_eta: 0.0529 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0739 | LR: 6.32e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.1638s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7250 | max_loss: 0.2514 | max_backward: 1.5956 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.8632 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0587 | Memory: rss_MB: 8032.9 | uss_MB: 7365.3 | child_rss_MB: 5019.4 | allocated_MB: 347.3 | reserved_MB: 23386.0 | inactive_split_MB: 278.6 +rank0 [2026-10-29 02:52:25,711] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:56:29,830] INFO : Step 22200/50000 rank0 | Train Loss: 1.2727 | Train Loss Unweighted: 0.6485 | Train Loss Components: Auxiliary: 0.2709 | Cardinality: 0.1498 | Classification: 0.1104 | Classification_binary: 0.2184 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1574 | Regression_eta: 0.0018 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.2077 | Regression_cos_phi: 0.0927 | Regression_energy: 0.0748 | Regression_eta: 0.0554 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0878 | LR: 6.29e-04 | DataLoad Time: 0.0164s | Model Forward Time: 7.6161s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0074 | max_forward: 0.7776 | max_loss: 0.1472 | max_backward: 1.7604 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.3305 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7158.4 | uss_MB: 5092.5 | child_rss_MB: 5016.1 | allocated_MB: 342.3 | reserved_MB: 23578.0 | inactive_split_MB: 277.9 +rank0 [2026-10-29 02:56:30,078] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 21% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 0% +6 10 0x7408, 62036 45.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:00:35,809] INFO : Step 22300/50000 rank0 | Train Loss: 1.1242 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2347 | Cardinality: 0.1367 | Classification: 0.1013 | Classification_binary: 0.1688 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1733 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0790 | Regression_eta: 0.0602 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0597 | LR: 6.26e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.4168s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7697 | max_loss: 0.1538 | max_backward: 1.3553 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.4896 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 7162.9 | uss_MB: 6498.9 | child_rss_MB: 4987.3 | allocated_MB: 341.2 | reserved_MB: 23578.0 | inactive_split_MB: 233.8 +rank0 [2026-10-29 03:00:36,065] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 46.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:04:33,398] INFO : Step 22400/50000 rank0 | Train Loss: 1.2357 | Train Loss Unweighted: 0.6255 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1493 | Classification: 0.1171 | Classification_binary: 0.1885 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1575 | Regression_eta: 0.0021 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0861 | Classification_binary: 0.1847 | Regression_cos_phi: 0.1138 | Regression_energy: 0.0783 | Regression_eta: 0.0576 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0610 | LR: 6.23e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5388s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0067 | max_forward: 0.6425 | max_loss: 0.1530 | max_backward: 1.4136 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.5876 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 7158.4 | uss_MB: 6496.8 | child_rss_MB: 5021.9 | allocated_MB: 342.5 | reserved_MB: 23578.0 | inactive_split_MB: 251.5 +rank0 [2026-10-29 03:04:33,641] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:08:27,309] INFO : Step 22500/50000 rank0 | Train Loss: 1.2633 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2638 | Cardinality: 0.1676 | Classification: 0.0954 | Classification_binary: 0.2181 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0020 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2133 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0751 | Regression_eta: 0.0649 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0652 | LR: 6.20e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.0518s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1522 | max_backward: 1.5631 | max_optimizer_step: 0.3261 | max_tensorboard_logging: 0.7552 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 7160.2 | uss_MB: 6502.4 | child_rss_MB: 5011.7 | allocated_MB: 342.9 | reserved_MB: 24052.0 | inactive_split_MB: 295.1 +rank0 [2026-10-29 03:08:27,572] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:12:19,486] INFO : Step 22600/50000 rank0 | Train Loss: 1.1636 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1329 | Classification: 0.0997 | Classification_binary: 0.1889 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1493 | Regression_eta: 0.0020 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1783 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0686 | Regression_eta: 0.0428 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0740 | LR: 6.18e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6621s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7375 | max_loss: 0.2403 | max_backward: 1.4584 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7163.5 | uss_MB: 6503.3 | child_rss_MB: 5024.4 | allocated_MB: 344.2 | reserved_MB: 24590.0 | inactive_split_MB: 288.5 +rank0 [2026-10-29 03:12:19,731] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 53.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:16:20,768] INFO : Step 22700/50000 rank0 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1212 | Classification: 0.0976 | Classification_binary: 0.1566 | HitGrouping: 0.2101 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1455 | Regression_eta: 0.0015 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1644 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0724 | Regression_eta: 0.0698 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0971 | LR: 6.15e-04 | DataLoad Time: 0.0166s | Model Forward Time: 7.6715s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.6124 | max_loss: 0.1376 | max_backward: 1.1766 | max_optimizer_step: 0.2237 | max_tensorboard_logging: 3.8568 | max_tensorboard_flush: 0.0169 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 7164.2 | uss_MB: 6160.1 | child_rss_MB: 5007.8 | allocated_MB: 341.6 | reserved_MB: 25058.0 | inactive_split_MB: 256.4 +rank0 [2026-10-29 03:16:21,016] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 45.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 85% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:20:30,525] INFO : Step 22800/50000 rank0 | Train Loss: 1.1840 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2533 | Cardinality: 0.1443 | Classification: 0.1191 | Classification_binary: 0.1924 | HitGrouping: 0.1959 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1444 | Regression_eta: 0.0016 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.2141 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0711 | Regression_eta: 0.0389 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0600 | LR: 6.12e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.2369s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7010 | max_loss: 0.2312 | max_backward: 1.6853 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8726 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 7164.4 | uss_MB: 6502.8 | child_rss_MB: 5021.2 | allocated_MB: 341.7 | reserved_MB: 25058.0 | inactive_split_MB: 281.3 +rank0 [2026-10-29 03:20:30,782] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 9% +2 6 0x7408, 15961 49.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 45.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 92% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:24:21,772] INFO : Step 22900/50000 rank0 | Train Loss: 1.2680 | Train Loss Unweighted: 0.6318 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1792 | Classification: 0.1020 | Classification_binary: 0.2074 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1580 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0879 | Classification_binary: 0.2018 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0781 | Regression_eta: 0.0463 | Regression_pt: 0.0827 | Regression_sin_phi: 0.1124 | LR: 6.09e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.6096s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 1.1637 | max_loss: 0.4031 | max_backward: 1.8442 | max_optimizer_step: 0.3320 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7165.6 | uss_MB: 6506.2 | child_rss_MB: 5023.4 | allocated_MB: 344.8 | reserved_MB: 25058.0 | inactive_split_MB: 300.1 +rank0 [2026-10-29 03:24:22,028] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:28:06,289] INFO : Step 23000/50000 rank0 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5484 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1129 | Classification: 0.0836 | Classification_binary: 0.1519 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1547 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0796 | Regression_eta: 0.0556 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0687 | LR: 6.06e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4482s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0063 | max_forward: 0.6963 | max_loss: 0.2382 | max_backward: 1.2870 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 7162.6 | uss_MB: 6506.3 | child_rss_MB: 5008.7 | allocated_MB: 340.3 | reserved_MB: 26066.0 | inactive_split_MB: 218.4 +rank0 [2026-10-29 03:28:06,532] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:28:06,554] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-23000.pth +rank0 [2026-10-29 03:28:06,884] INFO : Running validation on rank0 +rank0 [2026-10-29 03:28:06,884] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 03:28:06,884] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 03:28:56,656] INFO : VALIDATION | Step=23000/50000 | Train Loss=1.0575 | Valid Loss=1.1888 | Train Loss Unweighted=0.5484 | Valid Loss Unweighted=0.5953 | Stale=0 | ETA=1145.3m +rank0 [2026-10-29 03:28:56,656] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 03:28:56,656] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 03:28:56,656] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 03:28:56,656] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 03:28:56,656] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 03:28:56,656] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/ari=0.49036 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.609984 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/top1_competing=0.697804 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.616846 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0587931 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0423711 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 03:28:56,657] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/seconds_per_rank=21.4056 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 03:28:56,658] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 03:28:56,659] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 03:28:56,659] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding/ari=0.295712 +rank0 [2026-10-29 03:28:56,659] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 03:28:56,659] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding_minus_raw/ari=0.204089 +rank0 [2026-10-29 03:28:56,659] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 03:28:56,659] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 03:28:56,659] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 03:28:56,664] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:28:56,914] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 03:28:56,919] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 03:29:22,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 03:29:23,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 03:29:24,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 03:29:25,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 03:29:35,801] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 03:29:35,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:29:36,300] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 03:29:36,303] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 03:29:51,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 03:29:53,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 03:29:54,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 03:29:55,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 03:30:05,209] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 03:30:05,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:30:05,701] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 03:30:05,705] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 03:30:21,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 03:30:21,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 03:30:22,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 03:30:23,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 03:30:33,387] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 03:30:33,556] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits +rank0 [2026-10-29 03:30:36,751] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits +rank0 [2026-10-29 03:30:37,788] INFO : Plotted number of elements +rank0 [2026-10-29 03:30:39,346] INFO : Plotted jets +rank0 [2026-10-29 03:30:40,866] INFO : Plotted jet ratio +rank0 [2026-10-29 03:30:40,936] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 03:30:44,134] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 03:30:45,193] INFO : Plotted number of elements +rank0 [2026-10-29 03:30:46,758] INFO : Plotted jets +rank0 [2026-10-29 03:30:47,857] INFO : Plotted jet ratio +rank0 [2026-10-29 03:30:47,910] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits +rank0 [2026-10-29 03:30:50,311] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits +rank0 [2026-10-29 03:30:51,161] INFO : Plotted number of elements +rank0 [2026-10-29 03:30:52,612] INFO : Plotted jets +rank0 [2026-10-29 03:30:53,722] INFO : Plotted jet ratio +rank0 [2026-10-29 03:34:46,106] INFO : Step 23100/50000 rank0 | Train Loss: 1.2422 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1379 | Classification: 0.1246 | Classification_binary: 0.1795 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1482 | Regression_eta: 0.0023 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0677 | Regression_eta: 0.0749 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0688 | LR: 6.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5486s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7203 | max_loss: 0.1522 | max_backward: 1.1000 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.6183 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 8419.6 | uss_MB: 7758.7 | child_rss_MB: 5039.2 | allocated_MB: 344.7 | reserved_MB: 22598.0 | inactive_split_MB: 244.5 +rank0 [2026-10-29 03:34:46,348] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:38:34,209] INFO : Step 23200/50000 rank0 | Train Loss: 1.2507 | Train Loss Unweighted: 0.6359 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1400 | Classification: 0.1121 | Classification_binary: 0.2208 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1488 | Regression_eta: 0.0020 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.2142 | Regression_cos_phi: 0.1035 | Regression_energy: 0.0630 | Regression_eta: 0.0770 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0583 | LR: 6.01e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.2732s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0727 | max_forward: 0.9486 | max_loss: 0.1744 | max_backward: 1.7031 | max_optimizer_step: 0.2405 | max_tensorboard_logging: 0.6717 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1109 | Memory: rss_MB: 7225.3 | uss_MB: 6564.6 | child_rss_MB: 5046.8 | allocated_MB: 343.7 | reserved_MB: 23792.0 | inactive_split_MB: 223.2 +rank0 [2026-10-29 03:38:34,466] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:42:25,180] INFO : Step 23300/50000 rank0 | Train Loss: 1.1880 | Train Loss Unweighted: 0.5927 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1464 | Classification: 0.0987 | Classification_binary: 0.1851 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1524 | Regression_eta: 0.0019 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1753 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0682 | Regression_eta: 0.0463 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0645 | LR: 5.98e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4692s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7557 | max_loss: 0.1467 | max_backward: 1.4708 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.4854 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 7231.3 | uss_MB: 6572.4 | child_rss_MB: 5045.1 | allocated_MB: 342.6 | reserved_MB: 24534.0 | inactive_split_MB: 271.6 +rank0 [2026-10-29 03:42:25,421] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 3% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 12% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 97% +6 10 0x7408, 62036 46.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:46:12,056] INFO : Step 23400/50000 rank0 | Train Loss: 1.1136 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2402 | Cardinality: 0.1202 | Classification: 0.1070 | Classification_binary: 0.1497 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1644 | Regression_eta: 0.0017 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1452 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0747 | Regression_eta: 0.0382 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0485 | LR: 5.95e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4191s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.5743 | max_loss: 0.2297 | max_backward: 1.0127 | max_optimizer_step: 0.3321 | max_tensorboard_logging: 0.6182 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 7232.3 | uss_MB: 6571.7 | child_rss_MB: 5046.9 | allocated_MB: 340.5 | reserved_MB: 24848.0 | inactive_split_MB: 241.3 +rank0 [2026-10-29 03:46:12,296] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 29% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:50:07,950] INFO : Step 23500/50000 rank0 | Train Loss: 1.1601 | Train Loss Unweighted: 0.5799 | Train Loss Components: Auxiliary: 0.2477 | Cardinality: 0.1161 | Classification: 0.1057 | Classification_binary: 0.1852 | HitGrouping: 0.2029 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1416 | Regression_eta: 0.0017 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.2100 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0810 | Regression_eta: 0.0378 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0526 | LR: 5.92e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.0911s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8864 | max_loss: 0.1573 | max_backward: 1.9036 | max_optimizer_step: 0.2294 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7232.1 | uss_MB: 6574.4 | child_rss_MB: 5036.9 | allocated_MB: 341.9 | reserved_MB: 24848.0 | inactive_split_MB: 233.9 +rank0 [2026-10-29 03:50:08,191] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 46.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:53:52,888] INFO : Step 23600/50000 rank0 | Train Loss: 1.2402 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1447 | Classification: 0.1136 | Classification_binary: 0.2049 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1534 | Regression_eta: 0.0020 | Regression_pt: 0.1550 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1974 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0828 | Regression_eta: 0.0469 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0599 | LR: 5.90e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.6679s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8397 | max_loss: 0.1470 | max_backward: 1.4395 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7234.1 | uss_MB: 6575.7 | child_rss_MB: 5045.6 | allocated_MB: 342.5 | reserved_MB: 24848.0 | inactive_split_MB: 215.5 +rank0 [2026-10-29 03:53:53,134] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 46.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:57:36,418] INFO : Step 23700/50000 rank0 | Train Loss: 1.1009 | Train Loss Unweighted: 0.5682 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1218 | Classification: 0.0954 | Classification_binary: 0.1599 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1528 | Regression_eta: 0.0014 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0844 | Regression_eta: 0.0289 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0607 | LR: 5.87e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3193s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6589 | max_loss: 0.1387 | max_backward: 1.2152 | max_optimizer_step: 0.2251 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7230.0 | uss_MB: 6574.3 | child_rss_MB: 5051.4 | allocated_MB: 342.5 | reserved_MB: 25508.0 | inactive_split_MB: 226.3 +rank0 [2026-10-29 03:57:36,660] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 46.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:01:36,797] INFO : Step 23800/50000 rank0 | Train Loss: 1.1390 | Train Loss Unweighted: 0.5882 | Train Loss Components: Auxiliary: 0.2427 | Cardinality: 0.1196 | Classification: 0.1035 | Classification_binary: 0.1735 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.2071 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0682 | Regression_eta: 0.0439 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0583 | LR: 5.84e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.5580s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.7283 | max_loss: 0.1547 | max_backward: 1.2415 | max_optimizer_step: 0.2259 | max_tensorboard_logging: 0.5270 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7231.0 | uss_MB: 6110.2 | child_rss_MB: 5050.1 | allocated_MB: 345.2 | reserved_MB: 25508.0 | inactive_split_MB: 241.8 +rank0 [2026-10-29 04:01:37,044] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 46.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:05:43,003] INFO : Step 23900/50000 rank0 | Train Loss: 1.2349 | Train Loss Unweighted: 0.6078 | Train Loss Components: Auxiliary: 0.2621 | Cardinality: 0.1596 | Classification: 0.1044 | Classification_binary: 0.2077 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1451 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.2227 | Regression_cos_phi: 0.0963 | Regression_energy: 0.0651 | Regression_eta: 0.0418 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0580 | LR: 5.81e-04 | DataLoad Time: 0.0145s | Model Forward Time: 5.1367s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.9077 | max_loss: 0.1538 | max_backward: 1.9088 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7234.7 | uss_MB: 6579.6 | child_rss_MB: 5044.4 | allocated_MB: 342.1 | reserved_MB: 27474.0 | inactive_split_MB: 282.4 +rank0 [2026-10-29 04:05:43,242] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 46.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:09:40,803] INFO : Step 24000/50000 rank0 | Train Loss: 1.1074 | Train Loss Unweighted: 0.5714 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1170 | Classification: 0.1007 | Classification_binary: 0.1792 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1431 | Regression_eta: 0.0015 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0701 | Regression_eta: 0.0352 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0645 | LR: 5.78e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.8670s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 1.1428 | max_loss: 0.2613 | max_backward: 2.5767 | max_optimizer_step: 0.2298 | max_tensorboard_logging: 0.5998 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7236.7 | uss_MB: 6577.2 | child_rss_MB: 5038.1 | allocated_MB: 342.1 | reserved_MB: 27476.0 | inactive_split_MB: 218.6 +rank0 [2026-10-29 04:09:41,044] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 46.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:09:41,248] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-24000.pth +rank0 [2026-10-29 04:09:41,425] INFO : Running validation on rank0 +rank0 [2026-10-29 04:09:41,425] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 04:09:41,425] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 04:10:39,283] INFO : VALIDATION | Step=24000/50000 | Train Loss=1.1074 | Valid Loss=1.1681 | Train Loss Unweighted=0.5714 | Valid Loss Unweighted=0.5858 | Stale=0 | ETA=1102.1m +rank0 [2026-10-29 04:10:39,283] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:10:39,283] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 04:10:39,283] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 04:10:39,283] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:10:39,283] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 04:10:39,283] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 04:10:39,284] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 04:10:39,284] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 04:10:39,284] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 04:10:39,284] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 04:10:39,284] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 04:10:39,284] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 04:10:39,284] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 04:10:39,284] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/ari=0.490215 +rank0 [2026-10-29 04:10:39,284] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 04:10:39,284] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.593898 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/top1_competing=0.678627 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.630082 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0589374 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0556074 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 04:10:39,285] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/seconds_per_rank=21.584 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 04:10:39,286] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 04:10:39,287] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:10:39,287] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding/ari=0.279094 +rank0 [2026-10-29 04:10:39,287] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 04:10:39,287] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding_minus_raw/ari=0.187471 +rank0 [2026-10-29 04:10:39,287] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 04:10:39,287] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 04:10:39,287] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 04:10:39,291] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:10:39,546] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 04:10:39,549] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 04:11:05,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 04:11:06,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 04:11:07,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 04:11:10,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 04:11:20,427] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-29 04:11:20,671] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:11:21,257] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 04:11:21,261] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:11:37,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 04:11:38,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 04:11:39,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 04:11:40,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 04:11:50,516] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 04:11:50,531] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:11:51,026] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 04:11:51,028] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 04:12:06,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 04:12:07,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 04:12:08,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 04:12:08,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 04:12:18,929] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 04:12:18,946] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits +rank0 [2026-10-29 04:12:22,092] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits +rank0 [2026-10-29 04:12:23,094] INFO : Plotted number of elements +rank0 [2026-10-29 04:12:24,638] INFO : Plotted jets +rank0 [2026-10-29 04:12:25,783] INFO : Plotted jet ratio +rank0 [2026-10-29 04:12:25,840] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:12:29,367] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:12:30,405] INFO : Plotted number of elements +rank0 [2026-10-29 04:12:32,009] INFO : Plotted jets +rank0 [2026-10-29 04:12:33,239] INFO : Plotted jet ratio +rank0 [2026-10-29 04:12:33,305] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits +rank0 [2026-10-29 04:12:35,723] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits +rank0 [2026-10-29 04:12:36,676] INFO : Plotted number of elements +rank0 [2026-10-29 04:12:38,158] INFO : Plotted jets +rank0 [2026-10-29 04:12:39,270] INFO : Plotted jet ratio +rank0 [2026-10-29 04:16:38,892] INFO : Step 24100/50000 rank0 | Train Loss: 1.1244 | Train Loss Unweighted: 0.5584 | Train Loss Components: Auxiliary: 0.2373 | Cardinality: 0.1285 | Classification: 0.0978 | Classification_binary: 0.1606 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1888 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0716 | Regression_eta: 0.0394 | Regression_pt: 0.0735 | Regression_sin_phi: 0.1009 | LR: 5.75e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8032s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0860 | max_forward: 0.7361 | max_loss: 0.1432 | max_backward: 1.4268 | max_optimizer_step: 0.3451 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 8228.7 | uss_MB: 7570.1 | child_rss_MB: 5001.0 | allocated_MB: 343.2 | reserved_MB: 21608.0 | inactive_split_MB: 269.9 +rank0 [2026-10-29 04:16:39,132] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:20:37,614] INFO : Step 24200/50000 rank0 | Train Loss: 1.3067 | Train Loss Unweighted: 0.6341 | Train Loss Components: Auxiliary: 0.2739 | Cardinality: 0.1760 | Classification: 0.1145 | Classification_binary: 0.2101 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0018 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.2362 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0769 | Regression_eta: 0.0434 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0810 | LR: 5.73e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.9165s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1066 | max_forward: 1.0531 | max_loss: 0.2796 | max_backward: 2.3820 | max_optimizer_step: 0.2441 | max_tensorboard_logging: 0.6187 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 8233.6 | uss_MB: 7574.6 | child_rss_MB: 5010.3 | allocated_MB: 345.0 | reserved_MB: 23864.0 | inactive_split_MB: 203.0 +rank0 [2026-10-29 04:20:37,857] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:24:33,694] INFO : Step 24300/50000 rank0 | Train Loss: 1.2207 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1582 | Classification: 0.1151 | Classification_binary: 0.2013 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1450 | Regression_eta: 0.0017 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0814 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0689 | Regression_eta: 0.0400 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0754 | LR: 5.70e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.9242s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7395 | max_loss: 0.2490 | max_backward: 1.4969 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.6555 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7355.3 | uss_MB: 6697.1 | child_rss_MB: 5007.1 | allocated_MB: 344.8 | reserved_MB: 23864.0 | inactive_split_MB: 284.8 +rank0 [2026-10-29 04:24:33,955] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:28:19,935] INFO : Step 24400/50000 rank0 | Train Loss: 1.1288 | Train Loss Unweighted: 0.5921 | Train Loss Components: Auxiliary: 0.2439 | Cardinality: 0.1229 | Classification: 0.1017 | Classification_binary: 0.1613 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1631 | Regression_eta: 0.0015 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1517 | Regression_cos_phi: 0.0328 | Regression_energy: 0.0993 | Regression_eta: 0.0262 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0434 | LR: 5.67e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.7497s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.6685 | max_loss: 0.1564 | max_backward: 1.4736 | max_optimizer_step: 0.2223 | max_tensorboard_logging: 0.5369 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7357.1 | uss_MB: 6698.7 | child_rss_MB: 5016.5 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 267.2 +rank0 [2026-10-29 04:28:20,178] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:32:04,049] INFO : Step 24500/50000 rank0 | Train Loss: 1.2319 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2586 | Cardinality: 0.1511 | Classification: 0.1107 | Classification_binary: 0.1876 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1540 | Regression_eta: 0.0018 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.1967 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0679 | Regression_eta: 0.0356 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0539 | LR: 5.64e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7705s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0677 | max_forward: 0.7219 | max_loss: 0.1439 | max_backward: 1.4253 | max_optimizer_step: 0.3416 | max_tensorboard_logging: 0.5859 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 7356.5 | uss_MB: 6698.2 | child_rss_MB: 5010.5 | allocated_MB: 343.1 | reserved_MB: 23864.0 | inactive_split_MB: 275.8 +rank0 [2026-10-29 04:32:04,292] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:36:02,421] INFO : Step 24600/50000 rank0 | Train Loss: 1.2474 | Train Loss Unweighted: 0.6241 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1630 | Classification: 0.1074 | Classification_binary: 0.2102 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2186 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0687 | Regression_eta: 0.0471 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0665 | LR: 5.61e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2737s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7876 | max_loss: 0.1528 | max_backward: 1.7437 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.6204 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7358.2 | uss_MB: 6700.7 | child_rss_MB: 5006.8 | allocated_MB: 344.3 | reserved_MB: 23864.0 | inactive_split_MB: 285.7 +rank0 [2026-10-29 04:36:02,676] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:39:51,948] INFO : Step 24700/50000 rank0 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1193 | Classification: 0.0990 | Classification_binary: 0.1723 | HitGrouping: 0.2019 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1471 | Regression_eta: 0.0016 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1508 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0720 | Regression_eta: 0.0651 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0888 | LR: 5.58e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5998s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6978 | max_loss: 0.2493 | max_backward: 1.2925 | max_optimizer_step: 0.2314 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7361.3 | uss_MB: 6700.3 | child_rss_MB: 5009.8 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 223.9 +rank0 [2026-10-29 04:39:52,191] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 53.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:43:38,558] INFO : Step 24800/50000 rank0 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5554 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1264 | Classification: 0.0932 | Classification_binary: 0.1583 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1708 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0579 | Regression_eta: 0.0386 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0592 | LR: 5.56e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.0257s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6582 | max_loss: 0.2533 | max_backward: 1.5858 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.6923 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1086 | Memory: rss_MB: 7365.2 | uss_MB: 6703.3 | child_rss_MB: 5005.7 | allocated_MB: 342.9 | reserved_MB: 23864.0 | inactive_split_MB: 278.6 +rank0 [2026-10-29 04:43:38,801] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:47:37,062] INFO : Step 24900/50000 rank0 | Train Loss: 1.1348 | Train Loss Unweighted: 0.5816 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1381 | Classification: 0.1126 | Classification_binary: 0.1845 | HitGrouping: 0.1937 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1403 | Regression_eta: 0.0015 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1894 | Regression_cos_phi: 0.0423 | Regression_energy: 0.0650 | Regression_eta: 0.0457 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0489 | LR: 5.53e-04 | DataLoad Time: 0.0154s | Model Forward Time: 9.3670s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0067 | max_forward: 0.7276 | max_loss: 0.2411 | max_backward: 1.3958 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 5.1754 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1038 | Memory: rss_MB: 7366.2 | uss_MB: 6703.5 | child_rss_MB: 5007.2 | allocated_MB: 343.9 | reserved_MB: 23864.0 | inactive_split_MB: 239.7 +rank0 [2026-10-29 04:47:37,315] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 50.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 44.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:51:29,057] INFO : Step 25000/50000 rank0 | Train Loss: 1.1995 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1590 | Classification: 0.0891 | Classification_binary: 0.2009 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1421 | Regression_eta: 0.0018 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0735 | Regression_eta: 0.1030 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0897 | LR: 5.50e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8867s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7692 | max_loss: 0.2091 | max_backward: 1.3616 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6652 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1392 | Memory: rss_MB: 7366.4 | uss_MB: 6702.5 | child_rss_MB: 5014.2 | allocated_MB: 343.5 | reserved_MB: 23864.0 | inactive_split_MB: 293.8 +rank0 [2026-10-29 04:51:29,308] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:51:29,329] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-25000.pth +rank0 [2026-10-29 04:51:29,504] INFO : Running validation on rank0 +rank0 [2026-10-29 04:51:29,505] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 04:51:29,505] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 04:52:22,929] INFO : VALIDATION | Step=25000/50000 | Train Loss=1.1995 | Valid Loss=1.1824 | Train Loss Unweighted=0.5795 | Valid Loss Unweighted=0.5828 | Stale=1 | ETA=1059.0m +rank0 [2026-10-29 04:52:22,929] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:52:22,929] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 04:52:22,929] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 04:52:22,929] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:52:22,929] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/ari=0.509731 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.600683 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/top1_competing=0.687205 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.63216 +rank0 [2026-10-29 04:52:22,930] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0394219 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0576852 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/seconds_per_rank=21.2032 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 04:52:22,931] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding/ari=0.266639 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding_minus_raw/ari=0.175016 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 04:52:22,932] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 04:52:22,937] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:52:23,193] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 04:52:23,203] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 04:52:49,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 04:52:50,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 04:52:51,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 04:52:52,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 04:53:02,749] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 04:53:02,763] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:53:03,282] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 04:53:03,286] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:53:19,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 04:53:20,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 04:53:21,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 04:53:22,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 04:53:32,514] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 04:53:32,527] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:53:32,992] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 04:53:32,995] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 04:53:48,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 04:53:49,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 04:53:50,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 04:53:50,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 04:54:00,928] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 04:54:01,099] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits +rank0 [2026-10-29 04:54:04,288] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits +rank0 [2026-10-29 04:54:05,330] INFO : Plotted number of elements +rank0 [2026-10-29 04:54:06,877] INFO : Plotted jets +rank0 [2026-10-29 04:54:08,409] INFO : Plotted jet ratio +rank0 [2026-10-29 04:54:08,460] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:54:11,664] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:54:12,711] INFO : Plotted number of elements +rank0 [2026-10-29 04:54:14,271] INFO : Plotted jets +rank0 [2026-10-29 04:54:15,454] INFO : Plotted jet ratio +rank0 [2026-10-29 04:54:15,509] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits +rank0 [2026-10-29 04:54:17,948] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits +rank0 [2026-10-29 04:54:18,840] INFO : Plotted number of elements +rank0 [2026-10-29 04:54:20,332] INFO : Plotted jets +rank0 [2026-10-29 04:54:21,434] INFO : Plotted jet ratio +rank0 [2026-10-29 04:58:08,473] INFO : Step 25100/50000 rank0 | Train Loss: 1.0833 | Train Loss Unweighted: 0.5514 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1317 | Classification: 0.0920 | Classification_binary: 0.1568 | HitGrouping: 0.2008 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1491 | Regression_eta: 0.0016 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0604 | Regression_eta: 0.0538 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0437 | LR: 5.47e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4490s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7295 | max_loss: 0.1401 | max_backward: 1.2439 | max_optimizer_step: 0.2490 | max_tensorboard_logging: 0.4940 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8607.6 | uss_MB: 7944.9 | child_rss_MB: 5042.3 | allocated_MB: 342.3 | reserved_MB: 23608.0 | inactive_split_MB: 222.3 +rank0 [2026-10-29 04:58:08,718] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:01:54,081] INFO : Step 25200/50000 rank0 | Train Loss: 1.1886 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1523 | Classification: 0.1158 | Classification_binary: 0.1800 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1512 | Regression_eta: 0.0016 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.1947 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0785 | Regression_eta: 0.0443 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0457 | LR: 5.44e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.6086s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8134 | max_loss: 0.1416 | max_backward: 1.5095 | max_optimizer_step: 0.3471 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 8606.1 | uss_MB: 7945.9 | child_rss_MB: 5039.5 | allocated_MB: 343.6 | reserved_MB: 24590.0 | inactive_split_MB: 260.9 +rank0 [2026-10-29 05:01:54,324] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 94% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +6 10 0x7408, 62036 45.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 94% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 97% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:05:52,874] INFO : Step 25300/50000 rank0 | Train Loss: 1.2031 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2624 | Cardinality: 0.1455 | Classification: 0.1145 | Classification_binary: 0.1966 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1448 | Regression_eta: 0.0016 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0665 | Regression_eta: 0.0299 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0466 | LR: 5.42e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.1429s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.8219 | max_loss: 0.1491 | max_backward: 1.6406 | max_optimizer_step: 0.2735 | max_tensorboard_logging: 0.7204 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0488 | Memory: rss_MB: 7456.6 | uss_MB: 6796.5 | child_rss_MB: 5036.7 | allocated_MB: 341.8 | reserved_MB: 25572.0 | inactive_split_MB: 253.5 +rank0 [2026-10-29 05:05:53,126] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:09:45,726] INFO : Step 25400/50000 rank0 | Train Loss: 1.1194 | Train Loss Unweighted: 0.5655 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1346 | Classification: 0.0983 | Classification_binary: 0.1754 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1438 | Regression_eta: 0.0015 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0646 | Regression_eta: 0.0335 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0485 | LR: 5.39e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.8614s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.6986 | max_loss: 0.1530 | max_backward: 1.2606 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.8427 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7454.9 | uss_MB: 6795.9 | child_rss_MB: 5040.6 | allocated_MB: 345.2 | reserved_MB: 25572.0 | inactive_split_MB: 223.8 +rank0 [2026-10-29 05:09:45,974] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:13:34,732] INFO : Step 25500/50000 rank0 | Train Loss: 1.0289 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1068 | Classification: 0.1034 | Classification_binary: 0.1444 | HitGrouping: 0.1913 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1484 | Regression_eta: 0.0014 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.1750 | Regression_cos_phi: 0.0943 | Regression_energy: 0.0680 | Regression_eta: 0.0592 | Regression_pt: 0.0697 | Regression_sin_phi: 0.1118 | LR: 5.36e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.8363s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0554 | max_forward: 0.7008 | max_loss: 0.1433 | max_backward: 1.4571 | max_optimizer_step: 0.3349 | max_tensorboard_logging: 0.6083 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 7460.4 | uss_MB: 6802.0 | child_rss_MB: 5038.5 | allocated_MB: 345.4 | reserved_MB: 25572.0 | inactive_split_MB: 240.6 +rank0 [2026-10-29 05:13:34,976] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 45.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:17:32,956] INFO : Step 25600/50000 rank0 | Train Loss: 1.2028 | Train Loss Unweighted: 0.5968 | Train Loss Components: Auxiliary: 0.2590 | Cardinality: 0.1607 | Classification: 0.1135 | Classification_binary: 0.1926 | HitGrouping: 0.1978 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1429 | Regression_eta: 0.0016 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.2447 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0677 | Regression_eta: 0.0584 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0770 | LR: 5.33e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.4228s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8800 | max_loss: 0.1578 | max_backward: 1.8564 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 7459.9 | uss_MB: 6802.4 | child_rss_MB: 5035.4 | allocated_MB: 345.5 | reserved_MB: 26054.0 | inactive_split_MB: 284.5 +rank0 [2026-10-29 05:17:33,214] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 46.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:21:26,671] INFO : Step 25700/50000 rank0 | Train Loss: 1.1800 | Train Loss Unweighted: 0.5964 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1298 | Classification: 0.1086 | Classification_binary: 0.1961 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0019 | Regression_pt: 0.1444 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1691 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0838 | Regression_eta: 0.0468 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0588 | LR: 5.30e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.1888s | Phase Times: max_pre_step_barrier: 0.0086 | max_batch_to_device: 0.0137 | max_forward: 1.0340 | max_loss: 0.2483 | max_backward: 1.6704 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 0.7612 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7459.7 | uss_MB: 6801.4 | child_rss_MB: 5033.4 | allocated_MB: 343.5 | reserved_MB: 26054.0 | inactive_split_MB: 270.8 +rank0 [2026-10-29 05:21:26,917] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 45.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:25:17,170] INFO : Step 25800/50000 rank0 | Train Loss: 1.1337 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2388 | Cardinality: 0.1264 | Classification: 0.0898 | Classification_binary: 0.1617 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0019 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0996 | Regression_energy: 0.0760 | Regression_eta: 0.0842 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0940 | LR: 5.27e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.5809s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0986 | max_forward: 0.8129 | max_loss: 0.1662 | max_backward: 1.2979 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1103 | Memory: rss_MB: 7457.8 | uss_MB: 6799.8 | child_rss_MB: 5030.2 | allocated_MB: 341.8 | reserved_MB: 26054.0 | inactive_split_MB: 273.2 +rank0 [2026-10-29 05:25:17,428] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 46.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:29:08,853] INFO : Step 25900/50000 rank0 | Train Loss: 1.1019 | Train Loss Unweighted: 0.5481 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1165 | Classification: 0.0977 | Classification_binary: 0.1594 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1434 | Regression_eta: 0.0016 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1976 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0709 | Regression_eta: 0.0440 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0621 | LR: 5.25e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1611s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7034 | max_loss: 0.2469 | max_backward: 1.7130 | max_optimizer_step: 0.2363 | max_tensorboard_logging: 0.8382 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 7454.8 | uss_MB: 6800.6 | child_rss_MB: 5037.1 | allocated_MB: 343.0 | reserved_MB: 26054.0 | inactive_split_MB: 187.0 +rank0 [2026-10-29 05:29:09,097] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 46.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:33:08,134] INFO : Step 26000/50000 rank0 | Train Loss: 1.1708 | Train Loss Unweighted: 0.5680 | Train Loss Components: Auxiliary: 0.2498 | Cardinality: 0.1352 | Classification: 0.0999 | Classification_binary: 0.2021 | HitGrouping: 0.1994 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1304 | Regression_eta: 0.0021 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1964 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0615 | Regression_eta: 0.0495 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0648 | LR: 5.22e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.8796s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.7877 | max_loss: 0.1637 | max_backward: 1.6550 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.4954 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 7454.4 | uss_MB: 6802.1 | child_rss_MB: 5032.6 | allocated_MB: 345.0 | reserved_MB: 27098.0 | inactive_split_MB: 285.6 +rank0 [2026-10-29 05:33:08,392] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 46.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:33:08,413] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-26000.pth +rank0 [2026-10-29 05:33:08,586] INFO : Running validation on rank0 +rank0 [2026-10-29 05:33:08,586] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 05:33:08,586] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 05:33:57,406] INFO : VALIDATION | Step=26000/50000 | Train Loss=1.1708 | Valid Loss=1.1675 | Train Loss Unweighted=0.5680 | Valid Loss Unweighted=0.5767 | Stale=0 | ETA=1015.9m +rank0 [2026-10-29 05:33:57,406] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 05:33:57,406] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 05:33:57,406] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/ari=0.506927 +rank0 [2026-10-29 05:33:57,407] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.607351 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/top1_competing=0.696367 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.626024 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0422255 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0515492 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 05:33:57,408] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/seconds_per_rank=21.0823 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding/ari=0.281125 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding_minus_raw/ari=0.189502 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 05:33:57,409] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 05:33:57,410] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 05:33:57,415] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:33:57,679] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 05:33:57,690] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 05:34:24,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 05:34:25,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 05:34:26,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 05:34:27,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 05:34:37,793] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 05:34:38,072] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:34:38,538] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 05:34:38,542] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 05:34:54,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 05:34:55,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 05:34:56,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 05:34:58,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 05:35:08,017] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 05:35:08,028] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:35:08,510] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 05:35:08,514] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 05:35:24,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 05:35:24,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 05:35:25,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 05:35:26,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 05:35:36,462] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 05:35:36,600] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits +rank0 [2026-10-29 05:35:39,812] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits +rank0 [2026-10-29 05:35:40,860] INFO : Plotted number of elements +rank0 [2026-10-29 05:35:42,406] INFO : Plotted jets +rank0 [2026-10-29 05:35:43,556] INFO : Plotted jet ratio +rank0 [2026-10-29 05:35:43,614] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 05:35:46,813] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 05:35:47,839] INFO : Plotted number of elements +rank0 [2026-10-29 05:35:49,777] INFO : Plotted jets +rank0 [2026-10-29 05:35:50,892] INFO : Plotted jet ratio +rank0 [2026-10-29 05:35:50,947] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits +rank0 [2026-10-29 05:35:53,322] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits +rank0 [2026-10-29 05:35:54,163] INFO : Plotted number of elements +rank0 [2026-10-29 05:35:55,644] INFO : Plotted jets +rank0 [2026-10-29 05:35:56,790] INFO : Plotted jet ratio +rank0 [2026-10-29 05:39:50,290] INFO : Step 26100/50000 rank0 | Train Loss: 1.0974 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2307 | Cardinality: 0.1119 | Classification: 0.0956 | Classification_binary: 0.1689 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1513 | Regression_eta: 0.0014 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1657 | Regression_cos_phi: 0.1286 | Regression_energy: 0.0888 | Regression_eta: 0.0502 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0669 | LR: 5.19e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8219s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0117 | max_forward: 0.7200 | max_loss: 0.1501 | max_backward: 1.2286 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.8284 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 8630.6 | uss_MB: 7973.7 | child_rss_MB: 5008.3 | allocated_MB: 340.1 | reserved_MB: 21762.0 | inactive_split_MB: 215.7 +rank0 [2026-10-29 05:39:50,531] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:43:57,378] INFO : Step 26200/50000 rank0 | Train Loss: 1.0876 | Train Loss Unweighted: 0.5502 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1065 | Classification: 0.0968 | Classification_binary: 0.1527 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1476 | Regression_eta: 0.0018 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1526 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0582 | Regression_eta: 0.0353 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0694 | LR: 5.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1635s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6530 | max_loss: 0.1421 | max_backward: 1.3973 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.8695 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8630.6 | uss_MB: 7973.8 | child_rss_MB: 5031.7 | allocated_MB: 342.3 | reserved_MB: 22678.0 | inactive_split_MB: 242.5 +rank0 [2026-10-29 05:43:57,642] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 2% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +4 8 0x7408, 13395 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 46.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:47:48,682] INFO : Step 26300/50000 rank0 | Train Loss: 1.2051 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1512 | Classification: 0.1193 | Classification_binary: 0.1877 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1458 | Regression_eta: 0.0017 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0872 | Classification_binary: 0.2270 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0695 | Regression_eta: 0.0466 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0815 | LR: 5.13e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.1032s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0072 | max_forward: 0.7601 | max_loss: 0.2267 | max_backward: 1.8902 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7522.0 | uss_MB: 6871.8 | child_rss_MB: 4996.1 | allocated_MB: 345.4 | reserved_MB: 22678.0 | inactive_split_MB: 270.9 +rank0 [2026-10-29 05:47:48,936] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:51:48,030] INFO : Step 26400/50000 rank0 | Train Loss: 1.2256 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1648 | Classification: 0.1020 | Classification_binary: 0.1981 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1520 | Regression_eta: 0.0015 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0663 | Regression_eta: 0.0400 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0531 | LR: 5.10e-04 | DataLoad Time: 0.0087s | Model Forward Time: 5.1675s | Phase Times: max_pre_step_barrier: 0.0351 | max_batch_to_device: 0.0070 | max_forward: 0.8129 | max_loss: 0.1477 | max_backward: 1.4015 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8384 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 7520.2 | uss_MB: 6869.2 | child_rss_MB: 4985.0 | allocated_MB: 342.8 | reserved_MB: 22678.0 | inactive_split_MB: 196.9 +rank0 [2026-10-29 05:51:48,278] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 45.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:55:41,400] INFO : Step 26500/50000 rank0 | Train Loss: 1.1110 | Train Loss Unweighted: 0.5356 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1488 | Classification: 0.0891 | Classification_binary: 0.1546 | HitGrouping: 0.2012 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0894 | Regression_eta: 0.0616 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0607 | LR: 5.08e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6612s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6841 | max_loss: 0.2421 | max_backward: 1.1616 | max_optimizer_step: 0.2383 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7505.4 | uss_MB: 6872.6 | child_rss_MB: 4957.6 | allocated_MB: 340.6 | reserved_MB: 22678.0 | inactive_split_MB: 269.9 +rank0 [2026-10-29 05:55:41,647] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:59:33,794] INFO : Step 26600/50000 rank0 | Train Loss: 1.0895 | Train Loss Unweighted: 0.5529 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1261 | Classification: 0.1083 | Classification_binary: 0.1603 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1386 | Regression_eta: 0.0015 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0548 | Regression_eta: 0.0312 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0704 | LR: 5.05e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.1545s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0696 | max_forward: 0.6926 | max_loss: 0.1459 | max_backward: 1.4149 | max_optimizer_step: 0.2385 | max_tensorboard_logging: 0.8927 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7477.7 | uss_MB: 6873.1 | child_rss_MB: 4831.1 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 232.0 +rank0 [2026-10-29 05:59:34,040] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 3% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +2 6 0x7408, 15961 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 17% +4 8 0x7408, 13395 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 6% +6 10 0x7408, 62036 45.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 79% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 38% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:03:24,390] INFO : Step 26700/50000 rank0 | Train Loss: 1.2592 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1717 | Classification: 0.1100 | Classification_binary: 0.2137 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1420 | Regression_eta: 0.0017 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0555 | Regression_eta: 0.0406 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0677 | LR: 5.02e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.3093s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7750 | max_loss: 0.1526 | max_backward: 1.6427 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.8866 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7338.5 | uss_MB: 6135.7 | child_rss_MB: 4788.2 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 267.4 +rank0 [2026-10-29 06:03:24,635] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 19% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:07:17,338] INFO : Step 26800/50000 rank0 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5792 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1288 | Classification: 0.0911 | Classification_binary: 0.1633 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1600 | Regression_eta: 0.0015 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0835 | Regression_eta: 0.0448 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0461 | LR: 4.99e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6757s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7108 | max_loss: 0.1467 | max_backward: 1.2855 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7173 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 7253.0 | uss_MB: 6872.2 | child_rss_MB: 4809.0 | allocated_MB: 340.8 | reserved_MB: 22846.0 | inactive_split_MB: 253.2 +rank0 [2026-10-29 06:07:17,601] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:11:10,354] INFO : Step 26900/50000 rank0 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.0996 | Classification: 0.0867 | Classification_binary: 0.1428 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1483 | Regression_eta: 0.0018 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1644 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0793 | Regression_eta: 0.0428 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0729 | LR: 4.96e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5409s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0336 | max_forward: 0.6534 | max_loss: 0.1408 | max_backward: 1.3898 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0912 | Memory: rss_MB: 7214.0 | uss_MB: 6872.7 | child_rss_MB: 4790.0 | allocated_MB: 342.5 | reserved_MB: 22846.0 | inactive_split_MB: 233.5 +rank0 [2026-10-29 06:11:10,603] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 45.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:15:06,077] INFO : Step 27000/50000 rank0 | Train Loss: 1.0945 | Train Loss Unweighted: 0.5396 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1196 | Classification: 0.0957 | Classification_binary: 0.1912 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1247 | Regression_eta: 0.0015 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0605 | Regression_eta: 0.0408 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0630 | LR: 4.94e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.8600s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7531 | max_loss: 0.1689 | max_backward: 1.5554 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6017 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7214.1 | uss_MB: 6877.0 | child_rss_MB: 4792.2 | allocated_MB: 345.3 | reserved_MB: 23694.0 | inactive_split_MB: 251.5 +rank0 [2026-10-29 06:15:06,320] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 94% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +4 8 0x7408, 13395 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 46.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 72% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:15:06,410] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-27000.pth +rank0 [2026-10-29 06:15:06,676] INFO : Running validation on rank0 +rank0 [2026-10-29 06:15:06,676] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 06:15:06,676] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 06:16:00,820] INFO : VALIDATION | Step=27000/50000 | Train Loss=1.0945 | Valid Loss=1.1342 | Train Loss Unweighted=0.5396 | Valid Loss Unweighted=0.5639 | Stale=0 | ETA=973.4m +rank0 [2026-10-29 06:16:00,820] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 06:16:00,820] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 06:16:00,820] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 06:16:00,820] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 06:16:00,820] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 06:16:00,820] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 06:16:00,820] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 06:16:00,820] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 06:16:00,820] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/ari=0.560598 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.618874 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/top1_competing=0.710566 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.619367 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0114448 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.044892 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 06:16:00,821] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/seconds_per_rank=21.2462 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding/ari=0.2846 +rank0 [2026-10-29 06:16:00,822] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 06:16:00,823] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding_minus_raw/ari=0.192977 +rank0 [2026-10-29 06:16:00,823] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 06:16:00,823] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 06:16:00,823] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 06:16:00,829] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:16:01,085] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 06:16:01,101] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 06:16:29,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 06:16:30,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 06:16:32,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 06:16:33,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 06:16:43,142] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 06:16:43,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:16:43,719] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 06:16:43,723] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 06:16:59,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 06:17:01,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 06:17:02,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 06:17:03,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 06:17:13,237] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 06:17:13,248] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:17:13,777] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 06:17:13,781] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 06:17:29,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 06:17:30,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 06:17:30,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 06:17:31,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 06:17:41,703] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 06:17:41,719] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits +rank0 [2026-10-29 06:17:44,916] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits +rank0 [2026-10-29 06:17:45,948] INFO : Plotted number of elements +rank0 [2026-10-29 06:17:47,506] INFO : Plotted jets +rank0 [2026-10-29 06:17:48,662] INFO : Plotted jet ratio +rank0 [2026-10-29 06:17:48,731] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 06:17:51,991] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 06:17:53,033] INFO : Plotted number of elements +rank0 [2026-10-29 06:17:54,530] INFO : Plotted jets +rank0 [2026-10-29 06:17:56,051] INFO : Plotted jet ratio +rank0 [2026-10-29 06:17:56,137] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits +rank0 [2026-10-29 06:17:58,504] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits +rank0 [2026-10-29 06:17:59,342] INFO : Plotted number of elements +rank0 [2026-10-29 06:18:00,822] INFO : Plotted jets +rank0 [2026-10-29 06:18:01,967] INFO : Plotted jet ratio +rank0 [2026-10-29 06:21:52,626] INFO : Step 27100/50000 rank0 | Train Loss: 1.1577 | Train Loss Unweighted: 0.5595 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1497 | Classification: 0.0884 | Classification_binary: 0.1879 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1399 | Regression_eta: 0.0018 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1894 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0818 | Regression_eta: 0.0412 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0662 | LR: 4.91e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.6994s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8543 | max_loss: 0.1477 | max_backward: 1.8762 | max_optimizer_step: 0.2540 | max_tensorboard_logging: 0.9462 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 8306.7 | uss_MB: 7966.7 | child_rss_MB: 4814.1 | allocated_MB: 342.9 | reserved_MB: 21640.0 | inactive_split_MB: 268.0 +rank0 [2026-10-29 06:21:52,870] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:25:40,802] INFO : Step 27200/50000 rank0 | Train Loss: 1.0028 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1065 | Classification: 0.0834 | Classification_binary: 0.1354 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1443 | Regression_eta: 0.0015 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1396 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0626 | Regression_eta: 0.0920 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0777 | LR: 4.88e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.9166s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6839 | max_loss: 0.1724 | max_backward: 1.0628 | max_optimizer_step: 0.2515 | max_tensorboard_logging: 0.9641 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0689 | Memory: rss_MB: 8305.7 | uss_MB: 7964.4 | child_rss_MB: 4815.8 | allocated_MB: 340.5 | reserved_MB: 22612.0 | inactive_split_MB: 267.2 +rank0 [2026-10-29 06:25:41,056] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 4% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 83% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 9% +6 10 0x7408, 62036 46.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 1% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 74% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:29:19,389] INFO : Step 27300/50000 rank0 | Train Loss: 1.1913 | Train Loss Unweighted: 0.5857 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1563 | Classification: 0.1169 | Classification_binary: 0.1769 | HitGrouping: 0.1965 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1426 | Regression_eta: 0.0019 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1897 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0635 | Regression_eta: 0.0520 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0569 | LR: 4.85e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.8818s | Phase Times: max_pre_step_barrier: 0.0244 | max_batch_to_device: 0.0064 | max_forward: 0.6813 | max_loss: 0.1454 | max_backward: 1.4618 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7699 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 7290.8 | uss_MB: 6952.0 | child_rss_MB: 4811.5 | allocated_MB: 345.6 | reserved_MB: 23586.0 | inactive_split_MB: 267.9 +rank0 [2026-10-29 06:29:19,634] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:33:04,051] INFO : Step 27400/50000 rank0 | Train Loss: 1.1723 | Train Loss Unweighted: 0.5888 | Train Loss Components: Auxiliary: 0.2572 | Cardinality: 0.1400 | Classification: 0.1088 | Classification_binary: 0.2016 | HitGrouping: 0.1987 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1370 | Regression_eta: 0.0017 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1930 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0585 | Regression_eta: 0.0455 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0714 | LR: 4.82e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2887s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8048 | max_loss: 0.1475 | max_backward: 1.6655 | max_optimizer_step: 0.2279 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7292.0 | uss_MB: 6951.9 | child_rss_MB: 4814.0 | allocated_MB: 344.5 | reserved_MB: 23586.0 | inactive_split_MB: 279.5 +rank0 [2026-10-29 06:33:04,296] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:37:02,091] INFO : Step 27500/50000 rank0 | Train Loss: 1.0739 | Train Loss Unweighted: 0.5273 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1251 | Classification: 0.0933 | Classification_binary: 0.1655 | HitGrouping: 0.1957 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0650 | Regression_eta: 0.0381 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0794 | LR: 4.80e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8290s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7686 | max_loss: 0.1542 | max_backward: 1.3750 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.8628 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 7288.8 | uss_MB: 6945.7 | child_rss_MB: 4804.4 | allocated_MB: 343.6 | reserved_MB: 23586.0 | inactive_split_MB: 241.4 +rank0 [2026-10-29 06:37:02,337] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 41% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 93% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 46.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 98% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 62% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:40:56,246] INFO : Step 27600/50000 rank0 | Train Loss: 1.0507 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1163 | Classification: 0.1024 | Classification_binary: 0.1467 | HitGrouping: 0.1992 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1531 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0669 | Regression_eta: 0.0378 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0514 | LR: 4.77e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.2143s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0060 | max_forward: 0.6744 | max_loss: 0.2492 | max_backward: 1.6867 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.5257 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 7290.8 | uss_MB: 6853.7 | child_rss_MB: 4808.0 | allocated_MB: 341.2 | reserved_MB: 24480.0 | inactive_split_MB: 234.8 +rank0 [2026-10-29 06:40:56,488] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 11% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 91% +2 6 0x7408, 15961 49.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 44% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 45.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 1% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:44:48,587] INFO : Step 27700/50000 rank0 | Train Loss: 1.1881 | Train Loss Unweighted: 0.5834 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1544 | Classification: 0.1001 | Classification_binary: 0.1885 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1444 | Regression_eta: 0.0015 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1846 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0605 | Regression_eta: 0.0491 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0718 | LR: 4.74e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.0082s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7716 | max_loss: 0.1527 | max_backward: 1.4941 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 0.7272 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7291.1 | uss_MB: 6952.7 | child_rss_MB: 4804.6 | allocated_MB: 344.5 | reserved_MB: 24480.0 | inactive_split_MB: 194.2 +rank0 [2026-10-29 06:44:48,832] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 46.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:48:39,636] INFO : Step 27800/50000 rank0 | Train Loss: 1.1903 | Train Loss Unweighted: 0.5862 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1435 | Classification: 0.1032 | Classification_binary: 0.2006 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1387 | Regression_eta: 0.0018 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.1758 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0668 | Regression_eta: 0.0331 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0873 | LR: 4.71e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8920s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8137 | max_loss: 0.1509 | max_backward: 1.4748 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6041 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7291.1 | uss_MB: 6953.1 | child_rss_MB: 4808.7 | allocated_MB: 348.4 | reserved_MB: 24592.0 | inactive_split_MB: 226.8 +rank0 [2026-10-29 06:48:39,881] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 46.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:52:39,865] INFO : Step 27900/50000 rank0 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5521 | Train Loss Components: Auxiliary: 0.2284 | Cardinality: 0.1139 | Classification: 0.0902 | Classification_binary: 0.1544 | HitGrouping: 0.2057 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1519 | Regression_eta: 0.0017 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1564 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0750 | Regression_eta: 0.0336 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0606 | LR: 4.68e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.3891s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6397 | max_loss: 0.2456 | max_backward: 1.5234 | max_optimizer_step: 0.2276 | max_tensorboard_logging: 1.1626 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 7291.1 | uss_MB: 6955.2 | child_rss_MB: 4805.2 | allocated_MB: 342.4 | reserved_MB: 25612.0 | inactive_split_MB: 259.0 +rank0 [2026-10-29 06:52:40,115] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 45.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:56:31,517] INFO : Step 28000/50000 rank0 | Train Loss: 1.1111 | Train Loss Unweighted: 0.5401 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1274 | Classification: 0.1035 | Classification_binary: 0.1699 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1960 | Regression_cos_phi: 0.1000 | Regression_energy: 0.0609 | Regression_eta: 0.0549 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0677 | LR: 4.66e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.8422s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.7503 | max_loss: 0.1492 | max_backward: 1.7566 | max_optimizer_step: 0.2764 | max_tensorboard_logging: 1.2295 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 7291.2 | uss_MB: 5984.8 | child_rss_MB: 4794.8 | allocated_MB: 344.1 | reserved_MB: 25612.0 | inactive_split_MB: 243.2 +rank0 [2026-10-29 06:56:31,770] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 45.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 13% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:56:31,863] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-28000.pth +rank0 [2026-10-29 06:56:32,074] INFO : Running validation on rank0 +rank0 [2026-10-29 06:56:32,075] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 06:56:32,075] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 06:57:25,535] INFO : VALIDATION | Step=28000/50000 | Train Loss=1.1111 | Valid Loss=1.1212 | Train Loss Unweighted=0.5401 | Valid Loss Unweighted=0.5561 | Stale=0 | ETA=930.3m +rank0 [2026-10-29 06:57:25,535] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 06:57:25,535] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 06:57:25,535] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 06:57:25,535] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 06:57:25,535] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 06:57:25,535] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 06:57:25,535] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 06:57:25,535] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 06:57:25,535] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/ari=0.509386 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.59736 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/top1_competing=0.681685 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.615532 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.039767 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0410573 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 06:57:25,536] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/seconds_per_rank=21.6552 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 06:57:25,537] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 06:57:25,538] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 06:57:25,538] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding/ari=0.279933 +rank0 [2026-10-29 06:57:25,538] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 06:57:25,538] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding_minus_raw/ari=0.18831 +rank0 [2026-10-29 06:57:25,538] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 06:57:25,538] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 06:57:25,538] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 06:57:25,544] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:57:25,923] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 06:57:25,927] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 06:57:52,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 06:57:53,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 06:57:54,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 06:57:55,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 06:58:05,585] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 06:58:05,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:58:06,180] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 06:58:06,183] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 06:58:21,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 06:58:23,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 06:58:24,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 06:58:25,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 06:58:35,247] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 06:58:35,258] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:58:35,800] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 06:58:35,804] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 06:58:51,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 06:58:52,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 06:58:53,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 06:58:54,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 06:59:04,186] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 06:59:04,203] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits +rank0 [2026-10-29 06:59:07,446] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits +rank0 [2026-10-29 06:59:08,500] INFO : Plotted number of elements +rank0 [2026-10-29 06:59:10,041] INFO : Plotted jets +rank0 [2026-10-29 06:59:11,178] INFO : Plotted jet ratio +rank0 [2026-10-29 06:59:11,229] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 06:59:14,482] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 06:59:15,507] INFO : Plotted number of elements +rank0 [2026-10-29 06:59:16,999] INFO : Plotted jets +rank0 [2026-10-29 06:59:18,536] INFO : Plotted jet ratio +rank0 [2026-10-29 06:59:18,590] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits +rank0 [2026-10-29 06:59:20,959] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits +rank0 [2026-10-29 06:59:21,815] INFO : Plotted number of elements +rank0 [2026-10-29 06:59:23,327] INFO : Plotted jets +rank0 [2026-10-29 06:59:24,489] INFO : Plotted jet ratio +rank0 [2026-10-29 07:03:11,903] INFO : Step 28100/50000 rank0 | Train Loss: 1.1727 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1644 | Classification: 0.0934 | Classification_binary: 0.1929 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1365 | Regression_eta: 0.0015 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1734 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0688 | Regression_eta: 0.0391 | Regression_pt: 0.0723 | Regression_sin_phi: 0.1096 | LR: 4.63e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.6003s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8425 | max_loss: 0.2571 | max_backward: 1.8157 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 1.0134 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 8340.2 | uss_MB: 8005.6 | child_rss_MB: 4802.9 | allocated_MB: 344.8 | reserved_MB: 22140.0 | inactive_split_MB: 293.2 +rank0 [2026-10-29 07:03:12,148] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 45.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:07:01,807] INFO : Step 28200/50000 rank0 | Train Loss: 1.0520 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2296 | Cardinality: 0.1203 | Classification: 0.0934 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1466 | Regression_eta: 0.0014 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1349 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0671 | Regression_eta: 0.0392 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0468 | LR: 4.60e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9999s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7943 | max_loss: 0.1496 | max_backward: 1.4503 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 0.7872 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 8343.2 | uss_MB: 8006.4 | child_rss_MB: 4796.0 | allocated_MB: 341.0 | reserved_MB: 24034.0 | inactive_split_MB: 210.4 +rank0 [2026-10-29 07:07:02,058] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 46.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:10:53,414] INFO : Step 28300/50000 rank0 | Train Loss: 1.0602 | Train Loss Unweighted: 0.5453 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1043 | Classification: 0.0923 | Classification_binary: 0.1501 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1496 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1683 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0644 | Regression_eta: 0.0483 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0585 | LR: 4.57e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8880s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6482 | max_loss: 0.2266 | max_backward: 1.5658 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.6870 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0836 | Memory: rss_MB: 7393.0 | uss_MB: 7059.5 | child_rss_MB: 4787.3 | allocated_MB: 342.7 | reserved_MB: 25016.0 | inactive_split_MB: 243.9 +rank0 [2026-10-29 07:10:53,672] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 46.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:14:41,256] INFO : Step 28400/50000 rank0 | Train Loss: 1.1507 | Train Loss Unweighted: 0.5391 | Train Loss Components: Auxiliary: 0.2436 | Cardinality: 0.1500 | Classification: 0.0965 | Classification_binary: 0.1896 | HitGrouping: 0.1911 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1241 | Regression_eta: 0.0018 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1868 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0605 | Regression_eta: 0.0603 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0628 | LR: 4.55e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.3107s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7664 | max_loss: 0.1532 | max_backward: 1.7342 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.8455 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7394.0 | uss_MB: 7059.9 | child_rss_MB: 4785.1 | allocated_MB: 344.2 | reserved_MB: 25016.0 | inactive_split_MB: 278.9 +rank0 [2026-10-29 07:14:41,525] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +6 10 0x7408, 62036 46.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:18:29,665] INFO : Step 28500/50000 rank0 | Train Loss: 1.1595 | Train Loss Unweighted: 0.5640 | Train Loss Components: Auxiliary: 0.2492 | Cardinality: 0.1556 | Classification: 0.0927 | Classification_binary: 0.1908 | HitGrouping: 0.1954 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1384 | Regression_eta: 0.0015 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0532 | Regression_eta: 0.0477 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0560 | LR: 4.52e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9303s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.8033 | max_loss: 0.1665 | max_backward: 1.5231 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.6178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0709 | Memory: rss_MB: 7393.5 | uss_MB: 7056.4 | child_rss_MB: 4808.7 | allocated_MB: 347.4 | reserved_MB: 26000.0 | inactive_split_MB: 283.2 +rank0 [2026-10-29 07:18:29,929] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:22:18,442] INFO : Step 28600/50000 rank0 | Train Loss: 1.0672 | Train Loss Unweighted: 0.5222 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1172 | Classification: 0.0893 | Classification_binary: 0.1544 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0752 | Regression_eta: 0.0542 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0524 | LR: 4.49e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6802s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6704 | max_loss: 0.1829 | max_backward: 1.2353 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.8074 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7396.5 | uss_MB: 7057.8 | child_rss_MB: 4793.1 | allocated_MB: 342.3 | reserved_MB: 26000.0 | inactive_split_MB: 243.7 +rank0 [2026-10-29 07:22:18,699] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:26:10,879] INFO : Step 28700/50000 rank0 | Train Loss: 1.1535 | Train Loss Unweighted: 0.5811 | Train Loss Components: Auxiliary: 0.2469 | Cardinality: 0.1254 | Classification: 0.1220 | Classification_binary: 0.1661 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1442 | Regression_eta: 0.0021 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.1888 | Regression_cos_phi: 0.0801 | Regression_energy: 0.0653 | Regression_eta: 0.0568 | Regression_pt: 0.0692 | Regression_sin_phi: 0.1000 | LR: 4.46e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.3068s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6905 | max_loss: 0.2072 | max_backward: 1.6876 | max_optimizer_step: 0.2353 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 7397.5 | uss_MB: 7064.1 | child_rss_MB: 4802.4 | allocated_MB: 343.5 | reserved_MB: 26000.0 | inactive_split_MB: 234.1 +rank0 [2026-10-29 07:26:11,121] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:30:12,847] INFO : Step 28800/50000 rank0 | Train Loss: 1.2131 | Train Loss Unweighted: 0.5739 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1788 | Classification: 0.1057 | Classification_binary: 0.2127 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.2191 | Regression_cos_phi: 0.0947 | Regression_energy: 0.0626 | Regression_eta: 0.0646 | Regression_pt: 0.0647 | Regression_sin_phi: 0.1061 | LR: 4.44e-04 | DataLoad Time: 0.0089s | Model Forward Time: 7.5679s | Phase Times: max_pre_step_barrier: 0.0263 | max_batch_to_device: 0.0149 | max_forward: 1.2928 | max_loss: 0.2713 | max_backward: 3.0161 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.8895 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 7398.3 | uss_MB: 5722.3 | child_rss_MB: 4796.9 | allocated_MB: 343.8 | reserved_MB: 26000.0 | inactive_split_MB: 273.3 +rank0 [2026-10-29 07:30:13,092] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 50.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:34:12,373] INFO : Step 28900/50000 rank0 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1252 | Classification: 0.0967 | Classification_binary: 0.1637 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1509 | Regression_eta: 0.0015 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1725 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0675 | Regression_eta: 0.0432 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0684 | LR: 4.41e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.9541s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7512 | max_loss: 0.2189 | max_backward: 1.6229 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.9330 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7399.6 | uss_MB: 5634.1 | child_rss_MB: 4800.1 | allocated_MB: 339.6 | reserved_MB: 26000.0 | inactive_split_MB: 223.0 +rank0 [2026-10-29 07:34:12,618] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 4% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 1% +2 6 0x7408, 15961 49.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 96% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 45.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:38:06,786] INFO : Step 29000/50000 rank0 | Train Loss: 1.0238 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1070 | Classification: 0.0861 | Classification_binary: 0.1439 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1389 | Regression_eta: 0.0014 | Regression_pt: 0.1414 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0740 | Regression_eta: 0.0461 | Regression_pt: 0.0769 | Regression_sin_phi: 0.1123 | LR: 4.38e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6863s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.6636 | max_loss: 0.2429 | max_backward: 1.5394 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.5216 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7400.5 | uss_MB: 7067.2 | child_rss_MB: 4794.8 | allocated_MB: 345.2 | reserved_MB: 26984.0 | inactive_split_MB: 235.2 +rank0 [2026-10-29 07:38:07,027] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:38:07,050] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-29000.pth +rank0 [2026-10-29 07:38:07,225] INFO : Running validation on rank0 +rank0 [2026-10-29 07:38:07,225] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 07:38:07,225] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 07:38:58,470] INFO : VALIDATION | Step=29000/50000 | Train Loss=1.0238 | Valid Loss=1.1174 | Train Loss Unweighted=0.5134 | Valid Loss Unweighted=0.5479 | Stale=0 | ETA=887.5m +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 07:38:58,471] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/ari=0.534344 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.630284 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/top1_competing=0.723591 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.610696 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0148084 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0362216 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 07:38:58,472] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/seconds_per_rank=21.2727 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:38:58,473] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding/ari=0.266255 +rank0 [2026-10-29 07:38:58,474] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding/ari_threshold=0.9 +rank0 [2026-10-29 07:38:58,474] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding_minus_raw/ari=0.174632 +rank0 [2026-10-29 07:38:58,474] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 07:38:58,474] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 07:38:58,474] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 07:38:58,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:38:58,733] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 07:38:58,743] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 07:39:24,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 07:39:25,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 07:39:26,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 07:39:28,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 07:39:38,013] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 07:39:38,025] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:39:38,576] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 07:39:38,580] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 07:39:54,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 07:39:55,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 07:39:56,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 07:39:57,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 07:40:07,768] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 07:40:07,779] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:40:08,277] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 07:40:08,281] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 07:40:23,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 07:40:24,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 07:40:25,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 07:40:26,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 07:40:36,250] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 07:40:36,344] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits +rank0 [2026-10-29 07:40:39,591] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits +rank0 [2026-10-29 07:40:40,589] INFO : Plotted number of elements +rank0 [2026-10-29 07:40:42,142] INFO : Plotted jets +rank0 [2026-10-29 07:40:43,287] INFO : Plotted jet ratio +rank0 [2026-10-29 07:40:43,425] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 07:40:46,757] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 07:40:47,800] INFO : Plotted number of elements +rank0 [2026-10-29 07:40:49,308] INFO : Plotted jets +rank0 [2026-10-29 07:40:50,929] INFO : Plotted jet ratio +rank0 [2026-10-29 07:40:50,998] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits +rank0 [2026-10-29 07:40:53,488] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits +rank0 [2026-10-29 07:40:54,378] INFO : Plotted number of elements +rank0 [2026-10-29 07:40:56,031] INFO : Plotted jets +rank0 [2026-10-29 07:40:57,187] INFO : Plotted jet ratio +rank0 [2026-10-29 07:45:02,690] INFO : Step 29100/50000 rank0 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5808 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1279 | Classification: 0.1113 | Classification_binary: 0.1973 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1339 | Regression_eta: 0.0017 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0799 | Regression_energy: 0.0670 | Regression_eta: 0.0634 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0673 | LR: 4.35e-04 | DataLoad Time: 0.0146s | Model Forward Time: 5.5657s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.8060 | max_loss: 0.1503 | max_backward: 1.9210 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 8379.4 | uss_MB: 8043.2 | child_rss_MB: 4787.2 | allocated_MB: 344.6 | reserved_MB: 24404.0 | inactive_split_MB: 274.9 +rank0 [2026-10-29 07:45:02,937] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 24% 100% +2 6 0x7408, 15961 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +4 8 0x7408, 13395 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +6 10 0x7408, 62036 45.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:48:58,741] INFO : Step 29200/50000 rank0 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5633 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1265 | Classification: 0.0969 | Classification_binary: 0.1883 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1368 | Regression_eta: 0.0014 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2041 | Regression_cos_phi: 0.0857 | Regression_energy: 0.0758 | Regression_eta: 0.0521 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0858 | LR: 4.33e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.5217s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0742 | max_forward: 1.0303 | max_loss: 0.2183 | max_backward: 1.8745 | max_optimizer_step: 0.2501 | max_tensorboard_logging: 0.5570 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8379.4 | uss_MB: 8042.7 | child_rss_MB: 4798.0 | allocated_MB: 343.1 | reserved_MB: 24404.0 | inactive_split_MB: 228.1 +rank0 [2026-10-29 07:48:58,984] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 49.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 45.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:52:52,721] INFO : Step 29300/50000 rank0 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2330 | Cardinality: 0.1231 | Classification: 0.0898 | Classification_binary: 0.1516 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1428 | Regression_eta: 0.0017 | Regression_pt: 0.1453 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0647 | Regression_eta: 0.0363 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0436 | LR: 4.30e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.6048s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0067 | max_forward: 0.7139 | max_loss: 0.1578 | max_backward: 1.5096 | max_optimizer_step: 0.2586 | max_tensorboard_logging: 1.1324 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7473.7 | uss_MB: 6607.6 | child_rss_MB: 4797.1 | allocated_MB: 341.0 | reserved_MB: 24404.0 | inactive_split_MB: 247.0 +rank0 [2026-10-29 07:52:52,985] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 94% +2 6 0x7408, 15961 48.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 46.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 1% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:56:45,101] INFO : Step 29400/50000 rank0 | Train Loss: 1.1400 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1377 | Classification: 0.1135 | Classification_binary: 0.1661 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1386 | Regression_eta: 0.0019 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1672 | Regression_cos_phi: 0.0935 | Regression_energy: 0.0721 | Regression_eta: 0.0474 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0605 | LR: 4.27e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.0194s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.6285 | max_loss: 0.1485 | max_backward: 1.3787 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.9179 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7474.7 | uss_MB: 7139.2 | child_rss_MB: 4788.9 | allocated_MB: 343.3 | reserved_MB: 24404.0 | inactive_split_MB: 277.1 +rank0 [2026-10-29 07:56:45,343] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +2 6 0x7408, 15961 49.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:00:59,379] INFO : Step 29500/50000 rank0 | Train Loss: 1.1987 | Train Loss Unweighted: 0.6106 | Train Loss Components: Auxiliary: 0.2551 | Cardinality: 0.1277 | Classification: 0.1201 | Classification_binary: 0.2013 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1419 | Regression_eta: 0.0019 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.2072 | Regression_cos_phi: 0.1008 | Regression_energy: 0.0686 | Regression_eta: 0.0527 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0946 | LR: 4.24e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.6313s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0749 | max_forward: 0.8153 | max_loss: 0.3465 | max_backward: 2.8565 | max_optimizer_step: 0.3442 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7477.7 | uss_MB: 7142.6 | child_rss_MB: 4792.1 | allocated_MB: 343.9 | reserved_MB: 24404.0 | inactive_split_MB: 271.5 +rank0 [2026-10-29 08:00:59,625] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 45.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:05:09,495] INFO : Step 29600/50000 rank0 | Train Loss: 1.1065 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2391 | Cardinality: 0.1179 | Classification: 0.0927 | Classification_binary: 0.1666 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0018 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1809 | Regression_cos_phi: 0.1011 | Regression_energy: 0.0779 | Regression_eta: 0.0568 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0590 | LR: 4.22e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1174s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8025 | max_loss: 0.1485 | max_backward: 1.4674 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.8982 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 7476.5 | uss_MB: 7138.1 | child_rss_MB: 4790.4 | allocated_MB: 340.8 | reserved_MB: 24404.0 | inactive_split_MB: 290.7 +rank0 [2026-10-29 08:05:09,741] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:09:14,685] INFO : Step 29700/50000 rank0 | Train Loss: 1.0488 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2273 | Cardinality: 0.1414 | Classification: 0.0952 | Classification_binary: 0.1459 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1557 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0632 | Regression_eta: 0.0391 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0486 | LR: 4.19e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.3982s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0057 | max_forward: 0.7342 | max_loss: 0.1652 | max_backward: 1.4907 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.9743 | max_tensorboard_flush: 0.0140 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7477.5 | uss_MB: 6998.2 | child_rss_MB: 4799.0 | allocated_MB: 342.3 | reserved_MB: 24404.0 | inactive_split_MB: 236.6 +rank0 [2026-10-29 08:09:14,932] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:13:32,737] INFO : Step 29800/50000 rank0 | Train Loss: 1.1686 | Train Loss Unweighted: 0.5644 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1551 | Classification: 0.0952 | Classification_binary: 0.1922 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1357 | Regression_eta: 0.0014 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.2035 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0746 | Regression_eta: 0.0538 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0953 | LR: 4.16e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.3353s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7294 | max_loss: 0.1685 | max_backward: 1.6547 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9409 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7478.5 | uss_MB: 7143.5 | child_rss_MB: 4794.4 | allocated_MB: 345.0 | reserved_MB: 24404.0 | inactive_split_MB: 265.1 +rank0 [2026-10-29 08:13:32,983] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:17:38,294] INFO : Step 29900/50000 rank0 | Train Loss: 1.1305 | Train Loss Unweighted: 0.5666 | Train Loss Components: Auxiliary: 0.2471 | Cardinality: 0.1380 | Classification: 0.0944 | Classification_binary: 0.1750 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1464 | Regression_eta: 0.0015 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0837 | Regression_energy: 0.0757 | Regression_eta: 0.0633 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0669 | LR: 4.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1783s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0611 | max_forward: 0.8060 | max_loss: 0.1494 | max_backward: 1.3251 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 7479.5 | uss_MB: 7143.0 | child_rss_MB: 4800.7 | allocated_MB: 344.7 | reserved_MB: 26422.0 | inactive_split_MB: 298.4 +rank0 [2026-10-29 08:17:38,551] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:22:05,694] INFO : Step 30000/50000 rank0 | Train Loss: 1.0854 | Train Loss Unweighted: 0.5586 | Train Loss Components: Auxiliary: 0.2416 | Cardinality: 0.1337 | Classification: 0.1000 | Classification_binary: 0.1541 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1499 | Regression_eta: 0.0014 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1691 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0802 | Regression_eta: 0.0382 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0754 | LR: 4.11e-04 | DataLoad Time: 0.0205s | Model Forward Time: 7.7738s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0172 | max_forward: 0.9685 | max_loss: 0.2166 | max_backward: 4.0112 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.8430 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7461.9 | uss_MB: 7142.5 | child_rss_MB: 4797.1 | allocated_MB: 338.0 | reserved_MB: 26422.0 | inactive_split_MB: 288.0 +rank0 [2026-10-29 08:22:05,950] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 50.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:22:05,975] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-30000.pth +rank0 [2026-10-29 08:22:06,157] INFO : Running validation on rank0 +rank0 [2026-10-29 08:22:06,157] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 08:22:06,157] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 08:22:58,424] INFO : VALIDATION | Step=30000/50000 | Train Loss=1.0854 | Valid Loss=1.1453 | Train Loss Unweighted=0.5586 | Valid Loss Unweighted=0.5585 | Stale=1 | ETA=846.4m +rank0 [2026-10-29 08:22:58,424] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 08:22:58,424] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 08:22:58,424] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 08:22:58,424] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 08:22:58,424] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 08:22:58,424] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 08:22:58,424] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 08:22:58,424] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 08:22:58,424] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/ari=0.512889 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.623129 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/top1_competing=0.714198 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.624634 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0362638 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0501596 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 08:22:58,425] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/seconds_per_rank=21.6698 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 08:22:58,426] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/embedding/ari=0.282006 +rank0 [2026-10-29 08:22:58,427] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 08:22:58,427] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/embedding_minus_raw/ari=0.190383 +rank0 [2026-10-29 08:22:58,427] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 08:22:58,427] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 08:22:58,427] INFO : VALIDATION_PROBE | Step=30000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 08:22:58,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:22:58,694] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 08:22:58,711] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 08:23:24,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 08:23:25,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 08:23:26,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 08:23:28,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 08:23:38,378] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 08:23:38,390] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:23:38,901] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 08:23:38,905] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 08:23:55,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 08:23:56,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 08:23:57,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 08:23:58,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 08:24:08,101] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 08:24:08,113] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:24:08,618] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 08:24:08,622] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 08:24:24,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 08:24:25,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 08:24:26,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 08:24:27,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 08:24:37,665] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 08:24:37,681] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits +rank0 [2026-10-29 08:24:40,943] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits +rank0 [2026-10-29 08:24:41,991] INFO : Plotted number of elements +rank0 [2026-10-29 08:24:43,542] INFO : Plotted jets +rank0 [2026-10-29 08:24:44,692] INFO : Plotted jet ratio +rank0 [2026-10-29 08:24:44,758] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 08:24:48,018] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 08:24:49,015] INFO : Plotted number of elements +rank0 [2026-10-29 08:24:50,935] INFO : Plotted jets +rank0 [2026-10-29 08:24:52,031] INFO : Plotted jet ratio +rank0 [2026-10-29 08:24:52,082] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits +rank0 [2026-10-29 08:24:54,446] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits +rank0 [2026-10-29 08:24:55,309] INFO : Plotted number of elements +rank0 [2026-10-29 08:24:56,783] INFO : Plotted jets +rank0 [2026-10-29 08:24:57,914] INFO : Plotted jet ratio +rank0 [2026-10-29 08:29:01,022] INFO : Step 30100/50000 rank0 | Train Loss: 1.0762 | Train Loss Unweighted: 0.5358 | Train Loss Components: Auxiliary: 0.2378 | Cardinality: 0.1106 | Classification: 0.1097 | Classification_binary: 0.1679 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0800 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0604 | Regression_eta: 0.0274 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0499 | LR: 4.08e-04 | DataLoad Time: 0.0165s | Model Forward Time: 8.9068s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7352 | max_loss: 0.1443 | max_backward: 1.7375 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 1.7731 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.3497 | Memory: rss_MB: 7528.6 | uss_MB: 6331.6 | child_rss_MB: 4801.2 | allocated_MB: 344.1 | reserved_MB: 22922.0 | inactive_split_MB: 248.8 +rank0 [2026-10-29 08:29:01,268] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +2 6 0x7408, 15961 48.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 50.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 44.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:33:17,112] INFO : Step 30200/50000 rank0 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5304 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1421 | Classification: 0.0934 | Classification_binary: 0.1839 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0016 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.2036 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0608 | Regression_eta: 0.0396 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0632 | LR: 4.06e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.2514s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8008 | max_loss: 0.1560 | max_backward: 1.6706 | max_optimizer_step: 0.2663 | max_tensorboard_logging: 0.7725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0555 | Memory: rss_MB: 7535.6 | uss_MB: 7212.9 | child_rss_MB: 4775.8 | allocated_MB: 344.1 | reserved_MB: 23882.0 | inactive_split_MB: 274.8 +rank0 [2026-10-29 08:33:17,359] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 46.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:37:31,267] INFO : Step 30300/50000 rank0 | Train Loss: 1.0429 | Train Loss Unweighted: 0.5252 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1223 | Classification: 0.0922 | Classification_binary: 0.1566 | HitGrouping: 0.1929 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0519 | Regression_eta: 0.0519 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0574 | LR: 4.03e-04 | DataLoad Time: 0.0215s | Model Forward Time: 6.4090s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0060 | max_forward: 1.1719 | max_loss: 0.2884 | max_backward: 2.4994 | max_optimizer_step: 0.2255 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7536.6 | uss_MB: 7213.9 | child_rss_MB: 4771.6 | allocated_MB: 341.8 | reserved_MB: 23882.0 | inactive_split_MB: 270.7 +rank0 [2026-10-29 08:37:31,516] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 48.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 98% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 45.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:41:40,788] INFO : Step 30400/50000 rank0 | Train Loss: 1.0543 | Train Loss Unweighted: 0.5182 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1318 | Classification: 0.0889 | Classification_binary: 0.1487 | HitGrouping: 0.1973 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1382 | Regression_eta: 0.0014 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1849 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0618 | Regression_eta: 0.0518 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0841 | LR: 4.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.7601s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0150 | max_forward: 0.8845 | max_loss: 0.2482 | max_backward: 2.1745 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.5112 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7452.2 | uss_MB: 7214.6 | child_rss_MB: 4475.9 | allocated_MB: 344.7 | reserved_MB: 24842.0 | inactive_split_MB: 245.3 +rank0 [2026-10-29 08:41:41,039] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 48.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 50.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:45:49,784] INFO : Step 30500/50000 rank0 | Train Loss: 1.1589 | Train Loss Unweighted: 0.5596 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1407 | Classification: 0.1085 | Classification_binary: 0.1952 | HitGrouping: 0.1960 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.2143 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0672 | Regression_eta: 0.0425 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0467 | LR: 3.98e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.4432s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7791 | max_loss: 0.2528 | max_backward: 1.9433 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.5210 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0672 | Memory: rss_MB: 7432.7 | uss_MB: 7008.9 | child_rss_MB: 4374.7 | allocated_MB: 346.2 | reserved_MB: 26806.0 | inactive_split_MB: 227.8 +rank0 [2026-10-29 08:45:50,028] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 3% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 91% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 45.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:49:59,149] INFO : Step 30600/50000 rank0 | Train Loss: 1.1269 | Train Loss Unweighted: 0.5561 | Train Loss Components: Auxiliary: 0.2414 | Cardinality: 0.1456 | Classification: 0.0973 | Classification_binary: 0.1804 | HitGrouping: 0.1953 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1373 | Regression_eta: 0.0014 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1707 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0678 | Regression_eta: 0.0376 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0703 | LR: 3.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1304s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1011 | max_forward: 0.8824 | max_loss: 0.1534 | max_backward: 1.4258 | max_optimizer_step: 0.3067 | max_tensorboard_logging: 0.7925 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 7378.2 | uss_MB: 7215.1 | child_rss_MB: 4177.7 | allocated_MB: 345.8 | reserved_MB: 27800.0 | inactive_split_MB: 283.5 +rank0 [2026-10-29 08:49:59,395] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 97% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 3% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 32% +6 10 0x7408, 62036 46.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 97% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:54:17,886] INFO : Step 30700/50000 rank0 | Train Loss: 1.0989 | Train Loss Unweighted: 0.5515 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1377 | Classification: 0.0890 | Classification_binary: 0.1568 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0013 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0565 | Regression_eta: 0.0466 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0527 | LR: 3.92e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.0157s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0056 | max_forward: 0.7546 | max_loss: 0.2396 | max_backward: 1.3389 | max_optimizer_step: 0.2508 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0572 | Memory: rss_MB: 7380.2 | uss_MB: 7216.5 | child_rss_MB: 4178.9 | allocated_MB: 340.3 | reserved_MB: 27800.0 | inactive_split_MB: 223.2 +rank0 [2026-10-29 08:54:18,143] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:58:32,646] INFO : Step 30800/50000 rank0 | Train Loss: 1.1241 | Train Loss Unweighted: 0.5488 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1377 | Classification: 0.1081 | Classification_binary: 0.1717 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1317 | Regression_eta: 0.0015 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.1854 | Regression_cos_phi: 0.1041 | Regression_energy: 0.0644 | Regression_eta: 0.0395 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0669 | LR: 3.90e-04 | DataLoad Time: 0.0177s | Model Forward Time: 6.5216s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7775 | max_loss: 0.2735 | max_backward: 1.9298 | max_optimizer_step: 0.3489 | max_tensorboard_logging: 0.8756 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0726 | Memory: rss_MB: 7357.4 | uss_MB: 5945.6 | child_rss_MB: 4116.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 275.1 +rank0 [2026-10-29 08:58:32,893] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 98% +2 6 0x7408, 15961 48.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 29% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 98% +4 8 0x7408, 13395 51.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 10% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 27% +6 10 0x7408, 62036 45.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 98% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 16% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:02:41,653] INFO : Step 30900/50000 rank0 | Train Loss: 1.1501 | Train Loss Unweighted: 0.5762 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1300 | Classification: 0.0976 | Classification_binary: 0.1996 | HitGrouping: 0.2000 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1374 | Regression_eta: 0.0017 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1982 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0696 | Regression_eta: 0.0361 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0622 | LR: 3.87e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.3324s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.3511 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 1.0306 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 7360.4 | uss_MB: 7217.8 | child_rss_MB: 4118.5 | allocated_MB: 342.2 | reserved_MB: 27800.0 | inactive_split_MB: 198.3 +rank0 [2026-10-29 09:02:41,897] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 45.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:06:47,593] INFO : Step 31000/50000 rank0 | Train Loss: 1.1054 | Train Loss Unweighted: 0.5218 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1446 | Classification: 0.0888 | Classification_binary: 0.1693 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0016 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.1534 | Regression_cos_phi: 0.0869 | Regression_energy: 0.0528 | Regression_eta: 0.0468 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0571 | LR: 3.84e-04 | DataLoad Time: 0.0229s | Model Forward Time: 5.3808s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.8505 | max_loss: 0.2491 | max_backward: 1.6460 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9731 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7337.9 | uss_MB: 6674.8 | child_rss_MB: 4046.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 190.7 +rank0 [2026-10-29 09:06:47,839] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 3% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 49.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 46.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 98% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:06:47,921] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-31000.pth +rank0 [2026-10-29 09:06:48,109] INFO : Running validation on rank0 +rank0 [2026-10-29 09:06:48,110] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 09:06:48,110] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 09:07:39,565] INFO : VALIDATION | Step=31000/50000 | Train Loss=1.1054 | Valid Loss=1.0859 | Train Loss Unweighted=0.5218 | Valid Loss Unweighted=0.5368 | Stale=0 | ETA=805.5m +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 09:07:39,566] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/ari=0.535555 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/ari_threshold=0.75625 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.612222 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/top1_competing=0.700426 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.622318 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0135974 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0478434 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 09:07:39,567] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/seconds_per_rank=21.5495 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding/ari=0.278961 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding_minus_raw/ari=0.187338 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 09:07:39,568] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 09:07:39,569] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 09:07:39,574] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:07:40,210] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 09:07:40,226] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 09:08:08,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 09:08:09,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 09:08:10,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 09:08:11,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 09:08:22,004] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 09:08:22,016] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:08:22,480] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 09:08:22,484] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 09:08:39,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 09:08:40,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 09:08:41,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 09:08:42,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 09:08:52,831] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 09:08:52,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:08:53,432] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 09:08:53,436] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 09:09:09,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 09:09:10,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 09:09:10,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 09:09:11,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 09:09:21,554] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 09:09:21,573] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits +rank0 [2026-10-29 09:09:24,886] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits +rank0 [2026-10-29 09:09:25,978] INFO : Plotted number of elements +rank0 [2026-10-29 09:09:27,544] INFO : Plotted jets +rank0 [2026-10-29 09:09:28,660] INFO : Plotted jet ratio +rank0 [2026-10-29 09:09:28,725] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 09:09:31,944] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 09:09:32,946] INFO : Plotted number of elements +rank0 [2026-10-29 09:09:34,417] INFO : Plotted jets +rank0 [2026-10-29 09:09:35,538] INFO : Plotted jet ratio +rank0 [2026-10-29 09:09:35,594] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits +rank0 [2026-10-29 09:09:38,045] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits +rank0 [2026-10-29 09:09:38,890] INFO : Plotted number of elements +rank0 [2026-10-29 09:09:40,810] INFO : Plotted jets +rank0 [2026-10-29 09:09:41,911] INFO : Plotted jet ratio +rank0 [2026-10-29 09:13:45,974] INFO : Step 31100/50000 rank0 | Train Loss: 1.0306 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2182 | Cardinality: 0.1238 | Classification: 0.0932 | Classification_binary: 0.1415 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1214 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0600 | Regression_eta: 0.0340 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0650 | LR: 3.82e-04 | DataLoad Time: 0.0201s | Model Forward Time: 6.4533s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1068 | max_forward: 0.8067 | max_loss: 0.1436 | max_backward: 1.1427 | max_optimizer_step: 0.2576 | max_tensorboard_logging: 1.1475 | max_tensorboard_flush: 0.0054 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 8196.3 | uss_MB: 5921.9 | child_rss_MB: 4032.2 | allocated_MB: 343.8 | reserved_MB: 24492.0 | inactive_split_MB: 293.3 +rank0 [2026-10-29 09:13:46,221] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 17% +2 6 0x7408, 15961 48.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 5% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 0% +4 8 0x7408, 13395 50.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 98% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 98% +6 10 0x7408, 62036 45.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 2% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:17:45,581] INFO : Step 31200/50000 rank0 | Train Loss: 1.1102 | Train Loss Unweighted: 0.5369 | Train Loss Components: Auxiliary: 0.2472 | Cardinality: 0.1495 | Classification: 0.0959 | Classification_binary: 0.1708 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1328 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1898 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0660 | Regression_eta: 0.0357 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0439 | LR: 3.79e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.8531s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7127 | max_loss: 0.1476 | max_backward: 1.5707 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7428.7 | uss_MB: 7313.8 | child_rss_MB: 4003.5 | allocated_MB: 344.5 | reserved_MB: 24648.0 | inactive_split_MB: 235.5 +rank0 [2026-10-29 09:17:45,835] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 45.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:21:47,934] INFO : Step 31300/50000 rank0 | Train Loss: 1.1865 | Train Loss Unweighted: 0.5472 | Train Loss Components: Auxiliary: 0.2511 | Cardinality: 0.1748 | Classification: 0.0979 | Classification_binary: 0.1851 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0019 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0564 | Regression_eta: 0.0869 | Regression_pt: 0.0595 | Regression_sin_phi: 0.1239 | LR: 3.76e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.0777s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0064 | max_forward: 0.7815 | max_loss: 0.1495 | max_backward: 1.6330 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.5234 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7366.1 | uss_MB: 7313.9 | child_rss_MB: 3706.8 | allocated_MB: 344.4 | reserved_MB: 25640.0 | inactive_split_MB: 233.6 +rank0 [2026-10-29 09:21:48,184] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:25:43,548] INFO : Step 31400/50000 rank0 | Train Loss: 1.0272 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1191 | Classification: 0.0784 | Classification_binary: 0.1451 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1632 | Regression_cos_phi: 0.1029 | Regression_energy: 0.0687 | Regression_eta: 0.0667 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0738 | LR: 3.74e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.1846s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0063 | max_forward: 0.7252 | max_loss: 0.1478 | max_backward: 1.6173 | max_optimizer_step: 0.2404 | max_tensorboard_logging: 0.9541 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7335.5 | uss_MB: 7312.2 | child_rss_MB: 3625.5 | allocated_MB: 342.1 | reserved_MB: 25640.0 | inactive_split_MB: 296.4 +rank0 [2026-10-29 09:25:43,840] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:29:45,544] INFO : Step 31500/50000 rank0 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5247 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1206 | Classification: 0.1037 | Classification_binary: 0.1583 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1297 | Regression_eta: 0.0014 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0638 | Regression_eta: 0.0320 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0597 | LR: 3.71e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.1978s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6643 | max_loss: 0.2487 | max_backward: 1.5648 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.8248 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7336.0 | uss_MB: 7313.0 | child_rss_MB: 3638.4 | allocated_MB: 344.4 | reserved_MB: 26092.0 | inactive_split_MB: 238.2 +rank0 [2026-10-29 09:29:45,792] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:33:42,818] INFO : Step 31600/50000 rank0 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5471 | Train Loss Components: Auxiliary: 0.2424 | Cardinality: 0.1284 | Classification: 0.1033 | Classification_binary: 0.1828 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1282 | Regression_eta: 0.0017 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0593 | Regression_eta: 0.0350 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0564 | LR: 3.69e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.7613s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7603 | max_loss: 0.1471 | max_backward: 2.0475 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.9076 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7328.4 | uss_MB: 7314.4 | child_rss_MB: 3585.6 | allocated_MB: 341.5 | reserved_MB: 26640.0 | inactive_split_MB: 287.0 +rank0 [2026-10-29 09:33:43,060] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 5% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 46.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 91% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:37:48,564] INFO : Step 31700/50000 rank0 | Train Loss: 1.0533 | Train Loss Unweighted: 0.5009 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1416 | Classification: 0.0871 | Classification_binary: 0.1632 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1324 | Regression_cos_phi: 0.0327 | Regression_energy: 0.0513 | Regression_eta: 0.0282 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0412 | LR: 3.66e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8503s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7635 | max_loss: 0.1543 | max_backward: 1.2075 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7330.4 | uss_MB: 7312.9 | child_rss_MB: 3557.0 | allocated_MB: 341.9 | reserved_MB: 26640.0 | inactive_split_MB: 274.1 +rank0 [2026-10-29 09:37:48,809] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:45:41,246] INFO : Step 31800/50000 rank0 | Train Loss: 1.0385 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1148 | Classification: 0.0877 | Classification_binary: 0.1481 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1418 | Regression_eta: 0.0012 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0635 | Regression_eta: 0.0319 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0542 | LR: 3.64e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.7142s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0072 | max_forward: 0.6606 | max_loss: 0.1610 | max_backward: 1.1159 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.8258 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1417 | Memory: rss_MB: 7328.9 | uss_MB: 7312.1 | child_rss_MB: 3561.3 | allocated_MB: 345.5 | reserved_MB: 27080.0 | inactive_split_MB: 260.5 +rank0 [2026-10-29 09:45:41,508] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 46.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 1% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 46.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 43.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 23% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:49:44,880] INFO : Step 31900/50000 rank0 | Train Loss: 1.1464 | Train Loss Unweighted: 0.5585 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1372 | Classification: 0.1078 | Classification_binary: 0.1815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0017 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0611 | Regression_eta: 0.0460 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0685 | LR: 3.61e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.3052s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7718 | max_loss: 0.1553 | max_backward: 1.7857 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.6721 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7333.9 | uss_MB: 7320.8 | child_rss_MB: 3569.3 | allocated_MB: 343.8 | reserved_MB: 27080.0 | inactive_split_MB: 280.2 +rank0 [2026-10-29 09:49:45,134] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 1% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 45.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:53:39,040] INFO : Step 32000/50000 rank0 | Train Loss: 1.1115 | Train Loss Unweighted: 0.5351 | Train Loss Components: Auxiliary: 0.2393 | Cardinality: 0.1406 | Classification: 0.0954 | Classification_binary: 0.1748 | HitGrouping: 0.1912 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1302 | Regression_eta: 0.0018 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0584 | Regression_eta: 0.0455 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0759 | LR: 3.58e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.9727s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.9767 | max_loss: 0.6919 | max_backward: 2.1955 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.9277 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 7329.3 | uss_MB: 7313.9 | child_rss_MB: 3566.4 | allocated_MB: 344.2 | reserved_MB: 27080.0 | inactive_split_MB: 265.7 +rank0 [2026-10-29 09:53:39,313] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 6% +2 6 0x7408, 15961 48.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 93% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 51.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 8% +6 10 0x7408, 62036 45.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 72% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 38% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:53:39,362] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-32000.pth +rank0 [2026-10-29 09:53:39,995] INFO : Running validation on rank0 +rank0 [2026-10-29 09:53:39,995] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 09:53:39,995] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 09:54:29,287] INFO : VALIDATION | Step=32000/50000 | Train Loss=1.1115 | Valid Loss=1.0744 | Train Loss Unweighted=0.5351 | Valid Loss Unweighted=0.5314 | Stale=0 | ETA=765.6m +rank0 [2026-10-29 09:54:29,287] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:54:29,287] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 09:54:29,287] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 09:54:29,287] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:54:29,287] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/ari=0.50614 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.643141 +rank0 [2026-10-29 09:54:29,288] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/top1_competing=0.740657 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.633255 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.043013 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0587806 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 09:54:29,289] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/seconds_per_rank=21.4864 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding/ari=0.269863 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding/ari_threshold=0.8625 +rank0 [2026-10-29 09:54:29,290] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding_minus_raw/ari=0.17824 +rank0 [2026-10-29 09:54:29,291] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 09:54:29,291] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 09:54:29,291] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 09:54:29,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:54:29,549] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 09:54:29,555] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 09:54:57,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 09:54:58,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 09:54:59,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 09:55:00,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 09:55:10,887] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 09:55:10,899] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:55:11,414] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 09:55:11,417] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 09:55:27,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 09:55:28,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 09:55:29,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 09:55:31,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 09:55:41,225] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 09:55:41,236] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:55:41,776] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 09:55:41,780] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 09:55:57,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 09:55:58,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 09:55:58,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 09:55:59,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 09:56:09,536] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 09:56:09,556] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits +rank0 [2026-10-29 09:56:12,815] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits +rank0 [2026-10-29 09:56:13,844] INFO : Plotted number of elements +rank0 [2026-10-29 09:56:15,401] INFO : Plotted jets +rank0 [2026-10-29 09:56:16,550] INFO : Plotted jet ratio +rank0 [2026-10-29 09:56:16,604] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 09:56:19,967] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 09:56:21,008] INFO : Plotted number of elements +rank0 [2026-10-29 09:56:22,490] INFO : Plotted jets +rank0 [2026-10-29 09:56:23,585] INFO : Plotted jet ratio +rank0 [2026-10-29 09:56:23,637] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits +rank0 [2026-10-29 09:56:26,134] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits +rank0 [2026-10-29 09:56:27,496] INFO : Plotted number of elements +rank0 [2026-10-29 09:56:28,990] INFO : Plotted jets +rank0 [2026-10-29 09:56:30,108] INFO : Plotted jet ratio +rank0 [2026-10-29 10:00:20,246] INFO : Step 32100/50000 rank0 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1218 | Classification: 0.0914 | Classification_binary: 0.1509 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1333 | Regression_eta: 0.0016 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1324 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0688 | Regression_eta: 0.0352 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0594 | LR: 3.56e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.3662s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7098 | max_loss: 0.1454 | max_backward: 1.1394 | max_optimizer_step: 0.2650 | max_tensorboard_logging: 0.9832 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 8404.9 | uss_MB: 8386.6 | child_rss_MB: 3573.9 | allocated_MB: 344.0 | reserved_MB: 22148.0 | inactive_split_MB: 276.8 +rank0 [2026-10-29 10:00:20,491] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 32% 9% +2 6 0x7408, 15961 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +4 8 0x7408, 13395 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 45.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 31% 2% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:04:34,052] INFO : Step 32200/50000 rank0 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1241 | Classification: 0.1035 | Classification_binary: 0.1630 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1279 | Regression_eta: 0.0014 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1866 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0709 | Regression_eta: 0.0339 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0755 | LR: 3.53e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.3453s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0891 | max_forward: 0.7517 | max_loss: 0.1441 | max_backward: 1.7835 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.7808 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7418.2 | uss_MB: 7403.1 | child_rss_MB: 3576.2 | allocated_MB: 345.5 | reserved_MB: 23122.0 | inactive_split_MB: 233.4 +rank0 [2026-10-29 10:04:34,299] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 46.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:08:32,715] INFO : Step 32300/50000 rank0 | Train Loss: 1.1438 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2535 | Cardinality: 0.1335 | Classification: 0.1042 | Classification_binary: 0.1958 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1303 | Regression_eta: 0.0015 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1935 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0634 | Regression_eta: 0.0299 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0530 | LR: 3.51e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.5911s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7857 | max_loss: 0.1780 | max_backward: 1.7580 | max_optimizer_step: 0.2428 | max_tensorboard_logging: 0.8954 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 7421.3 | uss_MB: 7404.5 | child_rss_MB: 3576.8 | allocated_MB: 343.9 | reserved_MB: 24108.0 | inactive_split_MB: 280.1 +rank0 [2026-10-29 10:08:32,957] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 48.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 98% +4 8 0x7408, 13395 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 6% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 50% +6 10 0x7408, 62036 45.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 95% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:12:37,789] INFO : Step 32400/50000 rank0 | Train Loss: 1.0306 | Train Loss Unweighted: 0.5063 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1168 | Classification: 0.0867 | Classification_binary: 0.1599 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0013 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0582 | Regression_eta: 0.0349 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0520 | LR: 3.48e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.9613s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.1456 | max_backward: 1.3401 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.8752 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7420.4 | uss_MB: 7401.9 | child_rss_MB: 3574.4 | allocated_MB: 344.8 | reserved_MB: 24108.0 | inactive_split_MB: 271.0 +rank0 [2026-10-29 10:12:38,032] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:16:49,232] INFO : Step 32500/50000 rank0 | Train Loss: 1.0033 | Train Loss Unweighted: 0.5126 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.0995 | Classification: 0.1015 | Classification_binary: 0.1402 | HitGrouping: 0.1898 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1332 | Regression_eta: 0.0015 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1282 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0606 | Regression_eta: 0.0300 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0560 | LR: 3.46e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.7417s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6088 | max_loss: 0.1480 | max_backward: 1.0729 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.8045 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7420.4 | uss_MB: 7401.8 | child_rss_MB: 3575.0 | allocated_MB: 341.9 | reserved_MB: 25096.0 | inactive_split_MB: 268.1 +rank0 [2026-10-29 10:16:49,482] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:21:04,733] INFO : Step 32600/50000 rank0 | Train Loss: 1.0981 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1271 | Classification: 0.1053 | Classification_binary: 0.1773 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1306 | Regression_eta: 0.0014 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2014 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0597 | Regression_eta: 0.0314 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0460 | LR: 3.43e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.9390s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8529 | max_loss: 0.3369 | max_backward: 2.9831 | max_optimizer_step: 0.2386 | max_tensorboard_logging: 0.5557 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7420.7 | uss_MB: 5748.3 | child_rss_MB: 3585.6 | allocated_MB: 344.1 | reserved_MB: 25096.0 | inactive_split_MB: 287.9 +rank0 [2026-10-29 10:21:04,986] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 48.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +4 8 0x7408, 13395 50.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 47% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 17% +6 10 0x7408, 62036 45.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 30% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:25:23,880] INFO : Step 32700/50000 rank0 | Train Loss: 1.0956 | Train Loss Unweighted: 0.5307 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.1282 | Classification: 0.1011 | Classification_binary: 0.1731 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1572 | Regression_cos_phi: 0.1019 | Regression_energy: 0.0721 | Regression_eta: 0.0369 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0511 | LR: 3.41e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.7845s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0062 | max_forward: 0.8233 | max_loss: 0.1478 | max_backward: 1.5120 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0564 | Memory: rss_MB: 7420.7 | uss_MB: 7403.4 | child_rss_MB: 3588.8 | allocated_MB: 342.9 | reserved_MB: 25096.0 | inactive_split_MB: 272.6 +rank0 [2026-10-29 10:25:24,124] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 9% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 93% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 0% +4 8 0x7408, 13395 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 0% +6 10 0x7408, 62036 45.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:29:24,190] INFO : Step 32800/50000 rank0 | Train Loss: 1.0857 | Train Loss Unweighted: 0.5241 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1299 | Classification: 0.1011 | Classification_binary: 0.1551 | HitGrouping: 0.1982 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1327 | Regression_eta: 0.0016 | Regression_pt: 0.1319 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0532 | Regression_eta: 0.0504 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0708 | LR: 3.38e-04 | DataLoad Time: 0.0244s | Model Forward Time: 4.8999s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0179 | max_forward: 0.9787 | max_loss: 0.1782 | max_backward: 1.5075 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 0.5808 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.1577 | Memory: rss_MB: 7419.5 | uss_MB: 7401.5 | child_rss_MB: 3585.9 | allocated_MB: 345.1 | reserved_MB: 25096.0 | inactive_split_MB: 263.9 +rank0 [2026-10-29 10:29:24,452] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:33:27,071] INFO : Step 32900/50000 rank0 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1116 | Classification: 0.1066 | Classification_binary: 0.1631 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1300 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.1841 | Regression_cos_phi: 0.1041 | Regression_energy: 0.0486 | Regression_eta: 0.0639 | Regression_pt: 0.0519 | Regression_sin_phi: 0.1105 | LR: 3.36e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.3717s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 1.3284 | max_loss: 0.5826 | max_backward: 1.9391 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7423.7 | uss_MB: 7407.1 | child_rss_MB: 3589.0 | allocated_MB: 344.3 | reserved_MB: 26086.0 | inactive_split_MB: 212.1 +rank0 [2026-10-29 10:33:27,319] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:37:13,958] INFO : Step 33000/50000 rank0 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5373 | Train Loss Components: Auxiliary: 0.2310 | Cardinality: 0.1017 | Classification: 0.1099 | Classification_binary: 0.1844 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0610 | Regression_eta: 0.0281 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0460 | LR: 3.33e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.0492s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 0.7919 | max_loss: 0.2231 | max_backward: 1.5724 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.5025 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1340 | Memory: rss_MB: 7423.5 | uss_MB: 7406.2 | child_rss_MB: 3581.4 | allocated_MB: 344.2 | reserved_MB: 26086.0 | inactive_split_MB: 269.8 +rank0 [2026-10-29 10:37:14,205] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 53.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 46.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 5% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:37:14,228] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-33000.pth +rank0 [2026-10-29 10:37:14,432] INFO : Running validation on rank0 +rank0 [2026-10-29 10:37:14,432] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 10:37:14,433] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 10:38:04,381] INFO : VALIDATION | Step=33000/50000 | Train Loss=1.0508 | Valid Loss=1.0668 | Train Loss Unweighted=0.5373 | Valid Loss Unweighted=0.5242 | Stale=0 | ETA=723.6m +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 10:38:04,382] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/ari=0.531362 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.637596 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/top1_competing=0.731664 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.632378 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0177908 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0579028 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 10:38:04,383] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/seconds_per_rank=21.2946 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding/ari=0.310584 +rank0 [2026-10-29 10:38:04,384] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 10:38:04,385] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding_minus_raw/ari=0.218961 +rank0 [2026-10-29 10:38:04,385] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 10:38:04,385] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 10:38:04,385] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 10:38:04,390] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:38:04,641] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 10:38:04,653] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 10:38:31,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 10:38:33,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 10:38:34,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 10:38:35,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 10:38:45,820] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 10:38:45,832] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:38:46,345] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 10:38:46,349] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 10:39:02,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 10:39:03,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 10:39:04,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 10:39:05,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 10:39:15,775] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 10:39:15,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:39:16,280] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 10:39:16,284] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 10:39:32,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 10:39:32,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 10:39:33,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 10:39:34,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 10:39:44,296] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 10:39:44,312] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits +rank0 [2026-10-29 10:39:47,572] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits +rank0 [2026-10-29 10:39:48,628] INFO : Plotted number of elements +rank0 [2026-10-29 10:39:50,224] INFO : Plotted jets +rank0 [2026-10-29 10:39:51,432] INFO : Plotted jet ratio +rank0 [2026-10-29 10:39:51,496] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 10:39:54,828] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 10:39:55,881] INFO : Plotted number of elements +rank0 [2026-10-29 10:39:57,437] INFO : Plotted jets +rank0 [2026-10-29 10:39:58,508] INFO : Plotted jet ratio +rank0 [2026-10-29 10:39:58,575] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits +rank0 [2026-10-29 10:40:01,038] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits +rank0 [2026-10-29 10:40:02,371] INFO : Plotted number of elements +rank0 [2026-10-29 10:40:03,841] INFO : Plotted jets +rank0 [2026-10-29 10:40:04,959] INFO : Plotted jet ratio +rank0 [2026-10-29 10:43:57,209] INFO : Step 33100/50000 rank0 | Train Loss: 1.1040 | Train Loss Unweighted: 0.5268 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1427 | Classification: 0.0921 | Classification_binary: 0.1655 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1319 | Regression_eta: 0.0015 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.1814 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0670 | Regression_eta: 0.0435 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0494 | LR: 3.31e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.2468s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0325 | max_forward: 0.7755 | max_loss: 0.1478 | max_backward: 1.3916 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.6610 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0760 | Memory: rss_MB: 8784.9 | uss_MB: 8766.5 | child_rss_MB: 3557.4 | allocated_MB: 343.8 | reserved_MB: 23356.0 | inactive_split_MB: 267.3 +rank0 [2026-10-29 10:43:57,452] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 45.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:47:56,998] INFO : Step 33200/50000 rank0 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4955 | Train Loss Components: Auxiliary: 0.2164 | Cardinality: 0.1253 | Classification: 0.1001 | Classification_binary: 0.1407 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0019 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0537 | Regression_eta: 0.0353 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0659 | LR: 3.28e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.7810s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0401 | max_forward: 0.9954 | max_loss: 0.3116 | max_backward: 2.2139 | max_optimizer_step: 0.2657 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7500.4 | uss_MB: 7483.0 | child_rss_MB: 3560.3 | allocated_MB: 344.1 | reserved_MB: 23356.0 | inactive_split_MB: 241.6 +rank0 [2026-10-29 10:47:57,247] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 77% +2 6 0x7408, 15961 48.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 68% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 68% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 46% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +6 10 0x7408, 62036 45.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 49% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:51:55,818] INFO : Step 33300/50000 rank0 | Train Loss: 1.0288 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2302 | Cardinality: 0.1166 | Classification: 0.0975 | Classification_binary: 0.1653 | HitGrouping: 0.1885 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1220 | Regression_eta: 0.0012 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0491 | Regression_eta: 0.0239 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0492 | LR: 3.26e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2801s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7054 | max_loss: 0.2213 | max_backward: 1.6633 | max_optimizer_step: 0.2474 | max_tensorboard_logging: 0.8237 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0708 | Memory: rss_MB: 7500.4 | uss_MB: 7483.1 | child_rss_MB: 3556.5 | allocated_MB: 343.5 | reserved_MB: 23356.0 | inactive_split_MB: 261.9 +rank0 [2026-10-29 10:51:56,090] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:55:55,586] INFO : Step 33400/50000 rank0 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5337 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1249 | Classification: 0.0970 | Classification_binary: 0.1720 | HitGrouping: 0.1947 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0013 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1809 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0610 | Regression_eta: 0.0467 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0551 | LR: 3.23e-04 | DataLoad Time: 0.0158s | Model Forward Time: 6.1373s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0073 | max_forward: 1.2444 | max_loss: 0.2025 | max_backward: 2.5109 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7502.5 | uss_MB: 7485.1 | child_rss_MB: 3549.5 | allocated_MB: 347.6 | reserved_MB: 23358.0 | inactive_split_MB: 303.0 +rank0 [2026-10-29 10:55:55,852] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:59:54,254] INFO : Step 33500/50000 rank0 | Train Loss: 1.0164 | Train Loss Unweighted: 0.4956 | Train Loss Components: Auxiliary: 0.2212 | Cardinality: 0.1274 | Classification: 0.0875 | Classification_binary: 0.1421 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1309 | Regression_eta: 0.0013 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0479 | Regression_eta: 0.0392 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0527 | LR: 3.21e-04 | DataLoad Time: 0.0101s | Model Forward Time: 5.3005s | Phase Times: max_pre_step_barrier: 0.0271 | max_batch_to_device: 0.0558 | max_forward: 0.9437 | max_loss: 0.1437 | max_backward: 1.5429 | max_optimizer_step: 0.2447 | max_tensorboard_logging: 0.8833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7505.9 | uss_MB: 7486.7 | child_rss_MB: 3558.2 | allocated_MB: 341.2 | reserved_MB: 23358.0 | inactive_split_MB: 237.5 +rank0 [2026-10-29 10:59:54,520] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:05:23,590] INFO : Step 33600/50000 rank0 | Train Loss: 1.0074 | Train Loss Unweighted: 0.4988 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1109 | Classification: 0.0969 | Classification_binary: 0.1509 | HitGrouping: 0.1901 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0012 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0663 | Regression_eta: 0.0495 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0602 | LR: 3.19e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9742s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0970 | max_forward: 0.7667 | max_loss: 0.1478 | max_backward: 1.4192 | max_optimizer_step: 0.2426 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7500.7 | uss_MB: 7485.1 | child_rss_MB: 3550.7 | allocated_MB: 344.5 | reserved_MB: 24528.0 | inactive_split_MB: 238.9 +rank0 [2026-10-29 11:05:23,835] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:09:18,516] INFO : Step 33700/50000 rank0 | Train Loss: 1.1891 | Train Loss Unweighted: 0.5658 | Train Loss Components: Auxiliary: 0.2602 | Cardinality: 0.1587 | Classification: 0.1116 | Classification_binary: 0.1953 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1270 | Regression_eta: 0.0015 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2024 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0670 | Regression_eta: 0.0349 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0555 | LR: 3.16e-04 | DataLoad Time: 0.0109s | Model Forward Time: 5.5983s | Phase Times: max_pre_step_barrier: 0.0253 | max_batch_to_device: 0.0066 | max_forward: 1.3051 | max_loss: 0.1764 | max_backward: 1.7331 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.5706 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0895 | Memory: rss_MB: 7502.0 | uss_MB: 7487.9 | child_rss_MB: 3548.7 | allocated_MB: 347.7 | reserved_MB: 24528.0 | inactive_split_MB: 288.7 +rank0 [2026-10-29 11:09:18,762] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 5% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 58% +4 8 0x7408, 13395 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 46.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:13:15,470] INFO : Step 33800/50000 rank0 | Train Loss: 1.0442 | Train Loss Unweighted: 0.4970 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1421 | Classification: 0.0764 | Classification_binary: 0.1525 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1318 | Regression_eta: 0.0014 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1282 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0683 | Regression_eta: 0.0358 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0380 | LR: 3.14e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9674s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7370 | max_loss: 0.2458 | max_backward: 1.5902 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.6292 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1331 | Memory: rss_MB: 7496.1 | uss_MB: 7482.2 | child_rss_MB: 3539.4 | allocated_MB: 344.2 | reserved_MB: 24528.0 | inactive_split_MB: 271.5 +rank0 [2026-10-29 11:13:15,733] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 45.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:17:11,462] INFO : Step 33900/50000 rank0 | Train Loss: 1.0341 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1352 | Classification: 0.0865 | Classification_binary: 0.1441 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1254 | Regression_eta: 0.0016 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0472 | Regression_eta: 0.0334 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0507 | LR: 3.11e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.0442s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.6305 | max_loss: 0.1507 | max_backward: 1.4015 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9471 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0605 | Memory: rss_MB: 7498.1 | uss_MB: 7488.3 | child_rss_MB: 3543.5 | allocated_MB: 344.1 | reserved_MB: 24528.0 | inactive_split_MB: 268.7 +rank0 [2026-10-29 11:17:11,705] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 46.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:21:05,832] INFO : Step 34000/50000 rank0 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1309 | Classification: 0.0985 | Classification_binary: 0.1707 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1170 | Regression_eta: 0.0014 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0609 | Regression_eta: 0.0226 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0350 | LR: 3.09e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.3816s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7414 | max_loss: 0.1458 | max_backward: 1.7218 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 7499.1 | uss_MB: 7491.4 | child_rss_MB: 3548.4 | allocated_MB: 342.9 | reserved_MB: 24528.0 | inactive_split_MB: 277.5 +rank0 [2026-10-29 11:21:06,078] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 49.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 46.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:21:06,184] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-34000.pth +rank0 [2026-10-29 11:21:06,372] INFO : Running validation on rank0 +rank0 [2026-10-29 11:21:06,372] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 11:21:06,372] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 11:21:57,438] INFO : VALIDATION | Step=34000/50000 | Train Loss=1.0653 | Valid Loss=1.0715 | Train Loss Unweighted=0.5076 | Valid Loss Unweighted=0.5203 | Stale=1 | ETA=681.7m +rank0 [2026-10-29 11:21:57,438] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 11:21:57,438] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 11:21:57,438] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 11:21:57,438] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/ari=0.542383 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.648377 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/top1_competing=0.74465 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.631905 +rank0 [2026-10-29 11:21:57,439] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.00676928 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.05743 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/seconds_per_rank=21.4408 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 11:21:57,440] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding/ari=0.307032 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding_minus_raw/ari=0.215409 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 11:21:57,441] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 11:21:57,447] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:21:57,708] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 11:21:57,718] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 11:22:17,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 11:22:18,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 11:22:20,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 11:22:21,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 11:22:31,812] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 11:22:32,014] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:22:32,506] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 11:22:32,509] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 11:22:48,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 11:22:49,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 11:22:50,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 11:22:51,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 11:23:01,905] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 11:23:02,010] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:23:02,514] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 11:23:02,517] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 11:23:18,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 11:23:19,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 11:23:19,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 11:23:20,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 11:23:30,596] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 11:23:30,699] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits +rank0 [2026-10-29 11:23:33,943] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits +rank0 [2026-10-29 11:23:35,351] INFO : Plotted number of elements +rank0 [2026-10-29 11:23:36,949] INFO : Plotted jets +rank0 [2026-10-29 11:23:38,112] INFO : Plotted jet ratio +rank0 [2026-10-29 11:23:38,179] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 11:23:41,494] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 11:23:42,522] INFO : Plotted number of elements +rank0 [2026-10-29 11:23:44,035] INFO : Plotted jets +rank0 [2026-10-29 11:23:45,141] INFO : Plotted jet ratio +rank0 [2026-10-29 11:23:45,196] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits +rank0 [2026-10-29 11:23:47,759] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits +rank0 [2026-10-29 11:23:48,634] INFO : Plotted number of elements +rank0 [2026-10-29 11:23:50,145] INFO : Plotted jets +rank0 [2026-10-29 11:23:51,787] INFO : Plotted jet ratio +rank0 [2026-10-29 11:27:46,518] INFO : Step 34100/50000 rank0 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5582 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1263 | Classification: 0.1031 | Classification_binary: 0.1801 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1358 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.2021 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0693 | Regression_eta: 0.0377 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0736 | LR: 3.06e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.7038s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9025 | max_loss: 0.1870 | max_backward: 1.7732 | max_optimizer_step: 0.2826 | max_tensorboard_logging: 0.8215 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0738 | Memory: rss_MB: 9109.5 | uss_MB: 8140.4 | child_rss_MB: 3555.5 | allocated_MB: 342.7 | reserved_MB: 22124.0 | inactive_split_MB: 211.6 +rank0 [2026-10-29 11:27:46,767] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 3% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 46.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 63% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:31:41,648] INFO : Step 34200/50000 rank0 | Train Loss: 0.9931 | Train Loss Unweighted: 0.4794 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1232 | Classification: 0.0771 | Classification_binary: 0.1377 | HitGrouping: 0.1996 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1301 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0566 | Regression_eta: 0.0265 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0346 | LR: 3.04e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.4831s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.8809 | max_loss: 0.3036 | max_backward: 1.8297 | max_optimizer_step: 0.2897 | max_tensorboard_logging: 0.6835 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7608.6 | uss_MB: 6291.1 | child_rss_MB: 3559.3 | allocated_MB: 339.0 | reserved_MB: 22508.0 | inactive_split_MB: 220.4 +rank0 [2026-10-29 11:31:41,913] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 5% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +2 6 0x7408, 15961 48.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 1% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 95% +4 8 0x7408, 13395 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 5% +6 10 0x7408, 62036 45.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 11% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 92% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:35:38,508] INFO : Step 34300/50000 rank0 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4920 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1192 | Classification: 0.0965 | Classification_binary: 0.1484 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0620 | Regression_eta: 0.0397 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0474 | LR: 3.02e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9967s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7255 | max_loss: 0.1427 | max_backward: 1.3652 | max_optimizer_step: 0.2496 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7610.6 | uss_MB: 7596.5 | child_rss_MB: 3557.3 | allocated_MB: 345.0 | reserved_MB: 22508.0 | inactive_split_MB: 216.4 +rank0 [2026-10-29 11:35:38,778] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 45.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:39:28,732] INFO : Step 34400/50000 rank0 | Train Loss: 1.0662 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2383 | Cardinality: 0.1289 | Classification: 0.0974 | Classification_binary: 0.1749 | HitGrouping: 0.1864 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1281 | Regression_eta: 0.0014 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1961 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0587 | Regression_eta: 0.0492 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0737 | LR: 2.99e-04 | DataLoad Time: 0.0189s | Model Forward Time: 6.8228s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9438 | max_loss: 0.1560 | max_backward: 2.9359 | max_optimizer_step: 0.2526 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7613.7 | uss_MB: 6304.5 | child_rss_MB: 3550.6 | allocated_MB: 345.3 | reserved_MB: 23132.0 | inactive_split_MB: 282.7 +rank0 [2026-10-29 11:39:28,978] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 51.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 1% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 45.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:43:18,458] INFO : Step 34500/50000 rank0 | Train Loss: 1.0328 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1103 | Classification: 0.0867 | Classification_binary: 0.1518 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0014 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1507 | Regression_cos_phi: 0.0867 | Regression_energy: 0.0581 | Regression_eta: 0.0366 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0552 | LR: 2.97e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.3146s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.8153 | max_loss: 0.1429 | max_backward: 1.8299 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.5315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7615.7 | uss_MB: 6643.2 | child_rss_MB: 3549.5 | allocated_MB: 341.4 | reserved_MB: 23132.0 | inactive_split_MB: 238.6 +rank0 [2026-10-29 11:43:18,745] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 97% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:47:06,656] INFO : Step 34600/50000 rank0 | Train Loss: 1.0317 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1201 | Classification: 0.0877 | Classification_binary: 0.1434 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1359 | Regression_eta: 0.0014 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0735 | Regression_eta: 0.0333 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0479 | LR: 2.95e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.0005s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.6882 | max_loss: 0.1360 | max_backward: 1.3984 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5787 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7607.9 | uss_MB: 6553.3 | child_rss_MB: 3582.8 | allocated_MB: 343.8 | reserved_MB: 23132.0 | inactive_split_MB: 233.1 +rank0 [2026-10-29 11:47:06,903] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 7% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 48.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 83% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 98% +6 10 0x7408, 62036 46.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 1% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:51:13,574] INFO : Step 34700/50000 rank0 | Train Loss: 1.1152 | Train Loss Unweighted: 0.5285 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1612 | Classification: 0.0937 | Classification_binary: 0.1734 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1283 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1754 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0610 | Regression_eta: 0.0325 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0476 | LR: 2.92e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1452s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7859 | max_loss: 0.2571 | max_backward: 1.9119 | max_optimizer_step: 0.2516 | max_tensorboard_logging: 0.5032 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7610.6 | uss_MB: 7599.4 | child_rss_MB: 3559.6 | allocated_MB: 343.9 | reserved_MB: 23132.0 | inactive_split_MB: 446.8 +rank0 [2026-10-29 11:51:13,831] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:54:57,633] INFO : Step 34800/50000 rank0 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5142 | Train Loss Components: Auxiliary: 0.2371 | Cardinality: 0.1583 | Classification: 0.0857 | Classification_binary: 0.1750 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1255 | Regression_eta: 0.0013 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1457 | Regression_cos_phi: 0.0323 | Regression_energy: 0.0553 | Regression_eta: 0.0201 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0359 | LR: 2.90e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9683s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0423 | max_forward: 0.8214 | max_loss: 0.1472 | max_backward: 1.4019 | max_optimizer_step: 0.3234 | max_tensorboard_logging: 0.5950 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7613.6 | uss_MB: 7602.4 | child_rss_MB: 3541.0 | allocated_MB: 344.5 | reserved_MB: 23132.0 | inactive_split_MB: 201.5 +rank0 [2026-10-29 11:54:57,879] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 46.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:58:49,519] INFO : Step 34900/50000 rank0 | Train Loss: 0.9512 | Train Loss Unweighted: 0.4678 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1120 | Classification: 0.0788 | Classification_binary: 0.1318 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1262 | Regression_eta: 0.0014 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1436 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0653 | Regression_eta: 0.0361 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0640 | LR: 2.88e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9246s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0986 | max_forward: 0.7397 | max_loss: 0.1421 | max_backward: 1.3918 | max_optimizer_step: 0.3560 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7614.6 | uss_MB: 7602.0 | child_rss_MB: 3543.9 | allocated_MB: 341.5 | reserved_MB: 23500.0 | inactive_split_MB: 262.9 +rank0 [2026-10-29 11:58:49,768] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 45.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:02:44,468] INFO : Step 35000/50000 rank0 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1401 | Classification: 0.0852 | Classification_binary: 0.1555 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0012 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.2000 | Regression_cos_phi: 0.0961 | Regression_energy: 0.0619 | Regression_eta: 0.0484 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0801 | LR: 2.85e-04 | DataLoad Time: 0.0222s | Model Forward Time: 5.4986s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0159 | max_forward: 0.7961 | max_loss: 0.2615 | max_backward: 2.0104 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6382 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7618.3 | uss_MB: 7606.3 | child_rss_MB: 3544.7 | allocated_MB: 345.2 | reserved_MB: 23500.0 | inactive_split_MB: 284.5 +rank0 [2026-10-29 12:02:44,713] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 49.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +6 10 0x7408, 62036 46.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:02:44,742] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth +rank0 [2026-10-29 12:02:45,572] INFO : Running validation on rank0 +rank0 [2026-10-29 12:02:45,573] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 12:02:45,573] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 12:03:34,210] INFO : VALIDATION | Step=35000/50000 | Train Loss=1.0147 | Valid Loss=1.0626 | Train Loss Unweighted=0.4747 | Valid Loss Unweighted=0.5097 | Stale=0 | ETA=638.7m +rank0 [2026-10-29 12:03:34,210] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 12:03:34,210] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 12:03:34,210] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 12:03:34,210] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 12:03:34,210] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 12:03:34,210] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/ari=0.550708 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.638436 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/top1_competing=0.735681 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.621819 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.00155529 +rank0 [2026-10-29 12:03:34,211] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0473446 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 12:03:34,212] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/seconds_per_rank=21.6856 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding/ari=0.293681 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding/ari_threshold=0.9 +rank0 [2026-10-29 12:03:34,213] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding_minus_raw/ari=0.202058 +rank0 [2026-10-29 12:03:34,214] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 12:03:34,214] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 12:03:34,214] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 12:03:34,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 12:03:34,476] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 12:03:34,493] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 12:04:02,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 12:04:03,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 12:04:04,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 12:04:07,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 12:04:17,987] INFO : Time taken to make predictions on device 0 is: 0.30 min +rank0 [2026-10-29 12:04:17,999] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 12:04:18,512] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 12:04:18,516] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 12:04:35,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 12:04:36,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 12:04:37,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 12:04:38,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 12:04:48,289] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 12:04:48,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 12:04:48,828] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 12:04:48,831] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 12:05:04,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 12:05:05,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 12:05:06,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 12:05:06,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 12:05:16,775] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 12:05:16,791] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits +rank0 [2026-10-29 12:05:19,995] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits +rank0 [2026-10-29 12:05:21,053] INFO : Plotted number of elements +rank0 [2026-10-29 12:05:23,005] INFO : Plotted jets +rank0 [2026-10-29 12:05:24,141] INFO : Plotted jet ratio +rank0 [2026-10-29 12:05:24,206] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 12:05:27,480] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 12:05:28,529] INFO : Plotted number of elements +rank0 [2026-10-29 12:05:30,052] INFO : Plotted jets +rank0 [2026-10-29 12:05:31,163] INFO : Plotted jet ratio +rank0 [2026-10-29 12:05:31,218] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits +rank0 [2026-10-29 12:05:33,702] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits +rank0 [2026-10-29 12:05:34,550] INFO : Plotted number of elements +rank0 [2026-10-29 12:05:36,023] INFO : Plotted jets +rank0 [2026-10-29 12:05:37,148] INFO : Plotted jet ratio +rank0 [2026-10-29 12:09:35,345] INFO : Step 35100/50000 rank0 | Train Loss: 1.1060 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1387 | Classification: 0.0926 | Classification_binary: 0.1848 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0014 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1974 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0576 | Regression_eta: 0.0409 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0621 | LR: 2.83e-04 | DataLoad Time: 0.0152s | Model Forward Time: 7.5526s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.9070 | max_loss: 0.1513 | max_backward: 2.1746 | max_optimizer_step: 0.2850 | max_tensorboard_logging: 1.1439 | max_tensorboard_flush: 0.0024 | max_post_step_barrier: 0.0768 | Memory: rss_MB: 9217.3 | uss_MB: 7564.1 | child_rss_MB: 3570.6 | allocated_MB: 344.3 | reserved_MB: 22622.0 | inactive_split_MB: 852.9 +rank0 [2026-10-29 12:09:35,590] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 48.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 45.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:13:24,916] INFO : Step 35200/50000 rank0 | Train Loss: 1.0386 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1293 | Classification: 0.0826 | Classification_binary: 0.1559 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1247 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0520 | Regression_eta: 0.0349 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0584 | LR: 2.81e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.1674s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.8193 | max_loss: 0.1608 | max_backward: 1.3752 | max_optimizer_step: 0.2535 | max_tensorboard_logging: 0.9185 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0700 | Memory: rss_MB: 7738.0 | uss_MB: 7717.0 | child_rss_MB: 3546.7 | allocated_MB: 341.8 | reserved_MB: 25614.0 | inactive_split_MB: 192.0 +rank0 [2026-10-29 12:13:25,161] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 46.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:17:18,685] INFO : Step 35300/50000 rank0 | Train Loss: 0.9736 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1059 | Classification: 0.0970 | Classification_binary: 0.1371 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0016 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0551 | Regression_eta: 0.0484 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0748 | LR: 2.79e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2989s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6434 | max_loss: 0.2341 | max_backward: 1.6834 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0588 | Memory: rss_MB: 7740.0 | uss_MB: 7297.3 | child_rss_MB: 3543.2 | allocated_MB: 343.2 | reserved_MB: 25614.0 | inactive_split_MB: 226.8 +rank0 [2026-10-29 12:17:18,931] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 2% +2 6 0x7408, 15961 49.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 52.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +6 10 0x7408, 62036 45.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:21:06,656] INFO : Step 35400/50000 rank0 | Train Loss: 1.1186 | Train Loss Unweighted: 0.5363 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1393 | Classification: 0.1069 | Classification_binary: 0.1767 | HitGrouping: 0.1943 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0016 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0567 | Regression_eta: 0.0424 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0639 | LR: 2.76e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.4458s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0060 | max_forward: 0.7591 | max_loss: 0.1562 | max_backward: 1.6040 | max_optimizer_step: 0.2543 | max_tensorboard_logging: 0.9624 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.0563 | Memory: rss_MB: 7735.5 | uss_MB: 7248.0 | child_rss_MB: 3540.1 | allocated_MB: 345.3 | reserved_MB: 25762.0 | inactive_split_MB: 277.5 +rank0 [2026-10-29 12:21:06,903] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 46.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:24:54,761] INFO : Step 35500/50000 rank0 | Train Loss: 1.1216 | Train Loss Unweighted: 0.5466 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1527 | Classification: 0.0970 | Classification_binary: 0.1767 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1342 | Regression_eta: 0.0014 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1753 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0762 | Regression_eta: 0.0317 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0389 | LR: 2.74e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0496s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0648 | max_forward: 0.8748 | max_loss: 0.1544 | max_backward: 1.6387 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 0.5130 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0632 | Memory: rss_MB: 7738.5 | uss_MB: 7721.7 | child_rss_MB: 3543.4 | allocated_MB: 343.4 | reserved_MB: 25762.0 | inactive_split_MB: 284.4 +rank0 [2026-10-29 12:24:55,008] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 46.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:28:51,780] INFO : Step 35600/50000 rank0 | Train Loss: 1.0380 | Train Loss Unweighted: 0.5039 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1124 | Classification: 0.0922 | Classification_binary: 0.1405 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.1531 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0756 | Regression_eta: 0.0528 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0995 | LR: 2.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7090s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0559 | max_forward: 0.7376 | max_loss: 0.1415 | max_backward: 1.1754 | max_optimizer_step: 0.2524 | max_tensorboard_logging: 0.5225 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1603 | Memory: rss_MB: 7739.4 | uss_MB: 7724.9 | child_rss_MB: 3557.9 | allocated_MB: 343.8 | reserved_MB: 25762.0 | inactive_split_MB: 233.6 +rank0 [2026-10-29 12:28:52,024] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 46.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:32:55,165] INFO : Step 35700/50000 rank0 | Train Loss: 1.0352 | Train Loss Unweighted: 0.5000 | Train Loss Components: Auxiliary: 0.2288 | Cardinality: 0.1199 | Classification: 0.1043 | Classification_binary: 0.1492 | HitGrouping: 0.1850 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1212 | Regression_eta: 0.0015 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0806 | Classification_binary: 0.1937 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0592 | Regression_eta: 0.0692 | Regression_pt: 0.0620 | Regression_sin_phi: 0.1572 | LR: 2.70e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0536s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0360 | max_forward: 0.7358 | max_loss: 0.1419 | max_backward: 1.6761 | max_optimizer_step: 0.2531 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7741.1 | uss_MB: 7726.8 | child_rss_MB: 3543.5 | allocated_MB: 344.2 | reserved_MB: 26034.0 | inactive_split_MB: 271.8 +rank0 [2026-10-29 12:32:55,410] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 45.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:36:36,961] INFO : Step 35800/50000 rank0 | Train Loss: 1.0929 | Train Loss Unweighted: 0.5323 | Train Loss Components: Auxiliary: 0.2446 | Cardinality: 0.1241 | Classification: 0.0986 | Classification_binary: 0.1801 | HitGrouping: 0.1908 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0014 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1793 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0613 | Regression_eta: 0.0343 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0637 | LR: 2.68e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0717s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7732 | max_loss: 0.1506 | max_backward: 1.4048 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0581 | Memory: rss_MB: 7740.3 | uss_MB: 7206.4 | child_rss_MB: 3548.3 | allocated_MB: 344.1 | reserved_MB: 26034.0 | inactive_split_MB: 276.1 +rank0 [2026-10-29 12:47:12,911] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 42.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 43.0°C 119.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 44.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 41.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 14:48:22,193] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-29 14:48:22,207] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-29 14:48:45,046] INFO : Instantiating model +rank0 [2026-10-29 14:48:45,047] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank0 [2026-10-29 14:48:45,047] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-29 14:48:45,064] INFO : Input encoding initialization took 0.02s +rank0 [2026-10-29 14:48:45,064] INFO : nn0 parameters: 171776 +rank0 [2026-10-29 14:48:45,064] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-29 14:48:45,064] INFO : Initializing backbone layer 0 +rank0 [2026-10-29 14:48:45,065] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:48:45,068] INFO : Initializing backbone layer 1 +rank0 [2026-10-29 14:48:45,068] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:48:45,071] INFO : Initializing backbone layer 2 +rank0 [2026-10-29 14:48:45,071] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:48:45,074] INFO : Initializing backbone layer 3 +rank0 [2026-10-29 14:48:45,074] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:48:45,077] INFO : Initializing backbone layer 4 +rank0 [2026-10-29 14:48:45,078] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:48:45,081] INFO : Initializing backbone layer 5 +rank0 [2026-10-29 14:48:45,081] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-29 14:48:45,084] INFO : Backbone initialization took 0.02s +rank0 [2026-10-29 14:48:45,084] INFO : backbone parameters: 2374656 +rank0 [2026-10-29 14:48:45,084] INFO : Initializing output DNNs +rank0 [2026-10-29 14:48:45,133] INFO : Output DNNs initialization took 0.05s +rank0 [2026-10-29 14:48:45,133] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-29 14:48:45,133] INFO : jagged_attention enabled=True +rank0 [2026-10-29 14:48:45,133] INFO : flash_attn_varlen enabled=True +rank0 [2026-10-29 14:48:45,133] INFO : task_queries enabled=False +rank0 [2026-10-29 14:48:45,133] INFO : classification_norm parameters: 0 +rank0 [2026-10-29 14:48:45,133] INFO : regression_norm parameters: 0 +rank0 [2026-10-29 14:48:45,133] INFO : classification_query parameters: 0 +rank0 [2026-10-29 14:48:45,133] INFO : regression_query parameters: 0 +rank0 [2026-10-29 14:48:45,133] INFO : classification_readout parameters: 0 +rank0 [2026-10-29 14:48:45,133] INFO : regression_readout parameters: 0 +rank0 [2026-10-29 14:48:45,133] INFO : output_mode=set +rank0 [2026-10-29 14:48:45,134] INFO : set_decoder parameters: 4713812 +rank0 [2026-10-29 14:48:45,134] INFO : nn_binary_particle parameters: 0 +rank0 [2026-10-29 14:48:45,134] INFO : nn_pid parameters: 0 +rank0 [2026-10-29 14:48:45,134] INFO : nn_pt parameters: 0 +rank0 [2026-10-29 14:48:45,134] INFO : nn_eta parameters: 0 +rank0 [2026-10-29 14:48:45,134] INFO : nn_sin_phi parameters: 0 +rank0 [2026-10-29 14:48:45,134] INFO : nn_cos_phi parameters: 0 +rank0 [2026-10-29 14:48:45,134] INFO : nn_energy parameters: 0 +rank0 [2026-10-29 14:48:45,134] INFO : Total MLPF parameters: 7260244 +rank0 [2026-10-29 14:48:45,134] INFO : MLPF __init__ done +rank0 [2026-10-29 14:48:45,136] INFO : Instantiated model +rank0 [2026-10-29 14:48:45,136] INFO : Moving model to device rank=0 +rank0 [2026-10-29 14:48:45,172] INFO : Moved model to device rank=0 +rank0 [2026-10-29 14:48:45,173] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-29 14:48:45,514] INFO : Restoring training from step 35001 +rank0 [2026-10-29 14:48:45,520] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth +rank0 [2026-10-29 14:48:45,521] INFO : Configured model for SyncBatchNorm rank=0 +rank0 [2026-10-29 14:48:45,764] INFO : Configured model for DistributedDataParallel rank=0 +rank0 [2026-10-29 14:48:45,876] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank0 [2026-10-29 14:48:45,876] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank0 [2026-10-29 14:48:45,877] INFO : Trainable parameters: 7260244 +rank0 [2026-10-29 14:48:45,878] INFO : Non-trainable parameters: 0 +rank0 [2026-10-29 14:48:45,878] INFO : Total parameters: 7260244 +rank0 [2026-10-29 14:48:45,880] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank0 [2026-10-29 14:48:45,885] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187 +rank0 [2026-10-29 14:48:45,885] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187 +rank0 [2026-10-29 14:48:45,885] INFO : Getting dataloaders +rank0 [2026-10-29 14:48:45,885] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:48:50,510] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:48:50,618] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:48:50,713] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:48:50,841] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:48:50,925] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:48:51,023] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:48:51,434] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:48:51,518] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:48:51,622] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:48:51,701] INFO : train_dataset: cld_edm_ttbar_hits, 90900 +rank0 [2026-10-29 14:48:51,701] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:48:51,782] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:48:51,864] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:48:51,951] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:48:52,087] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:48:52,186] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:48:52,317] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:48:52,399] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:48:52,510] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:48:52,633] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:48:52,754] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90900 +rank0 [2026-10-29 14:48:52,754] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:48:52,842] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-29 14:48:53,227] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-29 14:48:53,306] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-29 14:48:53,387] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-29 14:48:53,467] INFO : train_dataset: cld_edm_qq_hits, 89998 +rank0 [2026-10-29 14:48:53,544] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-29 14:48:53,624] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-29 14:48:53,706] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-29 14:48:53,830] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-29 14:48:53,923] INFO : train_dataset: cld_edm_qq_hits, 90599 +rank0 [2026-10-29 14:48:53,924] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-29 14:48:53,924] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-29 14:48:53,924] INFO : train mix: physical: weight=21112 step_share=1.000 batches_per_epoch=21112 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-29 14:48:53,924] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:48:53,951] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:48:53,978] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:48:54,005] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:48:54,033] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:48:54,061] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:48:54,088] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:48:54,116] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:48:54,150] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:48:54,176] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:48:54,203] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:48:54,203] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:48:54,230] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:48:54,257] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:48:54,288] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:48:54,314] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:48:54,340] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:48:54,366] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:48:54,393] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:48:54,419] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:48:54,445] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:48:54,474] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:48:54,474] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:48:54,499] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:48:54,532] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:48:54,558] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:48:54,591] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:48:54,616] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:48:54,641] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:48:54,665] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:48:54,690] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:48:54,715] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:48:54,741] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:48:54,741] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-29 14:48:54,742] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-29 14:49:08,546] INFO : Got dataloaders +rank0 [2026-10-29 14:49:08,546] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13888] of epochs [1] +rank0 [2026-10-29 14:49:08,546] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank0 [2026-10-29 14:49:08,547] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank0 [2026-10-29 14:49:08,547] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank0 [2026-10-29 14:49:10,476] INFO : loader split=train rank=0 len=21112 +rank0 [2026-10-29 14:49:10,476] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-29 14:49:10,476] INFO : Starting training from step 35001 to 50000 on rank 0 of 8 +rank0 [2026-10-29 14:49:10,482] INFO : Creating train_iterator +rank0 [2026-10-29 14:49:10,482] INFO : ResumableSampler physical:train loading state: start_index=222208 +rank0 [2026-10-29 14:49:24,314] INFO : Created train_iterator +rank0 [2026-10-29 14:53:12,961] INFO : Step 35100/50000 rank0 | Train Loss: 1.0826 | Train Loss Unweighted: 0.5185 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1393 | Classification: 0.0961 | Classification_binary: 0.1822 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1902 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0514 | Regression_eta: 0.0376 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0456 | LR: 2.83e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.2026s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.7340 | max_loss: 0.1651 | max_backward: 1.4152 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4761 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4625.8 | uss_MB: 3388.6 | child_rss_MB: 5578.9 | allocated_MB: 372.6 | reserved_MB: 22674.0 | inactive_split_MB: 823.1 +rank0 [2026-10-29 14:53:13,197] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 194.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 191.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 55.0°C 196.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 14:56:46,173] INFO : Step 35200/50000 rank0 | Train Loss: 1.0562 | Train Loss Unweighted: 0.5125 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1301 | Classification: 0.0878 | Classification_binary: 0.1540 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1331 | Regression_eta: 0.0017 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0624 | Regression_eta: 0.0457 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0520 | LR: 2.81e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8045s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6042 | max_loss: 0.1411 | max_backward: 1.1580 | max_optimizer_step: 0.1447 | max_tensorboard_logging: 0.5108 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4625.0 | uss_MB: 3391.6 | child_rss_MB: 5591.1 | allocated_MB: 368.1 | reserved_MB: 25668.0 | inactive_split_MB: 192.7 +rank0 [2026-10-29 14:56:46,411] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 186.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 192.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 56.0°C 202.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:00:19,972] INFO : Step 35300/50000 rank0 | Train Loss: 0.9962 | Train Loss Unweighted: 0.4725 | Train Loss Components: Auxiliary: 0.2160 | Cardinality: 0.1172 | Classification: 0.0927 | Classification_binary: 0.1341 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1529 | Regression_cos_phi: 0.0907 | Regression_energy: 0.0647 | Regression_eta: 0.0481 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0839 | LR: 2.79e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6777s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.5940 | max_loss: 0.1358 | max_backward: 1.1911 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0027 | Memory: rss_MB: 4636.4 | uss_MB: 3402.6 | child_rss_MB: 5598.9 | allocated_MB: 371.5 | reserved_MB: 25668.0 | inactive_split_MB: 207.6 +rank0 [2026-10-29 15:00:20,221] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 183.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 55.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:03:54,496] INFO : Step 35400/50000 rank0 | Train Loss: 1.1270 | Train Loss Unweighted: 0.5445 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1356 | Classification: 0.0990 | Classification_binary: 0.1805 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1297 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1831 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0577 | Regression_eta: 0.0477 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0756 | LR: 2.76e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2929s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1507 | max_backward: 1.5023 | max_optimizer_step: 0.2459 | max_tensorboard_logging: 0.5679 | max_tensorboard_flush: 0.0064 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4646.3 | uss_MB: 3410.8 | child_rss_MB: 5599.4 | allocated_MB: 371.4 | reserved_MB: 25816.0 | inactive_split_MB: 263.6 +rank0 [2026-10-29 15:03:54,734] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 55.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:07:34,580] INFO : Step 35500/50000 rank0 | Train Loss: 1.0502 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1302 | Classification: 0.0904 | Classification_binary: 0.1703 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1250 | Regression_eta: 0.0013 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1658 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0676 | Regression_eta: 0.0271 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0364 | LR: 2.74e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3062s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0737 | max_forward: 0.7546 | max_loss: 0.1581 | max_backward: 1.4977 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4644.7 | uss_MB: 3412.9 | child_rss_MB: 5568.4 | allocated_MB: 371.1 | reserved_MB: 25816.0 | inactive_split_MB: 263.4 +rank0 [2026-10-29 15:07:34,833] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 55.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 48.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:11:20,064] INFO : Step 35600/50000 rank0 | Train Loss: 1.0258 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2226 | Cardinality: 0.1119 | Classification: 0.0922 | Classification_binary: 0.1418 | HitGrouping: 0.1950 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1286 | Regression_eta: 0.0015 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0820 | Classification_binary: 0.1592 | Regression_cos_phi: 0.0802 | Regression_energy: 0.0836 | Regression_eta: 0.0599 | Regression_pt: 0.0850 | Regression_sin_phi: 0.1084 | LR: 2.72e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9267s | Phase Times: max_pre_step_barrier: 0.0929 | max_batch_to_device: 0.0064 | max_forward: 0.5696 | max_loss: 0.1430 | max_backward: 1.1627 | max_optimizer_step: 0.1475 | max_tensorboard_logging: 0.5805 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4646.9 | uss_MB: 3412.7 | child_rss_MB: 5602.7 | allocated_MB: 370.5 | reserved_MB: 25816.0 | inactive_split_MB: 212.3 +rank0 [2026-10-29 15:11:20,302] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 55.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 48.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:15:09,246] INFO : Step 35700/50000 rank0 | Train Loss: 1.0162 | Train Loss Unweighted: 0.5013 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1185 | Classification: 0.0946 | Classification_binary: 0.1537 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1248 | Regression_eta: 0.0015 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0794 | Classification_binary: 0.1899 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0651 | Regression_eta: 0.0672 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0786 | LR: 2.70e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2686s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7397 | max_loss: 0.1413 | max_backward: 1.6688 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4646.6 | uss_MB: 3411.7 | child_rss_MB: 5604.2 | allocated_MB: 372.2 | reserved_MB: 26084.0 | inactive_split_MB: 249.8 +rank0 [2026-10-29 15:15:09,487] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 47.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:18:50,105] INFO : Step 35800/50000 rank0 | Train Loss: 1.0884 | Train Loss Unweighted: 0.5265 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1296 | Classification: 0.0948 | Classification_binary: 0.1803 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0588 | Regression_eta: 0.0316 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0417 | LR: 2.68e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.8960s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0199 | max_forward: 0.9402 | max_loss: 0.2389 | max_backward: 1.7889 | max_optimizer_step: 0.2466 | max_tensorboard_logging: 0.7071 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4643.4 | uss_MB: 3414.5 | child_rss_MB: 5602.6 | allocated_MB: 371.6 | reserved_MB: 26084.0 | inactive_split_MB: 257.5 +rank0 [2026-10-29 15:18:50,342] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:22:31,531] INFO : Step 35900/50000 rank0 | Train Loss: 1.0760 | Train Loss Unweighted: 0.5299 | Train Loss Components: Auxiliary: 0.2328 | Cardinality: 0.1277 | Classification: 0.0983 | Classification_binary: 0.1568 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1349 | Regression_eta: 0.0016 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0640 | Regression_eta: 0.0456 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0695 | LR: 2.65e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9915s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6268 | max_loss: 0.2373 | max_backward: 1.2570 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.4934 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0016 | Memory: rss_MB: 4639.4 | uss_MB: 3412.6 | child_rss_MB: 5605.5 | allocated_MB: 369.7 | reserved_MB: 26800.0 | inactive_split_MB: 294.3 +rank0 [2026-10-29 15:22:31,767] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 55.0°C 191.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:26:13,811] INFO : Step 36000/50000 rank0 | Train Loss: 1.0831 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1276 | Classification: 0.1031 | Classification_binary: 0.1434 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1452 | Regression_eta: 0.0017 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0680 | Regression_eta: 0.0349 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0298 | LR: 2.63e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8698s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6266 | max_loss: 0.1437 | max_backward: 1.3209 | max_optimizer_step: 0.1464 | max_tensorboard_logging: 0.4977 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4620.1 | uss_MB: 3411.5 | child_rss_MB: 5582.3 | allocated_MB: 368.7 | reserved_MB: 26800.0 | inactive_split_MB: 270.9 +rank0 [2026-10-29 15:26:14,047] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 93% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 55.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 48.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 17% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:26:14,093] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-36000.pth +rank0 [2026-10-29 15:26:14,295] INFO : Running validation on rank0 +rank0 [2026-10-29 15:26:14,296] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 15:26:14,296] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION | Step=36000/50000 | Train Loss=1.0831 | Valid Loss=1.0386 | Train Loss Unweighted=0.5411 | Valid Loss Unweighted=0.4996 | Stale=0 | ETA=14.7m +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 15:27:03,899] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/ari=0.538894 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.63737 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/top1_competing=0.732644 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.613621 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0102591 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0391466 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 15:27:03,900] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/seconds_per_rank=21.4216 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding/ari=0.295908 +rank0 [2026-10-29 15:27:03,901] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding/ari_threshold=0.9 +rank0 [2026-10-29 15:27:03,902] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding_minus_raw/ari=0.204285 +rank0 [2026-10-29 15:27:03,902] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 15:27:03,902] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 15:27:03,902] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 15:27:03,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:27:04,250] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 15:27:04,263] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 15:27:29,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 15:27:30,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 15:27:31,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 15:27:32,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 15:27:42,727] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 15:27:42,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:27:43,313] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 15:27:43,317] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 15:27:59,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 15:28:00,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 15:28:01,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 15:28:02,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 15:28:12,639] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 15:28:12,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:28:13,125] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 15:28:13,129] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 15:28:29,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 15:28:29,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 15:28:30,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 15:28:31,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 15:28:41,351] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 15:28:41,367] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits +rank0 [2026-10-29 15:28:44,544] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits +rank0 [2026-10-29 15:28:45,811] INFO : Plotted number of elements +rank0 [2026-10-29 15:28:47,417] INFO : Plotted jets +rank0 [2026-10-29 15:28:48,581] INFO : Plotted jet ratio +rank0 [2026-10-29 15:28:48,632] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 15:28:51,796] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 15:28:53,111] INFO : Plotted number of elements +rank0 [2026-10-29 15:28:54,599] INFO : Plotted jets +rank0 [2026-10-29 15:28:55,706] INFO : Plotted jet ratio +rank0 [2026-10-29 15:28:55,759] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits +rank0 [2026-10-29 15:28:58,069] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits +rank0 [2026-10-29 15:28:58,924] INFO : Plotted number of elements +rank0 [2026-10-29 15:29:00,438] INFO : Plotted jets +rank0 [2026-10-29 15:29:01,583] INFO : Plotted jet ratio +rank0 [2026-10-29 15:32:51,152] INFO : Step 36100/50000 rank0 | Train Loss: 1.0137 | Train Loss Unweighted: 0.5026 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1079 | Classification: 0.1014 | Classification_binary: 0.1522 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1216 | Regression_eta: 0.0013 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0512 | Regression_eta: 0.0266 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0608 | LR: 2.61e-04 | DataLoad Time: 0.0118s | Model Forward Time: 4.6839s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0065 | max_forward: 0.6968 | max_loss: 0.2782 | max_backward: 1.4278 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.9130 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6101.3 | uss_MB: 4907.2 | child_rss_MB: 5609.1 | allocated_MB: 372.5 | reserved_MB: 23174.0 | inactive_split_MB: 246.6 +rank0 [2026-10-29 15:32:51,391] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 2% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 23% 0% +4 8 0x7408, 13395 55.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:36:34,583] INFO : Step 36200/50000 rank0 | Train Loss: 1.0439 | Train Loss Unweighted: 0.5147 | Train Loss Components: Auxiliary: 0.2338 | Cardinality: 0.1304 | Classification: 0.0909 | Classification_binary: 0.1697 | HitGrouping: 0.1917 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1255 | Regression_eta: 0.0011 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1455 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0762 | Regression_eta: 0.0304 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0741 | LR: 2.59e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2393s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7440 | max_loss: 0.1544 | max_backward: 1.4037 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 6101.4 | uss_MB: 4909.6 | child_rss_MB: 5587.9 | allocated_MB: 367.8 | reserved_MB: 23174.0 | inactive_split_MB: 282.8 +rank0 [2026-10-29 15:36:34,825] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +4 8 0x7408, 13395 54.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 47.0°C 191.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:40:21,608] INFO : Step 36300/50000 rank0 | Train Loss: 1.0379 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1216 | Classification: 0.0886 | Classification_binary: 0.1447 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1266 | Regression_eta: 0.0016 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0606 | Regression_eta: 0.0304 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0610 | LR: 2.57e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0059s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6106 | max_loss: 0.2412 | max_backward: 1.3407 | max_optimizer_step: 0.1535 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1063 | Memory: rss_MB: 5450.1 | uss_MB: 4269.0 | child_rss_MB: 5611.1 | allocated_MB: 370.5 | reserved_MB: 23174.0 | inactive_split_MB: 274.2 +rank0 [2026-10-29 15:40:21,863] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +4 8 0x7408, 13395 55.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:44:06,924] INFO : Step 36400/50000 rank0 | Train Loss: 1.0182 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.0976 | Classification: 0.0939 | Classification_binary: 0.1520 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1167 | Regression_eta: 0.0016 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0693 | Regression_eta: 0.0308 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0497 | LR: 2.55e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1492s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.6772 | max_loss: 0.1711 | max_backward: 1.4058 | max_optimizer_step: 0.1518 | max_tensorboard_logging: 0.5460 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0089 | Memory: rss_MB: 5451.9 | uss_MB: 4272.1 | child_rss_MB: 5615.2 | allocated_MB: 371.8 | reserved_MB: 23382.0 | inactive_split_MB: 230.6 +rank0 [2026-10-29 15:44:07,161] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:47:54,699] INFO : Step 36500/50000 rank0 | Train Loss: 1.0953 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1370 | Classification: 0.0895 | Classification_binary: 0.1838 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1771 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0658 | Regression_eta: 0.0449 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0642 | LR: 2.52e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6149s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7475 | max_loss: 0.2537 | max_backward: 1.7352 | max_optimizer_step: 0.1520 | max_tensorboard_logging: 0.6267 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5440.5 | uss_MB: 4262.4 | child_rss_MB: 5614.9 | allocated_MB: 372.3 | reserved_MB: 24160.0 | inactive_split_MB: 275.1 +rank0 [2026-10-29 15:47:54,940] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 99% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 93% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 95% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:51:40,788] INFO : Step 36600/50000 rank0 | Train Loss: 0.9967 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1165 | Classification: 0.0804 | Classification_binary: 0.1438 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1366 | Regression_eta: 0.0013 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1383 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0727 | Regression_eta: 0.0326 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0588 | LR: 2.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3351s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6592 | max_loss: 0.2501 | max_backward: 1.4213 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5953 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5450.1 | uss_MB: 4272.8 | child_rss_MB: 5617.4 | allocated_MB: 367.8 | reserved_MB: 27180.0 | inactive_split_MB: 225.4 +rank0 [2026-10-29 15:51:41,028] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:55:26,884] INFO : Step 36700/50000 rank0 | Train Loss: 0.9770 | Train Loss Unweighted: 0.4817 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1070 | Classification: 0.0901 | Classification_binary: 0.1347 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1578 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0591 | Regression_eta: 0.0313 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0546 | LR: 2.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9615s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0062 | max_forward: 0.5676 | max_loss: 0.2381 | max_backward: 1.1343 | max_optimizer_step: 0.2417 | max_tensorboard_logging: 0.4693 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5448.5 | uss_MB: 4274.1 | child_rss_MB: 5620.1 | allocated_MB: 370.8 | reserved_MB: 27180.0 | inactive_split_MB: 235.2 +rank0 [2026-10-29 15:55:27,119] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +4 8 0x7408, 13395 55.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:59:21,632] INFO : Step 36800/50000 rank0 | Train Loss: 1.0234 | Train Loss Unweighted: 0.4967 | Train Loss Components: Auxiliary: 0.2267 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1662 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1197 | Regression_eta: 0.0013 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.2051 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0615 | Regression_eta: 0.0271 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0618 | LR: 2.46e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.1611s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 1.1394 | max_loss: 0.1518 | max_backward: 2.3088 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5450.3 | uss_MB: 4278.9 | child_rss_MB: 5629.5 | allocated_MB: 371.9 | reserved_MB: 27180.0 | inactive_split_MB: 230.7 +rank0 [2026-10-29 15:59:21,872] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:03:12,091] INFO : Step 36900/50000 rank0 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2268 | Cardinality: 0.1211 | Classification: 0.0855 | Classification_binary: 0.1637 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1219 | Regression_eta: 0.0015 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0490 | Regression_eta: 0.0306 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0553 | LR: 2.44e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1790s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0071 | max_forward: 0.7005 | max_loss: 0.1465 | max_backward: 1.2640 | max_optimizer_step: 0.1579 | max_tensorboard_logging: 0.4728 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5451.1 | uss_MB: 4280.1 | child_rss_MB: 5636.6 | allocated_MB: 372.3 | reserved_MB: 27180.0 | inactive_split_MB: 201.7 +rank0 [2026-10-29 16:03:12,331] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 51.0°C 194.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 23% 100% +4 8 0x7408, 13395 55.0°C 197.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:07:15,819] INFO : Step 37000/50000 rank0 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2194 | Cardinality: 0.1268 | Classification: 0.0926 | Classification_binary: 0.1402 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0014 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1312 | Regression_cos_phi: 0.0351 | Regression_energy: 0.0500 | Regression_eta: 0.0207 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0507 | LR: 2.42e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1943s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6075 | max_loss: 0.2475 | max_backward: 1.2956 | max_optimizer_step: 0.1585 | max_tensorboard_logging: 0.7613 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5445.1 | uss_MB: 4277.2 | child_rss_MB: 5637.5 | allocated_MB: 368.1 | reserved_MB: 27180.0 | inactive_split_MB: 226.3 +rank0 [2026-10-29 16:07:16,057] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 53.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:07:16,081] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-37000.pth +rank0 [2026-10-29 16:07:16,401] INFO : Running validation on rank0 +rank0 [2026-10-29 16:07:16,401] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 16:07:16,401] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 16:08:14,586] INFO : VALIDATION | Step=37000/50000 | Train Loss=1.0150 | Valid Loss=1.0384 | Train Loss Unweighted=0.4848 | Valid Loss Unweighted=0.4981 | Stale=0 | ETA=27.8m +rank0 [2026-10-29 16:08:14,586] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:08:14,586] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 16:08:14,586] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 16:08:14,586] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:08:14,586] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 16:08:14,586] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 16:08:14,586] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 16:08:14,586] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 16:08:14,586] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/ari=0.532485 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.641521 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/top1_competing=0.737074 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.627479 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0166677 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0530046 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 16:08:14,587] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/seconds_per_rank=21.1808 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 16:08:14,588] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:08:14,589] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding/ari=0.293732 +rank0 [2026-10-29 16:08:14,589] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding/ari_threshold=0.875 +rank0 [2026-10-29 16:08:14,589] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding_minus_raw/ari=0.202109 +rank0 [2026-10-29 16:08:14,589] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 16:08:14,589] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 16:08:14,589] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 16:08:14,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:08:14,858] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 16:08:14,863] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 16:08:39,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 16:08:40,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 16:08:41,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 16:08:43,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 16:08:53,045] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 16:08:53,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:08:53,527] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 16:08:53,530] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 16:09:10,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 16:09:11,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 16:09:12,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 16:09:13,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 16:09:23,576] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 16:09:23,587] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:09:24,143] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 16:09:24,146] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 16:09:39,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 16:09:40,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 16:09:41,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 16:09:42,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 16:09:52,145] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 16:09:52,163] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits +rank0 [2026-10-29 16:09:55,354] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits +rank0 [2026-10-29 16:09:56,390] INFO : Plotted number of elements +rank0 [2026-10-29 16:09:57,957] INFO : Plotted jets +rank0 [2026-10-29 16:09:59,432] INFO : Plotted jet ratio +rank0 [2026-10-29 16:09:59,550] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 16:10:02,745] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 16:10:03,726] INFO : Plotted number of elements +rank0 [2026-10-29 16:10:05,233] INFO : Plotted jets +rank0 [2026-10-29 16:10:06,323] INFO : Plotted jet ratio +rank0 [2026-10-29 16:10:06,374] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits +rank0 [2026-10-29 16:10:08,790] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits +rank0 [2026-10-29 16:10:09,630] INFO : Plotted number of elements +rank0 [2026-10-29 16:10:11,109] INFO : Plotted jets +rank0 [2026-10-29 16:10:12,651] INFO : Plotted jet ratio +rank0 [2026-10-29 16:13:56,976] INFO : Step 37100/50000 rank0 | Train Loss: 1.1150 | Train Loss Unweighted: 0.5431 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1226 | Classification: 0.1105 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1340 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1801 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0624 | Regression_eta: 0.0387 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0633 | LR: 2.40e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5136s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7539 | max_loss: 0.3988 | max_backward: 1.7903 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4747 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6593.7 | uss_MB: 5426.2 | child_rss_MB: 5642.5 | allocated_MB: 372.5 | reserved_MB: 23160.0 | inactive_split_MB: 265.5 +rank0 [2026-10-29 16:13:57,214] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +2 6 0x7408, 15961 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +4 8 0x7408, 13395 55.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:17:47,099] INFO : Step 37200/50000 rank0 | Train Loss: 1.0752 | Train Loss Unweighted: 0.5181 | Train Loss Components: Auxiliary: 0.2406 | Cardinality: 0.1258 | Classification: 0.0924 | Classification_binary: 0.1812 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1200 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0628 | Regression_eta: 0.0468 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0650 | LR: 2.38e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.8215s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0549 | max_forward: 0.8058 | max_loss: 0.1510 | max_backward: 1.8476 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5280 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6591.2 | uss_MB: 5426.9 | child_rss_MB: 5644.6 | allocated_MB: 371.5 | reserved_MB: 24140.0 | inactive_split_MB: 271.0 +rank0 [2026-10-29 16:17:47,337] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:21:31,266] INFO : Step 37300/50000 rank0 | Train Loss: 1.0230 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1563 | HitGrouping: 0.1981 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1213 | Regression_eta: 0.0016 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1576 | Regression_cos_phi: 0.0926 | Regression_energy: 0.0597 | Regression_eta: 0.0579 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0960 | LR: 2.36e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6757s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.6853 | max_loss: 0.1502 | max_backward: 1.3245 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 1.0657 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5580.1 | uss_MB: 4415.2 | child_rss_MB: 5668.9 | allocated_MB: 370.9 | reserved_MB: 24140.0 | inactive_split_MB: 261.7 +rank0 [2026-10-29 16:21:31,520] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:25:14,691] INFO : Step 37400/50000 rank0 | Train Loss: 1.0130 | Train Loss Unweighted: 0.4929 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1180 | Classification: 0.0961 | Classification_binary: 0.1390 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1273 | Regression_eta: 0.0015 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0613 | Regression_eta: 0.0480 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0898 | LR: 2.34e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1643s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.5424 | max_loss: 0.1464 | max_backward: 1.0792 | max_optimizer_step: 0.1675 | max_tensorboard_logging: 0.8668 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5585.6 | uss_MB: 4420.8 | child_rss_MB: 5670.0 | allocated_MB: 368.8 | reserved_MB: 25122.0 | inactive_split_MB: 232.7 +rank0 [2026-10-29 16:26:49,211] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 43.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 45.0°C 123.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 46.0°C 123.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 44.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:30:42,536] INFO : Step 37500/50000 rank0 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1315 | Classification: 0.0992 | Classification_binary: 0.1679 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1176 | Regression_eta: 0.0013 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1539 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0543 | Regression_eta: 0.0433 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0540 | LR: 2.32e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.9782s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.7240 | max_loss: 0.1472 | max_backward: 1.5381 | max_optimizer_step: 0.2707 | max_tensorboard_logging: 0.8554 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5585.8 | uss_MB: 4422.3 | child_rss_MB: 5642.8 | allocated_MB: 370.2 | reserved_MB: 26134.0 | inactive_split_MB: 179.7 +rank0 [2026-10-29 16:30:42,793] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:34:39,014] INFO : Step 37600/50000 rank0 | Train Loss: 1.0243 | Train Loss Unweighted: 0.4867 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1182 | Classification: 0.0808 | Classification_binary: 0.1619 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0690 | Regression_eta: 0.0458 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0586 | LR: 2.30e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8166s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.7069 | max_loss: 0.1543 | max_backward: 1.5139 | max_optimizer_step: 0.1704 | max_tensorboard_logging: 1.0159 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5587.9 | uss_MB: 4422.7 | child_rss_MB: 5672.0 | allocated_MB: 370.8 | reserved_MB: 26846.0 | inactive_split_MB: 273.3 +rank0 [2026-10-29 16:34:39,260] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 55.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:38:30,017] INFO : Step 37700/50000 rank0 | Train Loss: 0.9610 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2102 | Cardinality: 0.1049 | Classification: 0.0974 | Classification_binary: 0.1317 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0012 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0555 | Regression_eta: 0.0330 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0385 | LR: 2.28e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2374s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6233 | max_loss: 0.2433 | max_backward: 1.3306 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5644 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5591.0 | uss_MB: 4425.5 | child_rss_MB: 5643.2 | allocated_MB: 369.4 | reserved_MB: 28822.0 | inactive_split_MB: 231.5 +rank0 [2026-10-29 16:38:30,254] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:42:35,997] INFO : Step 37800/50000 rank0 | Train Loss: 0.9994 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1105 | Classification: 0.0952 | Classification_binary: 0.1458 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1161 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0615 | Regression_eta: 0.0460 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0535 | LR: 2.26e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5774s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.6659 | max_loss: 0.1473 | max_backward: 1.3840 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.9558 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5593.5 | uss_MB: 4427.0 | child_rss_MB: 5671.1 | allocated_MB: 371.5 | reserved_MB: 28822.0 | inactive_split_MB: 170.1 +rank0 [2026-10-29 16:42:36,232] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 55.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:46:40,928] INFO : Step 37900/50000 rank0 | Train Loss: 1.0837 | Train Loss Unweighted: 0.5094 | Train Loss Components: Auxiliary: 0.2372 | Cardinality: 0.1419 | Classification: 0.0937 | Classification_binary: 0.1795 | HitGrouping: 0.1848 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1160 | Regression_eta: 0.0015 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0586 | Regression_eta: 0.0357 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0551 | LR: 2.24e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.0289s | Phase Times: max_pre_step_barrier: 0.1076 | max_batch_to_device: 0.0071 | max_forward: 0.7572 | max_loss: 0.1547 | max_backward: 1.4632 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 0.9205 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 5593.0 | uss_MB: 4429.3 | child_rss_MB: 5670.1 | allocated_MB: 370.2 | reserved_MB: 28822.0 | inactive_split_MB: 265.3 +rank0 [2026-10-29 16:46:41,182] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 47.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:50:32,173] INFO : Step 38000/50000 rank0 | Train Loss: 0.9811 | Train Loss Unweighted: 0.4782 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1213 | Classification: 0.0781 | Classification_binary: 0.1413 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1277 | Regression_eta: 0.0012 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1184 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0562 | Regression_eta: 0.0344 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0502 | LR: 2.22e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.1594s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.6120 | max_loss: 0.2171 | max_backward: 1.2526 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5836 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5590.4 | uss_MB: 4430.0 | child_rss_MB: 5667.2 | allocated_MB: 368.7 | reserved_MB: 28822.0 | inactive_split_MB: 223.4 +rank0 [2026-10-29 16:50:32,410] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 1% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 90% +4 8 0x7408, 13395 55.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 1% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:50:32,532] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-38000.pth +rank0 [2026-10-29 16:50:32,720] INFO : Running validation on rank0 +rank0 [2026-10-29 16:50:32,720] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 16:50:32,720] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 16:51:24,263] INFO : VALIDATION | Step=38000/50000 | Train Loss=0.9811 | Valid Loss=1.0294 | Train Loss Unweighted=0.4782 | Valid Loss Unweighted=0.4962 | Stale=0 | ETA=38.6m +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 16:51:24,264] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/ari=0.51594 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.620485 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/top1_competing=0.714666 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.626746 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0332123 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.052271 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 16:51:24,265] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/seconds_per_rank=21.2698 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding/ari=0.305597 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding/ari_threshold=0.9 +rank0 [2026-10-29 16:51:24,266] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding_minus_raw/ari=0.213974 +rank0 [2026-10-29 16:51:24,267] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 16:51:24,267] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 16:51:24,267] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 16:51:24,273] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:51:24,539] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 16:51:24,557] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 16:51:50,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 16:51:52,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 16:51:53,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 16:51:54,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 16:52:04,240] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 16:52:04,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:52:05,008] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 16:52:05,012] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 16:52:21,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 16:52:22,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 16:52:23,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 16:52:24,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 16:52:34,960] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 16:52:34,973] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:52:35,418] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 16:52:35,423] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 16:52:51,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 16:52:51,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 16:52:52,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 16:52:53,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 16:53:03,480] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 16:53:03,522] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits +rank0 [2026-10-29 16:53:06,777] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits +rank0 [2026-10-29 16:53:07,815] INFO : Plotted number of elements +rank0 [2026-10-29 16:53:09,392] INFO : Plotted jets +rank0 [2026-10-29 16:53:10,588] INFO : Plotted jet ratio +rank0 [2026-10-29 16:53:10,658] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 16:53:13,912] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 16:53:15,269] INFO : Plotted number of elements +rank0 [2026-10-29 16:53:16,795] INFO : Plotted jets +rank0 [2026-10-29 16:53:17,904] INFO : Plotted jet ratio +rank0 [2026-10-29 16:53:17,962] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits +rank0 [2026-10-29 16:53:20,241] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits +rank0 [2026-10-29 16:53:21,097] INFO : Plotted number of elements +rank0 [2026-10-29 16:53:22,576] INFO : Plotted jets +rank0 [2026-10-29 16:53:23,690] INFO : Plotted jet ratio +rank0 [2026-10-29 16:57:14,759] INFO : Step 38100/50000 rank0 | Train Loss: 1.0158 | Train Loss Unweighted: 0.4912 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1220 | Classification: 0.0982 | Classification_binary: 0.1455 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1224 | Regression_eta: 0.0013 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0567 | Regression_eta: 0.0624 | Regression_pt: 0.0590 | Regression_sin_phi: 0.1605 | LR: 2.20e-04 | DataLoad Time: 0.0250s | Model Forward Time: 5.6039s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8406 | max_loss: 0.1820 | max_backward: 1.8536 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.3354 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0179 | Memory: rss_MB: 5714.6 | uss_MB: 4482.4 | child_rss_MB: 5669.8 | allocated_MB: 370.5 | reserved_MB: 24292.0 | inactive_split_MB: 257.8 +rank0 [2026-10-29 16:57:14,998] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 2% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 80% +2 6 0x7408, 15961 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +4 8 0x7408, 13395 54.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:01:02,588] INFO : Step 38200/50000 rank0 | Train Loss: 1.0273 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2292 | Cardinality: 0.1060 | Classification: 0.0997 | Classification_binary: 0.1650 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0396 | Regression_energy: 0.0486 | Regression_eta: 0.0485 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0669 | LR: 2.18e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.5337s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7758 | max_loss: 0.1794 | max_backward: 1.7969 | max_optimizer_step: 0.1713 | max_tensorboard_logging: 0.4698 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5717.4 | uss_MB: 4558.1 | child_rss_MB: 5664.1 | allocated_MB: 372.9 | reserved_MB: 26212.0 | inactive_split_MB: 221.1 +rank0 [2026-10-29 17:01:02,826] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:05:07,873] INFO : Step 38300/50000 rank0 | Train Loss: 1.0498 | Train Loss Unweighted: 0.5122 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1249 | Classification: 0.0913 | Classification_binary: 0.1583 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1294 | Regression_eta: 0.0013 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0544 | Regression_eta: 0.0423 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0480 | LR: 2.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4518s | Phase Times: max_pre_step_barrier: 0.1103 | max_batch_to_device: 0.0086 | max_forward: 0.6730 | max_loss: 0.1509 | max_backward: 1.3787 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.4782 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5715.4 | uss_MB: 4556.5 | child_rss_MB: 5665.1 | allocated_MB: 369.2 | reserved_MB: 26212.0 | inactive_split_MB: 171.8 +rank0 [2026-10-29 17:05:08,126] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 92% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:09:09,041] INFO : Step 38400/50000 rank0 | Train Loss: 0.9354 | Train Loss Unweighted: 0.4735 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.0968 | Classification: 0.0760 | Classification_binary: 0.1246 | HitGrouping: 0.1889 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1340 | Regression_eta: 0.0013 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1499 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0686 | Regression_eta: 0.0553 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0544 | LR: 2.14e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.3146s | Phase Times: max_pre_step_barrier: 0.1107 | max_batch_to_device: 0.0062 | max_forward: 0.6635 | max_loss: 0.1419 | max_backward: 1.4771 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 0.4619 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5721.1 | uss_MB: 4559.4 | child_rss_MB: 5662.2 | allocated_MB: 370.2 | reserved_MB: 26212.0 | inactive_split_MB: 267.8 +rank0 [2026-10-29 17:09:09,299] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 55.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:13:06,808] INFO : Step 38500/50000 rank0 | Train Loss: 1.0482 | Train Loss Unweighted: 0.5077 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1228 | Classification: 0.1147 | Classification_binary: 0.1473 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1208 | Regression_eta: 0.0016 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0527 | Regression_eta: 0.0327 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0692 | LR: 2.12e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.1738s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.6383 | max_loss: 0.1393 | max_backward: 1.3136 | max_optimizer_step: 0.1695 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 5717.0 | uss_MB: 4562.8 | child_rss_MB: 5658.1 | allocated_MB: 370.2 | reserved_MB: 27188.0 | inactive_split_MB: 210.0 +rank0 [2026-10-29 17:13:07,045] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:17:05,481] INFO : Step 38600/50000 rank0 | Train Loss: 1.0321 | Train Loss Unweighted: 0.4950 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1157 | Classification: 0.0910 | Classification_binary: 0.1735 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1130 | Regression_eta: 0.0014 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0329 | Classification_binary: 0.1531 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0486 | Regression_eta: 0.0363 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0592 | LR: 2.11e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.7273s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0127 | max_forward: 0.7516 | max_loss: 0.1551 | max_backward: 1.6059 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.7250 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5714.3 | uss_MB: 4563.8 | child_rss_MB: 5662.6 | allocated_MB: 372.6 | reserved_MB: 28172.0 | inactive_split_MB: 283.9 +rank0 [2026-10-29 17:17:05,719] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:20:59,959] INFO : Step 38700/50000 rank0 | Train Loss: 0.9420 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1147 | Classification: 0.0828 | Classification_binary: 0.1348 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1155 | Regression_eta: 0.0013 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0692 | Regression_eta: 0.0360 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0497 | LR: 2.09e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.3975s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.8052 | max_loss: 0.2133 | max_backward: 1.3215 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5712.0 | uss_MB: 4410.4 | child_rss_MB: 5659.5 | allocated_MB: 367.4 | reserved_MB: 28172.0 | inactive_split_MB: 206.7 +rank0 [2026-10-29 17:21:00,202] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 50.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 55.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 95% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:25:08,909] INFO : Step 38800/50000 rank0 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4825 | Train Loss Components: Auxiliary: 0.2201 | Cardinality: 0.1186 | Classification: 0.0965 | Classification_binary: 0.1429 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1198 | Regression_eta: 0.0012 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1247 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0517 | Regression_eta: 0.0336 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0466 | LR: 2.07e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.3161s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.9011 | max_loss: 0.1880 | max_backward: 1.7487 | max_optimizer_step: 0.2166 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5710.9 | uss_MB: 4431.1 | child_rss_MB: 5666.7 | allocated_MB: 371.5 | reserved_MB: 28172.0 | inactive_split_MB: 220.5 +rank0 [2026-10-29 17:25:09,166] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 47.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:29:08,278] INFO : Step 38900/50000 rank0 | Train Loss: 0.9986 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2211 | Cardinality: 0.1078 | Classification: 0.0988 | Classification_binary: 0.1578 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0493 | Regression_eta: 0.0427 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0788 | LR: 2.05e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.2290s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0073 | max_forward: 0.9800 | max_loss: 0.2159 | max_backward: 2.2158 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0079 | Memory: rss_MB: 5714.2 | uss_MB: 3754.5 | child_rss_MB: 5665.7 | allocated_MB: 373.3 | reserved_MB: 29164.0 | inactive_split_MB: 189.2 +rank0 [2026-10-29 17:29:08,521] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 5% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +6 10 0x7408, 62036 47.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 47% 1% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:33:03,054] INFO : Step 39000/50000 rank0 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4758 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1227 | Classification: 0.0847 | Classification_binary: 0.1518 | HitGrouping: 0.1853 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1179 | Regression_eta: 0.0013 | Regression_pt: 0.1189 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0302 | Regression_energy: 0.0556 | Regression_eta: 0.0229 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0400 | LR: 2.03e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.4518s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0515 | max_forward: 0.7410 | max_loss: 0.1790 | max_backward: 1.4655 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5713.4 | uss_MB: 4569.9 | child_rss_MB: 5663.9 | allocated_MB: 370.5 | reserved_MB: 29164.0 | inactive_split_MB: 192.2 +rank0 [2026-10-29 17:33:03,291] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 48.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:33:03,321] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-39000.pth +rank0 [2026-10-29 17:33:03,502] INFO : Running validation on rank0 +rank0 [2026-10-29 17:33:03,502] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 17:33:03,502] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 17:33:56,851] INFO : VALIDATION | Step=39000/50000 | Train Loss=0.9857 | Valid Loss=1.0069 | Train Loss Unweighted=0.4758 | Valid Loss Unweighted=0.4865 | Stale=0 | ETA=46.5m +rank0 [2026-10-29 17:33:56,851] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 17:33:56,851] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 17:33:56,851] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 17:33:56,851] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/ari=0.527499 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.642361 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/top1_competing=0.738461 +rank0 [2026-10-29 17:33:56,852] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.637081 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0216534 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0626065 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 17:33:56,853] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/seconds_per_rank=21.5068 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding/ari=0.326662 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding/ari_threshold=0.9 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding_minus_raw/ari=0.235039 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 17:33:56,854] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 17:33:56,859] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:33:57,116] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 17:33:57,126] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 17:34:24,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 17:34:25,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 17:34:26,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 17:34:28,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 17:34:38,061] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 17:34:38,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:34:38,630] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 17:34:38,634] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 17:34:54,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 17:34:55,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 17:34:56,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 17:34:57,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 17:35:07,772] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 17:35:07,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:35:08,263] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 17:35:08,267] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 17:35:24,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 17:35:25,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 17:35:26,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 17:35:26,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 17:35:36,869] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 17:35:36,885] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits +rank0 [2026-10-29 17:35:40,140] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits +rank0 [2026-10-29 17:35:41,137] INFO : Plotted number of elements +rank0 [2026-10-29 17:35:42,690] INFO : Plotted jets +rank0 [2026-10-29 17:35:44,171] INFO : Plotted jet ratio +rank0 [2026-10-29 17:35:44,261] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 17:35:47,452] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 17:35:48,446] INFO : Plotted number of elements +rank0 [2026-10-29 17:35:49,980] INFO : Plotted jets +rank0 [2026-10-29 17:35:51,144] INFO : Plotted jet ratio +rank0 [2026-10-29 17:35:51,196] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits +rank0 [2026-10-29 17:35:53,666] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits +rank0 [2026-10-29 17:35:54,585] INFO : Plotted number of elements +rank0 [2026-10-29 17:35:56,132] INFO : Plotted jets +rank0 [2026-10-29 17:35:57,277] INFO : Plotted jet ratio +rank0 [2026-10-29 17:39:51,533] INFO : Step 39100/50000 rank0 | Train Loss: 1.0043 | Train Loss Unweighted: 0.4886 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1149 | Classification: 0.0878 | Classification_binary: 0.1320 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1329 | Regression_eta: 0.0012 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0468 | Regression_eta: 0.0331 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0696 | LR: 2.01e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4659s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0736 | max_forward: 0.7590 | max_loss: 0.4062 | max_backward: 1.4330 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.4941 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6810.5 | uss_MB: 5664.9 | child_rss_MB: 5641.2 | allocated_MB: 371.1 | reserved_MB: 23656.0 | inactive_split_MB: 194.8 +rank0 [2026-10-29 17:39:51,773] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:43:45,243] INFO : Step 39200/50000 rank0 | Train Loss: 1.0325 | Train Loss Unweighted: 0.5174 | Train Loss Components: Auxiliary: 0.2315 | Cardinality: 0.1117 | Classification: 0.1086 | Classification_binary: 0.1481 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.1697 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0608 | Regression_eta: 0.0284 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0569 | LR: 2.00e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.3555s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0102 | max_forward: 0.7546 | max_loss: 0.4268 | max_backward: 1.4370 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5801.6 | uss_MB: 4660.4 | child_rss_MB: 5666.2 | allocated_MB: 371.2 | reserved_MB: 23934.0 | inactive_split_MB: 196.6 +rank0 [2026-10-29 17:43:45,484] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +4 8 0x7408, 13395 54.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 95% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:47:42,958] INFO : Step 39300/50000 rank0 | Train Loss: 1.0373 | Train Loss Unweighted: 0.5050 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1241 | Classification: 0.0956 | Classification_binary: 0.1745 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1150 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0591 | Regression_eta: 0.0266 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0708 | LR: 1.98e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.0460s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0844 | max_forward: 0.8662 | max_loss: 0.2199 | max_backward: 1.7416 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5805.0 | uss_MB: 4500.0 | child_rss_MB: 5642.3 | allocated_MB: 373.1 | reserved_MB: 23934.0 | inactive_split_MB: 256.8 +rank0 [2026-10-29 17:47:43,197] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 54.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:51:43,805] INFO : Step 39400/50000 rank0 | Train Loss: 0.9741 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2189 | Cardinality: 0.1078 | Classification: 0.0808 | Classification_binary: 0.1461 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0520 | Regression_eta: 0.0330 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0354 | LR: 1.96e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.2604s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.6778 | max_loss: 0.1515 | max_backward: 1.4156 | max_optimizer_step: 0.1740 | max_tensorboard_logging: 0.4653 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5807.5 | uss_MB: 4663.4 | child_rss_MB: 5665.4 | allocated_MB: 368.0 | reserved_MB: 23934.0 | inactive_split_MB: 199.1 +rank0 [2026-10-29 17:51:44,046] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 54.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:55:40,866] INFO : Step 39500/50000 rank0 | Train Loss: 0.9809 | Train Loss Unweighted: 0.4664 | Train Loss Components: Auxiliary: 0.2175 | Cardinality: 0.1105 | Classification: 0.0963 | Classification_binary: 0.1347 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1158 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1615 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0398 | Regression_eta: 0.0337 | Regression_pt: 0.0416 | Regression_sin_phi: 0.0547 | LR: 1.94e-04 | DataLoad Time: 0.0083s | Model Forward Time: 4.3569s | Phase Times: max_pre_step_barrier: 0.0259 | max_batch_to_device: 0.1050 | max_forward: 0.6813 | max_loss: 0.1541 | max_backward: 1.3449 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 0.5375 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0111 | Memory: rss_MB: 5806.7 | uss_MB: 4606.4 | child_rss_MB: 5664.0 | allocated_MB: 367.2 | reserved_MB: 24970.0 | inactive_split_MB: 230.9 +rank0 [2026-10-29 17:55:41,104] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 55.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:59:42,773] INFO : Step 39600/50000 rank0 | Train Loss: 1.0814 | Train Loss Unweighted: 0.5099 | Train Loss Components: Auxiliary: 0.2411 | Cardinality: 0.1344 | Classification: 0.0978 | Classification_binary: 0.1731 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1913 | Regression_cos_phi: 0.0893 | Regression_energy: 0.0591 | Regression_eta: 0.0594 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0692 | LR: 1.93e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8060s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8839 | max_loss: 0.1464 | max_backward: 1.8057 | max_optimizer_step: 0.2568 | max_tensorboard_logging: 0.5748 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5805.3 | uss_MB: 4666.8 | child_rss_MB: 5646.1 | allocated_MB: 370.6 | reserved_MB: 24970.0 | inactive_split_MB: 243.4 +rank0 [2026-10-29 17:59:43,011] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:03:56,025] INFO : Step 39700/50000 rank0 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4877 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1185 | Classification: 0.0909 | Classification_binary: 0.1674 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1134 | Regression_eta: 0.0013 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0585 | Regression_eta: 0.0501 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0653 | LR: 1.91e-04 | DataLoad Time: 0.0141s | Model Forward Time: 4.4924s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.6954 | max_loss: 0.2515 | max_backward: 1.5651 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.5599 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5806.9 | uss_MB: 4667.8 | child_rss_MB: 5665.2 | allocated_MB: 371.1 | reserved_MB: 24970.0 | inactive_split_MB: 244.9 +rank0 [2026-10-29 18:03:56,262] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 54.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:07:57,387] INFO : Step 39800/50000 rank0 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1022 | Classification: 0.0711 | Classification_binary: 0.1322 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0290 | Classification_binary: 0.1115 | Regression_cos_phi: 0.0328 | Regression_energy: 0.0560 | Regression_eta: 0.0214 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0357 | LR: 1.89e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.6711s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0063 | max_forward: 1.0012 | max_loss: 0.2859 | max_backward: 2.1063 | max_optimizer_step: 0.1800 | max_tensorboard_logging: 0.4881 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5810.9 | uss_MB: 3839.1 | child_rss_MB: 5666.9 | allocated_MB: 368.3 | reserved_MB: 24970.0 | inactive_split_MB: 241.7 +rank0 [2026-10-29 18:07:57,627] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 6% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +4 8 0x7408, 13395 54.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 91% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 1% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:11:54,826] INFO : Step 39900/50000 rank0 | Train Loss: 0.9667 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.2188 | Cardinality: 0.1055 | Classification: 0.0897 | Classification_binary: 0.1526 | HitGrouping: 0.1883 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1101 | Regression_eta: 0.0012 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1703 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0420 | Regression_eta: 0.0419 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0411 | LR: 1.88e-04 | DataLoad Time: 0.0202s | Model Forward Time: 7.6665s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.3186 | max_loss: 0.4377 | max_backward: 3.6189 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.6123 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 5815.3 | uss_MB: 4217.6 | child_rss_MB: 5667.9 | allocated_MB: 370.5 | reserved_MB: 24970.0 | inactive_split_MB: 229.4 +rank0 [2026-10-29 18:11:55,065] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 49.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 53.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 47.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:16:02,068] INFO : Step 40000/50000 rank0 | Train Loss: 1.0232 | Train Loss Unweighted: 0.5131 | Train Loss Components: Auxiliary: 0.2348 | Cardinality: 0.1100 | Classification: 0.0962 | Classification_binary: 0.1712 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1206 | Regression_eta: 0.0013 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0570 | Regression_eta: 0.0298 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0453 | LR: 1.86e-04 | DataLoad Time: 0.0181s | Model Forward Time: 8.2157s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0077 | max_forward: 0.8168 | max_loss: 0.2161 | max_backward: 1.5709 | max_optimizer_step: 0.1733 | max_tensorboard_logging: 1.1681 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.1427 | Memory: rss_MB: 5816.6 | uss_MB: 3786.6 | child_rss_MB: 5665.4 | allocated_MB: 375.8 | reserved_MB: 24970.0 | inactive_split_MB: 259.2 +rank0 [2026-10-29 18:16:02,306] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 53.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 47.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:16:02,327] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-40000.pth +rank0 [2026-10-29 18:16:02,527] INFO : Running validation on rank0 +rank0 [2026-10-29 18:16:02,527] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 18:16:02,527] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 18:16:54,184] INFO : VALIDATION | Step=40000/50000 | Train Loss=1.0232 | Valid Loss=1.0124 | Train Loss Unweighted=0.5131 | Valid Loss Unweighted=0.4833 | Stale=1 | ETA=51.9m +rank0 [2026-10-29 18:16:54,184] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:16:54,184] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 18:16:54,184] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 18:16:54,184] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:16:54,184] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 18:16:54,184] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/ari=0.533615 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.628498 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/top1_competing=0.72172 +rank0 [2026-10-29 18:16:54,185] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.63003 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0155376 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.055555 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 18:16:54,186] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/seconds_per_rank=21.5972 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding/ari=0.298925 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 18:16:54,187] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding_minus_raw/ari=0.207302 +rank0 [2026-10-29 18:16:54,188] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 18:16:54,188] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 18:16:54,188] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 18:16:54,193] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:16:54,444] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 18:16:54,447] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 18:17:19,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 18:17:20,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 18:17:21,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 18:17:22,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 18:17:32,643] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:17:33,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:17:33,663] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 18:17:33,667] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 18:17:50,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 18:17:51,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 18:17:52,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 18:17:53,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 18:18:03,260] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 18:18:03,274] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:18:03,744] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 18:18:03,748] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 18:18:19,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 18:18:20,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 18:18:20,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 18:18:21,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 18:18:31,676] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 18:18:31,799] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits +rank0 [2026-10-29 18:18:35,005] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits +rank0 [2026-10-29 18:18:36,014] INFO : Plotted number of elements +rank0 [2026-10-29 18:18:37,565] INFO : Plotted jets +rank0 [2026-10-29 18:18:39,031] INFO : Plotted jet ratio +rank0 [2026-10-29 18:18:39,092] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 18:18:42,330] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 18:18:43,357] INFO : Plotted number of elements +rank0 [2026-10-29 18:18:44,892] INFO : Plotted jets +rank0 [2026-10-29 18:18:46,022] INFO : Plotted jet ratio +rank0 [2026-10-29 18:18:46,072] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits +rank0 [2026-10-29 18:18:48,510] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits +rank0 [2026-10-29 18:18:49,359] INFO : Plotted number of elements +rank0 [2026-10-29 18:18:50,862] INFO : Plotted jets +rank0 [2026-10-29 18:18:52,415] INFO : Plotted jet ratio +rank0 [2026-10-29 18:22:47,441] INFO : Step 40100/50000 rank0 | Train Loss: 0.9349 | Train Loss Unweighted: 0.4412 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1220 | Classification: 0.0762 | Classification_binary: 0.1328 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1148 | Regression_eta: 0.0012 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1173 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0477 | Regression_eta: 0.0269 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0308 | LR: 1.84e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5733s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.7093 | max_loss: 0.3974 | max_backward: 1.3310 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.5555 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 7103.1 | uss_MB: 5953.7 | child_rss_MB: 5672.5 | allocated_MB: 367.6 | reserved_MB: 22178.0 | inactive_split_MB: 265.9 +rank0 [2026-10-29 18:22:47,691] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:26:52,808] INFO : Step 40200/50000 rank0 | Train Loss: 0.9570 | Train Loss Unweighted: 0.4556 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1119 | Classification: 0.0905 | Classification_binary: 0.1293 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0013 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0495 | Regression_eta: 0.0407 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0644 | LR: 1.83e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2094s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6287 | max_loss: 0.4129 | max_backward: 1.3901 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 0.5609 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5918.3 | uss_MB: 4768.5 | child_rss_MB: 5665.4 | allocated_MB: 370.8 | reserved_MB: 22294.0 | inactive_split_MB: 233.2 +rank0 [2026-10-29 18:26:53,044] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 47.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:31:00,291] INFO : Step 40300/50000 rank0 | Train Loss: 1.0209 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1232 | Classification: 0.0991 | Classification_binary: 0.1688 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1068 | Regression_eta: 0.0015 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0475 | Regression_eta: 0.0355 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0445 | LR: 1.81e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7194s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0749 | max_forward: 0.7563 | max_loss: 0.1523 | max_backward: 1.5452 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5632 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5917.0 | uss_MB: 4772.2 | child_rss_MB: 5652.3 | allocated_MB: 371.8 | reserved_MB: 23296.0 | inactive_split_MB: 234.2 +rank0 [2026-10-29 18:31:00,529] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:35:21,552] INFO : Step 40400/50000 rank0 | Train Loss: 1.0486 | Train Loss Unweighted: 0.5003 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1357 | Classification: 0.0888 | Classification_binary: 0.1690 | HitGrouping: 0.1879 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1196 | Regression_eta: 0.0015 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0685 | Regression_eta: 0.0611 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0329 | LR: 1.79e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.3541s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0072 | max_forward: 0.7368 | max_loss: 0.2443 | max_backward: 3.1596 | max_optimizer_step: 0.1851 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5913.1 | uss_MB: 3802.2 | child_rss_MB: 5663.8 | allocated_MB: 371.2 | reserved_MB: 23386.0 | inactive_split_MB: 254.8 +rank0 [2026-10-29 18:35:21,790] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:39:24,615] INFO : Step 40500/50000 rank0 | Train Loss: 0.9309 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.0955 | Classification: 0.0829 | Classification_binary: 0.1355 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0014 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1447 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0464 | Regression_eta: 0.0480 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0833 | LR: 1.78e-04 | DataLoad Time: 0.0105s | Model Forward Time: 4.4830s | Phase Times: max_pre_step_barrier: 0.0186 | max_batch_to_device: 0.0076 | max_forward: 0.6863 | max_loss: 0.2393 | max_backward: 1.3065 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.7004 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5916.1 | uss_MB: 4771.6 | child_rss_MB: 5669.9 | allocated_MB: 369.2 | reserved_MB: 24370.0 | inactive_split_MB: 237.3 +rank0 [2026-10-29 18:39:24,856] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 53.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:43:24,563] INFO : Step 40600/50000 rank0 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1087 | Classification: 0.0887 | Classification_binary: 0.1521 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0523 | Regression_eta: 0.0258 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0559 | LR: 1.76e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9833s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.1102 | max_forward: 0.7412 | max_loss: 0.1464 | max_backward: 1.4782 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 1.0719 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5917.7 | uss_MB: 4772.7 | child_rss_MB: 5655.3 | allocated_MB: 372.3 | reserved_MB: 24370.0 | inactive_split_MB: 234.4 +rank0 [2026-10-29 18:43:24,820] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 47.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:47:15,499] INFO : Step 40700/50000 rank0 | Train Loss: 1.0611 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1374 | Classification: 0.0904 | Classification_binary: 0.1773 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1190 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0544 | Regression_eta: 0.0385 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0582 | LR: 1.75e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9021s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7422 | max_loss: 0.1589 | max_backward: 1.5298 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5918.6 | uss_MB: 4774.9 | child_rss_MB: 5665.5 | allocated_MB: 372.7 | reserved_MB: 24370.0 | inactive_split_MB: 255.3 +rank0 [2026-10-29 18:47:15,753] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:51:04,044] INFO : Step 40800/50000 rank0 | Train Loss: 0.9701 | Train Loss Unweighted: 0.4785 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.0983 | Classification: 0.0901 | Classification_binary: 0.1373 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0705 | Regression_eta: 0.0432 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0841 | LR: 1.73e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3997s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5843 | max_loss: 0.1422 | max_backward: 1.0661 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.0389 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5915.8 | uss_MB: 4772.0 | child_rss_MB: 5661.2 | allocated_MB: 367.5 | reserved_MB: 24658.0 | inactive_split_MB: 205.4 +rank0 [2026-10-29 18:51:04,292] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:54:43,923] INFO : Step 40900/50000 rank0 | Train Loss: 0.9731 | Train Loss Unweighted: 0.4636 | Train Loss Components: Auxiliary: 0.2169 | Cardinality: 0.1073 | Classification: 0.0883 | Classification_binary: 0.1426 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1140 | Regression_eta: 0.0013 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0314 | Regression_energy: 0.0504 | Regression_eta: 0.0237 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0648 | LR: 1.72e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5452s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.1047 | max_forward: 0.7107 | max_loss: 0.1679 | max_backward: 1.3343 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.8664 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5919.0 | uss_MB: 4778.3 | child_rss_MB: 5681.4 | allocated_MB: 372.0 | reserved_MB: 25640.0 | inactive_split_MB: 260.9 +rank0 [2026-10-29 18:54:44,166] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:58:26,516] INFO : Step 41000/50000 rank0 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1031 | Classification: 0.0930 | Classification_binary: 0.1618 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1164 | Regression_eta: 0.0014 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1564 | Regression_cos_phi: 0.0355 | Regression_energy: 0.0550 | Regression_eta: 0.0265 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0498 | LR: 1.70e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1759s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.7604 | max_loss: 0.2558 | max_backward: 1.7603 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.8399 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 5926.0 | uss_MB: 4782.0 | child_rss_MB: 5654.3 | allocated_MB: 371.9 | reserved_MB: 25640.0 | inactive_split_MB: 195.8 +rank0 [2026-10-29 18:58:26,763] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:58:26,786] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-41000.pth +rank0 [2026-10-29 18:58:27,078] INFO : Running validation on rank0 +rank0 [2026-10-29 18:58:27,078] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 18:58:27,078] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 18:59:15,274] INFO : VALIDATION | Step=41000/50000 | Train Loss=1.0063 | Valid Loss=1.0002 | Train Loss Unweighted=0.4913 | Valid Loss Unweighted=0.4768 | Stale=0 | ETA=54.9m +rank0 [2026-10-29 18:59:15,274] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:59:15,274] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 18:59:15,274] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 18:59:15,274] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/ari=0.552223 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.653185 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/top1_competing=0.753808 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.623586 +rank0 [2026-10-29 18:59:15,275] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.00307047 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0491112 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/seconds_per_rank=21.2208 +rank0 [2026-10-29 18:59:15,276] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding/ari=0.270787 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding/ari_threshold=0.9125 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding_minus_raw/ari=0.179164 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 18:59:15,277] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 18:59:15,284] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:59:15,527] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 18:59:15,538] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 18:59:40,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 18:59:41,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 18:59:42,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 18:59:43,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 18:59:53,531] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 18:59:53,543] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:59:54,028] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 18:59:54,032] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 19:00:10,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 19:00:11,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 19:00:12,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 19:00:13,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 19:00:23,196] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 19:00:23,207] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:00:23,684] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 19:00:23,688] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 19:00:39,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 19:00:39,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 19:00:40,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 19:00:41,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 19:00:51,363] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 19:00:51,503] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits +rank0 [2026-10-29 19:00:54,645] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits +rank0 [2026-10-29 19:00:55,684] INFO : Plotted number of elements +rank0 [2026-10-29 19:00:57,241] INFO : Plotted jets +rank0 [2026-10-29 19:00:58,722] INFO : Plotted jet ratio +rank0 [2026-10-29 19:00:58,823] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 19:01:01,998] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 19:01:03,006] INFO : Plotted number of elements +rank0 [2026-10-29 19:01:04,520] INFO : Plotted jets +rank0 [2026-10-29 19:01:05,630] INFO : Plotted jet ratio +rank0 [2026-10-29 19:01:05,705] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits +rank0 [2026-10-29 19:01:08,123] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits +rank0 [2026-10-29 19:01:08,978] INFO : Plotted number of elements +rank0 [2026-10-29 19:01:10,469] INFO : Plotted jets +rank0 [2026-10-29 19:01:12,023] INFO : Plotted jet ratio +rank0 [2026-10-29 19:04:59,107] INFO : Step 41100/50000 rank0 | Train Loss: 1.0684 | Train Loss Unweighted: 0.5053 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1438 | Classification: 0.0856 | Classification_binary: 0.1616 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1271 | Regression_eta: 0.0014 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1722 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0699 | Regression_eta: 0.0342 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0677 | LR: 1.69e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.1815s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0066 | max_forward: 0.8006 | max_loss: 0.1726 | max_backward: 1.6699 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.6214 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0566 | Memory: rss_MB: 7184.9 | uss_MB: 5901.7 | child_rss_MB: 5647.6 | allocated_MB: 371.5 | reserved_MB: 22898.0 | inactive_split_MB: 258.4 +rank0 [2026-10-29 19:04:59,349] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 62% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:08:42,291] INFO : Step 41200/50000 rank0 | Train Loss: 0.9455 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1027 | Classification: 0.0887 | Classification_binary: 0.1242 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1223 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1264 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0627 | Regression_eta: 0.0591 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0642 | LR: 1.67e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0780s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0589 | max_forward: 0.6517 | max_loss: 0.1401 | max_backward: 1.1761 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5685 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5984.6 | uss_MB: 4847.1 | child_rss_MB: 5628.1 | allocated_MB: 370.2 | reserved_MB: 24878.0 | inactive_split_MB: 199.0 +rank0 [2026-10-29 19:08:42,528] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:12:29,064] INFO : Step 41300/50000 rank0 | Train Loss: 0.9373 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1011 | Classification: 0.0853 | Classification_binary: 0.1448 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0013 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2004 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0519 | Regression_eta: 0.0613 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0816 | LR: 1.66e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6268s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.8726 | max_loss: 0.1617 | max_backward: 1.6974 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5417 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5987.6 | uss_MB: 4852.4 | child_rss_MB: 5623.6 | allocated_MB: 370.7 | reserved_MB: 24878.0 | inactive_split_MB: 233.6 +rank0 [2026-10-29 19:12:29,316] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:16:16,107] INFO : Step 41400/50000 rank0 | Train Loss: 1.0355 | Train Loss Unweighted: 0.4965 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1363 | Classification: 0.0957 | Classification_binary: 0.1654 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1162 | Regression_eta: 0.0012 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0469 | Regression_eta: 0.0275 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0443 | LR: 1.64e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5610s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7129 | max_loss: 0.2460 | max_backward: 1.6340 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5967.8 | uss_MB: 4849.9 | child_rss_MB: 5563.5 | allocated_MB: 372.0 | reserved_MB: 24878.0 | inactive_split_MB: 305.4 +rank0 [2026-10-29 19:16:16,345] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:19:53,341] INFO : Step 41500/50000 rank0 | Train Loss: 0.9689 | Train Loss Unweighted: 0.4689 | Train Loss Components: Auxiliary: 0.2118 | Cardinality: 0.1040 | Classification: 0.0862 | Classification_binary: 0.1340 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1227 | Regression_eta: 0.0012 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1203 | Regression_cos_phi: 0.0802 | Regression_energy: 0.0642 | Regression_eta: 0.0316 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0539 | LR: 1.63e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.0736s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.6415 | max_loss: 0.1502 | max_backward: 1.1845 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5935.8 | uss_MB: 4853.4 | child_rss_MB: 5403.5 | allocated_MB: 368.5 | reserved_MB: 24878.0 | inactive_split_MB: 173.5 +rank0 [2026-10-29 19:19:53,598] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 60.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:23:41,061] INFO : Step 41600/50000 rank0 | Train Loss: 0.9559 | Train Loss Unweighted: 0.4625 | Train Loss Components: Auxiliary: 0.2141 | Cardinality: 0.1097 | Classification: 0.0872 | Classification_binary: 0.1298 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1211 | Regression_eta: 0.0012 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1255 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0642 | Regression_eta: 0.0468 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0546 | LR: 1.61e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.2536s | Phase Times: max_pre_step_barrier: 0.0755 | max_batch_to_device: 0.0062 | max_forward: 0.6849 | max_loss: 0.2047 | max_backward: 1.1646 | max_optimizer_step: 0.2235 | max_tensorboard_logging: 0.5806 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5909.3 | uss_MB: 4827.6 | child_rss_MB: 5308.7 | allocated_MB: 370.4 | reserved_MB: 24878.0 | inactive_split_MB: 287.4 +rank0 [2026-10-29 19:23:41,299] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 6% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 93% +2 6 0x7408, 15961 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 92% +6 10 0x7408, 62036 49.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:27:24,420] INFO : Step 41700/50000 rank0 | Train Loss: 1.0204 | Train Loss Unweighted: 0.4905 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1165 | Classification: 0.0860 | Classification_binary: 0.1552 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1668 | Regression_cos_phi: 0.0436 | Regression_energy: 0.0574 | Regression_eta: 0.0302 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0606 | LR: 1.60e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.7063s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0773 | max_forward: 0.7754 | max_loss: 0.1486 | max_backward: 1.5759 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 0.5773 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5907.3 | uss_MB: 4857.3 | child_rss_MB: 5302.2 | allocated_MB: 370.8 | reserved_MB: 24878.0 | inactive_split_MB: 238.3 +rank0 [2026-10-29 19:27:24,657] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 55.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 78% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:31:24,977] INFO : Step 41800/50000 rank0 | Train Loss: 1.0101 | Train Loss Unweighted: 0.4703 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1282 | Classification: 0.0818 | Classification_binary: 0.1618 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0299 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0599 | Regression_eta: 0.0353 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0403 | LR: 1.58e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6412s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8147 | max_loss: 0.4259 | max_backward: 1.4886 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1200 | Memory: rss_MB: 5899.0 | uss_MB: 4855.0 | child_rss_MB: 5276.1 | allocated_MB: 371.9 | reserved_MB: 25802.0 | inactive_split_MB: 266.5 +rank0 [2026-10-29 19:31:25,228] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:35:33,022] INFO : Step 41900/50000 rank0 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4672 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1185 | Classification: 0.0878 | Classification_binary: 0.1282 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1241 | Regression_eta: 0.0011 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1215 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0534 | Regression_eta: 0.0335 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0649 | LR: 1.57e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1372s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0473 | max_forward: 0.6783 | max_loss: 0.1431 | max_backward: 1.2347 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.5634 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5868.0 | uss_MB: 4858.2 | child_rss_MB: 5156.0 | allocated_MB: 370.8 | reserved_MB: 25802.0 | inactive_split_MB: 166.5 +rank0 [2026-10-29 19:35:33,261] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:39:32,697] INFO : Step 42000/50000 rank0 | Train Loss: 0.9837 | Train Loss Unweighted: 0.4719 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.1093 | Classification: 0.1015 | Classification_binary: 0.1413 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1128 | Regression_eta: 0.0014 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1396 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0556 | Regression_eta: 0.0347 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0761 | LR: 1.56e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.3190s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6848 | max_loss: 0.2509 | max_backward: 1.3746 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5866.0 | uss_MB: 4860.4 | child_rss_MB: 5161.1 | allocated_MB: 370.2 | reserved_MB: 25802.0 | inactive_split_MB: 219.9 +rank0 [2026-10-29 19:39:32,938] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 54.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:39:32,959] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-42000.pth +rank0 [2026-10-29 19:39:33,272] INFO : Running validation on rank0 +rank0 [2026-10-29 19:39:33,272] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 19:39:33,273] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION | Step=42000/50000 | Train Loss=0.9837 | Valid Loss=1.0030 | Train Loss Unweighted=0.4719 | Valid Loss Unweighted=0.4808 | Stale=1 | ETA=55.5m +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 19:40:23,917] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/ari=0.539037 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.630508 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/top1_competing=0.725283 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.643643 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0101161 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0691678 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 19:40:23,918] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/seconds_per_rank=21.64 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 19:40:23,919] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding/ari=0.322868 +rank0 [2026-10-29 19:40:23,920] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 19:40:23,920] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding_minus_raw/ari=0.231245 +rank0 [2026-10-29 19:40:23,920] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 19:40:23,920] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 19:40:23,920] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 19:40:23,925] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:40:24,179] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 19:40:24,194] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 19:40:52,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 19:40:53,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 19:40:55,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 19:40:56,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 19:41:06,163] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 19:41:06,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:41:06,795] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 19:41:06,798] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 19:41:23,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 19:41:25,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 19:41:27,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 19:41:28,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 19:41:38,156] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 19:41:38,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:41:38,674] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 19:41:38,678] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 19:41:54,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 19:41:54,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 19:41:55,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 19:41:56,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 19:42:06,295] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 19:42:06,537] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits +rank0 [2026-10-29 19:42:09,667] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits +rank0 [2026-10-29 19:42:10,700] INFO : Plotted number of elements +rank0 [2026-10-29 19:42:12,260] INFO : Plotted jets +rank0 [2026-10-29 19:42:13,439] INFO : Plotted jet ratio +rank0 [2026-10-29 19:42:13,512] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 19:42:16,842] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 19:42:17,827] INFO : Plotted number of elements +rank0 [2026-10-29 19:42:19,325] INFO : Plotted jets +rank0 [2026-10-29 19:42:20,436] INFO : Plotted jet ratio +rank0 [2026-10-29 19:42:20,499] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits +rank0 [2026-10-29 19:42:22,906] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits +rank0 [2026-10-29 19:42:23,739] INFO : Plotted number of elements +rank0 [2026-10-29 19:42:25,222] INFO : Plotted jets +rank0 [2026-10-29 19:42:26,362] INFO : Plotted jet ratio +rank0 [2026-10-29 19:46:30,233] INFO : Step 42100/50000 rank0 | Train Loss: 1.0639 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1434 | Classification: 0.0999 | Classification_binary: 0.1737 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1097 | Regression_eta: 0.0012 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0486 | Regression_eta: 0.0347 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0494 | LR: 1.54e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.8398s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.7424 | max_loss: 0.1586 | max_backward: 1.4434 | max_optimizer_step: 0.3014 | max_tensorboard_logging: 0.6771 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 7018.7 | uss_MB: 6010.1 | child_rss_MB: 5164.1 | allocated_MB: 370.8 | reserved_MB: 23298.0 | inactive_split_MB: 265.3 +rank0 [2026-10-29 19:46:30,470] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 55.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 47.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:50:22,313] INFO : Step 42200/50000 rank0 | Train Loss: 1.0152 | Train Loss Unweighted: 0.4831 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1169 | Classification: 0.0913 | Classification_binary: 0.1515 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1190 | Regression_eta: 0.0015 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1464 | Regression_cos_phi: 0.1108 | Regression_energy: 0.0699 | Regression_eta: 0.0482 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0635 | LR: 1.53e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.1937s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.6567 | max_loss: 0.2000 | max_backward: 1.2379 | max_optimizer_step: 0.1803 | max_tensorboard_logging: 0.5782 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5943.7 | uss_MB: 4948.2 | child_rss_MB: 5171.4 | allocated_MB: 370.9 | reserved_MB: 24984.0 | inactive_split_MB: 272.4 +rank0 [2026-10-29 19:50:22,552] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 5% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 95% +2 6 0x7408, 15961 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 54.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 2% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:51:24,069] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-29 19:51:24,069] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-29 19:54:16,123] INFO : Step 42300/50000 rank0 | Train Loss: 1.0224 | Train Loss Unweighted: 0.4869 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.1261 | Classification: 0.0943 | Classification_binary: 0.1704 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0487 | Regression_eta: 0.0391 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0538 | LR: 1.52e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4558s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7065 | max_loss: 0.2237 | max_backward: 1.4902 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.5353 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6010.9 | uss_MB: 5019.0 | child_rss_MB: 5170.2 | allocated_MB: 371.9 | reserved_MB: 24984.0 | inactive_split_MB: 183.6 +rank0 [2026-10-29 19:54:16,362] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 50.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 55.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:58:13,934] INFO : Step 42400/50000 rank0 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4895 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1253 | Classification: 0.0872 | Classification_binary: 0.1616 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1184 | Regression_eta: 0.0013 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1417 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0597 | Regression_eta: 0.0361 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0389 | LR: 1.50e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4460s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7436 | max_loss: 0.2477 | max_backward: 1.5279 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5169 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5933.8 | uss_MB: 4947.5 | child_rss_MB: 5164.9 | allocated_MB: 373.0 | reserved_MB: 24984.0 | inactive_split_MB: 275.6 +rank0 [2026-10-29 19:58:14,175] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 55.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:02:13,952] INFO : Step 42500/50000 rank0 | Train Loss: 0.9211 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1056 | Classification: 0.0789 | Classification_binary: 0.1296 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1162 | Regression_eta: 0.0011 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0678 | Regression_eta: 0.0307 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0528 | LR: 1.49e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.2227s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.6196 | max_loss: 0.2440 | max_backward: 1.3403 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 2.3828 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5926.8 | uss_MB: 4949.7 | child_rss_MB: 5158.5 | allocated_MB: 371.3 | reserved_MB: 25980.0 | inactive_split_MB: 274.9 +rank0 [2026-10-29 20:02:14,205] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 53.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:06:18,577] INFO : Step 42600/50000 rank0 | Train Loss: 0.9688 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1101 | Classification: 0.0874 | Classification_binary: 0.1472 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0012 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0502 | Regression_eta: 0.0299 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0315 | LR: 1.48e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6662s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6353 | max_loss: 0.2547 | max_backward: 1.3961 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 5918.3 | uss_MB: 4947.3 | child_rss_MB: 5167.3 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 272.5 +rank0 [2026-10-29 20:06:18,817] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 5% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 55.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:10:17,576] INFO : Step 42700/50000 rank0 | Train Loss: 1.0476 | Train Loss Unweighted: 0.4808 | Train Loss Components: Auxiliary: 0.2377 | Cardinality: 0.1284 | Classification: 0.0883 | Classification_binary: 0.1739 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1071 | Regression_eta: 0.0014 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1912 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0470 | Regression_eta: 0.0725 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0706 | LR: 1.47e-04 | DataLoad Time: 0.0125s | Model Forward Time: 6.4453s | Phase Times: max_pre_step_barrier: 0.0320 | max_batch_to_device: 0.0114 | max_forward: 2.1726 | max_loss: 0.5101 | max_backward: 2.7494 | max_optimizer_step: 0.2679 | max_tensorboard_logging: 0.6520 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5921.8 | uss_MB: 4952.4 | child_rss_MB: 5151.8 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 482.9 +rank0 [2026-10-29 20:10:17,814] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:14:17,771] INFO : Step 42800/50000 rank0 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4792 | Train Loss Components: Auxiliary: 0.2262 | Cardinality: 0.1277 | Classification: 0.0841 | Classification_binary: 0.1487 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1209 | Regression_eta: 0.0012 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1170 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0609 | Regression_eta: 0.0204 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0276 | LR: 1.45e-04 | DataLoad Time: 0.0217s | Model Forward Time: 6.9995s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0175 | max_forward: 1.3870 | max_loss: 0.5106 | max_backward: 1.9221 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 2.4017 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5924.3 | uss_MB: 4952.5 | child_rss_MB: 5165.0 | allocated_MB: 372.6 | reserved_MB: 25980.0 | inactive_split_MB: 234.4 +rank0 [2026-10-29 20:14:18,035] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:18:10,120] INFO : Step 42900/50000 rank0 | Train Loss: 0.8847 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.0993 | Classification: 0.0847 | Classification_binary: 0.1207 | HitGrouping: 0.1828 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1069 | Regression_eta: 0.0011 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0498 | Regression_eta: 0.0437 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0475 | LR: 1.44e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3954s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.6133 | max_loss: 0.1478 | max_backward: 1.3445 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.7551 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0762 | Memory: rss_MB: 5924.9 | uss_MB: 4957.1 | child_rss_MB: 5179.7 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 276.9 +rank0 [2026-10-29 20:18:10,373] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:22:02,904] INFO : Step 43000/50000 rank0 | Train Loss: 0.9800 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1096 | Classification: 0.0987 | Classification_binary: 0.1606 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0012 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0530 | Regression_eta: 0.0367 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0601 | LR: 1.43e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.6356s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0071 | max_forward: 0.7477 | max_loss: 0.1477 | max_backward: 1.5663 | max_optimizer_step: 0.1600 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 5929.7 | uss_MB: 4961.8 | child_rss_MB: 5171.4 | allocated_MB: 369.1 | reserved_MB: 25980.0 | inactive_split_MB: 209.4 +rank0 [2026-10-29 20:22:03,142] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 50.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:22:03,162] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-43000.pth +rank0 [2026-10-29 20:22:03,349] INFO : Running validation on rank0 +rank0 [2026-10-29 20:22:03,350] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 20:22:03,350] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 20:22:52,304] INFO : VALIDATION | Step=43000/50000 | Train Loss=0.9800 | Valid Loss=0.9825 | Train Loss Unweighted=0.4737 | Valid Loss Unweighted=0.4711 | Stale=0 | ETA=54.3m +rank0 [2026-10-29 20:22:52,304] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 20:22:52,304] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/ari=0.533102 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 20:22:52,305] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.654384 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/top1_competing=0.75405 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.625497 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0160509 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0510224 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 20:22:52,306] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/seconds_per_rank=21.5836 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding/ari=0.264422 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding/ari_threshold=0.925 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding_minus_raw/ari=0.172799 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 20:22:52,307] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 20:22:52,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:22:52,569] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 20:22:52,573] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 20:23:18,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 20:23:19,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 20:23:20,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 20:23:21,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 20:23:31,863] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 20:23:31,879] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:23:32,374] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 20:23:32,378] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 20:23:48,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 20:23:49,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 20:23:50,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 20:23:51,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 20:24:01,712] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 20:24:01,723] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:24:02,214] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 20:24:02,218] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 20:24:17,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 20:24:18,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 20:24:19,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 20:24:20,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 20:24:30,272] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 20:24:30,290] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits +rank0 [2026-10-29 20:24:33,478] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits +rank0 [2026-10-29 20:24:34,517] INFO : Plotted number of elements +rank0 [2026-10-29 20:24:36,084] INFO : Plotted jets +rank0 [2026-10-29 20:24:37,258] INFO : Plotted jet ratio +rank0 [2026-10-29 20:24:37,325] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 20:24:40,540] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 20:24:41,546] INFO : Plotted number of elements +rank0 [2026-10-29 20:24:43,434] INFO : Plotted jets +rank0 [2026-10-29 20:24:44,537] INFO : Plotted jet ratio +rank0 [2026-10-29 20:24:44,606] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits +rank0 [2026-10-29 20:24:46,925] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits +rank0 [2026-10-29 20:24:47,767] INFO : Plotted number of elements +rank0 [2026-10-29 20:24:49,265] INFO : Plotted jets +rank0 [2026-10-29 20:24:50,414] INFO : Plotted jet ratio +rank0 [2026-10-29 20:28:29,940] INFO : Step 43100/50000 rank0 | Train Loss: 0.9692 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1151 | Classification: 0.0766 | Classification_binary: 0.1586 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1062 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0557 | Regression_eta: 0.0285 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0559 | LR: 1.42e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5607s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.8051 | max_loss: 0.1438 | max_backward: 1.3224 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 7068.8 | uss_MB: 6101.3 | child_rss_MB: 5191.0 | allocated_MB: 370.5 | reserved_MB: 21096.0 | inactive_split_MB: 247.5 +rank0 [2026-10-29 20:28:30,184] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 28% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 50.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 55.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:32:31,460] INFO : Step 43200/50000 rank0 | Train Loss: 0.9431 | Train Loss Unweighted: 0.4458 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1175 | Classification: 0.0811 | Classification_binary: 0.1294 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0012 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1147 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0562 | Regression_eta: 0.0284 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0453 | LR: 1.40e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1686s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0120 | max_forward: 0.6342 | max_loss: 0.1417 | max_backward: 1.2752 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6068.4 | uss_MB: 5101.7 | child_rss_MB: 5194.2 | allocated_MB: 370.4 | reserved_MB: 22140.0 | inactive_split_MB: 198.0 +rank0 [2026-10-29 20:32:31,701] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 50.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:36:17,105] INFO : Step 43300/50000 rank0 | Train Loss: 1.0382 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1221 | Classification: 0.1031 | Classification_binary: 0.1555 | HitGrouping: 0.1894 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1173 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0576 | Regression_eta: 0.0347 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0432 | LR: 1.39e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9161s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0266 | max_forward: 0.7527 | max_loss: 0.1476 | max_backward: 1.6633 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.7701 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6071.5 | uss_MB: 5107.2 | child_rss_MB: 5175.9 | allocated_MB: 371.1 | reserved_MB: 23082.0 | inactive_split_MB: 236.4 +rank0 [2026-10-29 20:36:17,351] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:40:06,168] INFO : Step 43400/50000 rank0 | Train Loss: 1.0532 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1292 | Classification: 0.0981 | Classification_binary: 0.1785 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0015 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0424 | Regression_energy: 0.0460 | Regression_eta: 0.0252 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0344 | LR: 1.38e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9678s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0691 | max_forward: 0.8057 | max_loss: 0.1503 | max_backward: 1.8319 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.6703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6067.0 | uss_MB: 5102.4 | child_rss_MB: 5192.5 | allocated_MB: 371.5 | reserved_MB: 23082.0 | inactive_split_MB: 776.0 +rank0 [2026-10-29 20:40:06,405] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 47.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:43:57,820] INFO : Step 43500/50000 rank0 | Train Loss: 0.9308 | Train Loss Unweighted: 0.4531 | Train Loss Components: Auxiliary: 0.2084 | Cardinality: 0.0966 | Classification: 0.0846 | Classification_binary: 0.1384 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1135 | Regression_eta: 0.0013 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0593 | Regression_eta: 0.0374 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0467 | LR: 1.37e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3616s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0171 | max_forward: 0.6718 | max_loss: 0.1455 | max_backward: 1.4141 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 6070.4 | uss_MB: 5108.5 | child_rss_MB: 5191.0 | allocated_MB: 372.0 | reserved_MB: 23082.0 | inactive_split_MB: 228.0 +rank0 [2026-10-29 20:43:58,072] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:47:53,509] INFO : Step 43600/50000 rank0 | Train Loss: 0.9195 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.0926 | Classification: 0.0847 | Classification_binary: 0.1298 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1175 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0578 | Regression_eta: 0.0254 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0510 | LR: 1.36e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2403s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.6453 | max_loss: 0.1399 | max_backward: 1.3514 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0942 | Memory: rss_MB: 6067.9 | uss_MB: 5106.0 | child_rss_MB: 5190.1 | allocated_MB: 367.6 | reserved_MB: 23082.0 | inactive_split_MB: 270.4 +rank0 [2026-10-29 20:47:53,760] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:51:55,313] INFO : Step 43700/50000 rank0 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1181 | Classification: 0.0916 | Classification_binary: 0.1553 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1009 | Regression_eta: 0.0014 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1483 | Regression_cos_phi: 0.0344 | Regression_energy: 0.0466 | Regression_eta: 0.0424 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0474 | LR: 1.35e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.7625s | Phase Times: max_pre_step_barrier: 0.0199 | max_batch_to_device: 0.0121 | max_forward: 0.8005 | max_loss: 0.2526 | max_backward: 1.9159 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 6075.4 | uss_MB: 5110.5 | child_rss_MB: 5168.5 | allocated_MB: 371.6 | reserved_MB: 23674.0 | inactive_split_MB: 256.4 +rank0 [2026-10-29 20:51:55,553] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 47.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 38% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:55:51,903] INFO : Step 43800/50000 rank0 | Train Loss: 1.0325 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1341 | Classification: 0.0892 | Classification_binary: 0.1542 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0012 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1363 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0740 | Regression_eta: 0.0332 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0506 | LR: 1.34e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3962s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7618 | max_loss: 0.1715 | max_backward: 1.3818 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6073.5 | uss_MB: 5110.6 | child_rss_MB: 5186.9 | allocated_MB: 371.7 | reserved_MB: 23674.0 | inactive_split_MB: 282.3 +rank0 [2026-10-29 20:55:52,144] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 50.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 55.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:59:46,919] INFO : Step 43900/50000 rank0 | Train Loss: 0.9104 | Train Loss Unweighted: 0.4325 | Train Loss Components: Auxiliary: 0.2009 | Cardinality: 0.1078 | Classification: 0.0725 | Classification_binary: 0.1272 | HitGrouping: 0.1855 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1146 | Regression_eta: 0.0011 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1267 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0637 | Regression_eta: 0.0454 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0693 | LR: 1.33e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.0707s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.6145 | max_loss: 0.1412 | max_backward: 1.1415 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5764 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6071.8 | uss_MB: 5111.9 | child_rss_MB: 5178.4 | allocated_MB: 368.3 | reserved_MB: 23674.0 | inactive_split_MB: 193.1 +rank0 [2026-10-29 20:59:47,155] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:03:44,285] INFO : Step 44000/50000 rank0 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4573 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1078 | Classification: 0.0956 | Classification_binary: 0.1421 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1083 | Regression_eta: 0.0011 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1534 | Regression_cos_phi: 0.0407 | Regression_energy: 0.0448 | Regression_eta: 0.0319 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0750 | LR: 1.32e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7199s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0061 | max_forward: 1.1697 | max_loss: 0.2455 | max_backward: 1.7366 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6074.9 | uss_MB: 5118.1 | child_rss_MB: 5179.2 | allocated_MB: 370.7 | reserved_MB: 23674.0 | inactive_split_MB: 207.3 +rank0 [2026-10-29 21:03:44,525] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:03:44,548] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-44000.pth +rank0 [2026-10-29 21:03:44,754] INFO : Running validation on rank0 +rank0 [2026-10-29 21:03:44,754] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 21:03:44,755] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 21:04:33,255] INFO : VALIDATION | Step=44000/50000 | Train Loss=0.9391 | Valid Loss=0.9779 | Train Loss Unweighted=0.4573 | Valid Loss Unweighted=0.4662 | Stale=0 | ETA=51.2m +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 21:04:33,256] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/ari=0.540836 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.620588 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/top1_competing=0.714023 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.632153 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.00831649 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0576778 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 21:04:33,257] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/seconds_per_rank=21.6492 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding/ari=0.326337 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding/ari_threshold=0.9125 +rank0 [2026-10-29 21:04:33,258] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding_minus_raw/ari=0.234714 +rank0 [2026-10-29 21:04:33,259] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 21:04:33,259] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 21:04:33,259] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 21:04:33,264] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:04:33,522] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 21:04:33,527] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 21:04:58,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 21:05:00,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 21:05:01,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 21:05:02,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 21:05:12,498] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 21:05:12,510] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:05:12,999] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 21:05:13,003] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 21:05:28,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 21:05:29,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 21:05:30,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 21:05:32,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 21:05:42,049] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 21:05:42,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:05:42,734] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 21:05:42,738] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 21:05:58,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 21:05:59,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 21:06:00,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 21:06:00,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 21:06:10,855] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 21:06:10,872] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits +rank0 [2026-10-29 21:06:14,043] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits +rank0 [2026-10-29 21:06:15,068] INFO : Plotted number of elements +rank0 [2026-10-29 21:06:16,631] INFO : Plotted jets +rank0 [2026-10-29 21:06:17,799] INFO : Plotted jet ratio +rank0 [2026-10-29 21:06:17,850] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 21:06:20,992] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 21:06:21,972] INFO : Plotted number of elements +rank0 [2026-10-29 21:06:23,859] INFO : Plotted jets +rank0 [2026-10-29 21:06:25,014] INFO : Plotted jet ratio +rank0 [2026-10-29 21:06:25,065] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits +rank0 [2026-10-29 21:06:27,394] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits +rank0 [2026-10-29 21:06:28,222] INFO : Plotted number of elements +rank0 [2026-10-29 21:06:29,748] INFO : Plotted jets +rank0 [2026-10-29 21:06:30,919] INFO : Plotted jet ratio +rank0 [2026-10-29 21:10:20,832] INFO : Step 44100/50000 rank0 | Train Loss: 1.0009 | Train Loss Unweighted: 0.4605 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1400 | Classification: 0.0841 | Classification_binary: 0.1644 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1044 | Regression_eta: 0.0011 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0359 | Regression_energy: 0.0452 | Regression_eta: 0.0268 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0369 | LR: 1.31e-04 | DataLoad Time: 0.0196s | Model Forward Time: 6.2985s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 1.2763 | max_loss: 0.5349 | max_backward: 2.2676 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 1.2755 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 7076.5 | uss_MB: 6129.8 | child_rss_MB: 5164.2 | allocated_MB: 372.3 | reserved_MB: 24224.0 | inactive_split_MB: 272.2 +rank0 [2026-10-29 21:10:21,073] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 46% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 47.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 5% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:14:15,971] INFO : Step 44200/50000 rank0 | Train Loss: 0.9801 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1146 | Classification: 0.0882 | Classification_binary: 0.1482 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1167 | Regression_eta: 0.0014 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0748 | Regression_eta: 0.0370 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0599 | LR: 1.30e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.6628s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0132 | max_forward: 0.6993 | max_loss: 0.1574 | max_backward: 1.3862 | max_optimizer_step: 0.2915 | max_tensorboard_logging: 0.7724 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6137.2 | uss_MB: 5191.5 | child_rss_MB: 5174.4 | allocated_MB: 370.2 | reserved_MB: 26182.0 | inactive_split_MB: 176.5 +rank0 [2026-10-29 21:14:16,213] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 1% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 55.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 57% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 47.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:17:59,274] INFO : Step 44300/50000 rank0 | Train Loss: 0.8787 | Train Loss Unweighted: 0.4332 | Train Loss Components: Auxiliary: 0.1952 | Cardinality: 0.0950 | Classification: 0.0869 | Classification_binary: 0.1153 | HitGrouping: 0.1807 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0011 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1032 | Regression_cos_phi: 0.0346 | Regression_energy: 0.0481 | Regression_eta: 0.0229 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0286 | LR: 1.29e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4289s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0142 | max_forward: 0.7699 | max_loss: 0.2088 | max_backward: 1.4049 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 0.4839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 6137.2 | uss_MB: 4195.8 | child_rss_MB: 5178.8 | allocated_MB: 367.2 | reserved_MB: 27204.0 | inactive_split_MB: 205.8 +rank0 [2026-10-29 21:17:59,523] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 5% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 55% +2 6 0x7408, 15961 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 95% +4 8 0x7408, 13395 55.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:22:00,528] INFO : Step 44400/50000 rank0 | Train Loss: 0.9558 | Train Loss Unweighted: 0.4680 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.0905 | Classification: 0.0911 | Classification_binary: 0.1587 | HitGrouping: 0.1858 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0501 | Regression_eta: 0.0424 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0400 | LR: 1.28e-04 | DataLoad Time: 0.0213s | Model Forward Time: 7.0591s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0370 | max_forward: 0.7829 | max_loss: 0.1727 | max_backward: 1.7862 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 2.7361 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6138.4 | uss_MB: 5206.0 | child_rss_MB: 5161.4 | allocated_MB: 373.1 | reserved_MB: 27586.0 | inactive_split_MB: 176.3 +rank0 [2026-10-29 21:22:00,769] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 47.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 50.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 53.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 47.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:26:12,503] INFO : Step 44500/50000 rank0 | Train Loss: 0.9739 | Train Loss Unweighted: 0.4610 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1139 | Classification: 0.0849 | Classification_binary: 0.1578 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1077 | Regression_eta: 0.0012 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1497 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0559 | Regression_eta: 0.0348 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0761 | LR: 1.27e-04 | DataLoad Time: 0.0136s | Model Forward Time: 4.5676s | Phase Times: max_pre_step_barrier: 0.0250 | max_batch_to_device: 0.0065 | max_forward: 0.7930 | max_loss: 0.1668 | max_backward: 1.3363 | max_optimizer_step: 0.2871 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6136.3 | uss_MB: 5200.8 | child_rss_MB: 5177.3 | allocated_MB: 371.5 | reserved_MB: 27586.0 | inactive_split_MB: 195.8 +rank0 [2026-10-29 21:26:12,740] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 54.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 47.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:30:05,305] INFO : Step 44600/50000 rank0 | Train Loss: 0.8904 | Train Loss Unweighted: 0.4364 | Train Loss Components: Auxiliary: 0.2007 | Cardinality: 0.1041 | Classification: 0.0775 | Classification_binary: 0.1238 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1161 | Regression_eta: 0.0010 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1224 | Regression_cos_phi: 0.0380 | Regression_energy: 0.0505 | Regression_eta: 0.0242 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0315 | LR: 1.26e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2557s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.6364 | max_loss: 0.1431 | max_backward: 1.3712 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.4900 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6139.4 | uss_MB: 5205.0 | child_rss_MB: 5167.5 | allocated_MB: 368.4 | reserved_MB: 27586.0 | inactive_split_MB: 170.2 +rank0 [2026-10-29 21:30:05,547] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:33:52,256] INFO : Step 44700/50000 rank0 | Train Loss: 1.0003 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1252 | Classification: 0.0899 | Classification_binary: 0.1520 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1117 | Regression_eta: 0.0013 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0549 | Regression_eta: 0.0262 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0442 | LR: 1.25e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2958s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6952 | max_loss: 0.1455 | max_backward: 1.5015 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6145.1 | uss_MB: 5206.5 | child_rss_MB: 5162.4 | allocated_MB: 371.9 | reserved_MB: 28574.0 | inactive_split_MB: 214.1 +rank0 [2026-10-29 21:33:52,495] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 2% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 42% 0% +2 6 0x7408, 15961 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 85% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 80% +4 8 0x7408, 13395 54.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 83% +6 10 0x7408, 62036 47.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 98% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:37:56,850] INFO : Step 44800/50000 rank0 | Train Loss: 0.9665 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1115 | Classification: 0.0889 | Classification_binary: 0.1605 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1042 | Regression_eta: 0.0011 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1393 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0476 | Regression_eta: 0.0231 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0442 | LR: 1.24e-04 | DataLoad Time: 0.0147s | Model Forward Time: 6.7353s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7593 | max_loss: 0.1924 | max_backward: 2.8086 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.6087 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 6148.4 | uss_MB: 5210.8 | child_rss_MB: 5160.0 | allocated_MB: 370.8 | reserved_MB: 28574.0 | inactive_split_MB: 257.7 +rank0 [2026-10-29 21:37:57,095] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 47.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 49.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 52.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:41:37,564] INFO : Step 44900/50000 rank0 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.0886 | Classification_binary: 0.1466 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1121 | Regression_eta: 0.0013 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0380 | Regression_energy: 0.0520 | Regression_eta: 0.0254 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0434 | LR: 1.23e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.1682s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0101 | max_forward: 0.6531 | max_loss: 0.1431 | max_backward: 1.3230 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6145.9 | uss_MB: 5207.3 | child_rss_MB: 5182.8 | allocated_MB: 368.1 | reserved_MB: 28574.0 | inactive_split_MB: 178.8 +rank0 [2026-10-29 21:41:37,801] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:45:51,485] INFO : Step 45000/50000 rank0 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4443 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1053 | Classification: 0.0832 | Classification_binary: 0.1311 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0457 | Regression_eta: 0.0402 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0557 | LR: 1.22e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.8439s | Phase Times: max_pre_step_barrier: 0.0074 | max_batch_to_device: 0.0159 | max_forward: 1.1897 | max_loss: 0.2457 | max_backward: 1.2951 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.8342 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0919 | Memory: rss_MB: 6152.1 | uss_MB: 5213.7 | child_rss_MB: 5170.7 | allocated_MB: 368.8 | reserved_MB: 28576.0 | inactive_split_MB: 266.4 +rank0 [2026-10-29 21:45:51,737] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 47.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 54.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:45:51,757] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-45000.pth +rank0 [2026-10-29 21:45:51,938] INFO : Running validation on rank0 +rank0 [2026-10-29 21:45:51,938] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 21:45:51,938] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 21:46:41,087] INFO : VALIDATION | Step=45000/50000 | Train Loss=0.9391 | Valid Loss=0.9645 | Train Loss Unweighted=0.4443 | Valid Loss Unweighted=0.4564 | Stale=0 | ETA=46.4m +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 21:46:41,088] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/ari=0.5535 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.651608 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/top1_competing=0.750485 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.631657 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.00434764 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0571826 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 21:46:41,089] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/seconds_per_rank=21.6684 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding/ari=0.321628 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding/ari_threshold=0.9125 +rank0 [2026-10-29 21:46:41,090] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding_minus_raw/ari=0.230005 +rank0 [2026-10-29 21:46:41,091] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 21:46:41,091] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 21:46:41,091] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 21:46:41,096] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:46:41,355] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 21:46:41,367] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 21:47:08,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 21:47:10,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 21:47:11,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 21:47:13,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 21:47:23,114] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 21:47:23,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:47:23,605] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 21:47:23,608] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 21:47:40,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 21:47:41,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 21:47:42,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 21:47:43,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 21:47:53,391] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 21:47:53,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:47:53,967] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 21:47:53,971] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 21:48:10,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 21:48:11,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 21:48:12,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 21:48:12,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 21:48:22,793] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 21:48:22,810] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits +rank0 [2026-10-29 21:48:25,979] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits +rank0 [2026-10-29 21:48:27,012] INFO : Plotted number of elements +rank0 [2026-10-29 21:48:28,561] INFO : Plotted jets +rank0 [2026-10-29 21:48:29,752] INFO : Plotted jet ratio +rank0 [2026-10-29 21:48:29,824] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 21:48:33,008] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 21:48:34,025] INFO : Plotted number of elements +rank0 [2026-10-29 21:48:35,945] INFO : Plotted jets +rank0 [2026-10-29 21:48:37,077] INFO : Plotted jet ratio +rank0 [2026-10-29 21:48:37,134] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits +rank0 [2026-10-29 21:48:39,495] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits +rank0 [2026-10-29 21:48:40,378] INFO : Plotted number of elements +rank0 [2026-10-29 21:48:41,949] INFO : Plotted jets +rank0 [2026-10-29 21:48:43,089] INFO : Plotted jet ratio +rank0 [2026-10-29 21:52:57,144] INFO : Step 45100/50000 rank0 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1151 | Classification: 0.1010 | Classification_binary: 0.1562 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1118 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0512 | Regression_eta: 0.0342 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0675 | LR: 1.21e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.9944s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7307 | max_loss: 0.4409 | max_backward: 1.5823 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 7066.7 | uss_MB: 6122.4 | child_rss_MB: 5160.9 | allocated_MB: 373.1 | reserved_MB: 23280.0 | inactive_split_MB: 227.2 +rank0 [2026-10-29 21:52:57,385] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 54.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:56:58,245] INFO : Step 45200/50000 rank0 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1108 | Classification: 0.0815 | Classification_binary: 0.1593 | HitGrouping: 0.1884 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1223 | Regression_cos_phi: 0.0371 | Regression_energy: 0.0730 | Regression_eta: 0.0379 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0438 | LR: 1.20e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.0300s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.9482 | max_loss: 0.4476 | max_backward: 1.7165 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.4656 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7064.1 | uss_MB: 6120.3 | child_rss_MB: 5157.7 | allocated_MB: 373.0 | reserved_MB: 23338.0 | inactive_split_MB: 277.4 +rank0 [2026-10-29 21:57:15,029] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 45.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 48.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 50.0°C 124.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 44.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:01:29,581] INFO : Step 45300/50000 rank0 | Train Loss: 0.9312 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1141 | Classification: 0.0882 | Classification_binary: 0.1299 | HitGrouping: 0.1891 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0010 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0295 | Classification_binary: 0.1177 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0526 | Regression_eta: 0.0268 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0393 | LR: 1.19e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0530s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6106 | max_loss: 0.2394 | max_backward: 1.1660 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 6271.7 | uss_MB: 5329.4 | child_rss_MB: 5157.3 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 223.2 +rank0 [2026-10-29 22:01:29,833] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:05:38,128] INFO : Step 45400/50000 rank0 | Train Loss: 0.9251 | Train Loss Unweighted: 0.4554 | Train Loss Components: Auxiliary: 0.2112 | Cardinality: 0.1034 | Classification: 0.0967 | Classification_binary: 0.1405 | HitGrouping: 0.1801 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1072 | Regression_eta: 0.0012 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0511 | Regression_eta: 0.0439 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0611 | LR: 1.19e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6949s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7158 | max_loss: 0.2520 | max_backward: 1.7727 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6276.9 | uss_MB: 5335.5 | child_rss_MB: 5162.7 | allocated_MB: 372.6 | reserved_MB: 23338.0 | inactive_split_MB: 212.6 +rank0 [2026-10-29 22:05:38,371] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:09:50,539] INFO : Step 45500/50000 rank0 | Train Loss: 1.0136 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1238 | Classification: 0.0902 | Classification_binary: 0.1661 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1060 | Regression_eta: 0.0013 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0480 | Regression_eta: 0.0292 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0289 | LR: 1.18e-04 | DataLoad Time: 0.0183s | Model Forward Time: 6.8962s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0147 | max_forward: 1.9023 | max_loss: 0.2979 | max_backward: 3.5478 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5661 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 6272.0 | uss_MB: 4281.7 | child_rss_MB: 5159.4 | allocated_MB: 372.4 | reserved_MB: 23338.0 | inactive_split_MB: 256.1 +rank0 [2026-10-29 22:09:50,781] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 8% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 51.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 93% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 52.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 46.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 94% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 31% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:14:16,750] INFO : Step 45600/50000 rank0 | Train Loss: 0.9675 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2172 | Cardinality: 0.1193 | Classification: 0.0849 | Classification_binary: 0.1400 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1218 | Regression_eta: 0.0010 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1335 | Regression_cos_phi: 0.1020 | Regression_energy: 0.0776 | Regression_eta: 0.0319 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0535 | LR: 1.17e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.4464s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6781 | max_loss: 0.1576 | max_backward: 1.5089 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 0.5447 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6272.0 | uss_MB: 5337.5 | child_rss_MB: 5161.4 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 216.2 +rank0 [2026-10-29 22:14:16,990] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:18:16,327] INFO : Step 45700/50000 rank0 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1961 | Cardinality: 0.0908 | Classification: 0.0837 | Classification_binary: 0.1223 | HitGrouping: 0.1794 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1108 | Regression_eta: 0.0011 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1269 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0518 | Regression_eta: 0.0356 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0603 | LR: 1.16e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.1672s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6216 | max_loss: 0.1658 | max_backward: 1.1797 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6234.4 | uss_MB: 5336.4 | child_rss_MB: 5162.7 | allocated_MB: 370.8 | reserved_MB: 23338.0 | inactive_split_MB: 226.6 +rank0 [2026-10-29 22:18:16,573] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 27% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 5% +4 8 0x7408, 13395 55.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:22:20,254] INFO : Step 45800/50000 rank0 | Train Loss: 0.9372 | Train Loss Unweighted: 0.4491 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1073 | Classification: 0.0880 | Classification_binary: 0.1481 | HitGrouping: 0.1781 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1904 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0594 | Regression_eta: 0.0342 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0470 | LR: 1.16e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.9543s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8017 | max_loss: 0.2428 | max_backward: 1.9917 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0978 | Memory: rss_MB: 6240.7 | uss_MB: 5347.0 | child_rss_MB: 5162.1 | allocated_MB: 375.2 | reserved_MB: 23338.0 | inactive_split_MB: 268.8 +rank0 [2026-10-29 22:22:20,512] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:26:27,565] INFO : Step 45900/50000 rank0 | Train Loss: 0.9600 | Train Loss Unweighted: 0.4592 | Train Loss Components: Auxiliary: 0.2179 | Cardinality: 0.1027 | Classification: 0.0816 | Classification_binary: 0.1590 | HitGrouping: 0.1875 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0370 | Regression_energy: 0.0546 | Regression_eta: 0.0383 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0453 | LR: 1.15e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.4762s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 1.3210 | max_loss: 0.4294 | max_backward: 3.2770 | max_optimizer_step: 0.2740 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 6235.6 | uss_MB: 5343.7 | child_rss_MB: 5161.3 | allocated_MB: 370.5 | reserved_MB: 23338.0 | inactive_split_MB: 259.6 +rank0 [2026-10-29 22:26:27,806] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 50.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 53.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:30:41,572] INFO : Step 46000/50000 rank0 | Train Loss: 0.8490 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.0938 | Classification: 0.0817 | Classification_binary: 0.1110 | HitGrouping: 0.1819 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0010 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.0941 | Regression_cos_phi: 0.0248 | Regression_energy: 0.0449 | Regression_eta: 0.0228 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0488 | LR: 1.14e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2414s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.5955 | max_loss: 0.1430 | max_backward: 1.2483 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 0.6657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 6228.9 | uss_MB: 5344.9 | child_rss_MB: 5164.3 | allocated_MB: 365.7 | reserved_MB: 23338.0 | inactive_split_MB: 222.2 +rank0 [2026-10-29 22:30:41,826] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:30:41,849] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-46000.pth +rank0 [2026-10-29 22:30:42,032] INFO : Running validation on rank0 +rank0 [2026-10-29 22:30:42,033] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 22:30:42,033] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 22:31:33,637] INFO : VALIDATION | Step=46000/50000 | Train Loss=0.8490 | Valid Loss=0.9615 | Train Loss Unweighted=0.3997 | Valid Loss Unweighted=0.4541 | Stale=0 | ETA=40.2m +rank0 [2026-10-29 22:31:33,637] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 22:31:33,637] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 22:31:33,637] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 22:31:33,637] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/ari=0.528877 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.647185 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/top1_competing=0.744667 +rank0 [2026-10-29 22:31:33,638] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.634531 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0202757 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0600559 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/seconds_per_rank=21.6312 +rank0 [2026-10-29 22:31:33,639] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding/ari=0.29401 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding/ari_threshold=0.9 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding_minus_raw/ari=0.202386 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 22:31:33,640] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 22:31:33,646] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:31:33,901] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 22:31:33,905] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 22:32:00,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 22:32:01,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 22:32:03,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 22:32:05,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 22:32:15,270] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-29 22:32:15,389] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:32:16,223] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 22:32:16,227] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 22:32:32,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 22:32:33,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 22:32:34,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 22:32:35,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 22:32:45,533] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 22:32:45,544] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:32:46,016] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 22:32:46,019] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 22:33:01,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 22:33:02,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 22:33:03,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 22:33:04,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 22:33:14,267] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-29 22:33:14,283] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits +rank0 [2026-10-29 22:33:17,544] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits +rank0 [2026-10-29 22:33:18,663] INFO : Plotted number of elements +rank0 [2026-10-29 22:33:20,251] INFO : Plotted jets +rank0 [2026-10-29 22:33:21,422] INFO : Plotted jet ratio +rank0 [2026-10-29 22:33:21,475] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 22:33:24,642] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 22:33:25,662] INFO : Plotted number of elements +rank0 [2026-10-29 22:33:27,198] INFO : Plotted jets +rank0 [2026-10-29 22:33:28,678] INFO : Plotted jet ratio +rank0 [2026-10-29 22:33:28,731] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits +rank0 [2026-10-29 22:33:31,062] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits +rank0 [2026-10-29 22:33:31,908] INFO : Plotted number of elements +rank0 [2026-10-29 22:33:33,401] INFO : Plotted jets +rank0 [2026-10-29 22:33:34,570] INFO : Plotted jet ratio +rank0 [2026-10-29 22:37:45,115] INFO : Step 46100/50000 rank0 | Train Loss: 0.9299 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1082 | Classification: 0.0899 | Classification_binary: 0.1357 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1081 | Regression_eta: 0.0010 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1556 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0567 | Regression_eta: 0.0275 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0629 | LR: 1.13e-04 | DataLoad Time: 0.0188s | Model Forward Time: 7.6523s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 2.5146 | max_loss: 0.4579 | max_backward: 2.4958 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1014 | Memory: rss_MB: 6980.2 | uss_MB: 6096.7 | child_rss_MB: 5172.0 | allocated_MB: 371.5 | reserved_MB: 22784.0 | inactive_split_MB: 245.0 +rank0 [2026-10-29 22:37:45,366] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 53.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +6 10 0x7408, 62036 47.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:41:44,320] INFO : Step 46200/50000 rank0 | Train Loss: 0.9944 | Train Loss Unweighted: 0.4707 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1286 | Classification: 0.0940 | Classification_binary: 0.1623 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1052 | Regression_eta: 0.0011 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0477 | Regression_eta: 0.0288 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0343 | LR: 1.13e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.0110s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7685 | max_loss: 0.2359 | max_backward: 1.6855 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1253 | Memory: rss_MB: 6976.0 | uss_MB: 6094.5 | child_rss_MB: 5141.0 | allocated_MB: 373.1 | reserved_MB: 22784.0 | inactive_split_MB: 264.3 +rank0 [2026-10-29 22:41:44,561] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:45:39,749] INFO : Step 46300/50000 rank0 | Train Loss: 0.9564 | Train Loss Unweighted: 0.4515 | Train Loss Components: Auxiliary: 0.2114 | Cardinality: 0.1124 | Classification: 0.0869 | Classification_binary: 0.1435 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1420 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0557 | Regression_eta: 0.0229 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0453 | LR: 1.12e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4774s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6709 | max_loss: 0.2410 | max_backward: 1.2189 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.8303 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6276.5 | uss_MB: 5413.4 | child_rss_MB: 5109.4 | allocated_MB: 372.3 | reserved_MB: 23784.0 | inactive_split_MB: 228.2 +rank0 [2026-10-29 22:45:39,987] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 55.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:49:35,421] INFO : Step 46400/50000 rank0 | Train Loss: 0.9514 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1147 | Classification: 0.0822 | Classification_binary: 0.1248 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0010 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1349 | Regression_cos_phi: 0.0285 | Regression_energy: 0.0486 | Regression_eta: 0.0215 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0342 | LR: 1.11e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.7243s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.6748 | max_loss: 0.4371 | max_backward: 1.4077 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.8211 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6270.0 | uss_MB: 5413.5 | child_rss_MB: 5075.5 | allocated_MB: 374.2 | reserved_MB: 23988.0 | inactive_split_MB: 241.8 +rank0 [2026-10-29 22:49:35,671] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:53:42,368] INFO : Step 46500/50000 rank0 | Train Loss: 0.9517 | Train Loss Unweighted: 0.4566 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.0937 | Classification: 0.0928 | Classification_binary: 0.1533 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1031 | Regression_eta: 0.0013 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1512 | Regression_cos_phi: 0.0907 | Regression_energy: 0.0534 | Regression_eta: 0.0351 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0539 | LR: 1.11e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5950s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0087 | max_forward: 0.7136 | max_loss: 0.1507 | max_backward: 1.5229 | max_optimizer_step: 0.2767 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6272.9 | uss_MB: 5416.3 | child_rss_MB: 5079.6 | allocated_MB: 372.0 | reserved_MB: 23988.0 | inactive_split_MB: 248.1 +rank0 [2026-10-29 22:53:42,605] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 76% +2 6 0x7408, 15961 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 76% +4 8 0x7408, 13395 55.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 24% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 98% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:57:42,457] INFO : Step 46600/50000 rank0 | Train Loss: 0.9541 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.2166 | Cardinality: 0.1078 | Classification: 0.0915 | Classification_binary: 0.1560 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1389 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0412 | Regression_eta: 0.0442 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0524 | LR: 1.10e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3157s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7127 | max_loss: 0.1604 | max_backward: 1.3079 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4722 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 6261.6 | uss_MB: 5413.8 | child_rss_MB: 5045.9 | allocated_MB: 370.5 | reserved_MB: 23988.0 | inactive_split_MB: 179.8 +rank0 [2026-10-29 22:57:42,709] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 55.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:02:41,328] INFO : Step 46700/50000 rank0 | Train Loss: 0.9378 | Train Loss Unweighted: 0.4497 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.0977 | Classification: 0.0857 | Classification_binary: 0.1264 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0015 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1302 | Regression_cos_phi: 0.0363 | Regression_energy: 0.0516 | Regression_eta: 0.0293 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0460 | LR: 1.10e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4025s | Phase Times: max_pre_step_barrier: 0.0197 | max_batch_to_device: 0.0481 | max_forward: 0.6699 | max_loss: 0.2593 | max_backward: 1.4356 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5914 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6261.6 | uss_MB: 5415.0 | child_rss_MB: 5040.6 | allocated_MB: 368.9 | reserved_MB: 24784.0 | inactive_split_MB: 229.2 +rank0 [2026-10-29 23:02:41,565] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:06:37,199] INFO : Step 46800/50000 rank0 | Train Loss: 0.9152 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.2022 | Cardinality: 0.1011 | Classification: 0.0910 | Classification_binary: 0.1376 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0988 | Regression_eta: 0.0014 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1192 | Regression_cos_phi: 0.0327 | Regression_energy: 0.0443 | Regression_eta: 0.0394 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0460 | LR: 1.09e-04 | DataLoad Time: 0.0129s | Model Forward Time: 5.2374s | Phase Times: max_pre_step_barrier: 0.1216 | max_batch_to_device: 0.0144 | max_forward: 1.0361 | max_loss: 0.2737 | max_backward: 1.7951 | max_optimizer_step: 0.2978 | max_tensorboard_logging: 0.5843 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6263.1 | uss_MB: 5417.5 | child_rss_MB: 5044.7 | allocated_MB: 368.9 | reserved_MB: 24994.0 | inactive_split_MB: 204.7 +rank0 [2026-10-29 23:06:37,437] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:10:46,458] INFO : Step 46900/50000 rank0 | Train Loss: 0.9278 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2149 | Cardinality: 0.1073 | Classification: 0.0827 | Classification_binary: 0.1607 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1027 | Regression_eta: 0.0010 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1580 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0437 | Regression_eta: 0.0287 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0547 | LR: 1.09e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7265s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0446 | max_forward: 0.7815 | max_loss: 0.1553 | max_backward: 1.4732 | max_optimizer_step: 0.2929 | max_tensorboard_logging: 0.6668 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6256.7 | uss_MB: 5416.7 | child_rss_MB: 5041.4 | allocated_MB: 369.3 | reserved_MB: 24994.0 | inactive_split_MB: 274.7 +rank0 [2026-10-29 23:10:46,698] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 1% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 50.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:14:41,072] INFO : Step 47000/50000 rank0 | Train Loss: 0.9023 | Train Loss Unweighted: 0.4500 | Train Loss Components: Auxiliary: 0.2093 | Cardinality: 0.0953 | Classification: 0.0802 | Classification_binary: 0.1325 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1175 | Regression_eta: 0.0010 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0754 | Regression_eta: 0.0358 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0376 | LR: 1.08e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.1664s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6624 | max_loss: 0.1440 | max_backward: 1.2028 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 6259.7 | uss_MB: 5421.0 | child_rss_MB: 5043.0 | allocated_MB: 369.4 | reserved_MB: 25784.0 | inactive_split_MB: 224.5 +rank0 [2026-10-29 23:14:41,331] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 55.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:14:41,353] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-47000.pth +rank0 [2026-10-29 23:14:41,582] INFO : Running validation on rank0 +rank0 [2026-10-29 23:14:41,582] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 23:14:41,582] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION | Step=47000/50000 | Train Loss=0.9023 | Valid Loss=0.9592 | Train Loss Unweighted=0.4500 | Valid Loss Unweighted=0.4568 | Stale=0 | ETA=32.3m +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 23:15:33,538] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/ari=0.542743 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.639901 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/top1_competing=0.733194 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.625632 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.00640945 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0511572 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 23:15:33,539] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/seconds_per_rank=21.648 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding/ari=0.297018 +rank0 [2026-10-29 23:15:33,540] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding/ari_threshold=0.8875 +rank0 [2026-10-29 23:15:33,541] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding_minus_raw/ari=0.205395 +rank0 [2026-10-29 23:15:33,541] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 23:15:33,541] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 23:15:33,541] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 23:15:33,547] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:15:33,807] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 23:15:33,812] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 23:16:03,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 23:16:04,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 23:16:05,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 23:16:06,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 23:16:16,303] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 23:16:16,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:16:16,906] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 23:16:16,909] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 23:16:33,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 23:16:34,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 23:16:35,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 23:16:37,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 23:16:47,125] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 23:16:47,374] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:16:47,862] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 23:16:47,865] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 23:17:03,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 23:17:04,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 23:17:05,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 23:17:06,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 23:17:16,738] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 23:17:16,757] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits +rank0 [2026-10-29 23:17:20,348] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits +rank0 [2026-10-29 23:17:21,389] INFO : Plotted number of elements +rank0 [2026-10-29 23:17:22,964] INFO : Plotted jets +rank0 [2026-10-29 23:17:24,167] INFO : Plotted jet ratio +rank0 [2026-10-29 23:17:24,244] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 23:17:27,442] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 23:17:28,517] INFO : Plotted number of elements +rank0 [2026-10-29 23:17:30,103] INFO : Plotted jets +rank0 [2026-10-29 23:17:31,261] INFO : Plotted jet ratio +rank0 [2026-10-29 23:17:31,312] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits +rank0 [2026-10-29 23:17:33,815] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits +rank0 [2026-10-29 23:17:35,173] INFO : Plotted number of elements +rank0 [2026-10-29 23:17:36,662] INFO : Plotted jets +rank0 [2026-10-29 23:17:37,823] INFO : Plotted jet ratio +rank0 [2026-10-29 23:21:36,891] INFO : Step 47100/50000 rank0 | Train Loss: 0.9138 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.2058 | Cardinality: 0.0938 | Classification: 0.0946 | Classification_binary: 0.1243 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1103 | Regression_eta: 0.0011 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1306 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0380 | Regression_eta: 0.0237 | Regression_pt: 0.0399 | Regression_sin_phi: 0.0265 | LR: 1.07e-04 | DataLoad Time: 0.0112s | Model Forward Time: 5.0185s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0063 | max_forward: 0.8048 | max_loss: 0.2433 | max_backward: 1.3159 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 1.0353 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 6952.3 | uss_MB: 6113.6 | child_rss_MB: 5037.2 | allocated_MB: 368.1 | reserved_MB: 22106.0 | inactive_split_MB: 269.0 +rank0 [2026-10-29 23:21:37,130] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 46.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:25:46,389] INFO : Step 47200/50000 rank0 | Train Loss: 0.9875 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2263 | Cardinality: 0.1259 | Classification: 0.0903 | Classification_binary: 0.1586 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1110 | Regression_eta: 0.0012 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0318 | Regression_energy: 0.0473 | Regression_eta: 0.0355 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0510 | LR: 1.07e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.0480s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0486 | max_forward: 0.7581 | max_loss: 0.1506 | max_backward: 1.5914 | max_optimizer_step: 0.2300 | max_tensorboard_logging: 0.7953 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 6951.3 | uss_MB: 6117.8 | child_rss_MB: 5032.9 | allocated_MB: 370.5 | reserved_MB: 23104.0 | inactive_split_MB: 195.8 +rank0 [2026-10-29 23:25:46,628] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:29:54,548] INFO : Step 47300/50000 rank0 | Train Loss: 0.9350 | Train Loss Unweighted: 0.4467 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.0958 | Classification: 0.0857 | Classification_binary: 0.1512 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0529 | Regression_eta: 0.0461 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0541 | LR: 1.06e-04 | DataLoad Time: 0.0229s | Model Forward Time: 7.1818s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0184 | max_forward: 1.9890 | max_loss: 0.4771 | max_backward: 3.1656 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6309.1 | uss_MB: 5475.6 | child_rss_MB: 5028.4 | allocated_MB: 370.8 | reserved_MB: 23104.0 | inactive_split_MB: 271.1 +rank0 [2026-10-29 23:29:54,792] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 47.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:33:59,125] INFO : Step 47400/50000 rank0 | Train Loss: 0.8593 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.0807 | Classification: 0.0812 | Classification_binary: 0.1182 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1114 | Regression_eta: 0.0011 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1105 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0628 | Regression_eta: 0.0343 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0405 | LR: 1.06e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.7804s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0125 | max_forward: 0.7544 | max_loss: 0.1826 | max_backward: 1.4299 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8027 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6311.1 | uss_MB: 5475.9 | child_rss_MB: 5031.1 | allocated_MB: 369.9 | reserved_MB: 23104.0 | inactive_split_MB: 253.6 +rank0 [2026-10-29 23:33:59,367] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:37:53,970] INFO : Step 47500/50000 rank0 | Train Loss: 0.9089 | Train Loss Unweighted: 0.4235 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1000 | Classification: 0.0799 | Classification_binary: 0.1391 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1008 | Regression_eta: 0.0012 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0516 | Regression_eta: 0.0244 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0417 | LR: 1.06e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.9137s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0068 | max_forward: 0.7329 | max_loss: 0.1827 | max_backward: 1.5814 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9161 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6306.2 | uss_MB: 5478.3 | child_rss_MB: 5026.1 | allocated_MB: 371.7 | reserved_MB: 23104.0 | inactive_split_MB: 243.6 +rank0 [2026-10-29 23:37:54,212] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:41:48,870] INFO : Step 47600/50000 rank0 | Train Loss: 0.9781 | Train Loss Unweighted: 0.4635 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1226 | Classification: 0.0910 | Classification_binary: 0.1607 | HitGrouping: 0.1784 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1545 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0418 | Regression_eta: 0.0248 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0608 | LR: 1.05e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.1237s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8435 | max_loss: 0.1581 | max_backward: 1.8989 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5341 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6310.2 | uss_MB: 5483.0 | child_rss_MB: 5033.7 | allocated_MB: 370.6 | reserved_MB: 24122.0 | inactive_split_MB: 257.4 +rank0 [2026-10-29 23:41:49,114] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 99% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:46:00,893] INFO : Step 47700/50000 rank0 | Train Loss: 0.9311 | Train Loss Unweighted: 0.4523 | Train Loss Components: Auxiliary: 0.2105 | Cardinality: 0.0992 | Classification: 0.0889 | Classification_binary: 0.1321 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1138 | Regression_eta: 0.0012 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1258 | Regression_cos_phi: 0.0412 | Regression_energy: 0.0817 | Regression_eta: 0.0543 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0463 | LR: 1.05e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1501s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0068 | max_forward: 0.6708 | max_loss: 0.1613 | max_backward: 1.2326 | max_optimizer_step: 0.1946 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6306.7 | uss_MB: 5481.0 | child_rss_MB: 5031.8 | allocated_MB: 368.1 | reserved_MB: 24122.0 | inactive_split_MB: 276.0 +rank0 [2026-10-29 23:46:01,145] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 50.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:50:03,589] INFO : Step 47800/50000 rank0 | Train Loss: 0.9063 | Train Loss Unweighted: 0.4379 | Train Loss Components: Auxiliary: 0.2020 | Cardinality: 0.0932 | Classification: 0.0920 | Classification_binary: 0.1284 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0011 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0561 | Regression_eta: 0.0281 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0662 | LR: 1.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5185s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0610 | max_forward: 0.6626 | max_loss: 0.1349 | max_backward: 1.5091 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6302.7 | uss_MB: 5485.7 | child_rss_MB: 5029.8 | allocated_MB: 369.5 | reserved_MB: 24122.0 | inactive_split_MB: 236.5 +rank0 [2026-10-29 23:50:03,843] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:54:08,357] INFO : Step 47900/50000 rank0 | Train Loss: 0.9591 | Train Loss Unweighted: 0.4478 | Train Loss Components: Auxiliary: 0.2152 | Cardinality: 0.1022 | Classification: 0.0882 | Classification_binary: 0.1545 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1003 | Regression_eta: 0.0014 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1628 | Regression_cos_phi: 0.0298 | Regression_energy: 0.0482 | Regression_eta: 0.0306 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0359 | LR: 1.04e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.8808s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8744 | max_loss: 0.2241 | max_backward: 1.6370 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 6302.2 | uss_MB: 5483.9 | child_rss_MB: 5030.3 | allocated_MB: 370.5 | reserved_MB: 24122.0 | inactive_split_MB: 218.9 +rank0 [2026-10-29 23:54:08,599] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:00:29,669] INFO : Step 48000/50000 rank0 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1221 | Classification: 0.0823 | Classification_binary: 0.1525 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1079 | Regression_eta: 0.0017 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1379 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0518 | Regression_eta: 0.0265 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0407 | LR: 1.04e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7944s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0057 | max_forward: 0.7069 | max_loss: 0.2447 | max_backward: 1.4090 | max_optimizer_step: 0.1933 | max_tensorboard_logging: 0.8132 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1250 | Memory: rss_MB: 6301.2 | uss_MB: 5485.0 | child_rss_MB: 5024.5 | allocated_MB: 368.8 | reserved_MB: 24122.0 | inactive_split_MB: 269.6 +rank0 [2026-10-30 00:00:29,928] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:00:29,951] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-48000.pth +rank0 [2026-10-30 00:00:30,134] INFO : Running validation on rank0 +rank0 [2026-10-30 00:00:30,134] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 00:00:30,134] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 00:01:20,235] INFO : VALIDATION | Step=48000/50000 | Train Loss=0.9846 | Valid Loss=0.9575 | Train Loss Unweighted=0.4530 | Valid Loss Unweighted=0.4507 | Stale=0 | ETA=23.0m +rank0 [2026-10-30 00:01:20,235] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:01:20,235] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 00:01:20,235] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 00:01:20,235] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:01:20,235] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 00:01:20,235] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 00:01:20,235] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 00:01:20,235] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/ari=0.572458 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.644839 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/top1_competing=0.741147 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.638892 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0233051 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0644169 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 00:01:20,236] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/seconds_per_rank=21.3754 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:01:20,237] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 00:01:20,238] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 00:01:20,238] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:01:20,238] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding/ari=0.299821 +rank0 [2026-10-30 00:01:20,238] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding/ari_threshold=0.9125 +rank0 [2026-10-30 00:01:20,238] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding_minus_raw/ari=0.208198 +rank0 [2026-10-30 00:01:20,238] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 00:01:20,238] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 00:01:20,238] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 00:01:20,244] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:01:20,506] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 00:01:20,511] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 00:01:46,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 00:01:49,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 00:01:50,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 00:01:51,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 00:02:01,235] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-30 00:02:01,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:02:01,858] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 00:02:01,862] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 00:02:18,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 00:02:19,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 00:02:20,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 00:02:22,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 00:02:32,493] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-30 00:02:32,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:02:32,990] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 00:02:32,993] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 00:02:48,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 00:02:49,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 00:02:50,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 00:02:50,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 00:03:00,993] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-30 00:03:01,009] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits +rank0 [2026-10-30 00:03:04,545] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits +rank0 [2026-10-30 00:03:05,530] INFO : Plotted number of elements +rank0 [2026-10-30 00:03:07,174] INFO : Plotted jets +rank0 [2026-10-30 00:03:08,427] INFO : Plotted jet ratio +rank0 [2026-10-30 00:03:08,492] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 00:03:11,792] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 00:03:12,870] INFO : Plotted number of elements +rank0 [2026-10-30 00:03:14,449] INFO : Plotted jets +rank0 [2026-10-30 00:03:15,621] INFO : Plotted jet ratio +rank0 [2026-10-30 00:03:15,673] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits +rank0 [2026-10-30 00:03:18,149] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits +rank0 [2026-10-30 00:03:19,438] INFO : Plotted number of elements +rank0 [2026-10-30 00:03:20,928] INFO : Plotted jets +rank0 [2026-10-30 00:03:22,077] INFO : Plotted jet ratio +rank0 [2026-10-30 00:07:25,861] INFO : Step 48100/50000 rank0 | Train Loss: 0.9084 | Train Loss Unweighted: 0.4360 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1162 | Classification: 0.0769 | Classification_binary: 0.1204 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1183 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1138 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0538 | Regression_eta: 0.0271 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0367 | LR: 1.03e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5803s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1372 | max_backward: 1.2024 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.9158 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7374.2 | uss_MB: 6557.5 | child_rss_MB: 5038.7 | allocated_MB: 367.7 | reserved_MB: 24830.0 | inactive_split_MB: 222.3 +rank0 [2026-10-30 00:07:26,103] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 53.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 1% +6 10 0x7408, 62036 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:11:34,417] INFO : Step 48200/50000 rank0 | Train Loss: 0.9585 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2151 | Cardinality: 0.1199 | Classification: 0.0839 | Classification_binary: 0.1503 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0507 | Regression_eta: 0.0342 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0552 | LR: 1.03e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.2837s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7766 | max_loss: 0.1498 | max_backward: 1.7414 | max_optimizer_step: 0.3242 | max_tensorboard_logging: 0.7504 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 7374.2 | uss_MB: 6560.6 | child_rss_MB: 5031.5 | allocated_MB: 370.4 | reserved_MB: 24830.0 | inactive_split_MB: 179.3 +rank0 [2026-10-30 00:11:34,672] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 2% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 50.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 53.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 92% +6 10 0x7408, 62036 47.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:15:42,365] INFO : Step 48300/50000 rank0 | Train Loss: 1.0575 | Train Loss Unweighted: 0.4963 | Train Loss Components: Auxiliary: 0.2408 | Cardinality: 0.1341 | Classification: 0.0958 | Classification_binary: 0.1687 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1138 | Regression_eta: 0.0013 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0525 | Regression_eta: 0.0316 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0621 | LR: 1.03e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8486s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7497 | max_loss: 0.2201 | max_backward: 1.7233 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6378.2 | uss_MB: 5566.5 | child_rss_MB: 5032.3 | allocated_MB: 372.3 | reserved_MB: 24830.0 | inactive_split_MB: 269.6 +rank0 [2026-10-30 00:15:42,618] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:19:57,331] INFO : Step 48400/50000 rank0 | Train Loss: 0.9108 | Train Loss Unweighted: 0.4349 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1093 | Classification: 0.0691 | Classification_binary: 0.1311 | HitGrouping: 0.1830 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1153 | Regression_eta: 0.0013 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0271 | Classification_binary: 0.1044 | Regression_cos_phi: 0.0271 | Regression_energy: 0.0514 | Regression_eta: 0.0420 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0305 | LR: 1.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6201s | Phase Times: max_pre_step_barrier: 0.1141 | max_batch_to_device: 0.0064 | max_forward: 0.6489 | max_loss: 0.1390 | max_backward: 1.1801 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6705 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6380.2 | uss_MB: 5566.3 | child_rss_MB: 5030.1 | allocated_MB: 370.1 | reserved_MB: 24830.0 | inactive_split_MB: 261.5 +rank0 [2026-10-30 00:19:57,584] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:23:54,185] INFO : Step 48500/50000 rank0 | Train Loss: 0.9694 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1004 | Classification: 0.0844 | Classification_binary: 0.1367 | HitGrouping: 0.1928 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1194 | Regression_eta: 0.0012 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.1323 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0490 | Regression_eta: 0.0321 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0604 | LR: 1.02e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.3832s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 1.7512 | max_loss: 0.2678 | max_backward: 1.9538 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.4804 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 6374.4 | uss_MB: 5565.1 | child_rss_MB: 5023.6 | allocated_MB: 371.5 | reserved_MB: 26842.0 | inactive_split_MB: 176.5 +rank0 [2026-10-30 00:23:54,427] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:27:51,751] INFO : Step 48600/50000 rank0 | Train Loss: 0.9724 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1148 | Classification: 0.0854 | Classification_binary: 0.1551 | HitGrouping: 0.1844 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0429 | Regression_eta: 0.0322 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0641 | LR: 1.02e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6499s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0086 | max_forward: 0.6984 | max_loss: 0.2525 | max_backward: 1.6611 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6375.4 | uss_MB: 5569.8 | child_rss_MB: 5026.2 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 234.8 +rank0 [2026-10-30 00:27:51,989] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:31:53,068] INFO : Step 48700/50000 rank0 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4863 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1212 | Classification: 0.0968 | Classification_binary: 0.1569 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1143 | Regression_eta: 0.0012 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0721 | Regression_eta: 0.0286 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0656 | LR: 1.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5983s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7410 | max_loss: 0.2472 | max_backward: 1.4647 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.6585 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6316.1 | uss_MB: 5565.7 | child_rss_MB: 4823.4 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 275.8 +rank0 [2026-10-30 00:31:53,306] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:35:41,072] INFO : Step 48800/50000 rank0 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1981 | Cardinality: 0.0898 | Classification: 0.0814 | Classification_binary: 0.1196 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1094 | Regression_eta: 0.0012 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0552 | Regression_eta: 0.0394 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0728 | LR: 1.01e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4485s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.6122 | max_loss: 0.1775 | max_backward: 1.1932 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6316.1 | uss_MB: 5568.8 | child_rss_MB: 4813.9 | allocated_MB: 368.0 | reserved_MB: 26842.0 | inactive_split_MB: 263.7 +rank0 [2026-10-30 00:35:41,324] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 54.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:39:24,320] INFO : Step 48900/50000 rank0 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1005 | Classification: 0.0873 | Classification_binary: 0.1401 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0488 | Regression_eta: 0.0242 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0648 | LR: 1.01e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.6673s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0809 | max_forward: 0.7397 | max_loss: 0.1455 | max_backward: 1.5027 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6303.5 | uss_MB: 5491.8 | child_rss_MB: 4768.8 | allocated_MB: 373.0 | reserved_MB: 26842.0 | inactive_split_MB: 191.3 +rank0 [2026-10-30 00:39:24,561] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 55.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 98% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:43:09,612] INFO : Step 49000/50000 rank0 | Train Loss: 1.0019 | Train Loss Unweighted: 0.4657 | Train Loss Components: Auxiliary: 0.2334 | Cardinality: 0.1354 | Classification: 0.0893 | Classification_binary: 0.1587 | HitGrouping: 0.1838 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0010 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0532 | Regression_eta: 0.0332 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0561 | LR: 1.01e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2519s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1085 | max_forward: 0.8072 | max_loss: 0.1558 | max_backward: 1.7809 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.6082 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6305.0 | uss_MB: 5577.9 | child_rss_MB: 4749.4 | allocated_MB: 370.9 | reserved_MB: 26842.0 | inactive_split_MB: 271.3 +rank0 [2026-10-30 00:43:09,866] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 7% +1 5 0x7408, 51740 59.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 50.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 55.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 3% +6 10 0x7408, 62036 47.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 12% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:43:09,888] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-49000.pth +rank0 [2026-10-30 00:43:10,071] INFO : Running validation on rank0 +rank0 [2026-10-30 00:43:10,071] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 00:43:10,071] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 00:43:59,913] INFO : VALIDATION | Step=49000/50000 | Train Loss=1.0019 | Valid Loss=0.9592 | Train Loss Unweighted=0.4657 | Valid Loss Unweighted=0.4542 | Stale=1 | ETA=12.1m +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 00:43:59,914] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/ari=0.523197 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.639946 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/top1_competing=0.734245 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.639053 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0259562 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.064578 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 00:43:59,915] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/seconds_per_rank=21.4718 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding/ari=0.320273 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding/ari_threshold=0.9 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding_minus_raw/ari=0.22865 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 00:43:59,916] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 00:43:59,922] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:44:00,188] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 00:44:00,203] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 00:44:27,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 00:44:28,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 00:44:29,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 00:44:30,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 00:44:40,632] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-30 00:44:40,645] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:44:41,122] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 00:44:41,125] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 00:44:57,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 00:44:58,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 00:44:59,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 00:45:00,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 00:45:10,275] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-30 00:45:10,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:45:10,742] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 00:45:10,746] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 00:45:26,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 00:45:27,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 00:45:27,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 00:45:28,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 00:45:38,710] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-30 00:45:38,726] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits +rank0 [2026-10-30 00:45:42,005] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits +rank0 [2026-10-30 00:45:43,052] INFO : Plotted number of elements +rank0 [2026-10-30 00:45:44,618] INFO : Plotted jets +rank0 [2026-10-30 00:45:45,797] INFO : Plotted jet ratio +rank0 [2026-10-30 00:45:45,860] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 00:45:49,072] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 00:45:50,082] INFO : Plotted number of elements +rank0 [2026-10-30 00:45:51,585] INFO : Plotted jets +rank0 [2026-10-30 00:45:53,082] INFO : Plotted jet ratio +rank0 [2026-10-30 00:45:53,145] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits +rank0 [2026-10-30 00:45:55,408] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits +rank0 [2026-10-30 00:45:56,172] INFO : Plotted number of elements +rank0 [2026-10-30 00:45:57,695] INFO : Plotted jets +rank0 [2026-10-30 00:45:58,883] INFO : Plotted jet ratio +rank0 [2026-10-30 00:49:43,176] INFO : Step 49100/50000 rank0 | Train Loss: 0.9797 | Train Loss Unweighted: 0.4561 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1125 | Classification: 0.0877 | Classification_binary: 0.1433 | HitGrouping: 0.1905 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0013 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1241 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0525 | Regression_eta: 0.0277 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0676 | LR: 1.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3488s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6869 | max_loss: 0.1436 | max_backward: 1.2639 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.4741 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0455 | Memory: rss_MB: 7470.2 | uss_MB: 6734.9 | child_rss_MB: 4778.3 | allocated_MB: 370.4 | reserved_MB: 22116.0 | inactive_split_MB: 274.9 +rank0 [2026-10-30 00:49:43,428] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 55.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:53:47,917] INFO : Step 49200/50000 rank0 | Train Loss: 0.8929 | Train Loss Unweighted: 0.4393 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1020 | Classification: 0.0903 | Classification_binary: 0.1250 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0009 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1153 | Regression_cos_phi: 0.0248 | Regression_energy: 0.0492 | Regression_eta: 0.0242 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0345 | LR: 1.01e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5478s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6890 | max_loss: 0.2430 | max_backward: 1.3427 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.6347 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7470.5 | uss_MB: 6738.6 | child_rss_MB: 4771.6 | allocated_MB: 368.3 | reserved_MB: 24092.0 | inactive_split_MB: 187.3 +rank0 [2026-10-30 00:53:48,158] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:57:34,593] INFO : Step 49300/50000 rank0 | Train Loss: 1.0122 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1210 | Classification: 0.0951 | Classification_binary: 0.1621 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1066 | Regression_eta: 0.0015 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1791 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0567 | Regression_eta: 0.0365 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0599 | LR: 1.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5474s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1495 | max_backward: 1.5906 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6415.4 | uss_MB: 5684.6 | child_rss_MB: 4746.9 | allocated_MB: 370.9 | reserved_MB: 24092.0 | inactive_split_MB: 226.9 +rank0 [2026-10-30 00:57:34,832] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:01:33,931] INFO : Step 49400/50000 rank0 | Train Loss: 0.9790 | Train Loss Unweighted: 0.4695 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1137 | Classification: 0.0883 | Classification_binary: 0.1513 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1131 | Regression_eta: 0.0013 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1227 | Regression_cos_phi: 0.0236 | Regression_energy: 0.0533 | Regression_eta: 0.0215 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0446 | LR: 1.00e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.7402s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2433 | max_backward: 1.8823 | max_optimizer_step: 0.3081 | max_tensorboard_logging: 0.5951 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6413.4 | uss_MB: 5608.6 | child_rss_MB: 4765.4 | allocated_MB: 372.2 | reserved_MB: 24092.0 | inactive_split_MB: 278.9 +rank0 [2026-10-30 01:01:34,177] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 0% +2 6 0x7408, 15961 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 54.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 83% +6 10 0x7408, 62036 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:05:37,901] INFO : Step 49500/50000 rank0 | Train Loss: 0.9706 | Train Loss Unweighted: 0.4581 | Train Loss Components: Auxiliary: 0.2120 | Cardinality: 0.1158 | Classification: 0.0879 | Classification_binary: 0.1279 | HitGrouping: 0.1920 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1305 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0579 | Regression_eta: 0.0261 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0282 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5919s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6307 | max_loss: 0.1505 | max_backward: 1.3337 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.8584 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6409.4 | uss_MB: 5683.7 | child_rss_MB: 4759.9 | allocated_MB: 370.4 | reserved_MB: 24614.0 | inactive_split_MB: 171.1 +rank0 [2026-10-30 01:05:38,153] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:09:50,308] INFO : Step 49600/50000 rank0 | Train Loss: 0.9965 | Train Loss Unweighted: 0.4752 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1098 | Classification: 0.1016 | Classification_binary: 0.1437 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1133 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1580 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0552 | Regression_eta: 0.0452 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0602 | LR: 1.00e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7965s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.6933 | max_loss: 0.1496 | max_backward: 1.5639 | max_optimizer_step: 0.3077 | max_tensorboard_logging: 0.7702 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6412.2 | uss_MB: 5687.4 | child_rss_MB: 4755.4 | allocated_MB: 371.3 | reserved_MB: 24614.0 | inactive_split_MB: 250.0 +rank0 [2026-10-30 01:09:50,549] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 48.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 57.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 49.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:13:45,113] INFO : Step 49700/50000 rank0 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4775 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1108 | Classification: 0.1000 | Classification_binary: 0.1702 | HitGrouping: 0.1796 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1019 | Regression_eta: 0.0014 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.1633 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0398 | Regression_eta: 0.0512 | Regression_pt: 0.0411 | Regression_sin_phi: 0.0503 | LR: 1.00e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.9173s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7591 | max_loss: 0.1933 | max_backward: 1.7410 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 6407.4 | uss_MB: 5675.2 | child_rss_MB: 4703.7 | allocated_MB: 372.4 | reserved_MB: 24614.0 | inactive_split_MB: 259.5 +rank0 [2026-10-30 01:13:45,354] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 50.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:17:46,754] INFO : Step 49800/50000 rank0 | Train Loss: 0.9472 | Train Loss Unweighted: 0.4401 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1188 | Classification: 0.0821 | Classification_binary: 0.1427 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1184 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0537 | Regression_eta: 0.0268 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0535 | LR: 1.00e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4399s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7430 | max_loss: 0.2153 | max_backward: 1.0846 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.7647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6388.3 | uss_MB: 5552.5 | child_rss_MB: 4630.2 | allocated_MB: 368.0 | reserved_MB: 24614.0 | inactive_split_MB: 273.7 +rank0 [2026-10-30 01:17:46,996] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 18% +4 8 0x7408, 13395 55.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 61% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:21:40,484] INFO : Step 49900/50000 rank0 | Train Loss: 0.9411 | Train Loss Unweighted: 0.4586 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.0996 | Classification: 0.0961 | Classification_binary: 0.1284 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1145 | Regression_eta: 0.0013 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0605 | Regression_eta: 0.0375 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0675 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8632s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.6992 | max_loss: 0.1527 | max_backward: 1.3836 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8576 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6363.3 | uss_MB: 5692.1 | child_rss_MB: 4528.3 | allocated_MB: 368.7 | reserved_MB: 24614.0 | inactive_split_MB: 241.3 +rank0 [2026-10-30 01:21:40,728] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 50.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 53.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 47.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:25:30,068] INFO : Step 50000/50000 rank0 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2297 | Cardinality: 0.1196 | Classification: 0.0902 | Classification_binary: 0.1583 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1740 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0531 | Regression_eta: 0.0261 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0595 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9409s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.7146 | max_loss: 0.1481 | max_backward: 1.6207 | max_optimizer_step: 0.2721 | max_tensorboard_logging: 0.8195 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6364.5 | uss_MB: 5690.8 | child_rss_MB: 4519.4 | allocated_MB: 371.5 | reserved_MB: 25088.0 | inactive_split_MB: 268.9 +rank0 [2026-10-30 01:25:30,312] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 58.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:25:30,374] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-50000.pth +rank0 [2026-10-30 01:25:30,573] INFO : Running validation on rank0 +rank0 [2026-10-30 01:25:30,573] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 01:25:30,573] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 01:26:22,380] INFO : VALIDATION | Step=50000/50000 | Train Loss=1.0036 | Valid Loss=0.9516 | Train Loss Unweighted=0.4712 | Valid Loss Unweighted=0.4487 | Stale=0 | ETA=0.0m +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 01:26:22,381] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/ari=0.544807 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.646452 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/top1_competing=0.741599 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.637389 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.00434591 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0629145 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 01:26:22,382] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/seconds_per_rank=21.3965 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 01:26:22,383] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 01:26:22,384] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 01:26:22,384] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 01:26:22,384] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding/ari=0.306605 +rank0 [2026-10-30 01:26:22,384] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding/ari_threshold=0.925 +rank0 [2026-10-30 01:26:22,384] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding_minus_raw/ari=0.214982 +rank0 [2026-10-30 01:26:22,384] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 01:26:22,384] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 01:26:22,384] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 01:26:22,389] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:26:22,649] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 01:26:22,653] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 01:26:49,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 01:26:51,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 01:26:52,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 01:26:53,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 01:27:03,979] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-30 01:27:03,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:27:04,557] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 01:27:04,560] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 01:27:21,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 01:27:22,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 01:27:23,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 01:27:24,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 01:27:34,244] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-30 01:27:34,256] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:27:34,756] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 01:27:34,759] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 01:27:50,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 01:27:51,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 01:27:52,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 01:27:53,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 01:28:03,059] INFO : Time taken to make predictions on device 0 is: 0.23 min +rank0 [2026-10-30 01:28:03,076] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits +rank0 [2026-10-30 01:28:06,308] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits +rank0 [2026-10-30 01:28:07,719] INFO : Plotted number of elements +rank0 [2026-10-30 01:28:09,385] INFO : Plotted jets +rank0 [2026-10-30 01:28:10,636] INFO : Plotted jet ratio +rank0 [2026-10-30 01:28:10,715] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 01:28:14,042] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 01:28:15,045] INFO : Plotted number of elements +rank0 [2026-10-30 01:28:16,555] INFO : Plotted jets +rank0 [2026-10-30 01:28:17,672] INFO : Plotted jet ratio +rank0 [2026-10-30 01:28:17,723] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits +rank0 [2026-10-30 01:28:20,232] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits +rank0 [2026-10-30 01:28:21,076] INFO : Plotted number of elements +rank0 [2026-10-30 01:28:23,032] INFO : Plotted jets +rank0 [2026-10-30 01:28:24,177] INFO : Plotted jet ratio +rank0 [2026-10-30 01:28:24,613] INFO : Training completed. Total time on device 0: 639.236min diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.1 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.1 new file mode 100644 index 0000000000000000000000000000000000000000..8174d1f8db972d6cc7563336c67eeb2dff954a93 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.1 @@ -0,0 +1,2484 @@ +rank1 [2026-10-28 11:12:33,078] INFO : Initializing model with seed=12345; process seed=12346 +rank1 [2026-10-28 11:12:33,079] INFO : configured dtype=torch.bfloat16 for autocast +rank1 [2026-10-28 11:12:54,339] INFO : Instantiating model +rank1 [2026-10-28 11:12:54,339] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank1 [2026-10-28 11:12:54,339] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank1 [2026-10-28 11:12:54,358] INFO : Input encoding initialization took 0.02s +rank1 [2026-10-28 11:12:54,358] INFO : nn0 parameters: 171776 +rank1 [2026-10-28 11:12:54,358] INFO : Initializing shared backbone layers, num_convs=6 +rank1 [2026-10-28 11:12:54,358] INFO : Initializing backbone layer 0 +rank1 [2026-10-28 11:12:54,358] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:54,362] INFO : Initializing backbone layer 1 +rank1 [2026-10-28 11:12:54,362] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:54,365] INFO : Initializing backbone layer 2 +rank1 [2026-10-28 11:12:54,365] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:54,368] INFO : Initializing backbone layer 3 +rank1 [2026-10-28 11:12:54,368] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:54,371] INFO : Initializing backbone layer 4 +rank1 [2026-10-28 11:12:54,371] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:54,374] INFO : Initializing backbone layer 5 +rank1 [2026-10-28 11:12:54,374] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-28 11:12:54,377] INFO : Backbone initialization took 0.02s +rank1 [2026-10-28 11:12:54,377] INFO : backbone parameters: 2374656 +rank1 [2026-10-28 11:12:54,377] INFO : Initializing output DNNs +rank1 [2026-10-28 11:12:54,426] INFO : Output DNNs initialization took 0.05s +rank1 [2026-10-28 11:12:54,426] INFO : backbone_mode=BackboneMode.SHARED +rank1 [2026-10-28 11:12:54,426] INFO : jagged_attention enabled=True +rank1 [2026-10-28 11:12:54,426] INFO : flash_attn_varlen enabled=True +rank1 [2026-10-28 11:12:54,427] INFO : task_queries enabled=False +rank1 [2026-10-28 11:12:54,427] INFO : classification_norm parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : regression_norm parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : classification_query parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : regression_query parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : classification_readout parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : regression_readout parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : output_mode=set +rank1 [2026-10-28 11:12:54,427] INFO : set_decoder parameters: 4713812 +rank1 [2026-10-28 11:12:54,427] INFO : nn_binary_particle parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : nn_pid parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : nn_pt parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : nn_eta parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : nn_sin_phi parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : nn_cos_phi parameters: 0 +rank1 [2026-10-28 11:12:54,427] INFO : nn_energy parameters: 0 +rank1 [2026-10-28 11:12:54,428] INFO : Total MLPF parameters: 7260244 +rank1 [2026-10-28 11:12:54,428] INFO : MLPF __init__ done +rank1 [2026-10-28 11:12:54,430] INFO : Instantiated model +rank1 [2026-10-28 11:12:54,430] INFO : Moving model to device rank=1 +rank1 [2026-10-28 11:12:54,448] INFO : Moved model to device rank=1 +rank1 [2026-10-28 11:12:54,449] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank1 [2026-10-28 11:12:54,450] INFO : Configured model for SyncBatchNorm rank=1 +rank1 [2026-10-28 11:12:54,644] INFO : Configured model for DistributedDataParallel rank=1 +rank1 [2026-10-28 11:12:54,762] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank1 [2026-10-28 11:12:54,762] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank1 [2026-10-28 11:12:54,763] INFO : Trainable parameters: 7260244 +rank1 [2026-10-28 11:12:54,763] INFO : Non-trainable parameters: 0 +rank1 [2026-10-28 11:12:54,763] INFO : Total parameters: 7260244 +rank1 [2026-10-28 11:12:54,765] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank1 [2026-10-28 11:12:54,766] INFO : Getting dataloaders +rank1 [2026-10-28 11:12:54,766] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:03,737] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:04,812] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,147] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-28 11:13:06,147] INFO : Creating WeightedMixIterator with 1 data loaders. +rank1 [2026-10-28 11:13:06,147] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,444] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,699] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,950] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-28 11:13:06,950] INFO : Creating InterleavedIterator with 1 data loaders. +rank1 [2026-10-28 11:13:20,820] INFO : Got dataloaders +rank1 [2026-10-28 11:13:20,821] INFO : loader split=train rank=1 len=21112 +rank1 [2026-10-28 11:13:20,821] INFO : loader split=valid rank=1 len=12 +rank1 [2026-10-28 11:13:20,821] INFO : Starting training from step 1 to 50000 on rank 1 of 8 +rank1 [2026-10-28 11:13:20,821] INFO : Creating train_iterator +rank1 [2026-10-28 11:13:20,821] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-28 11:13:35,381] INFO : Created train_iterator +rank1 [2026-10-28 11:17:15,588] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5434697668626904, 'Classification': 0.9147019968181849, 'Regression_pt': 1.0888657218962907, 'Regression_eta': 0.019410275079426356, 'Regression_sin_phi': 0.00660347732307855, 'Regression_cos_phi': 0.006484371119877323, 'Regression_energy': 1.1263945941627025}, weights={'Classification_binary': 1.0, 'Classification': 0.5941495299339294, 'Regression_pt': 0.4991154968738556, 'Regression_eta': 27.99907684326172, 'Regression_sin_phi': 82.30054473876953, 'Regression_cos_phi': 83.812255859375, 'Regression_energy': 0.48248612880706787} +rank1 [2026-10-28 11:17:17,664] INFO : Step 100/50000 rank1 | Train Loss: 5.4941 | Train Loss Unweighted: 3.3573 | Train Loss Components: Auxiliary: 1.0155 | Cardinality: 0.6417 | Classification: 0.8373 | Classification_binary: 0.4757 | HitGrouping: 0.4797 | Regression_cos_phi: 0.0063 | Regression_energy: 1.0524 | Regression_eta: 0.0130 | Regression_pt: 0.9668 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8438 | Classification_binary: 0.5206 | Regression_cos_phi: 0.0070 | Regression_energy: 0.9589 | Regression_eta: 0.0115 | Regression_pt: 0.9104 | Regression_sin_phi: 0.0057 | LR: 1.00e-03 | DataLoad Time: 0.0156s | Model Forward Time: 3.0145s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.5597 | max_loss: 0.2615 | max_backward: 1.0157 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.4654 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4647.0 | uss_MB: 3401.7 | child_rss_MB: 5567.8 | allocated_MB: 341.0 | reserved_MB: 22914.0 | inactive_split_MB: 237.2 +rank1 [2026-10-28 11:20:50,235] INFO : Step 200/50000 rank1 | Train Loss: 4.4651 | Train Loss Unweighted: 2.7232 | Train Loss Components: Auxiliary: 0.8610 | Cardinality: 0.5153 | Classification: 0.6425 | Classification_binary: 0.5162 | HitGrouping: 0.4582 | Regression_cos_phi: 0.0043 | Regression_energy: 0.7694 | Regression_eta: 0.0086 | Regression_pt: 0.7776 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3653 | Classification_binary: 0.5707 | Regression_cos_phi: 0.3837 | Regression_energy: 0.3692 | Regression_eta: 0.1997 | Regression_pt: 0.3796 | Regression_sin_phi: 0.4173 | LR: 1.00e-03 | DataLoad Time: 0.0152s | Model Forward Time: 3.7253s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7368 | max_loss: 0.2463 | max_backward: 1.7032 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5468 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4651.2 | uss_MB: 3409.7 | child_rss_MB: 5571.3 | allocated_MB: 343.8 | reserved_MB: 23264.0 | inactive_split_MB: 284.1 +rank1 [2026-10-28 11:24:24,163] INFO : Step 300/50000 rank1 | Train Loss: 3.6383 | Train Loss Unweighted: 2.1819 | Train Loss Components: Auxiliary: 0.6931 | Cardinality: 0.3447 | Classification: 0.5089 | Classification_binary: 0.4824 | HitGrouping: 0.4305 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5908 | Regression_eta: 0.0074 | Regression_pt: 0.5852 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2990 | Classification_binary: 0.4692 | Regression_cos_phi: 0.3059 | Regression_energy: 0.2743 | Regression_eta: 0.2061 | Regression_pt: 0.2806 | Regression_sin_phi: 0.2416 | LR: 1.00e-03 | DataLoad Time: 0.0126s | Model Forward Time: 3.8747s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0315 | max_forward: 0.6855 | max_loss: 0.1568 | max_backward: 1.3632 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 1.0182 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4652.7 | uss_MB: 3415.2 | child_rss_MB: 5569.6 | allocated_MB: 341.4 | reserved_MB: 23916.0 | inactive_split_MB: 289.5 +rank1 [2026-10-28 11:27:58,396] INFO : Step 400/50000 rank1 | Train Loss: 3.5064 | Train Loss Unweighted: 1.9938 | Train Loss Components: Auxiliary: 0.6582 | Cardinality: 0.3738 | Classification: 0.4591 | Classification_binary: 0.4306 | HitGrouping: 0.4041 | Regression_cos_phi: 0.0039 | Regression_energy: 0.5433 | Regression_eta: 0.0080 | Regression_pt: 0.5456 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2893 | Classification_binary: 0.4268 | Regression_cos_phi: 0.3282 | Regression_energy: 0.2685 | Regression_eta: 0.2025 | Regression_pt: 0.2780 | Regression_sin_phi: 0.2421 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 3.7419s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.6100 | max_loss: 0.1486 | max_backward: 1.2396 | max_optimizer_step: 0.1474 | max_tensorboard_logging: 1.0157 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4650.3 | uss_MB: 3416.8 | child_rss_MB: 5584.9 | allocated_MB: 342.9 | reserved_MB: 23982.0 | inactive_split_MB: 297.5 +rank1 [2026-10-28 11:31:35,335] INFO : Step 500/50000 rank1 | Train Loss: 3.8899 | Train Loss Unweighted: 2.1856 | Train Loss Components: Auxiliary: 0.6982 | Cardinality: 0.3682 | Classification: 0.4707 | Classification_binary: 0.4836 | HitGrouping: 0.4071 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6048 | Regression_eta: 0.0103 | Regression_pt: 0.6070 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2983 | Classification_binary: 0.5348 | Regression_cos_phi: 0.4264 | Regression_energy: 0.2880 | Regression_eta: 0.3113 | Regression_pt: 0.3023 | Regression_sin_phi: 0.5280 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 4.1830s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.7460 | max_loss: 0.2622 | max_backward: 1.7372 | max_optimizer_step: 0.1478 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 4651.3 | uss_MB: 3418.2 | child_rss_MB: 5567.0 | allocated_MB: 344.2 | reserved_MB: 24988.0 | inactive_split_MB: 288.4 +rank1 [2026-10-28 11:35:12,860] INFO : Step 600/50000 rank1 | Train Loss: 3.5648 | Train Loss Unweighted: 2.0654 | Train Loss Components: Auxiliary: 0.6756 | Cardinality: 0.3628 | Classification: 0.4159 | Classification_binary: 0.5246 | HitGrouping: 0.3632 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5539 | Regression_eta: 0.0077 | Regression_pt: 0.5557 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2557 | Classification_binary: 0.5187 | Regression_cos_phi: 0.3594 | Regression_energy: 0.2727 | Regression_eta: 0.2448 | Regression_pt: 0.2822 | Regression_sin_phi: 0.3615 | LR: 1.00e-03 | DataLoad Time: 0.0155s | Model Forward Time: 4.1933s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0071 | max_forward: 0.7483 | max_loss: 0.2574 | max_backward: 1.6305 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.9709 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 4647.1 | uss_MB: 3418.2 | child_rss_MB: 5567.3 | allocated_MB: 344.6 | reserved_MB: 24988.0 | inactive_split_MB: 289.5 +rank1 [2026-10-28 11:38:47,057] INFO : Step 700/50000 rank1 | Train Loss: 3.0543 | Train Loss Unweighted: 1.8237 | Train Loss Components: Auxiliary: 0.6036 | Cardinality: 0.3094 | Classification: 0.3407 | Classification_binary: 0.4142 | HitGrouping: 0.3367 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5254 | Regression_eta: 0.0060 | Regression_pt: 0.5314 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1870 | Classification_binary: 0.3797 | Regression_cos_phi: 0.1941 | Regression_energy: 0.2447 | Regression_eta: 0.1156 | Regression_pt: 0.2576 | Regression_sin_phi: 0.1667 | LR: 1.00e-03 | DataLoad Time: 0.0171s | Model Forward Time: 3.4868s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.6445 | max_loss: 0.2136 | max_backward: 1.2179 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.7893 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4647.0 | uss_MB: 3416.3 | child_rss_MB: 5577.3 | allocated_MB: 341.4 | reserved_MB: 24988.0 | inactive_split_MB: 289.7 +rank1 [2026-10-28 11:42:25,056] INFO : Step 800/50000 rank1 | Train Loss: 3.1854 | Train Loss Unweighted: 1.9126 | Train Loss Components: Auxiliary: 0.6154 | Cardinality: 0.2934 | Classification: 0.4077 | Classification_binary: 0.4086 | HitGrouping: 0.3317 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5404 | Regression_eta: 0.0067 | Regression_pt: 0.5423 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2547 | Classification_binary: 0.4271 | Regression_cos_phi: 0.3116 | Regression_energy: 0.2795 | Regression_eta: 0.1463 | Regression_pt: 0.2841 | Regression_sin_phi: 0.1704 | LR: 9.99e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.3172s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.6526 | max_loss: 0.2001 | max_backward: 1.2539 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5713 | max_tensorboard_flush: 0.0138 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 4642.6 | uss_MB: 3416.3 | child_rss_MB: 5579.0 | allocated_MB: 346.1 | reserved_MB: 24988.0 | inactive_split_MB: 222.7 +rank1 [2026-10-28 11:46:07,652] INFO : Step 900/50000 rank1 | Train Loss: 3.4950 | Train Loss Unweighted: 2.0678 | Train Loss Components: Auxiliary: 0.6566 | Cardinality: 0.3469 | Classification: 0.4160 | Classification_binary: 0.4953 | HitGrouping: 0.3295 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5663 | Regression_eta: 0.0083 | Regression_pt: 0.5742 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2323 | Classification_binary: 0.5310 | Regression_cos_phi: 0.3115 | Regression_energy: 0.2971 | Regression_eta: 0.2466 | Regression_pt: 0.2933 | Regression_sin_phi: 0.2970 | LR: 9.99e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.8991s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8131 | max_loss: 0.2548 | max_backward: 1.8396 | max_optimizer_step: 0.1492 | max_tensorboard_logging: 0.4938 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 4643.4 | uss_MB: 3417.5 | child_rss_MB: 5588.2 | allocated_MB: 344.2 | reserved_MB: 26006.0 | inactive_split_MB: 273.7 +rank1 [2026-10-28 11:49:58,411] INFO : Step 1000/50000 rank1 | Train Loss: 3.0581 | Train Loss Unweighted: 1.8801 | Train Loss Components: Auxiliary: 0.5942 | Cardinality: 0.2760 | Classification: 0.3603 | Classification_binary: 0.4232 | HitGrouping: 0.3177 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5411 | Regression_eta: 0.0066 | Regression_pt: 0.5426 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1792 | Classification_binary: 0.3969 | Regression_cos_phi: 0.3032 | Regression_energy: 0.2589 | Regression_eta: 0.1897 | Regression_pt: 0.2718 | Regression_sin_phi: 0.2214 | LR: 9.99e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2160s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1006 | max_forward: 0.7513 | max_loss: 0.1459 | max_backward: 1.3658 | max_optimizer_step: 0.1511 | max_tensorboard_logging: 1.2506 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4639.1 | uss_MB: 3415.3 | child_rss_MB: 5584.2 | allocated_MB: 344.4 | reserved_MB: 26006.0 | inactive_split_MB: 271.1 +rank1 [2026-10-28 11:49:59,498] INFO : Running validation on rank1 +rank1 [2026-10-28 11:49:59,498] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 11:49:59,498] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 11:50:52,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:50:53,226] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 11:51:19,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 11:51:20,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 11:51:21,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 11:51:22,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 11:51:32,260] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 11:51:32,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:51:33,321] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 11:51:49,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 11:51:50,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 11:51:51,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 11:51:52,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 11:52:02,816] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 11:52:02,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:52:03,476] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 11:52:18,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 11:52:19,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 11:52:20,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 11:52:20,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 11:52:31,008] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 11:56:35,453] INFO : Step 1100/50000 rank1 | Train Loss: 2.9181 | Train Loss Unweighted: 1.8556 | Train Loss Components: Auxiliary: 0.5873 | Cardinality: 0.2540 | Classification: 0.3370 | Classification_binary: 0.3789 | HitGrouping: 0.3247 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5584 | Regression_eta: 0.0064 | Regression_pt: 0.5695 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1957 | Classification_binary: 0.3475 | Regression_cos_phi: 0.1636 | Regression_energy: 0.2371 | Regression_eta: 0.1433 | Regression_pt: 0.2479 | Regression_sin_phi: 0.1765 | LR: 9.99e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.9464s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8528 | max_loss: 0.1390 | max_backward: 1.4220 | max_optimizer_step: 0.2689 | max_tensorboard_logging: 0.9135 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 6058.3 | uss_MB: 4835.8 | child_rss_MB: 5619.2 | allocated_MB: 344.1 | reserved_MB: 21080.0 | inactive_split_MB: 227.9 +rank1 [2026-10-28 12:00:25,221] INFO : Step 1200/50000 rank1 | Train Loss: 3.1885 | Train Loss Unweighted: 1.9202 | Train Loss Components: Auxiliary: 0.6265 | Cardinality: 0.3516 | Classification: 0.3718 | Classification_binary: 0.4294 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5507 | Regression_eta: 0.0065 | Regression_pt: 0.5556 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2400 | Classification_binary: 0.4499 | Regression_cos_phi: 0.2794 | Regression_energy: 0.2772 | Regression_eta: 0.2041 | Regression_pt: 0.2864 | Regression_sin_phi: 0.2477 | LR: 9.99e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.4088s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0076 | max_forward: 0.7370 | max_loss: 0.3921 | max_backward: 1.7945 | max_optimizer_step: 0.1639 | max_tensorboard_logging: 1.0479 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 6061.1 | uss_MB: 4838.4 | child_rss_MB: 5619.8 | allocated_MB: 344.9 | reserved_MB: 23662.0 | inactive_split_MB: 241.9 +rank1 [2026-10-28 12:04:16,198] INFO : Step 1300/50000 rank1 | Train Loss: 3.2694 | Train Loss Unweighted: 1.9280 | Train Loss Components: Auxiliary: 0.6193 | Cardinality: 0.3461 | Classification: 0.3601 | Classification_binary: 0.4731 | HitGrouping: 0.3156 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5359 | Regression_eta: 0.0069 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2173 | Classification_binary: 0.4642 | Regression_cos_phi: 0.3175 | Regression_energy: 0.2815 | Regression_eta: 0.2065 | Regression_pt: 0.2931 | Regression_sin_phi: 0.2685 | LR: 9.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.5224s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0540 | max_forward: 0.9909 | max_loss: 0.3234 | max_backward: 2.2964 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 1.1252 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5498.9 | uss_MB: 4193.7 | child_rss_MB: 5645.6 | allocated_MB: 344.9 | reserved_MB: 24052.0 | inactive_split_MB: 299.1 +rank1 [2026-10-28 12:07:55,471] INFO : Step 1400/50000 rank1 | Train Loss: 2.8491 | Train Loss Unweighted: 1.7889 | Train Loss Components: Auxiliary: 0.5645 | Cardinality: 0.2711 | Classification: 0.3226 | Classification_binary: 0.3768 | HitGrouping: 0.3109 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5376 | Regression_eta: 0.0061 | Regression_pt: 0.5407 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1684 | Classification_binary: 0.3784 | Regression_cos_phi: 0.1863 | Regression_energy: 0.2846 | Regression_eta: 0.1614 | Regression_pt: 0.2907 | Regression_sin_phi: 0.1798 | LR: 9.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7568s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0094 | max_forward: 0.6378 | max_loss: 0.4388 | max_backward: 1.2886 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 0.9631 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 5501.2 | uss_MB: 4277.0 | child_rss_MB: 5652.8 | allocated_MB: 350.4 | reserved_MB: 24916.0 | inactive_split_MB: 216.1 +rank1 [2026-10-28 12:11:42,089] INFO : Step 1500/50000 rank1 | Train Loss: 2.8928 | Train Loss Unweighted: 1.7749 | Train Loss Components: Auxiliary: 0.5690 | Cardinality: 0.2892 | Classification: 0.3566 | Classification_binary: 0.3704 | HitGrouping: 0.3078 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5147 | Regression_eta: 0.0058 | Regression_pt: 0.5216 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2087 | Classification_binary: 0.3440 | Regression_cos_phi: 0.2166 | Regression_energy: 0.2309 | Regression_eta: 0.1145 | Regression_pt: 0.2449 | Regression_sin_phi: 0.1607 | LR: 9.98e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5602s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.6094 | max_loss: 0.4244 | max_backward: 1.4479 | max_optimizer_step: 0.1602 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5488.8 | uss_MB: 4270.8 | child_rss_MB: 5621.5 | allocated_MB: 342.2 | reserved_MB: 24916.0 | inactive_split_MB: 182.9 +rank1 [2026-10-28 12:15:27,599] INFO : Step 1600/50000 rank1 | Train Loss: 2.9805 | Train Loss Unweighted: 1.8512 | Train Loss Components: Auxiliary: 0.5725 | Cardinality: 0.2336 | Classification: 0.3605 | Classification_binary: 0.4404 | HitGrouping: 0.3159 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5142 | Regression_eta: 0.0065 | Regression_pt: 0.5235 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2104 | Classification_binary: 0.4888 | Regression_cos_phi: 0.2684 | Regression_energy: 0.2141 | Regression_eta: 0.2023 | Regression_pt: 0.2283 | Regression_sin_phi: 0.3377 | LR: 9.98e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.7180s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7382 | max_loss: 0.1451 | max_backward: 1.5359 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.4679 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5499.3 | uss_MB: 4281.1 | child_rss_MB: 5637.2 | allocated_MB: 343.2 | reserved_MB: 25046.0 | inactive_split_MB: 260.8 +rank1 [2026-10-28 12:19:19,666] INFO : Step 1700/50000 rank1 | Train Loss: 2.9423 | Train Loss Unweighted: 1.7603 | Train Loss Components: Auxiliary: 0.5618 | Cardinality: 0.2834 | Classification: 0.3350 | Classification_binary: 0.4141 | HitGrouping: 0.3155 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4976 | Regression_eta: 0.0064 | Regression_pt: 0.5011 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2008 | Classification_binary: 0.3745 | Regression_cos_phi: 0.2898 | Regression_energy: 0.2273 | Regression_eta: 0.1754 | Regression_pt: 0.2340 | Regression_sin_phi: 0.1952 | LR: 9.97e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.2301s | Phase Times: max_pre_step_barrier: 0.0070 | max_batch_to_device: 0.0065 | max_forward: 0.8351 | max_loss: 0.2466 | max_backward: 1.7292 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0857 | Memory: rss_MB: 5499.7 | uss_MB: 4254.7 | child_rss_MB: 5622.9 | allocated_MB: 342.0 | reserved_MB: 25046.0 | inactive_split_MB: 198.9 +rank1 [2026-10-28 12:23:11,816] INFO : Step 1800/50000 rank1 | Train Loss: 2.7182 | Train Loss Unweighted: 1.6765 | Train Loss Components: Auxiliary: 0.5305 | Cardinality: 0.2556 | Classification: 0.3337 | Classification_binary: 0.3613 | HitGrouping: 0.3103 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4855 | Regression_eta: 0.0054 | Regression_pt: 0.4854 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2283 | Classification_binary: 0.3613 | Regression_cos_phi: 0.1851 | Regression_energy: 0.2431 | Regression_eta: 0.1327 | Regression_pt: 0.2516 | Regression_sin_phi: 0.1724 | LR: 9.97e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0937s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6573 | max_loss: 0.2402 | max_backward: 1.4111 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 1.0385 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0959 | Memory: rss_MB: 5501.3 | uss_MB: 4283.5 | child_rss_MB: 5636.1 | allocated_MB: 340.6 | reserved_MB: 25046.0 | inactive_split_MB: 181.6 +rank1 [2026-10-28 12:26:53,741] INFO : Step 1900/50000 rank1 | Train Loss: 2.9893 | Train Loss Unweighted: 1.8606 | Train Loss Components: Auxiliary: 0.5839 | Cardinality: 0.2900 | Classification: 0.3568 | Classification_binary: 0.4041 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5399 | Regression_eta: 0.0063 | Regression_pt: 0.5477 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2104 | Classification_binary: 0.4348 | Regression_cos_phi: 0.2106 | Regression_energy: 0.2458 | Regression_eta: 0.2256 | Regression_pt: 0.2581 | Regression_sin_phi: 0.2612 | LR: 9.97e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8706s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.6296 | max_loss: 0.1482 | max_backward: 1.2931 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 1.0537 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 5503.8 | uss_MB: 4286.6 | child_rss_MB: 5619.9 | allocated_MB: 344.8 | reserved_MB: 25048.0 | inactive_split_MB: 225.2 +rank1 [2026-10-28 12:30:53,516] INFO : Step 2000/50000 rank1 | Train Loss: 3.0007 | Train Loss Unweighted: 1.8040 | Train Loss Components: Auxiliary: 0.5673 | Cardinality: 0.3091 | Classification: 0.3519 | Classification_binary: 0.4482 | HitGrouping: 0.3137 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4922 | Regression_eta: 0.0062 | Regression_pt: 0.4996 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1975 | Classification_binary: 0.4156 | Regression_cos_phi: 0.2664 | Regression_energy: 0.2484 | Regression_eta: 0.1351 | Regression_pt: 0.2482 | Regression_sin_phi: 0.2290 | LR: 9.96e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1668s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6813 | max_loss: 0.2416 | max_backward: 1.5165 | max_optimizer_step: 0.1635 | max_tensorboard_logging: 0.9842 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5502.9 | uss_MB: 4287.2 | child_rss_MB: 5669.7 | allocated_MB: 346.7 | reserved_MB: 25696.0 | inactive_split_MB: 287.8 +rank1 [2026-10-28 12:30:54,413] INFO : Running validation on rank1 +rank1 [2026-10-28 12:30:54,413] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 12:30:54,413] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 12:31:46,080] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:31:46,628] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 12:32:13,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 12:32:14,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 12:32:15,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 12:32:16,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 12:32:26,530] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 12:32:27,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:32:27,705] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 12:32:43,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 12:32:44,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 12:32:45,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 12:32:46,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 12:32:56,538] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 12:32:57,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:32:57,629] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 12:33:13,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 12:33:14,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 12:33:14,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 12:33:15,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 12:33:25,684] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 12:37:37,593] INFO : Step 2100/50000 rank1 | Train Loss: 2.6299 | Train Loss Unweighted: 1.6429 | Train Loss Components: Auxiliary: 0.5172 | Cardinality: 0.2489 | Classification: 0.3119 | Classification_binary: 0.3591 | HitGrouping: 0.3047 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4785 | Regression_eta: 0.0053 | Regression_pt: 0.4834 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1988 | Classification_binary: 0.3396 | Regression_cos_phi: 0.1953 | Regression_energy: 0.2094 | Regression_eta: 0.1348 | Regression_pt: 0.2213 | Regression_sin_phi: 0.1372 | LR: 9.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0177s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0685 | max_forward: 0.7343 | max_loss: 0.1422 | max_backward: 1.3363 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.0015 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6447.9 | uss_MB: 5231.1 | child_rss_MB: 5629.3 | allocated_MB: 341.1 | reserved_MB: 22546.0 | inactive_split_MB: 269.0 +rank1 [2026-10-28 12:41:24,842] INFO : Step 2200/50000 rank1 | Train Loss: 2.8483 | Train Loss Unweighted: 1.7068 | Train Loss Components: Auxiliary: 0.5372 | Cardinality: 0.2547 | Classification: 0.3411 | Classification_binary: 0.3679 | HitGrouping: 0.3030 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4876 | Regression_eta: 0.0061 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1857 | Classification_binary: 0.3517 | Regression_cos_phi: 0.2365 | Regression_energy: 0.2757 | Regression_eta: 0.1788 | Regression_pt: 0.2935 | Regression_sin_phi: 0.2995 | LR: 9.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.4917s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6352 | max_loss: 0.1414 | max_backward: 1.3119 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.4703 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6448.9 | uss_MB: 5233.9 | child_rss_MB: 5636.2 | allocated_MB: 341.4 | reserved_MB: 22598.0 | inactive_split_MB: 313.4 +rank1 [2026-10-28 12:45:27,860] INFO : Step 2300/50000 rank1 | Train Loss: 3.1410 | Train Loss Unweighted: 1.8301 | Train Loss Components: Auxiliary: 0.5918 | Cardinality: 0.3583 | Classification: 0.3517 | Classification_binary: 0.4348 | HitGrouping: 0.3083 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5121 | Regression_eta: 0.0065 | Regression_pt: 0.5184 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2259 | Classification_binary: 0.4730 | Regression_cos_phi: 0.3113 | Regression_energy: 0.2383 | Regression_eta: 0.2333 | Regression_pt: 0.2520 | Regression_sin_phi: 0.3297 | LR: 9.95e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8597s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.7258 | max_loss: 0.1520 | max_backward: 1.6288 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.6027 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0166 | Memory: rss_MB: 5633.9 | uss_MB: 4422.4 | child_rss_MB: 5644.1 | allocated_MB: 346.5 | reserved_MB: 23596.0 | inactive_split_MB: 273.5 +rank1 [2026-10-28 12:49:22,411] INFO : Step 2400/50000 rank1 | Train Loss: 2.8063 | Train Loss Unweighted: 1.7109 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.2683 | Classification: 0.3276 | Classification_binary: 0.3929 | HitGrouping: 0.3025 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4873 | Regression_eta: 0.0054 | Regression_pt: 0.4919 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2046 | Classification_binary: 0.3711 | Regression_cos_phi: 0.2311 | Regression_energy: 0.2325 | Regression_eta: 0.2040 | Regression_pt: 0.2445 | Regression_sin_phi: 0.2719 | LR: 9.95e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.5897s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0314 | max_forward: 0.7331 | max_loss: 0.1502 | max_backward: 1.4174 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.5071 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0157 | Memory: rss_MB: 5635.8 | uss_MB: 4426.4 | child_rss_MB: 5648.9 | allocated_MB: 344.4 | reserved_MB: 23596.0 | inactive_split_MB: 276.8 +rank1 [2026-10-28 12:53:10,911] INFO : Step 2500/50000 rank1 | Train Loss: 2.5300 | Train Loss Unweighted: 1.6034 | Train Loss Components: Auxiliary: 0.5018 | Cardinality: 0.2387 | Classification: 0.2872 | Classification_binary: 0.3313 | HitGrouping: 0.3000 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4857 | Regression_eta: 0.0042 | Regression_pt: 0.4904 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1514 | Classification_binary: 0.3378 | Regression_cos_phi: 0.2293 | Regression_energy: 0.2376 | Regression_eta: 0.1175 | Regression_pt: 0.2486 | Regression_sin_phi: 0.2197 | LR: 9.94e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.0256s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0177 | max_forward: 0.6104 | max_loss: 0.1394 | max_backward: 1.2397 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 1.1029 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5639.0 | uss_MB: 4422.1 | child_rss_MB: 5639.4 | allocated_MB: 342.5 | reserved_MB: 23596.0 | inactive_split_MB: 302.8 +rank1 [2026-10-28 12:56:55,919] INFO : Step 2600/50000 rank1 | Train Loss: 2.8992 | Train Loss Unweighted: 1.7075 | Train Loss Components: Auxiliary: 0.5471 | Cardinality: 0.3219 | Classification: 0.3420 | Classification_binary: 0.4005 | HitGrouping: 0.3027 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4737 | Regression_eta: 0.0057 | Regression_pt: 0.4796 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2536 | Classification_binary: 0.4334 | Regression_cos_phi: 0.2878 | Regression_energy: 0.2103 | Regression_eta: 0.2471 | Regression_pt: 0.2249 | Regression_sin_phi: 0.2858 | LR: 9.94e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.7068s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8849 | max_loss: 0.1495 | max_backward: 1.9136 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1254 | Memory: rss_MB: 5638.3 | uss_MB: 4434.4 | child_rss_MB: 5644.3 | allocated_MB: 343.4 | reserved_MB: 23596.0 | inactive_split_MB: 290.6 +rank1 [2026-10-28 13:00:46,527] INFO : Step 2700/50000 rank1 | Train Loss: 2.8481 | Train Loss Unweighted: 1.7200 | Train Loss Components: Auxiliary: 0.5468 | Cardinality: 0.2918 | Classification: 0.3199 | Classification_binary: 0.4344 | HitGrouping: 0.3029 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4756 | Regression_eta: 0.0057 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1715 | Classification_binary: 0.4306 | Regression_cos_phi: 0.2671 | Regression_energy: 0.2192 | Regression_eta: 0.1554 | Regression_pt: 0.2286 | Regression_sin_phi: 0.1891 | LR: 9.94e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.3235s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0064 | max_forward: 0.7554 | max_loss: 0.2027 | max_backward: 1.6333 | max_optimizer_step: 0.1633 | max_tensorboard_logging: 0.9978 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5637.9 | uss_MB: 4434.7 | child_rss_MB: 5643.6 | allocated_MB: 343.8 | reserved_MB: 23598.0 | inactive_split_MB: 284.2 +rank1 [2026-10-28 13:04:40,295] INFO : Step 2800/50000 rank1 | Train Loss: 2.6381 | Train Loss Unweighted: 1.5955 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2387 | Classification: 0.2848 | Classification_binary: 0.3420 | HitGrouping: 0.2974 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4746 | Regression_eta: 0.0056 | Regression_pt: 0.4829 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1338 | Classification_binary: 0.3089 | Regression_cos_phi: 0.2146 | Regression_energy: 0.2170 | Regression_eta: 0.1620 | Regression_pt: 0.2324 | Regression_sin_phi: 0.1920 | LR: 9.93e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7368s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6805 | max_loss: 0.2368 | max_backward: 1.1707 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5635.7 | uss_MB: 4432.9 | child_rss_MB: 5638.4 | allocated_MB: 341.2 | reserved_MB: 24554.0 | inactive_split_MB: 308.0 +rank1 [2026-10-28 13:08:36,459] INFO : Step 2900/50000 rank1 | Train Loss: 2.7831 | Train Loss Unweighted: 1.6528 | Train Loss Components: Auxiliary: 0.5323 | Cardinality: 0.3031 | Classification: 0.3069 | Classification_binary: 0.3681 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4798 | Regression_eta: 0.0056 | Regression_pt: 0.4869 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1945 | Classification_binary: 0.3910 | Regression_cos_phi: 0.2482 | Regression_energy: 0.2277 | Regression_eta: 0.2323 | Regression_pt: 0.2379 | Regression_sin_phi: 0.2844 | LR: 9.93e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.7856s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6242 | max_loss: 0.1475 | max_backward: 1.3573 | max_optimizer_step: 0.2831 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5637.0 | uss_MB: 4433.8 | child_rss_MB: 5645.5 | allocated_MB: 341.4 | reserved_MB: 25570.0 | inactive_split_MB: 292.0 +rank1 [2026-10-28 13:12:15,921] INFO : Step 3000/50000 rank1 | Train Loss: 2.9336 | Train Loss Unweighted: 1.7260 | Train Loss Components: Auxiliary: 0.5466 | Cardinality: 0.2829 | Classification: 0.3524 | Classification_binary: 0.4205 | HitGrouping: 0.3085 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4673 | Regression_eta: 0.0061 | Regression_pt: 0.4733 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2056 | Classification_binary: 0.4715 | Regression_cos_phi: 0.2629 | Regression_energy: 0.2402 | Regression_eta: 0.1865 | Regression_pt: 0.2508 | Regression_sin_phi: 0.2605 | LR: 9.92e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1514s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0386 | max_forward: 0.8057 | max_loss: 0.1488 | max_backward: 1.9376 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5635.9 | uss_MB: 4440.1 | child_rss_MB: 5640.1 | allocated_MB: 346.5 | reserved_MB: 25570.0 | inactive_split_MB: 281.4 +rank1 [2026-10-28 13:12:16,813] INFO : Running validation on rank1 +rank1 [2026-10-28 13:12:16,814] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 13:12:16,814] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 13:13:09,907] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:13:10,420] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 13:13:30,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 13:13:31,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 13:13:31,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 13:13:32,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 13:13:42,876] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 13:13:43,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:13:44,016] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 13:13:59,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 13:14:00,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 13:14:01,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 13:14:02,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 13:14:12,974] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 13:14:13,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:14:14,220] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 13:14:29,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 13:14:30,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 13:14:31,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 13:14:31,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 13:14:41,860] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 13:18:52,946] INFO : Step 3100/50000 rank1 | Train Loss: 2.7478 | Train Loss Unweighted: 1.6734 | Train Loss Components: Auxiliary: 0.5216 | Cardinality: 0.2531 | Classification: 0.2981 | Classification_binary: 0.3781 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4881 | Regression_eta: 0.0057 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1668 | Classification_binary: 0.3969 | Regression_cos_phi: 0.1987 | Regression_energy: 0.2098 | Regression_eta: 0.1354 | Regression_pt: 0.2174 | Regression_sin_phi: 0.1732 | LR: 9.91e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5560s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.6511 | max_loss: 0.3988 | max_backward: 1.4306 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 1.2631 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5749.3 | uss_MB: 4552.2 | child_rss_MB: 5658.3 | allocated_MB: 342.5 | reserved_MB: 23120.0 | inactive_split_MB: 194.6 +rank1 [2026-10-28 13:22:38,494] INFO : Step 3200/50000 rank1 | Train Loss: 2.5254 | Train Loss Unweighted: 1.5710 | Train Loss Components: Auxiliary: 0.4888 | Cardinality: 0.2129 | Classification: 0.2845 | Classification_binary: 0.3271 | HitGrouping: 0.2964 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4770 | Regression_eta: 0.0050 | Regression_pt: 0.4723 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1680 | Classification_binary: 0.3377 | Regression_cos_phi: 0.2189 | Regression_energy: 0.2397 | Regression_eta: 0.1352 | Regression_pt: 0.2471 | Regression_sin_phi: 0.2617 | LR: 9.91e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.5510s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6134 | max_loss: 0.2441 | max_backward: 1.2833 | max_optimizer_step: 0.2753 | max_tensorboard_logging: 0.5629 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5733.9 | uss_MB: 4558.8 | child_rss_MB: 5628.8 | allocated_MB: 343.6 | reserved_MB: 23340.0 | inactive_split_MB: 191.9 +rank1 [2026-10-28 13:26:30,566] INFO : Step 3300/50000 rank1 | Train Loss: 2.8037 | Train Loss Unweighted: 1.6635 | Train Loss Components: Auxiliary: 0.5345 | Cardinality: 0.2841 | Classification: 0.3310 | Classification_binary: 0.3857 | HitGrouping: 0.3002 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4633 | Regression_eta: 0.0058 | Regression_pt: 0.4718 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1892 | Classification_binary: 0.3797 | Regression_cos_phi: 0.2000 | Regression_energy: 0.2483 | Regression_eta: 0.1287 | Regression_pt: 0.2617 | Regression_sin_phi: 0.2336 | LR: 9.90e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2961s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9147 | max_loss: 0.1535 | max_backward: 1.9100 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.6631 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5735.1 | uss_MB: 4562.9 | child_rss_MB: 5661.7 | allocated_MB: 344.0 | reserved_MB: 23618.0 | inactive_split_MB: 250.0 +rank1 [2026-10-28 13:30:24,444] INFO : Step 3400/50000 rank1 | Train Loss: 2.8855 | Train Loss Unweighted: 1.7067 | Train Loss Components: Auxiliary: 0.5454 | Cardinality: 0.2859 | Classification: 0.3252 | Classification_binary: 0.4155 | HitGrouping: 0.2965 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4731 | Regression_eta: 0.0067 | Regression_pt: 0.4802 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1878 | Classification_binary: 0.3779 | Regression_cos_phi: 0.2716 | Regression_energy: 0.2261 | Regression_eta: 0.2541 | Regression_pt: 0.2323 | Regression_sin_phi: 0.2077 | LR: 9.90e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5648s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0438 | max_forward: 0.7271 | max_loss: 0.1506 | max_backward: 1.3643 | max_optimizer_step: 0.1716 | max_tensorboard_logging: 0.4827 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5729.2 | uss_MB: 4562.1 | child_rss_MB: 5658.8 | allocated_MB: 346.0 | reserved_MB: 23618.0 | inactive_split_MB: 235.6 +rank1 [2026-10-28 13:34:14,208] INFO : Step 3500/50000 rank1 | Train Loss: 2.4422 | Train Loss Unweighted: 1.4809 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.2120 | Classification: 0.2615 | Classification_binary: 0.3296 | HitGrouping: 0.2898 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4369 | Regression_eta: 0.0054 | Regression_pt: 0.4427 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1339 | Classification_binary: 0.2924 | Regression_cos_phi: 0.2343 | Regression_energy: 0.1986 | Regression_eta: 0.2023 | Regression_pt: 0.2093 | Regression_sin_phi: 0.2359 | LR: 9.89e-04 | DataLoad Time: 0.0147s | Model Forward Time: 3.7311s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0058 | max_forward: 0.7056 | max_loss: 0.1971 | max_backward: 1.3359 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.6307 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5720.3 | uss_MB: 4563.4 | child_rss_MB: 5630.0 | allocated_MB: 339.0 | reserved_MB: 24610.0 | inactive_split_MB: 280.8 +rank1 [2026-10-28 13:38:02,722] INFO : Step 3600/50000 rank1 | Train Loss: 2.5780 | Train Loss Unweighted: 1.5250 | Train Loss Components: Auxiliary: 0.4892 | Cardinality: 0.2463 | Classification: 0.3079 | Classification_binary: 0.3446 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4295 | Regression_eta: 0.0052 | Regression_pt: 0.4324 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1930 | Classification_binary: 0.3673 | Regression_cos_phi: 0.2594 | Regression_energy: 0.2406 | Regression_eta: 0.1201 | Regression_pt: 0.2509 | Regression_sin_phi: 0.2082 | LR: 9.89e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.4637s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1453 | max_backward: 1.4106 | max_optimizer_step: 0.1681 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0126 | Memory: rss_MB: 5726.7 | uss_MB: 4567.6 | child_rss_MB: 5631.8 | allocated_MB: 343.9 | reserved_MB: 24610.0 | inactive_split_MB: 244.5 +rank1 [2026-10-28 13:41:50,384] INFO : Step 3700/50000 rank1 | Train Loss: 2.9213 | Train Loss Unweighted: 1.6952 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.3153 | Classification: 0.3418 | Classification_binary: 0.4227 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4570 | Regression_eta: 0.0063 | Regression_pt: 0.4612 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1783 | Classification_binary: 0.4099 | Regression_cos_phi: 0.2629 | Regression_energy: 0.2145 | Regression_eta: 0.1834 | Regression_pt: 0.2251 | Regression_sin_phi: 0.3168 | LR: 9.88e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2196s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7710 | max_loss: 0.1507 | max_backward: 1.6430 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.8888 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0145 | Memory: rss_MB: 5726.2 | uss_MB: 4569.4 | child_rss_MB: 5661.5 | allocated_MB: 345.4 | reserved_MB: 24610.0 | inactive_split_MB: 268.9 +rank1 [2026-10-28 13:45:46,299] INFO : Step 3800/50000 rank1 | Train Loss: 2.6063 | Train Loss Unweighted: 1.5289 | Train Loss Components: Auxiliary: 0.4933 | Cardinality: 0.2661 | Classification: 0.2839 | Classification_binary: 0.3652 | HitGrouping: 0.2879 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4319 | Regression_eta: 0.0058 | Regression_pt: 0.4369 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1817 | Classification_binary: 0.3489 | Regression_cos_phi: 0.2458 | Regression_energy: 0.1870 | Regression_eta: 0.2992 | Regression_pt: 0.1884 | Regression_sin_phi: 0.2679 | LR: 9.87e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1714s | Phase Times: max_pre_step_barrier: 0.1134 | max_batch_to_device: 0.0063 | max_forward: 0.6784 | max_loss: 0.1451 | max_backward: 1.4269 | max_optimizer_step: 0.2795 | max_tensorboard_logging: 0.9060 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5726.3 | uss_MB: 4569.5 | child_rss_MB: 5664.0 | allocated_MB: 343.8 | reserved_MB: 24610.0 | inactive_split_MB: 202.2 +rank1 [2026-10-28 13:49:32,923] INFO : Step 3900/50000 rank1 | Train Loss: 2.2761 | Train Loss Unweighted: 1.4165 | Train Loss Components: Auxiliary: 0.4517 | Cardinality: 0.1938 | Classification: 0.2553 | Classification_binary: 0.2930 | HitGrouping: 0.2882 | Regression_cos_phi: 0.0020 | Regression_energy: 0.4264 | Regression_eta: 0.0045 | Regression_pt: 0.4331 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2040 | Classification_binary: 0.3246 | Regression_cos_phi: 0.2860 | Regression_energy: 0.1907 | Regression_eta: 0.1728 | Regression_pt: 0.2032 | Regression_sin_phi: 0.2527 | LR: 9.87e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8894s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6197 | max_loss: 0.1501 | max_backward: 1.2810 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.0075 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 5727.5 | uss_MB: 3764.5 | child_rss_MB: 5661.9 | allocated_MB: 343.5 | reserved_MB: 24610.0 | inactive_split_MB: 286.6 +rank1 [2026-10-28 13:53:23,685] INFO : Step 4000/50000 rank1 | Train Loss: 2.6760 | Train Loss Unweighted: 1.5903 | Train Loss Components: Auxiliary: 0.5112 | Cardinality: 0.2445 | Classification: 0.3070 | Classification_binary: 0.3812 | HitGrouping: 0.2906 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4413 | Regression_eta: 0.0062 | Regression_pt: 0.4492 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1914 | Classification_binary: 0.4032 | Regression_cos_phi: 0.1882 | Regression_energy: 0.2316 | Regression_eta: 0.1493 | Regression_pt: 0.2437 | Regression_sin_phi: 0.2303 | LR: 9.86e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.0142s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0492 | max_forward: 0.7834 | max_loss: 0.1586 | max_backward: 1.6903 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.5589 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5731.5 | uss_MB: 4577.2 | child_rss_MB: 5661.6 | allocated_MB: 345.5 | reserved_MB: 24610.0 | inactive_split_MB: 252.6 +rank1 [2026-10-28 13:53:24,556] INFO : Running validation on rank1 +rank1 [2026-10-28 13:53:24,556] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 13:53:24,557] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 13:54:13,892] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:54:14,418] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 13:54:39,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 13:54:41,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 13:54:42,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 13:54:42,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 13:54:52,965] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 13:54:53,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:54:53,947] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 13:55:10,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 13:55:11,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 13:55:12,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 13:55:12,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 13:55:23,007] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 13:55:23,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:55:23,752] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 13:55:39,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 13:55:39,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 13:55:40,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 13:55:41,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 13:55:51,281] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 13:59:55,049] INFO : Step 4100/50000 rank1 | Train Loss: 2.6923 | Train Loss Unweighted: 1.5640 | Train Loss Components: Auxiliary: 0.5058 | Cardinality: 0.2671 | Classification: 0.2981 | Classification_binary: 0.4073 | HitGrouping: 0.2911 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4210 | Regression_eta: 0.0055 | Regression_pt: 0.4265 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1889 | Classification_binary: 0.3816 | Regression_cos_phi: 0.2002 | Regression_energy: 0.2147 | Regression_eta: 0.1224 | Regression_pt: 0.2279 | Regression_sin_phi: 0.1808 | LR: 9.85e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1496s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0141 | max_forward: 0.7422 | max_loss: 0.3919 | max_backward: 1.3629 | max_optimizer_step: 0.1890 | max_tensorboard_logging: 0.9903 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6680.3 | uss_MB: 5525.0 | child_rss_MB: 5670.9 | allocated_MB: 345.2 | reserved_MB: 23430.0 | inactive_split_MB: 294.4 +rank1 [2026-10-28 14:03:47,899] INFO : Step 4200/50000 rank1 | Train Loss: 2.3415 | Train Loss Unweighted: 1.4424 | Train Loss Components: Auxiliary: 0.4542 | Cardinality: 0.2058 | Classification: 0.2540 | Classification_binary: 0.3143 | HitGrouping: 0.2846 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4310 | Regression_eta: 0.0050 | Regression_pt: 0.4337 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1592 | Classification_binary: 0.3043 | Regression_cos_phi: 0.2199 | Regression_energy: 0.2018 | Regression_eta: 0.1424 | Regression_pt: 0.2088 | Regression_sin_phi: 0.1631 | LR: 9.84e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.7401s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0168 | max_forward: 0.8335 | max_loss: 0.2425 | max_backward: 1.5586 | max_optimizer_step: 0.1744 | max_tensorboard_logging: 0.4781 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5904.9 | uss_MB: 4751.1 | child_rss_MB: 5672.4 | allocated_MB: 341.2 | reserved_MB: 24406.0 | inactive_split_MB: 273.9 +rank1 [2026-10-28 14:07:35,176] INFO : Step 4300/50000 rank1 | Train Loss: 2.5102 | Train Loss Unweighted: 1.4857 | Train Loss Components: Auxiliary: 0.4757 | Cardinality: 0.2309 | Classification: 0.2786 | Classification_binary: 0.3543 | HitGrouping: 0.2938 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4177 | Regression_eta: 0.0054 | Regression_pt: 0.4246 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1891 | Classification_binary: 0.3909 | Regression_cos_phi: 0.2802 | Regression_energy: 0.1737 | Regression_eta: 0.1720 | Regression_pt: 0.1810 | Regression_sin_phi: 0.2218 | LR: 9.84e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5997s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6520 | max_loss: 0.2453 | max_backward: 1.4084 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.5540 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5910.3 | uss_MB: 4757.7 | child_rss_MB: 5668.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 239.6 +rank1 [2026-10-28 14:11:23,917] INFO : Step 4400/50000 rank1 | Train Loss: 2.8258 | Train Loss Unweighted: 1.5691 | Train Loss Components: Auxiliary: 0.5223 | Cardinality: 0.3313 | Classification: 0.2877 | Classification_binary: 0.4120 | HitGrouping: 0.2902 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4255 | Regression_eta: 0.0063 | Regression_pt: 0.4316 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1759 | Classification_binary: 0.4483 | Regression_cos_phi: 0.2701 | Regression_energy: 0.2045 | Regression_eta: 0.1628 | Regression_pt: 0.2167 | Regression_sin_phi: 0.2146 | LR: 9.83e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.0439s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.7635 | max_loss: 0.2580 | max_backward: 1.7456 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.5221 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 5909.1 | uss_MB: 4757.0 | child_rss_MB: 5674.0 | allocated_MB: 344.6 | reserved_MB: 24406.0 | inactive_split_MB: 261.0 +rank1 [2026-10-28 14:15:17,054] INFO : Step 4500/50000 rank1 | Train Loss: 2.5622 | Train Loss Unweighted: 1.4664 | Train Loss Components: Auxiliary: 0.4942 | Cardinality: 0.3132 | Classification: 0.2685 | Classification_binary: 0.3695 | HitGrouping: 0.2876 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4070 | Regression_eta: 0.0048 | Regression_pt: 0.4117 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1325 | Classification_binary: 0.3257 | Regression_cos_phi: 0.1571 | Regression_energy: 0.1932 | Regression_eta: 0.1091 | Regression_pt: 0.2040 | Regression_sin_phi: 0.1510 | LR: 9.82e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.2158s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0462 | max_forward: 1.1330 | max_loss: 0.3526 | max_backward: 2.1744 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5571 | max_tensorboard_flush: 0.0016 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 5912.8 | uss_MB: 4736.1 | child_rss_MB: 5674.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 288.8 +rank1 [2026-10-28 14:19:09,440] INFO : Step 4600/50000 rank1 | Train Loss: 2.2516 | Train Loss Unweighted: 1.3548 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2151 | Classification: 0.2428 | Classification_binary: 0.2886 | HitGrouping: 0.2811 | Regression_cos_phi: 0.0019 | Regression_energy: 0.4059 | Regression_eta: 0.0042 | Regression_pt: 0.4088 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1213 | Classification_binary: 0.2740 | Regression_cos_phi: 0.1313 | Regression_energy: 0.1843 | Regression_eta: 0.0757 | Regression_pt: 0.1927 | Regression_sin_phi: 0.2038 | LR: 9.81e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7005s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0445 | max_forward: 0.8897 | max_loss: 0.2214 | max_backward: 1.8748 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5913.2 | uss_MB: 4741.7 | child_rss_MB: 5653.3 | allocated_MB: 341.6 | reserved_MB: 25392.0 | inactive_split_MB: 298.4 +rank1 [2026-10-28 14:23:11,224] INFO : Step 4700/50000 rank1 | Train Loss: 2.5991 | Train Loss Unweighted: 1.5142 | Train Loss Components: Auxiliary: 0.4914 | Cardinality: 0.2679 | Classification: 0.2997 | Classification_binary: 0.3514 | HitGrouping: 0.2867 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4228 | Regression_eta: 0.0061 | Regression_pt: 0.4290 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1809 | Classification_binary: 0.3649 | Regression_cos_phi: 0.1986 | Regression_energy: 0.2123 | Regression_eta: 0.1547 | Regression_pt: 0.2227 | Regression_sin_phi: 0.2177 | LR: 9.81e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8892s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7362 | max_loss: 0.2547 | max_backward: 1.8331 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5918.9 | uss_MB: 4764.8 | child_rss_MB: 5675.1 | allocated_MB: 343.1 | reserved_MB: 26384.0 | inactive_split_MB: 292.4 +rank1 [2026-10-28 14:27:13,059] INFO : Step 4800/50000 rank1 | Train Loss: 2.7077 | Train Loss Unweighted: 1.5200 | Train Loss Components: Auxiliary: 0.5118 | Cardinality: 0.3360 | Classification: 0.2606 | Classification_binary: 0.3918 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4264 | Regression_eta: 0.0053 | Regression_pt: 0.4305 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1363 | Classification_binary: 0.3602 | Regression_cos_phi: 0.1940 | Regression_energy: 0.2078 | Regression_eta: 0.1483 | Regression_pt: 0.2190 | Regression_sin_phi: 0.2244 | LR: 9.80e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2498s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7154 | max_loss: 0.1552 | max_backward: 1.4398 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.8979 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0854 | Memory: rss_MB: 5917.3 | uss_MB: 4226.4 | child_rss_MB: 5678.8 | allocated_MB: 346.1 | reserved_MB: 26384.0 | inactive_split_MB: 284.0 +rank1 [2026-10-28 14:31:09,413] INFO : Step 4900/50000 rank1 | Train Loss: 2.3682 | Train Loss Unweighted: 1.4242 | Train Loss Components: Auxiliary: 0.4616 | Cardinality: 0.2176 | Classification: 0.2570 | Classification_binary: 0.3149 | HitGrouping: 0.2900 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4191 | Regression_eta: 0.0047 | Regression_pt: 0.4239 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1520 | Classification_binary: 0.3263 | Regression_cos_phi: 0.2003 | Regression_energy: 0.1982 | Regression_eta: 0.1393 | Regression_pt: 0.2141 | Regression_sin_phi: 0.2086 | LR: 9.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.4623s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0516 | max_forward: 0.6608 | max_loss: 0.1430 | max_backward: 1.2657 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5916.2 | uss_MB: 4765.4 | child_rss_MB: 5677.6 | allocated_MB: 343.8 | reserved_MB: 26384.0 | inactive_split_MB: 201.9 +rank1 [2026-10-28 14:35:19,045] INFO : Step 5000/50000 rank1 | Train Loss: 2.4060 | Train Loss Unweighted: 1.4130 | Train Loss Components: Auxiliary: 0.4756 | Cardinality: 0.2573 | Classification: 0.2748 | Classification_binary: 0.3198 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4020 | Regression_eta: 0.0043 | Regression_pt: 0.4074 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1721 | Classification_binary: 0.3249 | Regression_cos_phi: 0.1581 | Regression_energy: 0.1742 | Regression_eta: 0.1179 | Regression_pt: 0.1794 | Regression_sin_phi: 0.1464 | LR: 9.78e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.8090s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6153 | max_loss: 0.1441 | max_backward: 1.2119 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.9741 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5914.8 | uss_MB: 4767.1 | child_rss_MB: 5669.5 | allocated_MB: 342.5 | reserved_MB: 27388.0 | inactive_split_MB: 218.4 +rank1 [2026-10-28 14:35:19,962] INFO : Running validation on rank1 +rank1 [2026-10-28 14:35:19,962] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 14:35:19,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 14:36:10,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:36:11,318] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 14:36:36,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 14:36:37,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 14:36:38,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 14:36:39,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 14:36:49,538] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 14:36:50,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:36:50,780] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 14:37:07,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 14:37:08,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 14:37:09,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 14:37:09,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 14:37:19,993] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 14:37:20,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:37:20,582] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 14:37:36,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 14:37:36,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 14:37:37,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 14:37:38,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 14:37:48,109] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 14:42:02,660] INFO : Step 5100/50000 rank1 | Train Loss: 2.7059 | Train Loss Unweighted: 1.5631 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2507 | Classification: 0.3068 | Classification_binary: 0.4110 | HitGrouping: 0.2884 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4148 | Regression_eta: 0.0060 | Regression_pt: 0.4186 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1816 | Classification_binary: 0.4155 | Regression_cos_phi: 0.2627 | Regression_energy: 0.1915 | Regression_eta: 0.1444 | Regression_pt: 0.2030 | Regression_sin_phi: 0.2555 | LR: 9.77e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4432s | Phase Times: max_pre_step_barrier: 0.1071 | max_batch_to_device: 0.0070 | max_forward: 0.7192 | max_loss: 0.2509 | max_backward: 1.6066 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.8859 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7032.2 | uss_MB: 5882.9 | child_rss_MB: 5662.0 | allocated_MB: 343.2 | reserved_MB: 23542.0 | inactive_split_MB: 236.2 +rank1 [2026-10-28 14:45:56,261] INFO : Step 5200/50000 rank1 | Train Loss: 2.4117 | Train Loss Unweighted: 1.4393 | Train Loss Components: Auxiliary: 0.4667 | Cardinality: 0.2215 | Classification: 0.2426 | Classification_binary: 0.3524 | HitGrouping: 0.2783 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4149 | Regression_eta: 0.0050 | Regression_pt: 0.4196 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1507 | Classification_binary: 0.3664 | Regression_cos_phi: 0.1315 | Regression_energy: 0.2137 | Regression_eta: 0.1360 | Regression_pt: 0.2240 | Regression_sin_phi: 0.1455 | LR: 9.76e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.9023s | Phase Times: max_pre_step_barrier: 0.1151 | max_batch_to_device: 0.0079 | max_forward: 0.8909 | max_loss: 0.1450 | max_backward: 1.3654 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.5791 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 6020.7 | uss_MB: 4875.4 | child_rss_MB: 5666.4 | allocated_MB: 346.1 | reserved_MB: 23690.0 | inactive_split_MB: 197.9 +rank1 [2026-10-28 14:50:03,850] INFO : Step 5300/50000 rank1 | Train Loss: 2.2417 | Train Loss Unweighted: 1.3238 | Train Loss Components: Auxiliary: 0.4353 | Cardinality: 0.2226 | Classification: 0.2387 | Classification_binary: 0.3027 | HitGrouping: 0.2808 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3844 | Regression_eta: 0.0041 | Regression_pt: 0.3895 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1862 | Classification_binary: 0.3238 | Regression_cos_phi: 0.2133 | Regression_energy: 0.1964 | Regression_eta: 0.1501 | Regression_pt: 0.2044 | Regression_sin_phi: 0.2492 | LR: 9.75e-04 | DataLoad Time: 0.0237s | Model Forward Time: 4.1965s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0155 | max_forward: 1.3549 | max_loss: 0.4272 | max_backward: 1.7517 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 6024.3 | uss_MB: 4878.1 | child_rss_MB: 5664.8 | allocated_MB: 341.7 | reserved_MB: 25496.0 | inactive_split_MB: 195.4 +rank1 [2026-10-28 14:54:26,617] INFO : Step 5400/50000 rank1 | Train Loss: 2.5857 | Train Loss Unweighted: 1.4820 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2715 | Classification: 0.2898 | Classification_binary: 0.3613 | HitGrouping: 0.2829 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4076 | Regression_eta: 0.0053 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1834 | Classification_binary: 0.4162 | Regression_cos_phi: 0.2270 | Regression_energy: 0.1788 | Regression_eta: 0.1384 | Regression_pt: 0.1899 | Regression_sin_phi: 0.2285 | LR: 9.74e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.8622s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7472 | max_loss: 0.1664 | max_backward: 1.6216 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4956 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6028.8 | uss_MB: 4880.2 | child_rss_MB: 5668.3 | allocated_MB: 345.8 | reserved_MB: 25496.0 | inactive_split_MB: 226.8 +rank1 [2026-10-28 14:58:29,775] INFO : Step 5500/50000 rank1 | Train Loss: 2.4383 | Train Loss Unweighted: 1.4361 | Train Loss Components: Auxiliary: 0.4636 | Cardinality: 0.2531 | Classification: 0.2547 | Classification_binary: 0.3846 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3900 | Regression_eta: 0.0045 | Regression_pt: 0.3974 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1648 | Classification_binary: 0.3767 | Regression_cos_phi: 0.1803 | Regression_energy: 0.1854 | Regression_eta: 0.1361 | Regression_pt: 0.1967 | Regression_sin_phi: 0.1562 | LR: 9.73e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3516s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7547 | max_loss: 0.2915 | max_backward: 1.8143 | max_optimizer_step: 0.2819 | max_tensorboard_logging: 0.5958 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6029.1 | uss_MB: 4882.8 | child_rss_MB: 5667.9 | allocated_MB: 342.5 | reserved_MB: 26496.0 | inactive_split_MB: 471.5 +rank1 [2026-10-28 15:02:37,794] INFO : Step 5600/50000 rank1 | Train Loss: 2.2622 | Train Loss Unweighted: 1.3177 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.1990 | Classification: 0.2257 | Classification_binary: 0.3268 | HitGrouping: 0.2800 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3758 | Regression_eta: 0.0051 | Regression_pt: 0.3799 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1193 | Classification_binary: 0.3075 | Regression_cos_phi: 0.1905 | Regression_energy: 0.1747 | Regression_eta: 0.0976 | Regression_pt: 0.1818 | Regression_sin_phi: 0.1586 | LR: 9.72e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1696s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0347 | max_forward: 1.1910 | max_loss: 0.2555 | max_backward: 1.5431 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 6024.4 | uss_MB: 4885.9 | child_rss_MB: 5669.8 | allocated_MB: 340.8 | reserved_MB: 26496.0 | inactive_split_MB: 191.2 +rank1 [2026-10-28 15:06:24,089] INFO : Step 5700/50000 rank1 | Train Loss: 2.4588 | Train Loss Unweighted: 1.4026 | Train Loss Components: Auxiliary: 0.4666 | Cardinality: 0.2348 | Classification: 0.2768 | Classification_binary: 0.3365 | HitGrouping: 0.2916 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3864 | Regression_eta: 0.0055 | Regression_pt: 0.3923 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1422 | Classification_binary: 0.3155 | Regression_cos_phi: 0.1909 | Regression_energy: 0.1830 | Regression_eta: 0.1537 | Regression_pt: 0.1900 | Regression_sin_phi: 0.1621 | LR: 9.71e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.5061s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6547 | max_loss: 0.1874 | max_backward: 1.2748 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6027.1 | uss_MB: 4889.2 | child_rss_MB: 5674.0 | allocated_MB: 344.4 | reserved_MB: 26496.0 | inactive_split_MB: 224.5 +rank1 [2026-10-28 15:10:14,851] INFO : Step 5800/50000 rank1 | Train Loss: 2.4947 | Train Loss Unweighted: 1.4616 | Train Loss Components: Auxiliary: 0.4752 | Cardinality: 0.2715 | Classification: 0.2623 | Classification_binary: 0.3729 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4032 | Regression_eta: 0.0047 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1798 | Classification_binary: 0.3874 | Regression_cos_phi: 0.2378 | Regression_energy: 0.2065 | Regression_eta: 0.1490 | Regression_pt: 0.2176 | Regression_sin_phi: 0.2127 | LR: 9.70e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.7394s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.9768 | max_loss: 0.2512 | max_backward: 2.0573 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.6978 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6026.0 | uss_MB: 4048.8 | child_rss_MB: 5650.8 | allocated_MB: 343.4 | reserved_MB: 27494.0 | inactive_split_MB: 198.3 +rank1 [2026-10-28 15:14:12,312] INFO : Step 5900/50000 rank1 | Train Loss: 2.3800 | Train Loss Unweighted: 1.3838 | Train Loss Components: Auxiliary: 0.4549 | Cardinality: 0.2203 | Classification: 0.2438 | Classification_binary: 0.3435 | HitGrouping: 0.2757 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3900 | Regression_eta: 0.0050 | Regression_pt: 0.3965 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1174 | Classification_binary: 0.3086 | Regression_cos_phi: 0.2058 | Regression_energy: 0.1996 | Regression_eta: 0.1364 | Regression_pt: 0.2132 | Regression_sin_phi: 0.1905 | LR: 9.69e-04 | DataLoad Time: 0.0131s | Model Forward Time: 4.7292s | Phase Times: max_pre_step_barrier: 0.0158 | max_batch_to_device: 0.0069 | max_forward: 1.0355 | max_loss: 0.2311 | max_backward: 2.2749 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5131 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 6029.8 | uss_MB: 3868.6 | child_rss_MB: 5658.5 | allocated_MB: 342.1 | reserved_MB: 27494.0 | inactive_split_MB: 281.2 +rank1 [2026-10-28 15:18:13,895] INFO : Step 6000/50000 rank1 | Train Loss: 2.1372 | Train Loss Unweighted: 1.3015 | Train Loss Components: Auxiliary: 0.4179 | Cardinality: 0.1916 | Classification: 0.2091 | Classification_binary: 0.2848 | HitGrouping: 0.2746 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3966 | Regression_eta: 0.0041 | Regression_pt: 0.4029 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1349 | Classification_binary: 0.2540 | Regression_cos_phi: 0.1886 | Regression_energy: 0.1841 | Regression_eta: 0.1352 | Regression_pt: 0.1982 | Regression_sin_phi: 0.2248 | LR: 9.68e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6791s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6457 | max_loss: 0.2157 | max_backward: 1.4984 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.5590 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6033.8 | uss_MB: 4891.5 | child_rss_MB: 5658.2 | allocated_MB: 343.1 | reserved_MB: 27494.0 | inactive_split_MB: 260.7 +rank1 [2026-10-28 15:18:14,994] INFO : Running validation on rank1 +rank1 [2026-10-28 15:18:14,994] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 15:18:14,994] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 15:19:17,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:19:17,849] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 15:19:43,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 15:19:44,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 15:19:45,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 15:19:46,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 15:19:56,624] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 15:19:57,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:19:58,136] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 15:20:14,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 15:20:14,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 15:20:15,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 15:20:16,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 15:20:26,820] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 15:20:27,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:20:28,075] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 15:20:43,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 15:20:44,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 15:20:45,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 15:20:45,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 15:20:55,992] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 15:25:29,841] INFO : Step 6100/50000 rank1 | Train Loss: 2.4040 | Train Loss Unweighted: 1.3907 | Train Loss Components: Auxiliary: 0.4661 | Cardinality: 0.2610 | Classification: 0.2427 | Classification_binary: 0.3401 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3968 | Regression_eta: 0.0047 | Regression_pt: 0.4016 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1438 | Classification_binary: 0.3295 | Regression_cos_phi: 0.1968 | Regression_energy: 0.1971 | Regression_eta: 0.1820 | Regression_pt: 0.2119 | Regression_sin_phi: 0.1917 | LR: 9.67e-04 | DataLoad Time: 0.0209s | Model Forward Time: 6.3861s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0073 | max_forward: 1.0567 | max_loss: 0.2452 | max_backward: 2.4106 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.9836 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 7214.6 | uss_MB: 5292.3 | child_rss_MB: 5654.8 | allocated_MB: 344.4 | reserved_MB: 23648.0 | inactive_split_MB: 194.8 +rank1 [2026-10-28 15:29:38,612] INFO : Step 6200/50000 rank1 | Train Loss: 2.4942 | Train Loss Unweighted: 1.4287 | Train Loss Components: Auxiliary: 0.4744 | Cardinality: 0.2537 | Classification: 0.2580 | Classification_binary: 0.3817 | HitGrouping: 0.2748 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3867 | Regression_eta: 0.0050 | Regression_pt: 0.3920 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1227 | Classification_binary: 0.3441 | Regression_cos_phi: 0.2113 | Regression_energy: 0.1772 | Regression_eta: 0.1368 | Regression_pt: 0.1875 | Regression_sin_phi: 0.2074 | LR: 9.66e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0639s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7838 | max_loss: 0.1532 | max_backward: 1.6314 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6093.8 | uss_MB: 4537.3 | child_rss_MB: 5656.7 | allocated_MB: 343.8 | reserved_MB: 23830.0 | inactive_split_MB: 215.7 +rank1 [2026-10-28 15:34:14,250] INFO : Step 6300/50000 rank1 | Train Loss: 2.2204 | Train Loss Unweighted: 1.2763 | Train Loss Components: Auxiliary: 0.4285 | Cardinality: 0.2506 | Classification: 0.2258 | Classification_binary: 0.2983 | HitGrouping: 0.2710 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3704 | Regression_eta: 0.0045 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1537 | Classification_binary: 0.3292 | Regression_cos_phi: 0.2451 | Regression_energy: 0.1719 | Regression_eta: 0.1145 | Regression_pt: 0.1766 | Regression_sin_phi: 0.2113 | LR: 9.65e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.6730s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.5854 | max_loss: 0.1923 | max_backward: 1.0848 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.8586 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6096.2 | uss_MB: 4957.2 | child_rss_MB: 5651.4 | allocated_MB: 339.7 | reserved_MB: 24826.0 | inactive_split_MB: 251.6 +rank1 [2026-10-28 15:38:39,608] INFO : Step 6400/50000 rank1 | Train Loss: 2.2882 | Train Loss Unweighted: 1.3014 | Train Loss Components: Auxiliary: 0.4348 | Cardinality: 0.2271 | Classification: 0.2431 | Classification_binary: 0.3058 | HitGrouping: 0.2702 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3679 | Regression_eta: 0.0046 | Regression_pt: 0.3751 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1626 | Classification_binary: 0.3305 | Regression_cos_phi: 0.2062 | Regression_energy: 0.1806 | Regression_eta: 0.1495 | Regression_pt: 0.1940 | Regression_sin_phi: 0.3963 | LR: 9.64e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.8574s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7489 | max_loss: 0.3608 | max_backward: 2.5327 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6095.6 | uss_MB: 4594.4 | child_rss_MB: 5649.8 | allocated_MB: 345.8 | reserved_MB: 24826.0 | inactive_split_MB: 253.0 +rank1 [2026-10-28 15:42:45,601] INFO : Step 6500/50000 rank1 | Train Loss: 2.4626 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.3022 | Classification: 0.2393 | Classification_binary: 0.3805 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3783 | Regression_eta: 0.0049 | Regression_pt: 0.3831 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1399 | Classification_binary: 0.3720 | Regression_cos_phi: 0.1908 | Regression_energy: 0.1745 | Regression_eta: 0.1305 | Regression_pt: 0.1831 | Regression_sin_phi: 0.1923 | LR: 9.63e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1896s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7600 | max_loss: 0.1555 | max_backward: 1.3910 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6094.6 | uss_MB: 4960.0 | child_rss_MB: 5656.1 | allocated_MB: 344.4 | reserved_MB: 24826.0 | inactive_split_MB: 269.0 +rank1 [2026-10-28 15:46:47,732] INFO : Step 6600/50000 rank1 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3617 | Train Loss Components: Auxiliary: 0.4493 | Cardinality: 0.2405 | Classification: 0.2289 | Classification_binary: 0.3388 | HitGrouping: 0.2650 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3914 | Regression_eta: 0.0045 | Regression_pt: 0.3935 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1185 | Classification_binary: 0.3385 | Regression_cos_phi: 0.2068 | Regression_energy: 0.1940 | Regression_eta: 0.1403 | Regression_pt: 0.2050 | Regression_sin_phi: 0.2173 | LR: 9.62e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7772s | Phase Times: max_pre_step_barrier: 0.1106 | max_batch_to_device: 0.0061 | max_forward: 0.7587 | max_loss: 0.2869 | max_backward: 2.2641 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1321 | Memory: rss_MB: 6092.2 | uss_MB: 4956.2 | child_rss_MB: 5651.6 | allocated_MB: 343.8 | reserved_MB: 25846.0 | inactive_split_MB: 267.9 +rank1 [2026-10-28 15:51:24,435] INFO : Step 6700/50000 rank1 | Train Loss: 2.1225 | Train Loss Unweighted: 1.2385 | Train Loss Components: Auxiliary: 0.4092 | Cardinality: 0.2100 | Classification: 0.2079 | Classification_binary: 0.2916 | HitGrouping: 0.2751 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3623 | Regression_eta: 0.0041 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1097 | Classification_binary: 0.2793 | Regression_cos_phi: 0.1877 | Regression_energy: 0.1554 | Regression_eta: 0.1184 | Regression_pt: 0.1610 | Regression_sin_phi: 0.1595 | LR: 9.61e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9919s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.6452 | max_loss: 0.1453 | max_backward: 1.2984 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.0269 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6098.1 | uss_MB: 4965.1 | child_rss_MB: 5657.8 | allocated_MB: 345.0 | reserved_MB: 25846.0 | inactive_split_MB: 258.8 +rank1 [2026-10-28 15:55:32,974] INFO : Step 6800/50000 rank1 | Train Loss: 2.4601 | Train Loss Unweighted: 1.3772 | Train Loss Components: Auxiliary: 0.4589 | Cardinality: 0.2427 | Classification: 0.2631 | Classification_binary: 0.3490 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3747 | Regression_eta: 0.0054 | Regression_pt: 0.3793 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1413 | Classification_binary: 0.3743 | Regression_cos_phi: 0.2216 | Regression_energy: 0.1863 | Regression_eta: 0.1684 | Regression_pt: 0.1948 | Regression_sin_phi: 0.2352 | LR: 9.60e-04 | DataLoad Time: 0.0237s | Model Forward Time: 5.6160s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0662 | max_forward: 1.6558 | max_loss: 0.3855 | max_backward: 2.1309 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 1.0709 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6092.6 | uss_MB: 4961.3 | child_rss_MB: 5647.9 | allocated_MB: 345.8 | reserved_MB: 26622.0 | inactive_split_MB: 194.2 +rank1 [2026-10-28 15:59:35,211] INFO : Step 6900/50000 rank1 | Train Loss: 2.3793 | Train Loss Unweighted: 1.3337 | Train Loss Components: Auxiliary: 0.4495 | Cardinality: 0.2757 | Classification: 0.2300 | Classification_binary: 0.3711 | HitGrouping: 0.2647 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3605 | Regression_eta: 0.0045 | Regression_pt: 0.3628 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1111 | Classification_binary: 0.3315 | Regression_cos_phi: 0.1304 | Regression_energy: 0.1884 | Regression_eta: 0.0836 | Regression_pt: 0.1953 | Regression_sin_phi: 0.1470 | LR: 9.58e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2972s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1120 | max_forward: 0.8190 | max_loss: 0.1496 | max_backward: 1.4356 | max_optimizer_step: 0.2847 | max_tensorboard_logging: 0.8440 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6096.9 | uss_MB: 4962.9 | child_rss_MB: 5656.8 | allocated_MB: 345.4 | reserved_MB: 26622.0 | inactive_split_MB: 300.5 +rank1 [2026-10-28 16:03:56,953] INFO : Step 7000/50000 rank1 | Train Loss: 2.2410 | Train Loss Unweighted: 1.2707 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2746 | Classification: 0.1970 | Classification_binary: 0.3002 | HitGrouping: 0.2655 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3795 | Regression_eta: 0.0045 | Regression_pt: 0.3852 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1310 | Classification_binary: 0.3161 | Regression_cos_phi: 0.1743 | Regression_energy: 0.1732 | Regression_eta: 0.1612 | Regression_pt: 0.1825 | Regression_sin_phi: 0.1927 | LR: 9.57e-04 | DataLoad Time: 0.0129s | Model Forward Time: 5.3952s | Phase Times: max_pre_step_barrier: 0.0229 | max_batch_to_device: 0.0154 | max_forward: 1.0393 | max_loss: 0.2251 | max_backward: 2.0451 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.9425 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6097.4 | uss_MB: 4965.6 | child_rss_MB: 5656.9 | allocated_MB: 345.5 | reserved_MB: 26622.0 | inactive_split_MB: 296.5 +rank1 [2026-10-28 16:03:58,343] INFO : Running validation on rank1 +rank1 [2026-10-28 16:03:58,344] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 16:03:58,344] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 16:04:51,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:04:52,382] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 16:05:18,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 16:05:19,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 16:05:20,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 16:05:21,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 16:05:31,718] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 16:05:32,028] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:05:32,553] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 16:05:48,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 16:05:49,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 16:05:50,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 16:05:51,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 16:06:01,121] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 16:06:02,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:06:02,710] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 16:06:18,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 16:06:19,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 16:06:19,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 16:06:20,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 16:06:30,568] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 16:10:49,043] INFO : Step 7100/50000 rank1 | Train Loss: 2.1086 | Train Loss Unweighted: 1.2525 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.1821 | Classification: 0.2186 | Classification_binary: 0.2903 | HitGrouping: 0.2613 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3655 | Regression_eta: 0.0041 | Regression_pt: 0.3696 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1303 | Classification_binary: 0.2966 | Regression_cos_phi: 0.1664 | Regression_energy: 0.1950 | Regression_eta: 0.1187 | Regression_pt: 0.2055 | Regression_sin_phi: 0.1817 | LR: 9.56e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.9392s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6376 | max_loss: 0.1424 | max_backward: 1.3436 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8454 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 7191.1 | uss_MB: 6057.4 | child_rss_MB: 5686.3 | allocated_MB: 345.6 | reserved_MB: 22618.0 | inactive_split_MB: 206.0 +rank1 [2026-10-28 16:15:05,704] INFO : Step 7200/50000 rank1 | Train Loss: 2.4589 | Train Loss Unweighted: 1.3873 | Train Loss Components: Auxiliary: 0.4658 | Cardinality: 0.2721 | Classification: 0.2648 | Classification_binary: 0.3806 | HitGrouping: 0.2633 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3642 | Regression_eta: 0.0051 | Regression_pt: 0.3675 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1883 | Classification_binary: 0.4444 | Regression_cos_phi: 0.2859 | Regression_energy: 0.1805 | Regression_eta: 0.1425 | Regression_pt: 0.1890 | Regression_sin_phi: 0.2168 | LR: 9.55e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3057s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 1.0191 | max_loss: 0.1674 | max_backward: 1.7813 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6183.3 | uss_MB: 5050.5 | child_rss_MB: 5662.0 | allocated_MB: 343.5 | reserved_MB: 22618.0 | inactive_split_MB: 315.3 +rank1 [2026-10-28 16:19:17,626] INFO : Step 7300/50000 rank1 | Train Loss: 2.1617 | Train Loss Unweighted: 1.2404 | Train Loss Components: Auxiliary: 0.4141 | Cardinality: 0.2258 | Classification: 0.1979 | Classification_binary: 0.3107 | HitGrouping: 0.2671 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3613 | Regression_eta: 0.0041 | Regression_pt: 0.3621 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1120 | Classification_binary: 0.3175 | Regression_cos_phi: 0.1580 | Regression_energy: 0.1835 | Regression_eta: 0.1148 | Regression_pt: 0.1888 | Regression_sin_phi: 0.1728 | LR: 9.53e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.7535s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0084 | max_forward: 0.7227 | max_loss: 0.1491 | max_backward: 1.4232 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.5902 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6180.9 | uss_MB: 5051.2 | child_rss_MB: 5696.9 | allocated_MB: 345.0 | reserved_MB: 23608.0 | inactive_split_MB: 295.0 +rank1 [2026-10-28 16:23:25,991] INFO : Step 7400/50000 rank1 | Train Loss: 2.0741 | Train Loss Unweighted: 1.2014 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2287 | Classification: 0.2035 | Classification_binary: 0.2781 | HitGrouping: 0.2646 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3549 | Regression_eta: 0.0040 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1274 | Classification_binary: 0.2821 | Regression_cos_phi: 0.2154 | Regression_energy: 0.1553 | Regression_eta: 0.1232 | Regression_pt: 0.1599 | Regression_sin_phi: 0.2068 | LR: 9.52e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6558s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6129 | max_loss: 0.2127 | max_backward: 1.2944 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6450 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6180.9 | uss_MB: 5049.5 | child_rss_MB: 5695.6 | allocated_MB: 343.8 | reserved_MB: 23608.0 | inactive_split_MB: 229.5 +rank1 [2026-10-28 16:27:18,559] INFO : Step 7500/50000 rank1 | Train Loss: 2.4640 | Train Loss Unweighted: 1.3686 | Train Loss Components: Auxiliary: 0.4637 | Cardinality: 0.2640 | Classification: 0.2570 | Classification_binary: 0.3594 | HitGrouping: 0.2687 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3684 | Regression_eta: 0.0054 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1483 | Classification_binary: 0.3805 | Regression_cos_phi: 0.2087 | Regression_energy: 0.1666 | Regression_eta: 0.1477 | Regression_pt: 0.1744 | Regression_sin_phi: 0.2559 | LR: 9.51e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1736s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0887 | max_forward: 0.7717 | max_loss: 0.1508 | max_backward: 1.3590 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.9634 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6179.4 | uss_MB: 5053.3 | child_rss_MB: 5684.8 | allocated_MB: 348.6 | reserved_MB: 23608.0 | inactive_split_MB: 283.7 +rank1 [2026-10-28 16:31:22,305] INFO : Step 7600/50000 rank1 | Train Loss: 2.3745 | Train Loss Unweighted: 1.3447 | Train Loss Components: Auxiliary: 0.4509 | Cardinality: 0.2616 | Classification: 0.2519 | Classification_binary: 0.3743 | HitGrouping: 0.2627 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3532 | Regression_eta: 0.0050 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1721 | Classification_binary: 0.3633 | Regression_cos_phi: 0.1699 | Regression_energy: 0.1702 | Regression_eta: 0.1274 | Regression_pt: 0.1778 | Regression_sin_phi: 0.1805 | LR: 9.50e-04 | DataLoad Time: 0.0162s | Model Forward Time: 16.0036s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.7345 | max_loss: 0.1617 | max_backward: 1.4897 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 12.6867 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6179.5 | uss_MB: 5051.2 | child_rss_MB: 5695.8 | allocated_MB: 345.3 | reserved_MB: 23608.0 | inactive_split_MB: 284.8 +rank1 [2026-10-28 16:35:23,036] INFO : Step 7700/50000 rank1 | Train Loss: 2.0023 | Train Loss Unweighted: 1.1475 | Train Loss Components: Auxiliary: 0.3866 | Cardinality: 0.2005 | Classification: 0.1968 | Classification_binary: 0.2784 | HitGrouping: 0.2562 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3315 | Regression_eta: 0.0040 | Regression_pt: 0.3329 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1276 | Classification_binary: 0.2342 | Regression_cos_phi: 0.1267 | Regression_energy: 0.1518 | Regression_eta: 0.0868 | Regression_pt: 0.1542 | Regression_sin_phi: 0.1394 | LR: 9.48e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.0206s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7891 | max_loss: 0.1927 | max_backward: 1.4291 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6182.9 | uss_MB: 5055.0 | child_rss_MB: 5702.0 | allocated_MB: 343.9 | reserved_MB: 23608.0 | inactive_split_MB: 279.7 +rank1 [2026-10-28 16:39:23,043] INFO : Step 7800/50000 rank1 | Train Loss: 2.0473 | Train Loss Unweighted: 1.2037 | Train Loss Components: Auxiliary: 0.4013 | Cardinality: 0.1891 | Classification: 0.2102 | Classification_binary: 0.2760 | HitGrouping: 0.2604 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3526 | Regression_eta: 0.0038 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1420 | Classification_binary: 0.2966 | Regression_cos_phi: 0.1631 | Regression_energy: 0.1820 | Regression_eta: 0.1235 | Regression_pt: 0.1920 | Regression_sin_phi: 0.1520 | LR: 9.47e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4342s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.7326 | max_loss: 0.2464 | max_backward: 1.8124 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9395 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6182.4 | uss_MB: 5059.4 | child_rss_MB: 5675.6 | allocated_MB: 343.5 | reserved_MB: 24612.0 | inactive_split_MB: 224.8 +rank1 [2026-10-28 16:43:21,637] INFO : Step 7900/50000 rank1 | Train Loss: 2.3872 | Train Loss Unweighted: 1.3279 | Train Loss Components: Auxiliary: 0.4525 | Cardinality: 0.2722 | Classification: 0.2401 | Classification_binary: 0.3580 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3573 | Regression_eta: 0.0051 | Regression_pt: 0.3627 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1722 | Classification_binary: 0.4111 | Regression_cos_phi: 0.2619 | Regression_energy: 0.1814 | Regression_eta: 0.1706 | Regression_pt: 0.1913 | Regression_sin_phi: 0.1942 | LR: 9.46e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.7842s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7032 | max_loss: 0.2052 | max_backward: 1.5183 | max_optimizer_step: 0.1922 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6185.2 | uss_MB: 5060.7 | child_rss_MB: 5695.7 | allocated_MB: 347.1 | reserved_MB: 24612.0 | inactive_split_MB: 279.0 +rank1 [2026-10-28 16:47:21,752] INFO : Step 8000/50000 rank1 | Train Loss: 2.1303 | Train Loss Unweighted: 1.2019 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2451 | Classification: 0.2031 | Classification_binary: 0.3087 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3382 | Regression_eta: 0.0041 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1282 | Classification_binary: 0.3104 | Regression_cos_phi: 0.1883 | Regression_energy: 0.1441 | Regression_eta: 0.1321 | Regression_pt: 0.1500 | Regression_sin_phi: 0.1691 | LR: 9.44e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.7980s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0554 | max_forward: 0.7608 | max_loss: 0.1617 | max_backward: 1.3810 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 6186.5 | uss_MB: 4730.8 | child_rss_MB: 5692.5 | allocated_MB: 344.2 | reserved_MB: 24612.0 | inactive_split_MB: 224.1 +rank1 [2026-10-28 16:47:22,744] INFO : Running validation on rank1 +rank1 [2026-10-28 16:47:22,744] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 16:47:22,744] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 16:48:13,809] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:48:14,392] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 16:48:40,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 16:48:41,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 16:48:42,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 16:48:43,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 16:48:53,593] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 16:48:53,945] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:48:54,469] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 16:49:10,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 16:49:11,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 16:49:12,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 16:49:13,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 16:49:23,913] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 16:49:24,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:49:24,963] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 16:49:40,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 16:49:41,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 16:49:41,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 16:49:42,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 16:49:52,632] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 16:54:09,126] INFO : Step 8100/50000 rank1 | Train Loss: 2.0320 | Train Loss Unweighted: 1.1744 | Train Loss Components: Auxiliary: 0.3893 | Cardinality: 0.1986 | Classification: 0.1938 | Classification_binary: 0.2691 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3510 | Regression_eta: 0.0039 | Regression_pt: 0.3525 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.2617 | Regression_cos_phi: 0.1096 | Regression_energy: 0.1855 | Regression_eta: 0.0633 | Regression_pt: 0.1930 | Regression_sin_phi: 0.1285 | LR: 9.43e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0441s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6251 | max_loss: 0.2465 | max_backward: 1.3564 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8507 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7244.9 | uss_MB: 6118.4 | child_rss_MB: 5660.8 | allocated_MB: 342.9 | reserved_MB: 23510.0 | inactive_split_MB: 258.1 +rank1 [2026-10-28 16:58:00,709] INFO : Step 8200/50000 rank1 | Train Loss: 2.1917 | Train Loss Unweighted: 1.2329 | Train Loss Components: Auxiliary: 0.4200 | Cardinality: 0.2405 | Classification: 0.2359 | Classification_binary: 0.3328 | HitGrouping: 0.2582 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3251 | Regression_eta: 0.0043 | Regression_pt: 0.3305 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1497 | Classification_binary: 0.3834 | Regression_cos_phi: 0.2144 | Regression_energy: 0.1695 | Regression_eta: 0.1385 | Regression_pt: 0.1800 | Regression_sin_phi: 0.1583 | LR: 9.42e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2792s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0064 | max_forward: 0.7679 | max_loss: 0.2484 | max_backward: 1.6620 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.8049 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1289 | Memory: rss_MB: 6232.6 | uss_MB: 5106.1 | child_rss_MB: 5673.2 | allocated_MB: 343.2 | reserved_MB: 23510.0 | inactive_split_MB: 292.4 +rank1 [2026-10-28 17:02:03,650] INFO : Step 8300/50000 rank1 | Train Loss: 2.1924 | Train Loss Unweighted: 1.2399 | Train Loss Components: Auxiliary: 0.4122 | Cardinality: 0.2039 | Classification: 0.2083 | Classification_binary: 0.3661 | HitGrouping: 0.2599 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3243 | Regression_eta: 0.0043 | Regression_pt: 0.3323 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1039 | Classification_binary: 0.3263 | Regression_cos_phi: 0.1349 | Regression_energy: 0.1451 | Regression_eta: 0.0980 | Regression_pt: 0.1556 | Regression_sin_phi: 0.1474 | LR: 9.40e-04 | DataLoad Time: 0.0236s | Model Forward Time: 4.8393s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0061 | max_forward: 0.9407 | max_loss: 0.2296 | max_backward: 1.9622 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6228.1 | uss_MB: 5055.3 | child_rss_MB: 5669.2 | allocated_MB: 344.6 | reserved_MB: 24580.0 | inactive_split_MB: 267.5 +rank1 [2026-10-28 17:06:07,748] INFO : Step 8400/50000 rank1 | Train Loss: 1.9542 | Train Loss Unweighted: 1.1189 | Train Loss Components: Auxiliary: 0.3756 | Cardinality: 0.1979 | Classification: 0.1926 | Classification_binary: 0.2783 | HitGrouping: 0.2585 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3198 | Regression_eta: 0.0035 | Regression_pt: 0.3209 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1423 | Classification_binary: 0.2626 | Regression_cos_phi: 0.1988 | Regression_energy: 0.1599 | Regression_eta: 0.1589 | Regression_pt: 0.1617 | Regression_sin_phi: 0.2649 | LR: 9.39e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2373s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6939 | max_loss: 0.1474 | max_backward: 1.5357 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 0.9892 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 6216.9 | uss_MB: 5103.3 | child_rss_MB: 5675.6 | allocated_MB: 341.1 | reserved_MB: 24580.0 | inactive_split_MB: 178.9 +rank1 [2026-10-28 17:10:17,598] INFO : Step 8500/50000 rank1 | Train Loss: 2.0642 | Train Loss Unweighted: 1.1724 | Train Loss Components: Auxiliary: 0.3964 | Cardinality: 0.2023 | Classification: 0.2152 | Classification_binary: 0.2897 | HitGrouping: 0.2607 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3289 | Regression_eta: 0.0041 | Regression_pt: 0.3302 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1374 | Classification_binary: 0.3190 | Regression_cos_phi: 0.2119 | Regression_energy: 0.1392 | Regression_eta: 0.1297 | Regression_pt: 0.1463 | Regression_sin_phi: 0.2089 | LR: 9.37e-04 | DataLoad Time: 0.0149s | Model Forward Time: 3.7406s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0069 | max_forward: 0.6424 | max_loss: 0.1972 | max_backward: 1.2711 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.7327 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6218.9 | uss_MB: 5108.3 | child_rss_MB: 5667.6 | allocated_MB: 344.4 | reserved_MB: 25586.0 | inactive_split_MB: 317.0 +rank1 [2026-10-28 17:14:24,635] INFO : Step 8600/50000 rank1 | Train Loss: 2.2820 | Train Loss Unweighted: 1.2960 | Train Loss Components: Auxiliary: 0.4378 | Cardinality: 0.2546 | Classification: 0.2294 | Classification_binary: 0.3478 | HitGrouping: 0.2584 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3521 | Regression_eta: 0.0046 | Regression_pt: 0.3576 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1291 | Classification_binary: 0.3350 | Regression_cos_phi: 0.2014 | Regression_energy: 0.1618 | Regression_eta: 0.1068 | Regression_pt: 0.1706 | Regression_sin_phi: 0.1864 | LR: 9.36e-04 | DataLoad Time: 0.0231s | Model Forward Time: 3.9587s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0069 | max_forward: 0.7396 | max_loss: 0.1527 | max_backward: 1.6088 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5007 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6218.9 | uss_MB: 5112.5 | child_rss_MB: 5664.7 | allocated_MB: 344.8 | reserved_MB: 25586.0 | inactive_split_MB: 283.2 +rank1 [2026-10-28 17:18:21,031] INFO : Step 8700/50000 rank1 | Train Loss: 2.0581 | Train Loss Unweighted: 1.1705 | Train Loss Components: Auxiliary: 0.4038 | Cardinality: 0.2463 | Classification: 0.2066 | Classification_binary: 0.3017 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3267 | Regression_eta: 0.0035 | Regression_pt: 0.3282 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1219 | Classification_binary: 0.2854 | Regression_cos_phi: 0.1518 | Regression_energy: 0.1613 | Regression_eta: 0.0842 | Regression_pt: 0.1703 | Regression_sin_phi: 0.1343 | LR: 9.34e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.7160s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6873 | max_loss: 0.2444 | max_backward: 1.2985 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6214.6 | uss_MB: 5113.2 | child_rss_MB: 5650.1 | allocated_MB: 343.7 | reserved_MB: 25586.0 | inactive_split_MB: 260.4 +rank1 [2026-10-28 17:22:21,714] INFO : Step 8800/50000 rank1 | Train Loss: 1.8415 | Train Loss Unweighted: 1.0591 | Train Loss Components: Auxiliary: 0.3597 | Cardinality: 0.1811 | Classification: 0.1734 | Classification_binary: 0.2490 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3144 | Regression_eta: 0.0034 | Regression_pt: 0.3153 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0966 | Classification_binary: 0.2382 | Regression_cos_phi: 0.1198 | Regression_energy: 0.1545 | Regression_eta: 0.1350 | Regression_pt: 0.1630 | Regression_sin_phi: 0.0921 | LR: 9.33e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.5733s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6737 | max_loss: 0.2385 | max_backward: 1.3073 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.5819 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6215.6 | uss_MB: 5114.6 | child_rss_MB: 5649.0 | allocated_MB: 344.1 | reserved_MB: 25586.0 | inactive_split_MB: 205.9 +rank1 [2026-10-28 17:26:43,765] INFO : Step 8900/50000 rank1 | Train Loss: 2.1706 | Train Loss Unweighted: 1.1768 | Train Loss Components: Auxiliary: 0.4167 | Cardinality: 0.3000 | Classification: 0.2064 | Classification_binary: 0.3037 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3284 | Regression_eta: 0.0039 | Regression_pt: 0.3304 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1238 | Classification_binary: 0.3000 | Regression_cos_phi: 0.1868 | Regression_energy: 0.1816 | Regression_eta: 0.1019 | Regression_pt: 0.1885 | Regression_sin_phi: 0.1396 | LR: 9.31e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.8173s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0073 | max_forward: 0.9973 | max_loss: 0.4040 | max_backward: 3.4126 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6194.6 | uss_MB: 5119.9 | child_rss_MB: 5654.4 | allocated_MB: 347.1 | reserved_MB: 25586.0 | inactive_split_MB: 275.1 +rank1 [2026-10-28 17:30:46,804] INFO : Step 9000/50000 rank1 | Train Loss: 2.1357 | Train Loss Unweighted: 1.1883 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.2351 | Classification: 0.2003 | Classification_binary: 0.3467 | HitGrouping: 0.2532 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3143 | Regression_eta: 0.0041 | Regression_pt: 0.3186 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1258 | Classification_binary: 0.3271 | Regression_cos_phi: 0.1894 | Regression_energy: 0.1428 | Regression_eta: 0.1335 | Regression_pt: 0.1504 | Regression_sin_phi: 0.1720 | LR: 9.30e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.7214s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.7873 | max_loss: 0.2178 | max_backward: 1.5407 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.3125 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 6192.6 | uss_MB: 5121.2 | child_rss_MB: 5660.4 | allocated_MB: 344.6 | reserved_MB: 25586.0 | inactive_split_MB: 289.9 +rank1 [2026-10-28 17:30:47,949] INFO : Running validation on rank1 +rank1 [2026-10-28 17:30:47,949] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 17:30:47,949] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 17:31:40,963] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:31:41,476] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 17:32:08,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 17:32:10,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 17:32:10,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 17:32:11,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 17:32:21,948] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-28 17:32:22,656] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:32:23,201] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 17:32:39,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 17:32:40,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 17:32:41,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 17:32:42,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 17:32:52,115] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 17:32:52,551] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:32:53,102] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 17:33:08,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 17:33:09,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 17:33:10,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 17:33:10,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 17:33:20,724] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 17:37:41,206] INFO : Step 9100/50000 rank1 | Train Loss: 1.8448 | Train Loss Unweighted: 1.0695 | Train Loss Components: Auxiliary: 0.3538 | Cardinality: 0.1722 | Classification: 0.1878 | Classification_binary: 0.2661 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3012 | Regression_eta: 0.0033 | Regression_pt: 0.3075 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1119 | Classification_binary: 0.2514 | Regression_cos_phi: 0.1693 | Regression_energy: 0.1410 | Regression_eta: 0.0927 | Regression_pt: 0.1440 | Regression_sin_phi: 0.1680 | LR: 9.28e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4457s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0153 | max_forward: 1.2776 | max_loss: 0.3881 | max_backward: 1.3802 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9203 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7188.5 | uss_MB: 6128.3 | child_rss_MB: 5636.7 | allocated_MB: 345.3 | reserved_MB: 22582.0 | inactive_split_MB: 275.2 +rank1 [2026-10-28 17:41:50,228] INFO : Step 9200/50000 rank1 | Train Loss: 1.9883 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3829 | Cardinality: 0.2086 | Classification: 0.1812 | Classification_binary: 0.2802 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3207 | Regression_eta: 0.0035 | Regression_pt: 0.3257 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1164 | Classification_binary: 0.2869 | Regression_cos_phi: 0.1802 | Regression_energy: 0.1499 | Regression_eta: 0.0810 | Regression_pt: 0.1562 | Regression_sin_phi: 0.1404 | LR: 9.27e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5155s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0073 | max_forward: 0.6839 | max_loss: 0.2414 | max_backward: 1.6336 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 1.0636 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6214.5 | uss_MB: 5161.4 | child_rss_MB: 5636.3 | allocated_MB: 345.6 | reserved_MB: 23224.0 | inactive_split_MB: 190.1 +rank1 [2026-10-28 17:45:52,428] INFO : Step 9300/50000 rank1 | Train Loss: 2.0749 | Train Loss Unweighted: 1.1591 | Train Loss Components: Auxiliary: 0.4049 | Cardinality: 0.2573 | Classification: 0.2065 | Classification_binary: 0.3180 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3111 | Regression_eta: 0.0036 | Regression_pt: 0.3160 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1117 | Classification_binary: 0.3248 | Regression_cos_phi: 0.1334 | Regression_energy: 0.1533 | Regression_eta: 0.0793 | Regression_pt: 0.1603 | Regression_sin_phi: 0.1270 | LR: 9.25e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2173s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8329 | max_loss: 0.1499 | max_backward: 1.4059 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.8766 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6214.5 | uss_MB: 5164.3 | child_rss_MB: 5654.0 | allocated_MB: 344.8 | reserved_MB: 24586.0 | inactive_split_MB: 287.2 +rank1 [2026-10-28 17:50:16,942] INFO : Step 9400/50000 rank1 | Train Loss: 1.8467 | Train Loss Unweighted: 1.0417 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1520 | Classification: 0.1837 | Classification_binary: 0.2977 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2742 | Regression_eta: 0.0040 | Regression_pt: 0.2784 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0994 | Classification_binary: 0.2861 | Regression_cos_phi: 0.1510 | Regression_energy: 0.1383 | Regression_eta: 0.0971 | Regression_pt: 0.1444 | Regression_sin_phi: 0.1761 | LR: 9.24e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.6045s | Phase Times: max_pre_step_barrier: 0.0180 | max_batch_to_device: 0.0062 | max_forward: 1.5076 | max_loss: 0.3748 | max_backward: 2.6247 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.5247 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6206.4 | uss_MB: 4649.5 | child_rss_MB: 5650.0 | allocated_MB: 343.9 | reserved_MB: 24586.0 | inactive_split_MB: 267.3 +rank1 [2026-10-28 17:54:18,159] INFO : Step 9500/50000 rank1 | Train Loss: 1.8199 | Train Loss Unweighted: 0.9995 | Train Loss Components: Auxiliary: 0.3464 | Cardinality: 0.1875 | Classification: 0.1651 | Classification_binary: 0.2558 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2851 | Regression_eta: 0.0035 | Regression_pt: 0.2864 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1111 | Classification_binary: 0.2840 | Regression_cos_phi: 0.1583 | Regression_energy: 0.1435 | Regression_eta: 0.1032 | Regression_pt: 0.1478 | Regression_sin_phi: 0.1550 | LR: 9.22e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.8036s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6532 | max_loss: 0.2385 | max_backward: 1.0043 | max_optimizer_step: 0.3001 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6205.2 | uss_MB: 5165.5 | child_rss_MB: 5654.5 | allocated_MB: 341.2 | reserved_MB: 24586.0 | inactive_split_MB: 180.8 +rank1 [2026-10-28 17:58:23,634] INFO : Step 9600/50000 rank1 | Train Loss: 2.0900 | Train Loss Unweighted: 1.1491 | Train Loss Components: Auxiliary: 0.4030 | Cardinality: 0.2464 | Classification: 0.2089 | Classification_binary: 0.3189 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3034 | Regression_eta: 0.0042 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1141 | Classification_binary: 0.3258 | Regression_cos_phi: 0.1635 | Regression_energy: 0.1518 | Regression_eta: 0.1148 | Regression_pt: 0.1626 | Regression_sin_phi: 0.1536 | LR: 9.21e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8839s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1083 | max_forward: 0.7381 | max_loss: 0.1457 | max_backward: 1.4890 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4970 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6206.2 | uss_MB: 5167.2 | child_rss_MB: 5637.2 | allocated_MB: 345.2 | reserved_MB: 25594.0 | inactive_split_MB: 190.6 +rank1 [2026-10-28 18:02:35,091] INFO : Step 9700/50000 rank1 | Train Loss: 2.1397 | Train Loss Unweighted: 1.1700 | Train Loss Components: Auxiliary: 0.4082 | Cardinality: 0.2377 | Classification: 0.1942 | Classification_binary: 0.3460 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3095 | Regression_eta: 0.0043 | Regression_pt: 0.3118 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1255 | Classification_binary: 0.3692 | Regression_cos_phi: 0.2041 | Regression_energy: 0.1562 | Regression_eta: 0.1367 | Regression_pt: 0.1636 | Regression_sin_phi: 0.1856 | LR: 9.19e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1031s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7744 | max_loss: 0.1497 | max_backward: 1.6846 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4710 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 6209.3 | uss_MB: 5170.1 | child_rss_MB: 5650.1 | allocated_MB: 345.9 | reserved_MB: 25594.0 | inactive_split_MB: 288.1 +rank1 [2026-10-28 18:06:42,229] INFO : Step 9800/50000 rank1 | Train Loss: 1.8589 | Train Loss Unweighted: 1.0245 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2213 | Classification: 0.1590 | Classification_binary: 0.2751 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2908 | Regression_eta: 0.0031 | Regression_pt: 0.2931 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2516 | Regression_cos_phi: 0.1002 | Regression_energy: 0.1278 | Regression_eta: 0.0771 | Regression_pt: 0.1337 | Regression_sin_phi: 0.1268 | LR: 9.17e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.1221s | Phase Times: max_pre_step_barrier: 0.1099 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.1713 | max_backward: 1.7200 | max_optimizer_step: 0.1898 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6188.3 | uss_MB: 5169.7 | child_rss_MB: 5642.5 | allocated_MB: 342.5 | reserved_MB: 26610.0 | inactive_split_MB: 264.5 +rank1 [2026-10-28 18:10:58,313] INFO : Step 9900/50000 rank1 | Train Loss: 1.8366 | Train Loss Unweighted: 1.0077 | Train Loss Components: Auxiliary: 0.3557 | Cardinality: 0.2091 | Classification: 0.1810 | Classification_binary: 0.2609 | HitGrouping: 0.2462 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2775 | Regression_eta: 0.0035 | Regression_pt: 0.2814 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1037 | Classification_binary: 0.3138 | Regression_cos_phi: 0.1407 | Regression_energy: 0.1360 | Regression_eta: 0.0888 | Regression_pt: 0.1421 | Regression_sin_phi: 0.1363 | LR: 9.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5792s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6346 | max_loss: 0.1692 | max_backward: 1.3542 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6188.4 | uss_MB: 5170.9 | child_rss_MB: 5645.5 | allocated_MB: 343.7 | reserved_MB: 26610.0 | inactive_split_MB: 205.7 +rank1 [2026-10-28 18:15:06,636] INFO : Step 10000/50000 rank1 | Train Loss: 2.0706 | Train Loss Unweighted: 1.1295 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2648 | Classification: 0.1819 | Classification_binary: 0.3225 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3077 | Regression_eta: 0.0037 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0798 | Classification_binary: 0.2796 | Regression_cos_phi: 0.1285 | Regression_energy: 0.1345 | Regression_eta: 0.0890 | Regression_pt: 0.1388 | Regression_sin_phi: 0.1247 | LR: 9.14e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.7113s | Phase Times: max_pre_step_barrier: 0.0136 | max_batch_to_device: 0.0708 | max_forward: 1.8842 | max_loss: 0.5484 | max_backward: 2.6620 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.5314 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6190.4 | uss_MB: 5174.2 | child_rss_MB: 5642.5 | allocated_MB: 345.2 | reserved_MB: 26610.0 | inactive_split_MB: 288.8 +rank1 [2026-10-28 18:15:07,626] INFO : Running validation on rank1 +rank1 [2026-10-28 18:15:07,626] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 18:15:07,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 18:15:55,924] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:15:56,538] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 18:16:22,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 18:16:23,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 18:16:24,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 18:16:26,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 18:16:36,077] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 18:16:36,560] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:16:37,046] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 18:16:52,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 18:16:53,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 18:16:55,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 18:16:56,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 18:17:06,701] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 18:17:09,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:17:10,078] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 18:17:25,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 18:17:26,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 18:17:27,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 18:17:27,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 18:17:37,907] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 18:22:01,597] INFO : Step 10100/50000 rank1 | Train Loss: 1.8115 | Train Loss Unweighted: 1.0221 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.1999 | Classification: 0.1589 | Classification_binary: 0.2815 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2868 | Regression_eta: 0.0032 | Regression_pt: 0.2886 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1100 | Classification_binary: 0.2717 | Regression_cos_phi: 0.1472 | Regression_energy: 0.1352 | Regression_eta: 0.0827 | Regression_pt: 0.1396 | Regression_sin_phi: 0.1460 | LR: 9.12e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9767s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7090 | max_loss: 0.1507 | max_backward: 1.2179 | max_optimizer_step: 0.3146 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 7166.8 | uss_MB: 6150.2 | child_rss_MB: 5645.8 | allocated_MB: 343.1 | reserved_MB: 21794.0 | inactive_split_MB: 243.8 +rank1 [2026-10-28 18:26:06,877] INFO : Step 10200/50000 rank1 | Train Loss: 1.6749 | Train Loss Unweighted: 0.9660 | Train Loss Components: Auxiliary: 0.3310 | Cardinality: 0.1773 | Classification: 0.1544 | Classification_binary: 0.2377 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2827 | Regression_eta: 0.0027 | Regression_pt: 0.2857 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1033 | Classification_binary: 0.2278 | Regression_cos_phi: 0.1079 | Regression_energy: 0.1180 | Regression_eta: 0.0652 | Regression_pt: 0.1236 | Regression_sin_phi: 0.1021 | LR: 9.11e-04 | DataLoad Time: 0.0114s | Model Forward Time: 4.0269s | Phase Times: max_pre_step_barrier: 0.0352 | max_batch_to_device: 0.0241 | max_forward: 0.6625 | max_loss: 0.1442 | max_backward: 1.3835 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7009 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1256 | Memory: rss_MB: 7164.8 | uss_MB: 6150.1 | child_rss_MB: 5645.0 | allocated_MB: 345.9 | reserved_MB: 21794.0 | inactive_split_MB: 292.4 +rank1 [2026-10-28 18:30:14,208] INFO : Step 10300/50000 rank1 | Train Loss: 1.9113 | Train Loss Unweighted: 1.0360 | Train Loss Components: Auxiliary: 0.3660 | Cardinality: 0.2108 | Classification: 0.1830 | Classification_binary: 0.2931 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0033 | Regression_pt: 0.2786 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0820 | Classification_binary: 0.2963 | Regression_cos_phi: 0.1208 | Regression_energy: 0.1298 | Regression_eta: 0.0610 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0998 | LR: 9.09e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9107s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7205 | max_loss: 0.1496 | max_backward: 1.6083 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 6287.2 | uss_MB: 5273.5 | child_rss_MB: 5650.9 | allocated_MB: 343.8 | reserved_MB: 22764.0 | inactive_split_MB: 257.9 +rank1 [2026-10-28 18:34:19,879] INFO : Step 10400/50000 rank1 | Train Loss: 1.9303 | Train Loss Unweighted: 1.0569 | Train Loss Components: Auxiliary: 0.3739 | Cardinality: 0.2318 | Classification: 0.1704 | Classification_binary: 0.3342 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2718 | Regression_eta: 0.0033 | Regression_pt: 0.2738 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1120 | Classification_binary: 0.3208 | Regression_cos_phi: 0.1797 | Regression_energy: 0.1379 | Regression_eta: 0.1232 | Regression_pt: 0.1435 | Regression_sin_phi: 0.1523 | LR: 9.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7367s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7465 | max_loss: 0.1499 | max_backward: 1.4212 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.4712 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6287.2 | uss_MB: 5272.0 | child_rss_MB: 5660.4 | allocated_MB: 345.6 | reserved_MB: 22792.0 | inactive_split_MB: 259.8 +rank1 [2026-10-28 18:38:24,242] INFO : Step 10500/50000 rank1 | Train Loss: 1.6601 | Train Loss Unweighted: 0.9268 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1892 | Classification: 0.1410 | Classification_binary: 0.2342 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2693 | Regression_eta: 0.0029 | Regression_pt: 0.2766 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1220 | Classification_binary: 0.2286 | Regression_cos_phi: 0.1262 | Regression_energy: 0.1318 | Regression_eta: 0.1022 | Regression_pt: 0.1408 | Regression_sin_phi: 0.1543 | LR: 9.06e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.5592s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 0.6364 | max_loss: 0.2260 | max_backward: 1.1629 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6452 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6284.6 | uss_MB: 5269.3 | child_rss_MB: 5635.7 | allocated_MB: 340.4 | reserved_MB: 22792.0 | inactive_split_MB: 230.3 +rank1 [2026-10-28 18:42:22,546] INFO : Step 10600/50000 rank1 | Train Loss: 1.9388 | Train Loss Unweighted: 1.0719 | Train Loss Components: Auxiliary: 0.3737 | Cardinality: 0.2008 | Classification: 0.1863 | Classification_binary: 0.2709 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3047 | Regression_eta: 0.0042 | Regression_pt: 0.3019 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1252 | Classification_binary: 0.2931 | Regression_cos_phi: 0.1853 | Regression_energy: 0.1428 | Regression_eta: 0.1165 | Regression_pt: 0.1500 | Regression_sin_phi: 0.1750 | LR: 9.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9689s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6880 | max_loss: 0.2458 | max_backward: 1.4354 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7481 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 6285.6 | uss_MB: 5273.7 | child_rss_MB: 5653.3 | allocated_MB: 344.6 | reserved_MB: 22798.0 | inactive_split_MB: 286.4 +rank1 [2026-10-28 18:46:28,872] INFO : Step 10700/50000 rank1 | Train Loss: 1.8955 | Train Loss Unweighted: 1.0230 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.2139 | Classification: 0.1752 | Classification_binary: 0.3186 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2598 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1090 | Classification_binary: 0.3585 | Regression_cos_phi: 0.1679 | Regression_energy: 0.1231 | Regression_eta: 0.0966 | Regression_pt: 0.1303 | Regression_sin_phi: 0.1345 | LR: 9.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1554s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0072 | max_forward: 1.0363 | max_loss: 0.1757 | max_backward: 2.3255 | max_optimizer_step: 0.1834 | max_tensorboard_logging: 0.7232 | max_tensorboard_flush: 0.0051 | max_post_step_barrier: 0.1576 | Memory: rss_MB: 6287.7 | uss_MB: 5274.9 | child_rss_MB: 5649.7 | allocated_MB: 342.1 | reserved_MB: 22798.0 | inactive_split_MB: 282.9 +rank1 [2026-10-28 18:50:28,323] INFO : Step 10800/50000 rank1 | Train Loss: 1.7679 | Train Loss Unweighted: 0.9563 | Train Loss Components: Auxiliary: 0.3446 | Cardinality: 0.2074 | Classification: 0.1518 | Classification_binary: 0.2766 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2590 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0964 | Classification_binary: 0.2640 | Regression_cos_phi: 0.1465 | Regression_energy: 0.1296 | Regression_eta: 0.0826 | Regression_pt: 0.1349 | Regression_sin_phi: 0.1308 | LR: 9.00e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7802s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0995 | max_forward: 0.7884 | max_loss: 0.2225 | max_backward: 2.0144 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.5230 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6288.7 | uss_MB: 5271.1 | child_rss_MB: 5654.7 | allocated_MB: 343.6 | reserved_MB: 23784.0 | inactive_split_MB: 286.4 +rank1 [2026-10-28 18:54:31,738] INFO : Step 10900/50000 rank1 | Train Loss: 1.6025 | Train Loss Unweighted: 0.8934 | Train Loss Components: Auxiliary: 0.3208 | Cardinality: 0.1729 | Classification: 0.1446 | Classification_binary: 0.2284 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2574 | Regression_eta: 0.0027 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1000 | Classification_binary: 0.2388 | Regression_cos_phi: 0.1167 | Regression_energy: 0.1059 | Regression_eta: 0.0636 | Regression_pt: 0.1101 | Regression_sin_phi: 0.1130 | LR: 8.99e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.4479s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.5765 | max_loss: 0.1490 | max_backward: 1.1882 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6288.0 | uss_MB: 5275.5 | child_rss_MB: 5639.7 | allocated_MB: 342.1 | reserved_MB: 23796.0 | inactive_split_MB: 272.7 +rank1 [2026-10-28 18:58:45,418] INFO : Step 11000/50000 rank1 | Train Loss: 1.9315 | Train Loss Unweighted: 1.0466 | Train Loss Components: Auxiliary: 0.3721 | Cardinality: 0.2078 | Classification: 0.1806 | Classification_binary: 0.3087 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0037 | Regression_pt: 0.2758 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1089 | Classification_binary: 0.3084 | Regression_cos_phi: 0.1770 | Regression_energy: 0.1255 | Regression_eta: 0.0953 | Regression_pt: 0.1310 | Regression_sin_phi: 0.1108 | LR: 8.97e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.1705s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7424 | max_loss: 0.1704 | max_backward: 1.5866 | max_optimizer_step: 0.2906 | max_tensorboard_logging: 0.6287 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6277.0 | uss_MB: 5281.8 | child_rss_MB: 5663.8 | allocated_MB: 347.8 | reserved_MB: 23796.0 | inactive_split_MB: 262.8 +rank1 [2026-10-28 18:58:46,359] INFO : Running validation on rank1 +rank1 [2026-10-28 18:58:46,359] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 18:58:46,359] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 18:59:40,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:59:40,901] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 19:00:07,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 19:00:09,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 19:00:10,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 19:00:11,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 19:00:21,415] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-28 19:00:21,677] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:00:22,258] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 19:00:38,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 19:00:39,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 19:00:40,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 19:00:41,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 19:00:51,701] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 19:00:52,166] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:00:52,895] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 19:01:08,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 19:01:08,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 19:01:09,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 19:01:10,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 19:01:20,319] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 19:05:41,392] INFO : Step 11100/50000 rank1 | Train Loss: 1.9118 | Train Loss Unweighted: 1.0064 | Train Loss Components: Auxiliary: 0.3723 | Cardinality: 0.2793 | Classification: 0.1563 | Classification_binary: 0.3082 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2660 | Regression_eta: 0.0034 | Regression_pt: 0.2693 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0888 | Classification_binary: 0.2956 | Regression_cos_phi: 0.1360 | Regression_energy: 0.1252 | Regression_eta: 0.1141 | Regression_pt: 0.1320 | Regression_sin_phi: 0.1147 | LR: 8.95e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0411s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.9217 | max_loss: 0.2304 | max_backward: 1.5570 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 7169.9 | uss_MB: 6179.3 | child_rss_MB: 5664.2 | allocated_MB: 343.2 | reserved_MB: 22146.0 | inactive_split_MB: 281.7 +rank1 [2026-10-28 19:09:39,137] INFO : Step 11200/50000 rank1 | Train Loss: 1.7366 | Train Loss Unweighted: 0.9397 | Train Loss Components: Auxiliary: 0.3529 | Cardinality: 0.2466 | Classification: 0.1384 | Classification_binary: 0.2548 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2689 | Regression_eta: 0.0027 | Regression_pt: 0.2722 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2156 | Regression_cos_phi: 0.0906 | Regression_energy: 0.1244 | Regression_eta: 0.0710 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0949 | LR: 8.93e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7178s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7793 | max_loss: 0.1825 | max_backward: 1.3743 | max_optimizer_step: 0.2128 | max_tensorboard_logging: 0.4676 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7167.9 | uss_MB: 6179.2 | child_rss_MB: 5641.7 | allocated_MB: 344.6 | reserved_MB: 22972.0 | inactive_split_MB: 273.4 +rank1 [2026-10-28 19:13:41,797] INFO : Step 11300/50000 rank1 | Train Loss: 1.6876 | Train Loss Unweighted: 0.9423 | Train Loss Components: Auxiliary: 0.3334 | Cardinality: 0.1862 | Classification: 0.1581 | Classification_binary: 0.2713 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2524 | Regression_eta: 0.0027 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1263 | Classification_binary: 0.3181 | Regression_cos_phi: 0.1274 | Regression_energy: 0.1294 | Regression_eta: 0.0855 | Regression_pt: 0.1385 | Regression_sin_phi: 0.1259 | LR: 8.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6829s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0223 | max_forward: 0.9315 | max_loss: 0.2308 | max_backward: 1.9859 | max_optimizer_step: 0.1928 | max_tensorboard_logging: 0.5681 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6388.7 | uss_MB: 5273.7 | child_rss_MB: 5658.4 | allocated_MB: 345.5 | reserved_MB: 24368.0 | inactive_split_MB: 269.6 +rank1 [2026-10-28 19:17:45,333] INFO : Step 11400/50000 rank1 | Train Loss: 1.8563 | Train Loss Unweighted: 0.9980 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.2376 | Classification: 0.1627 | Classification_binary: 0.3164 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2551 | Regression_eta: 0.0029 | Regression_pt: 0.2577 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0986 | Classification_binary: 0.3310 | Regression_cos_phi: 0.1462 | Regression_energy: 0.1193 | Regression_eta: 0.0792 | Regression_pt: 0.1238 | Regression_sin_phi: 0.1317 | LR: 8.89e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.0886s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.7684 | max_loss: 0.1511 | max_backward: 1.6966 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5034 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 6391.7 | uss_MB: 5402.6 | child_rss_MB: 5664.5 | allocated_MB: 343.2 | reserved_MB: 25642.0 | inactive_split_MB: 251.6 +rank1 [2026-10-28 19:21:55,043] INFO : Step 11500/50000 rank1 | Train Loss: 1.6323 | Train Loss Unweighted: 0.8976 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1755 | Classification: 0.1375 | Classification_binary: 0.2717 | HitGrouping: 0.2346 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2408 | Regression_eta: 0.0028 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1049 | Classification_binary: 0.2322 | Regression_cos_phi: 0.1188 | Regression_energy: 0.1074 | Regression_eta: 0.0795 | Regression_pt: 0.1105 | Regression_sin_phi: 0.1490 | LR: 8.88e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.6945s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7232 | max_loss: 0.1557 | max_backward: 1.4770 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6390.5 | uss_MB: 5401.3 | child_rss_MB: 5644.3 | allocated_MB: 345.3 | reserved_MB: 25642.0 | inactive_split_MB: 274.7 +rank1 [2026-10-28 19:25:54,594] INFO : Step 11600/50000 rank1 | Train Loss: 1.6133 | Train Loss Unweighted: 0.9062 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1718 | Classification: 0.1502 | Classification_binary: 0.2356 | HitGrouping: 0.2377 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2569 | Regression_eta: 0.0025 | Regression_pt: 0.2581 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0814 | Classification_binary: 0.2424 | Regression_cos_phi: 0.1188 | Regression_energy: 0.1064 | Regression_eta: 0.0605 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0884 | LR: 8.86e-04 | DataLoad Time: 0.0146s | Model Forward Time: 3.6511s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6646 | max_loss: 0.1710 | max_backward: 1.3208 | max_optimizer_step: 0.1963 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6391.5 | uss_MB: 5404.2 | child_rss_MB: 5665.3 | allocated_MB: 344.1 | reserved_MB: 26630.0 | inactive_split_MB: 209.9 +rank1 [2026-10-28 19:29:50,733] INFO : Step 11700/50000 rank1 | Train Loss: 1.7739 | Train Loss Unweighted: 0.9706 | Train Loss Components: Auxiliary: 0.3531 | Cardinality: 0.2418 | Classification: 0.1518 | Classification_binary: 0.2821 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2638 | Regression_eta: 0.0028 | Regression_pt: 0.2671 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0844 | Classification_binary: 0.2838 | Regression_cos_phi: 0.1418 | Regression_energy: 0.1188 | Regression_eta: 0.0968 | Regression_pt: 0.1241 | Regression_sin_phi: 0.1082 | LR: 8.84e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.4608s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8789 | max_loss: 0.2057 | max_backward: 2.3846 | max_optimizer_step: 0.1875 | max_tensorboard_logging: 1.0403 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6389.5 | uss_MB: 4921.9 | child_rss_MB: 5640.9 | allocated_MB: 343.8 | reserved_MB: 27624.0 | inactive_split_MB: 186.6 +rank1 [2026-10-28 19:33:54,004] INFO : Step 11800/50000 rank1 | Train Loss: 1.8223 | Train Loss Unweighted: 0.9745 | Train Loss Components: Auxiliary: 0.3609 | Cardinality: 0.2329 | Classification: 0.1589 | Classification_binary: 0.3100 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2473 | Regression_eta: 0.0028 | Regression_pt: 0.2523 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0937 | Classification_binary: 0.2818 | Regression_cos_phi: 0.0962 | Regression_energy: 0.1125 | Regression_eta: 0.0606 | Regression_pt: 0.1194 | Regression_sin_phi: 0.1039 | LR: 8.82e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.3911s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0070 | max_forward: 1.2302 | max_loss: 0.2767 | max_backward: 2.3745 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6390.5 | uss_MB: 5402.7 | child_rss_MB: 5661.5 | allocated_MB: 346.0 | reserved_MB: 27624.0 | inactive_split_MB: 280.5 +rank1 [2026-10-28 19:37:48,966] INFO : Step 11900/50000 rank1 | Train Loss: 1.5797 | Train Loss Unweighted: 0.8638 | Train Loss Components: Auxiliary: 0.3225 | Cardinality: 0.1935 | Classification: 0.1358 | Classification_binary: 0.2351 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2427 | Regression_eta: 0.0025 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.2229 | Regression_cos_phi: 0.0862 | Regression_energy: 0.1039 | Regression_eta: 0.0702 | Regression_pt: 0.1087 | Regression_sin_phi: 0.1025 | LR: 8.80e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.1387s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0787 | max_forward: 0.7011 | max_loss: 0.1484 | max_backward: 1.2002 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.9463 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 6390.3 | uss_MB: 5402.5 | child_rss_MB: 5642.2 | allocated_MB: 340.1 | reserved_MB: 28620.0 | inactive_split_MB: 264.1 +rank1 [2026-10-28 19:41:43,168] INFO : Step 12000/50000 rank1 | Train Loss: 1.6036 | Train Loss Unweighted: 0.8706 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1972 | Classification: 0.1481 | Classification_binary: 0.2459 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2330 | Regression_eta: 0.0025 | Regression_pt: 0.2385 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2355 | Regression_cos_phi: 0.0902 | Regression_energy: 0.1163 | Regression_eta: 0.0549 | Regression_pt: 0.1243 | Regression_sin_phi: 0.1027 | LR: 8.78e-04 | DataLoad Time: 0.0146s | Model Forward Time: 3.6992s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0072 | max_forward: 0.6194 | max_loss: 0.1546 | max_backward: 1.4628 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5143 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6383.3 | uss_MB: 5409.4 | child_rss_MB: 5646.2 | allocated_MB: 342.9 | reserved_MB: 28620.0 | inactive_split_MB: 215.8 +rank1 [2026-10-28 19:41:44,122] INFO : Running validation on rank1 +rank1 [2026-10-28 19:41:44,122] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 19:41:44,122] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 19:42:35,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:42:35,719] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 19:43:02,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 19:43:04,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 19:43:05,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 19:43:06,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 19:43:16,706] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-28 19:43:17,093] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:43:17,653] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 19:43:34,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 19:43:35,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 19:43:36,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 19:43:36,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 19:43:46,986] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 19:43:48,085] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:43:48,652] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 19:44:04,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 19:44:04,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 19:44:05,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 19:44:06,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 19:44:16,279] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 19:48:34,084] INFO : Step 12100/50000 rank1 | Train Loss: 1.8149 | Train Loss Unweighted: 0.9793 | Train Loss Components: Auxiliary: 0.3632 | Cardinality: 0.2324 | Classification: 0.1506 | Classification_binary: 0.3149 | HitGrouping: 0.2337 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2518 | Regression_eta: 0.0029 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.3217 | Regression_cos_phi: 0.1068 | Regression_energy: 0.1232 | Regression_eta: 0.0800 | Regression_pt: 0.1284 | Regression_sin_phi: 0.1517 | LR: 8.76e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5272s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8305 | max_loss: 0.1678 | max_backward: 1.6498 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8610 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7131.3 | uss_MB: 6150.6 | child_rss_MB: 5646.8 | allocated_MB: 345.8 | reserved_MB: 23512.0 | inactive_split_MB: 299.6 +rank1 [2026-10-28 19:52:54,660] INFO : Step 12200/50000 rank1 | Train Loss: 1.6619 | Train Loss Unweighted: 0.9043 | Train Loss Components: Auxiliary: 0.3290 | Cardinality: 0.1906 | Classification: 0.1445 | Classification_binary: 0.2674 | HitGrouping: 0.2303 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2435 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2594 | Regression_cos_phi: 0.1094 | Regression_energy: 0.1009 | Regression_eta: 0.0800 | Regression_pt: 0.1028 | Regression_sin_phi: 0.1291 | LR: 8.74e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8234s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.7862 | max_loss: 0.1438 | max_backward: 1.5991 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.8843 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7133.3 | uss_MB: 5953.1 | child_rss_MB: 5653.1 | allocated_MB: 341.2 | reserved_MB: 23512.0 | inactive_split_MB: 206.8 +rank1 [2026-10-28 19:56:47,338] INFO : Step 12300/50000 rank1 | Train Loss: 1.4699 | Train Loss Unweighted: 0.8181 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1474 | Classification: 0.1404 | Classification_binary: 0.2280 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2222 | Regression_eta: 0.0028 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1009 | Classification_binary: 0.2601 | Regression_cos_phi: 0.1203 | Regression_energy: 0.1082 | Regression_eta: 0.1235 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0765 | LR: 8.72e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.7233s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6177 | max_loss: 0.2232 | max_backward: 1.1179 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 0.7974 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6497.9 | uss_MB: 5527.4 | child_rss_MB: 5654.1 | allocated_MB: 343.2 | reserved_MB: 24286.0 | inactive_split_MB: 290.2 +rank1 [2026-10-28 20:00:36,190] INFO : Step 12400/50000 rank1 | Train Loss: 1.6135 | Train Loss Unweighted: 0.8782 | Train Loss Components: Auxiliary: 0.3265 | Cardinality: 0.2056 | Classification: 0.1546 | Classification_binary: 0.2540 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2308 | Regression_eta: 0.0025 | Regression_pt: 0.2337 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0813 | Classification_binary: 0.2624 | Regression_cos_phi: 0.0869 | Regression_energy: 0.0897 | Regression_eta: 0.0561 | Regression_pt: 0.0976 | Regression_sin_phi: 0.1174 | LR: 8.70e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.9936s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6928 | max_loss: 0.1486 | max_backward: 1.3860 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6497.9 | uss_MB: 5528.1 | child_rss_MB: 5671.7 | allocated_MB: 344.8 | reserved_MB: 24510.0 | inactive_split_MB: 295.5 +rank1 [2026-10-28 20:04:30,061] INFO : Step 12500/50000 rank1 | Train Loss: 1.6527 | Train Loss Unweighted: 0.8797 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1890 | Classification: 0.1391 | Classification_binary: 0.2779 | HitGrouping: 0.2315 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2273 | Regression_eta: 0.0030 | Regression_pt: 0.2295 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0802 | Classification_binary: 0.2730 | Regression_cos_phi: 0.1382 | Regression_energy: 0.1205 | Regression_eta: 0.0863 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0894 | LR: 8.68e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1681s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0535 | max_forward: 0.7747 | max_loss: 0.1534 | max_backward: 1.3339 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6494.1 | uss_MB: 5528.8 | child_rss_MB: 5649.7 | allocated_MB: 347.2 | reserved_MB: 24510.0 | inactive_split_MB: 456.8 +rank1 [2026-10-28 20:08:30,520] INFO : Step 12600/50000 rank1 | Train Loss: 1.4969 | Train Loss Unweighted: 0.8185 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1595 | Classification: 0.1210 | Classification_binary: 0.2357 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2279 | Regression_eta: 0.0024 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.2219 | Regression_cos_phi: 0.0879 | Regression_energy: 0.1056 | Regression_eta: 0.0632 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0622 | LR: 8.66e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.7596s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0062 | max_forward: 0.6571 | max_loss: 0.1429 | max_backward: 1.4364 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6482.1 | uss_MB: 5531.3 | child_rss_MB: 5638.2 | allocated_MB: 342.9 | reserved_MB: 25526.0 | inactive_split_MB: 234.6 +rank1 [2026-10-28 20:12:34,583] INFO : Step 12700/50000 rank1 | Train Loss: 1.6234 | Train Loss Unweighted: 0.8661 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.2004 | Classification: 0.1459 | Classification_binary: 0.2387 | HitGrouping: 0.2403 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2360 | Regression_eta: 0.0027 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0920 | Classification_binary: 0.2133 | Regression_cos_phi: 0.1309 | Regression_energy: 0.1293 | Regression_eta: 0.0961 | Regression_pt: 0.1400 | Regression_sin_phi: 0.1335 | LR: 8.64e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.4790s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6221 | max_loss: 0.1846 | max_backward: 1.1554 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6469.4 | uss_MB: 5531.3 | child_rss_MB: 5648.7 | allocated_MB: 345.9 | reserved_MB: 26286.0 | inactive_split_MB: 241.7 +rank1 [2026-10-28 20:16:22,944] INFO : Step 12800/50000 rank1 | Train Loss: 1.7877 | Train Loss Unweighted: 0.9398 | Train Loss Components: Auxiliary: 0.3651 | Cardinality: 0.2327 | Classification: 0.1495 | Classification_binary: 0.3094 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2357 | Regression_eta: 0.0027 | Regression_pt: 0.2394 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2919 | Regression_cos_phi: 0.1027 | Regression_energy: 0.1125 | Regression_eta: 0.0442 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0882 | LR: 8.62e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.2578s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.8468 | max_loss: 0.1534 | max_backward: 1.7420 | max_optimizer_step: 0.3013 | max_tensorboard_logging: 0.5725 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6458.4 | uss_MB: 5535.9 | child_rss_MB: 5651.5 | allocated_MB: 345.6 | reserved_MB: 26286.0 | inactive_split_MB: 284.4 +rank1 [2026-10-28 20:20:13,787] INFO : Step 12900/50000 rank1 | Train Loss: 1.5881 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.3191 | Cardinality: 0.2029 | Classification: 0.1304 | Classification_binary: 0.2564 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2291 | Regression_eta: 0.0023 | Regression_pt: 0.2312 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2110 | Regression_cos_phi: 0.0713 | Regression_energy: 0.1325 | Regression_eta: 0.0591 | Regression_pt: 0.1403 | Regression_sin_phi: 0.1105 | LR: 8.60e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9624s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7180 | max_loss: 0.2380 | max_backward: 1.4488 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6932 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6444.6 | uss_MB: 5531.5 | child_rss_MB: 5646.1 | allocated_MB: 341.8 | reserved_MB: 26286.0 | inactive_split_MB: 252.2 +rank1 [2026-10-28 20:24:06,549] INFO : Step 13000/50000 rank1 | Train Loss: 1.5217 | Train Loss Unweighted: 0.8547 | Train Loss Components: Auxiliary: 0.3141 | Cardinality: 0.1693 | Classification: 0.1382 | Classification_binary: 0.2135 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2470 | Regression_eta: 0.0024 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2313 | Regression_cos_phi: 0.0937 | Regression_energy: 0.1079 | Regression_eta: 0.0521 | Regression_pt: 0.1145 | Regression_sin_phi: 0.1203 | LR: 8.58e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8300s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7101 | max_loss: 0.1763 | max_backward: 1.1845 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 0.8282 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6429.6 | uss_MB: 5536.9 | child_rss_MB: 5653.9 | allocated_MB: 339.8 | reserved_MB: 26286.0 | inactive_split_MB: 213.4 +rank1 [2026-10-28 20:24:07,449] INFO : Running validation on rank1 +rank1 [2026-10-28 20:24:07,449] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 20:24:07,449] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 20:24:58,458] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:24:59,078] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 20:25:24,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 20:25:25,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 20:25:26,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 20:25:27,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 20:25:37,564] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 20:25:38,113] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:25:38,652] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 20:25:54,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 20:25:55,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 20:25:56,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 20:25:57,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 20:26:07,349] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 20:26:08,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:26:09,008] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 20:26:24,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 20:26:25,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 20:26:26,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 20:26:26,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 20:26:36,784] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 20:30:47,340] INFO : Step 13100/50000 rank1 | Train Loss: 1.6168 | Train Loss Unweighted: 0.8649 | Train Loss Components: Auxiliary: 0.3213 | Cardinality: 0.1869 | Classification: 0.1467 | Classification_binary: 0.2696 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2210 | Regression_eta: 0.0027 | Regression_pt: 0.2221 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0878 | Classification_binary: 0.2542 | Regression_cos_phi: 0.0933 | Regression_energy: 0.1049 | Regression_eta: 0.0648 | Regression_pt: 0.1083 | Regression_sin_phi: 0.1096 | LR: 8.56e-04 | DataLoad Time: 0.0106s | Model Forward Time: 6.4003s | Phase Times: max_pre_step_barrier: 0.0337 | max_batch_to_device: 0.0136 | max_forward: 1.2842 | max_loss: 0.2583 | max_backward: 3.1281 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.9677 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7302.9 | uss_MB: 6406.9 | child_rss_MB: 5646.9 | allocated_MB: 341.8 | reserved_MB: 22940.0 | inactive_split_MB: 270.2 +rank1 [2026-10-28 20:34:56,293] INFO : Step 13200/50000 rank1 | Train Loss: 1.6996 | Train Loss Unweighted: 0.8855 | Train Loss Components: Auxiliary: 0.3386 | Cardinality: 0.2474 | Classification: 0.1410 | Classification_binary: 0.2769 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2299 | Regression_eta: 0.0027 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.2595 | Regression_cos_phi: 0.0898 | Regression_energy: 0.1113 | Regression_eta: 0.0652 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0992 | LR: 8.54e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.1539s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0072 | max_forward: 0.7543 | max_loss: 0.1532 | max_backward: 1.3317 | max_optimizer_step: 0.3180 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7300.0 | uss_MB: 6407.2 | child_rss_MB: 5649.5 | allocated_MB: 343.4 | reserved_MB: 23942.0 | inactive_split_MB: 249.6 +rank1 [2026-10-28 20:38:53,368] INFO : Step 13300/50000 rank1 | Train Loss: 1.4876 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2970 | Cardinality: 0.2018 | Classification: 0.1100 | Classification_binary: 0.2192 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2205 | Regression_eta: 0.0026 | Regression_pt: 0.2230 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1981 | Regression_cos_phi: 0.0828 | Regression_energy: 0.1027 | Regression_eta: 0.0525 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0728 | LR: 8.52e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.6371s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0259 | max_forward: 0.9208 | max_loss: 0.2245 | max_backward: 1.9523 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6473.0 | uss_MB: 4937.1 | child_rss_MB: 5535.0 | allocated_MB: 341.7 | reserved_MB: 24474.0 | inactive_split_MB: 265.4 +rank1 [2026-10-28 20:42:52,944] INFO : Step 13400/50000 rank1 | Train Loss: 1.4253 | Train Loss Unweighted: 0.7944 | Train Loss Components: Auxiliary: 0.2889 | Cardinality: 0.1585 | Classification: 0.1406 | Classification_binary: 0.2158 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2158 | Regression_eta: 0.0022 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2314 | Regression_cos_phi: 0.1152 | Regression_energy: 0.0986 | Regression_eta: 0.0763 | Regression_pt: 0.1026 | Regression_sin_phi: 0.1308 | LR: 8.50e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7382s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6533 | max_loss: 0.2500 | max_backward: 1.4458 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.5454 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6474.0 | uss_MB: 5610.2 | child_rss_MB: 5538.3 | allocated_MB: 341.6 | reserved_MB: 25334.0 | inactive_split_MB: 266.6 +rank1 [2026-10-28 20:47:05,664] INFO : Step 13500/50000 rank1 | Train Loss: 1.6021 | Train Loss Unweighted: 0.8749 | Train Loss Components: Auxiliary: 0.3241 | Cardinality: 0.1832 | Classification: 0.1473 | Classification_binary: 0.2801 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2200 | Regression_eta: 0.0024 | Regression_pt: 0.2225 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0946 | Classification_binary: 0.3066 | Regression_cos_phi: 0.0952 | Regression_energy: 0.1102 | Regression_eta: 0.0632 | Regression_pt: 0.1158 | Regression_sin_phi: 0.1009 | LR: 8.48e-04 | DataLoad Time: 0.0145s | Model Forward Time: 6.1085s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.7859 | max_loss: 0.3303 | max_backward: 2.5111 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4974 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6412.7 | uss_MB: 5395.8 | child_rss_MB: 5343.5 | allocated_MB: 344.3 | reserved_MB: 25574.0 | inactive_split_MB: 284.1 +rank1 [2026-10-28 20:51:04,211] INFO : Step 13600/50000 rank1 | Train Loss: 1.5296 | Train Loss Unweighted: 0.8114 | Train Loss Components: Auxiliary: 0.3050 | Cardinality: 0.1872 | Classification: 0.1291 | Classification_binary: 0.2450 | HitGrouping: 0.2218 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2158 | Regression_eta: 0.0025 | Regression_pt: 0.2165 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2244 | Regression_cos_phi: 0.0867 | Regression_energy: 0.1051 | Regression_eta: 0.0602 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0724 | LR: 8.45e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.6657s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.7508 | max_loss: 0.1507 | max_backward: 1.2891 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6404.7 | uss_MB: 5608.6 | child_rss_MB: 5283.3 | allocated_MB: 345.0 | reserved_MB: 26608.0 | inactive_split_MB: 257.3 +rank1 [2026-10-28 20:55:12,337] INFO : Step 13700/50000 rank1 | Train Loss: 1.4515 | Train Loss Unweighted: 0.7895 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1575 | Classification: 0.1261 | Classification_binary: 0.2121 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2219 | Regression_eta: 0.0023 | Regression_pt: 0.2245 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1869 | Regression_cos_phi: 0.1121 | Regression_energy: 0.1171 | Regression_eta: 0.0522 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0677 | LR: 8.43e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.8533s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.6804 | max_loss: 0.2128 | max_backward: 1.1331 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6398.1 | uss_MB: 5609.6 | child_rss_MB: 5245.6 | allocated_MB: 342.5 | reserved_MB: 26608.0 | inactive_split_MB: 241.7 +rank1 [2026-10-28 20:59:19,101] INFO : Step 13800/50000 rank1 | Train Loss: 1.5060 | Train Loss Unweighted: 0.8320 | Train Loss Components: Auxiliary: 0.3084 | Cardinality: 0.1762 | Classification: 0.1368 | Classification_binary: 0.2567 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2156 | Regression_eta: 0.0023 | Regression_pt: 0.2184 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2849 | Regression_cos_phi: 0.1095 | Regression_energy: 0.1108 | Regression_eta: 0.0741 | Regression_pt: 0.1166 | Regression_sin_phi: 0.1164 | LR: 8.41e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4121s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7967 | max_loss: 0.1491 | max_backward: 1.5976 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6392.1 | uss_MB: 5613.3 | child_rss_MB: 5216.1 | allocated_MB: 342.1 | reserved_MB: 26608.0 | inactive_split_MB: 241.3 +rank1 [2026-10-28 21:03:21,776] INFO : Step 13900/50000 rank1 | Train Loss: 1.6108 | Train Loss Unweighted: 0.8469 | Train Loss Components: Auxiliary: 0.3204 | Cardinality: 0.1942 | Classification: 0.1303 | Classification_binary: 0.2751 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2168 | Regression_eta: 0.0028 | Regression_pt: 0.2192 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.2397 | Regression_cos_phi: 0.0862 | Regression_energy: 0.1142 | Regression_eta: 0.0527 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0893 | LR: 8.39e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.7331s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8010 | max_loss: 0.1552 | max_backward: 2.5526 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 1.2862 | max_tensorboard_flush: 0.0041 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 6389.1 | uss_MB: 5603.5 | child_rss_MB: 5217.8 | allocated_MB: 345.6 | reserved_MB: 26608.0 | inactive_split_MB: 292.8 +rank1 [2026-10-28 21:07:29,379] INFO : Step 14000/50000 rank1 | Train Loss: 1.3659 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1707 | Classification: 0.0974 | Classification_binary: 0.1997 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2149 | Regression_eta: 0.0023 | Regression_pt: 0.2181 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1820 | Regression_cos_phi: 0.0700 | Regression_energy: 0.1091 | Regression_eta: 0.0536 | Regression_pt: 0.1135 | Regression_sin_phi: 0.0876 | LR: 8.37e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.4523s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7574 | max_loss: 0.1908 | max_backward: 1.5870 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6393.2 | uss_MB: 4632.1 | child_rss_MB: 5217.4 | allocated_MB: 342.3 | reserved_MB: 26608.0 | inactive_split_MB: 238.4 +rank1 [2026-10-28 21:07:30,351] INFO : Running validation on rank1 +rank1 [2026-10-28 21:07:30,351] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 21:07:30,351] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 21:08:21,199] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:08:21,763] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 21:08:49,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 21:08:50,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 21:08:51,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 21:08:52,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 21:09:02,298] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 21:09:05,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:09:06,443] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 21:09:22,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 21:09:23,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 21:09:23,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 21:09:25,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 21:09:35,355] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 21:09:36,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:09:36,854] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 21:09:52,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 21:09:53,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 21:09:53,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 21:09:54,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 21:10:04,600] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 21:14:16,859] INFO : Step 14100/50000 rank1 | Train Loss: 1.5416 | Train Loss Unweighted: 0.8176 | Train Loss Components: Auxiliary: 0.3012 | Cardinality: 0.1751 | Classification: 0.1363 | Classification_binary: 0.2297 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2177 | Regression_eta: 0.0028 | Regression_pt: 0.2283 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2341 | Regression_cos_phi: 0.0937 | Regression_energy: 0.1151 | Regression_eta: 0.0484 | Regression_pt: 0.1249 | Regression_sin_phi: 0.0966 | LR: 8.35e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.3189s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7718 | max_loss: 0.1710 | max_backward: 1.6376 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 7412.7 | uss_MB: 6170.5 | child_rss_MB: 5197.6 | allocated_MB: 347.1 | reserved_MB: 23970.0 | inactive_split_MB: 244.9 +rank1 [2026-10-28 21:18:17,876] INFO : Step 14200/50000 rank1 | Train Loss: 1.6164 | Train Loss Unweighted: 0.8542 | Train Loss Components: Auxiliary: 0.3190 | Cardinality: 0.1950 | Classification: 0.1478 | Classification_binary: 0.2794 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2094 | Regression_eta: 0.0027 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2698 | Regression_cos_phi: 0.0946 | Regression_energy: 0.0958 | Regression_eta: 0.0670 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0964 | LR: 8.32e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.3718s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1153 | max_forward: 0.8788 | max_loss: 0.1615 | max_backward: 1.5286 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7413.4 | uss_MB: 6634.1 | child_rss_MB: 5195.0 | allocated_MB: 344.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.9 +rank1 [2026-10-28 21:22:21,673] INFO : Step 14300/50000 rank1 | Train Loss: 1.4246 | Train Loss Unweighted: 0.7744 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1685 | Classification: 0.1198 | Classification_binary: 0.2357 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2084 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.2037 | Regression_cos_phi: 0.0845 | Regression_energy: 0.1068 | Regression_eta: 0.0563 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0693 | LR: 8.30e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.8794s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7183 | max_loss: 0.1494 | max_backward: 1.3371 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6454.4 | uss_MB: 5677.1 | child_rss_MB: 5184.2 | allocated_MB: 341.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.1 +rank1 [2026-10-28 21:26:25,336] INFO : Step 14400/50000 rank1 | Train Loss: 1.3539 | Train Loss Unweighted: 0.7267 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1402 | Classification: 0.1303 | Classification_binary: 0.2015 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1957 | Regression_eta: 0.0022 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2252 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0951 | Regression_eta: 0.0563 | Regression_pt: 0.0988 | Regression_sin_phi: 0.1059 | LR: 8.28e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6677s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6350 | max_loss: 0.2071 | max_backward: 1.2597 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6458.7 | uss_MB: 5681.9 | child_rss_MB: 5188.8 | allocated_MB: 346.1 | reserved_MB: 24962.0 | inactive_split_MB: 231.6 +rank1 [2026-10-28 21:30:23,756] INFO : Step 14500/50000 rank1 | Train Loss: 1.5651 | Train Loss Unweighted: 0.7967 | Train Loss Components: Auxiliary: 0.3070 | Cardinality: 0.2312 | Classification: 0.1364 | Classification_binary: 0.2565 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1984 | Regression_eta: 0.0026 | Regression_pt: 0.2005 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0840 | Classification_binary: 0.2827 | Regression_cos_phi: 0.0995 | Regression_energy: 0.1054 | Regression_eta: 0.0604 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0830 | LR: 8.26e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.4671s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0069 | max_forward: 0.8745 | max_loss: 0.1717 | max_backward: 1.8485 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 1.4922 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6443.5 | uss_MB: 5682.8 | child_rss_MB: 5113.1 | allocated_MB: 344.2 | reserved_MB: 25374.0 | inactive_split_MB: 206.9 +rank1 [2026-10-28 21:34:34,947] INFO : Step 14600/50000 rank1 | Train Loss: 1.6485 | Train Loss Unweighted: 0.8413 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.2375 | Classification: 0.1249 | Classification_binary: 0.2902 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2104 | Regression_eta: 0.0023 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.2749 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0961 | Regression_eta: 0.0518 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0805 | LR: 8.24e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.3683s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0068 | max_forward: 0.7866 | max_loss: 0.1515 | max_backward: 1.5469 | max_optimizer_step: 0.3128 | max_tensorboard_logging: 0.7882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6427.1 | uss_MB: 5683.8 | child_rss_MB: 5049.1 | allocated_MB: 345.7 | reserved_MB: 25374.0 | inactive_split_MB: 292.3 +rank1 [2026-10-28 21:38:39,704] INFO : Step 14700/50000 rank1 | Train Loss: 1.3645 | Train Loss Unweighted: 0.7326 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1507 | Classification: 0.1080 | Classification_binary: 0.2034 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1857 | Regression_cos_phi: 0.1062 | Regression_energy: 0.0899 | Regression_eta: 0.0516 | Regression_pt: 0.0961 | Regression_sin_phi: 0.1019 | LR: 8.21e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9106s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6571 | max_loss: 0.1494 | max_backward: 1.2978 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7751 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0922 | Memory: rss_MB: 6423.6 | uss_MB: 5684.9 | child_rss_MB: 5040.9 | allocated_MB: 341.1 | reserved_MB: 25374.0 | inactive_split_MB: 264.9 +rank1 [2026-10-28 21:42:53,742] INFO : Step 14800/50000 rank1 | Train Loss: 1.3987 | Train Loss Unweighted: 0.7335 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1827 | Classification: 0.1136 | Classification_binary: 0.2244 | HitGrouping: 0.2204 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1949 | Regression_eta: 0.0022 | Regression_pt: 0.1963 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1829 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0787 | Regression_eta: 0.0437 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0501 | LR: 8.19e-04 | DataLoad Time: 0.0237s | Model Forward Time: 5.2138s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.1679 | max_loss: 0.2171 | max_backward: 2.8051 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 6420.4 | uss_MB: 4434.9 | child_rss_MB: 5026.4 | allocated_MB: 345.2 | reserved_MB: 25374.0 | inactive_split_MB: 222.5 +rank1 [2026-10-28 21:47:06,022] INFO : Step 14900/50000 rank1 | Train Loss: 1.5046 | Train Loss Unweighted: 0.7991 | Train Loss Components: Auxiliary: 0.3055 | Cardinality: 0.1834 | Classification: 0.1311 | Classification_binary: 0.2643 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1982 | Regression_eta: 0.0023 | Regression_pt: 0.2008 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2556 | Regression_cos_phi: 0.1204 | Regression_energy: 0.1080 | Regression_eta: 0.0609 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0906 | LR: 8.17e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0366s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0071 | max_forward: 0.7642 | max_loss: 0.1561 | max_backward: 1.5975 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6421.6 | uss_MB: 5685.9 | child_rss_MB: 5030.2 | allocated_MB: 345.4 | reserved_MB: 25374.0 | inactive_split_MB: 289.7 +rank1 [2026-10-28 21:51:16,625] INFO : Step 15000/50000 rank1 | Train Loss: 1.3836 | Train Loss Unweighted: 0.7341 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1614 | Classification: 0.1182 | Classification_binary: 0.2252 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1930 | Regression_eta: 0.0023 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2206 | Regression_cos_phi: 0.1048 | Regression_energy: 0.0942 | Regression_eta: 0.0614 | Regression_pt: 0.0983 | Regression_sin_phi: 0.0620 | LR: 8.15e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.7143s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0066 | max_forward: 0.6828 | max_loss: 0.1453 | max_backward: 1.3064 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1064 | Memory: rss_MB: 6423.8 | uss_MB: 5686.3 | child_rss_MB: 5021.1 | allocated_MB: 342.6 | reserved_MB: 25374.0 | inactive_split_MB: 262.7 +rank1 [2026-10-28 21:51:17,568] INFO : Running validation on rank1 +rank1 [2026-10-28 21:51:17,568] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 21:51:17,568] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 21:52:06,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:52:07,026] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 21:52:33,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 21:52:34,263] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 21:52:35,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 21:52:36,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 21:52:46,128] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 21:52:46,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:52:47,273] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 21:53:03,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 21:53:04,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 21:53:05,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 21:53:06,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 21:53:16,158] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 21:53:16,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:53:17,340] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 21:53:32,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 21:53:33,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 21:53:34,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 21:53:34,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 21:53:44,848] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 21:57:58,526] INFO : Step 15100/50000 rank1 | Train Loss: 1.3706 | Train Loss Unweighted: 0.7159 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1535 | Classification: 0.1271 | Classification_binary: 0.1954 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1940 | Regression_eta: 0.0022 | Regression_pt: 0.1948 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.2286 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0949 | Regression_eta: 0.0666 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0875 | LR: 8.12e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.0260s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.6658 | max_loss: 0.2401 | max_backward: 1.3105 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8277 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7654.5 | uss_MB: 6910.5 | child_rss_MB: 5047.2 | allocated_MB: 341.1 | reserved_MB: 22096.0 | inactive_split_MB: 215.5 +rank1 [2026-10-28 22:01:59,131] INFO : Step 15200/50000 rank1 | Train Loss: 1.5024 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3072 | Cardinality: 0.2080 | Classification: 0.1273 | Classification_binary: 0.2385 | HitGrouping: 0.2164 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1989 | Regression_eta: 0.0022 | Regression_pt: 0.2007 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0817 | Classification_binary: 0.2677 | Regression_cos_phi: 0.1166 | Regression_energy: 0.1046 | Regression_eta: 0.0685 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0857 | LR: 8.10e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.5457s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7502 | max_loss: 0.1534 | max_backward: 1.8002 | max_optimizer_step: 0.2270 | max_tensorboard_logging: 0.8682 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 7649.1 | uss_MB: 6910.3 | child_rss_MB: 5045.3 | allocated_MB: 345.6 | reserved_MB: 22420.0 | inactive_split_MB: 257.7 +rank1 [2026-10-28 22:05:42,232] INFO : Step 15300/50000 rank1 | Train Loss: 1.4920 | Train Loss Unweighted: 0.7880 | Train Loss Components: Auxiliary: 0.3065 | Cardinality: 0.1970 | Classification: 0.1161 | Classification_binary: 0.2589 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2026 | Regression_eta: 0.0022 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2637 | Regression_cos_phi: 0.0901 | Regression_energy: 0.1127 | Regression_eta: 0.0668 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0814 | LR: 8.08e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5783s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0192 | max_forward: 0.7642 | max_loss: 0.1530 | max_backward: 1.6292 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 0.9125 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6524.0 | uss_MB: 5797.3 | child_rss_MB: 5044.3 | allocated_MB: 341.2 | reserved_MB: 23096.0 | inactive_split_MB: 271.7 +rank1 [2026-10-28 22:09:28,150] INFO : Step 15400/50000 rank1 | Train Loss: 1.3549 | Train Loss Unweighted: 0.7210 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1572 | Classification: 0.1148 | Classification_binary: 0.2052 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1983 | Regression_eta: 0.0022 | Regression_pt: 0.1984 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2055 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0761 | Regression_eta: 0.0604 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0831 | LR: 8.05e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.7038s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6594 | max_loss: 0.2185 | max_backward: 1.2416 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6511.0 | uss_MB: 5790.6 | child_rss_MB: 5047.2 | allocated_MB: 340.8 | reserved_MB: 23394.0 | inactive_split_MB: 166.6 +rank1 [2026-10-28 22:13:12,490] INFO : Step 15500/50000 rank1 | Train Loss: 1.4313 | Train Loss Unweighted: 0.7527 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1666 | Classification: 0.1321 | Classification_binary: 0.2182 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1979 | Regression_eta: 0.0025 | Regression_pt: 0.1996 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0649 | Regression_energy: 0.1026 | Regression_eta: 0.0437 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0555 | LR: 8.03e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2549s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0792 | max_forward: 0.7874 | max_loss: 0.1514 | max_backward: 1.8676 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5380 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6511.0 | uss_MB: 5805.3 | child_rss_MB: 5052.2 | allocated_MB: 345.6 | reserved_MB: 24092.0 | inactive_split_MB: 260.5 +rank1 [2026-10-28 22:17:01,248] INFO : Step 15600/50000 rank1 | Train Loss: 1.5305 | Train Loss Unweighted: 0.8109 | Train Loss Components: Auxiliary: 0.3137 | Cardinality: 0.1790 | Classification: 0.1304 | Classification_binary: 0.2721 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2011 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0802 | Classification_binary: 0.2652 | Regression_cos_phi: 0.1167 | Regression_energy: 0.0884 | Regression_eta: 0.0612 | Regression_pt: 0.0926 | Regression_sin_phi: 0.1017 | LR: 8.01e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2102s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7645 | max_loss: 0.2230 | max_backward: 1.7015 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1398 | Memory: rss_MB: 6508.8 | uss_MB: 5803.4 | child_rss_MB: 5052.3 | allocated_MB: 344.1 | reserved_MB: 24402.0 | inactive_split_MB: 287.9 +rank1 [2026-10-28 22:20:46,546] INFO : Step 15700/50000 rank1 | Train Loss: 1.4083 | Train Loss Unweighted: 0.7205 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1775 | Classification: 0.1238 | Classification_binary: 0.2322 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1785 | Regression_eta: 0.0024 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0843 | Classification_binary: 0.2509 | Regression_cos_phi: 0.0925 | Regression_energy: 0.0812 | Regression_eta: 0.0590 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0958 | LR: 7.98e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.0320s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2392 | max_backward: 1.6606 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6506.8 | uss_MB: 5802.2 | child_rss_MB: 5055.5 | allocated_MB: 344.4 | reserved_MB: 25380.0 | inactive_split_MB: 228.2 +rank1 [2026-10-28 22:24:39,043] INFO : Step 15800/50000 rank1 | Train Loss: 1.2988 | Train Loss Unweighted: 0.6968 | Train Loss Components: Auxiliary: 0.2671 | Cardinality: 0.1436 | Classification: 0.1112 | Classification_binary: 0.1916 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1933 | Regression_eta: 0.0020 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.2308 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0819 | Regression_eta: 0.0552 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0719 | LR: 7.96e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7266s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6109 | max_loss: 0.2015 | max_backward: 1.1291 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6506.5 | uss_MB: 5803.6 | child_rss_MB: 5051.3 | allocated_MB: 341.6 | reserved_MB: 25380.0 | inactive_split_MB: 263.9 +rank1 [2026-10-28 22:28:35,066] INFO : Step 15900/50000 rank1 | Train Loss: 1.4020 | Train Loss Unweighted: 0.7322 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1800 | Classification: 0.1295 | Classification_binary: 0.2346 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1816 | Regression_eta: 0.0023 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0914 | Classification_binary: 0.2546 | Regression_cos_phi: 0.1106 | Regression_energy: 0.0797 | Regression_eta: 0.1011 | Regression_pt: 0.0831 | Regression_sin_phi: 0.1170 | LR: 7.94e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2696s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0933 | max_forward: 0.7823 | max_loss: 0.1445 | max_backward: 1.4740 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6508.5 | uss_MB: 5807.0 | child_rss_MB: 5047.2 | allocated_MB: 342.9 | reserved_MB: 25380.0 | inactive_split_MB: 283.5 +rank1 [2026-10-28 22:32:40,676] INFO : Step 16000/50000 rank1 | Train Loss: 1.4978 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3126 | Cardinality: 0.2093 | Classification: 0.1190 | Classification_binary: 0.2580 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1920 | Regression_eta: 0.0021 | Regression_pt: 0.1969 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2523 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0857 | Regression_eta: 0.0582 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0725 | LR: 7.91e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4907s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.7474 | max_loss: 0.1892 | max_backward: 1.4114 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.7981 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6509.3 | uss_MB: 5675.5 | child_rss_MB: 5048.3 | allocated_MB: 342.7 | reserved_MB: 25380.0 | inactive_split_MB: 292.9 +rank1 [2026-10-28 22:32:41,701] INFO : Running validation on rank1 +rank1 [2026-10-28 22:32:41,702] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 22:32:41,702] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 22:33:34,622] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:33:35,150] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 22:34:01,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 22:34:02,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 22:34:02,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 22:34:03,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 22:34:13,929] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 22:34:14,445] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:34:14,938] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 22:34:30,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 22:34:31,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 22:34:32,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 22:34:33,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 22:34:43,461] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 22:34:44,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:34:44,856] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 22:35:00,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 22:35:01,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 22:35:01,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 22:35:02,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 22:35:12,520] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 22:39:22,135] INFO : Step 16100/50000 rank1 | Train Loss: 1.3441 | Train Loss Unweighted: 0.7192 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1631 | Classification: 0.1197 | Classification_binary: 0.2058 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1931 | Regression_eta: 0.0020 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.1906 | Regression_cos_phi: 0.1028 | Regression_energy: 0.1099 | Regression_eta: 0.0482 | Regression_pt: 0.1159 | Regression_sin_phi: 0.1357 | LR: 7.89e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1525s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0584 | max_forward: 0.7257 | max_loss: 0.1421 | max_backward: 1.2590 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.0197 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6577.1 | uss_MB: 5871.7 | child_rss_MB: 5053.6 | allocated_MB: 341.3 | reserved_MB: 23342.0 | inactive_split_MB: 221.6 +rank1 [2026-10-28 22:43:09,260] INFO : Step 16200/50000 rank1 | Train Loss: 1.3737 | Train Loss Unweighted: 0.7229 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1695 | Classification: 0.1213 | Classification_binary: 0.2167 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.2455 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0904 | Regression_eta: 0.0502 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0707 | LR: 7.86e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0513s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.1062 | max_forward: 0.7366 | max_loss: 0.1469 | max_backward: 1.4999 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.6015 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6580.7 | uss_MB: 5882.6 | child_rss_MB: 5053.8 | allocated_MB: 342.5 | reserved_MB: 24304.0 | inactive_split_MB: 205.9 +rank1 [2026-10-28 22:46:54,663] INFO : Step 16300/50000 rank1 | Train Loss: 1.4671 | Train Loss Unweighted: 0.7592 | Train Loss Components: Auxiliary: 0.3008 | Cardinality: 0.1775 | Classification: 0.1342 | Classification_binary: 0.2702 | HitGrouping: 0.2148 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0023 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0926 | Classification_binary: 0.3085 | Regression_cos_phi: 0.0937 | Regression_energy: 0.0908 | Regression_eta: 0.0575 | Regression_pt: 0.0938 | Regression_sin_phi: 0.1125 | LR: 7.84e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4601s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7718 | max_loss: 0.1628 | max_backward: 1.7017 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.8513 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6574.7 | uss_MB: 5880.8 | child_rss_MB: 5053.1 | allocated_MB: 345.8 | reserved_MB: 24304.0 | inactive_split_MB: 281.0 +rank1 [2026-10-28 22:50:39,547] INFO : Step 16400/50000 rank1 | Train Loss: 1.4200 | Train Loss Unweighted: 0.7161 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.2086 | Classification: 0.1206 | Classification_binary: 0.2158 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0024 | Regression_pt: 0.1885 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0854 | Regression_eta: 0.0525 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0746 | LR: 7.81e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.8031s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7570 | max_loss: 0.1546 | max_backward: 1.2568 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5199 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1436 | Memory: rss_MB: 6571.7 | uss_MB: 5879.8 | child_rss_MB: 5049.7 | allocated_MB: 342.7 | reserved_MB: 25268.0 | inactive_split_MB: 213.6 +rank1 [2026-10-28 22:54:25,077] INFO : Step 16500/50000 rank1 | Train Loss: 1.2201 | Train Loss Unweighted: 0.6592 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1195 | Classification: 0.1029 | Classification_binary: 0.1835 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1836 | Regression_eta: 0.0021 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0751 | Classification_binary: 0.1588 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0925 | Regression_eta: 0.0809 | Regression_pt: 0.0958 | Regression_sin_phi: 0.0588 | LR: 7.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3271s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0128 | max_forward: 0.7866 | max_loss: 0.2200 | max_backward: 1.9590 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.4913 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6572.2 | uss_MB: 5885.7 | child_rss_MB: 5046.8 | allocated_MB: 343.6 | reserved_MB: 26238.0 | inactive_split_MB: 248.8 +rank1 [2026-10-28 22:58:25,296] INFO : Step 16600/50000 rank1 | Train Loss: 1.4335 | Train Loss Unweighted: 0.7657 | Train Loss Components: Auxiliary: 0.2949 | Cardinality: 0.1698 | Classification: 0.1386 | Classification_binary: 0.2312 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1941 | Regression_eta: 0.0021 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0908 | Classification_binary: 0.2446 | Regression_cos_phi: 0.1231 | Regression_energy: 0.1038 | Regression_eta: 0.0598 | Regression_pt: 0.1085 | Regression_sin_phi: 0.1114 | LR: 7.77e-04 | DataLoad Time: 0.0150s | Model Forward Time: 6.0006s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0139 | max_forward: 1.0329 | max_loss: 0.3279 | max_backward: 2.5489 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.5244 | max_tensorboard_flush: 0.0014 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6575.3 | uss_MB: 4925.2 | child_rss_MB: 5045.0 | allocated_MB: 344.2 | reserved_MB: 27228.0 | inactive_split_MB: 219.6 +rank1 [2026-10-28 23:02:38,763] INFO : Step 16700/50000 rank1 | Train Loss: 1.4062 | Train Loss Unweighted: 0.7378 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1840 | Classification: 0.1203 | Classification_binary: 0.2439 | HitGrouping: 0.2108 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1839 | Regression_eta: 0.0020 | Regression_pt: 0.1855 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0849 | Classification_binary: 0.2404 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0862 | Regression_eta: 0.0555 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0947 | LR: 7.74e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0466s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7336 | max_loss: 0.2460 | max_backward: 1.6747 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6577.4 | uss_MB: 5888.0 | child_rss_MB: 5049.7 | allocated_MB: 344.6 | reserved_MB: 27228.0 | inactive_split_MB: 279.4 +rank1 [2026-10-28 23:06:47,051] INFO : Step 16800/50000 rank1 | Train Loss: 1.3172 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1533 | Classification: 0.1173 | Classification_binary: 0.1949 | HitGrouping: 0.2175 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1819 | Regression_eta: 0.0027 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1721 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0817 | Regression_eta: 0.0643 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0816 | LR: 7.72e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.6846s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0962 | max_forward: 0.7772 | max_loss: 0.1423 | max_backward: 1.3121 | max_optimizer_step: 0.2117 | max_tensorboard_logging: 0.4865 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6580.1 | uss_MB: 5886.7 | child_rss_MB: 5043.1 | allocated_MB: 342.1 | reserved_MB: 27250.0 | inactive_split_MB: 200.8 +rank1 [2026-10-28 23:11:05,115] INFO : Step 16900/50000 rank1 | Train Loss: 1.3132 | Train Loss Unweighted: 0.6868 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1474 | Classification: 0.1260 | Classification_binary: 0.1988 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1779 | Regression_eta: 0.0021 | Regression_pt: 0.1798 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0695 | Regression_eta: 0.0433 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0760 | LR: 7.69e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.7602s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7203 | max_loss: 0.1660 | max_backward: 1.1458 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6581.3 | uss_MB: 5889.6 | child_rss_MB: 5044.4 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 193.9 +rank1 [2026-10-28 23:15:09,202] INFO : Step 17000/50000 rank1 | Train Loss: 1.3575 | Train Loss Unweighted: 0.7115 | Train Loss Components: Auxiliary: 0.2906 | Cardinality: 0.1781 | Classification: 0.1179 | Classification_binary: 0.2325 | HitGrouping: 0.2089 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0018 | Regression_pt: 0.1792 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0873 | Classification_binary: 0.2213 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0839 | Regression_eta: 0.0377 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0585 | LR: 7.67e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4168s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.9008 | max_loss: 0.1474 | max_backward: 1.9229 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6575.6 | uss_MB: 5894.0 | child_rss_MB: 5025.6 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 278.9 +rank1 [2026-10-28 23:15:10,040] INFO : Running validation on rank1 +rank1 [2026-10-28 23:15:10,040] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 23:15:10,040] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 23:16:00,503] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:16:01,060] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 23:16:27,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 23:16:28,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 23:16:29,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 23:16:30,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 23:16:40,600] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 23:16:42,017] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:16:42,520] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 23:16:58,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 23:16:59,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 23:17:00,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 23:17:01,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 23:17:11,305] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 23:17:11,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:17:12,023] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 23:17:27,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 23:17:28,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 23:17:28,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 23:17:29,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 23:17:39,412] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-28 23:21:59,514] INFO : Step 17100/50000 rank1 | Train Loss: 1.3790 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1704 | Classification: 0.1078 | Classification_binary: 0.2095 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1968 | Regression_eta: 0.0020 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0916 | Classification_binary: 0.1850 | Regression_cos_phi: 0.1155 | Regression_energy: 0.1125 | Regression_eta: 0.0591 | Regression_pt: 0.1191 | Regression_sin_phi: 0.1192 | LR: 7.64e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9167s | Phase Times: max_pre_step_barrier: 0.0234 | max_batch_to_device: 0.0065 | max_forward: 0.8292 | max_loss: 0.1455 | max_backward: 1.4833 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6666.8 | uss_MB: 5307.9 | child_rss_MB: 5009.6 | allocated_MB: 343.8 | reserved_MB: 22688.0 | inactive_split_MB: 254.6 +rank1 [2026-10-28 23:26:06,862] INFO : Step 17200/50000 rank1 | Train Loss: 1.2786 | Train Loss Unweighted: 0.6598 | Train Loss Components: Auxiliary: 0.2576 | Cardinality: 0.1378 | Classification: 0.1161 | Classification_binary: 0.1910 | HitGrouping: 0.2191 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0025 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1924 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0922 | Regression_eta: 0.0433 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0877 | LR: 7.62e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8626s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6470 | max_loss: 0.2505 | max_backward: 1.4380 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6320 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6664.1 | uss_MB: 5982.7 | child_rss_MB: 5028.0 | allocated_MB: 344.1 | reserved_MB: 22688.0 | inactive_split_MB: 295.9 +rank1 [2026-10-28 23:30:05,515] INFO : Step 17300/50000 rank1 | Train Loss: 1.3571 | Train Loss Unweighted: 0.6976 | Train Loss Components: Auxiliary: 0.2789 | Cardinality: 0.1665 | Classification: 0.1193 | Classification_binary: 0.2287 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1724 | Regression_eta: 0.0021 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0780 | Classification_binary: 0.2241 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0696 | Regression_eta: 0.0485 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0900 | LR: 7.59e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5436s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7276 | max_loss: 0.2520 | max_backward: 1.8579 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8525 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6666.1 | uss_MB: 5121.5 | child_rss_MB: 5026.9 | allocated_MB: 344.2 | reserved_MB: 23616.0 | inactive_split_MB: 266.5 +rank1 [2026-10-28 23:34:25,421] INFO : Step 17400/50000 rank1 | Train Loss: 1.3668 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2811 | Cardinality: 0.1760 | Classification: 0.1195 | Classification_binary: 0.2351 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1732 | Regression_eta: 0.0019 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2366 | Regression_cos_phi: 0.0935 | Regression_energy: 0.0750 | Regression_eta: 0.0467 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0898 | LR: 7.57e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4952s | Phase Times: max_pre_step_barrier: 0.1038 | max_batch_to_device: 0.0306 | max_forward: 0.7782 | max_loss: 0.2429 | max_backward: 1.3743 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.8348 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0757 | Memory: rss_MB: 6667.1 | uss_MB: 5984.6 | child_rss_MB: 5026.4 | allocated_MB: 343.3 | reserved_MB: 24604.0 | inactive_split_MB: 262.4 +rank1 [2026-10-28 23:38:51,401] INFO : Step 17500/50000 rank1 | Train Loss: 1.3114 | Train Loss Unweighted: 0.6526 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.1481 | Classification: 0.1145 | Classification_binary: 0.1884 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1714 | Regression_eta: 0.0024 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0781 | Regression_eta: 0.0424 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0548 | LR: 7.54e-04 | DataLoad Time: 0.0229s | Model Forward Time: 3.9139s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.6394 | max_loss: 0.1427 | max_backward: 1.2079 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.8375 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 6667.1 | uss_MB: 5986.2 | child_rss_MB: 5023.0 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 240.0 +rank1 [2026-10-28 23:43:03,144] INFO : Step 17600/50000 rank1 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2688 | Cardinality: 0.1362 | Classification: 0.1273 | Classification_binary: 0.2065 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0019 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2035 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0951 | Regression_eta: 0.0386 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0685 | LR: 7.52e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6102s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.1120 | max_forward: 1.2883 | max_loss: 0.2624 | max_backward: 1.5380 | max_optimizer_step: 0.2147 | max_tensorboard_logging: 0.7644 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 6665.9 | uss_MB: 5985.5 | child_rss_MB: 5028.5 | allocated_MB: 344.2 | reserved_MB: 25596.0 | inactive_split_MB: 237.9 +rank1 [2026-10-28 23:47:02,149] INFO : Step 17700/50000 rank1 | Train Loss: 1.3491 | Train Loss Unweighted: 0.6939 | Train Loss Components: Auxiliary: 0.2835 | Cardinality: 0.1622 | Classification: 0.1186 | Classification_binary: 0.2310 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0022 | Regression_pt: 0.1701 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2585 | Regression_cos_phi: 0.1397 | Regression_energy: 0.0716 | Regression_eta: 0.0707 | Regression_pt: 0.0732 | Regression_sin_phi: 0.1137 | LR: 7.49e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.6544s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0704 | max_forward: 0.9841 | max_loss: 0.1994 | max_backward: 1.8911 | max_optimizer_step: 0.2227 | max_tensorboard_logging: 0.5849 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6667.9 | uss_MB: 5989.0 | child_rss_MB: 5031.2 | allocated_MB: 345.5 | reserved_MB: 25596.0 | inactive_split_MB: 287.6 +rank1 [2026-10-28 23:51:17,534] INFO : Step 17800/50000 rank1 | Train Loss: 1.3063 | Train Loss Unweighted: 0.6692 | Train Loss Components: Auxiliary: 0.2664 | Cardinality: 0.1579 | Classification: 0.1042 | Classification_binary: 0.2130 | HitGrouping: 0.2150 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1741 | Regression_eta: 0.0019 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0859 | Regression_eta: 0.0564 | Regression_pt: 0.0896 | Regression_sin_phi: 0.1148 | LR: 7.47e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8695s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.7351 | max_loss: 0.1686 | max_backward: 1.4616 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5023 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 6666.0 | uss_MB: 5988.8 | child_rss_MB: 5032.9 | allocated_MB: 341.0 | reserved_MB: 25596.0 | inactive_split_MB: 197.8 +rank1 [2026-10-28 23:55:22,791] INFO : Step 17900/50000 rank1 | Train Loss: 1.2213 | Train Loss Unweighted: 0.6433 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1427 | Classification: 0.1147 | Classification_binary: 0.1725 | HitGrouping: 0.2070 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.2070 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0741 | Regression_eta: 0.0396 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0625 | LR: 7.44e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8184s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7249 | max_loss: 0.1579 | max_backward: 1.1981 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6654.3 | uss_MB: 5990.9 | child_rss_MB: 5033.3 | allocated_MB: 341.6 | reserved_MB: 25596.0 | inactive_split_MB: 297.9 +rank1 [2026-10-28 23:59:20,282] INFO : Step 18000/50000 rank1 | Train Loss: 1.3597 | Train Loss Unweighted: 0.7083 | Train Loss Components: Auxiliary: 0.2781 | Cardinality: 0.1599 | Classification: 0.1247 | Classification_binary: 0.2275 | HitGrouping: 0.2122 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0021 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.2716 | Regression_cos_phi: 0.0876 | Regression_energy: 0.0905 | Regression_eta: 0.0561 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0867 | LR: 7.42e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.6438s | Phase Times: max_pre_step_barrier: 0.0816 | max_batch_to_device: 0.0068 | max_forward: 0.8980 | max_loss: 0.1586 | max_backward: 1.9185 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.7546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6654.3 | uss_MB: 5995.0 | child_rss_MB: 5035.3 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 254.8 +rank1 [2026-10-28 23:59:21,238] INFO : Running validation on rank1 +rank1 [2026-10-28 23:59:21,238] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 23:59:21,238] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 00:00:12,211] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:00:12,739] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 00:00:39,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 00:00:40,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 00:00:41,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 00:00:42,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 00:00:52,312] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 00:00:52,802] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:00:53,359] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 00:01:09,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 00:01:10,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 00:01:11,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 00:01:12,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 00:01:22,160] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 00:01:22,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:01:23,291] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 00:01:38,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 00:01:39,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 00:01:40,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 00:01:41,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 00:01:51,016] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 00:05:57,253] INFO : Step 18100/50000 rank1 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7216 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1666 | Classification: 0.1199 | Classification_binary: 0.2340 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1807 | Regression_eta: 0.0024 | Regression_pt: 0.1824 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1924 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0869 | Regression_eta: 0.0643 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0809 | LR: 7.39e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.5275s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.9127 | max_loss: 0.4546 | max_backward: 1.7406 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.7463 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7789.9 | uss_MB: 7130.3 | child_rss_MB: 5031.5 | allocated_MB: 344.6 | reserved_MB: 22256.0 | inactive_split_MB: 291.4 +rank1 [2026-10-29 00:09:47,529] INFO : Step 18200/50000 rank1 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6468 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1359 | Classification: 0.1208 | Classification_binary: 0.1803 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1700 | Regression_eta: 0.0020 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0897 | Classification_binary: 0.2001 | Regression_cos_phi: 0.1203 | Regression_energy: 0.0772 | Regression_eta: 0.0882 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0771 | LR: 7.36e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7613s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7114 | max_loss: 0.2369 | max_backward: 1.1051 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6784.1 | uss_MB: 6119.0 | child_rss_MB: 5031.9 | allocated_MB: 341.2 | reserved_MB: 23772.0 | inactive_split_MB: 170.7 +rank1 [2026-10-29 00:13:32,739] INFO : Step 18300/50000 rank1 | Train Loss: 1.3100 | Train Loss Unweighted: 0.6891 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1522 | Classification: 0.1230 | Classification_binary: 0.1938 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1834 | Regression_eta: 0.0019 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1994 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0960 | Regression_eta: 0.0497 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0849 | LR: 7.34e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.1476s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7219 | max_loss: 0.1491 | max_backward: 1.5825 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.7390 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 6786.1 | uss_MB: 6124.1 | child_rss_MB: 5031.6 | allocated_MB: 344.4 | reserved_MB: 24778.0 | inactive_split_MB: 234.4 +rank1 [2026-10-29 00:17:16,846] INFO : Step 18400/50000 rank1 | Train Loss: 1.3667 | Train Loss Unweighted: 0.7337 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1448 | Classification: 0.1277 | Classification_binary: 0.2393 | HitGrouping: 0.2095 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1805 | Regression_eta: 0.0022 | Regression_pt: 0.1818 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2212 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0755 | Regression_eta: 0.0457 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0640 | LR: 7.31e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4870s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1744 | max_backward: 1.7308 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6782.0 | uss_MB: 6123.4 | child_rss_MB: 5027.3 | allocated_MB: 344.6 | reserved_MB: 24778.0 | inactive_split_MB: 281.4 +rank1 [2026-10-29 00:21:03,776] INFO : Step 18500/50000 rank1 | Train Loss: 1.2528 | Train Loss Unweighted: 0.6816 | Train Loss Components: Auxiliary: 0.2631 | Cardinality: 0.1350 | Classification: 0.1140 | Classification_binary: 0.1973 | HitGrouping: 0.2066 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1821 | Regression_eta: 0.0017 | Regression_pt: 0.1847 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2162 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0802 | Regression_eta: 0.0448 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0790 | LR: 7.29e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2632s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0113 | max_forward: 0.8281 | max_loss: 0.2418 | max_backward: 1.4683 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7561 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1431 | Memory: rss_MB: 6783.0 | uss_MB: 6122.8 | child_rss_MB: 5026.7 | allocated_MB: 343.3 | reserved_MB: 24778.0 | inactive_split_MB: 226.5 +rank1 [2026-10-29 00:24:44,690] INFO : Step 18600/50000 rank1 | Train Loss: 1.1881 | Train Loss Unweighted: 0.6335 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1331 | Classification: 0.1084 | Classification_binary: 0.1723 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1746 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1962 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0846 | Regression_eta: 0.0412 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0786 | LR: 7.26e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8512s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7299 | max_loss: 0.2279 | max_backward: 1.0799 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8080 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6782.4 | uss_MB: 6120.8 | child_rss_MB: 5022.4 | allocated_MB: 341.0 | reserved_MB: 24778.0 | inactive_split_MB: 229.3 +rank1 [2026-10-29 00:28:24,928] INFO : Step 18700/50000 rank1 | Train Loss: 1.3122 | Train Loss Unweighted: 0.6534 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1756 | Classification: 0.1225 | Classification_binary: 0.2098 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1571 | Regression_eta: 0.0020 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2443 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0911 | Regression_eta: 0.0579 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0899 | LR: 7.24e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.6268s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9129 | max_loss: 0.1606 | max_backward: 1.9397 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6785.3 | uss_MB: 6128.9 | child_rss_MB: 5021.8 | allocated_MB: 344.8 | reserved_MB: 24778.0 | inactive_split_MB: 239.8 +rank1 [2026-10-29 00:32:05,525] INFO : Step 18800/50000 rank1 | Train Loss: 1.2696 | Train Loss Unweighted: 0.6643 | Train Loss Components: Auxiliary: 0.2674 | Cardinality: 0.1602 | Classification: 0.1070 | Classification_binary: 0.2200 | HitGrouping: 0.2058 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1652 | Regression_eta: 0.0018 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0681 | Regression_eta: 0.0430 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0430 | LR: 7.21e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6469s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7444 | max_loss: 0.2527 | max_backward: 1.7167 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.9324 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 6785.1 | uss_MB: 6124.1 | child_rss_MB: 5021.8 | allocated_MB: 345.4 | reserved_MB: 25044.0 | inactive_split_MB: 284.4 +rank1 [2026-10-29 00:35:49,544] INFO : Step 18900/50000 rank1 | Train Loss: 1.1718 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1382 | Classification: 0.0996 | Classification_binary: 0.1741 | HitGrouping: 0.2036 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1652 | Regression_eta: 0.0017 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0742 | Regression_eta: 0.0825 | Regression_pt: 0.0801 | Regression_sin_phi: 0.1045 | LR: 7.18e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.0756s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0056 | max_forward: 0.6830 | max_loss: 0.1421 | max_backward: 1.2812 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.8931 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6786.1 | uss_MB: 6126.4 | child_rss_MB: 5006.2 | allocated_MB: 338.7 | reserved_MB: 25044.0 | inactive_split_MB: 232.6 +rank1 [2026-10-29 00:39:31,233] INFO : Step 19000/50000 rank1 | Train Loss: 1.2397 | Train Loss Unweighted: 0.6317 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1277 | Classification: 0.1188 | Classification_binary: 0.1866 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1600 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.1930 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0838 | Regression_eta: 0.0496 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0959 | LR: 7.16e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2170s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6593 | max_loss: 0.2467 | max_backward: 1.6203 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7732 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6787.7 | uss_MB: 6128.3 | child_rss_MB: 5026.8 | allocated_MB: 342.2 | reserved_MB: 25790.0 | inactive_split_MB: 229.6 +rank1 [2026-10-29 00:39:32,174] INFO : Running validation on rank1 +rank1 [2026-10-29 00:39:32,174] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 00:39:32,174] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 00:40:23,807] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:40:24,368] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 00:40:50,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 00:40:51,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 00:40:52,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 00:40:53,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 00:41:03,193] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 00:41:03,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:41:04,089] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 00:41:20,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 00:41:21,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 00:41:21,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 00:41:22,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 00:41:32,937] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 00:41:33,291] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:41:33,791] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 00:41:49,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 00:41:49,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 00:41:50,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 00:41:51,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 00:42:01,235] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 00:46:01,996] INFO : Step 19100/50000 rank1 | Train Loss: 1.3562 | Train Loss Unweighted: 0.6933 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1644 | Classification: 0.1204 | Classification_binary: 0.2426 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1627 | Regression_eta: 0.0023 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2209 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0672 | Regression_eta: 0.0411 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0530 | LR: 7.13e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.5915s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7796 | max_loss: 0.1634 | max_backward: 1.6942 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.8232 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 8027.6 | uss_MB: 7365.9 | child_rss_MB: 5027.2 | allocated_MB: 345.2 | reserved_MB: 23900.0 | inactive_split_MB: 240.8 +rank1 [2026-10-29 00:49:48,721] INFO : Step 19200/50000 rank1 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5946 | Train Loss Components: Auxiliary: 0.2384 | Cardinality: 0.1391 | Classification: 0.0965 | Classification_binary: 0.1984 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1472 | Regression_eta: 0.0018 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0626 | Regression_eta: 0.0416 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0446 | LR: 7.10e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9575s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7138 | max_loss: 0.2391 | max_backward: 1.2379 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.7395 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6876.1 | uss_MB: 6216.1 | child_rss_MB: 5025.0 | allocated_MB: 341.3 | reserved_MB: 23900.0 | inactive_split_MB: 206.8 +rank1 [2026-10-29 00:53:30,060] INFO : Step 19300/50000 rank1 | Train Loss: 1.1768 | Train Loss Unweighted: 0.6083 | Train Loss Components: Auxiliary: 0.2422 | Cardinality: 0.1345 | Classification: 0.1035 | Classification_binary: 0.1709 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0018 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0816 | Regression_eta: 0.0529 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0755 | LR: 7.08e-04 | DataLoad Time: 0.0214s | Model Forward Time: 3.8966s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0503 | max_forward: 0.7163 | max_loss: 0.1442 | max_backward: 1.3745 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6878.1 | uss_MB: 6217.4 | child_rss_MB: 5020.2 | allocated_MB: 341.0 | reserved_MB: 23900.0 | inactive_split_MB: 218.1 +rank1 [2026-10-29 00:57:11,121] INFO : Step 19400/50000 rank1 | Train Loss: 1.4590 | Train Loss Unweighted: 0.7109 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.2328 | Classification: 0.1237 | Classification_binary: 0.2265 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1770 | Regression_eta: 0.0021 | Regression_pt: 0.1794 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0951 | Classification_binary: 0.2673 | Regression_cos_phi: 0.1239 | Regression_energy: 0.0821 | Regression_eta: 0.0692 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0983 | LR: 7.05e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.4238s | Phase Times: max_pre_step_barrier: 0.1118 | max_batch_to_device: 0.0070 | max_forward: 0.7578 | max_loss: 0.2459 | max_backward: 1.7975 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6876.3 | uss_MB: 6217.1 | child_rss_MB: 5016.1 | allocated_MB: 347.7 | reserved_MB: 24186.0 | inactive_split_MB: 276.3 +rank1 [2026-10-29 01:00:48,549] INFO : Step 19500/50000 rank1 | Train Loss: 1.3753 | Train Loss Unweighted: 0.6794 | Train Loss Components: Auxiliary: 0.2810 | Cardinality: 0.1865 | Classification: 0.1153 | Classification_binary: 0.2310 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1637 | Regression_eta: 0.0023 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.2359 | Regression_cos_phi: 0.1249 | Regression_energy: 0.0758 | Regression_eta: 0.0816 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0869 | LR: 7.02e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.8968s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7197 | max_loss: 0.2164 | max_backward: 1.3937 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.4987 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6876.3 | uss_MB: 6219.5 | child_rss_MB: 5017.7 | allocated_MB: 345.2 | reserved_MB: 24186.0 | inactive_split_MB: 275.2 +rank1 [2026-10-29 01:04:42,200] INFO : Step 19600/50000 rank1 | Train Loss: 1.2086 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1557 | Classification: 0.0942 | Classification_binary: 0.1768 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1735 | Regression_eta: 0.0015 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.1696 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0829 | Regression_eta: 0.0488 | Regression_pt: 0.0874 | Regression_sin_phi: 0.1071 | LR: 7.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7331s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6937 | max_loss: 0.1942 | max_backward: 1.2727 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6877.2 | uss_MB: 6219.0 | child_rss_MB: 5004.9 | allocated_MB: 340.6 | reserved_MB: 24186.0 | inactive_split_MB: 277.9 +rank1 [2026-10-29 01:08:29,168] INFO : Step 19700/50000 rank1 | Train Loss: 1.2115 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1478 | Classification: 0.1094 | Classification_binary: 0.1919 | HitGrouping: 0.2027 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.2074 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0668 | Regression_eta: 0.0658 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0993 | LR: 6.97e-04 | DataLoad Time: 0.0145s | Model Forward Time: 3.8692s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0914 | max_forward: 0.7091 | max_loss: 0.1429 | max_backward: 1.4681 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0652 | Memory: rss_MB: 6879.2 | uss_MB: 6222.3 | child_rss_MB: 5016.1 | allocated_MB: 344.0 | reserved_MB: 24630.0 | inactive_split_MB: 143.8 +rank1 [2026-10-29 01:12:36,832] INFO : Step 19800/50000 rank1 | Train Loss: 1.2919 | Train Loss Unweighted: 0.6584 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1453 | Classification: 0.1132 | Classification_binary: 0.2375 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0020 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2514 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0700 | Regression_eta: 0.0480 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0761 | LR: 6.94e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.4516s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0074 | max_forward: 0.7828 | max_loss: 0.2508 | max_backward: 1.8604 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6879.1 | uss_MB: 6223.6 | child_rss_MB: 5015.7 | allocated_MB: 348.0 | reserved_MB: 25634.0 | inactive_split_MB: 277.9 +rank1 [2026-10-29 01:16:27,497] INFO : Step 19900/50000 rank1 | Train Loss: 1.2010 | Train Loss Unweighted: 0.6167 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1346 | Classification: 0.0885 | Classification_binary: 0.1961 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1633 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0775 | Regression_eta: 0.0507 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0643 | LR: 6.92e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9747s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6671 | max_loss: 0.2548 | max_backward: 1.5862 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6881.1 | uss_MB: 6224.6 | child_rss_MB: 5016.9 | allocated_MB: 340.3 | reserved_MB: 25634.0 | inactive_split_MB: 201.7 +rank1 [2026-10-29 01:20:16,835] INFO : Step 20000/50000 rank1 | Train Loss: 1.2077 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1487 | Classification: 0.1030 | Classification_binary: 0.1729 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1639 | Regression_eta: 0.0019 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0698 | Regression_eta: 0.0470 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0573 | LR: 6.89e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8142s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.6570 | max_loss: 0.1550 | max_backward: 1.4848 | max_optimizer_step: 0.2205 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6884.1 | uss_MB: 6226.8 | child_rss_MB: 5009.1 | allocated_MB: 341.8 | reserved_MB: 26642.0 | inactive_split_MB: 237.7 +rank1 [2026-10-29 01:20:17,835] INFO : Running validation on rank1 +rank1 [2026-10-29 01:20:17,835] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 01:20:17,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 01:21:07,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:21:08,247] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 01:21:34,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 01:21:35,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 01:21:36,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 01:21:37,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 01:21:47,732] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 01:21:48,205] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:21:48,733] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 01:22:04,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 01:22:05,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 01:22:06,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 01:22:07,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 01:22:17,280] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 01:22:18,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:22:18,870] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 01:22:34,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 01:22:35,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 01:22:35,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 01:22:36,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 01:22:46,397] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 01:26:57,041] INFO : Step 20100/50000 rank1 | Train Loss: 1.3825 | Train Loss Unweighted: 0.6739 | Train Loss Components: Auxiliary: 0.2830 | Cardinality: 0.1854 | Classification: 0.1189 | Classification_binary: 0.2349 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1561 | Regression_eta: 0.0022 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.2533 | Regression_cos_phi: 0.1088 | Regression_energy: 0.0714 | Regression_eta: 0.0687 | Regression_pt: 0.0759 | Regression_sin_phi: 0.1140 | LR: 6.86e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.2482s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.9496 | max_loss: 0.1965 | max_backward: 1.6425 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.5117 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1294 | Memory: rss_MB: 8120.6 | uss_MB: 7458.3 | child_rss_MB: 5010.7 | allocated_MB: 344.8 | reserved_MB: 21328.0 | inactive_split_MB: 198.7 +rank1 [2026-10-29 01:30:57,230] INFO : Step 20200/50000 rank1 | Train Loss: 1.3220 | Train Loss Unweighted: 0.6624 | Train Loss Components: Auxiliary: 0.2728 | Cardinality: 0.1545 | Classification: 0.1109 | Classification_binary: 0.2232 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1618 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2097 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0701 | Regression_eta: 0.0394 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0580 | LR: 6.84e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8620s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.9337 | max_loss: 0.2287 | max_backward: 1.4619 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6969.7 | uss_MB: 6305.7 | child_rss_MB: 5014.0 | allocated_MB: 344.4 | reserved_MB: 21328.0 | inactive_split_MB: 287.4 +rank1 [2026-10-29 01:34:46,196] INFO : Step 20300/50000 rank1 | Train Loss: 1.2742 | Train Loss Unweighted: 0.6629 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1615 | Classification: 0.1132 | Classification_binary: 0.1937 | HitGrouping: 0.2129 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1761 | Regression_eta: 0.0018 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.1707 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0907 | Regression_eta: 0.0602 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0621 | LR: 6.81e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8212s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6857 | max_loss: 0.2423 | max_backward: 1.2822 | max_optimizer_step: 0.2837 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0419 | Memory: rss_MB: 6967.2 | uss_MB: 6303.7 | child_rss_MB: 5019.2 | allocated_MB: 340.8 | reserved_MB: 21826.0 | inactive_split_MB: 239.4 +rank1 [2026-10-29 01:38:43,604] INFO : Step 20400/50000 rank1 | Train Loss: 1.2743 | Train Loss Unweighted: 0.6521 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1507 | Classification: 0.1279 | Classification_binary: 0.1990 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1604 | Regression_eta: 0.0019 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1089 | Classification_binary: 0.1972 | Regression_cos_phi: 0.1332 | Regression_energy: 0.0688 | Regression_eta: 0.0625 | Regression_pt: 0.0707 | Regression_sin_phi: 0.1049 | LR: 6.78e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.7478s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0283 | max_forward: 1.3648 | max_loss: 0.1816 | max_backward: 1.7875 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 6969.3 | uss_MB: 6306.4 | child_rss_MB: 5010.2 | allocated_MB: 343.0 | reserved_MB: 22154.0 | inactive_split_MB: 243.5 +rank1 [2026-10-29 01:42:48,353] INFO : Step 20500/50000 rank1 | Train Loss: 1.2712 | Train Loss Unweighted: 0.6509 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1409 | Classification: 0.1147 | Classification_binary: 0.2202 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1558 | Regression_eta: 0.0019 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0815 | Classification_binary: 0.2480 | Regression_cos_phi: 0.0981 | Regression_energy: 0.0804 | Regression_eta: 0.0789 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0922 | LR: 6.76e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2551s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7205 | max_loss: 0.2283 | max_backward: 1.6623 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.6254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0704 | Memory: rss_MB: 6969.0 | uss_MB: 6306.9 | child_rss_MB: 5000.2 | allocated_MB: 345.0 | reserved_MB: 22154.0 | inactive_split_MB: 208.2 +rank1 [2026-10-29 01:46:45,623] INFO : Step 20600/50000 rank1 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6338 | Train Loss Components: Auxiliary: 0.2625 | Cardinality: 0.1623 | Classification: 0.1101 | Classification_binary: 0.1900 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0018 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0822 | Regression_eta: 0.0501 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0482 | LR: 6.73e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.9343s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 1.1520 | max_loss: 0.2849 | max_backward: 2.1471 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.6141 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6963.6 | uss_MB: 6301.3 | child_rss_MB: 5002.5 | allocated_MB: 341.4 | reserved_MB: 22154.0 | inactive_split_MB: 183.5 +rank1 [2026-10-29 01:50:54,217] INFO : Step 20700/50000 rank1 | Train Loss: 1.1613 | Train Loss Unweighted: 0.6066 | Train Loss Components: Auxiliary: 0.2395 | Cardinality: 0.1327 | Classification: 0.1061 | Classification_binary: 0.1677 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0016 | Regression_pt: 0.1666 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0831 | Classification_binary: 0.1847 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0791 | Regression_eta: 0.0557 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0865 | LR: 6.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5972s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6276 | max_loss: 0.2330 | max_backward: 1.4954 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6965.5 | uss_MB: 5936.8 | child_rss_MB: 5004.9 | allocated_MB: 341.0 | reserved_MB: 22154.0 | inactive_split_MB: 285.0 +rank1 [2026-10-29 01:55:01,768] INFO : Step 20800/50000 rank1 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1493 | Classification: 0.1205 | Classification_binary: 0.2020 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.2360 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0607 | Regression_eta: 0.0829 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0925 | LR: 6.67e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.2093s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.7679 | max_loss: 0.1604 | max_backward: 1.7694 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.4914 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6972.0 | uss_MB: 6311.7 | child_rss_MB: 5027.1 | allocated_MB: 344.1 | reserved_MB: 22154.0 | inactive_split_MB: 258.7 +rank1 [2026-10-29 01:59:06,393] INFO : Step 20900/50000 rank1 | Train Loss: 1.2934 | Train Loss Unweighted: 0.6525 | Train Loss Components: Auxiliary: 0.2719 | Cardinality: 0.1620 | Classification: 0.1167 | Classification_binary: 0.2121 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1591 | Regression_eta: 0.0021 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0737 | Regression_eta: 0.0418 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0430 | LR: 6.65e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1333s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7787 | max_loss: 0.1747 | max_backward: 1.6678 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 6974.0 | uss_MB: 5772.6 | child_rss_MB: 5037.2 | allocated_MB: 343.6 | reserved_MB: 23208.0 | inactive_split_MB: 296.4 +rank1 [2026-10-29 02:03:08,877] INFO : Step 21000/50000 rank1 | Train Loss: 1.1402 | Train Loss Unweighted: 0.5874 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1236 | Classification: 0.0967 | Classification_binary: 0.1701 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1583 | Regression_eta: 0.0017 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0608 | Regression_eta: 0.0479 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0583 | LR: 6.62e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.8977s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6629 | max_loss: 0.1833 | max_backward: 1.4689 | max_optimizer_step: 0.2206 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6973.2 | uss_MB: 6313.8 | child_rss_MB: 5014.5 | allocated_MB: 340.4 | reserved_MB: 23208.0 | inactive_split_MB: 264.1 +rank1 [2026-10-29 02:03:09,906] INFO : Running validation on rank1 +rank1 [2026-10-29 02:03:09,906] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 02:03:09,906] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 02:04:00,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:04:01,076] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 02:04:27,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 02:04:28,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 02:04:29,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 02:04:30,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 02:04:40,125] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 02:04:40,804] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:04:41,351] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 02:04:57,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 02:04:58,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 02:04:59,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 02:05:00,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 02:05:10,030] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 02:05:10,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:05:11,301] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 02:05:27,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 02:05:27,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 02:05:28,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 02:05:32,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 02:05:42,868] INFO : Time taken to make predictions on device 1 is: 0.29 min +rank1 [2026-10-29 02:10:06,179] INFO : Step 21100/50000 rank1 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6632 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1574 | Classification: 0.1277 | Classification_binary: 0.2026 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0024 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0825 | Classification_binary: 0.1761 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0768 | Regression_eta: 0.0698 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0696 | LR: 6.59e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2648s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7159 | max_loss: 0.1411 | max_backward: 1.5805 | max_optimizer_step: 0.3396 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8247.8 | uss_MB: 7584.2 | child_rss_MB: 5046.5 | allocated_MB: 343.8 | reserved_MB: 21372.0 | inactive_split_MB: 255.4 +rank1 [2026-10-29 02:10:38,255] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank1 [2026-10-29 02:10:38,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-29 02:14:15,483] INFO : Step 21200/50000 rank1 | Train Loss: 1.2354 | Train Loss Unweighted: 0.6366 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1422 | Classification: 0.1056 | Classification_binary: 0.1876 | HitGrouping: 0.2030 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0017 | Regression_pt: 0.1709 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.1819 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0942 | Regression_eta: 0.0570 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0846 | LR: 6.56e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.1583s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7385 | max_loss: 0.2183 | max_backward: 1.4276 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.7483 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7137.9 | uss_MB: 6469.9 | child_rss_MB: 5019.5 | allocated_MB: 342.5 | reserved_MB: 23286.0 | inactive_split_MB: 273.0 +rank1 [2026-10-29 02:18:17,991] INFO : Step 21300/50000 rank1 | Train Loss: 1.1511 | Train Loss Unweighted: 0.5782 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1240 | Classification: 0.1021 | Classification_binary: 0.1706 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1505 | Regression_eta: 0.0017 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1478 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0728 | Regression_eta: 0.0324 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0547 | LR: 6.54e-04 | DataLoad Time: 0.0240s | Model Forward Time: 4.4905s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0290 | max_forward: 1.3238 | max_loss: 0.2105 | max_backward: 1.6672 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.6878 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7140.4 | uss_MB: 6475.6 | child_rss_MB: 5012.2 | allocated_MB: 343.8 | reserved_MB: 23286.0 | inactive_split_MB: 272.8 +rank1 [2026-10-29 02:22:14,589] INFO : Step 21400/50000 rank1 | Train Loss: 1.2993 | Train Loss Unweighted: 0.6458 | Train Loss Components: Auxiliary: 0.2724 | Cardinality: 0.1796 | Classification: 0.1204 | Classification_binary: 0.2158 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1518 | Regression_eta: 0.0018 | Regression_pt: 0.1542 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2417 | Regression_cos_phi: 0.0905 | Regression_energy: 0.0686 | Regression_eta: 0.0489 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0764 | LR: 6.51e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5461s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8696 | max_loss: 0.1572 | max_backward: 1.8002 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 0.6992 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7114.7 | uss_MB: 6081.0 | child_rss_MB: 5015.7 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 201.9 +rank1 [2026-10-29 02:26:14,444] INFO : Step 21500/50000 rank1 | Train Loss: 1.2839 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2693 | Cardinality: 0.1478 | Classification: 0.1058 | Classification_binary: 0.2162 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0777 | Classification_binary: 0.2161 | Regression_cos_phi: 0.1206 | Regression_energy: 0.0923 | Regression_eta: 0.0826 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0735 | LR: 6.48e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7750s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.8684 | max_loss: 0.1577 | max_backward: 1.9355 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.7593 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0648 | Memory: rss_MB: 7121.8 | uss_MB: 6458.0 | child_rss_MB: 5014.4 | allocated_MB: 341.9 | reserved_MB: 25248.0 | inactive_split_MB: 284.1 +rank1 [2026-10-29 02:30:00,713] INFO : Step 21600/50000 rank1 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1290 | Classification: 0.1062 | Classification_binary: 0.1618 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1486 | Regression_eta: 0.0018 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0845 | Regression_eta: 0.0534 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0857 | LR: 6.45e-04 | DataLoad Time: 0.0217s | Model Forward Time: 3.9649s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0061 | max_forward: 0.7663 | max_loss: 0.1883 | max_backward: 1.4257 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.4997 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7120.6 | uss_MB: 6456.9 | child_rss_MB: 5032.2 | allocated_MB: 340.5 | reserved_MB: 25248.0 | inactive_split_MB: 234.8 +rank1 [2026-10-29 02:33:57,426] INFO : Step 21700/50000 rank1 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6283 | Train Loss Components: Auxiliary: 0.2577 | Cardinality: 0.1575 | Classification: 0.1186 | Classification_binary: 0.1853 | HitGrouping: 0.2068 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1594 | Regression_eta: 0.0022 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0751 | Regression_eta: 0.0745 | Regression_pt: 0.0777 | Regression_sin_phi: 0.1073 | LR: 6.43e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.7324s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.6796 | max_loss: 0.1417 | max_backward: 1.3881 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7122.2 | uss_MB: 6458.0 | child_rss_MB: 5034.2 | allocated_MB: 343.8 | reserved_MB: 25248.0 | inactive_split_MB: 244.5 +rank1 [2026-10-29 02:37:54,439] INFO : Step 21800/50000 rank1 | Train Loss: 1.2260 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2691 | Cardinality: 0.1439 | Classification: 0.1105 | Classification_binary: 0.2043 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1576 | Regression_eta: 0.0016 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.2162 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0733 | Regression_eta: 0.0450 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0891 | LR: 6.40e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.9214s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 0.8043 | max_loss: 0.1895 | max_backward: 2.4027 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.9442 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7124.3 | uss_MB: 5827.3 | child_rss_MB: 5037.0 | allocated_MB: 344.5 | reserved_MB: 25248.0 | inactive_split_MB: 277.8 +rank1 [2026-10-29 02:41:42,512] INFO : Step 21900/50000 rank1 | Train Loss: 1.2119 | Train Loss Unweighted: 0.5952 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1861 | Classification: 0.1033 | Classification_binary: 0.1865 | HitGrouping: 0.1989 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0015 | Regression_pt: 0.1511 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0727 | Regression_eta: 0.0519 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0422 | LR: 6.37e-04 | DataLoad Time: 0.0130s | Model Forward Time: 4.5649s | Phase Times: max_pre_step_barrier: 0.0174 | max_batch_to_device: 0.0163 | max_forward: 0.9684 | max_loss: 0.1448 | max_backward: 1.6354 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.8990 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 7124.6 | uss_MB: 6460.4 | child_rss_MB: 5033.8 | allocated_MB: 342.7 | reserved_MB: 25248.0 | inactive_split_MB: 192.3 +rank1 [2026-10-29 02:45:41,732] INFO : Step 22000/50000 rank1 | Train Loss: 1.1094 | Train Loss Unweighted: 0.5651 | Train Loss Components: Auxiliary: 0.2344 | Cardinality: 0.1248 | Classification: 0.1037 | Classification_binary: 0.1579 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1497 | Regression_eta: 0.0017 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.1828 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0674 | Regression_eta: 0.0670 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0732 | LR: 6.34e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.4209s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.6825 | max_loss: 0.1653 | max_backward: 1.4396 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.0256 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 7124.1 | uss_MB: 6463.7 | child_rss_MB: 5009.1 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 247.3 +rank1 [2026-10-29 02:45:42,810] INFO : Running validation on rank1 +rank1 [2026-10-29 02:45:42,810] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 02:45:42,810] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 02:46:37,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:46:38,046] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 02:46:57,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 02:46:58,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 02:46:59,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 02:47:00,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 02:47:10,697] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 02:47:11,416] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:47:11,959] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 02:47:27,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 02:47:28,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 02:47:29,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 02:47:30,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 02:47:40,450] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 02:47:41,459] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:47:42,031] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 02:47:57,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 02:47:58,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 02:47:59,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 02:47:59,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 02:48:09,780] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 02:52:24,844] INFO : Step 22100/50000 rank1 | Train Loss: 1.2514 | Train Loss Unweighted: 0.6219 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1530 | Classification: 0.1140 | Classification_binary: 0.2076 | HitGrouping: 0.2031 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1470 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0661 | Regression_eta: 0.0334 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0444 | LR: 6.32e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5418s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7250 | max_loss: 0.2514 | max_backward: 1.5956 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.8632 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0587 | Memory: rss_MB: 8032.9 | uss_MB: 7365.3 | child_rss_MB: 5019.4 | allocated_MB: 347.3 | reserved_MB: 23386.0 | inactive_split_MB: 278.6 +rank1 [2026-10-29 02:56:29,122] INFO : Step 22200/50000 rank1 | Train Loss: 1.2727 | Train Loss Unweighted: 0.6485 | Train Loss Components: Auxiliary: 0.2709 | Cardinality: 0.1498 | Classification: 0.1104 | Classification_binary: 0.2184 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1574 | Regression_eta: 0.0018 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.2131 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0842 | Regression_eta: 0.0471 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0783 | LR: 6.29e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.9692s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0074 | max_forward: 0.7776 | max_loss: 0.1472 | max_backward: 1.7604 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.3305 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7158.4 | uss_MB: 5092.5 | child_rss_MB: 5016.1 | allocated_MB: 342.3 | reserved_MB: 23578.0 | inactive_split_MB: 277.9 +rank1 [2026-10-29 03:00:35,125] INFO : Step 22300/50000 rank1 | Train Loss: 1.1242 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2347 | Cardinality: 0.1367 | Classification: 0.1013 | Classification_binary: 0.1688 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0683 | Regression_eta: 0.0345 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0472 | LR: 6.26e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.7792s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7697 | max_loss: 0.1538 | max_backward: 1.3553 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.4896 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 7162.9 | uss_MB: 6498.9 | child_rss_MB: 4987.3 | allocated_MB: 341.2 | reserved_MB: 23578.0 | inactive_split_MB: 233.8 +rank1 [2026-10-29 03:04:32,773] INFO : Step 22400/50000 rank1 | Train Loss: 1.2357 | Train Loss Unweighted: 0.6255 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1493 | Classification: 0.1171 | Classification_binary: 0.1885 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1575 | Regression_eta: 0.0021 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.2020 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0856 | Regression_eta: 0.0415 | Regression_pt: 0.0908 | Regression_sin_phi: 0.1119 | LR: 6.23e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9153s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0067 | max_forward: 0.6425 | max_loss: 0.1530 | max_backward: 1.4136 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.5876 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 7158.4 | uss_MB: 6496.8 | child_rss_MB: 5021.9 | allocated_MB: 342.5 | reserved_MB: 23578.0 | inactive_split_MB: 251.5 +rank1 [2026-10-29 03:08:26,684] INFO : Step 22500/50000 rank1 | Train Loss: 1.2633 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2638 | Cardinality: 0.1676 | Classification: 0.0954 | Classification_binary: 0.2181 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0020 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.2204 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0809 | Regression_eta: 0.0527 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0736 | LR: 6.20e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.4277s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1522 | max_backward: 1.5631 | max_optimizer_step: 0.3261 | max_tensorboard_logging: 0.7552 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 7160.2 | uss_MB: 6502.4 | child_rss_MB: 5011.7 | allocated_MB: 342.9 | reserved_MB: 24052.0 | inactive_split_MB: 295.1 +rank1 [2026-10-29 03:12:18,754] INFO : Step 22600/50000 rank1 | Train Loss: 1.1636 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1329 | Classification: 0.0997 | Classification_binary: 0.1889 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1493 | Regression_eta: 0.0020 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1678 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0587 | Regression_eta: 0.0752 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0574 | LR: 6.18e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9316s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7375 | max_loss: 0.2403 | max_backward: 1.4584 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7163.5 | uss_MB: 6503.3 | child_rss_MB: 5024.4 | allocated_MB: 344.2 | reserved_MB: 24590.0 | inactive_split_MB: 288.5 +rank1 [2026-10-29 03:16:20,064] INFO : Step 22700/50000 rank1 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1212 | Classification: 0.0976 | Classification_binary: 0.1566 | HitGrouping: 0.2101 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1455 | Regression_eta: 0.0015 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0669 | Regression_eta: 0.0276 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0511 | LR: 6.15e-04 | DataLoad Time: 0.0149s | Model Forward Time: 6.9710s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.6124 | max_loss: 0.1376 | max_backward: 1.1766 | max_optimizer_step: 0.2237 | max_tensorboard_logging: 3.8568 | max_tensorboard_flush: 0.0169 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 7164.2 | uss_MB: 6160.1 | child_rss_MB: 5007.8 | allocated_MB: 341.6 | reserved_MB: 25058.0 | inactive_split_MB: 256.4 +rank1 [2026-10-29 03:20:29,740] INFO : Step 22800/50000 rank1 | Train Loss: 1.1840 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2533 | Cardinality: 0.1443 | Classification: 0.1191 | Classification_binary: 0.1924 | HitGrouping: 0.1959 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1444 | Regression_eta: 0.0016 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0858 | Classification_binary: 0.2233 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0691 | Regression_eta: 0.0382 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0828 | LR: 6.12e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4525s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7010 | max_loss: 0.2312 | max_backward: 1.6853 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8726 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 7164.4 | uss_MB: 6502.8 | child_rss_MB: 5021.2 | allocated_MB: 341.7 | reserved_MB: 25058.0 | inactive_split_MB: 281.3 +rank1 [2026-10-29 03:24:21,160] INFO : Step 22900/50000 rank1 | Train Loss: 1.2680 | Train Loss Unweighted: 0.6318 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1792 | Classification: 0.1020 | Classification_binary: 0.2074 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1580 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0686 | Regression_eta: 0.0322 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0578 | LR: 6.09e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9993s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 1.1637 | max_loss: 0.4031 | max_backward: 1.8442 | max_optimizer_step: 0.3320 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7165.6 | uss_MB: 6506.2 | child_rss_MB: 5023.4 | allocated_MB: 344.8 | reserved_MB: 25058.0 | inactive_split_MB: 300.1 +rank1 [2026-10-29 03:28:05,592] INFO : Step 23000/50000 rank1 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5484 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1129 | Classification: 0.0836 | Classification_binary: 0.1519 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1547 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0424 | Regression_energy: 0.0840 | Regression_eta: 0.0340 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0453 | LR: 6.06e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7526s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0063 | max_forward: 0.6963 | max_loss: 0.2382 | max_backward: 1.2870 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 7162.6 | uss_MB: 6506.3 | child_rss_MB: 5008.7 | allocated_MB: 340.3 | reserved_MB: 26066.0 | inactive_split_MB: 218.4 +rank1 [2026-10-29 03:28:06,533] INFO : Running validation on rank1 +rank1 [2026-10-29 03:28:06,533] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 03:28:06,533] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 03:28:56,135] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:28:56,639] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 03:29:22,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 03:29:23,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 03:29:24,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 03:29:25,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 03:29:35,249] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 03:29:35,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:29:36,300] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 03:29:52,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 03:29:53,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 03:29:54,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 03:29:55,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 03:30:05,018] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 03:30:05,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:30:05,702] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 03:30:21,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 03:30:22,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 03:30:22,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 03:30:23,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 03:30:33,539] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 03:34:45,381] INFO : Step 23100/50000 rank1 | Train Loss: 1.2422 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1379 | Classification: 0.1246 | Classification_binary: 0.1795 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1482 | Regression_eta: 0.0023 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0954 | Classification_binary: 0.1901 | Regression_cos_phi: 0.1123 | Regression_energy: 0.0676 | Regression_eta: 0.1112 | Regression_pt: 0.0701 | Regression_sin_phi: 0.1175 | LR: 6.04e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8252s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7203 | max_loss: 0.1522 | max_backward: 1.1000 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.6183 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 8419.6 | uss_MB: 7758.7 | child_rss_MB: 5039.2 | allocated_MB: 344.7 | reserved_MB: 22598.0 | inactive_split_MB: 244.5 +rank1 [2026-10-29 03:38:33,525] INFO : Step 23200/50000 rank1 | Train Loss: 1.2507 | Train Loss Unweighted: 0.6359 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1400 | Classification: 0.1121 | Classification_binary: 0.2208 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1488 | Regression_eta: 0.0020 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2384 | Regression_cos_phi: 0.1198 | Regression_energy: 0.0625 | Regression_eta: 0.0605 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0782 | LR: 6.01e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.5953s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0727 | max_forward: 0.9486 | max_loss: 0.1744 | max_backward: 1.7031 | max_optimizer_step: 0.2405 | max_tensorboard_logging: 0.6717 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1109 | Memory: rss_MB: 7225.3 | uss_MB: 6564.6 | child_rss_MB: 5046.8 | allocated_MB: 343.7 | reserved_MB: 23792.0 | inactive_split_MB: 223.2 +rank1 [2026-10-29 03:42:24,564] INFO : Step 23300/50000 rank1 | Train Loss: 1.1880 | Train Loss Unweighted: 0.5927 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1464 | Classification: 0.0987 | Classification_binary: 0.1851 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1524 | Regression_eta: 0.0019 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0709 | Regression_eta: 0.0638 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0472 | LR: 5.98e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8604s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7557 | max_loss: 0.1467 | max_backward: 1.4708 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.4854 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 7231.3 | uss_MB: 6572.4 | child_rss_MB: 5045.1 | allocated_MB: 342.6 | reserved_MB: 24534.0 | inactive_split_MB: 271.6 +rank1 [2026-10-29 03:46:11,297] INFO : Step 23400/50000 rank1 | Train Loss: 1.1136 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2402 | Cardinality: 0.1202 | Classification: 0.1070 | Classification_binary: 0.1497 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1644 | Regression_eta: 0.0017 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0865 | Regression_eta: 0.0823 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0661 | LR: 5.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.6675s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.5743 | max_loss: 0.2297 | max_backward: 1.0127 | max_optimizer_step: 0.3321 | max_tensorboard_logging: 0.6182 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 7232.3 | uss_MB: 6571.7 | child_rss_MB: 5046.9 | allocated_MB: 340.5 | reserved_MB: 24848.0 | inactive_split_MB: 241.3 +rank1 [2026-10-29 03:50:07,301] INFO : Step 23500/50000 rank1 | Train Loss: 1.1601 | Train Loss Unweighted: 0.5799 | Train Loss Components: Auxiliary: 0.2477 | Cardinality: 0.1161 | Classification: 0.1057 | Classification_binary: 0.1852 | HitGrouping: 0.2029 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1416 | Regression_eta: 0.0017 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0667 | Regression_eta: 0.0410 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0866 | LR: 5.92e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4415s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8864 | max_loss: 0.1573 | max_backward: 1.9036 | max_optimizer_step: 0.2294 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7232.1 | uss_MB: 6574.4 | child_rss_MB: 5036.9 | allocated_MB: 341.9 | reserved_MB: 24848.0 | inactive_split_MB: 233.9 +rank1 [2026-10-29 03:53:52,163] INFO : Step 23600/50000 rank1 | Train Loss: 1.2402 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1447 | Classification: 0.1136 | Classification_binary: 0.2049 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1534 | Regression_eta: 0.0020 | Regression_pt: 0.1550 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0769 | Classification_binary: 0.1967 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0670 | Regression_eta: 0.0601 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0666 | LR: 5.90e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.9440s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8397 | max_loss: 0.1470 | max_backward: 1.4395 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7234.1 | uss_MB: 6575.7 | child_rss_MB: 5045.6 | allocated_MB: 342.5 | reserved_MB: 24848.0 | inactive_split_MB: 215.5 +rank1 [2026-10-29 03:57:35,779] INFO : Step 23700/50000 rank1 | Train Loss: 1.1009 | Train Loss Unweighted: 0.5682 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1218 | Classification: 0.0954 | Classification_binary: 0.1599 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1528 | Regression_eta: 0.0014 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1588 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0612 | Regression_eta: 0.0329 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0475 | LR: 5.87e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6817s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6589 | max_loss: 0.1387 | max_backward: 1.2152 | max_optimizer_step: 0.2251 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7230.0 | uss_MB: 6574.3 | child_rss_MB: 5051.4 | allocated_MB: 342.5 | reserved_MB: 25508.0 | inactive_split_MB: 226.3 +rank1 [2026-10-29 04:01:36,000] INFO : Step 23800/50000 rank1 | Train Loss: 1.1390 | Train Loss Unweighted: 0.5882 | Train Loss Components: Auxiliary: 0.2427 | Cardinality: 0.1196 | Classification: 0.1035 | Classification_binary: 0.1735 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1788 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0769 | Regression_eta: 0.0505 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0566 | LR: 5.84e-04 | DataLoad Time: 0.0230s | Model Forward Time: 3.7614s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.7283 | max_loss: 0.1547 | max_backward: 1.2415 | max_optimizer_step: 0.2259 | max_tensorboard_logging: 0.5270 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7231.0 | uss_MB: 6110.2 | child_rss_MB: 5050.1 | allocated_MB: 345.2 | reserved_MB: 25508.0 | inactive_split_MB: 241.8 +rank1 [2026-10-29 04:05:42,354] INFO : Step 23900/50000 rank1 | Train Loss: 1.2349 | Train Loss Unweighted: 0.6078 | Train Loss Components: Auxiliary: 0.2621 | Cardinality: 0.1596 | Classification: 0.1044 | Classification_binary: 0.2077 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1451 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2331 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0863 | Regression_eta: 0.0571 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0934 | LR: 5.81e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4879s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.9077 | max_loss: 0.1538 | max_backward: 1.9088 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7234.7 | uss_MB: 6579.6 | child_rss_MB: 5044.4 | allocated_MB: 342.1 | reserved_MB: 27474.0 | inactive_split_MB: 282.4 +rank1 [2026-10-29 04:09:40,154] INFO : Step 24000/50000 rank1 | Train Loss: 1.1074 | Train Loss Unweighted: 0.5714 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1170 | Classification: 0.1007 | Classification_binary: 0.1792 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1431 | Regression_eta: 0.0015 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1522 | Regression_cos_phi: 0.0331 | Regression_energy: 0.0725 | Regression_eta: 0.0288 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0435 | LR: 5.78e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.2207s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 1.1428 | max_loss: 0.2613 | max_backward: 2.5767 | max_optimizer_step: 0.2298 | max_tensorboard_logging: 0.5998 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7236.7 | uss_MB: 6577.2 | child_rss_MB: 5038.1 | allocated_MB: 342.1 | reserved_MB: 27476.0 | inactive_split_MB: 218.6 +rank1 [2026-10-29 04:09:41,046] INFO : Running validation on rank1 +rank1 [2026-10-29 04:09:41,046] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 04:09:41,046] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 04:10:38,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:10:39,335] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 04:11:05,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 04:11:06,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 04:11:07,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 04:11:10,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 04:11:20,241] INFO : Time taken to make predictions on device 1 is: 0.28 min +rank1 [2026-10-29 04:11:20,671] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:11:21,254] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 04:11:37,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 04:11:38,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 04:11:39,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 04:11:40,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 04:11:50,090] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:11:50,532] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:11:51,029] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 04:12:06,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 04:12:06,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 04:12:07,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 04:12:08,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 04:12:18,324] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 04:16:38,288] INFO : Step 24100/50000 rank1 | Train Loss: 1.1244 | Train Loss Unweighted: 0.5584 | Train Loss Components: Auxiliary: 0.2373 | Cardinality: 0.1285 | Classification: 0.0978 | Classification_binary: 0.1606 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.1951 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0832 | Regression_eta: 0.0577 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0867 | LR: 5.75e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1994s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0860 | max_forward: 0.7361 | max_loss: 0.1432 | max_backward: 1.4268 | max_optimizer_step: 0.3451 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 8228.7 | uss_MB: 7570.1 | child_rss_MB: 5001.0 | allocated_MB: 343.2 | reserved_MB: 21608.0 | inactive_split_MB: 269.9 +rank1 [2026-10-29 04:20:36,777] INFO : Step 24200/50000 rank1 | Train Loss: 1.3067 | Train Loss Unweighted: 0.6341 | Train Loss Components: Auxiliary: 0.2739 | Cardinality: 0.1760 | Classification: 0.1145 | Classification_binary: 0.2101 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0018 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2052 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0685 | Regression_eta: 0.0445 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0668 | LR: 5.73e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.0800s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1066 | max_forward: 1.0531 | max_loss: 0.2796 | max_backward: 2.3820 | max_optimizer_step: 0.2441 | max_tensorboard_logging: 0.6187 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 8233.6 | uss_MB: 7574.6 | child_rss_MB: 5010.3 | allocated_MB: 345.0 | reserved_MB: 23864.0 | inactive_split_MB: 203.0 +rank1 [2026-10-29 04:24:33,010] INFO : Step 24300/50000 rank1 | Train Loss: 1.2207 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1582 | Classification: 0.1151 | Classification_binary: 0.2013 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1450 | Regression_eta: 0.0017 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2065 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0679 | Regression_eta: 0.0440 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0706 | LR: 5.70e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3071s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7395 | max_loss: 0.2490 | max_backward: 1.4969 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.6555 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7355.3 | uss_MB: 6697.1 | child_rss_MB: 5007.1 | allocated_MB: 344.8 | reserved_MB: 23864.0 | inactive_split_MB: 284.8 +rank1 [2026-10-29 04:28:19,172] INFO : Step 24400/50000 rank1 | Train Loss: 1.1288 | Train Loss Unweighted: 0.5921 | Train Loss Components: Auxiliary: 0.2439 | Cardinality: 0.1229 | Classification: 0.1017 | Classification_binary: 0.1613 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1631 | Regression_eta: 0.0015 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1767 | Regression_cos_phi: 0.0822 | Regression_energy: 0.0829 | Regression_eta: 0.0388 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0694 | LR: 5.67e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.9883s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.6685 | max_loss: 0.1564 | max_backward: 1.4736 | max_optimizer_step: 0.2223 | max_tensorboard_logging: 0.5369 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7357.1 | uss_MB: 6698.7 | child_rss_MB: 5016.5 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 267.2 +rank1 [2026-10-29 04:32:03,440] INFO : Step 24500/50000 rank1 | Train Loss: 1.2319 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2586 | Cardinality: 0.1511 | Classification: 0.1107 | Classification_binary: 0.1876 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1540 | Regression_eta: 0.0018 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1870 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0825 | Regression_eta: 0.0454 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0753 | LR: 5.64e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1626s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0677 | max_forward: 0.7219 | max_loss: 0.1439 | max_backward: 1.4253 | max_optimizer_step: 0.3416 | max_tensorboard_logging: 0.5859 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 7356.5 | uss_MB: 6698.2 | child_rss_MB: 5010.5 | allocated_MB: 343.1 | reserved_MB: 23864.0 | inactive_split_MB: 275.8 +rank1 [2026-10-29 04:36:01,553] INFO : Step 24600/50000 rank1 | Train Loss: 1.2474 | Train Loss Unweighted: 0.6241 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1630 | Classification: 0.1074 | Classification_binary: 0.2102 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.2057 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0611 | Regression_eta: 0.0434 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0792 | LR: 5.61e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5143s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7876 | max_loss: 0.1528 | max_backward: 1.7437 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.6204 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7358.2 | uss_MB: 6700.7 | child_rss_MB: 5006.8 | allocated_MB: 344.3 | reserved_MB: 23864.0 | inactive_split_MB: 285.7 +rank1 [2026-10-29 04:39:51,298] INFO : Step 24700/50000 rank1 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1193 | Classification: 0.0990 | Classification_binary: 0.1723 | HitGrouping: 0.2019 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1471 | Regression_eta: 0.0016 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.1224 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0642 | Regression_eta: 0.0390 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0487 | LR: 5.58e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9515s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6978 | max_loss: 0.2493 | max_backward: 1.2925 | max_optimizer_step: 0.2314 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7361.3 | uss_MB: 6700.3 | child_rss_MB: 5009.8 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 223.9 +rank1 [2026-10-29 04:43:37,808] INFO : Step 24800/50000 rank1 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5554 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1264 | Classification: 0.0932 | Classification_binary: 0.1583 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2050 | Regression_cos_phi: 0.0905 | Regression_energy: 0.0883 | Regression_eta: 0.0485 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0823 | LR: 5.56e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.2782s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6582 | max_loss: 0.2533 | max_backward: 1.5858 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.6923 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1086 | Memory: rss_MB: 7365.2 | uss_MB: 6703.3 | child_rss_MB: 5005.7 | allocated_MB: 342.9 | reserved_MB: 23864.0 | inactive_split_MB: 278.6 +rank1 [2026-10-29 04:47:36,372] INFO : Step 24900/50000 rank1 | Train Loss: 1.1348 | Train Loss Unweighted: 0.5816 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1381 | Classification: 0.1126 | Classification_binary: 0.1845 | HitGrouping: 0.1937 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1403 | Regression_eta: 0.0015 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2015 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0628 | Regression_eta: 0.0474 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0690 | LR: 5.53e-04 | DataLoad Time: 0.0150s | Model Forward Time: 8.7045s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0067 | max_forward: 0.7276 | max_loss: 0.2411 | max_backward: 1.3958 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 5.1754 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1038 | Memory: rss_MB: 7366.2 | uss_MB: 6703.5 | child_rss_MB: 5007.2 | allocated_MB: 343.9 | reserved_MB: 23864.0 | inactive_split_MB: 239.7 +rank1 [2026-10-29 04:51:28,373] INFO : Step 25000/50000 rank1 | Train Loss: 1.1995 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1590 | Classification: 0.0891 | Classification_binary: 0.2009 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1421 | Regression_eta: 0.0018 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0631 | Regression_eta: 0.0365 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0551 | LR: 5.50e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2036s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7692 | max_loss: 0.2091 | max_backward: 1.3616 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6652 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1392 | Memory: rss_MB: 7366.4 | uss_MB: 6702.5 | child_rss_MB: 5014.2 | allocated_MB: 343.5 | reserved_MB: 23864.0 | inactive_split_MB: 293.8 +rank1 [2026-10-29 04:51:29,309] INFO : Running validation on rank1 +rank1 [2026-10-29 04:51:29,309] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 04:51:29,309] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 04:52:22,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:52:22,823] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 04:52:49,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 04:52:50,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 04:52:51,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 04:52:52,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 04:53:02,358] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:53:02,763] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:53:03,282] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 04:53:19,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 04:53:20,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 04:53:21,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 04:53:22,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 04:53:32,137] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:53:32,527] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:53:32,995] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 04:53:48,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 04:53:49,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 04:53:50,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 04:53:50,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 04:54:00,840] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 04:58:07,811] INFO : Step 25100/50000 rank1 | Train Loss: 1.0833 | Train Loss Unweighted: 0.5514 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1317 | Classification: 0.0920 | Classification_binary: 0.1568 | HitGrouping: 0.2008 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1491 | Regression_eta: 0.0016 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.1433 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0559 | Regression_eta: 0.0463 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0787 | LR: 5.47e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7884s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7295 | max_loss: 0.1401 | max_backward: 1.2439 | max_optimizer_step: 0.2490 | max_tensorboard_logging: 0.4940 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8607.6 | uss_MB: 7944.9 | child_rss_MB: 5042.3 | allocated_MB: 342.3 | reserved_MB: 23608.0 | inactive_split_MB: 222.3 +rank1 [2026-10-29 05:01:53,213] INFO : Step 25200/50000 rank1 | Train Loss: 1.1886 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1523 | Classification: 0.1158 | Classification_binary: 0.1800 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1512 | Regression_eta: 0.0016 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.2228 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0644 | Regression_eta: 0.0427 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0815 | LR: 5.44e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7334s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8134 | max_loss: 0.1416 | max_backward: 1.5095 | max_optimizer_step: 0.3471 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 8606.1 | uss_MB: 7945.9 | child_rss_MB: 5039.5 | allocated_MB: 343.6 | reserved_MB: 24590.0 | inactive_split_MB: 260.9 +rank1 [2026-10-29 05:05:52,267] INFO : Step 25300/50000 rank1 | Train Loss: 1.2031 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2624 | Cardinality: 0.1455 | Classification: 0.1145 | Classification_binary: 0.1966 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1448 | Regression_eta: 0.0016 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1962 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0668 | Regression_eta: 0.0402 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0771 | LR: 5.42e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5362s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.8219 | max_loss: 0.1491 | max_backward: 1.6406 | max_optimizer_step: 0.2735 | max_tensorboard_logging: 0.7204 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0488 | Memory: rss_MB: 7456.6 | uss_MB: 6796.5 | child_rss_MB: 5036.7 | allocated_MB: 341.8 | reserved_MB: 25572.0 | inactive_split_MB: 253.5 +rank1 [2026-10-29 05:09:44,971] INFO : Step 25400/50000 rank1 | Train Loss: 1.1194 | Train Loss Unweighted: 0.5655 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1346 | Classification: 0.0983 | Classification_binary: 0.1754 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1438 | Regression_eta: 0.0015 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0611 | Regression_eta: 0.0450 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0563 | LR: 5.39e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.1073s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.6986 | max_loss: 0.1530 | max_backward: 1.2606 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.8427 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7454.9 | uss_MB: 6795.9 | child_rss_MB: 5040.6 | allocated_MB: 345.2 | reserved_MB: 25572.0 | inactive_split_MB: 223.8 +rank1 [2026-10-29 05:13:34,055] INFO : Step 25500/50000 rank1 | Train Loss: 1.0289 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1068 | Classification: 0.1034 | Classification_binary: 0.1444 | HitGrouping: 0.1913 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1484 | Regression_eta: 0.0014 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1138 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0578 | Regression_eta: 0.0225 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0254 | LR: 5.36e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1591s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0554 | max_forward: 0.7008 | max_loss: 0.1433 | max_backward: 1.4571 | max_optimizer_step: 0.3349 | max_tensorboard_logging: 0.6083 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 7460.4 | uss_MB: 6802.0 | child_rss_MB: 5038.5 | allocated_MB: 345.4 | reserved_MB: 25572.0 | inactive_split_MB: 240.6 +rank1 [2026-10-29 05:17:32,215] INFO : Step 25600/50000 rank1 | Train Loss: 1.2028 | Train Loss Unweighted: 0.5968 | Train Loss Components: Auxiliary: 0.2590 | Cardinality: 0.1607 | Classification: 0.1135 | Classification_binary: 0.1926 | HitGrouping: 0.1978 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1429 | Regression_eta: 0.0016 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2010 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0621 | Regression_eta: 0.0586 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0841 | LR: 5.33e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6834s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8800 | max_loss: 0.1578 | max_backward: 1.8564 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 7459.9 | uss_MB: 6802.4 | child_rss_MB: 5035.4 | allocated_MB: 345.5 | reserved_MB: 26054.0 | inactive_split_MB: 284.5 +rank1 [2026-10-29 05:21:26,002] INFO : Step 25700/50000 rank1 | Train Loss: 1.1800 | Train Loss Unweighted: 0.5964 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1298 | Classification: 0.1086 | Classification_binary: 0.1961 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0019 | Regression_pt: 0.1444 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1681 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0672 | Regression_eta: 0.0363 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0462 | LR: 5.30e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.5222s | Phase Times: max_pre_step_barrier: 0.0086 | max_batch_to_device: 0.0137 | max_forward: 1.0340 | max_loss: 0.2483 | max_backward: 1.6704 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 0.7612 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7459.7 | uss_MB: 6801.4 | child_rss_MB: 5033.4 | allocated_MB: 343.5 | reserved_MB: 26054.0 | inactive_split_MB: 270.8 +rank1 [2026-10-29 05:25:16,514] INFO : Step 25800/50000 rank1 | Train Loss: 1.1337 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2388 | Cardinality: 0.1264 | Classification: 0.0898 | Classification_binary: 0.1617 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0019 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0613 | Regression_eta: 0.0513 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0780 | LR: 5.27e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9355s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0986 | max_forward: 0.8129 | max_loss: 0.1662 | max_backward: 1.2979 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1103 | Memory: rss_MB: 7457.8 | uss_MB: 6799.8 | child_rss_MB: 5030.2 | allocated_MB: 341.8 | reserved_MB: 26054.0 | inactive_split_MB: 273.2 +rank1 [2026-10-29 05:29:08,212] INFO : Step 25900/50000 rank1 | Train Loss: 1.1019 | Train Loss Unweighted: 0.5481 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1165 | Classification: 0.0977 | Classification_binary: 0.1594 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1434 | Regression_eta: 0.0016 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1460 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0638 | Regression_eta: 0.0458 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0882 | LR: 5.25e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5201s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7034 | max_loss: 0.2469 | max_backward: 1.7130 | max_optimizer_step: 0.2363 | max_tensorboard_logging: 0.8382 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 7454.8 | uss_MB: 6800.6 | child_rss_MB: 5037.1 | allocated_MB: 343.0 | reserved_MB: 26054.0 | inactive_split_MB: 187.0 +rank1 [2026-10-29 05:33:07,468] INFO : Step 26000/50000 rank1 | Train Loss: 1.1708 | Train Loss Unweighted: 0.5680 | Train Loss Components: Auxiliary: 0.2498 | Cardinality: 0.1352 | Classification: 0.0999 | Classification_binary: 0.2021 | HitGrouping: 0.1994 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1304 | Regression_eta: 0.0021 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1996 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0608 | Regression_eta: 0.0578 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0807 | LR: 5.22e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2230s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.7877 | max_loss: 0.1637 | max_backward: 1.6550 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.4954 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 7454.4 | uss_MB: 6802.1 | child_rss_MB: 5032.6 | allocated_MB: 345.0 | reserved_MB: 27098.0 | inactive_split_MB: 285.6 +rank1 [2026-10-29 05:33:08,393] INFO : Running validation on rank1 +rank1 [2026-10-29 05:33:08,393] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 05:33:08,393] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 05:33:56,766] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:33:57,245] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 05:34:24,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 05:34:25,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 05:34:26,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 05:34:27,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 05:34:37,464] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 05:34:38,072] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:34:38,538] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 05:34:54,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 05:34:55,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 05:34:56,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 05:34:57,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 05:35:07,657] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 05:35:08,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:35:08,510] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 05:35:24,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 05:35:24,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 05:35:25,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 05:35:26,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 05:35:36,183] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 05:39:49,627] INFO : Step 26100/50000 rank1 | Train Loss: 1.0974 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2307 | Cardinality: 0.1119 | Classification: 0.0956 | Classification_binary: 0.1689 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1513 | Regression_eta: 0.0014 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1496 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0757 | Regression_eta: 0.0471 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0549 | LR: 5.19e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.1611s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0117 | max_forward: 0.7200 | max_loss: 0.1501 | max_backward: 1.2286 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.8284 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 8630.6 | uss_MB: 7973.7 | child_rss_MB: 5008.3 | allocated_MB: 340.1 | reserved_MB: 21762.0 | inactive_split_MB: 215.7 +rank1 [2026-10-29 05:43:56,539] INFO : Step 26200/50000 rank1 | Train Loss: 1.0876 | Train Loss Unweighted: 0.5502 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1065 | Classification: 0.0968 | Classification_binary: 0.1527 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1476 | Regression_eta: 0.0018 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0866 | Regression_eta: 0.0572 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0901 | LR: 5.16e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3259s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6530 | max_loss: 0.1421 | max_backward: 1.3973 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.8695 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8630.6 | uss_MB: 7973.8 | child_rss_MB: 5031.7 | allocated_MB: 342.3 | reserved_MB: 22678.0 | inactive_split_MB: 242.5 +rank1 [2026-10-29 05:47:48,031] INFO : Step 26300/50000 rank1 | Train Loss: 1.2051 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1512 | Classification: 0.1193 | Classification_binary: 0.1877 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1458 | Regression_eta: 0.0017 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0871 | Classification_binary: 0.1897 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0700 | Regression_eta: 0.0453 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0775 | LR: 5.13e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.4701s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0072 | max_forward: 0.7601 | max_loss: 0.2267 | max_backward: 1.8902 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7522.0 | uss_MB: 6871.8 | child_rss_MB: 4996.1 | allocated_MB: 345.4 | reserved_MB: 22678.0 | inactive_split_MB: 270.9 +rank1 [2026-10-29 05:51:47,278] INFO : Step 26400/50000 rank1 | Train Loss: 1.2256 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1648 | Classification: 0.1020 | Classification_binary: 0.1981 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1520 | Regression_eta: 0.0015 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1922 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0762 | Regression_eta: 0.0439 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0387 | LR: 5.10e-04 | DataLoad Time: 0.0090s | Model Forward Time: 4.4042s | Phase Times: max_pre_step_barrier: 0.0351 | max_batch_to_device: 0.0070 | max_forward: 0.8129 | max_loss: 0.1477 | max_backward: 1.4015 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8384 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 7520.2 | uss_MB: 6869.2 | child_rss_MB: 4985.0 | allocated_MB: 342.8 | reserved_MB: 22678.0 | inactive_split_MB: 196.9 +rank1 [2026-10-29 05:55:40,759] INFO : Step 26500/50000 rank1 | Train Loss: 1.1110 | Train Loss Unweighted: 0.5356 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1488 | Classification: 0.0891 | Classification_binary: 0.1546 | HitGrouping: 0.2012 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1377 | Regression_cos_phi: 0.1249 | Regression_energy: 0.0752 | Regression_eta: 0.0356 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0837 | LR: 5.08e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0236s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6841 | max_loss: 0.2421 | max_backward: 1.1616 | max_optimizer_step: 0.2383 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7505.4 | uss_MB: 6872.6 | child_rss_MB: 4957.6 | allocated_MB: 340.6 | reserved_MB: 22678.0 | inactive_split_MB: 269.9 +rank1 [2026-10-29 05:59:33,069] INFO : Step 26600/50000 rank1 | Train Loss: 1.0895 | Train Loss Unweighted: 0.5529 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1261 | Classification: 0.1083 | Classification_binary: 0.1603 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1386 | Regression_eta: 0.0015 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0698 | Regression_eta: 0.0492 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0846 | LR: 5.05e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.4530s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0696 | max_forward: 0.6926 | max_loss: 0.1459 | max_backward: 1.4149 | max_optimizer_step: 0.2385 | max_tensorboard_logging: 0.8927 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7477.7 | uss_MB: 6873.1 | child_rss_MB: 4831.1 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 232.0 +rank1 [2026-10-29 06:03:23,755] INFO : Step 26700/50000 rank1 | Train Loss: 1.2592 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1717 | Classification: 0.1100 | Classification_binary: 0.2137 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1420 | Regression_eta: 0.0017 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2508 | Regression_cos_phi: 0.1096 | Regression_energy: 0.0743 | Regression_eta: 0.0569 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0785 | LR: 5.02e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.6754s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7750 | max_loss: 0.1526 | max_backward: 1.6427 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.8866 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7338.5 | uss_MB: 6135.7 | child_rss_MB: 4788.2 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 267.4 +rank1 [2026-10-29 06:07:16,644] INFO : Step 26800/50000 rank1 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5792 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1288 | Classification: 0.0911 | Classification_binary: 0.1633 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1600 | Regression_eta: 0.0015 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0802 | Regression_eta: 0.0436 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0889 | LR: 4.99e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0414s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7108 | max_loss: 0.1467 | max_backward: 1.2855 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7173 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 7253.0 | uss_MB: 6872.2 | child_rss_MB: 4809.0 | allocated_MB: 340.8 | reserved_MB: 22846.0 | inactive_split_MB: 253.2 +rank1 [2026-10-29 06:11:09,652] INFO : Step 26900/50000 rank1 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.0996 | Classification: 0.0867 | Classification_binary: 0.1428 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1483 | Regression_eta: 0.0018 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1386 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0716 | Regression_eta: 0.0421 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0629 | LR: 4.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8394s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0336 | max_forward: 0.6534 | max_loss: 0.1408 | max_backward: 1.3898 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0912 | Memory: rss_MB: 7214.0 | uss_MB: 6872.7 | child_rss_MB: 4790.0 | allocated_MB: 342.5 | reserved_MB: 22846.0 | inactive_split_MB: 233.5 +rank1 [2026-10-29 06:15:05,369] INFO : Step 27000/50000 rank1 | Train Loss: 1.0945 | Train Loss Unweighted: 0.5396 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1196 | Classification: 0.0957 | Classification_binary: 0.1912 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1247 | Regression_eta: 0.0015 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2288 | Regression_cos_phi: 0.1056 | Regression_energy: 0.0796 | Regression_eta: 0.0615 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0830 | LR: 4.94e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.1553s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7531 | max_loss: 0.1689 | max_backward: 1.5554 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6017 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7214.1 | uss_MB: 6877.0 | child_rss_MB: 4792.2 | allocated_MB: 345.3 | reserved_MB: 23694.0 | inactive_split_MB: 251.5 +rank1 [2026-10-29 06:15:06,390] INFO : Running validation on rank1 +rank1 [2026-10-29 06:15:06,390] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 06:15:06,390] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 06:16:00,134] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:16:00,711] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 06:16:29,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 06:16:30,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 06:16:31,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 06:16:32,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 06:16:42,724] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 06:16:43,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:16:43,719] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 06:16:59,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 06:17:00,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 06:17:01,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 06:17:02,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 06:17:12,640] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 06:17:13,248] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:17:13,780] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 06:17:29,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 06:17:30,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 06:17:30,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 06:17:31,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 06:17:41,618] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 06:21:51,977] INFO : Step 27100/50000 rank1 | Train Loss: 1.1577 | Train Loss Unweighted: 0.5595 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1497 | Classification: 0.0884 | Classification_binary: 0.1879 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1399 | Regression_eta: 0.0018 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1906 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0547 | Regression_eta: 0.0476 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0654 | LR: 4.91e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.0515s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8543 | max_loss: 0.1477 | max_backward: 1.8762 | max_optimizer_step: 0.2540 | max_tensorboard_logging: 0.9462 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 8306.7 | uss_MB: 7966.7 | child_rss_MB: 4814.1 | allocated_MB: 342.9 | reserved_MB: 21640.0 | inactive_split_MB: 268.0 +rank1 [2026-10-29 06:25:39,974] INFO : Step 27200/50000 rank1 | Train Loss: 1.0028 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1065 | Classification: 0.0834 | Classification_binary: 0.1354 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1443 | Regression_eta: 0.0015 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1199 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0668 | Regression_eta: 0.0359 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0515 | LR: 4.88e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.0916s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6839 | max_loss: 0.1724 | max_backward: 1.0628 | max_optimizer_step: 0.2515 | max_tensorboard_logging: 0.9641 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0689 | Memory: rss_MB: 8305.7 | uss_MB: 7964.4 | child_rss_MB: 4815.8 | allocated_MB: 340.5 | reserved_MB: 22612.0 | inactive_split_MB: 267.2 +rank1 [2026-10-29 06:29:18,757] INFO : Step 27300/50000 rank1 | Train Loss: 1.1913 | Train Loss Unweighted: 0.5857 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1563 | Classification: 0.1169 | Classification_binary: 0.1769 | HitGrouping: 0.1965 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1426 | Regression_eta: 0.0019 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0644 | Regression_eta: 0.0443 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0633 | LR: 4.85e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2511s | Phase Times: max_pre_step_barrier: 0.0244 | max_batch_to_device: 0.0064 | max_forward: 0.6813 | max_loss: 0.1454 | max_backward: 1.4618 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7699 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 7290.8 | uss_MB: 6952.0 | child_rss_MB: 4811.5 | allocated_MB: 345.6 | reserved_MB: 23586.0 | inactive_split_MB: 267.9 +rank1 [2026-10-29 06:33:03,429] INFO : Step 27400/50000 rank1 | Train Loss: 1.1723 | Train Loss Unweighted: 0.5888 | Train Loss Components: Auxiliary: 0.2572 | Cardinality: 0.1400 | Classification: 0.1088 | Classification_binary: 0.2016 | HitGrouping: 0.1987 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1370 | Regression_eta: 0.0017 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0653 | Regression_eta: 0.0455 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0612 | LR: 4.82e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6677s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8048 | max_loss: 0.1475 | max_backward: 1.6655 | max_optimizer_step: 0.2279 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7292.0 | uss_MB: 6951.9 | child_rss_MB: 4814.0 | allocated_MB: 344.5 | reserved_MB: 23586.0 | inactive_split_MB: 279.5 +rank1 [2026-10-29 06:37:01,482] INFO : Step 27500/50000 rank1 | Train Loss: 1.0739 | Train Loss Unweighted: 0.5273 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1251 | Classification: 0.0933 | Classification_binary: 0.1655 | HitGrouping: 0.1957 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0575 | Regression_eta: 0.0550 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0520 | LR: 4.80e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2235s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7686 | max_loss: 0.1542 | max_backward: 1.3750 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.8628 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 7288.8 | uss_MB: 6945.7 | child_rss_MB: 4804.4 | allocated_MB: 343.6 | reserved_MB: 23586.0 | inactive_split_MB: 241.4 +rank1 [2026-10-29 06:40:55,529] INFO : Step 27600/50000 rank1 | Train Loss: 1.0507 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1163 | Classification: 0.1024 | Classification_binary: 0.1467 | HitGrouping: 0.1992 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.1702 | Regression_cos_phi: 0.1030 | Regression_energy: 0.0726 | Regression_eta: 0.0525 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0957 | LR: 4.77e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4922s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0060 | max_forward: 0.6744 | max_loss: 0.2492 | max_backward: 1.6867 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.5257 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 7290.8 | uss_MB: 6853.7 | child_rss_MB: 4808.0 | allocated_MB: 341.2 | reserved_MB: 24480.0 | inactive_split_MB: 234.8 +rank1 [2026-10-29 06:44:47,871] INFO : Step 27700/50000 rank1 | Train Loss: 1.1881 | Train Loss Unweighted: 0.5834 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1544 | Classification: 0.1001 | Classification_binary: 0.1885 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1444 | Regression_eta: 0.0015 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1862 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0625 | Regression_eta: 0.0382 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0632 | LR: 4.74e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2920s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7716 | max_loss: 0.1527 | max_backward: 1.4941 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 0.7272 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7291.1 | uss_MB: 6952.7 | child_rss_MB: 4804.6 | allocated_MB: 344.5 | reserved_MB: 24480.0 | inactive_split_MB: 194.2 +rank1 [2026-10-29 06:48:38,929] INFO : Step 27800/50000 rank1 | Train Loss: 1.1903 | Train Loss Unweighted: 0.5862 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1435 | Classification: 0.1032 | Classification_binary: 0.2006 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1387 | Regression_eta: 0.0018 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2175 | Regression_cos_phi: 0.1275 | Regression_energy: 0.0688 | Regression_eta: 0.0480 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0597 | LR: 4.71e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1874s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8137 | max_loss: 0.1509 | max_backward: 1.4748 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6041 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7291.1 | uss_MB: 6953.1 | child_rss_MB: 4808.7 | allocated_MB: 348.4 | reserved_MB: 24592.0 | inactive_split_MB: 226.8 +rank1 [2026-10-29 06:52:39,106] INFO : Step 27900/50000 rank1 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5521 | Train Loss Components: Auxiliary: 0.2284 | Cardinality: 0.1139 | Classification: 0.0902 | Classification_binary: 0.1544 | HitGrouping: 0.2057 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1519 | Regression_eta: 0.0017 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0833 | Regression_eta: 0.0302 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0767 | LR: 4.68e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6327s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6397 | max_loss: 0.2456 | max_backward: 1.5234 | max_optimizer_step: 0.2276 | max_tensorboard_logging: 1.1626 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 7291.1 | uss_MB: 6955.2 | child_rss_MB: 4805.2 | allocated_MB: 342.4 | reserved_MB: 25612.0 | inactive_split_MB: 259.0 +rank1 [2026-10-29 06:56:30,760] INFO : Step 28000/50000 rank1 | Train Loss: 1.1111 | Train Loss Unweighted: 0.5401 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1274 | Classification: 0.1035 | Classification_binary: 0.1699 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0930 | Regression_energy: 0.0721 | Regression_eta: 0.0552 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0773 | LR: 4.66e-04 | DataLoad Time: 0.0225s | Model Forward Time: 5.0940s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.7503 | max_loss: 0.1492 | max_backward: 1.7566 | max_optimizer_step: 0.2764 | max_tensorboard_logging: 1.2295 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 7291.2 | uss_MB: 5984.8 | child_rss_MB: 4794.8 | allocated_MB: 344.1 | reserved_MB: 25612.0 | inactive_split_MB: 243.2 +rank1 [2026-10-29 06:56:31,843] INFO : Running validation on rank1 +rank1 [2026-10-29 06:56:31,843] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 06:56:31,843] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 06:57:25,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:57:25,560] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 06:57:52,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 06:57:53,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 06:57:54,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 06:57:55,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 06:58:05,195] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 06:58:05,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:58:06,179] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 06:58:21,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 06:58:22,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 06:58:23,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 06:58:24,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 06:58:34,600] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 06:58:35,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:58:35,800] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 06:58:51,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 06:58:52,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 06:58:52,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 06:58:53,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 06:59:03,512] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 07:03:11,251] INFO : Step 28100/50000 rank1 | Train Loss: 1.1727 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1644 | Classification: 0.0934 | Classification_binary: 0.1929 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1365 | Regression_eta: 0.0015 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1980 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0625 | Regression_eta: 0.0377 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0424 | LR: 4.63e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9497s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8425 | max_loss: 0.2571 | max_backward: 1.8157 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 1.0134 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 8340.2 | uss_MB: 8005.6 | child_rss_MB: 4802.9 | allocated_MB: 344.8 | reserved_MB: 22140.0 | inactive_split_MB: 293.2 +rank1 [2026-10-29 07:07:01,163] INFO : Step 28200/50000 rank1 | Train Loss: 1.0520 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2296 | Cardinality: 0.1203 | Classification: 0.0934 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1466 | Regression_eta: 0.0014 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0600 | Regression_eta: 0.0359 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0474 | LR: 4.60e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3562s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7943 | max_loss: 0.1496 | max_backward: 1.4503 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 0.7872 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 8343.2 | uss_MB: 8006.4 | child_rss_MB: 4796.0 | allocated_MB: 341.0 | reserved_MB: 24034.0 | inactive_split_MB: 210.4 +rank1 [2026-10-29 07:10:52,729] INFO : Step 28300/50000 rank1 | Train Loss: 1.0602 | Train Loss Unweighted: 0.5453 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1043 | Classification: 0.0923 | Classification_binary: 0.1501 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1496 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0799 | Classification_binary: 0.1809 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0753 | Regression_eta: 0.0526 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0884 | LR: 4.57e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2453s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6482 | max_loss: 0.2266 | max_backward: 1.5658 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.6870 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0836 | Memory: rss_MB: 7393.0 | uss_MB: 7059.5 | child_rss_MB: 4787.3 | allocated_MB: 342.7 | reserved_MB: 25016.0 | inactive_split_MB: 243.9 +rank1 [2026-10-29 07:14:40,574] INFO : Step 28400/50000 rank1 | Train Loss: 1.1507 | Train Loss Unweighted: 0.5391 | Train Loss Components: Auxiliary: 0.2436 | Cardinality: 0.1500 | Classification: 0.0965 | Classification_binary: 0.1896 | HitGrouping: 0.1911 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1241 | Regression_eta: 0.0018 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.2038 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0564 | Regression_eta: 0.0407 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0763 | LR: 4.55e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6302s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7664 | max_loss: 0.1532 | max_backward: 1.7342 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.8455 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7394.0 | uss_MB: 7059.9 | child_rss_MB: 4785.1 | allocated_MB: 344.2 | reserved_MB: 25016.0 | inactive_split_MB: 278.9 +rank1 [2026-10-29 07:18:29,002] INFO : Step 28500/50000 rank1 | Train Loss: 1.1595 | Train Loss Unweighted: 0.5640 | Train Loss Components: Auxiliary: 0.2492 | Cardinality: 0.1556 | Classification: 0.0927 | Classification_binary: 0.1908 | HitGrouping: 0.1954 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1384 | Regression_eta: 0.0015 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1931 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0643 | Regression_eta: 0.0579 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0638 | LR: 4.52e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2719s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.8033 | max_loss: 0.1665 | max_backward: 1.5231 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.6178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0709 | Memory: rss_MB: 7393.5 | uss_MB: 7056.4 | child_rss_MB: 4808.7 | allocated_MB: 347.4 | reserved_MB: 26000.0 | inactive_split_MB: 283.2 +rank1 [2026-10-29 07:22:17,774] INFO : Step 28600/50000 rank1 | Train Loss: 1.0672 | Train Loss Unweighted: 0.5222 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1172 | Classification: 0.0893 | Classification_binary: 0.1544 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0687 | Regression_eta: 0.0413 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0603 | LR: 4.49e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.0216s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6704 | max_loss: 0.1829 | max_backward: 1.2353 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.8074 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7396.5 | uss_MB: 7057.8 | child_rss_MB: 4793.1 | allocated_MB: 342.3 | reserved_MB: 26000.0 | inactive_split_MB: 243.7 +rank1 [2026-10-29 07:26:10,154] INFO : Step 28700/50000 rank1 | Train Loss: 1.1535 | Train Loss Unweighted: 0.5811 | Train Loss Components: Auxiliary: 0.2469 | Cardinality: 0.1254 | Classification: 0.1220 | Classification_binary: 0.1661 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1442 | Regression_eta: 0.0021 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1625 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0652 | Regression_eta: 0.0265 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0538 | LR: 4.46e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.5828s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6905 | max_loss: 0.2072 | max_backward: 1.6876 | max_optimizer_step: 0.2353 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 7397.5 | uss_MB: 7064.1 | child_rss_MB: 4802.4 | allocated_MB: 343.5 | reserved_MB: 26000.0 | inactive_split_MB: 234.1 +rank1 [2026-10-29 07:30:12,150] INFO : Step 28800/50000 rank1 | Train Loss: 1.2131 | Train Loss Unweighted: 0.5739 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1788 | Classification: 0.1057 | Classification_binary: 0.2127 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0643 | Regression_eta: 0.0301 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0533 | LR: 4.44e-04 | DataLoad Time: 0.0129s | Model Forward Time: 6.8518s | Phase Times: max_pre_step_barrier: 0.0263 | max_batch_to_device: 0.0149 | max_forward: 1.2928 | max_loss: 0.2713 | max_backward: 3.0161 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.8895 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 7398.3 | uss_MB: 5722.3 | child_rss_MB: 4796.9 | allocated_MB: 343.8 | reserved_MB: 26000.0 | inactive_split_MB: 273.3 +rank1 [2026-10-29 07:34:11,570] INFO : Step 28900/50000 rank1 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1252 | Classification: 0.0967 | Classification_binary: 0.1637 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1509 | Regression_eta: 0.0015 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0939 | Regression_energy: 0.0767 | Regression_eta: 0.0395 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0668 | LR: 4.41e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.1531s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7512 | max_loss: 0.2189 | max_backward: 1.6229 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.9330 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7399.6 | uss_MB: 5634.1 | child_rss_MB: 4800.1 | allocated_MB: 339.6 | reserved_MB: 26000.0 | inactive_split_MB: 223.0 +rank1 [2026-10-29 07:38:06,079] INFO : Step 29000/50000 rank1 | Train Loss: 1.0238 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1070 | Classification: 0.0861 | Classification_binary: 0.1439 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1389 | Regression_eta: 0.0014 | Regression_pt: 0.1414 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1554 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0665 | Regression_eta: 0.0331 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0625 | LR: 4.38e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9817s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.6636 | max_loss: 0.2429 | max_backward: 1.5394 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.5216 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7400.5 | uss_MB: 7067.2 | child_rss_MB: 4794.8 | allocated_MB: 345.2 | reserved_MB: 26984.0 | inactive_split_MB: 235.2 +rank1 [2026-10-29 07:38:07,029] INFO : Running validation on rank1 +rank1 [2026-10-29 07:38:07,029] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 07:38:07,029] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 07:38:57,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:38:58,451] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 07:39:24,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 07:39:25,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 07:39:26,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 07:39:27,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 07:39:37,592] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 07:39:38,025] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:39:38,580] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 07:39:54,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 07:39:55,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 07:39:56,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 07:39:57,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 07:40:07,546] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 07:40:07,779] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:40:08,276] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 07:40:24,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 07:40:24,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 07:40:25,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 07:40:26,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 07:40:36,284] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 07:45:01,997] INFO : Step 29100/50000 rank1 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5808 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1279 | Classification: 0.1113 | Classification_binary: 0.1973 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1339 | Regression_eta: 0.0017 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2057 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0653 | Regression_eta: 0.0439 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0865 | LR: 4.35e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.8736s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.8060 | max_loss: 0.1503 | max_backward: 1.9210 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 8379.4 | uss_MB: 8043.2 | child_rss_MB: 4787.2 | allocated_MB: 344.6 | reserved_MB: 24404.0 | inactive_split_MB: 274.9 +rank1 [2026-10-29 07:48:58,055] INFO : Step 29200/50000 rank1 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5633 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1265 | Classification: 0.0969 | Classification_binary: 0.1883 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1368 | Regression_eta: 0.0014 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0604 | Regression_eta: 0.0328 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0456 | LR: 4.33e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8371s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0742 | max_forward: 1.0303 | max_loss: 0.2183 | max_backward: 1.8745 | max_optimizer_step: 0.2501 | max_tensorboard_logging: 0.5570 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8379.4 | uss_MB: 8042.7 | child_rss_MB: 4798.0 | allocated_MB: 343.1 | reserved_MB: 24404.0 | inactive_split_MB: 228.1 +rank1 [2026-10-29 07:52:51,792] INFO : Step 29300/50000 rank1 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2330 | Cardinality: 0.1231 | Classification: 0.0898 | Classification_binary: 0.1516 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1428 | Regression_eta: 0.0017 | Regression_pt: 0.1453 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0606 | Regression_eta: 0.0537 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0502 | LR: 4.30e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.7363s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0067 | max_forward: 0.7139 | max_loss: 0.1578 | max_backward: 1.5096 | max_optimizer_step: 0.2586 | max_tensorboard_logging: 1.1324 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7473.7 | uss_MB: 6607.6 | child_rss_MB: 4797.1 | allocated_MB: 341.0 | reserved_MB: 24404.0 | inactive_split_MB: 247.0 +rank1 [2026-10-29 07:56:44,367] INFO : Step 29400/50000 rank1 | Train Loss: 1.1400 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1377 | Classification: 0.1135 | Classification_binary: 0.1661 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1386 | Regression_eta: 0.0019 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.1608 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0613 | Regression_eta: 0.0335 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0522 | LR: 4.27e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.2872s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.6285 | max_loss: 0.1485 | max_backward: 1.3787 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.9179 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7474.7 | uss_MB: 7139.2 | child_rss_MB: 4788.9 | allocated_MB: 343.3 | reserved_MB: 24404.0 | inactive_split_MB: 277.1 +rank1 [2026-10-29 08:00:58,645] INFO : Step 29500/50000 rank1 | Train Loss: 1.1987 | Train Loss Unweighted: 0.6106 | Train Loss Components: Auxiliary: 0.2551 | Cardinality: 0.1277 | Classification: 0.1201 | Classification_binary: 0.2013 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1419 | Regression_eta: 0.0019 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2032 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0647 | Regression_eta: 0.0418 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0751 | LR: 4.24e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.8984s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0749 | max_forward: 0.8153 | max_loss: 0.3465 | max_backward: 2.8565 | max_optimizer_step: 0.3442 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7477.7 | uss_MB: 7142.6 | child_rss_MB: 4792.1 | allocated_MB: 343.9 | reserved_MB: 24404.0 | inactive_split_MB: 271.5 +rank1 [2026-10-29 08:05:08,851] INFO : Step 29600/50000 rank1 | Train Loss: 1.1065 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2391 | Cardinality: 0.1179 | Classification: 0.0927 | Classification_binary: 0.1666 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0018 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0761 | Regression_eta: 0.0651 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0539 | LR: 4.22e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4743s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8025 | max_loss: 0.1485 | max_backward: 1.4674 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.8982 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 7476.5 | uss_MB: 7138.1 | child_rss_MB: 4790.4 | allocated_MB: 340.8 | reserved_MB: 24404.0 | inactive_split_MB: 290.7 +rank1 [2026-10-29 08:09:13,801] INFO : Step 29700/50000 rank1 | Train Loss: 1.0488 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2273 | Cardinality: 0.1414 | Classification: 0.0952 | Classification_binary: 0.1459 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1432 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0777 | Regression_eta: 0.0362 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0340 | LR: 4.19e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.5175s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0057 | max_forward: 0.7342 | max_loss: 0.1652 | max_backward: 1.4907 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.9743 | max_tensorboard_flush: 0.0140 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7477.5 | uss_MB: 6998.2 | child_rss_MB: 4799.0 | allocated_MB: 342.3 | reserved_MB: 24404.0 | inactive_split_MB: 236.6 +rank1 [2026-10-29 08:13:32,079] INFO : Step 29800/50000 rank1 | Train Loss: 1.1686 | Train Loss Unweighted: 0.5644 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1551 | Classification: 0.0952 | Classification_binary: 0.1922 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1357 | Regression_eta: 0.0014 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0613 | Regression_eta: 0.0421 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0657 | LR: 4.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6773s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7294 | max_loss: 0.1685 | max_backward: 1.6547 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9409 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7478.5 | uss_MB: 7143.5 | child_rss_MB: 4794.4 | allocated_MB: 345.0 | reserved_MB: 24404.0 | inactive_split_MB: 265.1 +rank1 [2026-10-29 08:17:37,375] INFO : Step 29900/50000 rank1 | Train Loss: 1.1305 | Train Loss Unweighted: 0.5666 | Train Loss Components: Auxiliary: 0.2471 | Cardinality: 0.1380 | Classification: 0.0944 | Classification_binary: 0.1750 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1464 | Regression_eta: 0.0015 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0739 | Regression_eta: 0.0343 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0486 | LR: 4.14e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.3524s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0611 | max_forward: 0.8060 | max_loss: 0.1494 | max_backward: 1.3251 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 7479.5 | uss_MB: 7143.0 | child_rss_MB: 4800.7 | allocated_MB: 344.7 | reserved_MB: 26422.0 | inactive_split_MB: 298.4 +rank1 [2026-10-29 08:22:04,989] INFO : Step 30000/50000 rank1 | Train Loss: 1.0854 | Train Loss Unweighted: 0.5586 | Train Loss Components: Auxiliary: 0.2416 | Cardinality: 0.1337 | Classification: 0.1000 | Classification_binary: 0.1541 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1499 | Regression_eta: 0.0014 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0807 | Regression_eta: 0.0540 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0606 | LR: 4.11e-04 | DataLoad Time: 0.0198s | Model Forward Time: 7.0753s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0172 | max_forward: 0.9685 | max_loss: 0.2166 | max_backward: 4.0112 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.8430 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7461.9 | uss_MB: 7142.5 | child_rss_MB: 4797.1 | allocated_MB: 338.0 | reserved_MB: 26422.0 | inactive_split_MB: 288.0 +rank1 [2026-10-29 08:22:05,951] INFO : Running validation on rank1 +rank1 [2026-10-29 08:22:05,952] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 08:22:05,952] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 08:22:57,875] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:22:58,364] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 08:23:24,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 08:23:25,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 08:23:26,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 08:23:28,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 08:23:38,207] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 08:23:38,390] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:23:38,904] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 08:23:55,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 08:23:56,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 08:23:57,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 08:23:58,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 08:24:08,061] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 08:24:08,113] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:24:08,615] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 08:24:24,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 08:24:24,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 08:24:25,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 08:24:26,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 08:24:36,561] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 08:29:00,406] INFO : Step 30100/50000 rank1 | Train Loss: 1.0762 | Train Loss Unweighted: 0.5358 | Train Loss Components: Auxiliary: 0.2378 | Cardinality: 0.1106 | Classification: 0.1097 | Classification_binary: 0.1679 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.1913 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0601 | Regression_eta: 0.0567 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0809 | LR: 4.08e-04 | DataLoad Time: 0.0167s | Model Forward Time: 8.2922s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7352 | max_loss: 0.1443 | max_backward: 1.7375 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 1.7731 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.3497 | Memory: rss_MB: 7528.6 | uss_MB: 6331.6 | child_rss_MB: 4801.2 | allocated_MB: 344.1 | reserved_MB: 22922.0 | inactive_split_MB: 248.8 +rank1 [2026-10-29 08:33:16,461] INFO : Step 30200/50000 rank1 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5304 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1421 | Classification: 0.0934 | Classification_binary: 0.1839 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0016 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1674 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0564 | Regression_eta: 0.0324 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0433 | LR: 4.06e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6002s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8008 | max_loss: 0.1560 | max_backward: 1.6706 | max_optimizer_step: 0.2663 | max_tensorboard_logging: 0.7725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0555 | Memory: rss_MB: 7535.6 | uss_MB: 7212.9 | child_rss_MB: 4775.8 | allocated_MB: 344.1 | reserved_MB: 23882.0 | inactive_split_MB: 274.8 +rank1 [2026-10-29 08:37:30,482] INFO : Step 30300/50000 rank1 | Train Loss: 1.0429 | Train Loss Unweighted: 0.5252 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1223 | Classification: 0.0922 | Classification_binary: 0.1566 | HitGrouping: 0.1929 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1313 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0654 | Regression_eta: 0.0389 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0559 | LR: 4.03e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.6249s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0060 | max_forward: 1.1719 | max_loss: 0.2884 | max_backward: 2.4994 | max_optimizer_step: 0.2255 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7536.6 | uss_MB: 7213.9 | child_rss_MB: 4771.6 | allocated_MB: 341.8 | reserved_MB: 23882.0 | inactive_split_MB: 270.7 +rank1 [2026-10-29 08:41:40,106] INFO : Step 30400/50000 rank1 | Train Loss: 1.0543 | Train Loss Unweighted: 0.5182 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1318 | Classification: 0.0889 | Classification_binary: 0.1487 | HitGrouping: 0.1973 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1382 | Regression_eta: 0.0014 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0700 | Regression_eta: 0.0573 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0751 | LR: 4.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0792s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0150 | max_forward: 0.8845 | max_loss: 0.2482 | max_backward: 2.1745 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.5112 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7452.2 | uss_MB: 7214.6 | child_rss_MB: 4475.9 | allocated_MB: 344.7 | reserved_MB: 24842.0 | inactive_split_MB: 245.3 +rank1 [2026-10-29 08:45:49,048] INFO : Step 30500/50000 rank1 | Train Loss: 1.1589 | Train Loss Unweighted: 0.5596 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1407 | Classification: 0.1085 | Classification_binary: 0.1952 | HitGrouping: 0.1960 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2276 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0589 | Regression_eta: 0.0435 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0633 | LR: 3.98e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.7058s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7791 | max_loss: 0.2528 | max_backward: 1.9433 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.5210 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0672 | Memory: rss_MB: 7432.7 | uss_MB: 7008.9 | child_rss_MB: 4374.7 | allocated_MB: 346.2 | reserved_MB: 26806.0 | inactive_split_MB: 227.8 +rank1 [2026-10-29 08:49:58,495] INFO : Step 30600/50000 rank1 | Train Loss: 1.1269 | Train Loss Unweighted: 0.5561 | Train Loss Components: Auxiliary: 0.2414 | Cardinality: 0.1456 | Classification: 0.0973 | Classification_binary: 0.1804 | HitGrouping: 0.1953 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1373 | Regression_eta: 0.0014 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0720 | Regression_eta: 0.0363 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0517 | LR: 3.95e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4783s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1011 | max_forward: 0.8824 | max_loss: 0.1534 | max_backward: 1.4258 | max_optimizer_step: 0.3067 | max_tensorboard_logging: 0.7925 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 7378.2 | uss_MB: 7215.1 | child_rss_MB: 4177.7 | allocated_MB: 345.8 | reserved_MB: 27800.0 | inactive_split_MB: 283.5 +rank1 [2026-10-29 08:54:17,062] INFO : Step 30700/50000 rank1 | Train Loss: 1.0989 | Train Loss Unweighted: 0.5515 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1377 | Classification: 0.0890 | Classification_binary: 0.1568 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0013 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0623 | Regression_eta: 0.0360 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0589 | LR: 3.92e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2947s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0056 | max_forward: 0.7546 | max_loss: 0.2396 | max_backward: 1.3389 | max_optimizer_step: 0.2508 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0572 | Memory: rss_MB: 7380.2 | uss_MB: 7216.5 | child_rss_MB: 4178.9 | allocated_MB: 340.3 | reserved_MB: 27800.0 | inactive_split_MB: 223.2 +rank1 [2026-10-29 08:58:31,924] INFO : Step 30800/50000 rank1 | Train Loss: 1.1241 | Train Loss Unweighted: 0.5488 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1377 | Classification: 0.1081 | Classification_binary: 0.1717 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1317 | Regression_eta: 0.0015 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.1806 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0546 | Regression_eta: 0.0666 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0682 | LR: 3.90e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.8024s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7775 | max_loss: 0.2735 | max_backward: 1.9298 | max_optimizer_step: 0.3489 | max_tensorboard_logging: 0.8756 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0726 | Memory: rss_MB: 7357.4 | uss_MB: 5945.6 | child_rss_MB: 4116.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 275.1 +rank1 [2026-10-29 09:02:40,911] INFO : Step 30900/50000 rank1 | Train Loss: 1.1501 | Train Loss Unweighted: 0.5762 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1300 | Classification: 0.0976 | Classification_binary: 0.1996 | HitGrouping: 0.2000 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1374 | Regression_eta: 0.0017 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.2254 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0647 | Regression_eta: 0.0658 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0763 | LR: 3.87e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.5959s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.3511 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 1.0306 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 7360.4 | uss_MB: 7217.8 | child_rss_MB: 4118.5 | allocated_MB: 342.2 | reserved_MB: 27800.0 | inactive_split_MB: 198.3 +rank1 [2026-10-29 09:06:46,925] INFO : Step 31000/50000 rank1 | Train Loss: 1.1054 | Train Loss Unweighted: 0.5218 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1446 | Classification: 0.0888 | Classification_binary: 0.1693 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0016 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1503 | Regression_cos_phi: 0.0913 | Regression_energy: 0.0651 | Regression_eta: 0.0475 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0554 | LR: 3.84e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.7152s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.8505 | max_loss: 0.2491 | max_backward: 1.6460 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9731 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7337.9 | uss_MB: 6674.8 | child_rss_MB: 4046.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 190.7 +rank1 [2026-10-29 09:06:47,902] INFO : Running validation on rank1 +rank1 [2026-10-29 09:06:47,902] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 09:06:47,902] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 09:07:39,009] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:07:39,555] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 09:08:08,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 09:08:09,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 09:08:10,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 09:08:11,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 09:08:21,384] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 09:08:22,016] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:08:22,479] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 09:08:38,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 09:08:40,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 09:08:41,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 09:08:42,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 09:08:52,494] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 09:08:52,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:08:53,433] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 09:09:08,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 09:09:09,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 09:09:10,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 09:09:10,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 09:09:20,935] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 09:13:45,022] INFO : Step 31100/50000 rank1 | Train Loss: 1.0306 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2182 | Cardinality: 0.1238 | Classification: 0.0932 | Classification_binary: 0.1415 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1726 | Regression_cos_phi: 0.1166 | Regression_energy: 0.0634 | Regression_eta: 0.0793 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0710 | LR: 3.82e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.5105s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1068 | max_forward: 0.8067 | max_loss: 0.1436 | max_backward: 1.1427 | max_optimizer_step: 0.2576 | max_tensorboard_logging: 1.1475 | max_tensorboard_flush: 0.0054 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 8196.3 | uss_MB: 5921.9 | child_rss_MB: 4032.2 | allocated_MB: 343.8 | reserved_MB: 24492.0 | inactive_split_MB: 293.3 +rank1 [2026-10-29 09:17:44,913] INFO : Step 31200/50000 rank1 | Train Loss: 1.1102 | Train Loss Unweighted: 0.5369 | Train Loss Components: Auxiliary: 0.2472 | Cardinality: 0.1495 | Classification: 0.0959 | Classification_binary: 0.1708 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1328 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0401 | Regression_energy: 0.0666 | Regression_eta: 0.0308 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0364 | LR: 3.79e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.1853s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7127 | max_loss: 0.1476 | max_backward: 1.5707 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7428.7 | uss_MB: 7313.8 | child_rss_MB: 4003.5 | allocated_MB: 344.5 | reserved_MB: 24648.0 | inactive_split_MB: 235.5 +rank1 [2026-10-29 09:21:47,088] INFO : Step 31300/50000 rank1 | Train Loss: 1.1865 | Train Loss Unweighted: 0.5472 | Train Loss Components: Auxiliary: 0.2511 | Cardinality: 0.1748 | Classification: 0.0979 | Classification_binary: 0.1851 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0019 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2082 | Regression_cos_phi: 0.1221 | Regression_energy: 0.0783 | Regression_eta: 0.0825 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0947 | LR: 3.76e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.2296s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0064 | max_forward: 0.7815 | max_loss: 0.1495 | max_backward: 1.6330 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.5234 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7366.1 | uss_MB: 7313.9 | child_rss_MB: 3706.8 | allocated_MB: 344.4 | reserved_MB: 25640.0 | inactive_split_MB: 233.6 +rank1 [2026-10-29 09:25:42,887] INFO : Step 31400/50000 rank1 | Train Loss: 1.0272 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1191 | Classification: 0.0784 | Classification_binary: 0.1451 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1281 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0826 | Regression_eta: 0.0381 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0633 | LR: 3.74e-04 | DataLoad Time: 0.0137s | Model Forward Time: 4.5298s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0063 | max_forward: 0.7252 | max_loss: 0.1478 | max_backward: 1.6173 | max_optimizer_step: 0.2404 | max_tensorboard_logging: 0.9541 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7335.5 | uss_MB: 7312.2 | child_rss_MB: 3625.5 | allocated_MB: 342.1 | reserved_MB: 25640.0 | inactive_split_MB: 296.4 +rank1 [2026-10-29 09:29:44,710] INFO : Step 31500/50000 rank1 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5247 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1206 | Classification: 0.1037 | Classification_binary: 0.1583 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1297 | Regression_eta: 0.0014 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0547 | Regression_eta: 0.0555 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0605 | LR: 3.71e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.3661s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6643 | max_loss: 0.2487 | max_backward: 1.5648 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.8248 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7336.0 | uss_MB: 7313.0 | child_rss_MB: 3638.4 | allocated_MB: 344.4 | reserved_MB: 26092.0 | inactive_split_MB: 238.2 +rank1 [2026-10-29 09:33:42,079] INFO : Step 31600/50000 rank1 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5471 | Train Loss Components: Auxiliary: 0.2424 | Cardinality: 0.1284 | Classification: 0.1033 | Classification_binary: 0.1828 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1282 | Regression_eta: 0.0017 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1978 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0540 | Regression_eta: 0.0405 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0640 | LR: 3.69e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.0263s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7603 | max_loss: 0.1471 | max_backward: 2.0475 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.9076 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7328.4 | uss_MB: 7314.4 | child_rss_MB: 3585.6 | allocated_MB: 341.5 | reserved_MB: 26640.0 | inactive_split_MB: 287.0 +rank1 [2026-10-29 09:37:47,826] INFO : Step 31700/50000 rank1 | Train Loss: 1.0533 | Train Loss Unweighted: 0.5009 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1416 | Classification: 0.0871 | Classification_binary: 0.1632 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1628 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0570 | Regression_eta: 0.0451 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0768 | LR: 3.66e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.1104s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7635 | max_loss: 0.1543 | max_backward: 1.2075 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7330.4 | uss_MB: 7312.9 | child_rss_MB: 3557.0 | allocated_MB: 341.9 | reserved_MB: 26640.0 | inactive_split_MB: 274.1 +rank1 [2026-10-29 09:45:40,561] INFO : Step 31800/50000 rank1 | Train Loss: 1.0385 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1148 | Classification: 0.0877 | Classification_binary: 0.1481 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1418 | Regression_eta: 0.0012 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.1636 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0708 | Regression_eta: 0.0319 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0554 | LR: 3.64e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.0360s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0072 | max_forward: 0.6606 | max_loss: 0.1610 | max_backward: 1.1159 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.8258 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1417 | Memory: rss_MB: 7328.9 | uss_MB: 7312.1 | child_rss_MB: 3561.3 | allocated_MB: 345.5 | reserved_MB: 27080.0 | inactive_split_MB: 260.5 +rank1 [2026-10-29 09:49:44,090] INFO : Step 31900/50000 rank1 | Train Loss: 1.1464 | Train Loss Unweighted: 0.5585 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1372 | Classification: 0.1078 | Classification_binary: 0.1815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0017 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1730 | Regression_cos_phi: 0.1173 | Regression_energy: 0.0602 | Regression_eta: 0.0479 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0683 | LR: 3.61e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.5153s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7718 | max_loss: 0.1553 | max_backward: 1.7857 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.6721 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7333.9 | uss_MB: 7320.8 | child_rss_MB: 3569.3 | allocated_MB: 343.8 | reserved_MB: 27080.0 | inactive_split_MB: 280.2 +rank1 [2026-10-29 09:53:38,294] INFO : Step 32000/50000 rank1 | Train Loss: 1.1115 | Train Loss Unweighted: 0.5351 | Train Loss Components: Auxiliary: 0.2393 | Cardinality: 0.1406 | Classification: 0.0954 | Classification_binary: 0.1748 | HitGrouping: 0.1912 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1302 | Regression_eta: 0.0018 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0673 | Regression_eta: 0.0358 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0520 | LR: 3.58e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.2448s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.9767 | max_loss: 0.6919 | max_backward: 2.1955 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.9277 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 7329.3 | uss_MB: 7313.9 | child_rss_MB: 3566.4 | allocated_MB: 344.2 | reserved_MB: 27080.0 | inactive_split_MB: 265.7 +rank1 [2026-10-29 09:53:39,334] INFO : Running validation on rank1 +rank1 [2026-10-29 09:53:39,334] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 09:53:39,334] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 09:54:28,723] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:54:29,292] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 09:54:57,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 09:54:58,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 09:54:59,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 09:55:00,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 09:55:10,451] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 09:55:10,899] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:55:11,415] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 09:55:27,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 09:55:28,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 09:55:29,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 09:55:30,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 09:55:40,471] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 09:55:41,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:55:41,777] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 09:55:57,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 09:55:58,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 09:55:58,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 09:55:59,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 09:56:09,529] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 10:00:19,240] INFO : Step 32100/50000 rank1 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1218 | Classification: 0.0914 | Classification_binary: 0.1509 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1333 | Regression_eta: 0.0016 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0655 | Regression_eta: 0.0457 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0717 | LR: 3.56e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.3634s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7098 | max_loss: 0.1454 | max_backward: 1.1394 | max_optimizer_step: 0.2650 | max_tensorboard_logging: 0.9832 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 8404.9 | uss_MB: 8386.6 | child_rss_MB: 3573.9 | allocated_MB: 344.0 | reserved_MB: 22148.0 | inactive_split_MB: 276.8 +rank1 [2026-10-29 10:04:33,375] INFO : Step 32200/50000 rank1 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1241 | Classification: 0.1035 | Classification_binary: 0.1630 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1279 | Regression_eta: 0.0014 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0584 | Regression_eta: 0.0340 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0751 | LR: 3.53e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.6708s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0891 | max_forward: 0.7517 | max_loss: 0.1441 | max_backward: 1.7835 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.7808 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7418.2 | uss_MB: 7403.1 | child_rss_MB: 3576.2 | allocated_MB: 345.5 | reserved_MB: 23122.0 | inactive_split_MB: 233.4 +rank1 [2026-10-29 10:08:31,627] INFO : Step 32300/50000 rank1 | Train Loss: 1.1438 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2535 | Cardinality: 0.1335 | Classification: 0.1042 | Classification_binary: 0.1958 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1303 | Regression_eta: 0.0015 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0687 | Regression_eta: 0.0589 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0853 | LR: 3.51e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.7851s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7857 | max_loss: 0.1780 | max_backward: 1.7580 | max_optimizer_step: 0.2428 | max_tensorboard_logging: 0.8954 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 7421.3 | uss_MB: 7404.5 | child_rss_MB: 3576.8 | allocated_MB: 343.9 | reserved_MB: 24108.0 | inactive_split_MB: 280.1 +rank1 [2026-10-29 10:12:37,148] INFO : Step 32400/50000 rank1 | Train Loss: 1.0306 | Train Loss Unweighted: 0.5063 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1168 | Classification: 0.0867 | Classification_binary: 0.1599 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0013 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1617 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0554 | Regression_eta: 0.0335 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0740 | LR: 3.48e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.3228s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.1456 | max_backward: 1.3401 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.8752 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7420.4 | uss_MB: 7401.9 | child_rss_MB: 3574.4 | allocated_MB: 344.8 | reserved_MB: 24108.0 | inactive_split_MB: 271.0 +rank1 [2026-10-29 10:16:48,486] INFO : Step 32500/50000 rank1 | Train Loss: 1.0033 | Train Loss Unweighted: 0.5126 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.0995 | Classification: 0.1015 | Classification_binary: 0.1402 | HitGrouping: 0.1898 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1332 | Regression_eta: 0.0015 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1508 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0682 | Regression_eta: 0.0447 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0666 | LR: 3.46e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.9966s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6088 | max_loss: 0.1480 | max_backward: 1.0729 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.8045 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7420.4 | uss_MB: 7401.8 | child_rss_MB: 3575.0 | allocated_MB: 341.9 | reserved_MB: 25096.0 | inactive_split_MB: 268.1 +rank1 [2026-10-29 10:21:04,062] INFO : Step 32600/50000 rank1 | Train Loss: 1.0981 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1271 | Classification: 0.1053 | Classification_binary: 0.1773 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1306 | Regression_eta: 0.0014 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1745 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0568 | Regression_eta: 0.0298 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0630 | LR: 3.43e-04 | DataLoad Time: 0.0158s | Model Forward Time: 6.2933s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8529 | max_loss: 0.3369 | max_backward: 2.9831 | max_optimizer_step: 0.2386 | max_tensorboard_logging: 0.5557 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7420.7 | uss_MB: 5748.3 | child_rss_MB: 3585.6 | allocated_MB: 344.1 | reserved_MB: 25096.0 | inactive_split_MB: 287.9 +rank1 [2026-10-29 10:25:23,203] INFO : Step 32700/50000 rank1 | Train Loss: 1.0956 | Train Loss Unweighted: 0.5307 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.1282 | Classification: 0.1011 | Classification_binary: 0.1731 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1768 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0719 | Regression_eta: 0.0444 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0733 | LR: 3.41e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.1100s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0062 | max_forward: 0.8233 | max_loss: 0.1478 | max_backward: 1.5120 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0564 | Memory: rss_MB: 7420.7 | uss_MB: 7403.4 | child_rss_MB: 3588.8 | allocated_MB: 342.9 | reserved_MB: 25096.0 | inactive_split_MB: 272.6 +rank1 [2026-10-29 10:29:23,500] INFO : Step 32800/50000 rank1 | Train Loss: 1.0857 | Train Loss Unweighted: 0.5241 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1299 | Classification: 0.1011 | Classification_binary: 0.1551 | HitGrouping: 0.1982 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1327 | Regression_eta: 0.0016 | Regression_pt: 0.1319 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1569 | Regression_cos_phi: 0.1058 | Regression_energy: 0.0757 | Regression_eta: 0.0479 | Regression_pt: 0.0785 | Regression_sin_phi: 0.1065 | LR: 3.38e-04 | DataLoad Time: 0.0245s | Model Forward Time: 4.2373s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0179 | max_forward: 0.9787 | max_loss: 0.1782 | max_backward: 1.5075 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 0.5808 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.1577 | Memory: rss_MB: 7419.5 | uss_MB: 7401.5 | child_rss_MB: 3585.9 | allocated_MB: 345.1 | reserved_MB: 25096.0 | inactive_split_MB: 263.9 +rank1 [2026-10-29 10:33:26,398] INFO : Step 32900/50000 rank1 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1116 | Classification: 0.1066 | Classification_binary: 0.1631 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1300 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.1964 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0529 | Regression_eta: 0.0556 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0788 | LR: 3.36e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.6988s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 1.3284 | max_loss: 0.5826 | max_backward: 1.9391 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7423.7 | uss_MB: 7407.1 | child_rss_MB: 3589.0 | allocated_MB: 344.3 | reserved_MB: 26086.0 | inactive_split_MB: 212.1 +rank1 [2026-10-29 10:37:13,221] INFO : Step 33000/50000 rank1 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5373 | Train Loss Components: Auxiliary: 0.2310 | Cardinality: 0.1017 | Classification: 0.1099 | Classification_binary: 0.1844 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0615 | Regression_eta: 0.0659 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0727 | LR: 3.33e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.3110s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 0.7919 | max_loss: 0.2231 | max_backward: 1.5724 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.5025 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1340 | Memory: rss_MB: 7423.5 | uss_MB: 7406.2 | child_rss_MB: 3581.4 | allocated_MB: 344.2 | reserved_MB: 26086.0 | inactive_split_MB: 269.8 +rank1 [2026-10-29 10:37:14,207] INFO : Running validation on rank1 +rank1 [2026-10-29 10:37:14,207] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 10:37:14,207] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 10:38:03,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:38:04,389] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 10:38:31,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 10:38:33,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 10:38:34,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 10:38:35,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 10:38:45,468] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 10:38:45,832] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:38:46,353] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 10:39:02,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 10:39:03,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 10:39:04,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 10:39:05,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 10:39:15,495] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 10:39:15,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:39:16,281] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 10:39:31,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 10:39:32,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 10:39:33,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 10:39:33,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 10:39:43,993] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 10:43:56,311] INFO : Step 33100/50000 rank1 | Train Loss: 1.1040 | Train Loss Unweighted: 0.5268 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1427 | Classification: 0.0921 | Classification_binary: 0.1655 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1319 | Regression_eta: 0.0015 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1447 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0568 | Regression_eta: 0.0425 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0495 | LR: 3.31e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3493s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0325 | max_forward: 0.7755 | max_loss: 0.1478 | max_backward: 1.3916 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.6610 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0760 | Memory: rss_MB: 8784.9 | uss_MB: 8766.5 | child_rss_MB: 3557.4 | allocated_MB: 343.8 | reserved_MB: 23356.0 | inactive_split_MB: 267.3 +rank1 [2026-10-29 10:47:56,313] INFO : Step 33200/50000 rank1 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4955 | Train Loss Components: Auxiliary: 0.2164 | Cardinality: 0.1253 | Classification: 0.1001 | Classification_binary: 0.1407 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0019 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0630 | Regression_eta: 0.0641 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0667 | LR: 3.28e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.0980s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0401 | max_forward: 0.9954 | max_loss: 0.3116 | max_backward: 2.2139 | max_optimizer_step: 0.2657 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7500.4 | uss_MB: 7483.0 | child_rss_MB: 3560.3 | allocated_MB: 344.1 | reserved_MB: 23356.0 | inactive_split_MB: 241.6 +rank1 [2026-10-29 10:51:55,085] INFO : Step 33300/50000 rank1 | Train Loss: 1.0288 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2302 | Cardinality: 0.1166 | Classification: 0.0975 | Classification_binary: 0.1653 | HitGrouping: 0.1885 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1220 | Regression_eta: 0.0012 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1751 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0601 | Regression_eta: 0.0297 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0682 | LR: 3.26e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5496s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7054 | max_loss: 0.2213 | max_backward: 1.6633 | max_optimizer_step: 0.2474 | max_tensorboard_logging: 0.8237 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0708 | Memory: rss_MB: 7500.4 | uss_MB: 7483.1 | child_rss_MB: 3556.5 | allocated_MB: 343.5 | reserved_MB: 23356.0 | inactive_split_MB: 261.9 +rank1 [2026-10-29 10:55:54,911] INFO : Step 33400/50000 rank1 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5337 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1249 | Classification: 0.0970 | Classification_binary: 0.1720 | HitGrouping: 0.1947 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0013 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0517 | Regression_eta: 0.0370 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0375 | LR: 3.23e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.4614s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0073 | max_forward: 1.2444 | max_loss: 0.2025 | max_backward: 2.5109 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7502.5 | uss_MB: 7485.1 | child_rss_MB: 3549.5 | allocated_MB: 347.6 | reserved_MB: 23358.0 | inactive_split_MB: 303.0 +rank1 [2026-10-29 10:59:53,595] INFO : Step 33500/50000 rank1 | Train Loss: 1.0164 | Train Loss Unweighted: 0.4956 | Train Loss Components: Auxiliary: 0.2212 | Cardinality: 0.1274 | Classification: 0.0875 | Classification_binary: 0.1421 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1309 | Regression_eta: 0.0013 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1323 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0634 | Regression_eta: 0.0340 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0316 | LR: 3.21e-04 | DataLoad Time: 0.0130s | Model Forward Time: 4.6378s | Phase Times: max_pre_step_barrier: 0.0271 | max_batch_to_device: 0.0558 | max_forward: 0.9437 | max_loss: 0.1437 | max_backward: 1.5429 | max_optimizer_step: 0.2447 | max_tensorboard_logging: 0.8833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7505.9 | uss_MB: 7486.7 | child_rss_MB: 3558.2 | allocated_MB: 341.2 | reserved_MB: 23358.0 | inactive_split_MB: 237.5 +rank1 [2026-10-29 11:05:22,802] INFO : Step 33600/50000 rank1 | Train Loss: 1.0074 | Train Loss Unweighted: 0.4988 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1109 | Classification: 0.0969 | Classification_binary: 0.1509 | HitGrouping: 0.1901 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0012 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1704 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0581 | Regression_eta: 0.0387 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0614 | LR: 3.19e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1886s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0970 | max_forward: 0.7667 | max_loss: 0.1478 | max_backward: 1.4192 | max_optimizer_step: 0.2426 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7500.7 | uss_MB: 7485.1 | child_rss_MB: 3550.7 | allocated_MB: 344.5 | reserved_MB: 24528.0 | inactive_split_MB: 238.9 +rank1 [2026-10-29 11:09:17,748] INFO : Step 33700/50000 rank1 | Train Loss: 1.1891 | Train Loss Unweighted: 0.5658 | Train Loss Components: Auxiliary: 0.2602 | Cardinality: 0.1587 | Classification: 0.1116 | Classification_binary: 0.1953 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1270 | Regression_eta: 0.0015 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2122 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0557 | Regression_eta: 0.0505 | Regression_pt: 0.0580 | Regression_sin_phi: 0.1392 | LR: 3.16e-04 | DataLoad Time: 0.0141s | Model Forward Time: 4.8290s | Phase Times: max_pre_step_barrier: 0.0253 | max_batch_to_device: 0.0066 | max_forward: 1.3051 | max_loss: 0.1764 | max_backward: 1.7331 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.5706 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0895 | Memory: rss_MB: 7502.0 | uss_MB: 7487.9 | child_rss_MB: 3548.7 | allocated_MB: 347.7 | reserved_MB: 24528.0 | inactive_split_MB: 288.7 +rank1 [2026-10-29 11:13:14,784] INFO : Step 33800/50000 rank1 | Train Loss: 1.0442 | Train Loss Unweighted: 0.4970 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1421 | Classification: 0.0764 | Classification_binary: 0.1525 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1318 | Regression_eta: 0.0014 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0692 | Regression_eta: 0.0420 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0586 | LR: 3.14e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2984s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7370 | max_loss: 0.2458 | max_backward: 1.5902 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.6292 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1331 | Memory: rss_MB: 7496.1 | uss_MB: 7482.2 | child_rss_MB: 3539.4 | allocated_MB: 344.2 | reserved_MB: 24528.0 | inactive_split_MB: 271.5 +rank1 [2026-10-29 11:17:10,800] INFO : Step 33900/50000 rank1 | Train Loss: 1.0341 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1352 | Classification: 0.0865 | Classification_binary: 0.1441 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1254 | Regression_eta: 0.0016 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1634 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0694 | Regression_eta: 0.0396 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0450 | LR: 3.11e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3849s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.6305 | max_loss: 0.1507 | max_backward: 1.4015 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9471 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0605 | Memory: rss_MB: 7498.1 | uss_MB: 7488.3 | child_rss_MB: 3543.5 | allocated_MB: 344.1 | reserved_MB: 24528.0 | inactive_split_MB: 268.7 +rank1 [2026-10-29 11:21:05,085] INFO : Step 34000/50000 rank1 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1309 | Classification: 0.0985 | Classification_binary: 0.1707 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1170 | Regression_eta: 0.0014 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1975 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0612 | Regression_eta: 0.0339 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0598 | LR: 3.09e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.6377s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7414 | max_loss: 0.1458 | max_backward: 1.7218 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 7499.1 | uss_MB: 7491.4 | child_rss_MB: 3548.4 | allocated_MB: 342.9 | reserved_MB: 24528.0 | inactive_split_MB: 277.5 +rank1 [2026-10-29 11:21:06,162] INFO : Running validation on rank1 +rank1 [2026-10-29 11:21:06,162] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 11:21:06,162] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 11:21:56,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:21:57,376] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 11:22:17,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 11:22:18,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 11:22:19,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 11:22:21,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 11:22:31,448] INFO : Time taken to make predictions on device 1 is: 0.28 min +rank1 [2026-10-29 11:22:32,014] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:22:32,507] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 11:22:48,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 11:22:49,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 11:22:50,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 11:22:51,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 11:23:01,145] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 11:23:02,010] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:23:02,514] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 11:23:18,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 11:23:19,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 11:23:19,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 11:23:20,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 11:23:30,408] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 11:27:45,673] INFO : Step 34100/50000 rank1 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5582 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1263 | Classification: 0.1031 | Classification_binary: 0.1801 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1358 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1570 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0735 | Regression_eta: 0.0497 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0596 | LR: 3.06e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8617s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9025 | max_loss: 0.1870 | max_backward: 1.7732 | max_optimizer_step: 0.2826 | max_tensorboard_logging: 0.8215 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0738 | Memory: rss_MB: 9109.5 | uss_MB: 8140.4 | child_rss_MB: 3555.5 | allocated_MB: 342.7 | reserved_MB: 22124.0 | inactive_split_MB: 211.6 +rank1 [2026-10-29 11:31:40,886] INFO : Step 34200/50000 rank1 | Train Loss: 0.9931 | Train Loss Unweighted: 0.4794 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1232 | Classification: 0.0771 | Classification_binary: 0.1377 | HitGrouping: 0.1996 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1301 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0558 | Regression_eta: 0.1006 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0560 | LR: 3.04e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.7212s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.8809 | max_loss: 0.3036 | max_backward: 1.8297 | max_optimizer_step: 0.2897 | max_tensorboard_logging: 0.6835 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7608.6 | uss_MB: 6291.1 | child_rss_MB: 3559.3 | allocated_MB: 339.0 | reserved_MB: 22508.0 | inactive_split_MB: 220.4 +rank1 [2026-10-29 11:35:37,832] INFO : Step 34300/50000 rank1 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4920 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1192 | Classification: 0.0965 | Classification_binary: 0.1484 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0604 | Regression_eta: 0.0479 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0759 | LR: 3.02e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.3211s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7255 | max_loss: 0.1427 | max_backward: 1.3652 | max_optimizer_step: 0.2496 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7610.6 | uss_MB: 7596.5 | child_rss_MB: 3557.3 | allocated_MB: 345.0 | reserved_MB: 22508.0 | inactive_split_MB: 216.4 +rank1 [2026-10-29 11:39:28,026] INFO : Step 34400/50000 rank1 | Train Loss: 1.0662 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2383 | Cardinality: 0.1289 | Classification: 0.0974 | Classification_binary: 0.1749 | HitGrouping: 0.1864 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1281 | Regression_eta: 0.0014 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1787 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0709 | Regression_eta: 0.0454 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0568 | LR: 2.99e-04 | DataLoad Time: 0.0187s | Model Forward Time: 6.1256s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9438 | max_loss: 0.1560 | max_backward: 2.9359 | max_optimizer_step: 0.2526 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7613.7 | uss_MB: 6304.5 | child_rss_MB: 3550.6 | allocated_MB: 345.3 | reserved_MB: 23132.0 | inactive_split_MB: 282.7 +rank1 [2026-10-29 11:43:17,761] INFO : Step 34500/50000 rank1 | Train Loss: 1.0328 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1103 | Classification: 0.0867 | Classification_binary: 0.1518 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0014 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0602 | Regression_eta: 0.0300 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0782 | LR: 2.97e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.6245s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.8153 | max_loss: 0.1429 | max_backward: 1.8299 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.5315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7615.7 | uss_MB: 6643.2 | child_rss_MB: 3549.5 | allocated_MB: 341.4 | reserved_MB: 23132.0 | inactive_split_MB: 238.6 +rank1 [2026-10-29 11:47:05,943] INFO : Step 34600/50000 rank1 | Train Loss: 1.0317 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1201 | Classification: 0.0877 | Classification_binary: 0.1434 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1359 | Regression_eta: 0.0014 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1251 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0521 | Regression_eta: 0.0242 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0428 | LR: 2.95e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2921s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.6882 | max_loss: 0.1360 | max_backward: 1.3984 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5787 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7607.9 | uss_MB: 6553.3 | child_rss_MB: 3582.8 | allocated_MB: 343.8 | reserved_MB: 23132.0 | inactive_split_MB: 233.1 +rank1 [2026-10-29 11:51:12,923] INFO : Step 34700/50000 rank1 | Train Loss: 1.1152 | Train Loss Unweighted: 0.5285 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1612 | Classification: 0.0937 | Classification_binary: 0.1734 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1283 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0570 | Regression_eta: 0.0509 | Regression_pt: 0.0599 | Regression_sin_phi: 0.1022 | LR: 2.92e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5247s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7859 | max_loss: 0.2571 | max_backward: 1.9119 | max_optimizer_step: 0.2516 | max_tensorboard_logging: 0.5032 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7610.6 | uss_MB: 7599.4 | child_rss_MB: 3559.6 | allocated_MB: 343.9 | reserved_MB: 23132.0 | inactive_split_MB: 446.8 +rank1 [2026-10-29 11:54:56,884] INFO : Step 34800/50000 rank1 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5142 | Train Loss Components: Auxiliary: 0.2371 | Cardinality: 0.1583 | Classification: 0.0857 | Classification_binary: 0.1750 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1255 | Regression_eta: 0.0013 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1665 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0718 | Regression_eta: 0.0513 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0566 | LR: 2.90e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2204s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0423 | max_forward: 0.8214 | max_loss: 0.1472 | max_backward: 1.4019 | max_optimizer_step: 0.3234 | max_tensorboard_logging: 0.5950 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7613.6 | uss_MB: 7602.4 | child_rss_MB: 3541.0 | allocated_MB: 344.5 | reserved_MB: 23132.0 | inactive_split_MB: 201.5 +rank1 [2026-10-29 11:58:48,856] INFO : Step 34900/50000 rank1 | Train Loss: 0.9512 | Train Loss Unweighted: 0.4678 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1120 | Classification: 0.0788 | Classification_binary: 0.1318 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1262 | Regression_eta: 0.0014 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1274 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0611 | Regression_eta: 0.0324 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0384 | LR: 2.88e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.2610s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0986 | max_forward: 0.7397 | max_loss: 0.1421 | max_backward: 1.3918 | max_optimizer_step: 0.3560 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7614.6 | uss_MB: 7602.0 | child_rss_MB: 3543.9 | allocated_MB: 341.5 | reserved_MB: 23500.0 | inactive_split_MB: 262.9 +rank1 [2026-10-29 12:02:43,664] INFO : Step 35000/50000 rank1 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1401 | Classification: 0.0852 | Classification_binary: 0.1555 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0012 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.1845 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0608 | Regression_eta: 0.0509 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0835 | LR: 2.85e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7046s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0159 | max_forward: 0.7961 | max_loss: 0.2615 | max_backward: 2.0104 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6382 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7618.3 | uss_MB: 7606.3 | child_rss_MB: 3544.7 | allocated_MB: 345.2 | reserved_MB: 23500.0 | inactive_split_MB: 284.5 +rank1 [2026-10-29 12:02:44,718] INFO : Running validation on rank1 +rank1 [2026-10-29 12:02:44,718] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 12:02:44,718] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 12:03:33,663] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 12:03:34,352] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 12:04:01,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 12:04:03,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 12:04:04,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 12:04:05,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 12:04:15,892] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 12:04:17,999] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 12:04:18,510] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 12:04:34,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 12:04:35,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 12:04:36,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 12:04:37,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 12:04:47,494] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 12:04:48,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 12:04:48,821] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 12:05:04,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 12:05:05,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 12:05:05,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 12:05:06,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 12:05:16,489] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 12:09:34,463] INFO : Step 35100/50000 rank1 | Train Loss: 1.1060 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1387 | Classification: 0.0926 | Classification_binary: 0.1848 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0014 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0835 | Classification_binary: 0.1985 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0567 | Regression_eta: 0.0724 | Regression_pt: 0.0602 | Regression_sin_phi: 0.1028 | LR: 2.83e-04 | DataLoad Time: 0.0140s | Model Forward Time: 6.6717s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.9070 | max_loss: 0.1513 | max_backward: 2.1746 | max_optimizer_step: 0.2850 | max_tensorboard_logging: 1.1439 | max_tensorboard_flush: 0.0024 | max_post_step_barrier: 0.0768 | Memory: rss_MB: 9217.3 | uss_MB: 7564.1 | child_rss_MB: 3570.6 | allocated_MB: 344.3 | reserved_MB: 22622.0 | inactive_split_MB: 852.9 +rank1 [2026-10-29 12:13:24,279] INFO : Step 35200/50000 rank1 | Train Loss: 1.0386 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1293 | Classification: 0.0826 | Classification_binary: 0.1559 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1247 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1399 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0691 | Regression_eta: 0.0453 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0475 | LR: 2.81e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.5308s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.8193 | max_loss: 0.1608 | max_backward: 1.3752 | max_optimizer_step: 0.2535 | max_tensorboard_logging: 0.9185 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0700 | Memory: rss_MB: 7738.0 | uss_MB: 7717.0 | child_rss_MB: 3546.7 | allocated_MB: 341.8 | reserved_MB: 25614.0 | inactive_split_MB: 192.0 +rank1 [2026-10-29 12:17:18,034] INFO : Step 35300/50000 rank1 | Train Loss: 0.9736 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1059 | Classification: 0.0970 | Classification_binary: 0.1371 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0016 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1350 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0451 | Regression_eta: 0.0254 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0618 | LR: 2.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6491s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6434 | max_loss: 0.2341 | max_backward: 1.6834 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0588 | Memory: rss_MB: 7740.0 | uss_MB: 7297.3 | child_rss_MB: 3543.2 | allocated_MB: 343.2 | reserved_MB: 25614.0 | inactive_split_MB: 226.8 +rank1 [2026-10-29 12:21:06,025] INFO : Step 35400/50000 rank1 | Train Loss: 1.1186 | Train Loss Unweighted: 0.5363 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1393 | Classification: 0.1069 | Classification_binary: 0.1767 | HitGrouping: 0.1943 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0016 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1667 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0663 | Regression_eta: 0.0294 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0510 | LR: 2.76e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.8168s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0060 | max_forward: 0.7591 | max_loss: 0.1562 | max_backward: 1.6040 | max_optimizer_step: 0.2543 | max_tensorboard_logging: 0.9624 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.0563 | Memory: rss_MB: 7735.5 | uss_MB: 7248.0 | child_rss_MB: 3540.1 | allocated_MB: 345.3 | reserved_MB: 25762.0 | inactive_split_MB: 277.5 +rank1 [2026-10-29 12:24:54,118] INFO : Step 35500/50000 rank1 | Train Loss: 1.1216 | Train Loss Unweighted: 0.5466 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1527 | Classification: 0.0970 | Classification_binary: 0.1767 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1342 | Regression_eta: 0.0014 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1593 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0547 | Regression_eta: 0.0394 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0569 | LR: 2.74e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4079s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0648 | max_forward: 0.8748 | max_loss: 0.1544 | max_backward: 1.6387 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 0.5130 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0632 | Memory: rss_MB: 7738.5 | uss_MB: 7721.7 | child_rss_MB: 3543.4 | allocated_MB: 343.4 | reserved_MB: 25762.0 | inactive_split_MB: 284.4 +rank1 [2026-10-29 12:28:50,968] INFO : Step 35600/50000 rank1 | Train Loss: 1.0380 | Train Loss Unweighted: 0.5039 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1124 | Classification: 0.0922 | Classification_binary: 0.1405 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1353 | Regression_cos_phi: 0.1012 | Regression_energy: 0.0634 | Regression_eta: 0.0364 | Regression_pt: 0.0657 | Regression_sin_phi: 0.1003 | LR: 2.72e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8982s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0559 | max_forward: 0.7376 | max_loss: 0.1415 | max_backward: 1.1754 | max_optimizer_step: 0.2524 | max_tensorboard_logging: 0.5225 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1603 | Memory: rss_MB: 7739.4 | uss_MB: 7724.9 | child_rss_MB: 3557.9 | allocated_MB: 343.8 | reserved_MB: 25762.0 | inactive_split_MB: 233.6 +rank1 [2026-10-29 12:32:54,485] INFO : Step 35700/50000 rank1 | Train Loss: 1.0352 | Train Loss Unweighted: 0.5000 | Train Loss Components: Auxiliary: 0.2288 | Cardinality: 0.1199 | Classification: 0.1043 | Classification_binary: 0.1492 | HitGrouping: 0.1850 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1212 | Regression_eta: 0.0015 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0540 | Regression_eta: 0.0388 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0475 | LR: 2.70e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3750s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0360 | max_forward: 0.7358 | max_loss: 0.1419 | max_backward: 1.6761 | max_optimizer_step: 0.2531 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7741.1 | uss_MB: 7726.8 | child_rss_MB: 3543.5 | allocated_MB: 344.2 | reserved_MB: 26034.0 | inactive_split_MB: 271.8 +rank1 [2026-10-29 12:36:36,039] INFO : Step 35800/50000 rank1 | Train Loss: 1.0929 | Train Loss Unweighted: 0.5323 | Train Loss Components: Auxiliary: 0.2446 | Cardinality: 0.1241 | Classification: 0.0986 | Classification_binary: 0.1801 | HitGrouping: 0.1908 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0014 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1912 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0595 | Regression_eta: 0.0317 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0408 | LR: 2.68e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1522s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7732 | max_loss: 0.1506 | max_backward: 1.4048 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0581 | Memory: rss_MB: 7740.3 | uss_MB: 7206.4 | child_rss_MB: 3548.3 | allocated_MB: 344.1 | reserved_MB: 26034.0 | inactive_split_MB: 276.1 +rank1 [2026-10-29 14:48:22,160] INFO : Initializing model with seed=12345; process seed=12346 +rank1 [2026-10-29 14:48:22,177] INFO : configured dtype=torch.bfloat16 for autocast +rank1 [2026-10-29 14:48:45,047] INFO : Instantiating model +rank1 [2026-10-29 14:48:45,047] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank1 [2026-10-29 14:48:45,047] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank1 [2026-10-29 14:48:45,065] INFO : Input encoding initialization took 0.02s +rank1 [2026-10-29 14:48:45,065] INFO : nn0 parameters: 171776 +rank1 [2026-10-29 14:48:45,065] INFO : Initializing shared backbone layers, num_convs=6 +rank1 [2026-10-29 14:48:45,065] INFO : Initializing backbone layer 0 +rank1 [2026-10-29 14:48:45,065] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:48:45,071] INFO : Initializing backbone layer 1 +rank1 [2026-10-29 14:48:45,071] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:48:45,076] INFO : Initializing backbone layer 2 +rank1 [2026-10-29 14:48:45,076] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:48:45,082] INFO : Initializing backbone layer 3 +rank1 [2026-10-29 14:48:45,082] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:48:45,087] INFO : Initializing backbone layer 4 +rank1 [2026-10-29 14:48:45,087] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:48:45,092] INFO : Initializing backbone layer 5 +rank1 [2026-10-29 14:48:45,092] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank1 [2026-10-29 14:48:45,097] INFO : Backbone initialization took 0.03s +rank1 [2026-10-29 14:48:45,098] INFO : backbone parameters: 2374656 +rank1 [2026-10-29 14:48:45,098] INFO : Initializing output DNNs +rank1 [2026-10-29 14:48:45,135] INFO : Output DNNs initialization took 0.04s +rank1 [2026-10-29 14:48:45,135] INFO : backbone_mode=BackboneMode.SHARED +rank1 [2026-10-29 14:48:45,135] INFO : jagged_attention enabled=True +rank1 [2026-10-29 14:48:45,135] INFO : flash_attn_varlen enabled=True +rank1 [2026-10-29 14:48:45,135] INFO : task_queries enabled=False +rank1 [2026-10-29 14:48:45,135] INFO : classification_norm parameters: 0 +rank1 [2026-10-29 14:48:45,135] INFO : regression_norm parameters: 0 +rank1 [2026-10-29 14:48:45,135] INFO : classification_query parameters: 0 +rank1 [2026-10-29 14:48:45,136] INFO : regression_query parameters: 0 +rank1 [2026-10-29 14:48:45,136] INFO : classification_readout parameters: 0 +rank1 [2026-10-29 14:48:45,136] INFO : regression_readout parameters: 0 +rank1 [2026-10-29 14:48:45,136] INFO : output_mode=set +rank1 [2026-10-29 14:48:45,136] INFO : set_decoder parameters: 4713812 +rank1 [2026-10-29 14:48:45,136] INFO : nn_binary_particle parameters: 0 +rank1 [2026-10-29 14:48:45,136] INFO : nn_pid parameters: 0 +rank1 [2026-10-29 14:48:45,136] INFO : nn_pt parameters: 0 +rank1 [2026-10-29 14:48:45,136] INFO : nn_eta parameters: 0 +rank1 [2026-10-29 14:48:45,136] INFO : nn_sin_phi parameters: 0 +rank1 [2026-10-29 14:48:45,136] INFO : nn_cos_phi parameters: 0 +rank1 [2026-10-29 14:48:45,136] INFO : nn_energy parameters: 0 +rank1 [2026-10-29 14:48:45,137] INFO : Total MLPF parameters: 7260244 +rank1 [2026-10-29 14:48:45,137] INFO : MLPF __init__ done +rank1 [2026-10-29 14:48:45,137] INFO : Instantiated model +rank1 [2026-10-29 14:48:45,137] INFO : Moving model to device rank=1 +rank1 [2026-10-29 14:48:45,174] INFO : Moved model to device rank=1 +rank1 [2026-10-29 14:48:45,175] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank1 [2026-10-29 14:48:45,514] INFO : Restoring training from step 35001 +rank1 [2026-10-29 14:48:45,520] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth +rank1 [2026-10-29 14:48:45,521] INFO : Configured model for SyncBatchNorm rank=1 +rank1 [2026-10-29 14:48:45,764] INFO : Configured model for DistributedDataParallel rank=1 +rank1 [2026-10-29 14:48:45,874] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank1 [2026-10-29 14:48:45,875] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank1 [2026-10-29 14:48:45,876] INFO : Trainable parameters: 7260244 +rank1 [2026-10-29 14:48:45,876] INFO : Non-trainable parameters: 0 +rank1 [2026-10-29 14:48:45,876] INFO : Total parameters: 7260244 +rank1 [2026-10-29 14:48:45,879] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank1 [2026-10-29 14:48:45,879] INFO : Getting dataloaders +rank1 [2026-10-29 14:48:45,879] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:48:51,663] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:48:52,987] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:48:53,925] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-29 14:48:53,925] INFO : Creating WeightedMixIterator with 1 data loaders. +rank1 [2026-10-29 14:48:53,925] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:48:54,206] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:48:54,467] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:48:54,728] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-29 14:48:54,729] INFO : Creating InterleavedIterator with 1 data loaders. +rank1 [2026-10-29 14:49:08,505] INFO : Got dataloaders +rank1 [2026-10-29 14:49:08,505] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13888] of epochs [1] +rank1 [2026-10-29 14:49:08,505] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank1 [2026-10-29 14:49:08,505] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank1 [2026-10-29 14:49:08,505] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank1 [2026-10-29 14:49:10,195] INFO : loader split=train rank=1 len=21112 +rank1 [2026-10-29 14:49:10,195] INFO : loader split=valid rank=1 len=12 +rank1 [2026-10-29 14:49:10,195] INFO : Starting training from step 35001 to 50000 on rank 1 of 8 +rank1 [2026-10-29 14:49:10,195] INFO : Creating train_iterator +rank1 [2026-10-29 14:49:10,195] INFO : ResumableSampler physical:train loading state: start_index=222208 +rank1 [2026-10-29 14:49:24,147] INFO : Created train_iterator +rank1 [2026-10-29 14:53:12,251] INFO : Step 35100/50000 rank1 | Train Loss: 1.0826 | Train Loss Unweighted: 0.5185 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1393 | Classification: 0.0961 | Classification_binary: 0.1822 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.1975 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0573 | Regression_eta: 0.0667 | Regression_pt: 0.0610 | Regression_sin_phi: 0.1023 | LR: 2.83e-04 | DataLoad Time: 0.0137s | Model Forward Time: 3.4938s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.7340 | max_loss: 0.1651 | max_backward: 1.4152 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4761 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4625.8 | uss_MB: 3388.6 | child_rss_MB: 5578.9 | allocated_MB: 372.6 | reserved_MB: 22674.0 | inactive_split_MB: 823.1 +rank1 [2026-10-29 14:56:45,511] INFO : Step 35200/50000 rank1 | Train Loss: 1.0562 | Train Loss Unweighted: 0.5125 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1301 | Classification: 0.0878 | Classification_binary: 0.1540 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1331 | Regression_eta: 0.0017 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0671 | Regression_eta: 0.0365 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0487 | LR: 2.81e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.1436s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6042 | max_loss: 0.1411 | max_backward: 1.1580 | max_optimizer_step: 0.1447 | max_tensorboard_logging: 0.5108 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4625.0 | uss_MB: 3391.6 | child_rss_MB: 5591.1 | allocated_MB: 368.1 | reserved_MB: 25668.0 | inactive_split_MB: 192.7 +rank1 [2026-10-29 15:00:19,320] INFO : Step 35300/50000 rank1 | Train Loss: 0.9962 | Train Loss Unweighted: 0.4725 | Train Loss Components: Auxiliary: 0.2160 | Cardinality: 0.1172 | Classification: 0.0927 | Classification_binary: 0.1341 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0543 | Regression_eta: 0.0250 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0512 | LR: 2.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.0273s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.5940 | max_loss: 0.1358 | max_backward: 1.1911 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0027 | Memory: rss_MB: 4636.4 | uss_MB: 3402.6 | child_rss_MB: 5598.9 | allocated_MB: 371.5 | reserved_MB: 25668.0 | inactive_split_MB: 207.6 +rank1 [2026-10-29 15:03:53,852] INFO : Step 35400/50000 rank1 | Train Loss: 1.1270 | Train Loss Unweighted: 0.5445 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1356 | Classification: 0.0990 | Classification_binary: 0.1805 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1297 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1714 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0765 | Regression_eta: 0.0475 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0449 | LR: 2.76e-04 | DataLoad Time: 0.0142s | Model Forward Time: 3.6510s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1507 | max_backward: 1.5023 | max_optimizer_step: 0.2459 | max_tensorboard_logging: 0.5679 | max_tensorboard_flush: 0.0064 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4646.3 | uss_MB: 3410.8 | child_rss_MB: 5599.4 | allocated_MB: 371.4 | reserved_MB: 25816.0 | inactive_split_MB: 263.6 +rank1 [2026-10-29 15:07:33,750] INFO : Step 35500/50000 rank1 | Train Loss: 1.0502 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1302 | Classification: 0.0904 | Classification_binary: 0.1703 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1250 | Regression_eta: 0.0013 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0319 | Regression_energy: 0.0502 | Regression_eta: 0.0318 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0441 | LR: 2.74e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.6103s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0737 | max_forward: 0.7546 | max_loss: 0.1581 | max_backward: 1.4977 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4644.7 | uss_MB: 3412.9 | child_rss_MB: 5568.4 | allocated_MB: 371.1 | reserved_MB: 25816.0 | inactive_split_MB: 263.4 +rank1 [2026-10-29 15:11:19,389] INFO : Step 35600/50000 rank1 | Train Loss: 1.0258 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2226 | Cardinality: 0.1119 | Classification: 0.0922 | Classification_binary: 0.1418 | HitGrouping: 0.1950 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1286 | Regression_eta: 0.0015 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1325 | Regression_cos_phi: 0.1021 | Regression_energy: 0.0707 | Regression_eta: 0.0360 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0624 | LR: 2.72e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.2544s | Phase Times: max_pre_step_barrier: 0.0929 | max_batch_to_device: 0.0064 | max_forward: 0.5696 | max_loss: 0.1430 | max_backward: 1.1627 | max_optimizer_step: 0.1475 | max_tensorboard_logging: 0.5805 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4646.9 | uss_MB: 3412.7 | child_rss_MB: 5602.7 | allocated_MB: 370.5 | reserved_MB: 25816.0 | inactive_split_MB: 212.3 +rank1 [2026-10-29 15:15:08,570] INFO : Step 35700/50000 rank1 | Train Loss: 1.0162 | Train Loss Unweighted: 0.5013 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1185 | Classification: 0.0946 | Classification_binary: 0.1537 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1248 | Regression_eta: 0.0015 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0569 | Regression_eta: 0.0446 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0460 | LR: 2.70e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5934s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7397 | max_loss: 0.1413 | max_backward: 1.6688 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4646.6 | uss_MB: 3411.7 | child_rss_MB: 5604.2 | allocated_MB: 372.2 | reserved_MB: 26084.0 | inactive_split_MB: 249.8 +rank1 [2026-10-29 15:18:49,408] INFO : Step 35800/50000 rank1 | Train Loss: 1.0884 | Train Loss Unweighted: 0.5265 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1296 | Classification: 0.0948 | Classification_binary: 0.1803 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1775 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0604 | Regression_eta: 0.0410 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0457 | LR: 2.68e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.2011s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0199 | max_forward: 0.9402 | max_loss: 0.2389 | max_backward: 1.7889 | max_optimizer_step: 0.2466 | max_tensorboard_logging: 0.7071 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4643.4 | uss_MB: 3414.5 | child_rss_MB: 5602.6 | allocated_MB: 371.6 | reserved_MB: 26084.0 | inactive_split_MB: 257.5 +rank1 [2026-10-29 15:22:30,828] INFO : Step 35900/50000 rank1 | Train Loss: 1.0760 | Train Loss Unweighted: 0.5299 | Train Loss Components: Auxiliary: 0.2328 | Cardinality: 0.1277 | Classification: 0.0983 | Classification_binary: 0.1568 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1349 | Regression_eta: 0.0016 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1447 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0668 | Regression_eta: 0.0417 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0861 | LR: 2.65e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.2896s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6268 | max_loss: 0.2373 | max_backward: 1.2570 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.4934 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0016 | Memory: rss_MB: 4639.4 | uss_MB: 3412.6 | child_rss_MB: 5605.5 | allocated_MB: 369.7 | reserved_MB: 26800.0 | inactive_split_MB: 294.3 +rank1 [2026-10-29 15:26:13,159] INFO : Step 36000/50000 rank1 | Train Loss: 1.0831 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1276 | Classification: 0.1031 | Classification_binary: 0.1434 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1452 | Regression_eta: 0.0017 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1236 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0651 | Regression_eta: 0.0261 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0408 | LR: 2.63e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.2191s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6266 | max_loss: 0.1437 | max_backward: 1.3209 | max_optimizer_step: 0.1464 | max_tensorboard_logging: 0.4977 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4620.1 | uss_MB: 3411.5 | child_rss_MB: 5582.3 | allocated_MB: 368.7 | reserved_MB: 26800.0 | inactive_split_MB: 270.9 +rank1 [2026-10-29 15:26:14,074] INFO : Running validation on rank1 +rank1 [2026-10-29 15:26:14,074] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 15:26:14,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 15:27:03,341] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:27:03,871] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 15:27:29,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 15:27:30,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 15:27:31,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 15:27:32,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 15:27:42,339] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 15:27:42,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:27:43,314] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 15:27:59,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 15:28:00,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 15:28:01,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 15:28:02,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 15:28:12,343] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 15:28:12,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:28:13,229] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 15:28:28,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 15:28:29,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 15:28:30,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 15:28:30,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 15:28:40,947] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 15:32:50,524] INFO : Step 36100/50000 rank1 | Train Loss: 1.0137 | Train Loss Unweighted: 0.5026 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1079 | Classification: 0.1014 | Classification_binary: 0.1522 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1216 | Regression_eta: 0.0013 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.1837 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0605 | Regression_eta: 0.0416 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0538 | LR: 2.61e-04 | DataLoad Time: 0.0118s | Model Forward Time: 4.0563s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0065 | max_forward: 0.6968 | max_loss: 0.2782 | max_backward: 1.4278 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.9130 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6101.3 | uss_MB: 4907.2 | child_rss_MB: 5609.1 | allocated_MB: 372.5 | reserved_MB: 23174.0 | inactive_split_MB: 246.6 +rank1 [2026-10-29 15:36:33,882] INFO : Step 36200/50000 rank1 | Train Loss: 1.0439 | Train Loss Unweighted: 0.5147 | Train Loss Components: Auxiliary: 0.2338 | Cardinality: 0.1304 | Classification: 0.0909 | Classification_binary: 0.1697 | HitGrouping: 0.1917 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1255 | Regression_eta: 0.0011 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1659 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0585 | Regression_eta: 0.0263 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0528 | LR: 2.59e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5411s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7440 | max_loss: 0.1544 | max_backward: 1.4037 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 6101.4 | uss_MB: 4909.6 | child_rss_MB: 5587.9 | allocated_MB: 367.8 | reserved_MB: 23174.0 | inactive_split_MB: 282.8 +rank1 [2026-10-29 15:40:20,924] INFO : Step 36300/50000 rank1 | Train Loss: 1.0379 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1216 | Classification: 0.0886 | Classification_binary: 0.1447 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1266 | Regression_eta: 0.0016 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1201 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0619 | Regression_eta: 0.0707 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0746 | LR: 2.57e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.3296s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6106 | max_loss: 0.2412 | max_backward: 1.3407 | max_optimizer_step: 0.1535 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1063 | Memory: rss_MB: 5450.1 | uss_MB: 4269.0 | child_rss_MB: 5611.1 | allocated_MB: 370.5 | reserved_MB: 23174.0 | inactive_split_MB: 274.2 +rank1 [2026-10-29 15:44:06,295] INFO : Step 36400/50000 rank1 | Train Loss: 1.0182 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.0976 | Classification: 0.0939 | Classification_binary: 0.1520 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1167 | Regression_eta: 0.0016 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0633 | Regression_eta: 0.0321 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0804 | LR: 2.55e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5215s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.6772 | max_loss: 0.1711 | max_backward: 1.4058 | max_optimizer_step: 0.1518 | max_tensorboard_logging: 0.5460 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0089 | Memory: rss_MB: 5451.9 | uss_MB: 4272.1 | child_rss_MB: 5615.2 | allocated_MB: 371.8 | reserved_MB: 23382.0 | inactive_split_MB: 230.6 +rank1 [2026-10-29 15:47:54,065] INFO : Step 36500/50000 rank1 | Train Loss: 1.0953 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1370 | Classification: 0.0895 | Classification_binary: 0.1838 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0554 | Regression_eta: 0.0600 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0890 | LR: 2.52e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9829s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7475 | max_loss: 0.2537 | max_backward: 1.7352 | max_optimizer_step: 0.1520 | max_tensorboard_logging: 0.6267 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5440.5 | uss_MB: 4262.4 | child_rss_MB: 5614.9 | allocated_MB: 372.3 | reserved_MB: 24160.0 | inactive_split_MB: 275.1 +rank1 [2026-10-29 15:51:40,079] INFO : Step 36600/50000 rank1 | Train Loss: 0.9967 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1165 | Classification: 0.0804 | Classification_binary: 0.1438 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1366 | Regression_eta: 0.0013 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1131 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0597 | Regression_eta: 0.0320 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0457 | LR: 2.50e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.6284s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6592 | max_loss: 0.2501 | max_backward: 1.4213 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5953 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5450.1 | uss_MB: 4272.8 | child_rss_MB: 5617.4 | allocated_MB: 367.8 | reserved_MB: 27180.0 | inactive_split_MB: 225.4 +rank1 [2026-10-29 15:55:26,196] INFO : Step 36700/50000 rank1 | Train Loss: 0.9770 | Train Loss Unweighted: 0.4817 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1070 | Classification: 0.0901 | Classification_binary: 0.1347 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0652 | Regression_eta: 0.0373 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0278 | LR: 2.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.2746s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0062 | max_forward: 0.5676 | max_loss: 0.2381 | max_backward: 1.1343 | max_optimizer_step: 0.2417 | max_tensorboard_logging: 0.4693 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5448.5 | uss_MB: 4274.1 | child_rss_MB: 5620.1 | allocated_MB: 370.8 | reserved_MB: 27180.0 | inactive_split_MB: 235.2 +rank1 [2026-10-29 15:59:20,983] INFO : Step 36800/50000 rank1 | Train Loss: 1.0234 | Train Loss Unweighted: 0.4967 | Train Loss Components: Auxiliary: 0.2267 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1662 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1197 | Regression_eta: 0.0013 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1806 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0637 | Regression_eta: 0.0340 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0507 | LR: 2.46e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.5128s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 1.1394 | max_loss: 0.1518 | max_backward: 2.3088 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5450.3 | uss_MB: 4278.9 | child_rss_MB: 5629.5 | allocated_MB: 371.9 | reserved_MB: 27180.0 | inactive_split_MB: 230.7 +rank1 [2026-10-29 16:03:11,411] INFO : Step 36900/50000 rank1 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2268 | Cardinality: 0.1211 | Classification: 0.0855 | Classification_binary: 0.1637 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1219 | Regression_eta: 0.0015 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1447 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0606 | Regression_eta: 0.0406 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0506 | LR: 2.44e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.5009s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0071 | max_forward: 0.7005 | max_loss: 0.1465 | max_backward: 1.2640 | max_optimizer_step: 0.1579 | max_tensorboard_logging: 0.4728 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5451.1 | uss_MB: 4280.1 | child_rss_MB: 5636.6 | allocated_MB: 372.3 | reserved_MB: 27180.0 | inactive_split_MB: 201.7 +rank1 [2026-10-29 16:07:15,145] INFO : Step 37000/50000 rank1 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2194 | Cardinality: 0.1268 | Classification: 0.0926 | Classification_binary: 0.1402 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0014 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1337 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0690 | Regression_eta: 0.0247 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0519 | LR: 2.42e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.5221s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6075 | max_loss: 0.2475 | max_backward: 1.2956 | max_optimizer_step: 0.1585 | max_tensorboard_logging: 0.7613 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5445.1 | uss_MB: 4277.2 | child_rss_MB: 5637.5 | allocated_MB: 368.1 | reserved_MB: 27180.0 | inactive_split_MB: 226.3 +rank1 [2026-10-29 16:07:16,059] INFO : Running validation on rank1 +rank1 [2026-10-29 16:07:16,059] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 16:07:16,059] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 16:08:14,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:08:14,583] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 16:08:39,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 16:08:40,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 16:08:41,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 16:08:42,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 16:08:52,733] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 16:08:53,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:08:53,527] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 16:09:09,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 16:09:10,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 16:09:11,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 16:09:12,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 16:09:22,336] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 16:09:23,587] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:09:24,062] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 16:09:39,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 16:09:40,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 16:09:40,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 16:09:41,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 16:09:51,452] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 16:13:56,322] INFO : Step 37100/50000 rank1 | Train Loss: 1.1150 | Train Loss Unweighted: 0.5431 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1226 | Classification: 0.1105 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1340 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1592 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0595 | Regression_eta: 0.0305 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0808 | LR: 2.40e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8589s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7539 | max_loss: 0.3988 | max_backward: 1.7903 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4747 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6593.7 | uss_MB: 5426.2 | child_rss_MB: 5642.5 | allocated_MB: 372.5 | reserved_MB: 23160.0 | inactive_split_MB: 265.5 +rank1 [2026-10-29 16:17:46,448] INFO : Step 37200/50000 rank1 | Train Loss: 1.0752 | Train Loss Unweighted: 0.5181 | Train Loss Components: Auxiliary: 0.2406 | Cardinality: 0.1258 | Classification: 0.0924 | Classification_binary: 0.1812 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1200 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0620 | Regression_eta: 0.0503 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0746 | LR: 2.38e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.1722s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0549 | max_forward: 0.8058 | max_loss: 0.1510 | max_backward: 1.8476 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5280 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6591.2 | uss_MB: 5426.9 | child_rss_MB: 5644.6 | allocated_MB: 371.5 | reserved_MB: 24140.0 | inactive_split_MB: 271.0 +rank1 [2026-10-29 16:21:30,618] INFO : Step 37300/50000 rank1 | Train Loss: 1.0230 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1563 | HitGrouping: 0.1981 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1213 | Regression_eta: 0.0016 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1364 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0749 | Regression_eta: 0.0393 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0541 | LR: 2.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0290s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.6853 | max_loss: 0.1502 | max_backward: 1.3245 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 1.0657 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5580.1 | uss_MB: 4415.2 | child_rss_MB: 5668.9 | allocated_MB: 370.9 | reserved_MB: 24140.0 | inactive_split_MB: 261.7 +rank1 [2026-10-29 16:25:14,031] INFO : Step 37400/50000 rank1 | Train Loss: 1.0130 | Train Loss Unweighted: 0.4929 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1180 | Classification: 0.0961 | Classification_binary: 0.1390 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1273 | Regression_eta: 0.0015 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0713 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0586 | Regression_eta: 0.0416 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0695 | LR: 2.34e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5047s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.5424 | max_loss: 0.1464 | max_backward: 1.0792 | max_optimizer_step: 0.1675 | max_tensorboard_logging: 0.8668 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5585.6 | uss_MB: 4420.8 | child_rss_MB: 5670.0 | allocated_MB: 368.8 | reserved_MB: 25122.0 | inactive_split_MB: 232.7 +rank1 [2026-10-29 16:30:41,706] INFO : Step 37500/50000 rank1 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1315 | Classification: 0.0992 | Classification_binary: 0.1679 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1176 | Regression_eta: 0.0013 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0525 | Regression_eta: 0.0465 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0627 | LR: 2.32e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1491s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.7240 | max_loss: 0.1472 | max_backward: 1.5381 | max_optimizer_step: 0.2707 | max_tensorboard_logging: 0.8554 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5585.8 | uss_MB: 4422.3 | child_rss_MB: 5642.8 | allocated_MB: 370.2 | reserved_MB: 26134.0 | inactive_split_MB: 179.7 +rank1 [2026-10-29 16:34:38,362] INFO : Step 37600/50000 rank1 | Train Loss: 1.0243 | Train Loss Unweighted: 0.4867 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1182 | Classification: 0.0808 | Classification_binary: 0.1619 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1659 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0550 | Regression_eta: 0.0355 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0539 | LR: 2.30e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1664s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.7069 | max_loss: 0.1543 | max_backward: 1.5139 | max_optimizer_step: 0.1704 | max_tensorboard_logging: 1.0159 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5587.9 | uss_MB: 4422.7 | child_rss_MB: 5672.0 | allocated_MB: 370.8 | reserved_MB: 26846.0 | inactive_split_MB: 273.3 +rank1 [2026-10-29 16:38:29,294] INFO : Step 37700/50000 rank1 | Train Loss: 0.9610 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2102 | Cardinality: 0.1049 | Classification: 0.0974 | Classification_binary: 0.1317 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0012 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0545 | Regression_eta: 0.0372 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0568 | LR: 2.28e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.5159s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6233 | max_loss: 0.2433 | max_backward: 1.3306 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5644 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5591.0 | uss_MB: 4425.5 | child_rss_MB: 5643.2 | allocated_MB: 369.4 | reserved_MB: 28822.0 | inactive_split_MB: 231.5 +rank1 [2026-10-29 16:42:35,307] INFO : Step 37800/50000 rank1 | Train Loss: 0.9994 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1105 | Classification: 0.0952 | Classification_binary: 0.1458 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1161 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0427 | Regression_eta: 0.0342 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0823 | LR: 2.26e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8892s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.6659 | max_loss: 0.1473 | max_backward: 1.3840 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.9558 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5593.5 | uss_MB: 4427.0 | child_rss_MB: 5671.1 | allocated_MB: 371.5 | reserved_MB: 28822.0 | inactive_split_MB: 170.1 +rank1 [2026-10-29 16:46:40,061] INFO : Step 37900/50000 rank1 | Train Loss: 1.0837 | Train Loss Unweighted: 0.5094 | Train Loss Components: Auxiliary: 0.2372 | Cardinality: 0.1419 | Classification: 0.0937 | Classification_binary: 0.1795 | HitGrouping: 0.1848 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1160 | Regression_eta: 0.0015 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0438 | Regression_eta: 0.0311 | Regression_pt: 0.0452 | Regression_sin_phi: 0.0591 | LR: 2.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2202s | Phase Times: max_pre_step_barrier: 0.1076 | max_batch_to_device: 0.0071 | max_forward: 0.7572 | max_loss: 0.1547 | max_backward: 1.4632 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 0.9205 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 5593.0 | uss_MB: 4429.3 | child_rss_MB: 5670.1 | allocated_MB: 370.2 | reserved_MB: 28822.0 | inactive_split_MB: 265.3 +rank1 [2026-10-29 16:50:31,519] INFO : Step 38000/50000 rank1 | Train Loss: 0.9811 | Train Loss Unweighted: 0.4782 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1213 | Classification: 0.0781 | Classification_binary: 0.1413 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1277 | Regression_eta: 0.0012 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1363 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0642 | Regression_eta: 0.0348 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0407 | LR: 2.22e-04 | DataLoad Time: 0.0224s | Model Forward Time: 3.5071s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.6120 | max_loss: 0.2171 | max_backward: 1.2526 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5836 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5590.4 | uss_MB: 4430.0 | child_rss_MB: 5667.2 | allocated_MB: 368.7 | reserved_MB: 28822.0 | inactive_split_MB: 223.4 +rank1 [2026-10-29 16:50:32,412] INFO : Running validation on rank1 +rank1 [2026-10-29 16:50:32,412] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 16:50:32,412] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 16:51:23,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:51:24,263] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 16:51:51,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 16:51:52,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 16:51:53,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 16:51:54,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 16:52:04,019] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 16:52:04,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:52:05,011] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 16:52:21,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 16:52:22,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 16:52:23,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 16:52:24,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 16:52:34,125] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 16:52:34,973] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:52:35,419] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 16:52:51,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 16:52:51,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 16:52:52,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 16:52:53,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 16:53:03,322] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 16:57:14,095] INFO : Step 38100/50000 rank1 | Train Loss: 1.0158 | Train Loss Unweighted: 0.4912 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1220 | Classification: 0.0982 | Classification_binary: 0.1455 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1224 | Regression_eta: 0.0013 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0644 | Regression_eta: 0.0331 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0467 | LR: 2.20e-04 | DataLoad Time: 0.0236s | Model Forward Time: 4.9426s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8406 | max_loss: 0.1820 | max_backward: 1.8536 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.3354 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0179 | Memory: rss_MB: 5714.6 | uss_MB: 4482.4 | child_rss_MB: 5669.8 | allocated_MB: 370.5 | reserved_MB: 24292.0 | inactive_split_MB: 257.8 +rank1 [2026-10-29 17:01:01,945] INFO : Step 38200/50000 rank1 | Train Loss: 1.0273 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2292 | Cardinality: 0.1060 | Classification: 0.0997 | Classification_binary: 0.1650 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1735 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0507 | Regression_eta: 0.0452 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0435 | LR: 2.18e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8906s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7758 | max_loss: 0.1794 | max_backward: 1.7969 | max_optimizer_step: 0.1713 | max_tensorboard_logging: 0.4698 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5717.4 | uss_MB: 4558.1 | child_rss_MB: 5664.1 | allocated_MB: 372.9 | reserved_MB: 26212.0 | inactive_split_MB: 221.1 +rank1 [2026-10-29 17:05:07,130] INFO : Step 38300/50000 rank1 | Train Loss: 1.0498 | Train Loss Unweighted: 0.5122 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1249 | Classification: 0.0913 | Classification_binary: 0.1583 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1294 | Regression_eta: 0.0013 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1562 | Regression_cos_phi: 0.1459 | Regression_energy: 0.0713 | Regression_eta: 0.0462 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0710 | LR: 2.16e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7357s | Phase Times: max_pre_step_barrier: 0.1103 | max_batch_to_device: 0.0086 | max_forward: 0.6730 | max_loss: 0.1509 | max_backward: 1.3787 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.4782 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5715.4 | uss_MB: 4556.5 | child_rss_MB: 5665.1 | allocated_MB: 369.2 | reserved_MB: 26212.0 | inactive_split_MB: 171.8 +rank1 [2026-10-29 17:09:08,354] INFO : Step 38400/50000 rank1 | Train Loss: 0.9354 | Train Loss Unweighted: 0.4735 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.0968 | Classification: 0.0760 | Classification_binary: 0.1246 | HitGrouping: 0.1889 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1340 | Regression_eta: 0.0013 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1015 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0603 | Regression_eta: 0.0250 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0424 | LR: 2.14e-04 | DataLoad Time: 0.0217s | Model Forward Time: 3.6834s | Phase Times: max_pre_step_barrier: 0.1107 | max_batch_to_device: 0.0062 | max_forward: 0.6635 | max_loss: 0.1419 | max_backward: 1.4771 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 0.4619 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5721.1 | uss_MB: 4559.4 | child_rss_MB: 5662.2 | allocated_MB: 370.2 | reserved_MB: 26212.0 | inactive_split_MB: 267.8 +rank1 [2026-10-29 17:13:06,091] INFO : Step 38500/50000 rank1 | Train Loss: 1.0482 | Train Loss Unweighted: 0.5077 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1228 | Classification: 0.1147 | Classification_binary: 0.1473 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1208 | Regression_eta: 0.0016 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0690 | Regression_eta: 0.0385 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0498 | LR: 2.12e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.4604s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.6383 | max_loss: 0.1393 | max_backward: 1.3136 | max_optimizer_step: 0.1695 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 5717.0 | uss_MB: 4562.8 | child_rss_MB: 5658.1 | allocated_MB: 370.2 | reserved_MB: 27188.0 | inactive_split_MB: 210.0 +rank1 [2026-10-29 17:17:04,837] INFO : Step 38600/50000 rank1 | Train Loss: 1.0321 | Train Loss Unweighted: 0.4950 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1157 | Classification: 0.0910 | Classification_binary: 0.1735 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1130 | Regression_eta: 0.0014 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1723 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0574 | Regression_eta: 0.0294 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0528 | LR: 2.11e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.0838s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0127 | max_forward: 0.7516 | max_loss: 0.1551 | max_backward: 1.6059 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.7250 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5714.3 | uss_MB: 4563.8 | child_rss_MB: 5662.6 | allocated_MB: 372.6 | reserved_MB: 28172.0 | inactive_split_MB: 283.9 +rank1 [2026-10-29 17:20:59,239] INFO : Step 38700/50000 rank1 | Train Loss: 0.9420 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1147 | Classification: 0.0828 | Classification_binary: 0.1348 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1155 | Regression_eta: 0.0013 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0586 | Regression_eta: 0.0296 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0603 | LR: 2.09e-04 | DataLoad Time: 0.0223s | Model Forward Time: 3.6808s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.8052 | max_loss: 0.2133 | max_backward: 1.3215 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5712.0 | uss_MB: 4410.4 | child_rss_MB: 5659.5 | allocated_MB: 367.4 | reserved_MB: 28172.0 | inactive_split_MB: 206.7 +rank1 [2026-10-29 17:25:08,218] INFO : Step 38800/50000 rank1 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4825 | Train Loss Components: Auxiliary: 0.2201 | Cardinality: 0.1186 | Classification: 0.0965 | Classification_binary: 0.1429 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1198 | Regression_eta: 0.0012 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0556 | Regression_eta: 0.0211 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0364 | LR: 2.07e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.6551s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.9011 | max_loss: 0.1880 | max_backward: 1.7487 | max_optimizer_step: 0.2166 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5710.9 | uss_MB: 4431.1 | child_rss_MB: 5666.7 | allocated_MB: 371.5 | reserved_MB: 28172.0 | inactive_split_MB: 220.5 +rank1 [2026-10-29 17:29:07,491] INFO : Step 38900/50000 rank1 | Train Loss: 0.9986 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2211 | Cardinality: 0.1078 | Classification: 0.0988 | Classification_binary: 0.1578 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1890 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0528 | Regression_eta: 0.0456 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0792 | LR: 2.05e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.4420s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0073 | max_forward: 0.9800 | max_loss: 0.2159 | max_backward: 2.2158 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0079 | Memory: rss_MB: 5714.2 | uss_MB: 3754.5 | child_rss_MB: 5665.7 | allocated_MB: 373.3 | reserved_MB: 29164.0 | inactive_split_MB: 189.2 +rank1 [2026-10-29 17:33:02,383] INFO : Step 39000/50000 rank1 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4758 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1227 | Classification: 0.0847 | Classification_binary: 0.1518 | HitGrouping: 0.1853 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1179 | Regression_eta: 0.0013 | Regression_pt: 0.1189 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0713 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0582 | Regression_eta: 0.0374 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0595 | LR: 2.03e-04 | DataLoad Time: 0.0190s | Model Forward Time: 3.7824s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0515 | max_forward: 0.7410 | max_loss: 0.1790 | max_backward: 1.4655 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5713.4 | uss_MB: 4569.9 | child_rss_MB: 5663.9 | allocated_MB: 370.5 | reserved_MB: 29164.0 | inactive_split_MB: 192.2 +rank1 [2026-10-29 17:33:03,293] INFO : Running validation on rank1 +rank1 [2026-10-29 17:33:03,293] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 17:33:56,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:33:56,829] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 17:34:24,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 17:34:25,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 17:34:26,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 17:34:27,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 17:34:37,868] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 17:34:38,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:34:38,631] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 17:34:54,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 17:34:55,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 17:34:56,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 17:34:57,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 17:35:07,448] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 17:35:07,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:35:08,264] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 17:35:23,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 17:35:24,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 17:35:25,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 17:35:25,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 17:35:35,808] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 17:39:50,704] INFO : Step 39100/50000 rank1 | Train Loss: 1.0043 | Train Loss Unweighted: 0.4886 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1149 | Classification: 0.0878 | Classification_binary: 0.1320 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1329 | Regression_eta: 0.0012 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1157 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0615 | Regression_eta: 0.0231 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0410 | LR: 2.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.6394s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0736 | max_forward: 0.7590 | max_loss: 0.4062 | max_backward: 1.4330 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.4941 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6810.5 | uss_MB: 5664.9 | child_rss_MB: 5641.2 | allocated_MB: 371.1 | reserved_MB: 23656.0 | inactive_split_MB: 194.8 +rank1 [2026-10-29 17:43:44,583] INFO : Step 39200/50000 rank1 | Train Loss: 1.0325 | Train Loss Unweighted: 0.5174 | Train Loss Components: Auxiliary: 0.2315 | Cardinality: 0.1117 | Classification: 0.1086 | Classification_binary: 0.1481 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0580 | Regression_eta: 0.0288 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0409 | LR: 2.00e-04 | DataLoad Time: 0.0215s | Model Forward Time: 3.6942s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0102 | max_forward: 0.7546 | max_loss: 0.4268 | max_backward: 1.4370 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5801.6 | uss_MB: 4660.4 | child_rss_MB: 5666.2 | allocated_MB: 371.2 | reserved_MB: 23934.0 | inactive_split_MB: 196.6 +rank1 [2026-10-29 17:47:41,748] INFO : Step 39300/50000 rank1 | Train Loss: 1.0373 | Train Loss Unweighted: 0.5050 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1241 | Classification: 0.0956 | Classification_binary: 0.1745 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1150 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1912 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0583 | Regression_eta: 0.0404 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0561 | LR: 1.98e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.2660s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0844 | max_forward: 0.8662 | max_loss: 0.2199 | max_backward: 1.7416 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5805.0 | uss_MB: 4500.0 | child_rss_MB: 5642.3 | allocated_MB: 373.1 | reserved_MB: 23934.0 | inactive_split_MB: 256.8 +rank1 [2026-10-29 17:51:43,102] INFO : Step 39400/50000 rank1 | Train Loss: 0.9741 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2189 | Cardinality: 0.1078 | Classification: 0.0808 | Classification_binary: 0.1461 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0501 | Regression_eta: 0.0227 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0661 | LR: 1.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5591s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.6778 | max_loss: 0.1515 | max_backward: 1.4156 | max_optimizer_step: 0.1740 | max_tensorboard_logging: 0.4653 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5807.5 | uss_MB: 4663.4 | child_rss_MB: 5665.4 | allocated_MB: 368.0 | reserved_MB: 23934.0 | inactive_split_MB: 199.1 +rank1 [2026-10-29 17:55:40,086] INFO : Step 39500/50000 rank1 | Train Loss: 0.9809 | Train Loss Unweighted: 0.4664 | Train Loss Components: Auxiliary: 0.2175 | Cardinality: 0.1105 | Classification: 0.0963 | Classification_binary: 0.1347 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1158 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1064 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0623 | Regression_eta: 0.0203 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0301 | LR: 1.94e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5617s | Phase Times: max_pre_step_barrier: 0.0259 | max_batch_to_device: 0.1050 | max_forward: 0.6813 | max_loss: 0.1541 | max_backward: 1.3449 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 0.5375 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0111 | Memory: rss_MB: 5806.7 | uss_MB: 4606.4 | child_rss_MB: 5664.0 | allocated_MB: 367.2 | reserved_MB: 24970.0 | inactive_split_MB: 230.9 +rank1 [2026-10-29 17:59:42,116] INFO : Step 39600/50000 rank1 | Train Loss: 1.0814 | Train Loss Unweighted: 0.5099 | Train Loss Components: Auxiliary: 0.2411 | Cardinality: 0.1344 | Classification: 0.0978 | Classification_binary: 0.1731 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0564 | Regression_eta: 0.0296 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0691 | LR: 1.93e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1501s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8839 | max_loss: 0.1464 | max_backward: 1.8057 | max_optimizer_step: 0.2568 | max_tensorboard_logging: 0.5748 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5805.3 | uss_MB: 4666.8 | child_rss_MB: 5646.1 | allocated_MB: 370.6 | reserved_MB: 24970.0 | inactive_split_MB: 243.4 +rank1 [2026-10-29 18:03:55,339] INFO : Step 39700/50000 rank1 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4877 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1185 | Classification: 0.0909 | Classification_binary: 0.1674 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1134 | Regression_eta: 0.0013 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0542 | Regression_eta: 0.0326 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0393 | LR: 1.91e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8041s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.6954 | max_loss: 0.2515 | max_backward: 1.5651 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.5599 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5806.9 | uss_MB: 4667.8 | child_rss_MB: 5665.2 | allocated_MB: 371.1 | reserved_MB: 24970.0 | inactive_split_MB: 244.9 +rank1 [2026-10-29 18:07:56,659] INFO : Step 39800/50000 rank1 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1022 | Classification: 0.0711 | Classification_binary: 0.1322 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1293 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0448 | Regression_eta: 0.0279 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0471 | LR: 1.89e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9399s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0063 | max_forward: 1.0012 | max_loss: 0.2859 | max_backward: 2.1063 | max_optimizer_step: 0.1800 | max_tensorboard_logging: 0.4881 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5810.9 | uss_MB: 3839.1 | child_rss_MB: 5666.9 | allocated_MB: 368.3 | reserved_MB: 24970.0 | inactive_split_MB: 241.7 +rank1 [2026-10-29 18:11:54,160] INFO : Step 39900/50000 rank1 | Train Loss: 0.9667 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.2188 | Cardinality: 0.1055 | Classification: 0.0897 | Classification_binary: 0.1526 | HitGrouping: 0.1883 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1101 | Regression_eta: 0.0012 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1493 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0574 | Regression_eta: 0.0272 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0429 | LR: 1.88e-04 | DataLoad Time: 0.0194s | Model Forward Time: 6.9954s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.3186 | max_loss: 0.4377 | max_backward: 3.6189 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.6123 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 5815.3 | uss_MB: 4217.6 | child_rss_MB: 5667.9 | allocated_MB: 370.5 | reserved_MB: 24970.0 | inactive_split_MB: 229.4 +rank1 [2026-10-29 18:15:59,759] INFO : Step 40000/50000 rank1 | Train Loss: 1.0232 | Train Loss Unweighted: 0.5131 | Train Loss Components: Auxiliary: 0.2348 | Cardinality: 0.1100 | Classification: 0.0962 | Classification_binary: 0.1712 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1206 | Regression_eta: 0.0013 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1799 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0599 | Regression_eta: 0.0462 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0442 | LR: 1.86e-04 | DataLoad Time: 0.0195s | Model Forward Time: 7.4717s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0077 | max_forward: 0.8168 | max_loss: 0.2161 | max_backward: 1.5709 | max_optimizer_step: 0.1733 | max_tensorboard_logging: 1.1681 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.1427 | Memory: rss_MB: 5816.6 | uss_MB: 3786.6 | child_rss_MB: 5665.4 | allocated_MB: 375.8 | reserved_MB: 24970.0 | inactive_split_MB: 259.2 +rank1 [2026-10-29 18:16:02,308] INFO : Running validation on rank1 +rank1 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 18:16:53,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:16:54,180] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 18:17:19,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 18:17:20,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 18:17:21,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 18:17:22,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 18:17:32,951] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 18:17:33,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:17:33,663] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 18:17:49,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 18:17:50,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 18:17:51,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 18:17:52,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 18:18:02,409] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 18:18:03,274] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:18:03,745] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 18:18:19,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 18:18:20,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 18:18:20,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 18:18:21,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 18:18:31,467] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 18:22:46,459] INFO : Step 40100/50000 rank1 | Train Loss: 0.9349 | Train Loss Unweighted: 0.4412 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1220 | Classification: 0.0762 | Classification_binary: 0.1328 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1148 | Regression_eta: 0.0012 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1160 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0522 | Regression_eta: 0.0228 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0446 | LR: 1.84e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.6883s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.7093 | max_loss: 0.3974 | max_backward: 1.3310 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.5555 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 7103.1 | uss_MB: 5953.7 | child_rss_MB: 5672.5 | allocated_MB: 367.6 | reserved_MB: 22178.0 | inactive_split_MB: 265.9 +rank1 [2026-10-29 18:26:52,186] INFO : Step 40200/50000 rank1 | Train Loss: 0.9570 | Train Loss Unweighted: 0.4556 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1119 | Classification: 0.0905 | Classification_binary: 0.1293 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0013 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0460 | Regression_eta: 0.0580 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0910 | LR: 1.83e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5889s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6287 | max_loss: 0.4129 | max_backward: 1.3901 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 0.5609 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5918.3 | uss_MB: 4768.5 | child_rss_MB: 5665.4 | allocated_MB: 370.8 | reserved_MB: 22294.0 | inactive_split_MB: 233.2 +rank1 [2026-10-29 18:30:59,535] INFO : Step 40300/50000 rank1 | Train Loss: 1.0209 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1232 | Classification: 0.0991 | Classification_binary: 0.1688 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1068 | Regression_eta: 0.0015 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1821 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0485 | Regression_eta: 0.0429 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0660 | LR: 1.81e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9649s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0749 | max_forward: 0.7563 | max_loss: 0.1523 | max_backward: 1.5452 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5632 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5917.0 | uss_MB: 4772.2 | child_rss_MB: 5652.3 | allocated_MB: 371.8 | reserved_MB: 23296.0 | inactive_split_MB: 234.2 +rank1 [2026-10-29 18:35:20,931] INFO : Step 40400/50000 rank1 | Train Loss: 1.0486 | Train Loss Unweighted: 0.5003 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1357 | Classification: 0.0888 | Classification_binary: 0.1690 | HitGrouping: 0.1879 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1196 | Regression_eta: 0.0015 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0525 | Regression_eta: 0.0393 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0508 | LR: 1.79e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.7348s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0072 | max_forward: 0.7368 | max_loss: 0.2443 | max_backward: 3.1596 | max_optimizer_step: 0.1851 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5913.1 | uss_MB: 3802.2 | child_rss_MB: 5663.8 | allocated_MB: 371.2 | reserved_MB: 23386.0 | inactive_split_MB: 254.8 +rank1 [2026-10-29 18:39:23,871] INFO : Step 40500/50000 rank1 | Train Loss: 0.9309 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.0955 | Classification: 0.0829 | Classification_binary: 0.1355 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0014 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0465 | Regression_eta: 0.0276 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0254 | LR: 1.78e-04 | DataLoad Time: 0.0140s | Model Forward Time: 3.7329s | Phase Times: max_pre_step_barrier: 0.0186 | max_batch_to_device: 0.0076 | max_forward: 0.6863 | max_loss: 0.2393 | max_backward: 1.3065 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.7004 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5916.1 | uss_MB: 4771.6 | child_rss_MB: 5669.9 | allocated_MB: 369.2 | reserved_MB: 24370.0 | inactive_split_MB: 237.3 +rank1 [2026-10-29 18:43:23,923] INFO : Step 40600/50000 rank1 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1087 | Classification: 0.0887 | Classification_binary: 0.1521 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0499 | Regression_eta: 0.0549 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0814 | LR: 1.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3449s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.1102 | max_forward: 0.7412 | max_loss: 0.1464 | max_backward: 1.4782 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 1.0719 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5917.7 | uss_MB: 4772.7 | child_rss_MB: 5655.3 | allocated_MB: 372.3 | reserved_MB: 24370.0 | inactive_split_MB: 234.4 +rank1 [2026-10-29 18:47:14,842] INFO : Step 40700/50000 rank1 | Train Loss: 1.0611 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1374 | Classification: 0.0904 | Classification_binary: 0.1773 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1190 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1753 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0578 | Regression_eta: 0.0531 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0624 | LR: 1.75e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2475s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7422 | max_loss: 0.1589 | max_backward: 1.5298 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5918.6 | uss_MB: 4774.9 | child_rss_MB: 5665.5 | allocated_MB: 372.7 | reserved_MB: 24370.0 | inactive_split_MB: 255.3 +rank1 [2026-10-29 18:51:03,369] INFO : Step 40800/50000 rank1 | Train Loss: 0.9701 | Train Loss Unweighted: 0.4785 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.0983 | Classification: 0.0901 | Classification_binary: 0.1373 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1164 | Regression_cos_phi: 0.0278 | Regression_energy: 0.0654 | Regression_eta: 0.0235 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0427 | LR: 1.73e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.7269s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5843 | max_loss: 0.1422 | max_backward: 1.0661 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.0389 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5915.8 | uss_MB: 4772.0 | child_rss_MB: 5661.2 | allocated_MB: 367.5 | reserved_MB: 24658.0 | inactive_split_MB: 205.4 +rank1 [2026-10-29 18:54:43,248] INFO : Step 40900/50000 rank1 | Train Loss: 0.9731 | Train Loss Unweighted: 0.4636 | Train Loss Components: Auxiliary: 0.2169 | Cardinality: 0.1073 | Classification: 0.0883 | Classification_binary: 0.1426 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1140 | Regression_eta: 0.0013 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1710 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0564 | Regression_eta: 0.0401 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0658 | LR: 1.72e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8711s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.1047 | max_forward: 0.7107 | max_loss: 0.1679 | max_backward: 1.3343 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.8664 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5919.0 | uss_MB: 4778.3 | child_rss_MB: 5681.4 | allocated_MB: 372.0 | reserved_MB: 25640.0 | inactive_split_MB: 260.9 +rank1 [2026-10-29 18:58:25,753] INFO : Step 41000/50000 rank1 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1031 | Classification: 0.0930 | Classification_binary: 0.1618 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1164 | Regression_eta: 0.0014 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.1527 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0556 | Regression_eta: 0.0353 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0485 | LR: 1.70e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.4160s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.7604 | max_loss: 0.2558 | max_backward: 1.7603 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.8399 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 5926.0 | uss_MB: 4782.0 | child_rss_MB: 5654.3 | allocated_MB: 371.9 | reserved_MB: 25640.0 | inactive_split_MB: 195.8 +rank1 [2026-10-29 18:58:26,765] INFO : Running validation on rank1 +rank1 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 18:59:14,712] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:59:15,273] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 18:59:40,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 18:59:41,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 18:59:42,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 18:59:43,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 18:59:53,339] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 18:59:53,543] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:59:54,024] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 19:00:09,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 19:00:10,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 19:00:11,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 19:00:12,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 19:00:22,743] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 19:00:23,207] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:00:23,684] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 19:00:39,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 19:00:40,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 19:00:40,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 19:00:41,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 19:00:51,486] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 19:04:58,256] INFO : Step 41100/50000 rank1 | Train Loss: 1.0684 | Train Loss Unweighted: 0.5053 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1438 | Classification: 0.0856 | Classification_binary: 0.1616 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1271 | Regression_eta: 0.0014 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0534 | Regression_eta: 0.0377 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0801 | LR: 1.69e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.3315s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0066 | max_forward: 0.8006 | max_loss: 0.1726 | max_backward: 1.6699 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.6214 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0566 | Memory: rss_MB: 7184.9 | uss_MB: 5901.7 | child_rss_MB: 5647.6 | allocated_MB: 371.5 | reserved_MB: 22898.0 | inactive_split_MB: 258.4 +rank1 [2026-10-29 19:08:41,622] INFO : Step 41200/50000 rank1 | Train Loss: 0.9455 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1027 | Classification: 0.0887 | Classification_binary: 0.1242 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1223 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1194 | Regression_cos_phi: 0.0259 | Regression_energy: 0.0672 | Regression_eta: 0.0224 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0395 | LR: 1.67e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.4118s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0589 | max_forward: 0.6517 | max_loss: 0.1401 | max_backward: 1.1761 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5685 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5984.6 | uss_MB: 4847.1 | child_rss_MB: 5628.1 | allocated_MB: 370.2 | reserved_MB: 24878.0 | inactive_split_MB: 199.0 +rank1 [2026-10-29 19:12:28,247] INFO : Step 41300/50000 rank1 | Train Loss: 0.9373 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1011 | Classification: 0.0853 | Classification_binary: 0.1448 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0013 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1306 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0435 | Regression_eta: 0.0267 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0470 | LR: 1.66e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8976s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.8726 | max_loss: 0.1617 | max_backward: 1.6974 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5417 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5987.6 | uss_MB: 4852.4 | child_rss_MB: 5623.6 | allocated_MB: 370.7 | reserved_MB: 24878.0 | inactive_split_MB: 233.6 +rank1 [2026-10-29 19:16:15,464] INFO : Step 41400/50000 rank1 | Train Loss: 1.0355 | Train Loss Unweighted: 0.4965 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1363 | Classification: 0.0957 | Classification_binary: 0.1654 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1162 | Regression_eta: 0.0012 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0552 | Regression_eta: 0.0319 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0472 | LR: 1.64e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9193s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7129 | max_loss: 0.2460 | max_backward: 1.6340 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5967.8 | uss_MB: 4849.9 | child_rss_MB: 5563.5 | allocated_MB: 372.0 | reserved_MB: 24878.0 | inactive_split_MB: 305.4 +rank1 [2026-10-29 19:19:52,655] INFO : Step 41500/50000 rank1 | Train Loss: 0.9689 | Train Loss Unweighted: 0.4689 | Train Loss Components: Auxiliary: 0.2118 | Cardinality: 0.1040 | Classification: 0.0862 | Classification_binary: 0.1340 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1227 | Regression_eta: 0.0012 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1367 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0635 | Regression_eta: 0.0259 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0641 | LR: 1.63e-04 | DataLoad Time: 0.0190s | Model Forward Time: 3.4477s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.6415 | max_loss: 0.1502 | max_backward: 1.1845 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5935.8 | uss_MB: 4853.4 | child_rss_MB: 5403.5 | allocated_MB: 368.5 | reserved_MB: 24878.0 | inactive_split_MB: 173.5 +rank1 [2026-10-29 19:23:40,393] INFO : Step 41600/50000 rank1 | Train Loss: 0.9559 | Train Loss Unweighted: 0.4625 | Train Loss Components: Auxiliary: 0.2141 | Cardinality: 0.1097 | Classification: 0.0872 | Classification_binary: 0.1298 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1211 | Regression_eta: 0.0012 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.1550 | Regression_cos_phi: 0.0794 | Regression_energy: 0.0679 | Regression_eta: 0.0413 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0698 | LR: 1.61e-04 | DataLoad Time: 0.0190s | Model Forward Time: 3.5886s | Phase Times: max_pre_step_barrier: 0.0755 | max_batch_to_device: 0.0062 | max_forward: 0.6849 | max_loss: 0.2047 | max_backward: 1.1646 | max_optimizer_step: 0.2235 | max_tensorboard_logging: 0.5806 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5909.3 | uss_MB: 4827.6 | child_rss_MB: 5308.7 | allocated_MB: 370.4 | reserved_MB: 24878.0 | inactive_split_MB: 287.4 +rank1 [2026-10-29 19:27:23,742] INFO : Step 41700/50000 rank1 | Train Loss: 1.0204 | Train Loss Unweighted: 0.4905 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1165 | Classification: 0.0860 | Classification_binary: 0.1552 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0551 | Regression_eta: 0.0363 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0712 | LR: 1.60e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.0312s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0773 | max_forward: 0.7754 | max_loss: 0.1486 | max_backward: 1.5759 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 0.5773 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5907.3 | uss_MB: 4857.3 | child_rss_MB: 5302.2 | allocated_MB: 370.8 | reserved_MB: 24878.0 | inactive_split_MB: 238.3 +rank1 [2026-10-29 19:31:24,300] INFO : Step 41800/50000 rank1 | Train Loss: 1.0101 | Train Loss Unweighted: 0.4703 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1282 | Classification: 0.0818 | Classification_binary: 0.1618 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0436 | Regression_energy: 0.0595 | Regression_eta: 0.0362 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0487 | LR: 1.58e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9762s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8147 | max_loss: 0.4259 | max_backward: 1.4886 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1200 | Memory: rss_MB: 5899.0 | uss_MB: 4855.0 | child_rss_MB: 5276.1 | allocated_MB: 371.9 | reserved_MB: 25802.0 | inactive_split_MB: 266.5 +rank1 [2026-10-29 19:35:32,346] INFO : Step 41900/50000 rank1 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4672 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1185 | Classification: 0.0878 | Classification_binary: 0.1282 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1241 | Regression_eta: 0.0011 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1208 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0538 | Regression_eta: 0.0324 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0435 | LR: 1.57e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4625s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0473 | max_forward: 0.6783 | max_loss: 0.1431 | max_backward: 1.2347 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.5634 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5868.0 | uss_MB: 4858.2 | child_rss_MB: 5156.0 | allocated_MB: 370.8 | reserved_MB: 25802.0 | inactive_split_MB: 166.5 +rank1 [2026-10-29 19:39:32,016] INFO : Step 42000/50000 rank1 | Train Loss: 0.9837 | Train Loss Unweighted: 0.4719 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.1093 | Classification: 0.1015 | Classification_binary: 0.1413 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1128 | Regression_eta: 0.0014 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1407 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0522 | Regression_eta: 0.0298 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0418 | LR: 1.56e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.6408s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6848 | max_loss: 0.2509 | max_backward: 1.3746 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5866.0 | uss_MB: 4860.4 | child_rss_MB: 5161.1 | allocated_MB: 370.2 | reserved_MB: 25802.0 | inactive_split_MB: 219.9 +rank1 [2026-10-29 19:39:32,940] INFO : Running validation on rank1 +rank1 [2026-10-29 19:39:32,940] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 19:39:32,940] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 19:40:23,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:40:23,889] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 19:40:52,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 19:40:53,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 19:40:54,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 19:40:55,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 19:41:05,710] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 19:41:06,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:41:06,794] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 19:41:23,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 19:41:25,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 19:41:26,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 19:41:27,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 19:41:37,352] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 19:41:38,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:41:38,673] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 19:41:54,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 19:41:55,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 19:41:55,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 19:41:56,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 19:42:06,519] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 19:46:29,429] INFO : Step 42100/50000 rank1 | Train Loss: 1.0639 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1434 | Classification: 0.0999 | Classification_binary: 0.1737 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1097 | Regression_eta: 0.0012 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1665 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0621 | Regression_eta: 0.0313 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0374 | LR: 1.54e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0358s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.7424 | max_loss: 0.1586 | max_backward: 1.4434 | max_optimizer_step: 0.3014 | max_tensorboard_logging: 0.6771 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 7018.7 | uss_MB: 6010.1 | child_rss_MB: 5164.1 | allocated_MB: 370.8 | reserved_MB: 23298.0 | inactive_split_MB: 265.3 +rank1 [2026-10-29 19:50:21,661] INFO : Step 42200/50000 rank1 | Train Loss: 1.0152 | Train Loss Unweighted: 0.4831 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1169 | Classification: 0.0913 | Classification_binary: 0.1515 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1190 | Regression_eta: 0.0015 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0340 | Regression_energy: 0.0593 | Regression_eta: 0.0391 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0616 | LR: 1.53e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.5432s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.6567 | max_loss: 0.2000 | max_backward: 1.2379 | max_optimizer_step: 0.1803 | max_tensorboard_logging: 0.5782 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5943.7 | uss_MB: 4948.2 | child_rss_MB: 5171.4 | allocated_MB: 370.9 | reserved_MB: 24984.0 | inactive_split_MB: 272.4 +rank1 [2026-10-29 19:51:24,069] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank1 [2026-10-29 19:51:24,069] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-29 19:54:15,429] INFO : Step 42300/50000 rank1 | Train Loss: 1.0224 | Train Loss Unweighted: 0.4869 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.1261 | Classification: 0.0943 | Classification_binary: 0.1704 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0493 | Regression_eta: 0.0269 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0529 | LR: 1.52e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7707s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7065 | max_loss: 0.2237 | max_backward: 1.4902 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.5353 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6010.9 | uss_MB: 5019.0 | child_rss_MB: 5170.2 | allocated_MB: 371.9 | reserved_MB: 24984.0 | inactive_split_MB: 183.6 +rank1 [2026-10-29 19:58:13,284] INFO : Step 42400/50000 rank1 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4895 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1253 | Classification: 0.0872 | Classification_binary: 0.1616 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1184 | Regression_eta: 0.0013 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0633 | Regression_eta: 0.0302 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0340 | LR: 1.50e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7974s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7436 | max_loss: 0.2477 | max_backward: 1.5279 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5169 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5933.8 | uss_MB: 4947.5 | child_rss_MB: 5164.9 | allocated_MB: 373.0 | reserved_MB: 24984.0 | inactive_split_MB: 275.6 +rank1 [2026-10-29 20:02:13,260] INFO : Step 42500/50000 rank1 | Train Loss: 0.9211 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1056 | Classification: 0.0789 | Classification_binary: 0.1296 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1162 | Regression_eta: 0.0011 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0570 | Regression_eta: 0.0329 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0521 | LR: 1.49e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.5538s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.6196 | max_loss: 0.2440 | max_backward: 1.3403 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 2.3828 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5926.8 | uss_MB: 4949.7 | child_rss_MB: 5158.5 | allocated_MB: 371.3 | reserved_MB: 25980.0 | inactive_split_MB: 274.9 +rank1 [2026-10-29 20:06:17,860] INFO : Step 42600/50000 rank1 | Train Loss: 0.9688 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1101 | Classification: 0.0874 | Classification_binary: 0.1472 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0012 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0548 | Regression_eta: 0.0285 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0328 | LR: 1.48e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.9507s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6353 | max_loss: 0.2547 | max_backward: 1.3961 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 5918.3 | uss_MB: 4947.3 | child_rss_MB: 5167.3 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 272.5 +rank1 [2026-10-29 20:10:16,915] INFO : Step 42700/50000 rank1 | Train Loss: 1.0476 | Train Loss Unweighted: 0.4808 | Train Loss Components: Auxiliary: 0.2377 | Cardinality: 0.1284 | Classification: 0.0883 | Classification_binary: 0.1739 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1071 | Regression_eta: 0.0014 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0468 | Regression_eta: 0.0266 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0303 | LR: 1.47e-04 | DataLoad Time: 0.0128s | Model Forward Time: 5.7845s | Phase Times: max_pre_step_barrier: 0.0320 | max_batch_to_device: 0.0114 | max_forward: 2.1726 | max_loss: 0.5101 | max_backward: 2.7494 | max_optimizer_step: 0.2679 | max_tensorboard_logging: 0.6520 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5921.8 | uss_MB: 4952.4 | child_rss_MB: 5151.8 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 482.9 +rank1 [2026-10-29 20:14:17,050] INFO : Step 42800/50000 rank1 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4792 | Train Loss Components: Auxiliary: 0.2262 | Cardinality: 0.1277 | Classification: 0.0841 | Classification_binary: 0.1487 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1209 | Regression_eta: 0.0012 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0459 | Regression_eta: 0.0292 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0508 | LR: 1.45e-04 | DataLoad Time: 0.0210s | Model Forward Time: 6.2804s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0175 | max_forward: 1.3870 | max_loss: 0.5106 | max_backward: 1.9221 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 2.4017 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5924.3 | uss_MB: 4952.5 | child_rss_MB: 5165.0 | allocated_MB: 372.6 | reserved_MB: 25980.0 | inactive_split_MB: 234.4 +rank1 [2026-10-29 20:18:09,439] INFO : Step 42900/50000 rank1 | Train Loss: 0.8847 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.0993 | Classification: 0.0847 | Classification_binary: 0.1207 | HitGrouping: 0.1828 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1069 | Regression_eta: 0.0011 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.1181 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0505 | Regression_eta: 0.0244 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0291 | LR: 1.44e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7392s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.6133 | max_loss: 0.1478 | max_backward: 1.3445 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.7551 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0762 | Memory: rss_MB: 5924.9 | uss_MB: 4957.1 | child_rss_MB: 5179.7 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 276.9 +rank1 [2026-10-29 20:22:02,172] INFO : Step 43000/50000 rank1 | Train Loss: 0.9800 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1096 | Classification: 0.0987 | Classification_binary: 0.1606 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0012 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1564 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0485 | Regression_eta: 0.0380 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0549 | LR: 1.43e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.9055s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0071 | max_forward: 0.7477 | max_loss: 0.1477 | max_backward: 1.5663 | max_optimizer_step: 0.1600 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 5929.7 | uss_MB: 4961.8 | child_rss_MB: 5171.4 | allocated_MB: 369.1 | reserved_MB: 25980.0 | inactive_split_MB: 209.4 +rank1 [2026-10-29 20:22:03,144] INFO : Running validation on rank1 +rank1 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 20:22:51,801] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:22:52,331] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 20:23:18,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 20:23:19,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 20:23:20,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 20:23:21,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 20:23:31,512] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 20:23:31,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:23:32,370] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 20:23:48,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 20:23:49,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 20:23:50,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 20:23:51,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 20:24:01,083] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 20:24:01,723] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:24:02,214] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 20:24:17,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 20:24:18,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 20:24:19,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 20:24:19,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 20:24:29,795] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 20:28:29,104] INFO : Step 43100/50000 rank1 | Train Loss: 0.9692 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1151 | Classification: 0.0766 | Classification_binary: 0.1586 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1062 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0436 | Regression_eta: 0.0385 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0413 | LR: 1.42e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7256s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.8051 | max_loss: 0.1438 | max_backward: 1.3224 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 7068.8 | uss_MB: 6101.3 | child_rss_MB: 5191.0 | allocated_MB: 370.5 | reserved_MB: 21096.0 | inactive_split_MB: 247.5 +rank1 [2026-10-29 20:32:30,812] INFO : Step 43200/50000 rank1 | Train Loss: 0.9431 | Train Loss Unweighted: 0.4458 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1175 | Classification: 0.0811 | Classification_binary: 0.1294 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0012 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1334 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0659 | Regression_eta: 0.0496 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0459 | LR: 1.40e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.5223s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0120 | max_forward: 0.6342 | max_loss: 0.1417 | max_backward: 1.2752 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6068.4 | uss_MB: 5101.7 | child_rss_MB: 5194.2 | allocated_MB: 370.4 | reserved_MB: 22140.0 | inactive_split_MB: 198.0 +rank1 [2026-10-29 20:36:16,428] INFO : Step 43300/50000 rank1 | Train Loss: 1.0382 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1221 | Classification: 0.1031 | Classification_binary: 0.1555 | HitGrouping: 0.1894 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1173 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1608 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0654 | Regression_eta: 0.0287 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0623 | LR: 1.39e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2402s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0266 | max_forward: 0.7527 | max_loss: 0.1476 | max_backward: 1.6633 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.7701 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6071.5 | uss_MB: 5107.2 | child_rss_MB: 5175.9 | allocated_MB: 371.1 | reserved_MB: 23082.0 | inactive_split_MB: 236.4 +rank1 [2026-10-29 20:40:05,496] INFO : Step 43400/50000 rank1 | Train Loss: 1.0532 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1292 | Classification: 0.0981 | Classification_binary: 0.1785 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0015 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0466 | Regression_eta: 0.0221 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0562 | LR: 1.38e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2964s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0691 | max_forward: 0.8057 | max_loss: 0.1503 | max_backward: 1.8319 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.6703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6067.0 | uss_MB: 5102.4 | child_rss_MB: 5192.5 | allocated_MB: 371.5 | reserved_MB: 23082.0 | inactive_split_MB: 776.0 +rank1 [2026-10-29 20:43:57,118] INFO : Step 43500/50000 rank1 | Train Loss: 0.9308 | Train Loss Unweighted: 0.4531 | Train Loss Components: Auxiliary: 0.2084 | Cardinality: 0.0966 | Classification: 0.0846 | Classification_binary: 0.1384 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1135 | Regression_eta: 0.0013 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1310 | Regression_cos_phi: 0.0337 | Regression_energy: 0.0493 | Regression_eta: 0.0333 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0297 | LR: 1.37e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7362s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0171 | max_forward: 0.6718 | max_loss: 0.1455 | max_backward: 1.4141 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 6070.4 | uss_MB: 5108.5 | child_rss_MB: 5191.0 | allocated_MB: 372.0 | reserved_MB: 23082.0 | inactive_split_MB: 228.0 +rank1 [2026-10-29 20:47:52,743] INFO : Step 43600/50000 rank1 | Train Loss: 0.9195 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.0926 | Classification: 0.0847 | Classification_binary: 0.1298 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1175 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0323 | Classification_binary: 0.1240 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0436 | Regression_eta: 0.0624 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0467 | LR: 1.36e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5158s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.6453 | max_loss: 0.1399 | max_backward: 1.3514 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0942 | Memory: rss_MB: 6067.9 | uss_MB: 5106.0 | child_rss_MB: 5190.1 | allocated_MB: 367.6 | reserved_MB: 23082.0 | inactive_split_MB: 270.4 +rank1 [2026-10-29 20:51:54,645] INFO : Step 43700/50000 rank1 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1181 | Classification: 0.0916 | Classification_binary: 0.1553 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1009 | Regression_eta: 0.0014 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0499 | Regression_eta: 0.0578 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0773 | LR: 1.35e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0947s | Phase Times: max_pre_step_barrier: 0.0199 | max_batch_to_device: 0.0121 | max_forward: 0.8005 | max_loss: 0.2526 | max_backward: 1.9159 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 6075.4 | uss_MB: 5110.5 | child_rss_MB: 5168.5 | allocated_MB: 371.6 | reserved_MB: 23674.0 | inactive_split_MB: 256.4 +rank1 [2026-10-29 20:55:51,187] INFO : Step 43800/50000 rank1 | Train Loss: 1.0325 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1341 | Classification: 0.0892 | Classification_binary: 0.1542 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0012 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0586 | Regression_eta: 0.0329 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0432 | LR: 1.34e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.6836s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7618 | max_loss: 0.1715 | max_backward: 1.3818 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6073.5 | uss_MB: 5110.6 | child_rss_MB: 5186.9 | allocated_MB: 371.7 | reserved_MB: 23674.0 | inactive_split_MB: 282.3 +rank1 [2026-10-29 20:59:46,232] INFO : Step 43900/50000 rank1 | Train Loss: 0.9104 | Train Loss Unweighted: 0.4325 | Train Loss Components: Auxiliary: 0.2009 | Cardinality: 0.1078 | Classification: 0.0725 | Classification_binary: 0.1272 | HitGrouping: 0.1855 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1146 | Regression_eta: 0.0011 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0515 | Regression_eta: 0.0297 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0677 | LR: 1.33e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.3862s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.6145 | max_loss: 0.1412 | max_backward: 1.1415 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5764 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6071.8 | uss_MB: 5111.9 | child_rss_MB: 5178.4 | allocated_MB: 368.3 | reserved_MB: 23674.0 | inactive_split_MB: 193.1 +rank1 [2026-10-29 21:03:43,512] INFO : Step 44000/50000 rank1 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4573 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1078 | Classification: 0.0956 | Classification_binary: 0.1421 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1083 | Regression_eta: 0.0011 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1320 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0465 | Regression_eta: 0.0224 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0462 | LR: 1.32e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.9485s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0061 | max_forward: 1.1697 | max_loss: 0.2455 | max_backward: 1.7366 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6074.9 | uss_MB: 5118.1 | child_rss_MB: 5179.2 | allocated_MB: 370.7 | reserved_MB: 23674.0 | inactive_split_MB: 207.3 +rank1 [2026-10-29 21:03:44,527] INFO : Running validation on rank1 +rank1 [2026-10-29 21:03:44,527] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 21:03:44,527] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 21:04:32,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:04:33,227] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 21:04:59,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 21:05:00,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 21:05:01,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 21:05:02,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 21:05:12,235] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 21:05:12,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:05:12,998] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 21:05:28,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 21:05:29,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 21:05:30,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 21:05:31,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 21:05:41,844] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 21:05:42,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:05:42,732] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 21:05:58,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 21:05:58,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 21:05:59,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 21:06:00,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 21:06:10,225] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 21:10:20,117] INFO : Step 44100/50000 rank1 | Train Loss: 1.0009 | Train Loss Unweighted: 0.4605 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1400 | Classification: 0.0841 | Classification_binary: 0.1644 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1044 | Regression_eta: 0.0011 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1707 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0656 | Regression_eta: 0.0316 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0420 | LR: 1.31e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.5840s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 1.2763 | max_loss: 0.5349 | max_backward: 2.2676 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 1.2755 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 7076.5 | uss_MB: 6129.8 | child_rss_MB: 5164.2 | allocated_MB: 372.3 | reserved_MB: 24224.0 | inactive_split_MB: 272.2 +rank1 [2026-10-29 21:14:15,311] INFO : Step 44200/50000 rank1 | Train Loss: 0.9801 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1146 | Classification: 0.0882 | Classification_binary: 0.1482 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1167 | Regression_eta: 0.0014 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0561 | Regression_eta: 0.0431 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0610 | LR: 1.30e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.0051s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0132 | max_forward: 0.6993 | max_loss: 0.1574 | max_backward: 1.3862 | max_optimizer_step: 0.2915 | max_tensorboard_logging: 0.7724 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6137.2 | uss_MB: 5191.5 | child_rss_MB: 5174.4 | allocated_MB: 370.2 | reserved_MB: 26182.0 | inactive_split_MB: 176.5 +rank1 [2026-10-29 21:17:58,579] INFO : Step 44300/50000 rank1 | Train Loss: 0.8787 | Train Loss Unweighted: 0.4332 | Train Loss Components: Auxiliary: 0.1952 | Cardinality: 0.0950 | Classification: 0.0869 | Classification_binary: 0.1153 | HitGrouping: 0.1807 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0011 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0550 | Regression_eta: 0.0230 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0399 | LR: 1.29e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.7452s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0142 | max_forward: 0.7699 | max_loss: 0.2088 | max_backward: 1.4049 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 0.4839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 6137.2 | uss_MB: 4195.8 | child_rss_MB: 5178.8 | allocated_MB: 367.2 | reserved_MB: 27204.0 | inactive_split_MB: 205.8 +rank1 [2026-10-29 21:21:59,794] INFO : Step 44400/50000 rank1 | Train Loss: 0.9558 | Train Loss Unweighted: 0.4680 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.0905 | Classification: 0.0911 | Classification_binary: 0.1587 | HitGrouping: 0.1858 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0436 | Regression_energy: 0.0538 | Regression_eta: 0.0311 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0355 | LR: 1.28e-04 | DataLoad Time: 0.0194s | Model Forward Time: 6.3289s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0370 | max_forward: 0.7829 | max_loss: 0.1727 | max_backward: 1.7862 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 2.7361 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6138.4 | uss_MB: 5206.0 | child_rss_MB: 5161.4 | allocated_MB: 373.1 | reserved_MB: 27586.0 | inactive_split_MB: 176.3 +rank1 [2026-10-29 21:26:11,867] INFO : Step 44500/50000 rank1 | Train Loss: 0.9739 | Train Loss Unweighted: 0.4610 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1139 | Classification: 0.0849 | Classification_binary: 0.1578 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1077 | Regression_eta: 0.0012 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0585 | Regression_eta: 0.0293 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0614 | LR: 1.27e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.9307s | Phase Times: max_pre_step_barrier: 0.0250 | max_batch_to_device: 0.0065 | max_forward: 0.7930 | max_loss: 0.1668 | max_backward: 1.3363 | max_optimizer_step: 0.2871 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6136.3 | uss_MB: 5200.8 | child_rss_MB: 5177.3 | allocated_MB: 371.5 | reserved_MB: 27586.0 | inactive_split_MB: 195.8 +rank1 [2026-10-29 21:30:04,561] INFO : Step 44600/50000 rank1 | Train Loss: 0.8904 | Train Loss Unweighted: 0.4364 | Train Loss Components: Auxiliary: 0.2007 | Cardinality: 0.1041 | Classification: 0.0775 | Classification_binary: 0.1238 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1161 | Regression_eta: 0.0010 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1233 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0587 | Regression_eta: 0.0300 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0599 | LR: 1.26e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.5137s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.6364 | max_loss: 0.1431 | max_backward: 1.3712 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.4900 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6139.4 | uss_MB: 5205.0 | child_rss_MB: 5167.5 | allocated_MB: 368.4 | reserved_MB: 27586.0 | inactive_split_MB: 170.2 +rank1 [2026-10-29 21:33:51,597] INFO : Step 44700/50000 rank1 | Train Loss: 1.0003 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1252 | Classification: 0.0899 | Classification_binary: 0.1520 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1117 | Regression_eta: 0.0013 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0566 | Regression_energy: 0.0510 | Regression_eta: 0.0346 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0766 | LR: 1.25e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6388s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6952 | max_loss: 0.1455 | max_backward: 1.5015 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6145.1 | uss_MB: 5206.5 | child_rss_MB: 5162.4 | allocated_MB: 371.9 | reserved_MB: 28574.0 | inactive_split_MB: 214.1 +rank1 [2026-10-29 21:37:55,987] INFO : Step 44800/50000 rank1 | Train Loss: 0.9665 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1115 | Classification: 0.0889 | Classification_binary: 0.1605 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1042 | Regression_eta: 0.0011 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1585 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0471 | Regression_eta: 0.0296 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0584 | LR: 1.24e-04 | DataLoad Time: 0.0133s | Model Forward Time: 5.8760s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7593 | max_loss: 0.1924 | max_backward: 2.8086 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.6087 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 6148.4 | uss_MB: 5210.8 | child_rss_MB: 5160.0 | allocated_MB: 370.8 | reserved_MB: 28574.0 | inactive_split_MB: 257.7 +rank1 [2026-10-29 21:41:36,915] INFO : Step 44900/50000 rank1 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.0886 | Classification_binary: 0.1466 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1121 | Regression_eta: 0.0013 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0710 | Regression_eta: 0.0339 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0516 | LR: 1.23e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.5195s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0101 | max_forward: 0.6531 | max_loss: 0.1431 | max_backward: 1.3230 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6145.9 | uss_MB: 5207.3 | child_rss_MB: 5182.8 | allocated_MB: 368.1 | reserved_MB: 28574.0 | inactive_split_MB: 178.8 +rank1 [2026-10-29 21:45:50,735] INFO : Step 45000/50000 rank1 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4443 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1053 | Classification: 0.0832 | Classification_binary: 0.1311 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0533 | Regression_eta: 0.0291 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0411 | LR: 1.22e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.1238s | Phase Times: max_pre_step_barrier: 0.0074 | max_batch_to_device: 0.0159 | max_forward: 1.1897 | max_loss: 0.2457 | max_backward: 1.2951 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.8342 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0919 | Memory: rss_MB: 6152.1 | uss_MB: 5213.7 | child_rss_MB: 5170.7 | allocated_MB: 368.8 | reserved_MB: 28576.0 | inactive_split_MB: 266.4 +rank1 [2026-10-29 21:45:51,739] INFO : Running validation on rank1 +rank1 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 21:46:40,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:46:41,116] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 21:47:08,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 21:47:10,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 21:47:11,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 21:47:12,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 21:47:22,592] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 21:47:23,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:47:23,688] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 21:47:40,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 21:47:41,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 21:47:42,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 21:47:43,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 21:47:53,145] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 21:47:53,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:47:53,966] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 21:48:09,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 21:48:10,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 21:48:11,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 21:48:12,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 21:48:22,263] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 21:52:56,451] INFO : Step 45100/50000 rank1 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1151 | Classification: 0.1010 | Classification_binary: 0.1562 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1118 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0604 | Regression_eta: 0.0414 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0641 | LR: 1.21e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3037s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7307 | max_loss: 0.4409 | max_backward: 1.5823 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 7066.7 | uss_MB: 6122.4 | child_rss_MB: 5160.9 | allocated_MB: 373.1 | reserved_MB: 23280.0 | inactive_split_MB: 227.2 +rank1 [2026-10-29 21:56:57,436] INFO : Step 45200/50000 rank1 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1108 | Classification: 0.0815 | Classification_binary: 0.1593 | HitGrouping: 0.1884 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0655 | Regression_eta: 0.0334 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0518 | LR: 1.20e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.2205s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.9482 | max_loss: 0.4476 | max_backward: 1.7165 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.4656 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7064.1 | uss_MB: 6120.3 | child_rss_MB: 5157.7 | allocated_MB: 373.0 | reserved_MB: 23338.0 | inactive_split_MB: 277.4 +rank1 [2026-10-29 22:01:28,909] INFO : Step 45300/50000 rank1 | Train Loss: 0.9312 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1141 | Classification: 0.0882 | Classification_binary: 0.1299 | HitGrouping: 0.1891 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0010 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1051 | Regression_cos_phi: 0.0276 | Regression_energy: 0.0582 | Regression_eta: 0.0220 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0348 | LR: 1.19e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.4285s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6106 | max_loss: 0.2394 | max_backward: 1.1660 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 6271.7 | uss_MB: 5329.4 | child_rss_MB: 5157.3 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 223.2 +rank1 [2026-10-29 22:05:37,387] INFO : Step 45400/50000 rank1 | Train Loss: 0.9251 | Train Loss Unweighted: 0.4554 | Train Loss Components: Auxiliary: 0.2112 | Cardinality: 0.1034 | Classification: 0.0967 | Classification_binary: 0.1405 | HitGrouping: 0.1801 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1072 | Regression_eta: 0.0012 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0515 | Regression_eta: 0.0294 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0411 | LR: 1.19e-04 | DataLoad Time: 0.0148s | Model Forward Time: 3.9544s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7158 | max_loss: 0.2520 | max_backward: 1.7727 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6276.9 | uss_MB: 5335.5 | child_rss_MB: 5162.7 | allocated_MB: 372.6 | reserved_MB: 23338.0 | inactive_split_MB: 212.6 +rank1 [2026-10-29 22:09:49,791] INFO : Step 45500/50000 rank1 | Train Loss: 1.0136 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1238 | Classification: 0.0902 | Classification_binary: 0.1661 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1060 | Regression_eta: 0.0013 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1550 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0455 | Regression_eta: 0.0267 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0540 | LR: 1.18e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.1597s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0147 | max_forward: 1.9023 | max_loss: 0.2979 | max_backward: 3.5478 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5661 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 6272.0 | uss_MB: 4281.7 | child_rss_MB: 5159.4 | allocated_MB: 372.4 | reserved_MB: 23338.0 | inactive_split_MB: 256.1 +rank1 [2026-10-29 22:14:16,023] INFO : Step 45600/50000 rank1 | Train Loss: 0.9675 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2172 | Cardinality: 0.1193 | Classification: 0.0849 | Classification_binary: 0.1400 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1218 | Regression_eta: 0.0010 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1142 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0462 | Regression_eta: 0.0274 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0423 | LR: 1.17e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.7202s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6781 | max_loss: 0.1576 | max_backward: 1.5089 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 0.5447 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6272.0 | uss_MB: 5337.5 | child_rss_MB: 5161.4 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 216.2 +rank1 [2026-10-29 22:18:15,631] INFO : Step 45700/50000 rank1 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1961 | Cardinality: 0.0908 | Classification: 0.0837 | Classification_binary: 0.1223 | HitGrouping: 0.1794 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1108 | Regression_eta: 0.0011 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1047 | Regression_cos_phi: 0.0280 | Regression_energy: 0.0425 | Regression_eta: 0.0294 | Regression_pt: 0.0449 | Regression_sin_phi: 0.0277 | LR: 1.16e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.4722s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6216 | max_loss: 0.1658 | max_backward: 1.1797 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6234.4 | uss_MB: 5336.4 | child_rss_MB: 5162.7 | allocated_MB: 370.8 | reserved_MB: 23338.0 | inactive_split_MB: 226.6 +rank1 [2026-10-29 22:22:19,536] INFO : Step 45800/50000 rank1 | Train Loss: 0.9372 | Train Loss Unweighted: 0.4491 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1073 | Classification: 0.0880 | Classification_binary: 0.1481 | HitGrouping: 0.1781 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0428 | Regression_eta: 0.0320 | Regression_pt: 0.0452 | Regression_sin_phi: 0.0524 | LR: 1.16e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2367s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8017 | max_loss: 0.2428 | max_backward: 1.9917 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0978 | Memory: rss_MB: 6240.7 | uss_MB: 5347.0 | child_rss_MB: 5162.1 | allocated_MB: 375.2 | reserved_MB: 23338.0 | inactive_split_MB: 268.8 +rank1 [2026-10-29 22:26:26,931] INFO : Step 45900/50000 rank1 | Train Loss: 0.9600 | Train Loss Unweighted: 0.4592 | Train Loss Components: Auxiliary: 0.2179 | Cardinality: 0.1027 | Classification: 0.0816 | Classification_binary: 0.1590 | HitGrouping: 0.1875 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0536 | Regression_eta: 0.0484 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0505 | LR: 1.15e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.8439s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 1.3210 | max_loss: 0.4294 | max_backward: 3.2770 | max_optimizer_step: 0.2740 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 6235.6 | uss_MB: 5343.7 | child_rss_MB: 5161.3 | allocated_MB: 370.5 | reserved_MB: 23338.0 | inactive_split_MB: 259.6 +rank1 [2026-10-29 22:30:40,887] INFO : Step 46000/50000 rank1 | Train Loss: 0.8490 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.0938 | Classification: 0.0817 | Classification_binary: 0.1110 | HitGrouping: 0.1819 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0010 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1114 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0445 | Regression_eta: 0.0315 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0399 | LR: 1.14e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.5939s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.5955 | max_loss: 0.1430 | max_backward: 1.2483 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 0.6657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 6228.9 | uss_MB: 5344.9 | child_rss_MB: 5164.3 | allocated_MB: 365.7 | reserved_MB: 23338.0 | inactive_split_MB: 222.2 +rank1 [2026-10-29 22:30:41,828] INFO : Running validation on rank1 +rank1 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 22:31:32,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:31:33,520] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 22:32:00,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 22:32:01,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 22:32:03,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 22:32:04,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 22:32:14,976] INFO : Time taken to make predictions on device 1 is: 0.28 min +rank1 [2026-10-29 22:32:15,389] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:32:16,222] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 22:32:31,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 22:32:32,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 22:32:33,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 22:32:34,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 22:32:44,758] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 22:32:45,544] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:32:46,016] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 22:33:01,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 22:33:01,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 22:33:02,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 22:33:03,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 22:33:13,211] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-29 22:37:44,432] INFO : Step 46100/50000 rank1 | Train Loss: 0.9299 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1082 | Classification: 0.0899 | Classification_binary: 0.1357 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1081 | Regression_eta: 0.0010 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1446 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0551 | Regression_eta: 0.0350 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0544 | LR: 1.13e-04 | DataLoad Time: 0.0194s | Model Forward Time: 6.9695s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 2.5146 | max_loss: 0.4579 | max_backward: 2.4958 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1014 | Memory: rss_MB: 6980.2 | uss_MB: 6096.7 | child_rss_MB: 5172.0 | allocated_MB: 371.5 | reserved_MB: 22784.0 | inactive_split_MB: 245.0 +rank1 [2026-10-29 22:41:43,511] INFO : Step 46200/50000 rank1 | Train Loss: 0.9944 | Train Loss Unweighted: 0.4707 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1286 | Classification: 0.0940 | Classification_binary: 0.1623 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1052 | Regression_eta: 0.0011 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0463 | Regression_eta: 0.0331 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0844 | LR: 1.13e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2038s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7685 | max_loss: 0.2359 | max_backward: 1.6855 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1253 | Memory: rss_MB: 6976.0 | uss_MB: 6094.5 | child_rss_MB: 5141.0 | allocated_MB: 373.1 | reserved_MB: 22784.0 | inactive_split_MB: 264.3 +rank1 [2026-10-29 22:45:39,134] INFO : Step 46300/50000 rank1 | Train Loss: 0.9564 | Train Loss Unweighted: 0.4515 | Train Loss Components: Auxiliary: 0.2114 | Cardinality: 0.1124 | Classification: 0.0869 | Classification_binary: 0.1435 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.1343 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0560 | Regression_eta: 0.0339 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0552 | LR: 1.12e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8636s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6709 | max_loss: 0.2410 | max_backward: 1.2189 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.8303 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6276.5 | uss_MB: 5413.4 | child_rss_MB: 5109.4 | allocated_MB: 372.3 | reserved_MB: 23784.0 | inactive_split_MB: 228.2 +rank1 [2026-10-29 22:49:34,649] INFO : Step 46400/50000 rank1 | Train Loss: 0.9514 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1147 | Classification: 0.0822 | Classification_binary: 0.1248 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0010 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0565 | Regression_eta: 0.0288 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0396 | LR: 1.11e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.9546s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.6748 | max_loss: 0.4371 | max_backward: 1.4077 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.8211 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6270.0 | uss_MB: 5413.5 | child_rss_MB: 5075.5 | allocated_MB: 374.2 | reserved_MB: 23988.0 | inactive_split_MB: 241.8 +rank1 [2026-10-29 22:53:41,742] INFO : Step 46500/50000 rank1 | Train Loss: 0.9517 | Train Loss Unweighted: 0.4566 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.0937 | Classification: 0.0928 | Classification_binary: 0.1533 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1031 | Regression_eta: 0.0013 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0548 | Regression_eta: 0.0337 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0845 | LR: 1.11e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.9732s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0087 | max_forward: 0.7136 | max_loss: 0.1507 | max_backward: 1.5229 | max_optimizer_step: 0.2767 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6272.9 | uss_MB: 5416.3 | child_rss_MB: 5079.6 | allocated_MB: 372.0 | reserved_MB: 23988.0 | inactive_split_MB: 248.1 +rank1 [2026-10-29 22:57:41,614] INFO : Step 46600/50000 rank1 | Train Loss: 0.9541 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.2166 | Cardinality: 0.1078 | Classification: 0.0915 | Classification_binary: 0.1560 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0424 | Regression_energy: 0.0533 | Regression_eta: 0.0350 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0463 | LR: 1.10e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.5736s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7127 | max_loss: 0.1604 | max_backward: 1.3079 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4722 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 6261.6 | uss_MB: 5413.8 | child_rss_MB: 5045.9 | allocated_MB: 370.5 | reserved_MB: 23988.0 | inactive_split_MB: 179.8 +rank1 [2026-10-29 23:02:40,690] INFO : Step 46700/50000 rank1 | Train Loss: 0.9378 | Train Loss Unweighted: 0.4497 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.0977 | Classification: 0.0857 | Classification_binary: 0.1264 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0015 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1011 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0478 | Regression_eta: 0.0776 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0547 | LR: 1.10e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.7665s | Phase Times: max_pre_step_barrier: 0.0197 | max_batch_to_device: 0.0481 | max_forward: 0.6699 | max_loss: 0.2593 | max_backward: 1.4356 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5914 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6261.6 | uss_MB: 5415.0 | child_rss_MB: 5040.6 | allocated_MB: 368.9 | reserved_MB: 24784.0 | inactive_split_MB: 229.2 +rank1 [2026-10-29 23:06:36,578] INFO : Step 46800/50000 rank1 | Train Loss: 0.9152 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.2022 | Cardinality: 0.1011 | Classification: 0.0910 | Classification_binary: 0.1376 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0988 | Regression_eta: 0.0014 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0436 | Regression_eta: 0.0441 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0553 | LR: 1.09e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.6079s | Phase Times: max_pre_step_barrier: 0.1216 | max_batch_to_device: 0.0144 | max_forward: 1.0361 | max_loss: 0.2737 | max_backward: 1.7951 | max_optimizer_step: 0.2978 | max_tensorboard_logging: 0.5843 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6263.1 | uss_MB: 5417.5 | child_rss_MB: 5044.7 | allocated_MB: 368.9 | reserved_MB: 24994.0 | inactive_split_MB: 204.7 +rank1 [2026-10-29 23:10:45,824] INFO : Step 46900/50000 rank1 | Train Loss: 0.9278 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2149 | Cardinality: 0.1073 | Classification: 0.0827 | Classification_binary: 0.1607 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1027 | Regression_eta: 0.0010 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1541 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0413 | Regression_eta: 0.0263 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0453 | LR: 1.09e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0938s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0446 | max_forward: 0.7815 | max_loss: 0.1553 | max_backward: 1.4732 | max_optimizer_step: 0.2929 | max_tensorboard_logging: 0.6668 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6256.7 | uss_MB: 5416.7 | child_rss_MB: 5041.4 | allocated_MB: 369.3 | reserved_MB: 24994.0 | inactive_split_MB: 274.7 +rank1 [2026-10-29 23:14:40,387] INFO : Step 47000/50000 rank1 | Train Loss: 0.9023 | Train Loss Unweighted: 0.4500 | Train Loss Components: Auxiliary: 0.2093 | Cardinality: 0.0953 | Classification: 0.0802 | Classification_binary: 0.1325 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1175 | Regression_eta: 0.0010 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1087 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0457 | Regression_eta: 0.0264 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0255 | LR: 1.08e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5084s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6624 | max_loss: 0.1440 | max_backward: 1.2028 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 6259.7 | uss_MB: 5421.0 | child_rss_MB: 5043.0 | allocated_MB: 369.4 | reserved_MB: 25784.0 | inactive_split_MB: 224.5 +rank1 [2026-10-29 23:14:41,333] INFO : Running validation on rank1 +rank1 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 23:15:32,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:15:33,149] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 23:16:02,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 23:16:03,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 23:16:04,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 23:16:05,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 23:16:15,820] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 23:16:16,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:16:16,905] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 23:16:32,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 23:16:34,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 23:16:35,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 23:16:36,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 23:16:46,507] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 23:16:47,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:16:47,863] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 23:17:03,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 23:17:03,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 23:17:04,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 23:17:05,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 23:17:15,352] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 23:21:36,066] INFO : Step 47100/50000 rank1 | Train Loss: 0.9138 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.2058 | Cardinality: 0.0938 | Classification: 0.0946 | Classification_binary: 0.1243 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1103 | Regression_eta: 0.0011 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0413 | Regression_eta: 0.0525 | Regression_pt: 0.0449 | Regression_sin_phi: 0.0736 | LR: 1.07e-04 | DataLoad Time: 0.0109s | Model Forward Time: 4.1951s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0063 | max_forward: 0.8048 | max_loss: 0.2433 | max_backward: 1.3159 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 1.0353 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 6952.3 | uss_MB: 6113.6 | child_rss_MB: 5037.2 | allocated_MB: 368.1 | reserved_MB: 22106.0 | inactive_split_MB: 269.0 +rank1 [2026-10-29 23:25:45,623] INFO : Step 47200/50000 rank1 | Train Loss: 0.9875 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2263 | Cardinality: 0.1259 | Classification: 0.0903 | Classification_binary: 0.1586 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1110 | Regression_eta: 0.0012 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1527 | Regression_cos_phi: 0.0321 | Regression_energy: 0.0510 | Regression_eta: 0.0320 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0557 | LR: 1.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2831s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0486 | max_forward: 0.7581 | max_loss: 0.1506 | max_backward: 1.5914 | max_optimizer_step: 0.2300 | max_tensorboard_logging: 0.7953 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 6951.3 | uss_MB: 6117.8 | child_rss_MB: 5032.9 | allocated_MB: 370.5 | reserved_MB: 23104.0 | inactive_split_MB: 195.8 +rank1 [2026-10-29 23:29:53,764] INFO : Step 47300/50000 rank1 | Train Loss: 0.9350 | Train Loss Unweighted: 0.4467 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.0958 | Classification: 0.0857 | Classification_binary: 0.1512 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0387 | Regression_eta: 0.0282 | Regression_pt: 0.0396 | Regression_sin_phi: 0.0585 | LR: 1.06e-04 | DataLoad Time: 0.0208s | Model Forward Time: 6.4018s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0184 | max_forward: 1.9890 | max_loss: 0.4771 | max_backward: 3.1656 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6309.1 | uss_MB: 5475.6 | child_rss_MB: 5028.4 | allocated_MB: 370.8 | reserved_MB: 23104.0 | inactive_split_MB: 271.1 +rank1 [2026-10-29 23:33:58,356] INFO : Step 47400/50000 rank1 | Train Loss: 0.8593 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.0807 | Classification: 0.0812 | Classification_binary: 0.1182 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1114 | Regression_eta: 0.0011 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1110 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0472 | Regression_eta: 0.0393 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0583 | LR: 1.06e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.0117s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0125 | max_forward: 0.7544 | max_loss: 0.1826 | max_backward: 1.4299 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8027 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6311.1 | uss_MB: 5475.9 | child_rss_MB: 5031.1 | allocated_MB: 369.9 | reserved_MB: 23104.0 | inactive_split_MB: 253.6 +rank1 [2026-10-29 23:37:53,326] INFO : Step 47500/50000 rank1 | Train Loss: 0.9089 | Train Loss Unweighted: 0.4235 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1000 | Classification: 0.0799 | Classification_binary: 0.1391 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1008 | Regression_eta: 0.0012 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1519 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0457 | Regression_eta: 0.0355 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0630 | LR: 1.06e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2745s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0068 | max_forward: 0.7329 | max_loss: 0.1827 | max_backward: 1.5814 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9161 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6306.2 | uss_MB: 5478.3 | child_rss_MB: 5026.1 | allocated_MB: 371.7 | reserved_MB: 23104.0 | inactive_split_MB: 243.6 +rank1 [2026-10-29 23:41:48,171] INFO : Step 47600/50000 rank1 | Train Loss: 0.9781 | Train Loss Unweighted: 0.4635 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1226 | Classification: 0.0910 | Classification_binary: 0.1607 | HitGrouping: 0.1784 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0437 | Regression_eta: 0.0434 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0604 | LR: 1.05e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.4256s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8435 | max_loss: 0.1581 | max_backward: 1.8989 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5341 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6310.2 | uss_MB: 5483.0 | child_rss_MB: 5033.7 | allocated_MB: 370.6 | reserved_MB: 24122.0 | inactive_split_MB: 257.4 +rank1 [2026-10-29 23:46:00,211] INFO : Step 47700/50000 rank1 | Train Loss: 0.9311 | Train Loss Unweighted: 0.4523 | Train Loss Components: Auxiliary: 0.2105 | Cardinality: 0.0992 | Classification: 0.0889 | Classification_binary: 0.1321 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1138 | Regression_eta: 0.0012 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1209 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0576 | Regression_eta: 0.0346 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0797 | LR: 1.05e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.5042s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0068 | max_forward: 0.6708 | max_loss: 0.1613 | max_backward: 1.2326 | max_optimizer_step: 0.1946 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6306.7 | uss_MB: 5481.0 | child_rss_MB: 5031.8 | allocated_MB: 368.1 | reserved_MB: 24122.0 | inactive_split_MB: 276.0 +rank1 [2026-10-29 23:50:02,910] INFO : Step 47800/50000 rank1 | Train Loss: 0.9063 | Train Loss Unweighted: 0.4379 | Train Loss Components: Auxiliary: 0.2020 | Cardinality: 0.0932 | Classification: 0.0920 | Classification_binary: 0.1284 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0011 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1145 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0508 | Regression_eta: 0.0304 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0381 | LR: 1.04e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.8433s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0610 | max_forward: 0.6626 | max_loss: 0.1349 | max_backward: 1.5091 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6302.7 | uss_MB: 5485.7 | child_rss_MB: 5029.8 | allocated_MB: 369.5 | reserved_MB: 24122.0 | inactive_split_MB: 236.5 +rank1 [2026-10-29 23:54:07,643] INFO : Step 47900/50000 rank1 | Train Loss: 0.9591 | Train Loss Unweighted: 0.4478 | Train Loss Components: Auxiliary: 0.2152 | Cardinality: 0.1022 | Classification: 0.0882 | Classification_binary: 0.1545 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1003 | Regression_eta: 0.0014 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0883 | Regression_energy: 0.0479 | Regression_eta: 0.0722 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0733 | LR: 1.04e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1706s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8744 | max_loss: 0.2241 | max_backward: 1.6370 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 6302.2 | uss_MB: 5483.9 | child_rss_MB: 5030.3 | allocated_MB: 370.5 | reserved_MB: 24122.0 | inactive_split_MB: 218.9 +rank1 [2026-10-30 00:00:28,988] INFO : Step 48000/50000 rank1 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1221 | Classification: 0.0823 | Classification_binary: 0.1525 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1079 | Regression_eta: 0.0017 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0415 | Regression_eta: 0.0279 | Regression_pt: 0.0426 | Regression_sin_phi: 0.0408 | LR: 1.04e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.1287s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0057 | max_forward: 0.7069 | max_loss: 0.2447 | max_backward: 1.4090 | max_optimizer_step: 0.1933 | max_tensorboard_logging: 0.8132 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1250 | Memory: rss_MB: 6301.2 | uss_MB: 5485.0 | child_rss_MB: 5024.5 | allocated_MB: 368.8 | reserved_MB: 24122.0 | inactive_split_MB: 269.6 +rank1 [2026-10-30 00:00:29,930] INFO : Running validation on rank1 +rank1 [2026-10-30 00:00:29,930] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 00:00:29,930] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 00:01:19,557] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:01:20,065] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 00:01:46,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 00:01:48,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 00:01:49,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 00:01:50,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 00:02:00,492] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-30 00:02:01,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:02:01,858] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 00:02:17,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 00:02:18,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 00:02:19,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 00:02:20,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 00:02:30,698] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-30 00:02:32,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:02:32,991] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 00:02:48,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 00:02:49,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 00:02:49,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 00:02:50,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 00:03:00,631] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-30 00:07:25,203] INFO : Step 48100/50000 rank1 | Train Loss: 0.9084 | Train Loss Unweighted: 0.4360 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1162 | Classification: 0.0769 | Classification_binary: 0.1204 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1183 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1161 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0591 | Regression_eta: 0.0207 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0396 | LR: 1.03e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9231s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1372 | max_backward: 1.2024 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.9158 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7374.2 | uss_MB: 6557.5 | child_rss_MB: 5038.7 | allocated_MB: 367.7 | reserved_MB: 24830.0 | inactive_split_MB: 222.3 +rank1 [2026-10-30 00:11:33,472] INFO : Step 48200/50000 rank1 | Train Loss: 0.9585 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2151 | Cardinality: 0.1199 | Classification: 0.0839 | Classification_binary: 0.1503 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0585 | Regression_eta: 0.0374 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0544 | LR: 1.03e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4375s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7766 | max_loss: 0.1498 | max_backward: 1.7414 | max_optimizer_step: 0.3242 | max_tensorboard_logging: 0.7504 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 7374.2 | uss_MB: 6560.6 | child_rss_MB: 5031.5 | allocated_MB: 370.4 | reserved_MB: 24830.0 | inactive_split_MB: 179.3 +rank1 [2026-10-30 00:15:41,698] INFO : Step 48300/50000 rank1 | Train Loss: 1.0575 | Train Loss Unweighted: 0.4963 | Train Loss Components: Auxiliary: 0.2408 | Cardinality: 0.1341 | Classification: 0.0958 | Classification_binary: 0.1687 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1138 | Regression_eta: 0.0013 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0523 | Regression_eta: 0.0396 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0431 | LR: 1.03e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2036s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7497 | max_loss: 0.2201 | max_backward: 1.7233 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6378.2 | uss_MB: 5566.5 | child_rss_MB: 5032.3 | allocated_MB: 372.3 | reserved_MB: 24830.0 | inactive_split_MB: 269.6 +rank1 [2026-10-30 00:19:56,563] INFO : Step 48400/50000 rank1 | Train Loss: 0.9108 | Train Loss Unweighted: 0.4349 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1093 | Classification: 0.0691 | Classification_binary: 0.1311 | HitGrouping: 0.1830 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1153 | Regression_eta: 0.0013 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1287 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0639 | Regression_eta: 0.0304 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0601 | LR: 1.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8541s | Phase Times: max_pre_step_barrier: 0.1141 | max_batch_to_device: 0.0064 | max_forward: 0.6489 | max_loss: 0.1390 | max_backward: 1.1801 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6705 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6380.2 | uss_MB: 5566.3 | child_rss_MB: 5030.1 | allocated_MB: 370.1 | reserved_MB: 24830.0 | inactive_split_MB: 261.5 +rank1 [2026-10-30 00:23:53,558] INFO : Step 48500/50000 rank1 | Train Loss: 0.9694 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1004 | Classification: 0.0844 | Classification_binary: 0.1367 | HitGrouping: 0.1928 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1194 | Regression_eta: 0.0012 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1428 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0460 | Regression_eta: 0.0338 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0608 | LR: 1.02e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.7557s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 1.7512 | max_loss: 0.2678 | max_backward: 1.9538 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.4804 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 6374.4 | uss_MB: 5565.1 | child_rss_MB: 5023.6 | allocated_MB: 371.5 | reserved_MB: 26842.0 | inactive_split_MB: 176.5 +rank1 [2026-10-30 00:27:51,108] INFO : Step 48600/50000 rank1 | Train Loss: 0.9724 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1148 | Classification: 0.0854 | Classification_binary: 0.1551 | HitGrouping: 0.1844 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0545 | Regression_eta: 0.0369 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0707 | LR: 1.02e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0067s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0086 | max_forward: 0.6984 | max_loss: 0.2525 | max_backward: 1.6611 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6375.4 | uss_MB: 5569.8 | child_rss_MB: 5026.2 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 234.8 +rank1 [2026-10-30 00:31:52,432] INFO : Step 48700/50000 rank1 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4863 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1212 | Classification: 0.0968 | Classification_binary: 0.1569 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1143 | Regression_eta: 0.0012 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0502 | Regression_eta: 0.0247 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0546 | LR: 1.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9634s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7410 | max_loss: 0.2472 | max_backward: 1.4647 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.6585 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6316.1 | uss_MB: 5565.7 | child_rss_MB: 4823.4 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 275.8 +rank1 [2026-10-30 00:35:40,395] INFO : Step 48800/50000 rank1 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1981 | Cardinality: 0.0898 | Classification: 0.0814 | Classification_binary: 0.1196 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1094 | Regression_eta: 0.0012 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1343 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0526 | Regression_eta: 0.0342 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0421 | LR: 1.01e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.7732s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.6122 | max_loss: 0.1775 | max_backward: 1.1932 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6316.1 | uss_MB: 5568.8 | child_rss_MB: 4813.9 | allocated_MB: 368.0 | reserved_MB: 26842.0 | inactive_split_MB: 263.7 +rank1 [2026-10-30 00:39:23,571] INFO : Step 48900/50000 rank1 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1005 | Classification: 0.0873 | Classification_binary: 0.1401 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1708 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0476 | Regression_eta: 0.0341 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0309 | LR: 1.01e-04 | DataLoad Time: 0.0184s | Model Forward Time: 3.9273s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0809 | max_forward: 0.7397 | max_loss: 0.1455 | max_backward: 1.5027 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6303.5 | uss_MB: 5491.8 | child_rss_MB: 4768.8 | allocated_MB: 373.0 | reserved_MB: 26842.0 | inactive_split_MB: 191.3 +rank1 [2026-10-30 00:43:08,795] INFO : Step 49000/50000 rank1 | Train Loss: 1.0019 | Train Loss Unweighted: 0.4657 | Train Loss Components: Auxiliary: 0.2334 | Cardinality: 0.1354 | Classification: 0.0893 | Classification_binary: 0.1587 | HitGrouping: 0.1838 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0010 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1581 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0521 | Regression_eta: 0.0283 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0550 | LR: 1.01e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4535s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1085 | max_forward: 0.8072 | max_loss: 0.1558 | max_backward: 1.7809 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.6082 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6305.0 | uss_MB: 5577.9 | child_rss_MB: 4749.4 | allocated_MB: 370.9 | reserved_MB: 26842.0 | inactive_split_MB: 271.3 +rank1 [2026-10-30 00:43:09,868] INFO : Running validation on rank1 +rank1 [2026-10-30 00:43:09,868] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 00:43:09,868] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 00:43:59,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:43:59,787] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 00:44:27,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 00:44:28,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 00:44:29,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 00:44:30,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 00:44:40,139] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 00:44:40,645] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:44:41,122] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 00:44:57,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 00:44:58,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 00:44:58,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 00:44:59,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 00:45:09,949] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-30 00:45:10,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:45:10,743] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 00:45:25,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 00:45:26,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 00:45:27,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 00:45:28,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 00:45:38,020] INFO : Time taken to make predictions on device 1 is: 0.22 min +rank1 [2026-10-30 00:49:42,469] INFO : Step 49100/50000 rank1 | Train Loss: 0.9797 | Train Loss Unweighted: 0.4561 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1125 | Classification: 0.0877 | Classification_binary: 0.1433 | HitGrouping: 0.1905 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0013 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1304 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0543 | Regression_eta: 0.0321 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0431 | LR: 1.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.7304s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6869 | max_loss: 0.1436 | max_backward: 1.2639 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.4741 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0455 | Memory: rss_MB: 7470.2 | uss_MB: 6734.9 | child_rss_MB: 4778.3 | allocated_MB: 370.4 | reserved_MB: 22116.0 | inactive_split_MB: 274.9 +rank1 [2026-10-30 00:53:47,122] INFO : Step 49200/50000 rank1 | Train Loss: 0.8929 | Train Loss Unweighted: 0.4393 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1020 | Classification: 0.0903 | Classification_binary: 0.1250 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0009 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1440 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0634 | Regression_eta: 0.0318 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0342 | LR: 1.01e-04 | DataLoad Time: 0.0146s | Model Forward Time: 3.7554s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6890 | max_loss: 0.2430 | max_backward: 1.3427 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.6347 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7470.5 | uss_MB: 6738.6 | child_rss_MB: 4771.6 | allocated_MB: 368.3 | reserved_MB: 24092.0 | inactive_split_MB: 187.3 +rank1 [2026-10-30 00:57:33,941] INFO : Step 49300/50000 rank1 | Train Loss: 1.0122 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1210 | Classification: 0.0951 | Classification_binary: 0.1621 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1066 | Regression_eta: 0.0015 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0383 | Regression_energy: 0.0484 | Regression_eta: 0.0319 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0694 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8969s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1495 | max_backward: 1.5906 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6415.4 | uss_MB: 5684.6 | child_rss_MB: 4746.9 | allocated_MB: 370.9 | reserved_MB: 24092.0 | inactive_split_MB: 226.9 +rank1 [2026-10-30 01:01:33,225] INFO : Step 49400/50000 rank1 | Train Loss: 0.9790 | Train Loss Unweighted: 0.4695 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1137 | Classification: 0.0883 | Classification_binary: 0.1513 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1131 | Regression_eta: 0.0013 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0484 | Regression_eta: 0.0268 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0495 | LR: 1.00e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.0358s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2433 | max_backward: 1.8823 | max_optimizer_step: 0.3081 | max_tensorboard_logging: 0.5951 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6413.4 | uss_MB: 5608.6 | child_rss_MB: 4765.4 | allocated_MB: 372.2 | reserved_MB: 24092.0 | inactive_split_MB: 278.9 +rank1 [2026-10-30 01:05:37,233] INFO : Step 49500/50000 rank1 | Train Loss: 0.9706 | Train Loss Unweighted: 0.4581 | Train Loss Components: Auxiliary: 0.2120 | Cardinality: 0.1158 | Classification: 0.0879 | Classification_binary: 0.1279 | HitGrouping: 0.1920 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1247 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0501 | Regression_eta: 0.0312 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0409 | LR: 1.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9629s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6307 | max_loss: 0.1505 | max_backward: 1.3337 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.8584 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6409.4 | uss_MB: 5683.7 | child_rss_MB: 4759.9 | allocated_MB: 370.4 | reserved_MB: 24614.0 | inactive_split_MB: 171.1 +rank1 [2026-10-30 01:09:49,675] INFO : Step 49600/50000 rank1 | Train Loss: 0.9965 | Train Loss Unweighted: 0.4752 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1098 | Classification: 0.1016 | Classification_binary: 0.1437 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1133 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0611 | Regression_eta: 0.0387 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0604 | LR: 1.00e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1640s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.6933 | max_loss: 0.1496 | max_backward: 1.5639 | max_optimizer_step: 0.3077 | max_tensorboard_logging: 0.7702 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6412.2 | uss_MB: 5687.4 | child_rss_MB: 4755.4 | allocated_MB: 371.3 | reserved_MB: 24614.0 | inactive_split_MB: 250.0 +rank1 [2026-10-30 01:13:44,417] INFO : Step 49700/50000 rank1 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4775 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1108 | Classification: 0.1000 | Classification_binary: 0.1702 | HitGrouping: 0.1796 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1019 | Regression_eta: 0.0014 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.1700 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0516 | Regression_eta: 0.0344 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0571 | LR: 1.00e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.2319s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7591 | max_loss: 0.1933 | max_backward: 1.7410 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 6407.4 | uss_MB: 5675.2 | child_rss_MB: 4703.7 | allocated_MB: 372.4 | reserved_MB: 24614.0 | inactive_split_MB: 259.5 +rank1 [2026-10-30 01:17:46,084] INFO : Step 49800/50000 rank1 | Train Loss: 0.9472 | Train Loss Unweighted: 0.4401 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1188 | Classification: 0.0821 | Classification_binary: 0.1427 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1293 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0523 | Regression_eta: 0.0247 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0563 | LR: 1.00e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.7716s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7430 | max_loss: 0.2153 | max_backward: 1.0846 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.7647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6388.3 | uss_MB: 5552.5 | child_rss_MB: 4630.2 | allocated_MB: 368.0 | reserved_MB: 24614.0 | inactive_split_MB: 273.7 +rank1 [2026-10-30 01:21:39,660] INFO : Step 49900/50000 rank1 | Train Loss: 0.9411 | Train Loss Unweighted: 0.4586 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.0996 | Classification: 0.0961 | Classification_binary: 0.1284 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1145 | Regression_eta: 0.0013 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1247 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0504 | Regression_eta: 0.0679 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0497 | LR: 1.00e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0426s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.6992 | max_loss: 0.1527 | max_backward: 1.3836 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8576 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6363.3 | uss_MB: 5692.1 | child_rss_MB: 4528.3 | allocated_MB: 368.7 | reserved_MB: 24614.0 | inactive_split_MB: 241.3 +rank1 [2026-10-30 01:25:29,378] INFO : Step 50000/50000 rank1 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2297 | Cardinality: 0.1196 | Classification: 0.0902 | Classification_binary: 0.1583 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1576 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0420 | Regression_eta: 0.0387 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0557 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2529s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.7146 | max_loss: 0.1481 | max_backward: 1.6207 | max_optimizer_step: 0.2721 | max_tensorboard_logging: 0.8195 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6364.5 | uss_MB: 5690.8 | child_rss_MB: 4519.4 | allocated_MB: 371.5 | reserved_MB: 25088.0 | inactive_split_MB: 268.9 +rank1 [2026-10-30 01:25:30,313] INFO : Running validation on rank1 +rank1 [2026-10-30 01:25:30,313] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 01:26:21,849] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:26:22,339] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 01:26:50,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 01:26:51,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 01:26:52,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 01:26:53,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 01:27:03,480] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-30 01:27:03,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:27:04,490] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 01:27:20,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 01:27:21,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 01:27:22,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 01:27:23,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 01:27:33,241] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-30 01:27:34,256] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:27:34,756] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 01:27:50,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 01:27:51,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 01:27:51,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 01:27:52,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 01:28:02,647] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 01:28:03,360] INFO : Training completed. Total time on device 1: 638.886min diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.2 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.2 new file mode 100644 index 0000000000000000000000000000000000000000..da8057de814fc9be6d06cd5d3c8b1f2594395e34 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.2 @@ -0,0 +1,2484 @@ +rank2 [2026-10-28 11:12:33,111] INFO : Initializing model with seed=12345; process seed=12347 +rank2 [2026-10-28 11:12:33,112] INFO : configured dtype=torch.bfloat16 for autocast +rank2 [2026-10-28 11:12:54,339] INFO : Instantiating model +rank2 [2026-10-28 11:12:54,339] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank2 [2026-10-28 11:12:54,339] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank2 [2026-10-28 11:12:54,358] INFO : Input encoding initialization took 0.02s +rank2 [2026-10-28 11:12:54,358] INFO : nn0 parameters: 171776 +rank2 [2026-10-28 11:12:54,358] INFO : Initializing shared backbone layers, num_convs=6 +rank2 [2026-10-28 11:12:54,358] INFO : Initializing backbone layer 0 +rank2 [2026-10-28 11:12:54,358] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:54,362] INFO : Initializing backbone layer 1 +rank2 [2026-10-28 11:12:54,362] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:54,365] INFO : Initializing backbone layer 2 +rank2 [2026-10-28 11:12:54,365] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:54,368] INFO : Initializing backbone layer 3 +rank2 [2026-10-28 11:12:54,368] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:54,371] INFO : Initializing backbone layer 4 +rank2 [2026-10-28 11:12:54,371] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:54,374] INFO : Initializing backbone layer 5 +rank2 [2026-10-28 11:12:54,375] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-28 11:12:54,378] INFO : Backbone initialization took 0.02s +rank2 [2026-10-28 11:12:54,378] INFO : backbone parameters: 2374656 +rank2 [2026-10-28 11:12:54,378] INFO : Initializing output DNNs +rank2 [2026-10-28 11:12:54,427] INFO : Output DNNs initialization took 0.05s +rank2 [2026-10-28 11:12:54,427] INFO : backbone_mode=BackboneMode.SHARED +rank2 [2026-10-28 11:12:54,427] INFO : jagged_attention enabled=True +rank2 [2026-10-28 11:12:54,427] INFO : flash_attn_varlen enabled=True +rank2 [2026-10-28 11:12:54,427] INFO : task_queries enabled=False +rank2 [2026-10-28 11:12:54,427] INFO : classification_norm parameters: 0 +rank2 [2026-10-28 11:12:54,427] INFO : regression_norm parameters: 0 +rank2 [2026-10-28 11:12:54,427] INFO : classification_query parameters: 0 +rank2 [2026-10-28 11:12:54,427] INFO : regression_query parameters: 0 +rank2 [2026-10-28 11:12:54,427] INFO : classification_readout parameters: 0 +rank2 [2026-10-28 11:12:54,427] INFO : regression_readout parameters: 0 +rank2 [2026-10-28 11:12:54,428] INFO : output_mode=set +rank2 [2026-10-28 11:12:54,428] INFO : set_decoder parameters: 4713812 +rank2 [2026-10-28 11:12:54,428] INFO : nn_binary_particle parameters: 0 +rank2 [2026-10-28 11:12:54,428] INFO : nn_pid parameters: 0 +rank2 [2026-10-28 11:12:54,428] INFO : nn_pt parameters: 0 +rank2 [2026-10-28 11:12:54,428] INFO : nn_eta parameters: 0 +rank2 [2026-10-28 11:12:54,428] INFO : nn_sin_phi parameters: 0 +rank2 [2026-10-28 11:12:54,428] INFO : nn_cos_phi parameters: 0 +rank2 [2026-10-28 11:12:54,428] INFO : nn_energy parameters: 0 +rank2 [2026-10-28 11:12:54,429] INFO : Total MLPF parameters: 7260244 +rank2 [2026-10-28 11:12:54,429] INFO : MLPF __init__ done +rank2 [2026-10-28 11:12:54,430] INFO : Instantiated model +rank2 [2026-10-28 11:12:54,430] INFO : Moving model to device rank=2 +rank2 [2026-10-28 11:12:54,449] INFO : Moved model to device rank=2 +rank2 [2026-10-28 11:12:54,450] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank2 [2026-10-28 11:12:54,451] INFO : Configured model for SyncBatchNorm rank=2 +rank2 [2026-10-28 11:12:54,646] INFO : Configured model for DistributedDataParallel rank=2 +rank2 [2026-10-28 11:12:54,764] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank2 [2026-10-28 11:12:54,764] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank2 [2026-10-28 11:12:54,765] INFO : Trainable parameters: 7260244 +rank2 [2026-10-28 11:12:54,765] INFO : Non-trainable parameters: 0 +rank2 [2026-10-28 11:12:54,765] INFO : Total parameters: 7260244 +rank2 [2026-10-28 11:12:54,768] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank2 [2026-10-28 11:12:54,768] INFO : Getting dataloaders +rank2 [2026-10-28 11:12:54,768] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:03,736] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:04,812] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,149] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-28 11:13:06,150] INFO : Creating WeightedMixIterator with 1 data loaders. +rank2 [2026-10-28 11:13:06,150] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,436] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,693] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,957] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-28 11:13:06,958] INFO : Creating InterleavedIterator with 1 data loaders. +rank2 [2026-10-28 11:13:21,268] INFO : Got dataloaders +rank2 [2026-10-28 11:13:21,268] INFO : loader split=train rank=2 len=21112 +rank2 [2026-10-28 11:13:21,268] INFO : loader split=valid rank=2 len=12 +rank2 [2026-10-28 11:13:21,268] INFO : Starting training from step 1 to 50000 on rank 2 of 8 +rank2 [2026-10-28 11:13:21,268] INFO : Creating train_iterator +rank2 [2026-10-28 11:13:21,269] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-28 11:13:36,047] INFO : Created train_iterator +rank2 [2026-10-28 11:17:15,587] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5434697668626904, 'Classification': 0.9147019968181849, 'Regression_pt': 1.0888657218962907, 'Regression_eta': 0.019410275079426356, 'Regression_sin_phi': 0.00660347732307855, 'Regression_cos_phi': 0.006484371119877323, 'Regression_energy': 1.1263945941627025}, weights={'Classification_binary': 1.0, 'Classification': 0.5941495299339294, 'Regression_pt': 0.4991154968738556, 'Regression_eta': 27.99907684326172, 'Regression_sin_phi': 82.30054473876953, 'Regression_cos_phi': 83.812255859375, 'Regression_energy': 0.48248612880706787} +rank2 [2026-10-28 11:17:17,665] INFO : Step 100/50000 rank2 | Train Loss: 5.4941 | Train Loss Unweighted: 3.3573 | Train Loss Components: Auxiliary: 1.0155 | Cardinality: 0.6417 | Classification: 0.8373 | Classification_binary: 0.4757 | HitGrouping: 0.4797 | Regression_cos_phi: 0.0063 | Regression_energy: 1.0524 | Regression_eta: 0.0130 | Regression_pt: 0.9668 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8661 | Classification_binary: 0.4525 | Regression_cos_phi: 0.0049 | Regression_energy: 1.0871 | Regression_eta: 0.0103 | Regression_pt: 1.0591 | Regression_sin_phi: 0.0055 | LR: 1.00e-03 | DataLoad Time: 0.0148s | Model Forward Time: 3.0165s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.5597 | max_loss: 0.2615 | max_backward: 1.0157 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.4654 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4647.0 | uss_MB: 3401.7 | child_rss_MB: 5567.8 | allocated_MB: 341.0 | reserved_MB: 22914.0 | inactive_split_MB: 237.2 +rank2 [2026-10-28 11:20:50,235] INFO : Step 200/50000 rank2 | Train Loss: 4.4651 | Train Loss Unweighted: 2.7232 | Train Loss Components: Auxiliary: 0.8610 | Cardinality: 0.5153 | Classification: 0.6425 | Classification_binary: 0.5162 | HitGrouping: 0.4582 | Regression_cos_phi: 0.0043 | Regression_energy: 0.7694 | Regression_eta: 0.0086 | Regression_pt: 0.7776 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3928 | Classification_binary: 0.5284 | Regression_cos_phi: 0.4363 | Regression_energy: 0.3900 | Regression_eta: 0.2625 | Regression_pt: 0.4118 | Regression_sin_phi: 0.3990 | LR: 1.00e-03 | DataLoad Time: 0.0147s | Model Forward Time: 3.7256s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7368 | max_loss: 0.2463 | max_backward: 1.7032 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5468 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4651.2 | uss_MB: 3409.7 | child_rss_MB: 5571.3 | allocated_MB: 343.8 | reserved_MB: 23264.0 | inactive_split_MB: 284.1 +rank2 [2026-10-28 11:24:24,163] INFO : Step 300/50000 rank2 | Train Loss: 3.6383 | Train Loss Unweighted: 2.1819 | Train Loss Components: Auxiliary: 0.6931 | Cardinality: 0.3447 | Classification: 0.5089 | Classification_binary: 0.4824 | HitGrouping: 0.4305 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5908 | Regression_eta: 0.0074 | Regression_pt: 0.5852 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3084 | Classification_binary: 0.4792 | Regression_cos_phi: 0.2992 | Regression_energy: 0.2561 | Regression_eta: 0.2089 | Regression_pt: 0.2621 | Regression_sin_phi: 0.2857 | LR: 1.00e-03 | DataLoad Time: 0.0148s | Model Forward Time: 3.8723s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0315 | max_forward: 0.6855 | max_loss: 0.1568 | max_backward: 1.3632 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 1.0182 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4652.7 | uss_MB: 3415.2 | child_rss_MB: 5569.6 | allocated_MB: 341.4 | reserved_MB: 23916.0 | inactive_split_MB: 289.5 +rank2 [2026-10-28 11:27:58,396] INFO : Step 400/50000 rank2 | Train Loss: 3.5064 | Train Loss Unweighted: 1.9938 | Train Loss Components: Auxiliary: 0.6582 | Cardinality: 0.3738 | Classification: 0.4591 | Classification_binary: 0.4306 | HitGrouping: 0.4041 | Regression_cos_phi: 0.0039 | Regression_energy: 0.5433 | Regression_eta: 0.0080 | Regression_pt: 0.5456 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2733 | Classification_binary: 0.4032 | Regression_cos_phi: 0.2761 | Regression_energy: 0.2560 | Regression_eta: 0.1643 | Regression_pt: 0.2601 | Regression_sin_phi: 0.1920 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 3.7416s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.6100 | max_loss: 0.1486 | max_backward: 1.2396 | max_optimizer_step: 0.1474 | max_tensorboard_logging: 1.0157 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4650.3 | uss_MB: 3416.8 | child_rss_MB: 5584.9 | allocated_MB: 342.9 | reserved_MB: 23982.0 | inactive_split_MB: 297.5 +rank2 [2026-10-28 11:31:35,335] INFO : Step 500/50000 rank2 | Train Loss: 3.8899 | Train Loss Unweighted: 2.1856 | Train Loss Components: Auxiliary: 0.6982 | Cardinality: 0.3682 | Classification: 0.4707 | Classification_binary: 0.4836 | HitGrouping: 0.4071 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6048 | Regression_eta: 0.0103 | Regression_pt: 0.6070 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2758 | Classification_binary: 0.5340 | Regression_cos_phi: 0.4077 | Regression_energy: 0.2841 | Regression_eta: 0.3069 | Regression_pt: 0.2939 | Regression_sin_phi: 0.4104 | LR: 1.00e-03 | DataLoad Time: 0.0155s | Model Forward Time: 4.1831s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.7460 | max_loss: 0.2622 | max_backward: 1.7372 | max_optimizer_step: 0.1478 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 4651.3 | uss_MB: 3418.2 | child_rss_MB: 5567.0 | allocated_MB: 344.2 | reserved_MB: 24988.0 | inactive_split_MB: 288.4 +rank2 [2026-10-28 11:35:12,860] INFO : Step 600/50000 rank2 | Train Loss: 3.5648 | Train Loss Unweighted: 2.0654 | Train Loss Components: Auxiliary: 0.6756 | Cardinality: 0.3628 | Classification: 0.4159 | Classification_binary: 0.5246 | HitGrouping: 0.3632 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5539 | Regression_eta: 0.0077 | Regression_pt: 0.5557 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2668 | Classification_binary: 0.5323 | Regression_cos_phi: 0.3666 | Regression_energy: 0.2799 | Regression_eta: 0.2147 | Regression_pt: 0.2881 | Regression_sin_phi: 0.4581 | LR: 1.00e-03 | DataLoad Time: 0.0146s | Model Forward Time: 4.1940s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0071 | max_forward: 0.7483 | max_loss: 0.2574 | max_backward: 1.6305 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.9709 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 4647.1 | uss_MB: 3418.2 | child_rss_MB: 5567.3 | allocated_MB: 344.6 | reserved_MB: 24988.0 | inactive_split_MB: 289.5 +rank2 [2026-10-28 11:38:47,058] INFO : Step 700/50000 rank2 | Train Loss: 3.0543 | Train Loss Unweighted: 1.8237 | Train Loss Components: Auxiliary: 0.6036 | Cardinality: 0.3094 | Classification: 0.3407 | Classification_binary: 0.4142 | HitGrouping: 0.3367 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5254 | Regression_eta: 0.0060 | Regression_pt: 0.5314 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2034 | Classification_binary: 0.4007 | Regression_cos_phi: 0.1922 | Regression_energy: 0.2532 | Regression_eta: 0.1801 | Regression_pt: 0.2600 | Regression_sin_phi: 0.2754 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 3.4875s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.6445 | max_loss: 0.2136 | max_backward: 1.2179 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.7893 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4647.0 | uss_MB: 3416.3 | child_rss_MB: 5577.3 | allocated_MB: 341.4 | reserved_MB: 24988.0 | inactive_split_MB: 289.7 +rank2 [2026-10-28 11:42:25,056] INFO : Step 800/50000 rank2 | Train Loss: 3.1854 | Train Loss Unweighted: 1.9126 | Train Loss Components: Auxiliary: 0.6154 | Cardinality: 0.2934 | Classification: 0.4077 | Classification_binary: 0.4086 | HitGrouping: 0.3317 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5404 | Regression_eta: 0.0067 | Regression_pt: 0.5423 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2165 | Classification_binary: 0.4320 | Regression_cos_phi: 0.3027 | Regression_energy: 0.2246 | Regression_eta: 0.2034 | Regression_pt: 0.2335 | Regression_sin_phi: 0.2402 | LR: 9.99e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.3175s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.6526 | max_loss: 0.2001 | max_backward: 1.2539 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5713 | max_tensorboard_flush: 0.0138 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 4642.6 | uss_MB: 3416.3 | child_rss_MB: 5579.0 | allocated_MB: 346.1 | reserved_MB: 24988.0 | inactive_split_MB: 222.7 +rank2 [2026-10-28 11:46:07,653] INFO : Step 900/50000 rank2 | Train Loss: 3.4950 | Train Loss Unweighted: 2.0678 | Train Loss Components: Auxiliary: 0.6566 | Cardinality: 0.3469 | Classification: 0.4160 | Classification_binary: 0.4953 | HitGrouping: 0.3295 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5663 | Regression_eta: 0.0083 | Regression_pt: 0.5742 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2363 | Classification_binary: 0.4667 | Regression_cos_phi: 0.2996 | Regression_energy: 0.2645 | Regression_eta: 0.2250 | Regression_pt: 0.2757 | Regression_sin_phi: 0.3263 | LR: 9.99e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.8986s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8131 | max_loss: 0.2548 | max_backward: 1.8396 | max_optimizer_step: 0.1492 | max_tensorboard_logging: 0.4938 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 4643.4 | uss_MB: 3417.5 | child_rss_MB: 5588.2 | allocated_MB: 344.2 | reserved_MB: 26006.0 | inactive_split_MB: 273.7 +rank2 [2026-10-28 11:49:58,411] INFO : Step 1000/50000 rank2 | Train Loss: 3.0581 | Train Loss Unweighted: 1.8801 | Train Loss Components: Auxiliary: 0.5942 | Cardinality: 0.2760 | Classification: 0.3603 | Classification_binary: 0.4232 | HitGrouping: 0.3177 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5411 | Regression_eta: 0.0066 | Regression_pt: 0.5426 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2048 | Classification_binary: 0.4118 | Regression_cos_phi: 0.2651 | Regression_energy: 0.2203 | Regression_eta: 0.1427 | Regression_pt: 0.2302 | Regression_sin_phi: 0.2232 | LR: 9.99e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2152s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1006 | max_forward: 0.7513 | max_loss: 0.1459 | max_backward: 1.3658 | max_optimizer_step: 0.1511 | max_tensorboard_logging: 1.2506 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4639.1 | uss_MB: 3415.3 | child_rss_MB: 5584.2 | allocated_MB: 344.4 | reserved_MB: 26006.0 | inactive_split_MB: 271.1 +rank2 [2026-10-28 11:49:59,499] INFO : Running validation on rank2 +rank2 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 11:50:52,705] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:50:53,226] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 11:51:19,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 11:51:20,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 11:51:21,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 11:51:22,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 11:51:32,440] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 11:51:32,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:51:33,322] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 11:51:49,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 11:51:50,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 11:51:51,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 11:51:52,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 11:52:02,257] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 11:52:02,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:52:03,477] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 11:52:19,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 11:52:19,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 11:52:20,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 11:52:20,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 11:52:30,956] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 11:56:35,454] INFO : Step 1100/50000 rank2 | Train Loss: 2.9181 | Train Loss Unweighted: 1.8556 | Train Loss Components: Auxiliary: 0.5873 | Cardinality: 0.2540 | Classification: 0.3370 | Classification_binary: 0.3789 | HitGrouping: 0.3247 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5584 | Regression_eta: 0.0064 | Regression_pt: 0.5695 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1912 | Classification_binary: 0.4008 | Regression_cos_phi: 0.2256 | Regression_energy: 0.2604 | Regression_eta: 0.1471 | Regression_pt: 0.2754 | Regression_sin_phi: 0.1936 | LR: 9.99e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9464s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8528 | max_loss: 0.1390 | max_backward: 1.4220 | max_optimizer_step: 0.2689 | max_tensorboard_logging: 0.9135 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 6058.3 | uss_MB: 4835.8 | child_rss_MB: 5619.2 | allocated_MB: 344.1 | reserved_MB: 21080.0 | inactive_split_MB: 227.9 +rank2 [2026-10-28 12:00:25,221] INFO : Step 1200/50000 rank2 | Train Loss: 3.1885 | Train Loss Unweighted: 1.9202 | Train Loss Components: Auxiliary: 0.6265 | Cardinality: 0.3516 | Classification: 0.3718 | Classification_binary: 0.4294 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5507 | Regression_eta: 0.0065 | Regression_pt: 0.5556 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1951 | Classification_binary: 0.4126 | Regression_cos_phi: 0.1897 | Regression_energy: 0.2466 | Regression_eta: 0.1751 | Regression_pt: 0.2563 | Regression_sin_phi: 0.2481 | LR: 9.99e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.4091s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0076 | max_forward: 0.7370 | max_loss: 0.3921 | max_backward: 1.7945 | max_optimizer_step: 0.1639 | max_tensorboard_logging: 1.0479 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 6061.1 | uss_MB: 4838.4 | child_rss_MB: 5619.8 | allocated_MB: 344.9 | reserved_MB: 23662.0 | inactive_split_MB: 241.9 +rank2 [2026-10-28 12:04:16,197] INFO : Step 1300/50000 rank2 | Train Loss: 3.2694 | Train Loss Unweighted: 1.9280 | Train Loss Components: Auxiliary: 0.6193 | Cardinality: 0.3461 | Classification: 0.3601 | Classification_binary: 0.4731 | HitGrouping: 0.3156 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5359 | Regression_eta: 0.0069 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2105 | Classification_binary: 0.4210 | Regression_cos_phi: 0.2981 | Regression_energy: 0.2654 | Regression_eta: 0.1981 | Regression_pt: 0.2785 | Regression_sin_phi: 0.2376 | LR: 9.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.5217s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0540 | max_forward: 0.9909 | max_loss: 0.3234 | max_backward: 2.2964 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 1.1252 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5498.9 | uss_MB: 4193.7 | child_rss_MB: 5645.6 | allocated_MB: 344.9 | reserved_MB: 24052.0 | inactive_split_MB: 299.1 +rank2 [2026-10-28 12:07:55,472] INFO : Step 1400/50000 rank2 | Train Loss: 2.8491 | Train Loss Unweighted: 1.7889 | Train Loss Components: Auxiliary: 0.5645 | Cardinality: 0.2711 | Classification: 0.3226 | Classification_binary: 0.3768 | HitGrouping: 0.3109 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5376 | Regression_eta: 0.0061 | Regression_pt: 0.5407 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1939 | Classification_binary: 0.3270 | Regression_cos_phi: 0.2424 | Regression_energy: 0.2268 | Regression_eta: 0.1707 | Regression_pt: 0.2359 | Regression_sin_phi: 0.2431 | LR: 9.98e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7580s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0094 | max_forward: 0.6378 | max_loss: 0.4388 | max_backward: 1.2886 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 0.9631 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 5501.2 | uss_MB: 4277.0 | child_rss_MB: 5652.8 | allocated_MB: 350.4 | reserved_MB: 24916.0 | inactive_split_MB: 216.1 +rank2 [2026-10-28 12:11:42,090] INFO : Step 1500/50000 rank2 | Train Loss: 2.8928 | Train Loss Unweighted: 1.7749 | Train Loss Components: Auxiliary: 0.5690 | Cardinality: 0.2892 | Classification: 0.3566 | Classification_binary: 0.3704 | HitGrouping: 0.3078 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5147 | Regression_eta: 0.0058 | Regression_pt: 0.5216 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2143 | Classification_binary: 0.4120 | Regression_cos_phi: 0.2410 | Regression_energy: 0.2499 | Regression_eta: 0.1447 | Regression_pt: 0.2642 | Regression_sin_phi: 0.2260 | LR: 9.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5611s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.6094 | max_loss: 0.4244 | max_backward: 1.4479 | max_optimizer_step: 0.1602 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5488.8 | uss_MB: 4270.8 | child_rss_MB: 5621.5 | allocated_MB: 342.2 | reserved_MB: 24916.0 | inactive_split_MB: 182.9 +rank2 [2026-10-28 12:15:27,599] INFO : Step 1600/50000 rank2 | Train Loss: 2.9805 | Train Loss Unweighted: 1.8512 | Train Loss Components: Auxiliary: 0.5725 | Cardinality: 0.2336 | Classification: 0.3605 | Classification_binary: 0.4404 | HitGrouping: 0.3159 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5142 | Regression_eta: 0.0065 | Regression_pt: 0.5235 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2145 | Classification_binary: 0.4239 | Regression_cos_phi: 0.2134 | Regression_energy: 0.2523 | Regression_eta: 0.1943 | Regression_pt: 0.2615 | Regression_sin_phi: 0.2542 | LR: 9.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7171s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7382 | max_loss: 0.1451 | max_backward: 1.5359 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.4679 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5499.3 | uss_MB: 4281.1 | child_rss_MB: 5637.2 | allocated_MB: 343.2 | reserved_MB: 25046.0 | inactive_split_MB: 260.8 +rank2 [2026-10-28 12:19:19,666] INFO : Step 1700/50000 rank2 | Train Loss: 2.9423 | Train Loss Unweighted: 1.7603 | Train Loss Components: Auxiliary: 0.5618 | Cardinality: 0.2834 | Classification: 0.3350 | Classification_binary: 0.4141 | HitGrouping: 0.3155 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4976 | Regression_eta: 0.0064 | Regression_pt: 0.5011 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1968 | Classification_binary: 0.4371 | Regression_cos_phi: 0.2371 | Regression_energy: 0.2136 | Regression_eta: 0.2052 | Regression_pt: 0.2191 | Regression_sin_phi: 0.3436 | LR: 9.97e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.2295s | Phase Times: max_pre_step_barrier: 0.0070 | max_batch_to_device: 0.0065 | max_forward: 0.8351 | max_loss: 0.2466 | max_backward: 1.7292 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0857 | Memory: rss_MB: 5499.7 | uss_MB: 4254.7 | child_rss_MB: 5622.9 | allocated_MB: 342.0 | reserved_MB: 25046.0 | inactive_split_MB: 198.9 +rank2 [2026-10-28 12:23:11,816] INFO : Step 1800/50000 rank2 | Train Loss: 2.7182 | Train Loss Unweighted: 1.6765 | Train Loss Components: Auxiliary: 0.5305 | Cardinality: 0.2556 | Classification: 0.3337 | Classification_binary: 0.3613 | HitGrouping: 0.3103 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4855 | Regression_eta: 0.0054 | Regression_pt: 0.4854 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1847 | Classification_binary: 0.3547 | Regression_cos_phi: 0.2228 | Regression_energy: 0.2101 | Regression_eta: 0.1313 | Regression_pt: 0.2180 | Regression_sin_phi: 0.2035 | LR: 9.97e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0942s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6573 | max_loss: 0.2402 | max_backward: 1.4111 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 1.0385 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0959 | Memory: rss_MB: 5501.3 | uss_MB: 4283.5 | child_rss_MB: 5636.1 | allocated_MB: 340.6 | reserved_MB: 25046.0 | inactive_split_MB: 181.6 +rank2 [2026-10-28 12:26:53,741] INFO : Step 1900/50000 rank2 | Train Loss: 2.9893 | Train Loss Unweighted: 1.8606 | Train Loss Components: Auxiliary: 0.5839 | Cardinality: 0.2900 | Classification: 0.3568 | Classification_binary: 0.4041 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5399 | Regression_eta: 0.0063 | Regression_pt: 0.5477 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2220 | Classification_binary: 0.4438 | Regression_cos_phi: 0.2324 | Regression_energy: 0.2361 | Regression_eta: 0.1778 | Regression_pt: 0.2500 | Regression_sin_phi: 0.2519 | LR: 9.97e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8704s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.6296 | max_loss: 0.1482 | max_backward: 1.2931 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 1.0537 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 5503.8 | uss_MB: 4286.6 | child_rss_MB: 5619.9 | allocated_MB: 344.8 | reserved_MB: 25048.0 | inactive_split_MB: 225.2 +rank2 [2026-10-28 12:30:53,516] INFO : Step 2000/50000 rank2 | Train Loss: 3.0007 | Train Loss Unweighted: 1.8040 | Train Loss Components: Auxiliary: 0.5673 | Cardinality: 0.3091 | Classification: 0.3519 | Classification_binary: 0.4482 | HitGrouping: 0.3137 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4922 | Regression_eta: 0.0062 | Regression_pt: 0.4996 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2159 | Classification_binary: 0.4339 | Regression_cos_phi: 0.2448 | Regression_energy: 0.2419 | Regression_eta: 0.1888 | Regression_pt: 0.2580 | Regression_sin_phi: 0.2418 | LR: 9.96e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1678s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6813 | max_loss: 0.2416 | max_backward: 1.5165 | max_optimizer_step: 0.1635 | max_tensorboard_logging: 0.9842 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5502.9 | uss_MB: 4287.2 | child_rss_MB: 5669.7 | allocated_MB: 346.7 | reserved_MB: 25696.0 | inactive_split_MB: 287.8 +rank2 [2026-10-28 12:30:54,414] INFO : Running validation on rank2 +rank2 [2026-10-28 12:30:54,414] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 12:30:54,414] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 12:31:46,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:31:46,631] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 12:32:13,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 12:32:14,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 12:32:15,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 12:32:16,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 12:32:26,895] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 12:32:27,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:32:27,705] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 12:32:43,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 12:32:44,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 12:32:45,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 12:32:46,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 12:32:56,712] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 12:32:57,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:32:57,629] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 12:33:13,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 12:33:14,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 12:33:15,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 12:33:16,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 12:33:26,060] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 12:37:37,593] INFO : Step 2100/50000 rank2 | Train Loss: 2.6299 | Train Loss Unweighted: 1.6429 | Train Loss Components: Auxiliary: 0.5172 | Cardinality: 0.2489 | Classification: 0.3119 | Classification_binary: 0.3591 | HitGrouping: 0.3047 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4785 | Regression_eta: 0.0053 | Regression_pt: 0.4834 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1944 | Classification_binary: 0.3380 | Regression_cos_phi: 0.2038 | Regression_energy: 0.2326 | Regression_eta: 0.1421 | Regression_pt: 0.2464 | Regression_sin_phi: 0.2177 | LR: 9.96e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0184s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0685 | max_forward: 0.7343 | max_loss: 0.1422 | max_backward: 1.3363 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.0015 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6447.9 | uss_MB: 5231.1 | child_rss_MB: 5629.3 | allocated_MB: 341.1 | reserved_MB: 22546.0 | inactive_split_MB: 269.0 +rank2 [2026-10-28 12:41:24,841] INFO : Step 2200/50000 rank2 | Train Loss: 2.8483 | Train Loss Unweighted: 1.7068 | Train Loss Components: Auxiliary: 0.5372 | Cardinality: 0.2547 | Classification: 0.3411 | Classification_binary: 0.3679 | HitGrouping: 0.3030 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4876 | Regression_eta: 0.0061 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2277 | Classification_binary: 0.3923 | Regression_cos_phi: 0.3425 | Regression_energy: 0.2104 | Regression_eta: 0.1696 | Regression_pt: 0.2207 | Regression_sin_phi: 0.2070 | LR: 9.96e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.4922s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6352 | max_loss: 0.1414 | max_backward: 1.3119 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.4703 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6448.9 | uss_MB: 5233.9 | child_rss_MB: 5636.2 | allocated_MB: 341.4 | reserved_MB: 22598.0 | inactive_split_MB: 313.4 +rank2 [2026-10-28 12:45:27,861] INFO : Step 2300/50000 rank2 | Train Loss: 3.1410 | Train Loss Unweighted: 1.8301 | Train Loss Components: Auxiliary: 0.5918 | Cardinality: 0.3583 | Classification: 0.3517 | Classification_binary: 0.4348 | HitGrouping: 0.3083 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5121 | Regression_eta: 0.0065 | Regression_pt: 0.5184 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1814 | Classification_binary: 0.4387 | Regression_cos_phi: 0.2845 | Regression_energy: 0.2398 | Regression_eta: 0.1679 | Regression_pt: 0.2526 | Regression_sin_phi: 0.2325 | LR: 9.95e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.8619s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.7258 | max_loss: 0.1520 | max_backward: 1.6288 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.6027 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0166 | Memory: rss_MB: 5633.9 | uss_MB: 4422.4 | child_rss_MB: 5644.1 | allocated_MB: 346.5 | reserved_MB: 23596.0 | inactive_split_MB: 273.5 +rank2 [2026-10-28 12:49:22,412] INFO : Step 2400/50000 rank2 | Train Loss: 2.8063 | Train Loss Unweighted: 1.7109 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.2683 | Classification: 0.3276 | Classification_binary: 0.3929 | HitGrouping: 0.3025 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4873 | Regression_eta: 0.0054 | Regression_pt: 0.4919 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1908 | Classification_binary: 0.3775 | Regression_cos_phi: 0.2032 | Regression_energy: 0.2473 | Regression_eta: 0.1383 | Regression_pt: 0.2572 | Regression_sin_phi: 0.2366 | LR: 9.95e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.5896s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0314 | max_forward: 0.7331 | max_loss: 0.1502 | max_backward: 1.4174 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.5071 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0157 | Memory: rss_MB: 5635.8 | uss_MB: 4426.4 | child_rss_MB: 5648.9 | allocated_MB: 344.4 | reserved_MB: 23596.0 | inactive_split_MB: 276.8 +rank2 [2026-10-28 12:53:10,912] INFO : Step 2500/50000 rank2 | Train Loss: 2.5300 | Train Loss Unweighted: 1.6034 | Train Loss Components: Auxiliary: 0.5018 | Cardinality: 0.2387 | Classification: 0.2872 | Classification_binary: 0.3313 | HitGrouping: 0.3000 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4857 | Regression_eta: 0.0042 | Regression_pt: 0.4904 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1396 | Classification_binary: 0.3198 | Regression_cos_phi: 0.1878 | Regression_energy: 0.2530 | Regression_eta: 0.1072 | Regression_pt: 0.2681 | Regression_sin_phi: 0.1812 | LR: 9.94e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.0260s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0177 | max_forward: 0.6104 | max_loss: 0.1394 | max_backward: 1.2397 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 1.1029 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5639.0 | uss_MB: 4422.1 | child_rss_MB: 5639.4 | allocated_MB: 342.5 | reserved_MB: 23596.0 | inactive_split_MB: 302.8 +rank2 [2026-10-28 12:56:55,919] INFO : Step 2600/50000 rank2 | Train Loss: 2.8992 | Train Loss Unweighted: 1.7075 | Train Loss Components: Auxiliary: 0.5471 | Cardinality: 0.3219 | Classification: 0.3420 | Classification_binary: 0.4005 | HitGrouping: 0.3027 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4737 | Regression_eta: 0.0057 | Regression_pt: 0.4796 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2067 | Classification_binary: 0.4035 | Regression_cos_phi: 0.2662 | Regression_energy: 0.2269 | Regression_eta: 0.1364 | Regression_pt: 0.2386 | Regression_sin_phi: 0.2142 | LR: 9.94e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7078s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8849 | max_loss: 0.1495 | max_backward: 1.9136 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1254 | Memory: rss_MB: 5638.3 | uss_MB: 4434.4 | child_rss_MB: 5644.3 | allocated_MB: 343.4 | reserved_MB: 23596.0 | inactive_split_MB: 290.6 +rank2 [2026-10-28 13:00:46,528] INFO : Step 2700/50000 rank2 | Train Loss: 2.8481 | Train Loss Unweighted: 1.7200 | Train Loss Components: Auxiliary: 0.5468 | Cardinality: 0.2918 | Classification: 0.3199 | Classification_binary: 0.4344 | HitGrouping: 0.3029 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4756 | Regression_eta: 0.0057 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1896 | Classification_binary: 0.4469 | Regression_cos_phi: 0.2098 | Regression_energy: 0.2146 | Regression_eta: 0.1585 | Regression_pt: 0.2238 | Regression_sin_phi: 0.2950 | LR: 9.94e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.3244s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0064 | max_forward: 0.7554 | max_loss: 0.2027 | max_backward: 1.6333 | max_optimizer_step: 0.1633 | max_tensorboard_logging: 0.9978 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5637.9 | uss_MB: 4434.7 | child_rss_MB: 5643.6 | allocated_MB: 343.8 | reserved_MB: 23598.0 | inactive_split_MB: 284.2 +rank2 [2026-10-28 13:04:40,295] INFO : Step 2800/50000 rank2 | Train Loss: 2.6381 | Train Loss Unweighted: 1.5955 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2387 | Classification: 0.2848 | Classification_binary: 0.3420 | HitGrouping: 0.2974 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4746 | Regression_eta: 0.0056 | Regression_pt: 0.4829 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1547 | Classification_binary: 0.3297 | Regression_cos_phi: 0.1365 | Regression_energy: 0.2123 | Regression_eta: 0.1117 | Regression_pt: 0.2232 | Regression_sin_phi: 0.2057 | LR: 9.93e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.7381s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6805 | max_loss: 0.2368 | max_backward: 1.1707 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5635.7 | uss_MB: 4432.9 | child_rss_MB: 5638.4 | allocated_MB: 341.2 | reserved_MB: 24554.0 | inactive_split_MB: 308.0 +rank2 [2026-10-28 13:08:36,459] INFO : Step 2900/50000 rank2 | Train Loss: 2.7831 | Train Loss Unweighted: 1.6528 | Train Loss Components: Auxiliary: 0.5323 | Cardinality: 0.3031 | Classification: 0.3069 | Classification_binary: 0.3681 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4798 | Regression_eta: 0.0056 | Regression_pt: 0.4869 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1379 | Classification_binary: 0.3451 | Regression_cos_phi: 0.2411 | Regression_energy: 0.2071 | Regression_eta: 0.1256 | Regression_pt: 0.2182 | Regression_sin_phi: 0.1845 | LR: 9.93e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7855s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6242 | max_loss: 0.1475 | max_backward: 1.3573 | max_optimizer_step: 0.2831 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5637.0 | uss_MB: 4433.8 | child_rss_MB: 5645.5 | allocated_MB: 341.4 | reserved_MB: 25570.0 | inactive_split_MB: 292.0 +rank2 [2026-10-28 13:12:15,920] INFO : Step 3000/50000 rank2 | Train Loss: 2.9336 | Train Loss Unweighted: 1.7260 | Train Loss Components: Auxiliary: 0.5466 | Cardinality: 0.2829 | Classification: 0.3524 | Classification_binary: 0.4205 | HitGrouping: 0.3085 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4673 | Regression_eta: 0.0061 | Regression_pt: 0.4733 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2071 | Classification_binary: 0.4466 | Regression_cos_phi: 0.2371 | Regression_energy: 0.2187 | Regression_eta: 0.1585 | Regression_pt: 0.2301 | Regression_sin_phi: 0.2376 | LR: 9.92e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1514s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0386 | max_forward: 0.8057 | max_loss: 0.1488 | max_backward: 1.9376 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5635.9 | uss_MB: 4440.1 | child_rss_MB: 5640.1 | allocated_MB: 346.5 | reserved_MB: 25570.0 | inactive_split_MB: 281.4 +rank2 [2026-10-28 13:12:16,814] INFO : Running validation on rank2 +rank2 [2026-10-28 13:12:16,814] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 13:12:16,815] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 13:13:09,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:13:10,420] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 13:13:30,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 13:13:31,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 13:13:32,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 13:13:33,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 13:13:43,280] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:13:43,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:13:44,017] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 13:13:59,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 13:14:00,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 13:14:01,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 13:14:02,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 13:14:12,988] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:14:13,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:14:14,219] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 13:14:29,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 13:14:30,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 13:14:30,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 13:14:31,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 13:14:41,654] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 13:18:52,945] INFO : Step 3100/50000 rank2 | Train Loss: 2.7478 | Train Loss Unweighted: 1.6734 | Train Loss Components: Auxiliary: 0.5216 | Cardinality: 0.2531 | Classification: 0.2981 | Classification_binary: 0.3781 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4881 | Regression_eta: 0.0057 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1866 | Classification_binary: 0.3720 | Regression_cos_phi: 0.1914 | Regression_energy: 0.2184 | Regression_eta: 0.1703 | Regression_pt: 0.2329 | Regression_sin_phi: 0.2744 | LR: 9.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5544s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.6511 | max_loss: 0.3988 | max_backward: 1.4306 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 1.2631 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5749.3 | uss_MB: 4552.2 | child_rss_MB: 5658.3 | allocated_MB: 342.5 | reserved_MB: 23120.0 | inactive_split_MB: 194.6 +rank2 [2026-10-28 13:22:38,494] INFO : Step 3200/50000 rank2 | Train Loss: 2.5254 | Train Loss Unweighted: 1.5710 | Train Loss Components: Auxiliary: 0.4888 | Cardinality: 0.2129 | Classification: 0.2845 | Classification_binary: 0.3271 | HitGrouping: 0.2964 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4770 | Regression_eta: 0.0050 | Regression_pt: 0.4723 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1463 | Classification_binary: 0.3176 | Regression_cos_phi: 0.2549 | Regression_energy: 0.2235 | Regression_eta: 0.1460 | Regression_pt: 0.2308 | Regression_sin_phi: 0.2116 | LR: 9.91e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5503s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6134 | max_loss: 0.2441 | max_backward: 1.2833 | max_optimizer_step: 0.2753 | max_tensorboard_logging: 0.5629 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5733.9 | uss_MB: 4558.8 | child_rss_MB: 5628.8 | allocated_MB: 343.6 | reserved_MB: 23340.0 | inactive_split_MB: 191.9 +rank2 [2026-10-28 13:26:30,566] INFO : Step 3300/50000 rank2 | Train Loss: 2.8037 | Train Loss Unweighted: 1.6635 | Train Loss Components: Auxiliary: 0.5345 | Cardinality: 0.2841 | Classification: 0.3310 | Classification_binary: 0.3857 | HitGrouping: 0.3002 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4633 | Regression_eta: 0.0058 | Regression_pt: 0.4718 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1879 | Classification_binary: 0.4063 | Regression_cos_phi: 0.2418 | Regression_energy: 0.2242 | Regression_eta: 0.1568 | Regression_pt: 0.2352 | Regression_sin_phi: 0.2289 | LR: 9.90e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2962s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9147 | max_loss: 0.1535 | max_backward: 1.9100 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.6631 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5735.1 | uss_MB: 4562.9 | child_rss_MB: 5661.7 | allocated_MB: 344.0 | reserved_MB: 23618.0 | inactive_split_MB: 250.0 +rank2 [2026-10-28 13:30:24,443] INFO : Step 3400/50000 rank2 | Train Loss: 2.8855 | Train Loss Unweighted: 1.7067 | Train Loss Components: Auxiliary: 0.5454 | Cardinality: 0.2859 | Classification: 0.3252 | Classification_binary: 0.4155 | HitGrouping: 0.2965 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4731 | Regression_eta: 0.0067 | Regression_pt: 0.4802 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2145 | Classification_binary: 0.4319 | Regression_cos_phi: 0.2707 | Regression_energy: 0.2224 | Regression_eta: 0.2013 | Regression_pt: 0.2329 | Regression_sin_phi: 0.2939 | LR: 9.90e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.5647s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0438 | max_forward: 0.7271 | max_loss: 0.1506 | max_backward: 1.3643 | max_optimizer_step: 0.1716 | max_tensorboard_logging: 0.4827 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5729.2 | uss_MB: 4562.1 | child_rss_MB: 5658.8 | allocated_MB: 346.0 | reserved_MB: 23618.0 | inactive_split_MB: 235.6 +rank2 [2026-10-28 13:34:14,209] INFO : Step 3500/50000 rank2 | Train Loss: 2.4422 | Train Loss Unweighted: 1.4809 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.2120 | Classification: 0.2615 | Classification_binary: 0.3296 | HitGrouping: 0.2898 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4369 | Regression_eta: 0.0054 | Regression_pt: 0.4427 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1507 | Classification_binary: 0.3026 | Regression_cos_phi: 0.1897 | Regression_energy: 0.2345 | Regression_eta: 0.1203 | Regression_pt: 0.2482 | Regression_sin_phi: 0.1378 | LR: 9.89e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7304s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0058 | max_forward: 0.7056 | max_loss: 0.1971 | max_backward: 1.3359 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.6307 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5720.3 | uss_MB: 4563.4 | child_rss_MB: 5630.0 | allocated_MB: 339.0 | reserved_MB: 24610.0 | inactive_split_MB: 280.8 +rank2 [2026-10-28 13:38:02,721] INFO : Step 3600/50000 rank2 | Train Loss: 2.5780 | Train Loss Unweighted: 1.5250 | Train Loss Components: Auxiliary: 0.4892 | Cardinality: 0.2463 | Classification: 0.3079 | Classification_binary: 0.3446 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4295 | Regression_eta: 0.0052 | Regression_pt: 0.4324 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1970 | Classification_binary: 0.3592 | Regression_cos_phi: 0.2392 | Regression_energy: 0.2011 | Regression_eta: 0.1460 | Regression_pt: 0.2085 | Regression_sin_phi: 0.2862 | LR: 9.89e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.4638s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1453 | max_backward: 1.4106 | max_optimizer_step: 0.1681 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0126 | Memory: rss_MB: 5726.7 | uss_MB: 4567.6 | child_rss_MB: 5631.8 | allocated_MB: 343.9 | reserved_MB: 24610.0 | inactive_split_MB: 244.5 +rank2 [2026-10-28 13:41:50,385] INFO : Step 3700/50000 rank2 | Train Loss: 2.9213 | Train Loss Unweighted: 1.6952 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.3153 | Classification: 0.3418 | Classification_binary: 0.4227 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4570 | Regression_eta: 0.0063 | Regression_pt: 0.4612 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1872 | Classification_binary: 0.4213 | Regression_cos_phi: 0.2205 | Regression_energy: 0.2151 | Regression_eta: 0.1346 | Regression_pt: 0.2250 | Regression_sin_phi: 0.2620 | LR: 9.88e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2206s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7710 | max_loss: 0.1507 | max_backward: 1.6430 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.8888 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0145 | Memory: rss_MB: 5726.2 | uss_MB: 4569.4 | child_rss_MB: 5661.5 | allocated_MB: 345.4 | reserved_MB: 24610.0 | inactive_split_MB: 268.9 +rank2 [2026-10-28 13:45:46,299] INFO : Step 3800/50000 rank2 | Train Loss: 2.6063 | Train Loss Unweighted: 1.5289 | Train Loss Components: Auxiliary: 0.4933 | Cardinality: 0.2661 | Classification: 0.2839 | Classification_binary: 0.3652 | HitGrouping: 0.2879 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4319 | Regression_eta: 0.0058 | Regression_pt: 0.4369 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1524 | Classification_binary: 0.3582 | Regression_cos_phi: 0.2089 | Regression_energy: 0.2146 | Regression_eta: 0.1395 | Regression_pt: 0.2242 | Regression_sin_phi: 0.2037 | LR: 9.87e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1717s | Phase Times: max_pre_step_barrier: 0.1134 | max_batch_to_device: 0.0063 | max_forward: 0.6784 | max_loss: 0.1451 | max_backward: 1.4269 | max_optimizer_step: 0.2795 | max_tensorboard_logging: 0.9060 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5726.3 | uss_MB: 4569.5 | child_rss_MB: 5664.0 | allocated_MB: 343.8 | reserved_MB: 24610.0 | inactive_split_MB: 202.2 +rank2 [2026-10-28 13:49:32,923] INFO : Step 3900/50000 rank2 | Train Loss: 2.2761 | Train Loss Unweighted: 1.4165 | Train Loss Components: Auxiliary: 0.4517 | Cardinality: 0.1938 | Classification: 0.2553 | Classification_binary: 0.2930 | HitGrouping: 0.2882 | Regression_cos_phi: 0.0020 | Regression_energy: 0.4264 | Regression_eta: 0.0045 | Regression_pt: 0.4331 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1737 | Classification_binary: 0.3116 | Regression_cos_phi: 0.1589 | Regression_energy: 0.1908 | Regression_eta: 0.1366 | Regression_pt: 0.1984 | Regression_sin_phi: 0.2360 | LR: 9.87e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8909s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6197 | max_loss: 0.1501 | max_backward: 1.2810 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.0075 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 5727.5 | uss_MB: 3764.5 | child_rss_MB: 5661.9 | allocated_MB: 343.5 | reserved_MB: 24610.0 | inactive_split_MB: 286.6 +rank2 [2026-10-28 13:53:23,686] INFO : Step 4000/50000 rank2 | Train Loss: 2.6760 | Train Loss Unweighted: 1.5903 | Train Loss Components: Auxiliary: 0.5112 | Cardinality: 0.2445 | Classification: 0.3070 | Classification_binary: 0.3812 | HitGrouping: 0.2906 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4413 | Regression_eta: 0.0062 | Regression_pt: 0.4492 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1930 | Classification_binary: 0.4133 | Regression_cos_phi: 0.2267 | Regression_energy: 0.2051 | Regression_eta: 0.2081 | Regression_pt: 0.2139 | Regression_sin_phi: 0.2760 | LR: 9.86e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.0139s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0492 | max_forward: 0.7834 | max_loss: 0.1586 | max_backward: 1.6903 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.5589 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5731.5 | uss_MB: 4577.2 | child_rss_MB: 5661.6 | allocated_MB: 345.5 | reserved_MB: 24610.0 | inactive_split_MB: 252.6 +rank2 [2026-10-28 13:53:24,557] INFO : Running validation on rank2 +rank2 [2026-10-28 13:53:24,557] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 13:53:24,558] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 13:54:13,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:54:14,418] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 13:54:39,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 13:54:41,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 13:54:42,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 13:54:43,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 13:54:53,044] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:54:53,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:54:53,949] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 13:55:09,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 13:55:10,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 13:55:11,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 13:55:12,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 13:55:22,244] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 13:55:23,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:55:23,752] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 13:55:38,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 13:55:39,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 13:55:40,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 13:55:40,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 13:55:50,818] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 13:59:55,049] INFO : Step 4100/50000 rank2 | Train Loss: 2.6923 | Train Loss Unweighted: 1.5640 | Train Loss Components: Auxiliary: 0.5058 | Cardinality: 0.2671 | Classification: 0.2981 | Classification_binary: 0.4073 | HitGrouping: 0.2911 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4210 | Regression_eta: 0.0055 | Regression_pt: 0.4265 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1826 | Classification_binary: 0.3735 | Regression_cos_phi: 0.2329 | Regression_energy: 0.2068 | Regression_eta: 0.1452 | Regression_pt: 0.2153 | Regression_sin_phi: 0.2312 | LR: 9.85e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.1508s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0141 | max_forward: 0.7422 | max_loss: 0.3919 | max_backward: 1.3629 | max_optimizer_step: 0.1890 | max_tensorboard_logging: 0.9903 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6680.3 | uss_MB: 5525.0 | child_rss_MB: 5670.9 | allocated_MB: 345.2 | reserved_MB: 23430.0 | inactive_split_MB: 294.4 +rank2 [2026-10-28 14:03:47,900] INFO : Step 4200/50000 rank2 | Train Loss: 2.3415 | Train Loss Unweighted: 1.4424 | Train Loss Components: Auxiliary: 0.4542 | Cardinality: 0.2058 | Classification: 0.2540 | Classification_binary: 0.3143 | HitGrouping: 0.2846 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4310 | Regression_eta: 0.0050 | Regression_pt: 0.4337 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1608 | Classification_binary: 0.2763 | Regression_cos_phi: 0.2248 | Regression_energy: 0.2182 | Regression_eta: 0.1066 | Regression_pt: 0.2239 | Regression_sin_phi: 0.1451 | LR: 9.84e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.7412s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0168 | max_forward: 0.8335 | max_loss: 0.2425 | max_backward: 1.5586 | max_optimizer_step: 0.1744 | max_tensorboard_logging: 0.4781 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5904.9 | uss_MB: 4751.1 | child_rss_MB: 5672.4 | allocated_MB: 341.2 | reserved_MB: 24406.0 | inactive_split_MB: 273.9 +rank2 [2026-10-28 14:07:35,176] INFO : Step 4300/50000 rank2 | Train Loss: 2.5102 | Train Loss Unweighted: 1.4857 | Train Loss Components: Auxiliary: 0.4757 | Cardinality: 0.2309 | Classification: 0.2786 | Classification_binary: 0.3543 | HitGrouping: 0.2938 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4177 | Regression_eta: 0.0054 | Regression_pt: 0.4246 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1712 | Classification_binary: 0.3640 | Regression_cos_phi: 0.2226 | Regression_energy: 0.1962 | Regression_eta: 0.1509 | Regression_pt: 0.2076 | Regression_sin_phi: 0.2037 | LR: 9.84e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5995s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6520 | max_loss: 0.2453 | max_backward: 1.4084 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.5540 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5910.3 | uss_MB: 4757.7 | child_rss_MB: 5668.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 239.6 +rank2 [2026-10-28 14:11:23,917] INFO : Step 4400/50000 rank2 | Train Loss: 2.8258 | Train Loss Unweighted: 1.5691 | Train Loss Components: Auxiliary: 0.5223 | Cardinality: 0.3313 | Classification: 0.2877 | Classification_binary: 0.4120 | HitGrouping: 0.2902 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4255 | Regression_eta: 0.0063 | Regression_pt: 0.4316 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1521 | Classification_binary: 0.4158 | Regression_cos_phi: 0.2565 | Regression_energy: 0.2089 | Regression_eta: 0.1570 | Regression_pt: 0.2178 | Regression_sin_phi: 0.2462 | LR: 9.83e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.0427s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.7635 | max_loss: 0.2580 | max_backward: 1.7456 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.5221 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 5909.1 | uss_MB: 4757.0 | child_rss_MB: 5674.0 | allocated_MB: 344.6 | reserved_MB: 24406.0 | inactive_split_MB: 261.0 +rank2 [2026-10-28 14:15:17,054] INFO : Step 4500/50000 rank2 | Train Loss: 2.5622 | Train Loss Unweighted: 1.4664 | Train Loss Components: Auxiliary: 0.4942 | Cardinality: 0.3132 | Classification: 0.2685 | Classification_binary: 0.3695 | HitGrouping: 0.2876 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4070 | Regression_eta: 0.0048 | Regression_pt: 0.4117 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1381 | Classification_binary: 0.3583 | Regression_cos_phi: 0.1804 | Regression_energy: 0.1923 | Regression_eta: 0.1146 | Regression_pt: 0.2028 | Regression_sin_phi: 0.1980 | LR: 9.82e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.2155s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0462 | max_forward: 1.1330 | max_loss: 0.3526 | max_backward: 2.1744 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5571 | max_tensorboard_flush: 0.0016 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 5912.8 | uss_MB: 4736.1 | child_rss_MB: 5674.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 288.8 +rank2 [2026-10-28 14:19:09,440] INFO : Step 4600/50000 rank2 | Train Loss: 2.2516 | Train Loss Unweighted: 1.3548 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2151 | Classification: 0.2428 | Classification_binary: 0.2886 | HitGrouping: 0.2811 | Regression_cos_phi: 0.0019 | Regression_energy: 0.4059 | Regression_eta: 0.0042 | Regression_pt: 0.4088 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1666 | Classification_binary: 0.2994 | Regression_cos_phi: 0.1700 | Regression_energy: 0.1829 | Regression_eta: 0.1060 | Regression_pt: 0.1876 | Regression_sin_phi: 0.1734 | LR: 9.81e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6995s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0445 | max_forward: 0.8897 | max_loss: 0.2214 | max_backward: 1.8748 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5913.2 | uss_MB: 4741.7 | child_rss_MB: 5653.3 | allocated_MB: 341.6 | reserved_MB: 25392.0 | inactive_split_MB: 298.4 +rank2 [2026-10-28 14:23:11,224] INFO : Step 4700/50000 rank2 | Train Loss: 2.5991 | Train Loss Unweighted: 1.5142 | Train Loss Components: Auxiliary: 0.4914 | Cardinality: 0.2679 | Classification: 0.2997 | Classification_binary: 0.3514 | HitGrouping: 0.2867 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4228 | Regression_eta: 0.0061 | Regression_pt: 0.4290 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1876 | Classification_binary: 0.3930 | Regression_cos_phi: 0.2048 | Regression_energy: 0.2191 | Regression_eta: 0.2338 | Regression_pt: 0.2307 | Regression_sin_phi: 0.2148 | LR: 9.81e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8894s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7362 | max_loss: 0.2547 | max_backward: 1.8331 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5918.9 | uss_MB: 4764.8 | child_rss_MB: 5675.1 | allocated_MB: 343.1 | reserved_MB: 26384.0 | inactive_split_MB: 292.4 +rank2 [2026-10-28 14:27:13,057] INFO : Step 4800/50000 rank2 | Train Loss: 2.7077 | Train Loss Unweighted: 1.5200 | Train Loss Components: Auxiliary: 0.5118 | Cardinality: 0.3360 | Classification: 0.2606 | Classification_binary: 0.3918 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4264 | Regression_eta: 0.0053 | Regression_pt: 0.4305 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1818 | Classification_binary: 0.4248 | Regression_cos_phi: 0.2795 | Regression_energy: 0.2171 | Regression_eta: 0.1696 | Regression_pt: 0.2286 | Regression_sin_phi: 0.2969 | LR: 9.80e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.2494s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7154 | max_loss: 0.1552 | max_backward: 1.4398 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.8979 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0854 | Memory: rss_MB: 5917.3 | uss_MB: 4226.4 | child_rss_MB: 5678.8 | allocated_MB: 346.1 | reserved_MB: 26384.0 | inactive_split_MB: 284.0 +rank2 [2026-10-28 14:31:09,414] INFO : Step 4900/50000 rank2 | Train Loss: 2.3682 | Train Loss Unweighted: 1.4242 | Train Loss Components: Auxiliary: 0.4616 | Cardinality: 0.2176 | Classification: 0.2570 | Classification_binary: 0.3149 | HitGrouping: 0.2900 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4191 | Regression_eta: 0.0047 | Regression_pt: 0.4239 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1432 | Classification_binary: 0.2775 | Regression_cos_phi: 0.1753 | Regression_energy: 0.1924 | Regression_eta: 0.1211 | Regression_pt: 0.1984 | Regression_sin_phi: 0.1118 | LR: 9.79e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.4646s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0516 | max_forward: 0.6608 | max_loss: 0.1430 | max_backward: 1.2657 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5916.2 | uss_MB: 4765.4 | child_rss_MB: 5677.6 | allocated_MB: 343.8 | reserved_MB: 26384.0 | inactive_split_MB: 201.9 +rank2 [2026-10-28 14:35:19,045] INFO : Step 5000/50000 rank2 | Train Loss: 2.4060 | Train Loss Unweighted: 1.4130 | Train Loss Components: Auxiliary: 0.4756 | Cardinality: 0.2573 | Classification: 0.2748 | Classification_binary: 0.3198 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4020 | Regression_eta: 0.0043 | Regression_pt: 0.4074 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1911 | Classification_binary: 0.3274 | Regression_cos_phi: 0.2284 | Regression_energy: 0.2091 | Regression_eta: 0.1499 | Regression_pt: 0.2208 | Regression_sin_phi: 0.2468 | LR: 9.78e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8095s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6153 | max_loss: 0.1441 | max_backward: 1.2119 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.9741 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5914.8 | uss_MB: 4767.1 | child_rss_MB: 5669.5 | allocated_MB: 342.5 | reserved_MB: 27388.0 | inactive_split_MB: 218.4 +rank2 [2026-10-28 14:35:19,963] INFO : Running validation on rank2 +rank2 [2026-10-28 14:35:19,963] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 14:35:19,963] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 14:36:10,798] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:36:11,319] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 14:36:36,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 14:36:37,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 14:36:38,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 14:36:39,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 14:36:49,683] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 14:36:50,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:36:50,780] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 14:37:07,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 14:37:08,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 14:37:09,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 14:37:10,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 14:37:20,092] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 14:37:20,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:37:20,582] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 14:37:36,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 14:37:36,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 14:37:37,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 14:37:38,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 14:37:48,023] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 14:42:02,659] INFO : Step 5100/50000 rank2 | Train Loss: 2.7059 | Train Loss Unweighted: 1.5631 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2507 | Classification: 0.3068 | Classification_binary: 0.4110 | HitGrouping: 0.2884 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4148 | Regression_eta: 0.0060 | Regression_pt: 0.4186 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1796 | Classification_binary: 0.4130 | Regression_cos_phi: 0.2301 | Regression_energy: 0.1943 | Regression_eta: 0.1829 | Regression_pt: 0.2055 | Regression_sin_phi: 0.2032 | LR: 9.77e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4427s | Phase Times: max_pre_step_barrier: 0.1071 | max_batch_to_device: 0.0070 | max_forward: 0.7192 | max_loss: 0.2509 | max_backward: 1.6066 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.8859 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7032.2 | uss_MB: 5882.9 | child_rss_MB: 5662.0 | allocated_MB: 343.2 | reserved_MB: 23542.0 | inactive_split_MB: 236.2 +rank2 [2026-10-28 14:45:56,261] INFO : Step 5200/50000 rank2 | Train Loss: 2.4117 | Train Loss Unweighted: 1.4393 | Train Loss Components: Auxiliary: 0.4667 | Cardinality: 0.2215 | Classification: 0.2426 | Classification_binary: 0.3524 | HitGrouping: 0.2783 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4149 | Regression_eta: 0.0050 | Regression_pt: 0.4196 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1403 | Classification_binary: 0.3586 | Regression_cos_phi: 0.2745 | Regression_energy: 0.1614 | Regression_eta: 0.1443 | Regression_pt: 0.1673 | Regression_sin_phi: 0.1689 | LR: 9.76e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9031s | Phase Times: max_pre_step_barrier: 0.1151 | max_batch_to_device: 0.0079 | max_forward: 0.8909 | max_loss: 0.1450 | max_backward: 1.3654 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.5791 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 6020.7 | uss_MB: 4875.4 | child_rss_MB: 5666.4 | allocated_MB: 346.1 | reserved_MB: 23690.0 | inactive_split_MB: 197.9 +rank2 [2026-10-28 14:50:03,850] INFO : Step 5300/50000 rank2 | Train Loss: 2.2417 | Train Loss Unweighted: 1.3238 | Train Loss Components: Auxiliary: 0.4353 | Cardinality: 0.2226 | Classification: 0.2387 | Classification_binary: 0.3027 | HitGrouping: 0.2808 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3844 | Regression_eta: 0.0041 | Regression_pt: 0.3895 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1522 | Classification_binary: 0.2956 | Regression_cos_phi: 0.1825 | Regression_energy: 0.1805 | Regression_eta: 0.1368 | Regression_pt: 0.1942 | Regression_sin_phi: 0.1889 | LR: 9.75e-04 | DataLoad Time: 0.0235s | Model Forward Time: 4.1970s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0155 | max_forward: 1.3549 | max_loss: 0.4272 | max_backward: 1.7517 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 6024.3 | uss_MB: 4878.1 | child_rss_MB: 5664.8 | allocated_MB: 341.7 | reserved_MB: 25496.0 | inactive_split_MB: 195.4 +rank2 [2026-10-28 14:54:26,617] INFO : Step 5400/50000 rank2 | Train Loss: 2.5857 | Train Loss Unweighted: 1.4820 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2715 | Classification: 0.2898 | Classification_binary: 0.3613 | HitGrouping: 0.2829 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4076 | Regression_eta: 0.0053 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1702 | Classification_binary: 0.3488 | Regression_cos_phi: 0.2346 | Regression_energy: 0.1972 | Regression_eta: 0.1470 | Regression_pt: 0.2071 | Regression_sin_phi: 0.2431 | LR: 9.74e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.8620s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7472 | max_loss: 0.1664 | max_backward: 1.6216 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4956 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6028.8 | uss_MB: 4880.2 | child_rss_MB: 5668.3 | allocated_MB: 345.8 | reserved_MB: 25496.0 | inactive_split_MB: 226.8 +rank2 [2026-10-28 14:58:29,775] INFO : Step 5500/50000 rank2 | Train Loss: 2.4383 | Train Loss Unweighted: 1.4361 | Train Loss Components: Auxiliary: 0.4636 | Cardinality: 0.2531 | Classification: 0.2547 | Classification_binary: 0.3846 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3900 | Regression_eta: 0.0045 | Regression_pt: 0.3974 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1418 | Classification_binary: 0.3773 | Regression_cos_phi: 0.1887 | Regression_energy: 0.1834 | Regression_eta: 0.1248 | Regression_pt: 0.1906 | Regression_sin_phi: 0.2196 | LR: 9.73e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3513s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7547 | max_loss: 0.2915 | max_backward: 1.8143 | max_optimizer_step: 0.2819 | max_tensorboard_logging: 0.5958 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6029.1 | uss_MB: 4882.8 | child_rss_MB: 5667.9 | allocated_MB: 342.5 | reserved_MB: 26496.0 | inactive_split_MB: 471.5 +rank2 [2026-10-28 15:02:37,793] INFO : Step 5600/50000 rank2 | Train Loss: 2.2622 | Train Loss Unweighted: 1.3177 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.1990 | Classification: 0.2257 | Classification_binary: 0.3268 | HitGrouping: 0.2800 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3758 | Regression_eta: 0.0051 | Regression_pt: 0.3799 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1437 | Classification_binary: 0.3273 | Regression_cos_phi: 0.1651 | Regression_energy: 0.1794 | Regression_eta: 0.2128 | Regression_pt: 0.1892 | Regression_sin_phi: 0.1659 | LR: 9.72e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1696s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0347 | max_forward: 1.1910 | max_loss: 0.2555 | max_backward: 1.5431 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 6024.4 | uss_MB: 4885.9 | child_rss_MB: 5669.8 | allocated_MB: 340.8 | reserved_MB: 26496.0 | inactive_split_MB: 191.2 +rank2 [2026-10-28 15:06:24,089] INFO : Step 5700/50000 rank2 | Train Loss: 2.4588 | Train Loss Unweighted: 1.4026 | Train Loss Components: Auxiliary: 0.4666 | Cardinality: 0.2348 | Classification: 0.2768 | Classification_binary: 0.3365 | HitGrouping: 0.2916 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3864 | Regression_eta: 0.0055 | Regression_pt: 0.3923 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1437 | Classification_binary: 0.3180 | Regression_cos_phi: 0.1572 | Regression_energy: 0.1800 | Regression_eta: 0.1255 | Regression_pt: 0.1924 | Regression_sin_phi: 0.2101 | LR: 9.71e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.5065s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6547 | max_loss: 0.1874 | max_backward: 1.2748 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6027.1 | uss_MB: 4889.2 | child_rss_MB: 5674.0 | allocated_MB: 344.4 | reserved_MB: 26496.0 | inactive_split_MB: 224.5 +rank2 [2026-10-28 15:10:14,851] INFO : Step 5800/50000 rank2 | Train Loss: 2.4947 | Train Loss Unweighted: 1.4616 | Train Loss Components: Auxiliary: 0.4752 | Cardinality: 0.2715 | Classification: 0.2623 | Classification_binary: 0.3729 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4032 | Regression_eta: 0.0047 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1585 | Classification_binary: 0.3604 | Regression_cos_phi: 0.2346 | Regression_energy: 0.2047 | Regression_eta: 0.1376 | Regression_pt: 0.2197 | Regression_sin_phi: 0.2013 | LR: 9.70e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7374s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.9768 | max_loss: 0.2512 | max_backward: 2.0573 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.6978 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6026.0 | uss_MB: 4048.8 | child_rss_MB: 5650.8 | allocated_MB: 343.4 | reserved_MB: 27494.0 | inactive_split_MB: 198.3 +rank2 [2026-10-28 15:14:12,311] INFO : Step 5900/50000 rank2 | Train Loss: 2.3800 | Train Loss Unweighted: 1.3838 | Train Loss Components: Auxiliary: 0.4549 | Cardinality: 0.2203 | Classification: 0.2438 | Classification_binary: 0.3435 | HitGrouping: 0.2757 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3900 | Regression_eta: 0.0050 | Regression_pt: 0.3965 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1836 | Classification_binary: 0.3891 | Regression_cos_phi: 0.2194 | Regression_energy: 0.2228 | Regression_eta: 0.1779 | Regression_pt: 0.2293 | Regression_sin_phi: 0.2366 | LR: 9.69e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7210s | Phase Times: max_pre_step_barrier: 0.0158 | max_batch_to_device: 0.0069 | max_forward: 1.0355 | max_loss: 0.2311 | max_backward: 2.2749 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5131 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 6029.8 | uss_MB: 3868.6 | child_rss_MB: 5658.5 | allocated_MB: 342.1 | reserved_MB: 27494.0 | inactive_split_MB: 281.2 +rank2 [2026-10-28 15:18:13,895] INFO : Step 6000/50000 rank2 | Train Loss: 2.1372 | Train Loss Unweighted: 1.3015 | Train Loss Components: Auxiliary: 0.4179 | Cardinality: 0.1916 | Classification: 0.2091 | Classification_binary: 0.2848 | HitGrouping: 0.2746 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3966 | Regression_eta: 0.0041 | Regression_pt: 0.4029 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1381 | Classification_binary: 0.2732 | Regression_cos_phi: 0.1392 | Regression_energy: 0.1934 | Regression_eta: 0.1243 | Regression_pt: 0.2048 | Regression_sin_phi: 0.1644 | LR: 9.68e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6787s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6457 | max_loss: 0.2157 | max_backward: 1.4984 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.5590 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6033.8 | uss_MB: 4891.5 | child_rss_MB: 5658.2 | allocated_MB: 343.1 | reserved_MB: 27494.0 | inactive_split_MB: 260.7 +rank2 [2026-10-28 15:18:14,995] INFO : Running validation on rank2 +rank2 [2026-10-28 15:18:14,995] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 15:18:14,995] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 15:19:17,312] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:19:17,849] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 15:19:43,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 15:19:44,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 15:19:45,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 15:19:46,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 15:19:56,990] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 15:19:57,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:19:58,135] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 15:20:14,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 15:20:15,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 15:20:16,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 15:20:17,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 15:20:27,181] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 15:20:27,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:20:28,076] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 15:20:43,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 15:20:44,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 15:20:45,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 15:20:45,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 15:20:55,790] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 15:25:29,830] INFO : Step 6100/50000 rank2 | Train Loss: 2.4040 | Train Loss Unweighted: 1.3907 | Train Loss Components: Auxiliary: 0.4661 | Cardinality: 0.2610 | Classification: 0.2427 | Classification_binary: 0.3401 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3968 | Regression_eta: 0.0047 | Regression_pt: 0.4016 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1390 | Classification_binary: 0.3319 | Regression_cos_phi: 0.2463 | Regression_energy: 0.2099 | Regression_eta: 0.1261 | Regression_pt: 0.2107 | Regression_sin_phi: 0.1717 | LR: 9.67e-04 | DataLoad Time: 0.0209s | Model Forward Time: 6.3867s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0073 | max_forward: 1.0567 | max_loss: 0.2452 | max_backward: 2.4106 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.9836 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 7214.6 | uss_MB: 5292.3 | child_rss_MB: 5654.8 | allocated_MB: 344.4 | reserved_MB: 23648.0 | inactive_split_MB: 194.8 +rank2 [2026-10-28 15:29:38,612] INFO : Step 6200/50000 rank2 | Train Loss: 2.4942 | Train Loss Unweighted: 1.4287 | Train Loss Components: Auxiliary: 0.4744 | Cardinality: 0.2537 | Classification: 0.2580 | Classification_binary: 0.3817 | HitGrouping: 0.2748 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3867 | Regression_eta: 0.0050 | Regression_pt: 0.3920 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1520 | Classification_binary: 0.3984 | Regression_cos_phi: 0.2243 | Regression_energy: 0.1737 | Regression_eta: 0.1291 | Regression_pt: 0.1799 | Regression_sin_phi: 0.2057 | LR: 9.66e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.0655s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7838 | max_loss: 0.1532 | max_backward: 1.6314 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6093.8 | uss_MB: 4537.3 | child_rss_MB: 5656.7 | allocated_MB: 343.8 | reserved_MB: 23830.0 | inactive_split_MB: 215.7 +rank2 [2026-10-28 15:34:14,250] INFO : Step 6300/50000 rank2 | Train Loss: 2.2204 | Train Loss Unweighted: 1.2763 | Train Loss Components: Auxiliary: 0.4285 | Cardinality: 0.2506 | Classification: 0.2258 | Classification_binary: 0.2983 | HitGrouping: 0.2710 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3704 | Regression_eta: 0.0045 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1094 | Classification_binary: 0.2853 | Regression_cos_phi: 0.1697 | Regression_energy: 0.1814 | Regression_eta: 0.1527 | Regression_pt: 0.1859 | Regression_sin_phi: 0.1279 | LR: 9.65e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6717s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.5854 | max_loss: 0.1923 | max_backward: 1.0848 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.8586 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6096.2 | uss_MB: 4957.2 | child_rss_MB: 5651.4 | allocated_MB: 339.7 | reserved_MB: 24826.0 | inactive_split_MB: 251.6 +rank2 [2026-10-28 15:38:39,608] INFO : Step 6400/50000 rank2 | Train Loss: 2.2882 | Train Loss Unweighted: 1.3014 | Train Loss Components: Auxiliary: 0.4348 | Cardinality: 0.2271 | Classification: 0.2431 | Classification_binary: 0.3058 | HitGrouping: 0.2702 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3679 | Regression_eta: 0.0046 | Regression_pt: 0.3751 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1572 | Classification_binary: 0.3440 | Regression_cos_phi: 0.2239 | Regression_energy: 0.1962 | Regression_eta: 0.1552 | Regression_pt: 0.2085 | Regression_sin_phi: 0.2449 | LR: 9.64e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.8565s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7489 | max_loss: 0.3608 | max_backward: 2.5327 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6095.6 | uss_MB: 4594.4 | child_rss_MB: 5649.8 | allocated_MB: 345.8 | reserved_MB: 24826.0 | inactive_split_MB: 253.0 +rank2 [2026-10-28 15:42:45,602] INFO : Step 6500/50000 rank2 | Train Loss: 2.4626 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.3022 | Classification: 0.2393 | Classification_binary: 0.3805 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3783 | Regression_eta: 0.0049 | Regression_pt: 0.3831 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1420 | Classification_binary: 0.3957 | Regression_cos_phi: 0.1892 | Regression_energy: 0.1827 | Regression_eta: 0.1319 | Regression_pt: 0.1923 | Regression_sin_phi: 0.2125 | LR: 9.63e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1905s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7600 | max_loss: 0.1555 | max_backward: 1.3910 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6094.6 | uss_MB: 4960.0 | child_rss_MB: 5656.1 | allocated_MB: 344.4 | reserved_MB: 24826.0 | inactive_split_MB: 269.0 +rank2 [2026-10-28 15:46:47,732] INFO : Step 6600/50000 rank2 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3617 | Train Loss Components: Auxiliary: 0.4493 | Cardinality: 0.2405 | Classification: 0.2289 | Classification_binary: 0.3388 | HitGrouping: 0.2650 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3914 | Regression_eta: 0.0045 | Regression_pt: 0.3935 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1316 | Classification_binary: 0.3074 | Regression_cos_phi: 0.1384 | Regression_energy: 0.1681 | Regression_eta: 0.1205 | Regression_pt: 0.1747 | Regression_sin_phi: 0.1361 | LR: 9.62e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7760s | Phase Times: max_pre_step_barrier: 0.1106 | max_batch_to_device: 0.0061 | max_forward: 0.7587 | max_loss: 0.2869 | max_backward: 2.2641 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1321 | Memory: rss_MB: 6092.2 | uss_MB: 4956.2 | child_rss_MB: 5651.6 | allocated_MB: 343.8 | reserved_MB: 25846.0 | inactive_split_MB: 267.9 +rank2 [2026-10-28 15:51:24,435] INFO : Step 6700/50000 rank2 | Train Loss: 2.1225 | Train Loss Unweighted: 1.2385 | Train Loss Components: Auxiliary: 0.4092 | Cardinality: 0.2100 | Classification: 0.2079 | Classification_binary: 0.2916 | HitGrouping: 0.2751 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3623 | Regression_eta: 0.0041 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1352 | Classification_binary: 0.2687 | Regression_cos_phi: 0.1242 | Regression_energy: 0.1543 | Regression_eta: 0.0996 | Regression_pt: 0.1632 | Regression_sin_phi: 0.1308 | LR: 9.61e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9904s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.6452 | max_loss: 0.1453 | max_backward: 1.2984 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.0269 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6098.1 | uss_MB: 4965.1 | child_rss_MB: 5657.8 | allocated_MB: 345.0 | reserved_MB: 25846.0 | inactive_split_MB: 258.8 +rank2 [2026-10-28 15:55:32,974] INFO : Step 6800/50000 rank2 | Train Loss: 2.4601 | Train Loss Unweighted: 1.3772 | Train Loss Components: Auxiliary: 0.4589 | Cardinality: 0.2427 | Classification: 0.2631 | Classification_binary: 0.3490 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3747 | Regression_eta: 0.0054 | Regression_pt: 0.3793 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1505 | Classification_binary: 0.3585 | Regression_cos_phi: 0.2909 | Regression_energy: 0.1874 | Regression_eta: 0.1538 | Regression_pt: 0.1975 | Regression_sin_phi: 0.2616 | LR: 9.60e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.6177s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0662 | max_forward: 1.6558 | max_loss: 0.3855 | max_backward: 2.1309 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 1.0709 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6092.6 | uss_MB: 4961.3 | child_rss_MB: 5647.9 | allocated_MB: 345.8 | reserved_MB: 26622.0 | inactive_split_MB: 194.2 +rank2 [2026-10-28 15:59:35,211] INFO : Step 6900/50000 rank2 | Train Loss: 2.3793 | Train Loss Unweighted: 1.3337 | Train Loss Components: Auxiliary: 0.4495 | Cardinality: 0.2757 | Classification: 0.2300 | Classification_binary: 0.3711 | HitGrouping: 0.2647 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3605 | Regression_eta: 0.0045 | Regression_pt: 0.3628 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1106 | Classification_binary: 0.3749 | Regression_cos_phi: 0.1736 | Regression_energy: 0.1623 | Regression_eta: 0.1073 | Regression_pt: 0.1684 | Regression_sin_phi: 0.1732 | LR: 9.58e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2964s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1120 | max_forward: 0.8190 | max_loss: 0.1496 | max_backward: 1.4356 | max_optimizer_step: 0.2847 | max_tensorboard_logging: 0.8440 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6096.9 | uss_MB: 4962.9 | child_rss_MB: 5656.8 | allocated_MB: 345.4 | reserved_MB: 26622.0 | inactive_split_MB: 300.5 +rank2 [2026-10-28 16:03:56,953] INFO : Step 7000/50000 rank2 | Train Loss: 2.2410 | Train Loss Unweighted: 1.2707 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2746 | Classification: 0.1970 | Classification_binary: 0.3002 | HitGrouping: 0.2655 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3795 | Regression_eta: 0.0045 | Regression_pt: 0.3852 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1265 | Classification_binary: 0.3007 | Regression_cos_phi: 0.1911 | Regression_energy: 0.1681 | Regression_eta: 0.1449 | Regression_pt: 0.1719 | Regression_sin_phi: 0.2323 | LR: 9.57e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.3863s | Phase Times: max_pre_step_barrier: 0.0229 | max_batch_to_device: 0.0154 | max_forward: 1.0393 | max_loss: 0.2251 | max_backward: 2.0451 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.9425 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6097.4 | uss_MB: 4965.6 | child_rss_MB: 5656.9 | allocated_MB: 345.5 | reserved_MB: 26622.0 | inactive_split_MB: 296.5 +rank2 [2026-10-28 16:03:58,349] INFO : Running validation on rank2 +rank2 [2026-10-28 16:03:58,349] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 16:03:58,349] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 16:04:51,884] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:04:52,381] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 16:05:18,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 16:05:19,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 16:05:20,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 16:05:21,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 16:05:31,564] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 16:05:32,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:05:32,550] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 16:05:48,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 16:05:49,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 16:05:50,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 16:05:51,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 16:06:01,172] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 16:06:02,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:06:02,706] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 16:06:18,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 16:06:19,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 16:06:19,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 16:06:20,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 16:06:30,459] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 16:10:49,043] INFO : Step 7100/50000 rank2 | Train Loss: 2.1086 | Train Loss Unweighted: 1.2525 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.1821 | Classification: 0.2186 | Classification_binary: 0.2903 | HitGrouping: 0.2613 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3655 | Regression_eta: 0.0041 | Regression_pt: 0.3696 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1544 | Classification_binary: 0.3166 | Regression_cos_phi: 0.1830 | Regression_energy: 0.1892 | Regression_eta: 0.1455 | Regression_pt: 0.1966 | Regression_sin_phi: 0.1992 | LR: 9.56e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.9396s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6376 | max_loss: 0.1424 | max_backward: 1.3436 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8454 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 7191.1 | uss_MB: 6057.4 | child_rss_MB: 5686.3 | allocated_MB: 345.6 | reserved_MB: 22618.0 | inactive_split_MB: 206.0 +rank2 [2026-10-28 16:15:05,704] INFO : Step 7200/50000 rank2 | Train Loss: 2.4589 | Train Loss Unweighted: 1.3873 | Train Loss Components: Auxiliary: 0.4658 | Cardinality: 0.2721 | Classification: 0.2648 | Classification_binary: 0.3806 | HitGrouping: 0.2633 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3642 | Regression_eta: 0.0051 | Regression_pt: 0.3675 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1641 | Classification_binary: 0.3986 | Regression_cos_phi: 0.2166 | Regression_energy: 0.1679 | Regression_eta: 0.1403 | Regression_pt: 0.1772 | Regression_sin_phi: 0.2266 | LR: 9.55e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3065s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 1.0191 | max_loss: 0.1674 | max_backward: 1.7813 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6183.3 | uss_MB: 5050.5 | child_rss_MB: 5662.0 | allocated_MB: 343.5 | reserved_MB: 22618.0 | inactive_split_MB: 315.3 +rank2 [2026-10-28 16:19:17,626] INFO : Step 7300/50000 rank2 | Train Loss: 2.1617 | Train Loss Unweighted: 1.2404 | Train Loss Components: Auxiliary: 0.4141 | Cardinality: 0.2258 | Classification: 0.1979 | Classification_binary: 0.3107 | HitGrouping: 0.2671 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3613 | Regression_eta: 0.0041 | Regression_pt: 0.3621 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1040 | Classification_binary: 0.2828 | Regression_cos_phi: 0.1593 | Regression_energy: 0.2023 | Regression_eta: 0.1091 | Regression_pt: 0.2138 | Regression_sin_phi: 0.1769 | LR: 9.53e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.7533s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0084 | max_forward: 0.7227 | max_loss: 0.1491 | max_backward: 1.4232 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.5902 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6180.9 | uss_MB: 5051.2 | child_rss_MB: 5696.9 | allocated_MB: 345.0 | reserved_MB: 23608.0 | inactive_split_MB: 295.0 +rank2 [2026-10-28 16:23:25,991] INFO : Step 7400/50000 rank2 | Train Loss: 2.0741 | Train Loss Unweighted: 1.2014 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2287 | Classification: 0.2035 | Classification_binary: 0.2781 | HitGrouping: 0.2646 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3549 | Regression_eta: 0.0040 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1242 | Classification_binary: 0.2881 | Regression_cos_phi: 0.1721 | Regression_energy: 0.1695 | Regression_eta: 0.1211 | Regression_pt: 0.1755 | Regression_sin_phi: 0.1447 | LR: 9.52e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6554s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6129 | max_loss: 0.2127 | max_backward: 1.2944 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6450 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6180.9 | uss_MB: 5049.5 | child_rss_MB: 5695.6 | allocated_MB: 343.8 | reserved_MB: 23608.0 | inactive_split_MB: 229.5 +rank2 [2026-10-28 16:27:18,560] INFO : Step 7500/50000 rank2 | Train Loss: 2.4640 | Train Loss Unweighted: 1.3686 | Train Loss Components: Auxiliary: 0.4637 | Cardinality: 0.2640 | Classification: 0.2570 | Classification_binary: 0.3594 | HitGrouping: 0.2687 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3684 | Regression_eta: 0.0054 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1663 | Classification_binary: 0.3751 | Regression_cos_phi: 0.2604 | Regression_energy: 0.1670 | Regression_eta: 0.1602 | Regression_pt: 0.1726 | Regression_sin_phi: 0.2239 | LR: 9.51e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1740s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0887 | max_forward: 0.7717 | max_loss: 0.1508 | max_backward: 1.3590 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.9634 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6179.4 | uss_MB: 5053.3 | child_rss_MB: 5684.8 | allocated_MB: 348.6 | reserved_MB: 23608.0 | inactive_split_MB: 283.7 +rank2 [2026-10-28 16:31:22,305] INFO : Step 7600/50000 rank2 | Train Loss: 2.3745 | Train Loss Unweighted: 1.3447 | Train Loss Components: Auxiliary: 0.4509 | Cardinality: 0.2616 | Classification: 0.2519 | Classification_binary: 0.3743 | HitGrouping: 0.2627 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3532 | Regression_eta: 0.0050 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1306 | Classification_binary: 0.3661 | Regression_cos_phi: 0.2213 | Regression_energy: 0.1616 | Regression_eta: 0.1203 | Regression_pt: 0.1635 | Regression_sin_phi: 0.1797 | LR: 9.50e-04 | DataLoad Time: 0.0164s | Model Forward Time: 16.0036s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.7345 | max_loss: 0.1617 | max_backward: 1.4897 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 12.6867 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6179.5 | uss_MB: 5051.2 | child_rss_MB: 5695.8 | allocated_MB: 345.3 | reserved_MB: 23608.0 | inactive_split_MB: 284.8 +rank2 [2026-10-28 16:35:23,036] INFO : Step 7700/50000 rank2 | Train Loss: 2.0023 | Train Loss Unweighted: 1.1475 | Train Loss Components: Auxiliary: 0.3866 | Cardinality: 0.2005 | Classification: 0.1968 | Classification_binary: 0.2784 | HitGrouping: 0.2562 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3315 | Regression_eta: 0.0040 | Regression_pt: 0.3329 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1082 | Classification_binary: 0.2778 | Regression_cos_phi: 0.1486 | Regression_energy: 0.1641 | Regression_eta: 0.1363 | Regression_pt: 0.1697 | Regression_sin_phi: 0.1468 | LR: 9.48e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.0204s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7891 | max_loss: 0.1927 | max_backward: 1.4291 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6182.9 | uss_MB: 5055.0 | child_rss_MB: 5702.0 | allocated_MB: 343.9 | reserved_MB: 23608.0 | inactive_split_MB: 279.7 +rank2 [2026-10-28 16:39:23,043] INFO : Step 7800/50000 rank2 | Train Loss: 2.0473 | Train Loss Unweighted: 1.2037 | Train Loss Components: Auxiliary: 0.4013 | Cardinality: 0.1891 | Classification: 0.2102 | Classification_binary: 0.2760 | HitGrouping: 0.2604 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3526 | Regression_eta: 0.0038 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1353 | Classification_binary: 0.2920 | Regression_cos_phi: 0.2446 | Regression_energy: 0.1737 | Regression_eta: 0.1281 | Regression_pt: 0.1830 | Regression_sin_phi: 0.2056 | LR: 9.47e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.4338s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.7326 | max_loss: 0.2464 | max_backward: 1.8124 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9395 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6182.4 | uss_MB: 5059.4 | child_rss_MB: 5675.6 | allocated_MB: 343.5 | reserved_MB: 24612.0 | inactive_split_MB: 224.8 +rank2 [2026-10-28 16:43:21,636] INFO : Step 7900/50000 rank2 | Train Loss: 2.3872 | Train Loss Unweighted: 1.3279 | Train Loss Components: Auxiliary: 0.4525 | Cardinality: 0.2722 | Classification: 0.2401 | Classification_binary: 0.3580 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3573 | Regression_eta: 0.0051 | Regression_pt: 0.3627 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1516 | Classification_binary: 0.3603 | Regression_cos_phi: 0.2231 | Regression_energy: 0.1809 | Regression_eta: 0.1480 | Regression_pt: 0.1904 | Regression_sin_phi: 0.3193 | LR: 9.46e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7836s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7032 | max_loss: 0.2052 | max_backward: 1.5183 | max_optimizer_step: 0.1922 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6185.2 | uss_MB: 5060.7 | child_rss_MB: 5695.7 | allocated_MB: 347.1 | reserved_MB: 24612.0 | inactive_split_MB: 279.0 +rank2 [2026-10-28 16:47:21,753] INFO : Step 8000/50000 rank2 | Train Loss: 2.1303 | Train Loss Unweighted: 1.2019 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2451 | Classification: 0.2031 | Classification_binary: 0.3087 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3382 | Regression_eta: 0.0041 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1135 | Classification_binary: 0.2908 | Regression_cos_phi: 0.1798 | Regression_energy: 0.1611 | Regression_eta: 0.1210 | Regression_pt: 0.1704 | Regression_sin_phi: 0.1599 | LR: 9.44e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.7982s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0554 | max_forward: 0.7608 | max_loss: 0.1617 | max_backward: 1.3810 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 6186.5 | uss_MB: 4730.8 | child_rss_MB: 5692.5 | allocated_MB: 344.2 | reserved_MB: 24612.0 | inactive_split_MB: 224.1 +rank2 [2026-10-28 16:47:22,745] INFO : Running validation on rank2 +rank2 [2026-10-28 16:47:22,745] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 16:47:22,745] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 16:48:13,808] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:48:14,392] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 16:48:40,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 16:48:41,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 16:48:42,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 16:48:43,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 16:48:53,554] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 16:48:53,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:48:54,468] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 16:49:10,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 16:49:11,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 16:49:12,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 16:49:13,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 16:49:23,713] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 16:49:24,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:49:24,963] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 16:49:40,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 16:49:41,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 16:49:41,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 16:49:42,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 16:49:52,617] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 16:54:09,127] INFO : Step 8100/50000 rank2 | Train Loss: 2.0320 | Train Loss Unweighted: 1.1744 | Train Loss Components: Auxiliary: 0.3893 | Cardinality: 0.1986 | Classification: 0.1938 | Classification_binary: 0.2691 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3510 | Regression_eta: 0.0039 | Regression_pt: 0.3525 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1060 | Classification_binary: 0.2626 | Regression_cos_phi: 0.2572 | Regression_energy: 0.1726 | Regression_eta: 0.1456 | Regression_pt: 0.1797 | Regression_sin_phi: 0.2252 | LR: 9.43e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0440s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6251 | max_loss: 0.2465 | max_backward: 1.3564 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8507 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7244.9 | uss_MB: 6118.4 | child_rss_MB: 5660.8 | allocated_MB: 342.9 | reserved_MB: 23510.0 | inactive_split_MB: 258.1 +rank2 [2026-10-28 16:58:00,710] INFO : Step 8200/50000 rank2 | Train Loss: 2.1917 | Train Loss Unweighted: 1.2329 | Train Loss Components: Auxiliary: 0.4200 | Cardinality: 0.2405 | Classification: 0.2359 | Classification_binary: 0.3328 | HitGrouping: 0.2582 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3251 | Regression_eta: 0.0043 | Regression_pt: 0.3305 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1369 | Classification_binary: 0.4084 | Regression_cos_phi: 0.2197 | Regression_energy: 0.1707 | Regression_eta: 0.1397 | Regression_pt: 0.1823 | Regression_sin_phi: 0.2660 | LR: 9.42e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2792s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0064 | max_forward: 0.7679 | max_loss: 0.2484 | max_backward: 1.6620 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.8049 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1289 | Memory: rss_MB: 6232.6 | uss_MB: 5106.1 | child_rss_MB: 5673.2 | allocated_MB: 343.2 | reserved_MB: 23510.0 | inactive_split_MB: 292.4 +rank2 [2026-10-28 17:02:03,650] INFO : Step 8300/50000 rank2 | Train Loss: 2.1924 | Train Loss Unweighted: 1.2399 | Train Loss Components: Auxiliary: 0.4122 | Cardinality: 0.2039 | Classification: 0.2083 | Classification_binary: 0.3661 | HitGrouping: 0.2599 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3243 | Regression_eta: 0.0043 | Regression_pt: 0.3323 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1385 | Classification_binary: 0.3859 | Regression_cos_phi: 0.2318 | Regression_energy: 0.1547 | Regression_eta: 0.1324 | Regression_pt: 0.1613 | Regression_sin_phi: 0.2032 | LR: 9.40e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.8415s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0061 | max_forward: 0.9407 | max_loss: 0.2296 | max_backward: 1.9622 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6228.1 | uss_MB: 5055.3 | child_rss_MB: 5669.2 | allocated_MB: 344.6 | reserved_MB: 24580.0 | inactive_split_MB: 267.5 +rank2 [2026-10-28 17:06:07,749] INFO : Step 8400/50000 rank2 | Train Loss: 1.9542 | Train Loss Unweighted: 1.1189 | Train Loss Components: Auxiliary: 0.3756 | Cardinality: 0.1979 | Classification: 0.1926 | Classification_binary: 0.2783 | HitGrouping: 0.2585 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3198 | Regression_eta: 0.0035 | Regression_pt: 0.3209 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1248 | Classification_binary: 0.2498 | Regression_cos_phi: 0.1457 | Regression_energy: 0.1520 | Regression_eta: 0.0902 | Regression_pt: 0.1596 | Regression_sin_phi: 0.1045 | LR: 9.39e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2386s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6939 | max_loss: 0.1474 | max_backward: 1.5357 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 0.9892 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 6216.9 | uss_MB: 5103.3 | child_rss_MB: 5675.6 | allocated_MB: 341.1 | reserved_MB: 24580.0 | inactive_split_MB: 178.9 +rank2 [2026-10-28 17:10:17,599] INFO : Step 8500/50000 rank2 | Train Loss: 2.0642 | Train Loss Unweighted: 1.1724 | Train Loss Components: Auxiliary: 0.3964 | Cardinality: 0.2023 | Classification: 0.2152 | Classification_binary: 0.2897 | HitGrouping: 0.2607 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3289 | Regression_eta: 0.0041 | Regression_pt: 0.3302 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1417 | Classification_binary: 0.3117 | Regression_cos_phi: 0.2053 | Regression_energy: 0.1498 | Regression_eta: 0.1334 | Regression_pt: 0.1558 | Regression_sin_phi: 0.2293 | LR: 9.37e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7391s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0069 | max_forward: 0.6424 | max_loss: 0.1972 | max_backward: 1.2711 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.7327 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6218.9 | uss_MB: 5108.3 | child_rss_MB: 5667.6 | allocated_MB: 344.4 | reserved_MB: 25586.0 | inactive_split_MB: 317.0 +rank2 [2026-10-28 17:14:24,635] INFO : Step 8600/50000 rank2 | Train Loss: 2.2820 | Train Loss Unweighted: 1.2960 | Train Loss Components: Auxiliary: 0.4378 | Cardinality: 0.2546 | Classification: 0.2294 | Classification_binary: 0.3478 | HitGrouping: 0.2584 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3521 | Regression_eta: 0.0046 | Regression_pt: 0.3576 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1457 | Classification_binary: 0.3396 | Regression_cos_phi: 0.1447 | Regression_energy: 0.1729 | Regression_eta: 0.1193 | Regression_pt: 0.1813 | Regression_sin_phi: 0.1849 | LR: 9.36e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.9614s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0069 | max_forward: 0.7396 | max_loss: 0.1527 | max_backward: 1.6088 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5007 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6218.9 | uss_MB: 5112.5 | child_rss_MB: 5664.7 | allocated_MB: 344.8 | reserved_MB: 25586.0 | inactive_split_MB: 283.2 +rank2 [2026-10-28 17:18:21,031] INFO : Step 8700/50000 rank2 | Train Loss: 2.0581 | Train Loss Unweighted: 1.1705 | Train Loss Components: Auxiliary: 0.4038 | Cardinality: 0.2463 | Classification: 0.2066 | Classification_binary: 0.3017 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3267 | Regression_eta: 0.0035 | Regression_pt: 0.3282 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1220 | Classification_binary: 0.2991 | Regression_cos_phi: 0.1737 | Regression_energy: 0.1657 | Regression_eta: 0.0957 | Regression_pt: 0.1720 | Regression_sin_phi: 0.1345 | LR: 9.34e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.7163s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6873 | max_loss: 0.2444 | max_backward: 1.2985 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6214.6 | uss_MB: 5113.2 | child_rss_MB: 5650.1 | allocated_MB: 343.7 | reserved_MB: 25586.0 | inactive_split_MB: 260.4 +rank2 [2026-10-28 17:22:21,714] INFO : Step 8800/50000 rank2 | Train Loss: 1.8415 | Train Loss Unweighted: 1.0591 | Train Loss Components: Auxiliary: 0.3597 | Cardinality: 0.1811 | Classification: 0.1734 | Classification_binary: 0.2490 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3144 | Regression_eta: 0.0034 | Regression_pt: 0.3153 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0937 | Classification_binary: 0.2244 | Regression_cos_phi: 0.1461 | Regression_energy: 0.1546 | Regression_eta: 0.0838 | Regression_pt: 0.1603 | Regression_sin_phi: 0.1061 | LR: 9.33e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5727s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6737 | max_loss: 0.2385 | max_backward: 1.3073 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.5819 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6215.6 | uss_MB: 5114.6 | child_rss_MB: 5649.0 | allocated_MB: 344.1 | reserved_MB: 25586.0 | inactive_split_MB: 205.9 +rank2 [2026-10-28 17:26:43,764] INFO : Step 8900/50000 rank2 | Train Loss: 2.1706 | Train Loss Unweighted: 1.1768 | Train Loss Components: Auxiliary: 0.4167 | Cardinality: 0.3000 | Classification: 0.2064 | Classification_binary: 0.3037 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3284 | Regression_eta: 0.0039 | Regression_pt: 0.3304 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1370 | Classification_binary: 0.3332 | Regression_cos_phi: 0.1626 | Regression_energy: 0.1581 | Regression_eta: 0.1226 | Regression_pt: 0.1634 | Regression_sin_phi: 0.1602 | LR: 9.31e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.8179s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0073 | max_forward: 0.9973 | max_loss: 0.4040 | max_backward: 3.4126 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6194.6 | uss_MB: 5119.9 | child_rss_MB: 5654.4 | allocated_MB: 347.1 | reserved_MB: 25586.0 | inactive_split_MB: 275.1 +rank2 [2026-10-28 17:30:46,803] INFO : Step 9000/50000 rank2 | Train Loss: 2.1357 | Train Loss Unweighted: 1.1883 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.2351 | Classification: 0.2003 | Classification_binary: 0.3467 | HitGrouping: 0.2532 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3143 | Regression_eta: 0.0041 | Regression_pt: 0.3186 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1037 | Classification_binary: 0.3117 | Regression_cos_phi: 0.1238 | Regression_energy: 0.1602 | Regression_eta: 0.0943 | Regression_pt: 0.1666 | Regression_sin_phi: 0.1810 | LR: 9.30e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.7215s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.7873 | max_loss: 0.2178 | max_backward: 1.5407 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.3125 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 6192.6 | uss_MB: 5121.2 | child_rss_MB: 5660.4 | allocated_MB: 344.6 | reserved_MB: 25586.0 | inactive_split_MB: 289.9 +rank2 [2026-10-28 17:30:47,949] INFO : Running validation on rank2 +rank2 [2026-10-28 17:30:47,950] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 17:30:47,950] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 17:31:40,962] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:31:41,476] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 17:32:08,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 17:32:09,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 17:32:10,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 17:32:11,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 17:32:21,656] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 17:32:22,657] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:32:23,201] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 17:32:39,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 17:32:40,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 17:32:41,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 17:32:42,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 17:32:52,032] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 17:32:52,551] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:32:53,102] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 17:33:08,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 17:33:09,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 17:33:10,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 17:33:10,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 17:33:20,726] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 17:37:41,206] INFO : Step 9100/50000 rank2 | Train Loss: 1.8448 | Train Loss Unweighted: 1.0695 | Train Loss Components: Auxiliary: 0.3538 | Cardinality: 0.1722 | Classification: 0.1878 | Classification_binary: 0.2661 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3012 | Regression_eta: 0.0033 | Regression_pt: 0.3075 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1122 | Classification_binary: 0.2662 | Regression_cos_phi: 0.1261 | Regression_energy: 0.1445 | Regression_eta: 0.1168 | Regression_pt: 0.1577 | Regression_sin_phi: 0.1655 | LR: 9.28e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4315s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0153 | max_forward: 1.2776 | max_loss: 0.3881 | max_backward: 1.3802 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9203 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7188.5 | uss_MB: 6128.3 | child_rss_MB: 5636.7 | allocated_MB: 345.3 | reserved_MB: 22582.0 | inactive_split_MB: 275.2 +rank2 [2026-10-28 17:41:50,230] INFO : Step 9200/50000 rank2 | Train Loss: 1.9883 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3829 | Cardinality: 0.2086 | Classification: 0.1812 | Classification_binary: 0.2802 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3207 | Regression_eta: 0.0035 | Regression_pt: 0.3257 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0952 | Classification_binary: 0.2854 | Regression_cos_phi: 0.1334 | Regression_energy: 0.1464 | Regression_eta: 0.0743 | Regression_pt: 0.1508 | Regression_sin_phi: 0.1170 | LR: 9.27e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.5154s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0073 | max_forward: 0.6839 | max_loss: 0.2414 | max_backward: 1.6336 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 1.0636 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6214.5 | uss_MB: 5161.4 | child_rss_MB: 5636.3 | allocated_MB: 345.6 | reserved_MB: 23224.0 | inactive_split_MB: 190.1 +rank2 [2026-10-28 17:45:52,429] INFO : Step 9300/50000 rank2 | Train Loss: 2.0749 | Train Loss Unweighted: 1.1591 | Train Loss Components: Auxiliary: 0.4049 | Cardinality: 0.2573 | Classification: 0.2065 | Classification_binary: 0.3180 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3111 | Regression_eta: 0.0036 | Regression_pt: 0.3160 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0968 | Classification_binary: 0.2954 | Regression_cos_phi: 0.1788 | Regression_energy: 0.1533 | Regression_eta: 0.0852 | Regression_pt: 0.1643 | Regression_sin_phi: 0.1495 | LR: 9.25e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2176s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8329 | max_loss: 0.1499 | max_backward: 1.4059 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.8766 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6214.5 | uss_MB: 5164.3 | child_rss_MB: 5654.0 | allocated_MB: 344.8 | reserved_MB: 24586.0 | inactive_split_MB: 287.2 +rank2 [2026-10-28 17:50:16,942] INFO : Step 9400/50000 rank2 | Train Loss: 1.8467 | Train Loss Unweighted: 1.0417 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1520 | Classification: 0.1837 | Classification_binary: 0.2977 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2742 | Regression_eta: 0.0040 | Regression_pt: 0.2784 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1082 | Classification_binary: 0.2901 | Regression_cos_phi: 0.1604 | Regression_energy: 0.1325 | Regression_eta: 0.1108 | Regression_pt: 0.1394 | Regression_sin_phi: 0.1402 | LR: 9.24e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.5895s | Phase Times: max_pre_step_barrier: 0.0180 | max_batch_to_device: 0.0062 | max_forward: 1.5076 | max_loss: 0.3748 | max_backward: 2.6247 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.5247 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6206.4 | uss_MB: 4649.5 | child_rss_MB: 5650.0 | allocated_MB: 343.9 | reserved_MB: 24586.0 | inactive_split_MB: 267.3 +rank2 [2026-10-28 17:54:18,159] INFO : Step 9500/50000 rank2 | Train Loss: 1.8199 | Train Loss Unweighted: 0.9995 | Train Loss Components: Auxiliary: 0.3464 | Cardinality: 0.1875 | Classification: 0.1651 | Classification_binary: 0.2558 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2851 | Regression_eta: 0.0035 | Regression_pt: 0.2864 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1014 | Classification_binary: 0.2463 | Regression_cos_phi: 0.1381 | Regression_energy: 0.1365 | Regression_eta: 0.0870 | Regression_pt: 0.1446 | Regression_sin_phi: 0.1171 | LR: 9.22e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.8032s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6532 | max_loss: 0.2385 | max_backward: 1.0043 | max_optimizer_step: 0.3001 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6205.2 | uss_MB: 5165.5 | child_rss_MB: 5654.5 | allocated_MB: 341.2 | reserved_MB: 24586.0 | inactive_split_MB: 180.8 +rank2 [2026-10-28 17:58:23,634] INFO : Step 9600/50000 rank2 | Train Loss: 2.0900 | Train Loss Unweighted: 1.1491 | Train Loss Components: Auxiliary: 0.4030 | Cardinality: 0.2464 | Classification: 0.2089 | Classification_binary: 0.3189 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3034 | Regression_eta: 0.0042 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1392 | Classification_binary: 0.3617 | Regression_cos_phi: 0.1978 | Regression_energy: 0.1392 | Regression_eta: 0.1232 | Regression_pt: 0.1447 | Regression_sin_phi: 0.2392 | LR: 9.21e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8847s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1083 | max_forward: 0.7381 | max_loss: 0.1457 | max_backward: 1.4890 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4970 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6206.2 | uss_MB: 5167.2 | child_rss_MB: 5637.2 | allocated_MB: 345.2 | reserved_MB: 25594.0 | inactive_split_MB: 190.6 +rank2 [2026-10-28 18:02:35,091] INFO : Step 9700/50000 rank2 | Train Loss: 2.1397 | Train Loss Unweighted: 1.1700 | Train Loss Components: Auxiliary: 0.4082 | Cardinality: 0.2377 | Classification: 0.1942 | Classification_binary: 0.3460 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3095 | Regression_eta: 0.0043 | Regression_pt: 0.3118 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1002 | Classification_binary: 0.3096 | Regression_cos_phi: 0.1620 | Regression_energy: 0.1471 | Regression_eta: 0.1048 | Regression_pt: 0.1539 | Regression_sin_phi: 0.1696 | LR: 9.19e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1034s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7744 | max_loss: 0.1497 | max_backward: 1.6846 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4710 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 6209.3 | uss_MB: 5170.1 | child_rss_MB: 5650.1 | allocated_MB: 345.9 | reserved_MB: 25594.0 | inactive_split_MB: 288.1 +rank2 [2026-10-28 18:06:42,228] INFO : Step 9800/50000 rank2 | Train Loss: 1.8589 | Train Loss Unweighted: 1.0245 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2213 | Classification: 0.1590 | Classification_binary: 0.2751 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2908 | Regression_eta: 0.0031 | Regression_pt: 0.2931 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0818 | Classification_binary: 0.2511 | Regression_cos_phi: 0.1197 | Regression_energy: 0.1569 | Regression_eta: 0.1011 | Regression_pt: 0.1623 | Regression_sin_phi: 0.1629 | LR: 9.17e-04 | DataLoad Time: 0.0236s | Model Forward Time: 4.1207s | Phase Times: max_pre_step_barrier: 0.1099 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.1713 | max_backward: 1.7200 | max_optimizer_step: 0.1898 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6188.3 | uss_MB: 5169.7 | child_rss_MB: 5642.5 | allocated_MB: 342.5 | reserved_MB: 26610.0 | inactive_split_MB: 264.5 +rank2 [2026-10-28 18:10:58,313] INFO : Step 9900/50000 rank2 | Train Loss: 1.8366 | Train Loss Unweighted: 1.0077 | Train Loss Components: Auxiliary: 0.3557 | Cardinality: 0.2091 | Classification: 0.1810 | Classification_binary: 0.2609 | HitGrouping: 0.2462 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2775 | Regression_eta: 0.0035 | Regression_pt: 0.2814 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0936 | Classification_binary: 0.2592 | Regression_cos_phi: 0.1113 | Regression_energy: 0.1417 | Regression_eta: 0.0870 | Regression_pt: 0.1508 | Regression_sin_phi: 0.1530 | LR: 9.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5792s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6346 | max_loss: 0.1692 | max_backward: 1.3542 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6188.4 | uss_MB: 5170.9 | child_rss_MB: 5645.5 | allocated_MB: 343.7 | reserved_MB: 26610.0 | inactive_split_MB: 205.7 +rank2 [2026-10-28 18:15:06,636] INFO : Step 10000/50000 rank2 | Train Loss: 2.0706 | Train Loss Unweighted: 1.1295 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2648 | Classification: 0.1819 | Classification_binary: 0.3225 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3077 | Regression_eta: 0.0037 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1173 | Classification_binary: 0.3466 | Regression_cos_phi: 0.1712 | Regression_energy: 0.1578 | Regression_eta: 0.1020 | Regression_pt: 0.1638 | Regression_sin_phi: 0.1612 | LR: 9.14e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.7145s | Phase Times: max_pre_step_barrier: 0.0136 | max_batch_to_device: 0.0708 | max_forward: 1.8842 | max_loss: 0.5484 | max_backward: 2.6620 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.5314 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6190.4 | uss_MB: 5174.2 | child_rss_MB: 5642.5 | allocated_MB: 345.2 | reserved_MB: 26610.0 | inactive_split_MB: 288.8 +rank2 [2026-10-28 18:15:07,627] INFO : Running validation on rank2 +rank2 [2026-10-28 18:15:07,627] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 18:15:07,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 18:15:55,922] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:15:56,537] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 18:16:22,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 18:16:23,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 18:16:24,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 18:16:25,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 18:16:35,954] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 18:16:36,561] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:16:37,045] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 18:16:53,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 18:16:55,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 18:16:58,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 18:16:59,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 18:17:09,573] INFO : Time taken to make predictions on device 2 is: 0.31 min +rank2 [2026-10-28 18:17:09,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:17:10,077] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 18:17:26,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 18:17:26,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 18:17:27,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 18:17:28,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 18:17:38,080] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 18:22:01,597] INFO : Step 10100/50000 rank2 | Train Loss: 1.8115 | Train Loss Unweighted: 1.0221 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.1999 | Classification: 0.1589 | Classification_binary: 0.2815 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2868 | Regression_eta: 0.0032 | Regression_pt: 0.2886 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.2849 | Regression_cos_phi: 0.1463 | Regression_energy: 0.1564 | Regression_eta: 0.0824 | Regression_pt: 0.1642 | Regression_sin_phi: 0.1001 | LR: 9.12e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9758s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7090 | max_loss: 0.1507 | max_backward: 1.2179 | max_optimizer_step: 0.3146 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 7166.8 | uss_MB: 6150.2 | child_rss_MB: 5645.8 | allocated_MB: 343.1 | reserved_MB: 21794.0 | inactive_split_MB: 243.8 +rank2 [2026-10-28 18:26:06,876] INFO : Step 10200/50000 rank2 | Train Loss: 1.6749 | Train Loss Unweighted: 0.9660 | Train Loss Components: Auxiliary: 0.3310 | Cardinality: 0.1773 | Classification: 0.1544 | Classification_binary: 0.2377 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2827 | Regression_eta: 0.0027 | Regression_pt: 0.2857 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0979 | Classification_binary: 0.2674 | Regression_cos_phi: 0.1669 | Regression_energy: 0.1443 | Regression_eta: 0.0880 | Regression_pt: 0.1488 | Regression_sin_phi: 0.1542 | LR: 9.11e-04 | DataLoad Time: 0.0119s | Model Forward Time: 4.0153s | Phase Times: max_pre_step_barrier: 0.0352 | max_batch_to_device: 0.0241 | max_forward: 0.6625 | max_loss: 0.1442 | max_backward: 1.3835 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7009 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1256 | Memory: rss_MB: 7164.8 | uss_MB: 6150.1 | child_rss_MB: 5645.0 | allocated_MB: 345.9 | reserved_MB: 21794.0 | inactive_split_MB: 292.4 +rank2 [2026-10-28 18:30:14,208] INFO : Step 10300/50000 rank2 | Train Loss: 1.9113 | Train Loss Unweighted: 1.0360 | Train Loss Components: Auxiliary: 0.3660 | Cardinality: 0.2108 | Classification: 0.1830 | Classification_binary: 0.2931 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0033 | Regression_pt: 0.2786 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1059 | Classification_binary: 0.3052 | Regression_cos_phi: 0.2059 | Regression_energy: 0.1454 | Regression_eta: 0.1013 | Regression_pt: 0.1564 | Regression_sin_phi: 0.1578 | LR: 9.09e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9108s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7205 | max_loss: 0.1496 | max_backward: 1.6083 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 6287.2 | uss_MB: 5273.5 | child_rss_MB: 5650.9 | allocated_MB: 343.8 | reserved_MB: 22764.0 | inactive_split_MB: 257.9 +rank2 [2026-10-28 18:34:19,878] INFO : Step 10400/50000 rank2 | Train Loss: 1.9303 | Train Loss Unweighted: 1.0569 | Train Loss Components: Auxiliary: 0.3739 | Cardinality: 0.2318 | Classification: 0.1704 | Classification_binary: 0.3342 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2718 | Regression_eta: 0.0033 | Regression_pt: 0.2738 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0952 | Classification_binary: 0.3576 | Regression_cos_phi: 0.1434 | Regression_energy: 0.1321 | Regression_eta: 0.0842 | Regression_pt: 0.1390 | Regression_sin_phi: 0.1354 | LR: 9.07e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.7366s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7465 | max_loss: 0.1499 | max_backward: 1.4212 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.4712 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6287.2 | uss_MB: 5272.0 | child_rss_MB: 5660.4 | allocated_MB: 345.6 | reserved_MB: 22792.0 | inactive_split_MB: 259.8 +rank2 [2026-10-28 18:38:24,241] INFO : Step 10500/50000 rank2 | Train Loss: 1.6601 | Train Loss Unweighted: 0.9268 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1892 | Classification: 0.1410 | Classification_binary: 0.2342 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2693 | Regression_eta: 0.0029 | Regression_pt: 0.2766 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.2097 | Regression_cos_phi: 0.1070 | Regression_energy: 0.1123 | Regression_eta: 0.0528 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0941 | LR: 9.06e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.5593s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 0.6364 | max_loss: 0.2260 | max_backward: 1.1629 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6452 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6284.6 | uss_MB: 5269.3 | child_rss_MB: 5635.7 | allocated_MB: 340.4 | reserved_MB: 22792.0 | inactive_split_MB: 230.3 +rank2 [2026-10-28 18:42:22,546] INFO : Step 10600/50000 rank2 | Train Loss: 1.9388 | Train Loss Unweighted: 1.0719 | Train Loss Components: Auxiliary: 0.3737 | Cardinality: 0.2008 | Classification: 0.1863 | Classification_binary: 0.2709 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3047 | Regression_eta: 0.0042 | Regression_pt: 0.3019 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1303 | Classification_binary: 0.2952 | Regression_cos_phi: 0.2191 | Regression_energy: 0.1306 | Regression_eta: 0.1583 | Regression_pt: 0.1375 | Regression_sin_phi: 0.1995 | LR: 9.04e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9681s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6880 | max_loss: 0.2458 | max_backward: 1.4354 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7481 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 6285.6 | uss_MB: 5273.7 | child_rss_MB: 5653.3 | allocated_MB: 344.6 | reserved_MB: 22798.0 | inactive_split_MB: 286.4 +rank2 [2026-10-28 18:46:28,872] INFO : Step 10700/50000 rank2 | Train Loss: 1.8955 | Train Loss Unweighted: 1.0230 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.2139 | Classification: 0.1752 | Classification_binary: 0.3186 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2598 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1307 | Classification_binary: 0.3413 | Regression_cos_phi: 0.1532 | Regression_energy: 0.1368 | Regression_eta: 0.1265 | Regression_pt: 0.1455 | Regression_sin_phi: 0.2000 | LR: 9.02e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.1516s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0072 | max_forward: 1.0363 | max_loss: 0.1757 | max_backward: 2.3255 | max_optimizer_step: 0.1834 | max_tensorboard_logging: 0.7232 | max_tensorboard_flush: 0.0051 | max_post_step_barrier: 0.1576 | Memory: rss_MB: 6287.7 | uss_MB: 5274.9 | child_rss_MB: 5649.7 | allocated_MB: 342.1 | reserved_MB: 22798.0 | inactive_split_MB: 282.9 +rank2 [2026-10-28 18:50:28,323] INFO : Step 10800/50000 rank2 | Train Loss: 1.7679 | Train Loss Unweighted: 0.9563 | Train Loss Components: Auxiliary: 0.3446 | Cardinality: 0.2074 | Classification: 0.1518 | Classification_binary: 0.2766 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2590 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0973 | Classification_binary: 0.3068 | Regression_cos_phi: 0.1336 | Regression_energy: 0.1275 | Regression_eta: 0.0874 | Regression_pt: 0.1319 | Regression_sin_phi: 0.1294 | LR: 9.00e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.7791s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0995 | max_forward: 0.7884 | max_loss: 0.2225 | max_backward: 2.0144 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.5230 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6288.7 | uss_MB: 5271.1 | child_rss_MB: 5654.7 | allocated_MB: 343.6 | reserved_MB: 23784.0 | inactive_split_MB: 286.4 +rank2 [2026-10-28 18:54:31,738] INFO : Step 10900/50000 rank2 | Train Loss: 1.6025 | Train Loss Unweighted: 0.8934 | Train Loss Components: Auxiliary: 0.3208 | Cardinality: 0.1729 | Classification: 0.1446 | Classification_binary: 0.2284 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2574 | Regression_eta: 0.0027 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0996 | Classification_binary: 0.2628 | Regression_cos_phi: 0.1332 | Regression_energy: 0.1222 | Regression_eta: 0.0690 | Regression_pt: 0.1270 | Regression_sin_phi: 0.1334 | LR: 8.99e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4467s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.5765 | max_loss: 0.1490 | max_backward: 1.1882 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6288.0 | uss_MB: 5275.5 | child_rss_MB: 5639.7 | allocated_MB: 342.1 | reserved_MB: 23796.0 | inactive_split_MB: 272.7 +rank2 [2026-10-28 18:58:45,418] INFO : Step 11000/50000 rank2 | Train Loss: 1.9315 | Train Loss Unweighted: 1.0466 | Train Loss Components: Auxiliary: 0.3721 | Cardinality: 0.2078 | Classification: 0.1806 | Classification_binary: 0.3087 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0037 | Regression_pt: 0.2758 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0956 | Classification_binary: 0.3255 | Regression_cos_phi: 0.2480 | Regression_energy: 0.1297 | Regression_eta: 0.1057 | Regression_pt: 0.1344 | Regression_sin_phi: 0.1474 | LR: 8.97e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.1713s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7424 | max_loss: 0.1704 | max_backward: 1.5866 | max_optimizer_step: 0.2906 | max_tensorboard_logging: 0.6287 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6277.0 | uss_MB: 5281.8 | child_rss_MB: 5663.8 | allocated_MB: 347.8 | reserved_MB: 23796.0 | inactive_split_MB: 262.8 +rank2 [2026-10-28 18:58:46,360] INFO : Running validation on rank2 +rank2 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 18:59:40,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:59:40,900] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 19:00:07,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 19:00:09,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 19:00:10,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 19:00:11,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 19:00:21,376] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-28 19:00:21,678] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:00:22,258] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 19:00:38,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 19:00:39,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 19:00:40,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 19:00:41,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 19:00:51,604] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 19:00:52,166] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:00:52,895] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 19:01:08,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 19:01:09,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 19:01:09,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 19:01:10,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 19:01:20,652] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 19:05:41,393] INFO : Step 11100/50000 rank2 | Train Loss: 1.9118 | Train Loss Unweighted: 1.0064 | Train Loss Components: Auxiliary: 0.3723 | Cardinality: 0.2793 | Classification: 0.1563 | Classification_binary: 0.3082 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2660 | Regression_eta: 0.0034 | Regression_pt: 0.2693 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1025 | Classification_binary: 0.2952 | Regression_cos_phi: 0.1197 | Regression_energy: 0.1294 | Regression_eta: 0.0959 | Regression_pt: 0.1349 | Regression_sin_phi: 0.1479 | LR: 8.95e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0415s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.9217 | max_loss: 0.2304 | max_backward: 1.5570 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 7169.9 | uss_MB: 6179.3 | child_rss_MB: 5664.2 | allocated_MB: 343.2 | reserved_MB: 22146.0 | inactive_split_MB: 281.7 +rank2 [2026-10-28 19:09:39,138] INFO : Step 11200/50000 rank2 | Train Loss: 1.7366 | Train Loss Unweighted: 0.9397 | Train Loss Components: Auxiliary: 0.3529 | Cardinality: 0.2466 | Classification: 0.1384 | Classification_binary: 0.2548 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2689 | Regression_eta: 0.0027 | Regression_pt: 0.2722 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 0.2473 | Regression_cos_phi: 0.1157 | Regression_energy: 0.1246 | Regression_eta: 0.1010 | Regression_pt: 0.1325 | Regression_sin_phi: 0.1026 | LR: 8.93e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7186s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7793 | max_loss: 0.1825 | max_backward: 1.3743 | max_optimizer_step: 0.2128 | max_tensorboard_logging: 0.4676 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7167.9 | uss_MB: 6179.2 | child_rss_MB: 5641.7 | allocated_MB: 344.6 | reserved_MB: 22972.0 | inactive_split_MB: 273.4 +rank2 [2026-10-28 19:13:41,796] INFO : Step 11300/50000 rank2 | Train Loss: 1.6876 | Train Loss Unweighted: 0.9423 | Train Loss Components: Auxiliary: 0.3334 | Cardinality: 0.1862 | Classification: 0.1581 | Classification_binary: 0.2713 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2524 | Regression_eta: 0.0027 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.3022 | Regression_cos_phi: 0.1144 | Regression_energy: 0.1096 | Regression_eta: 0.0839 | Regression_pt: 0.1157 | Regression_sin_phi: 0.1492 | LR: 8.91e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6820s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0223 | max_forward: 0.9315 | max_loss: 0.2308 | max_backward: 1.9859 | max_optimizer_step: 0.1928 | max_tensorboard_logging: 0.5681 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6388.7 | uss_MB: 5273.7 | child_rss_MB: 5658.4 | allocated_MB: 345.5 | reserved_MB: 24368.0 | inactive_split_MB: 269.6 +rank2 [2026-10-28 19:17:45,333] INFO : Step 11400/50000 rank2 | Train Loss: 1.8563 | Train Loss Unweighted: 0.9980 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.2376 | Classification: 0.1627 | Classification_binary: 0.3164 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2551 | Regression_eta: 0.0029 | Regression_pt: 0.2577 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1139 | Classification_binary: 0.3500 | Regression_cos_phi: 0.1754 | Regression_energy: 0.1190 | Regression_eta: 0.0909 | Regression_pt: 0.1255 | Regression_sin_phi: 0.1465 | LR: 8.89e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.0880s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.7684 | max_loss: 0.1511 | max_backward: 1.6966 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5034 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 6391.7 | uss_MB: 5402.6 | child_rss_MB: 5664.5 | allocated_MB: 343.2 | reserved_MB: 25642.0 | inactive_split_MB: 251.6 +rank2 [2026-10-28 19:21:55,044] INFO : Step 11500/50000 rank2 | Train Loss: 1.6323 | Train Loss Unweighted: 0.8976 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1755 | Classification: 0.1375 | Classification_binary: 0.2717 | HitGrouping: 0.2346 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2408 | Regression_eta: 0.0028 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0815 | Classification_binary: 0.2579 | Regression_cos_phi: 0.1195 | Regression_energy: 0.1213 | Regression_eta: 0.0749 | Regression_pt: 0.1262 | Regression_sin_phi: 0.1037 | LR: 8.88e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6942s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7232 | max_loss: 0.1557 | max_backward: 1.4770 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6390.5 | uss_MB: 5401.3 | child_rss_MB: 5644.3 | allocated_MB: 345.3 | reserved_MB: 25642.0 | inactive_split_MB: 274.7 +rank2 [2026-10-28 19:25:54,593] INFO : Step 11600/50000 rank2 | Train Loss: 1.6133 | Train Loss Unweighted: 0.9062 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1718 | Classification: 0.1502 | Classification_binary: 0.2356 | HitGrouping: 0.2377 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2569 | Regression_eta: 0.0025 | Regression_pt: 0.2581 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0879 | Classification_binary: 0.2178 | Regression_cos_phi: 0.0889 | Regression_energy: 0.1336 | Regression_eta: 0.0766 | Regression_pt: 0.1382 | Regression_sin_phi: 0.1302 | LR: 8.86e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.6494s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6646 | max_loss: 0.1710 | max_backward: 1.3208 | max_optimizer_step: 0.1963 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6391.5 | uss_MB: 5404.2 | child_rss_MB: 5665.3 | allocated_MB: 344.1 | reserved_MB: 26630.0 | inactive_split_MB: 209.9 +rank2 [2026-10-28 19:29:50,737] INFO : Step 11700/50000 rank2 | Train Loss: 1.7739 | Train Loss Unweighted: 0.9706 | Train Loss Components: Auxiliary: 0.3531 | Cardinality: 0.2418 | Classification: 0.1518 | Classification_binary: 0.2821 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2638 | Regression_eta: 0.0028 | Regression_pt: 0.2671 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1014 | Classification_binary: 0.3110 | Regression_cos_phi: 0.1251 | Regression_energy: 0.1279 | Regression_eta: 0.0746 | Regression_pt: 0.1306 | Regression_sin_phi: 0.1212 | LR: 8.84e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.4609s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8789 | max_loss: 0.2057 | max_backward: 2.3846 | max_optimizer_step: 0.1875 | max_tensorboard_logging: 1.0403 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6389.5 | uss_MB: 4921.9 | child_rss_MB: 5640.9 | allocated_MB: 343.8 | reserved_MB: 27624.0 | inactive_split_MB: 186.6 +rank2 [2026-10-28 19:33:54,005] INFO : Step 11800/50000 rank2 | Train Loss: 1.8223 | Train Loss Unweighted: 0.9745 | Train Loss Components: Auxiliary: 0.3609 | Cardinality: 0.2329 | Classification: 0.1589 | Classification_binary: 0.3100 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2473 | Regression_eta: 0.0028 | Regression_pt: 0.2523 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0761 | Classification_binary: 0.3204 | Regression_cos_phi: 0.1336 | Regression_energy: 0.1223 | Regression_eta: 0.0643 | Regression_pt: 0.1273 | Regression_sin_phi: 0.1010 | LR: 8.82e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.3881s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0070 | max_forward: 1.2302 | max_loss: 0.2767 | max_backward: 2.3745 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6390.5 | uss_MB: 5402.7 | child_rss_MB: 5661.5 | allocated_MB: 346.0 | reserved_MB: 27624.0 | inactive_split_MB: 280.5 +rank2 [2026-10-28 19:37:48,976] INFO : Step 11900/50000 rank2 | Train Loss: 1.5797 | Train Loss Unweighted: 0.8638 | Train Loss Components: Auxiliary: 0.3225 | Cardinality: 0.1935 | Classification: 0.1358 | Classification_binary: 0.2351 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2427 | Regression_eta: 0.0025 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1972 | Regression_cos_phi: 0.0484 | Regression_energy: 0.1239 | Regression_eta: 0.0502 | Regression_pt: 0.1309 | Regression_sin_phi: 0.1195 | LR: 8.80e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.1376s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0787 | max_forward: 0.7011 | max_loss: 0.1484 | max_backward: 1.2002 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.9463 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 6390.3 | uss_MB: 5402.5 | child_rss_MB: 5642.2 | allocated_MB: 340.1 | reserved_MB: 28620.0 | inactive_split_MB: 264.1 +rank2 [2026-10-28 19:41:43,168] INFO : Step 12000/50000 rank2 | Train Loss: 1.6036 | Train Loss Unweighted: 0.8706 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1972 | Classification: 0.1481 | Classification_binary: 0.2459 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2330 | Regression_eta: 0.0025 | Regression_pt: 0.2385 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0937 | Classification_binary: 0.2664 | Regression_cos_phi: 0.1500 | Regression_energy: 0.1138 | Regression_eta: 0.0894 | Regression_pt: 0.1192 | Regression_sin_phi: 0.1073 | LR: 8.78e-04 | DataLoad Time: 0.0148s | Model Forward Time: 3.6989s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0072 | max_forward: 0.6194 | max_loss: 0.1546 | max_backward: 1.4628 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5143 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6383.3 | uss_MB: 5409.4 | child_rss_MB: 5646.2 | allocated_MB: 342.9 | reserved_MB: 28620.0 | inactive_split_MB: 215.8 +rank2 [2026-10-28 19:41:44,123] INFO : Running validation on rank2 +rank2 [2026-10-28 19:41:44,123] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 19:41:44,123] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 19:42:35,078] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:42:35,718] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 19:43:02,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 19:43:04,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 19:43:05,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 19:43:06,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 19:43:16,624] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-28 19:43:17,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:43:17,650] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 19:43:34,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 19:43:35,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 19:43:36,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 19:43:37,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 19:43:47,016] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 19:43:48,085] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:43:48,651] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 19:44:04,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 19:44:05,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 19:44:05,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 19:44:06,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 19:44:16,369] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 19:48:34,082] INFO : Step 12100/50000 rank2 | Train Loss: 1.8149 | Train Loss Unweighted: 0.9793 | Train Loss Components: Auxiliary: 0.3632 | Cardinality: 0.2324 | Classification: 0.1506 | Classification_binary: 0.3149 | HitGrouping: 0.2337 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2518 | Regression_eta: 0.0029 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0907 | Classification_binary: 0.2955 | Regression_cos_phi: 0.1465 | Regression_energy: 0.0973 | Regression_eta: 0.0897 | Regression_pt: 0.1026 | Regression_sin_phi: 0.1657 | LR: 8.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5268s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8305 | max_loss: 0.1678 | max_backward: 1.6498 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8610 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7131.3 | uss_MB: 6150.6 | child_rss_MB: 5646.8 | allocated_MB: 345.8 | reserved_MB: 23512.0 | inactive_split_MB: 299.6 +rank2 [2026-10-28 19:52:54,661] INFO : Step 12200/50000 rank2 | Train Loss: 1.6619 | Train Loss Unweighted: 0.9043 | Train Loss Components: Auxiliary: 0.3290 | Cardinality: 0.1906 | Classification: 0.1445 | Classification_binary: 0.2674 | HitGrouping: 0.2303 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2435 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1001 | Classification_binary: 0.2729 | Regression_cos_phi: 0.0918 | Regression_energy: 0.1360 | Regression_eta: 0.0756 | Regression_pt: 0.1415 | Regression_sin_phi: 0.1527 | LR: 8.74e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8240s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.7862 | max_loss: 0.1438 | max_backward: 1.5991 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.8843 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7133.3 | uss_MB: 5953.1 | child_rss_MB: 5653.1 | allocated_MB: 341.2 | reserved_MB: 23512.0 | inactive_split_MB: 206.8 +rank2 [2026-10-28 19:56:47,339] INFO : Step 12300/50000 rank2 | Train Loss: 1.4699 | Train Loss Unweighted: 0.8181 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1474 | Classification: 0.1404 | Classification_binary: 0.2280 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2222 | Regression_eta: 0.0028 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0799 | Classification_binary: 0.2272 | Regression_cos_phi: 0.0825 | Regression_energy: 0.1155 | Regression_eta: 0.0541 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0797 | LR: 8.72e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.7246s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6177 | max_loss: 0.2232 | max_backward: 1.1179 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 0.7974 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6497.9 | uss_MB: 5527.4 | child_rss_MB: 5654.1 | allocated_MB: 343.2 | reserved_MB: 24286.0 | inactive_split_MB: 290.2 +rank2 [2026-10-28 20:00:36,191] INFO : Step 12400/50000 rank2 | Train Loss: 1.6135 | Train Loss Unweighted: 0.8782 | Train Loss Components: Auxiliary: 0.3265 | Cardinality: 0.2056 | Classification: 0.1546 | Classification_binary: 0.2540 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2308 | Regression_eta: 0.0025 | Regression_pt: 0.2337 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0875 | Classification_binary: 0.2767 | Regression_cos_phi: 0.1031 | Regression_energy: 0.0991 | Regression_eta: 0.0773 | Regression_pt: 0.1060 | Regression_sin_phi: 0.1079 | LR: 8.70e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.9942s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6928 | max_loss: 0.1486 | max_backward: 1.3860 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6497.9 | uss_MB: 5528.1 | child_rss_MB: 5671.7 | allocated_MB: 344.8 | reserved_MB: 24510.0 | inactive_split_MB: 295.5 +rank2 [2026-10-28 20:04:30,061] INFO : Step 12500/50000 rank2 | Train Loss: 1.6527 | Train Loss Unweighted: 0.8797 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1890 | Classification: 0.1391 | Classification_binary: 0.2779 | HitGrouping: 0.2315 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2273 | Regression_eta: 0.0030 | Regression_pt: 0.2295 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0980 | Classification_binary: 0.2730 | Regression_cos_phi: 0.1165 | Regression_energy: 0.0912 | Regression_eta: 0.0797 | Regression_pt: 0.0970 | Regression_sin_phi: 0.1402 | LR: 8.68e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1689s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0535 | max_forward: 0.7747 | max_loss: 0.1534 | max_backward: 1.3339 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6494.1 | uss_MB: 5528.8 | child_rss_MB: 5649.7 | allocated_MB: 347.2 | reserved_MB: 24510.0 | inactive_split_MB: 456.8 +rank2 [2026-10-28 20:08:30,520] INFO : Step 12600/50000 rank2 | Train Loss: 1.4969 | Train Loss Unweighted: 0.8185 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1595 | Classification: 0.1210 | Classification_binary: 0.2357 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2279 | Regression_eta: 0.0024 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.1965 | Regression_cos_phi: 0.0833 | Regression_energy: 0.1272 | Regression_eta: 0.0502 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0853 | LR: 8.66e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7600s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0062 | max_forward: 0.6571 | max_loss: 0.1429 | max_backward: 1.4364 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6482.1 | uss_MB: 5531.3 | child_rss_MB: 5638.2 | allocated_MB: 342.9 | reserved_MB: 25526.0 | inactive_split_MB: 234.6 +rank2 [2026-10-28 20:12:34,582] INFO : Step 12700/50000 rank2 | Train Loss: 1.6234 | Train Loss Unweighted: 0.8661 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.2004 | Classification: 0.1459 | Classification_binary: 0.2387 | HitGrouping: 0.2403 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2360 | Regression_eta: 0.0027 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2212 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0965 | Regression_eta: 0.0673 | Regression_pt: 0.1020 | Regression_sin_phi: 0.1090 | LR: 8.64e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.4779s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6221 | max_loss: 0.1846 | max_backward: 1.1554 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6469.4 | uss_MB: 5531.3 | child_rss_MB: 5648.7 | allocated_MB: 345.9 | reserved_MB: 26286.0 | inactive_split_MB: 241.7 +rank2 [2026-10-28 20:16:22,944] INFO : Step 12800/50000 rank2 | Train Loss: 1.7877 | Train Loss Unweighted: 0.9398 | Train Loss Components: Auxiliary: 0.3651 | Cardinality: 0.2327 | Classification: 0.1495 | Classification_binary: 0.3094 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2357 | Regression_eta: 0.0027 | Regression_pt: 0.2394 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0933 | Classification_binary: 0.3257 | Regression_cos_phi: 0.1222 | Regression_energy: 0.0988 | Regression_eta: 0.0834 | Regression_pt: 0.1039 | Regression_sin_phi: 0.1485 | LR: 8.62e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.2586s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.8468 | max_loss: 0.1534 | max_backward: 1.7420 | max_optimizer_step: 0.3013 | max_tensorboard_logging: 0.5725 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6458.4 | uss_MB: 5535.9 | child_rss_MB: 5651.5 | allocated_MB: 345.6 | reserved_MB: 26286.0 | inactive_split_MB: 284.4 +rank2 [2026-10-28 20:20:13,787] INFO : Step 12900/50000 rank2 | Train Loss: 1.5881 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.3191 | Cardinality: 0.2029 | Classification: 0.1304 | Classification_binary: 0.2564 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2291 | Regression_eta: 0.0023 | Regression_pt: 0.2312 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2573 | Regression_cos_phi: 0.1310 | Regression_energy: 0.0895 | Regression_eta: 0.0677 | Regression_pt: 0.0938 | Regression_sin_phi: 0.1535 | LR: 8.60e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9617s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7180 | max_loss: 0.2380 | max_backward: 1.4488 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6932 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6444.6 | uss_MB: 5531.5 | child_rss_MB: 5646.1 | allocated_MB: 341.8 | reserved_MB: 26286.0 | inactive_split_MB: 252.2 +rank2 [2026-10-28 20:24:06,549] INFO : Step 13000/50000 rank2 | Train Loss: 1.5217 | Train Loss Unweighted: 0.8547 | Train Loss Components: Auxiliary: 0.3141 | Cardinality: 0.1693 | Classification: 0.1382 | Classification_binary: 0.2135 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2470 | Regression_eta: 0.0024 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2280 | Regression_cos_phi: 0.1117 | Regression_energy: 0.1376 | Regression_eta: 0.0779 | Regression_pt: 0.1457 | Regression_sin_phi: 0.0984 | LR: 8.58e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8297s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7101 | max_loss: 0.1763 | max_backward: 1.1845 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 0.8282 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6429.6 | uss_MB: 5536.9 | child_rss_MB: 5653.9 | allocated_MB: 339.8 | reserved_MB: 26286.0 | inactive_split_MB: 213.4 +rank2 [2026-10-28 20:24:07,450] INFO : Running validation on rank2 +rank2 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 20:24:58,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:24:59,078] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 20:25:24,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 20:25:25,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 20:25:26,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 20:25:27,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 20:25:37,817] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 20:25:38,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:25:38,655] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 20:25:54,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 20:25:55,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 20:25:56,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 20:25:57,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 20:26:07,328] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 20:26:08,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:26:09,008] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 20:26:24,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 20:26:25,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 20:26:26,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 20:26:26,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 20:26:36,701] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 20:30:47,349] INFO : Step 13100/50000 rank2 | Train Loss: 1.6168 | Train Loss Unweighted: 0.8649 | Train Loss Components: Auxiliary: 0.3213 | Cardinality: 0.1869 | Classification: 0.1467 | Classification_binary: 0.2696 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2210 | Regression_eta: 0.0027 | Regression_pt: 0.2221 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.2588 | Regression_cos_phi: 0.1586 | Regression_energy: 0.0984 | Regression_eta: 0.0712 | Regression_pt: 0.1012 | Regression_sin_phi: 0.0886 | LR: 8.56e-04 | DataLoad Time: 0.0143s | Model Forward Time: 6.3765s | Phase Times: max_pre_step_barrier: 0.0337 | max_batch_to_device: 0.0136 | max_forward: 1.2842 | max_loss: 0.2583 | max_backward: 3.1281 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.9677 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7302.9 | uss_MB: 6406.9 | child_rss_MB: 5646.9 | allocated_MB: 341.8 | reserved_MB: 22940.0 | inactive_split_MB: 270.2 +rank2 [2026-10-28 20:34:56,294] INFO : Step 13200/50000 rank2 | Train Loss: 1.6996 | Train Loss Unweighted: 0.8855 | Train Loss Components: Auxiliary: 0.3386 | Cardinality: 0.2474 | Classification: 0.1410 | Classification_binary: 0.2769 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2299 | Regression_eta: 0.0027 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2768 | Regression_cos_phi: 0.0812 | Regression_energy: 0.1093 | Regression_eta: 0.0728 | Regression_pt: 0.1147 | Regression_sin_phi: 0.1221 | LR: 8.54e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1567s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0072 | max_forward: 0.7543 | max_loss: 0.1532 | max_backward: 1.3317 | max_optimizer_step: 0.3180 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7300.0 | uss_MB: 6407.2 | child_rss_MB: 5649.5 | allocated_MB: 343.4 | reserved_MB: 23942.0 | inactive_split_MB: 249.6 +rank2 [2026-10-28 20:38:53,368] INFO : Step 13300/50000 rank2 | Train Loss: 1.4876 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2970 | Cardinality: 0.2018 | Classification: 0.1100 | Classification_binary: 0.2192 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2205 | Regression_eta: 0.0026 | Regression_pt: 0.2230 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1714 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0934 | Regression_eta: 0.0495 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0671 | LR: 8.52e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6365s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0259 | max_forward: 0.9208 | max_loss: 0.2245 | max_backward: 1.9523 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6473.0 | uss_MB: 4937.1 | child_rss_MB: 5535.0 | allocated_MB: 341.7 | reserved_MB: 24474.0 | inactive_split_MB: 265.4 +rank2 [2026-10-28 20:42:52,943] INFO : Step 13400/50000 rank2 | Train Loss: 1.4253 | Train Loss Unweighted: 0.7944 | Train Loss Components: Auxiliary: 0.2889 | Cardinality: 0.1585 | Classification: 0.1406 | Classification_binary: 0.2158 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2158 | Regression_eta: 0.0022 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2494 | Regression_cos_phi: 0.0928 | Regression_energy: 0.1057 | Regression_eta: 0.0671 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0995 | LR: 8.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7377s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6533 | max_loss: 0.2500 | max_backward: 1.4458 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.5454 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6474.0 | uss_MB: 5610.2 | child_rss_MB: 5538.3 | allocated_MB: 341.6 | reserved_MB: 25334.0 | inactive_split_MB: 266.6 +rank2 [2026-10-28 20:47:05,652] INFO : Step 13500/50000 rank2 | Train Loss: 1.6021 | Train Loss Unweighted: 0.8749 | Train Loss Components: Auxiliary: 0.3241 | Cardinality: 0.1832 | Classification: 0.1473 | Classification_binary: 0.2801 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2200 | Regression_eta: 0.0024 | Regression_pt: 0.2225 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0951 | Classification_binary: 0.2945 | Regression_cos_phi: 0.1019 | Regression_energy: 0.1129 | Regression_eta: 0.0807 | Regression_pt: 0.1198 | Regression_sin_phi: 0.1222 | LR: 8.48e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.1044s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.7859 | max_loss: 0.3303 | max_backward: 2.5111 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4974 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6412.7 | uss_MB: 5395.8 | child_rss_MB: 5343.5 | allocated_MB: 344.3 | reserved_MB: 25574.0 | inactive_split_MB: 284.1 +rank2 [2026-10-28 20:51:04,211] INFO : Step 13600/50000 rank2 | Train Loss: 1.5296 | Train Loss Unweighted: 0.8114 | Train Loss Components: Auxiliary: 0.3050 | Cardinality: 0.1872 | Classification: 0.1291 | Classification_binary: 0.2450 | HitGrouping: 0.2218 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2158 | Regression_eta: 0.0025 | Regression_pt: 0.2165 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0843 | Classification_binary: 0.2505 | Regression_cos_phi: 0.1594 | Regression_energy: 0.1122 | Regression_eta: 0.0866 | Regression_pt: 0.1167 | Regression_sin_phi: 0.1292 | LR: 8.45e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.6660s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.7508 | max_loss: 0.1507 | max_backward: 1.2891 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6404.7 | uss_MB: 5608.6 | child_rss_MB: 5283.3 | allocated_MB: 345.0 | reserved_MB: 26608.0 | inactive_split_MB: 257.3 +rank2 [2026-10-28 20:55:12,337] INFO : Step 13700/50000 rank2 | Train Loss: 1.4515 | Train Loss Unweighted: 0.7895 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1575 | Classification: 0.1261 | Classification_binary: 0.2121 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2219 | Regression_eta: 0.0023 | Regression_pt: 0.2245 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0821 | Classification_binary: 0.2355 | Regression_cos_phi: 0.1310 | Regression_energy: 0.1146 | Regression_eta: 0.0618 | Regression_pt: 0.1182 | Regression_sin_phi: 0.1045 | LR: 8.43e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.8548s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.6804 | max_loss: 0.2128 | max_backward: 1.1331 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6398.1 | uss_MB: 5609.6 | child_rss_MB: 5245.6 | allocated_MB: 342.5 | reserved_MB: 26608.0 | inactive_split_MB: 241.7 +rank2 [2026-10-28 20:59:19,101] INFO : Step 13800/50000 rank2 | Train Loss: 1.5060 | Train Loss Unweighted: 0.8320 | Train Loss Components: Auxiliary: 0.3084 | Cardinality: 0.1762 | Classification: 0.1368 | Classification_binary: 0.2567 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2156 | Regression_eta: 0.0023 | Regression_pt: 0.2184 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0852 | Classification_binary: 0.2651 | Regression_cos_phi: 0.0883 | Regression_energy: 0.1060 | Regression_eta: 0.0848 | Regression_pt: 0.1097 | Regression_sin_phi: 0.1025 | LR: 8.41e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4129s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7967 | max_loss: 0.1491 | max_backward: 1.5976 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6392.1 | uss_MB: 5613.3 | child_rss_MB: 5216.1 | allocated_MB: 342.1 | reserved_MB: 26608.0 | inactive_split_MB: 241.3 +rank2 [2026-10-28 21:03:21,785] INFO : Step 13900/50000 rank2 | Train Loss: 1.6108 | Train Loss Unweighted: 0.8469 | Train Loss Components: Auxiliary: 0.3204 | Cardinality: 0.1942 | Classification: 0.1303 | Classification_binary: 0.2751 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2168 | Regression_eta: 0.0028 | Regression_pt: 0.2192 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0866 | Classification_binary: 0.3015 | Regression_cos_phi: 0.1071 | Regression_energy: 0.0906 | Regression_eta: 0.0770 | Regression_pt: 0.0937 | Regression_sin_phi: 0.1001 | LR: 8.39e-04 | DataLoad Time: 0.0183s | Model Forward Time: 6.7348s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8010 | max_loss: 0.1552 | max_backward: 2.5526 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 1.2862 | max_tensorboard_flush: 0.0041 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 6389.1 | uss_MB: 5603.5 | child_rss_MB: 5217.8 | allocated_MB: 345.6 | reserved_MB: 26608.0 | inactive_split_MB: 292.8 +rank2 [2026-10-28 21:07:29,378] INFO : Step 14000/50000 rank2 | Train Loss: 1.3659 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1707 | Classification: 0.0974 | Classification_binary: 0.1997 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2149 | Regression_eta: 0.0023 | Regression_pt: 0.2181 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0932 | Regression_eta: 0.0552 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0863 | LR: 8.37e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.4510s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7574 | max_loss: 0.1908 | max_backward: 1.5870 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6393.2 | uss_MB: 4632.1 | child_rss_MB: 5217.4 | allocated_MB: 342.3 | reserved_MB: 26608.0 | inactive_split_MB: 238.4 +rank2 [2026-10-28 21:07:30,352] INFO : Running validation on rank2 +rank2 [2026-10-28 21:07:30,352] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 21:07:30,352] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 21:08:21,198] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:08:21,766] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 21:08:49,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 21:08:50,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 21:08:51,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 21:08:52,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 21:09:02,249] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 21:09:05,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:09:06,443] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 21:09:22,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 21:09:23,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 21:09:24,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 21:09:25,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 21:09:35,902] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 21:09:36,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:09:36,854] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 21:09:52,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 21:09:52,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 21:09:53,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 21:09:54,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 21:10:04,088] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 21:14:16,748] INFO : Step 14100/50000 rank2 | Train Loss: 1.5416 | Train Loss Unweighted: 0.8176 | Train Loss Components: Auxiliary: 0.3012 | Cardinality: 0.1751 | Classification: 0.1363 | Classification_binary: 0.2297 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2177 | Regression_eta: 0.0028 | Regression_pt: 0.2283 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2467 | Regression_cos_phi: 0.1026 | Regression_energy: 0.1047 | Regression_eta: 0.0595 | Regression_pt: 0.1133 | Regression_sin_phi: 0.1035 | LR: 8.35e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.2026s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7718 | max_loss: 0.1710 | max_backward: 1.6376 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 7412.7 | uss_MB: 6170.5 | child_rss_MB: 5197.6 | allocated_MB: 347.1 | reserved_MB: 23970.0 | inactive_split_MB: 244.9 +rank2 [2026-10-28 21:18:17,876] INFO : Step 14200/50000 rank2 | Train Loss: 1.6164 | Train Loss Unweighted: 0.8542 | Train Loss Components: Auxiliary: 0.3190 | Cardinality: 0.1950 | Classification: 0.1478 | Classification_binary: 0.2794 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2094 | Regression_eta: 0.0027 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0890 | Classification_binary: 0.2676 | Regression_cos_phi: 0.1026 | Regression_energy: 0.1032 | Regression_eta: 0.0660 | Regression_pt: 0.1073 | Regression_sin_phi: 0.1123 | LR: 8.32e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.3718s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1153 | max_forward: 0.8788 | max_loss: 0.1615 | max_backward: 1.5286 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7413.4 | uss_MB: 6634.1 | child_rss_MB: 5195.0 | allocated_MB: 344.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.9 +rank2 [2026-10-28 21:22:21,673] INFO : Step 14300/50000 rank2 | Train Loss: 1.4246 | Train Loss Unweighted: 0.7744 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1685 | Classification: 0.1198 | Classification_binary: 0.2357 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2084 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0868 | Classification_binary: 0.2485 | Regression_cos_phi: 0.1065 | Regression_energy: 0.0919 | Regression_eta: 0.0629 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0843 | LR: 8.30e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.8794s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7183 | max_loss: 0.1494 | max_backward: 1.3371 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6454.4 | uss_MB: 5677.1 | child_rss_MB: 5184.2 | allocated_MB: 341.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.1 +rank2 [2026-10-28 21:26:25,336] INFO : Step 14400/50000 rank2 | Train Loss: 1.3539 | Train Loss Unweighted: 0.7267 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1402 | Classification: 0.1303 | Classification_binary: 0.2015 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1957 | Regression_eta: 0.0022 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0791 | Classification_binary: 0.2053 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0838 | Regression_eta: 0.0480 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0959 | LR: 8.28e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6674s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6350 | max_loss: 0.2071 | max_backward: 1.2597 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6458.7 | uss_MB: 5681.9 | child_rss_MB: 5188.8 | allocated_MB: 346.1 | reserved_MB: 24962.0 | inactive_split_MB: 231.6 +rank2 [2026-10-28 21:30:23,756] INFO : Step 14500/50000 rank2 | Train Loss: 1.5651 | Train Loss Unweighted: 0.7967 | Train Loss Components: Auxiliary: 0.3070 | Cardinality: 0.2312 | Classification: 0.1364 | Classification_binary: 0.2565 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1984 | Regression_eta: 0.0026 | Regression_pt: 0.2005 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2387 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0885 | Regression_eta: 0.1478 | Regression_pt: 0.0902 | Regression_sin_phi: 0.1250 | LR: 8.26e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.4674s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0069 | max_forward: 0.8745 | max_loss: 0.1717 | max_backward: 1.8485 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 1.4922 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6443.5 | uss_MB: 5682.8 | child_rss_MB: 5113.1 | allocated_MB: 344.2 | reserved_MB: 25374.0 | inactive_split_MB: 206.9 +rank2 [2026-10-28 21:34:34,948] INFO : Step 14600/50000 rank2 | Train Loss: 1.6485 | Train Loss Unweighted: 0.8413 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.2375 | Classification: 0.1249 | Classification_binary: 0.2902 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2104 | Regression_eta: 0.0023 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2669 | Regression_cos_phi: 0.1087 | Regression_energy: 0.1030 | Regression_eta: 0.0549 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0865 | LR: 8.24e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3713s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0068 | max_forward: 0.7866 | max_loss: 0.1515 | max_backward: 1.5469 | max_optimizer_step: 0.3128 | max_tensorboard_logging: 0.7882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6427.1 | uss_MB: 5683.8 | child_rss_MB: 5049.1 | allocated_MB: 345.7 | reserved_MB: 25374.0 | inactive_split_MB: 292.3 +rank2 [2026-10-28 21:38:39,704] INFO : Step 14700/50000 rank2 | Train Loss: 1.3645 | Train Loss Unweighted: 0.7326 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1507 | Classification: 0.1080 | Classification_binary: 0.2034 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1960 | Regression_cos_phi: 0.0606 | Regression_energy: 0.1022 | Regression_eta: 0.0482 | Regression_pt: 0.1079 | Regression_sin_phi: 0.1008 | LR: 8.21e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9104s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6571 | max_loss: 0.1494 | max_backward: 1.2978 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7751 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0922 | Memory: rss_MB: 6423.6 | uss_MB: 5684.9 | child_rss_MB: 5040.9 | allocated_MB: 341.1 | reserved_MB: 25374.0 | inactive_split_MB: 264.9 +rank2 [2026-10-28 21:42:53,742] INFO : Step 14800/50000 rank2 | Train Loss: 1.3987 | Train Loss Unweighted: 0.7335 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1827 | Classification: 0.1136 | Classification_binary: 0.2244 | HitGrouping: 0.2204 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1949 | Regression_eta: 0.0022 | Regression_pt: 0.1963 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2200 | Regression_cos_phi: 0.0827 | Regression_energy: 0.1127 | Regression_eta: 0.0622 | Regression_pt: 0.1180 | Regression_sin_phi: 0.1022 | LR: 8.19e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.2140s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.1679 | max_loss: 0.2171 | max_backward: 2.8051 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 6420.4 | uss_MB: 4434.9 | child_rss_MB: 5026.4 | allocated_MB: 345.2 | reserved_MB: 25374.0 | inactive_split_MB: 222.5 +rank2 [2026-10-28 21:47:06,022] INFO : Step 14900/50000 rank2 | Train Loss: 1.5046 | Train Loss Unweighted: 0.7991 | Train Loss Components: Auxiliary: 0.3055 | Cardinality: 0.1834 | Classification: 0.1311 | Classification_binary: 0.2643 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1982 | Regression_eta: 0.0023 | Regression_pt: 0.2008 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2483 | Regression_cos_phi: 0.0966 | Regression_energy: 0.0939 | Regression_eta: 0.0625 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0790 | LR: 8.17e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.0348s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0071 | max_forward: 0.7642 | max_loss: 0.1561 | max_backward: 1.5975 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6421.6 | uss_MB: 5685.9 | child_rss_MB: 5030.2 | allocated_MB: 345.4 | reserved_MB: 25374.0 | inactive_split_MB: 289.7 +rank2 [2026-10-28 21:51:16,625] INFO : Step 15000/50000 rank2 | Train Loss: 1.3836 | Train Loss Unweighted: 0.7341 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1614 | Classification: 0.1182 | Classification_binary: 0.2252 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1930 | Regression_eta: 0.0023 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0916 | Classification_binary: 0.2271 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0876 | Regression_eta: 0.0654 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0646 | LR: 8.15e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.7144s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0066 | max_forward: 0.6828 | max_loss: 0.1453 | max_backward: 1.3064 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1064 | Memory: rss_MB: 6423.8 | uss_MB: 5686.3 | child_rss_MB: 5021.1 | allocated_MB: 342.6 | reserved_MB: 25374.0 | inactive_split_MB: 262.7 +rank2 [2026-10-28 21:51:17,569] INFO : Running validation on rank2 +rank2 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 21:52:06,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:52:07,026] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 21:52:33,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 21:52:34,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 21:52:35,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 21:52:36,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 21:52:46,388] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 21:52:46,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:52:47,272] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 21:53:03,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 21:53:04,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 21:53:05,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 21:53:06,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 21:53:16,343] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 21:53:16,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:53:17,339] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 21:53:32,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 21:53:33,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 21:53:33,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 21:53:34,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 21:53:44,625] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 21:57:58,525] INFO : Step 15100/50000 rank2 | Train Loss: 1.3706 | Train Loss Unweighted: 0.7159 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1535 | Classification: 0.1271 | Classification_binary: 0.1954 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1940 | Regression_eta: 0.0022 | Regression_pt: 0.1948 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1807 | Regression_cos_phi: 0.1051 | Regression_energy: 0.0942 | Regression_eta: 0.0465 | Regression_pt: 0.0991 | Regression_sin_phi: 0.1217 | LR: 8.12e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.0257s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.6658 | max_loss: 0.2401 | max_backward: 1.3105 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8277 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7654.5 | uss_MB: 6910.5 | child_rss_MB: 5047.2 | allocated_MB: 341.1 | reserved_MB: 22096.0 | inactive_split_MB: 215.5 +rank2 [2026-10-28 22:01:59,131] INFO : Step 15200/50000 rank2 | Train Loss: 1.5024 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3072 | Cardinality: 0.2080 | Classification: 0.1273 | Classification_binary: 0.2385 | HitGrouping: 0.2164 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1989 | Regression_eta: 0.0022 | Regression_pt: 0.2007 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0923 | Classification_binary: 0.2610 | Regression_cos_phi: 0.0831 | Regression_energy: 0.1002 | Regression_eta: 0.0622 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0875 | LR: 8.10e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.5444s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7502 | max_loss: 0.1534 | max_backward: 1.8002 | max_optimizer_step: 0.2270 | max_tensorboard_logging: 0.8682 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 7649.1 | uss_MB: 6910.3 | child_rss_MB: 5045.3 | allocated_MB: 345.6 | reserved_MB: 22420.0 | inactive_split_MB: 257.7 +rank2 [2026-10-28 22:05:42,233] INFO : Step 15300/50000 rank2 | Train Loss: 1.4920 | Train Loss Unweighted: 0.7880 | Train Loss Components: Auxiliary: 0.3065 | Cardinality: 0.1970 | Classification: 0.1161 | Classification_binary: 0.2589 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2026 | Regression_eta: 0.0022 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2530 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0928 | Regression_eta: 0.0476 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0741 | LR: 8.08e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5786s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0192 | max_forward: 0.7642 | max_loss: 0.1530 | max_backward: 1.6292 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 0.9125 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6524.0 | uss_MB: 5797.3 | child_rss_MB: 5044.3 | allocated_MB: 341.2 | reserved_MB: 23096.0 | inactive_split_MB: 271.7 +rank2 [2026-10-28 22:09:28,150] INFO : Step 15400/50000 rank2 | Train Loss: 1.3549 | Train Loss Unweighted: 0.7210 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1572 | Classification: 0.1148 | Classification_binary: 0.2052 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1983 | Regression_eta: 0.0022 | Regression_pt: 0.1984 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0840 | Classification_binary: 0.2155 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0984 | Regression_eta: 0.0857 | Regression_pt: 0.1007 | Regression_sin_phi: 0.1487 | LR: 8.05e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.7048s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6594 | max_loss: 0.2185 | max_backward: 1.2416 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6511.0 | uss_MB: 5790.6 | child_rss_MB: 5047.2 | allocated_MB: 340.8 | reserved_MB: 23394.0 | inactive_split_MB: 166.6 +rank2 [2026-10-28 22:13:12,490] INFO : Step 15500/50000 rank2 | Train Loss: 1.4313 | Train Loss Unweighted: 0.7527 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1666 | Classification: 0.1321 | Classification_binary: 0.2182 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1979 | Regression_eta: 0.0025 | Regression_pt: 0.1996 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.2236 | Regression_cos_phi: 0.1139 | Regression_energy: 0.0954 | Regression_eta: 0.1010 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0774 | LR: 8.03e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2552s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0792 | max_forward: 0.7874 | max_loss: 0.1514 | max_backward: 1.8676 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5380 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6511.0 | uss_MB: 5805.3 | child_rss_MB: 5052.2 | allocated_MB: 345.6 | reserved_MB: 24092.0 | inactive_split_MB: 260.5 +rank2 [2026-10-28 22:17:01,249] INFO : Step 15600/50000 rank2 | Train Loss: 1.5305 | Train Loss Unweighted: 0.8109 | Train Loss Components: Auxiliary: 0.3137 | Cardinality: 0.1790 | Classification: 0.1304 | Classification_binary: 0.2721 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2011 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.2676 | Regression_cos_phi: 0.0934 | Regression_energy: 0.0908 | Regression_eta: 0.0606 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0980 | LR: 8.01e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2126s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7645 | max_loss: 0.2230 | max_backward: 1.7015 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1398 | Memory: rss_MB: 6508.8 | uss_MB: 5803.4 | child_rss_MB: 5052.3 | allocated_MB: 344.1 | reserved_MB: 24402.0 | inactive_split_MB: 287.9 +rank2 [2026-10-28 22:20:46,546] INFO : Step 15700/50000 rank2 | Train Loss: 1.4083 | Train Loss Unweighted: 0.7205 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1775 | Classification: 0.1238 | Classification_binary: 0.2322 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1785 | Regression_eta: 0.0024 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1836 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0810 | Regression_eta: 0.0413 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0604 | LR: 7.98e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.0311s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2392 | max_backward: 1.6606 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6506.8 | uss_MB: 5802.2 | child_rss_MB: 5055.5 | allocated_MB: 344.4 | reserved_MB: 25380.0 | inactive_split_MB: 228.2 +rank2 [2026-10-28 22:24:39,044] INFO : Step 15800/50000 rank2 | Train Loss: 1.2988 | Train Loss Unweighted: 0.6968 | Train Loss Components: Auxiliary: 0.2671 | Cardinality: 0.1436 | Classification: 0.1112 | Classification_binary: 0.1916 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1933 | Regression_eta: 0.0020 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0974 | Regression_eta: 0.0492 | Regression_pt: 0.1043 | Regression_sin_phi: 0.0827 | LR: 7.96e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7268s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6109 | max_loss: 0.2015 | max_backward: 1.1291 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6506.5 | uss_MB: 5803.6 | child_rss_MB: 5051.3 | allocated_MB: 341.6 | reserved_MB: 25380.0 | inactive_split_MB: 263.9 +rank2 [2026-10-28 22:28:35,067] INFO : Step 15900/50000 rank2 | Train Loss: 1.4020 | Train Loss Unweighted: 0.7322 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1800 | Classification: 0.1295 | Classification_binary: 0.2346 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1816 | Regression_eta: 0.0023 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0794 | Classification_binary: 0.2502 | Regression_cos_phi: 0.0992 | Regression_energy: 0.1146 | Regression_eta: 0.0613 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0750 | LR: 7.94e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2709s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0933 | max_forward: 0.7823 | max_loss: 0.1445 | max_backward: 1.4740 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6508.5 | uss_MB: 5807.0 | child_rss_MB: 5047.2 | allocated_MB: 342.9 | reserved_MB: 25380.0 | inactive_split_MB: 283.5 +rank2 [2026-10-28 22:32:40,676] INFO : Step 16000/50000 rank2 | Train Loss: 1.4978 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3126 | Cardinality: 0.2093 | Classification: 0.1190 | Classification_binary: 0.2580 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1920 | Regression_eta: 0.0021 | Regression_pt: 0.1969 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0895 | Classification_binary: 0.2912 | Regression_cos_phi: 0.0964 | Regression_energy: 0.0843 | Regression_eta: 0.0606 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0934 | LR: 7.91e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4905s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.7474 | max_loss: 0.1892 | max_backward: 1.4114 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.7981 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6509.3 | uss_MB: 5675.5 | child_rss_MB: 5048.3 | allocated_MB: 342.7 | reserved_MB: 25380.0 | inactive_split_MB: 292.9 +rank2 [2026-10-28 22:32:41,708] INFO : Running validation on rank2 +rank2 [2026-10-28 22:32:41,709] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 22:32:41,709] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 22:33:34,621] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:33:35,151] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 22:34:01,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 22:34:02,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 22:34:03,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 22:34:04,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 22:34:14,056] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 22:34:14,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:34:14,939] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 22:34:31,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 22:34:32,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 22:34:33,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 22:34:34,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 22:34:44,155] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 22:34:44,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:34:44,854] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 22:35:00,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 22:35:01,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 22:35:01,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 22:35:02,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 22:35:12,364] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 22:39:22,135] INFO : Step 16100/50000 rank2 | Train Loss: 1.3441 | Train Loss Unweighted: 0.7192 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1631 | Classification: 0.1197 | Classification_binary: 0.2058 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1931 | Regression_eta: 0.0020 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.1862 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0951 | Regression_eta: 0.0543 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0777 | LR: 7.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1523s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0584 | max_forward: 0.7257 | max_loss: 0.1421 | max_backward: 1.2590 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.0197 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6577.1 | uss_MB: 5871.7 | child_rss_MB: 5053.6 | allocated_MB: 341.3 | reserved_MB: 23342.0 | inactive_split_MB: 221.6 +rank2 [2026-10-28 22:43:09,261] INFO : Step 16200/50000 rank2 | Train Loss: 1.3737 | Train Loss Unweighted: 0.7229 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1695 | Classification: 0.1213 | Classification_binary: 0.2167 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0913 | Regression_eta: 0.0579 | Regression_pt: 0.0960 | Regression_sin_phi: 0.1265 | LR: 7.86e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.0530s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.1062 | max_forward: 0.7366 | max_loss: 0.1469 | max_backward: 1.4999 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.6015 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6580.7 | uss_MB: 5882.6 | child_rss_MB: 5053.8 | allocated_MB: 342.5 | reserved_MB: 24304.0 | inactive_split_MB: 205.9 +rank2 [2026-10-28 22:46:54,662] INFO : Step 16300/50000 rank2 | Train Loss: 1.4671 | Train Loss Unweighted: 0.7592 | Train Loss Components: Auxiliary: 0.3008 | Cardinality: 0.1775 | Classification: 0.1342 | Classification_binary: 0.2702 | HitGrouping: 0.2148 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0023 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0896 | Classification_binary: 0.2884 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0805 | Regression_eta: 0.0598 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0910 | LR: 7.84e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4597s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7718 | max_loss: 0.1628 | max_backward: 1.7017 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.8513 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6574.7 | uss_MB: 5880.8 | child_rss_MB: 5053.1 | allocated_MB: 345.8 | reserved_MB: 24304.0 | inactive_split_MB: 281.0 +rank2 [2026-10-28 22:50:39,547] INFO : Step 16400/50000 rank2 | Train Loss: 1.4200 | Train Loss Unweighted: 0.7161 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.2086 | Classification: 0.1206 | Classification_binary: 0.2158 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0024 | Regression_pt: 0.1885 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.1923 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0798 | Regression_eta: 0.0589 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0619 | LR: 7.81e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.8036s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7570 | max_loss: 0.1546 | max_backward: 1.2568 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5199 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1436 | Memory: rss_MB: 6571.7 | uss_MB: 5879.8 | child_rss_MB: 5049.7 | allocated_MB: 342.7 | reserved_MB: 25268.0 | inactive_split_MB: 213.6 +rank2 [2026-10-28 22:54:25,076] INFO : Step 16500/50000 rank2 | Train Loss: 1.2201 | Train Loss Unweighted: 0.6592 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1195 | Classification: 0.1029 | Classification_binary: 0.1835 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1836 | Regression_eta: 0.0021 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0839 | Classification_binary: 0.1930 | Regression_cos_phi: 0.1046 | Regression_energy: 0.0896 | Regression_eta: 0.0649 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0835 | LR: 7.79e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3185s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0128 | max_forward: 0.7866 | max_loss: 0.2200 | max_backward: 1.9590 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.4913 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6572.2 | uss_MB: 5885.7 | child_rss_MB: 5046.8 | allocated_MB: 343.6 | reserved_MB: 26238.0 | inactive_split_MB: 248.8 +rank2 [2026-10-28 22:58:25,288] INFO : Step 16600/50000 rank2 | Train Loss: 1.4335 | Train Loss Unweighted: 0.7657 | Train Loss Components: Auxiliary: 0.2949 | Cardinality: 0.1698 | Classification: 0.1386 | Classification_binary: 0.2312 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1941 | Regression_eta: 0.0021 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0899 | Classification_binary: 0.2312 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0943 | Regression_eta: 0.0581 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0774 | LR: 7.77e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.9983s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0139 | max_forward: 1.0329 | max_loss: 0.3279 | max_backward: 2.5489 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.5244 | max_tensorboard_flush: 0.0014 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6575.3 | uss_MB: 4925.2 | child_rss_MB: 5045.0 | allocated_MB: 344.2 | reserved_MB: 27228.0 | inactive_split_MB: 219.6 +rank2 [2026-10-28 23:02:38,763] INFO : Step 16700/50000 rank2 | Train Loss: 1.4062 | Train Loss Unweighted: 0.7378 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1840 | Classification: 0.1203 | Classification_binary: 0.2439 | HitGrouping: 0.2108 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1839 | Regression_eta: 0.0020 | Regression_pt: 0.1855 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.2145 | Regression_cos_phi: 0.1072 | Regression_energy: 0.1012 | Regression_eta: 0.0510 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0633 | LR: 7.74e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0460s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7336 | max_loss: 0.2460 | max_backward: 1.6747 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6577.4 | uss_MB: 5888.0 | child_rss_MB: 5049.7 | allocated_MB: 344.6 | reserved_MB: 27228.0 | inactive_split_MB: 279.4 +rank2 [2026-10-28 23:06:47,051] INFO : Step 16800/50000 rank2 | Train Loss: 1.3172 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1533 | Classification: 0.1173 | Classification_binary: 0.1949 | HitGrouping: 0.2175 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1819 | Regression_eta: 0.0027 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2015 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0967 | Regression_eta: 0.0628 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0616 | LR: 7.72e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.6850s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0962 | max_forward: 0.7772 | max_loss: 0.1423 | max_backward: 1.3121 | max_optimizer_step: 0.2117 | max_tensorboard_logging: 0.4865 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6580.1 | uss_MB: 5886.7 | child_rss_MB: 5043.1 | allocated_MB: 342.1 | reserved_MB: 27250.0 | inactive_split_MB: 200.8 +rank2 [2026-10-28 23:11:05,116] INFO : Step 16900/50000 rank2 | Train Loss: 1.3132 | Train Loss Unweighted: 0.6868 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1474 | Classification: 0.1260 | Classification_binary: 0.1988 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1779 | Regression_eta: 0.0021 | Regression_pt: 0.1798 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0927 | Classification_binary: 0.2188 | Regression_cos_phi: 0.1020 | Regression_energy: 0.0833 | Regression_eta: 0.0673 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0897 | LR: 7.69e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.7616s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7203 | max_loss: 0.1660 | max_backward: 1.1458 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6581.3 | uss_MB: 5889.6 | child_rss_MB: 5044.4 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 193.9 +rank2 [2026-10-28 23:15:09,202] INFO : Step 17000/50000 rank2 | Train Loss: 1.3575 | Train Loss Unweighted: 0.7115 | Train Loss Components: Auxiliary: 0.2906 | Cardinality: 0.1781 | Classification: 0.1179 | Classification_binary: 0.2325 | HitGrouping: 0.2089 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0018 | Regression_pt: 0.1792 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0991 | Regression_eta: 0.0371 | Regression_pt: 0.1043 | Regression_sin_phi: 0.0694 | LR: 7.67e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4157s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.9008 | max_loss: 0.1474 | max_backward: 1.9229 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6575.6 | uss_MB: 5894.0 | child_rss_MB: 5025.6 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 278.9 +rank2 [2026-10-28 23:15:10,041] INFO : Running validation on rank2 +rank2 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 23:16:00,503] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:16:01,059] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 23:16:27,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 23:16:28,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 23:16:29,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 23:16:30,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 23:16:40,803] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 23:16:42,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:16:42,522] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 23:16:58,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 23:16:59,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 23:17:00,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 23:17:01,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 23:17:11,274] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 23:17:11,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:17:12,023] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 23:17:27,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 23:17:28,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 23:17:28,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 23:17:29,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 23:17:39,516] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-28 23:21:59,502] INFO : Step 17100/50000 rank2 | Train Loss: 1.3790 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1704 | Classification: 0.1078 | Classification_binary: 0.2095 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1968 | Regression_eta: 0.0020 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0880 | Regression_eta: 0.0549 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0929 | LR: 7.64e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.9058s | Phase Times: max_pre_step_barrier: 0.0234 | max_batch_to_device: 0.0065 | max_forward: 0.8292 | max_loss: 0.1455 | max_backward: 1.4833 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6666.8 | uss_MB: 5307.9 | child_rss_MB: 5009.6 | allocated_MB: 343.8 | reserved_MB: 22688.0 | inactive_split_MB: 254.6 +rank2 [2026-10-28 23:26:06,862] INFO : Step 17200/50000 rank2 | Train Loss: 1.2786 | Train Loss Unweighted: 0.6598 | Train Loss Components: Auxiliary: 0.2576 | Cardinality: 0.1378 | Classification: 0.1161 | Classification_binary: 0.1910 | HitGrouping: 0.2191 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0025 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.1690 | Regression_cos_phi: 0.1015 | Regression_energy: 0.0814 | Regression_eta: 0.0698 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0809 | LR: 7.62e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8628s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6470 | max_loss: 0.2505 | max_backward: 1.4380 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6320 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6664.1 | uss_MB: 5982.7 | child_rss_MB: 5028.0 | allocated_MB: 344.1 | reserved_MB: 22688.0 | inactive_split_MB: 295.9 +rank2 [2026-10-28 23:30:05,515] INFO : Step 17300/50000 rank2 | Train Loss: 1.3571 | Train Loss Unweighted: 0.6976 | Train Loss Components: Auxiliary: 0.2789 | Cardinality: 0.1665 | Classification: 0.1193 | Classification_binary: 0.2287 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1724 | Regression_eta: 0.0021 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2328 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0849 | Regression_eta: 0.0376 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0759 | LR: 7.59e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5430s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7276 | max_loss: 0.2520 | max_backward: 1.8579 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8525 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6666.1 | uss_MB: 5121.5 | child_rss_MB: 5026.9 | allocated_MB: 344.2 | reserved_MB: 23616.0 | inactive_split_MB: 266.5 +rank2 [2026-10-28 23:34:25,421] INFO : Step 17400/50000 rank2 | Train Loss: 1.3668 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2811 | Cardinality: 0.1760 | Classification: 0.1195 | Classification_binary: 0.2351 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1732 | Regression_eta: 0.0019 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0850 | Regression_eta: 0.0449 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0766 | LR: 7.57e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.4871s | Phase Times: max_pre_step_barrier: 0.1038 | max_batch_to_device: 0.0306 | max_forward: 0.7782 | max_loss: 0.2429 | max_backward: 1.3743 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.8348 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0757 | Memory: rss_MB: 6667.1 | uss_MB: 5984.6 | child_rss_MB: 5026.4 | allocated_MB: 343.3 | reserved_MB: 24604.0 | inactive_split_MB: 262.4 +rank2 [2026-10-28 23:38:51,401] INFO : Step 17500/50000 rank2 | Train Loss: 1.3114 | Train Loss Unweighted: 0.6526 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.1481 | Classification: 0.1145 | Classification_binary: 0.1884 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1714 | Regression_eta: 0.0024 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0794 | Classification_binary: 0.2113 | Regression_cos_phi: 0.2671 | Regression_energy: 0.0847 | Regression_eta: 0.1420 | Regression_pt: 0.0892 | Regression_sin_phi: 0.1893 | LR: 7.54e-04 | DataLoad Time: 0.0224s | Model Forward Time: 3.9141s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.6394 | max_loss: 0.1427 | max_backward: 1.2079 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.8375 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 6667.1 | uss_MB: 5986.2 | child_rss_MB: 5023.0 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 240.0 +rank2 [2026-10-28 23:43:03,145] INFO : Step 17600/50000 rank2 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2688 | Cardinality: 0.1362 | Classification: 0.1273 | Classification_binary: 0.2065 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0019 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2167 | Regression_cos_phi: 0.1439 | Regression_energy: 0.0812 | Regression_eta: 0.0734 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0772 | LR: 7.52e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.6080s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.1120 | max_forward: 1.2883 | max_loss: 0.2624 | max_backward: 1.5380 | max_optimizer_step: 0.2147 | max_tensorboard_logging: 0.7644 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 6665.9 | uss_MB: 5985.5 | child_rss_MB: 5028.5 | allocated_MB: 344.2 | reserved_MB: 25596.0 | inactive_split_MB: 237.9 +rank2 [2026-10-28 23:47:02,149] INFO : Step 17700/50000 rank2 | Train Loss: 1.3491 | Train Loss Unweighted: 0.6939 | Train Loss Components: Auxiliary: 0.2835 | Cardinality: 0.1622 | Classification: 0.1186 | Classification_binary: 0.2310 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0022 | Regression_pt: 0.1701 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.2512 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0826 | Regression_eta: 0.0812 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0974 | LR: 7.49e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.6542s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0704 | max_forward: 0.9841 | max_loss: 0.1994 | max_backward: 1.8911 | max_optimizer_step: 0.2227 | max_tensorboard_logging: 0.5849 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6667.9 | uss_MB: 5989.0 | child_rss_MB: 5031.2 | allocated_MB: 345.5 | reserved_MB: 25596.0 | inactive_split_MB: 287.6 +rank2 [2026-10-28 23:51:17,534] INFO : Step 17800/50000 rank2 | Train Loss: 1.3063 | Train Loss Unweighted: 0.6692 | Train Loss Components: Auxiliary: 0.2664 | Cardinality: 0.1579 | Classification: 0.1042 | Classification_binary: 0.2130 | HitGrouping: 0.2150 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1741 | Regression_eta: 0.0019 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.2028 | Regression_cos_phi: 0.0992 | Regression_energy: 0.0989 | Regression_eta: 0.0521 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0745 | LR: 7.47e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8701s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.7351 | max_loss: 0.1686 | max_backward: 1.4616 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5023 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 6666.0 | uss_MB: 5988.8 | child_rss_MB: 5032.9 | allocated_MB: 341.0 | reserved_MB: 25596.0 | inactive_split_MB: 197.8 +rank2 [2026-10-28 23:55:22,790] INFO : Step 17900/50000 rank2 | Train Loss: 1.2213 | Train Loss Unweighted: 0.6433 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1427 | Classification: 0.1147 | Classification_binary: 0.1725 | HitGrouping: 0.2070 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.1761 | Regression_cos_phi: 0.0924 | Regression_energy: 0.0804 | Regression_eta: 0.0502 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0570 | LR: 7.44e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8176s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7249 | max_loss: 0.1579 | max_backward: 1.1981 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6654.3 | uss_MB: 5990.9 | child_rss_MB: 5033.3 | allocated_MB: 341.6 | reserved_MB: 25596.0 | inactive_split_MB: 297.9 +rank2 [2026-10-28 23:59:20,282] INFO : Step 18000/50000 rank2 | Train Loss: 1.3597 | Train Loss Unweighted: 0.7083 | Train Loss Components: Auxiliary: 0.2781 | Cardinality: 0.1599 | Classification: 0.1247 | Classification_binary: 0.2275 | HitGrouping: 0.2122 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0021 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.2285 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0865 | Regression_eta: 0.0486 | Regression_pt: 0.0895 | Regression_sin_phi: 0.1180 | LR: 7.42e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6426s | Phase Times: max_pre_step_barrier: 0.0816 | max_batch_to_device: 0.0068 | max_forward: 0.8980 | max_loss: 0.1586 | max_backward: 1.9185 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.7546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6654.3 | uss_MB: 5995.0 | child_rss_MB: 5035.3 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 254.8 +rank2 [2026-10-28 23:59:21,239] INFO : Running validation on rank2 +rank2 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 00:00:12,209] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:00:12,738] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 00:00:39,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 00:00:40,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 00:00:41,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 00:00:42,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 00:00:52,215] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 00:00:52,803] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:00:53,359] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 00:01:09,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 00:01:10,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 00:01:11,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 00:01:12,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 00:01:22,052] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 00:01:22,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:01:23,292] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 00:01:38,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 00:01:39,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 00:01:40,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 00:01:40,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 00:01:50,911] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 00:05:57,253] INFO : Step 18100/50000 rank2 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7216 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1666 | Classification: 0.1199 | Classification_binary: 0.2340 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1807 | Regression_eta: 0.0024 | Regression_pt: 0.1824 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2175 | Regression_cos_phi: 0.0594 | Regression_energy: 0.1096 | Regression_eta: 0.0723 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0518 | LR: 7.39e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.5279s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.9127 | max_loss: 0.4546 | max_backward: 1.7406 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.7463 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7789.9 | uss_MB: 7130.3 | child_rss_MB: 5031.5 | allocated_MB: 344.6 | reserved_MB: 22256.0 | inactive_split_MB: 291.4 +rank2 [2026-10-29 00:09:47,529] INFO : Step 18200/50000 rank2 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6468 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1359 | Classification: 0.1208 | Classification_binary: 0.1803 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1700 | Regression_eta: 0.0020 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0938 | Regression_energy: 0.0827 | Regression_eta: 0.0459 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0686 | LR: 7.36e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7601s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7114 | max_loss: 0.2369 | max_backward: 1.1051 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6784.1 | uss_MB: 6119.0 | child_rss_MB: 5031.9 | allocated_MB: 341.2 | reserved_MB: 23772.0 | inactive_split_MB: 170.7 +rank2 [2026-10-29 00:13:32,739] INFO : Step 18300/50000 rank2 | Train Loss: 1.3100 | Train Loss Unweighted: 0.6891 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1522 | Classification: 0.1230 | Classification_binary: 0.1938 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1834 | Regression_eta: 0.0019 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.1885 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0801 | Regression_eta: 0.0538 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0870 | LR: 7.34e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.1476s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7219 | max_loss: 0.1491 | max_backward: 1.5825 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.7390 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 6786.1 | uss_MB: 6124.1 | child_rss_MB: 5031.6 | allocated_MB: 344.4 | reserved_MB: 24778.0 | inactive_split_MB: 234.4 +rank2 [2026-10-29 00:17:16,847] INFO : Step 18400/50000 rank2 | Train Loss: 1.3667 | Train Loss Unweighted: 0.7337 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1448 | Classification: 0.1277 | Classification_binary: 0.2393 | HitGrouping: 0.2095 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1805 | Regression_eta: 0.0022 | Regression_pt: 0.1818 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0892 | Classification_binary: 0.2576 | Regression_cos_phi: 0.0942 | Regression_energy: 0.1018 | Regression_eta: 0.0706 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0913 | LR: 7.31e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4873s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1744 | max_backward: 1.7308 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6782.0 | uss_MB: 6123.4 | child_rss_MB: 5027.3 | allocated_MB: 344.6 | reserved_MB: 24778.0 | inactive_split_MB: 281.4 +rank2 [2026-10-29 00:21:03,776] INFO : Step 18500/50000 rank2 | Train Loss: 1.2528 | Train Loss Unweighted: 0.6816 | Train Loss Components: Auxiliary: 0.2631 | Cardinality: 0.1350 | Classification: 0.1140 | Classification_binary: 0.1973 | HitGrouping: 0.2066 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1821 | Regression_eta: 0.0017 | Regression_pt: 0.1847 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0862 | Regression_eta: 0.0366 | Regression_pt: 0.0900 | Regression_sin_phi: 0.0699 | LR: 7.29e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.2638s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0113 | max_forward: 0.8281 | max_loss: 0.2418 | max_backward: 1.4683 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7561 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1431 | Memory: rss_MB: 6783.0 | uss_MB: 6122.8 | child_rss_MB: 5026.7 | allocated_MB: 343.3 | reserved_MB: 24778.0 | inactive_split_MB: 226.5 +rank2 [2026-10-29 00:24:44,691] INFO : Step 18600/50000 rank2 | Train Loss: 1.1881 | Train Loss Unweighted: 0.6335 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1331 | Classification: 0.1084 | Classification_binary: 0.1723 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1746 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0941 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0741 | Regression_eta: 0.0445 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0755 | LR: 7.26e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8519s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7299 | max_loss: 0.2279 | max_backward: 1.0799 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8080 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6782.4 | uss_MB: 6120.8 | child_rss_MB: 5022.4 | allocated_MB: 341.0 | reserved_MB: 24778.0 | inactive_split_MB: 229.3 +rank2 [2026-10-29 00:28:24,928] INFO : Step 18700/50000 rank2 | Train Loss: 1.3122 | Train Loss Unweighted: 0.6534 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1756 | Classification: 0.1225 | Classification_binary: 0.2098 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1571 | Regression_eta: 0.0020 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0921 | Classification_binary: 0.2028 | Regression_cos_phi: 0.1040 | Regression_energy: 0.0772 | Regression_eta: 0.0936 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0719 | LR: 7.24e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.6252s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9129 | max_loss: 0.1606 | max_backward: 1.9397 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6785.3 | uss_MB: 6128.9 | child_rss_MB: 5021.8 | allocated_MB: 344.8 | reserved_MB: 24778.0 | inactive_split_MB: 239.8 +rank2 [2026-10-29 00:32:05,525] INFO : Step 18800/50000 rank2 | Train Loss: 1.2696 | Train Loss Unweighted: 0.6643 | Train Loss Components: Auxiliary: 0.2674 | Cardinality: 0.1602 | Classification: 0.1070 | Classification_binary: 0.2200 | HitGrouping: 0.2058 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1652 | Regression_eta: 0.0018 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2289 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0772 | Regression_eta: 0.0523 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0682 | LR: 7.21e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.6487s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7444 | max_loss: 0.2527 | max_backward: 1.7167 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.9324 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 6785.1 | uss_MB: 6124.1 | child_rss_MB: 5021.8 | allocated_MB: 345.4 | reserved_MB: 25044.0 | inactive_split_MB: 284.4 +rank2 [2026-10-29 00:35:49,544] INFO : Step 18900/50000 rank2 | Train Loss: 1.1718 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1382 | Classification: 0.0996 | Classification_binary: 0.1741 | HitGrouping: 0.2036 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1652 | Regression_eta: 0.0017 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0584 | Regression_eta: 0.0325 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0544 | LR: 7.18e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0763s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0056 | max_forward: 0.6830 | max_loss: 0.1421 | max_backward: 1.2812 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.8931 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6786.1 | uss_MB: 6126.4 | child_rss_MB: 5006.2 | allocated_MB: 338.7 | reserved_MB: 25044.0 | inactive_split_MB: 232.6 +rank2 [2026-10-29 00:39:31,234] INFO : Step 19000/50000 rank2 | Train Loss: 1.2397 | Train Loss Unweighted: 0.6317 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1277 | Classification: 0.1188 | Classification_binary: 0.1866 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1600 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2019 | Regression_cos_phi: 0.1145 | Regression_energy: 0.0759 | Regression_eta: 0.0599 | Regression_pt: 0.0783 | Regression_sin_phi: 0.1011 | LR: 7.16e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.2166s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6593 | max_loss: 0.2467 | max_backward: 1.6203 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7732 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6787.7 | uss_MB: 6128.3 | child_rss_MB: 5026.8 | allocated_MB: 342.2 | reserved_MB: 25790.0 | inactive_split_MB: 229.6 +rank2 [2026-10-29 00:39:32,174] INFO : Running validation on rank2 +rank2 [2026-10-29 00:39:32,175] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 00:39:32,175] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 00:40:23,806] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:40:24,368] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 00:40:50,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 00:40:51,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 00:40:52,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 00:40:53,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 00:41:03,113] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 00:41:03,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:41:04,088] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 00:41:20,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 00:41:21,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 00:41:21,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 00:41:22,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 00:41:32,916] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 00:41:33,291] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:41:33,791] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 00:41:49,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 00:41:50,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 00:41:50,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 00:41:51,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 00:42:01,475] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 00:46:01,996] INFO : Step 19100/50000 rank2 | Train Loss: 1.3562 | Train Loss Unweighted: 0.6933 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1644 | Classification: 0.1204 | Classification_binary: 0.2426 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1627 | Regression_eta: 0.0023 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.2368 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0854 | Regression_eta: 0.0436 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0791 | LR: 7.13e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.5915s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7796 | max_loss: 0.1634 | max_backward: 1.6942 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.8232 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 8027.6 | uss_MB: 7365.9 | child_rss_MB: 5027.2 | allocated_MB: 345.2 | reserved_MB: 23900.0 | inactive_split_MB: 240.8 +rank2 [2026-10-29 00:49:48,721] INFO : Step 19200/50000 rank2 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5946 | Train Loss Components: Auxiliary: 0.2384 | Cardinality: 0.1391 | Classification: 0.0965 | Classification_binary: 0.1984 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1472 | Regression_eta: 0.0018 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0692 | Regression_eta: 0.0635 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0600 | LR: 7.10e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.9563s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7138 | max_loss: 0.2391 | max_backward: 1.2379 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.7395 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6876.1 | uss_MB: 6216.1 | child_rss_MB: 5025.0 | allocated_MB: 341.3 | reserved_MB: 23900.0 | inactive_split_MB: 206.8 +rank2 [2026-10-29 00:53:30,061] INFO : Step 19300/50000 rank2 | Train Loss: 1.1768 | Train Loss Unweighted: 0.6083 | Train Loss Components: Auxiliary: 0.2422 | Cardinality: 0.1345 | Classification: 0.1035 | Classification_binary: 0.1709 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0018 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2188 | Regression_cos_phi: 0.1213 | Regression_energy: 0.0800 | Regression_eta: 0.0742 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0944 | LR: 7.08e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.8976s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0503 | max_forward: 0.7163 | max_loss: 0.1442 | max_backward: 1.3745 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6878.1 | uss_MB: 6217.4 | child_rss_MB: 5020.2 | allocated_MB: 341.0 | reserved_MB: 23900.0 | inactive_split_MB: 218.1 +rank2 [2026-10-29 00:57:11,121] INFO : Step 19400/50000 rank2 | Train Loss: 1.4590 | Train Loss Unweighted: 0.7109 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.2328 | Classification: 0.1237 | Classification_binary: 0.2265 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1770 | Regression_eta: 0.0021 | Regression_pt: 0.1794 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2483 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0889 | Regression_eta: 0.0628 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0980 | LR: 7.05e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.4252s | Phase Times: max_pre_step_barrier: 0.1118 | max_batch_to_device: 0.0070 | max_forward: 0.7578 | max_loss: 0.2459 | max_backward: 1.7975 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6876.3 | uss_MB: 6217.1 | child_rss_MB: 5016.1 | allocated_MB: 347.7 | reserved_MB: 24186.0 | inactive_split_MB: 276.3 +rank2 [2026-10-29 01:00:48,549] INFO : Step 19500/50000 rank2 | Train Loss: 1.3753 | Train Loss Unweighted: 0.6794 | Train Loss Components: Auxiliary: 0.2810 | Cardinality: 0.1865 | Classification: 0.1153 | Classification_binary: 0.2310 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1637 | Regression_eta: 0.0023 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.2282 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0727 | Regression_eta: 0.0657 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0876 | LR: 7.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8979s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7197 | max_loss: 0.2164 | max_backward: 1.3937 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.4987 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6876.3 | uss_MB: 6219.5 | child_rss_MB: 5017.7 | allocated_MB: 345.2 | reserved_MB: 24186.0 | inactive_split_MB: 275.2 +rank2 [2026-10-29 01:04:42,200] INFO : Step 19600/50000 rank2 | Train Loss: 1.2086 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1557 | Classification: 0.0942 | Classification_binary: 0.1768 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1735 | Regression_eta: 0.0015 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0856 | Regression_eta: 0.0317 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0695 | LR: 7.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7336s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6937 | max_loss: 0.1942 | max_backward: 1.2727 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6877.2 | uss_MB: 6219.0 | child_rss_MB: 5004.9 | allocated_MB: 340.6 | reserved_MB: 24186.0 | inactive_split_MB: 277.9 +rank2 [2026-10-29 01:08:29,168] INFO : Step 19700/50000 rank2 | Train Loss: 1.2115 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1478 | Classification: 0.1094 | Classification_binary: 0.1919 | HitGrouping: 0.2027 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0802 | Regression_eta: 0.0769 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0738 | LR: 6.97e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8669s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0914 | max_forward: 0.7091 | max_loss: 0.1429 | max_backward: 1.4681 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0652 | Memory: rss_MB: 6879.2 | uss_MB: 6222.3 | child_rss_MB: 5016.1 | allocated_MB: 344.0 | reserved_MB: 24630.0 | inactive_split_MB: 143.8 +rank2 [2026-10-29 01:12:36,832] INFO : Step 19800/50000 rank2 | Train Loss: 1.2919 | Train Loss Unweighted: 0.6584 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1453 | Classification: 0.1132 | Classification_binary: 0.2375 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0020 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2484 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0668 | Regression_eta: 0.0565 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0799 | LR: 6.94e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.4528s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0074 | max_forward: 0.7828 | max_loss: 0.2508 | max_backward: 1.8604 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6879.1 | uss_MB: 6223.6 | child_rss_MB: 5015.7 | allocated_MB: 348.0 | reserved_MB: 25634.0 | inactive_split_MB: 277.9 +rank2 [2026-10-29 01:16:27,498] INFO : Step 19900/50000 rank2 | Train Loss: 1.2010 | Train Loss Unweighted: 0.6167 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1346 | Classification: 0.0885 | Classification_binary: 0.1961 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1633 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.2091 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0756 | Regression_eta: 0.0424 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0554 | LR: 6.92e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9747s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6671 | max_loss: 0.2548 | max_backward: 1.5862 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6881.1 | uss_MB: 6224.6 | child_rss_MB: 5016.9 | allocated_MB: 340.3 | reserved_MB: 25634.0 | inactive_split_MB: 201.7 +rank2 [2026-10-29 01:20:16,835] INFO : Step 20000/50000 rank2 | Train Loss: 1.2077 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1487 | Classification: 0.1030 | Classification_binary: 0.1729 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1639 | Regression_eta: 0.0019 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1857 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0964 | Regression_eta: 0.0608 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0562 | LR: 6.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8144s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.6570 | max_loss: 0.1550 | max_backward: 1.4848 | max_optimizer_step: 0.2205 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6884.1 | uss_MB: 6226.8 | child_rss_MB: 5009.1 | allocated_MB: 341.8 | reserved_MB: 26642.0 | inactive_split_MB: 237.7 +rank2 [2026-10-29 01:20:17,835] INFO : Running validation on rank2 +rank2 [2026-10-29 01:20:17,836] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 01:20:17,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 01:21:07,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:21:08,246] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 01:21:34,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 01:21:35,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 01:21:36,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 01:21:37,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 01:21:47,796] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 01:21:48,206] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:21:48,734] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 01:22:04,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 01:22:05,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 01:22:06,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 01:22:07,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 01:22:17,792] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 01:22:18,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:22:18,870] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 01:22:34,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 01:22:34,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 01:22:35,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 01:22:36,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 01:22:46,313] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 01:26:57,040] INFO : Step 20100/50000 rank2 | Train Loss: 1.3825 | Train Loss Unweighted: 0.6739 | Train Loss Components: Auxiliary: 0.2830 | Cardinality: 0.1854 | Classification: 0.1189 | Classification_binary: 0.2349 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1561 | Regression_eta: 0.0022 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2216 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0832 | Regression_eta: 0.0452 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0708 | LR: 6.86e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2457s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.9496 | max_loss: 0.1965 | max_backward: 1.6425 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.5117 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1294 | Memory: rss_MB: 8120.6 | uss_MB: 7458.3 | child_rss_MB: 5010.7 | allocated_MB: 344.8 | reserved_MB: 21328.0 | inactive_split_MB: 198.7 +rank2 [2026-10-29 01:30:57,230] INFO : Step 20200/50000 rank2 | Train Loss: 1.3220 | Train Loss Unweighted: 0.6624 | Train Loss Components: Auxiliary: 0.2728 | Cardinality: 0.1545 | Classification: 0.1109 | Classification_binary: 0.2232 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1618 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.2487 | Regression_cos_phi: 0.1326 | Regression_energy: 0.0816 | Regression_eta: 0.0592 | Regression_pt: 0.0840 | Regression_sin_phi: 0.1426 | LR: 6.84e-04 | DataLoad Time: 0.0144s | Model Forward Time: 3.8640s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.9337 | max_loss: 0.2287 | max_backward: 1.4619 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6969.7 | uss_MB: 6305.7 | child_rss_MB: 5014.0 | allocated_MB: 344.4 | reserved_MB: 21328.0 | inactive_split_MB: 287.4 +rank2 [2026-10-29 01:34:46,196] INFO : Step 20300/50000 rank2 | Train Loss: 1.2742 | Train Loss Unweighted: 0.6629 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1615 | Classification: 0.1132 | Classification_binary: 0.1937 | HitGrouping: 0.2129 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1761 | Regression_eta: 0.0018 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1971 | Regression_cos_phi: 0.0998 | Regression_energy: 0.0825 | Regression_eta: 0.0512 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0811 | LR: 6.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8215s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6857 | max_loss: 0.2423 | max_backward: 1.2822 | max_optimizer_step: 0.2837 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0419 | Memory: rss_MB: 6967.2 | uss_MB: 6303.7 | child_rss_MB: 5019.2 | allocated_MB: 340.8 | reserved_MB: 21826.0 | inactive_split_MB: 239.4 +rank2 [2026-10-29 01:38:43,604] INFO : Step 20400/50000 rank2 | Train Loss: 1.2743 | Train Loss Unweighted: 0.6521 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1507 | Classification: 0.1279 | Classification_binary: 0.1990 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1604 | Regression_eta: 0.0019 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2358 | Regression_cos_phi: 0.0932 | Regression_energy: 0.0763 | Regression_eta: 0.0654 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0892 | LR: 6.78e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.7464s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0283 | max_forward: 1.3648 | max_loss: 0.1816 | max_backward: 1.7875 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 6969.3 | uss_MB: 6306.4 | child_rss_MB: 5010.2 | allocated_MB: 343.0 | reserved_MB: 22154.0 | inactive_split_MB: 243.5 +rank2 [2026-10-29 01:42:48,353] INFO : Step 20500/50000 rank2 | Train Loss: 1.2712 | Train Loss Unweighted: 0.6509 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1409 | Classification: 0.1147 | Classification_binary: 0.2202 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1558 | Regression_eta: 0.0019 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.2191 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0583 | Regression_eta: 0.0525 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0630 | LR: 6.76e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.2568s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7205 | max_loss: 0.2283 | max_backward: 1.6623 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.6254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0704 | Memory: rss_MB: 6969.0 | uss_MB: 6306.9 | child_rss_MB: 5000.2 | allocated_MB: 345.0 | reserved_MB: 22154.0 | inactive_split_MB: 208.2 +rank2 [2026-10-29 01:46:45,622] INFO : Step 20600/50000 rank2 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6338 | Train Loss Components: Auxiliary: 0.2625 | Cardinality: 0.1623 | Classification: 0.1101 | Classification_binary: 0.1900 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0018 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1784 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0831 | Regression_eta: 0.0454 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0543 | LR: 6.73e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.9334s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 1.1520 | max_loss: 0.2849 | max_backward: 2.1471 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.6141 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6963.6 | uss_MB: 6301.3 | child_rss_MB: 5002.5 | allocated_MB: 341.4 | reserved_MB: 22154.0 | inactive_split_MB: 183.5 +rank2 [2026-10-29 01:50:54,179] INFO : Step 20700/50000 rank2 | Train Loss: 1.1613 | Train Loss Unweighted: 0.6066 | Train Loss Components: Auxiliary: 0.2395 | Cardinality: 0.1327 | Classification: 0.1061 | Classification_binary: 0.1677 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0016 | Regression_pt: 0.1666 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 0.1564 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0803 | Regression_eta: 0.0374 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0792 | LR: 6.70e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5593s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6276 | max_loss: 0.2330 | max_backward: 1.4954 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6965.5 | uss_MB: 5936.8 | child_rss_MB: 5004.9 | allocated_MB: 341.0 | reserved_MB: 22154.0 | inactive_split_MB: 285.0 +rank2 [2026-10-29 01:55:01,769] INFO : Step 20800/50000 rank2 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1493 | Classification: 0.1205 | Classification_binary: 0.2020 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1841 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0820 | Regression_eta: 0.0371 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0686 | LR: 6.67e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2110s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.7679 | max_loss: 0.1604 | max_backward: 1.7694 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.4914 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6972.0 | uss_MB: 6311.7 | child_rss_MB: 5027.1 | allocated_MB: 344.1 | reserved_MB: 22154.0 | inactive_split_MB: 258.7 +rank2 [2026-10-29 01:59:06,393] INFO : Step 20900/50000 rank2 | Train Loss: 1.2934 | Train Loss Unweighted: 0.6525 | Train Loss Components: Auxiliary: 0.2719 | Cardinality: 0.1620 | Classification: 0.1167 | Classification_binary: 0.2121 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1591 | Regression_eta: 0.0021 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0881 | Classification_binary: 0.2214 | Regression_cos_phi: 0.0882 | Regression_energy: 0.0867 | Regression_eta: 0.0831 | Regression_pt: 0.0900 | Regression_sin_phi: 0.1203 | LR: 6.65e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1335s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7787 | max_loss: 0.1747 | max_backward: 1.6678 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 6974.0 | uss_MB: 5772.6 | child_rss_MB: 5037.2 | allocated_MB: 343.6 | reserved_MB: 23208.0 | inactive_split_MB: 296.4 +rank2 [2026-10-29 02:03:08,876] INFO : Step 21000/50000 rank2 | Train Loss: 1.1402 | Train Loss Unweighted: 0.5874 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1236 | Classification: 0.0967 | Classification_binary: 0.1701 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1583 | Regression_eta: 0.0017 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1617 | Regression_cos_phi: 0.1085 | Regression_energy: 0.0705 | Regression_eta: 0.0480 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0684 | LR: 6.62e-04 | DataLoad Time: 0.0183s | Model Forward Time: 3.8973s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6629 | max_loss: 0.1833 | max_backward: 1.4689 | max_optimizer_step: 0.2206 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6973.2 | uss_MB: 6313.8 | child_rss_MB: 5014.5 | allocated_MB: 340.4 | reserved_MB: 23208.0 | inactive_split_MB: 264.1 +rank2 [2026-10-29 02:03:09,907] INFO : Running validation on rank2 +rank2 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 02:04:00,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:04:01,076] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 02:04:27,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 02:04:28,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 02:04:29,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 02:04:30,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 02:04:40,268] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 02:04:40,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:04:41,351] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 02:04:57,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 02:04:58,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 02:04:58,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 02:04:59,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 02:05:09,876] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 02:05:10,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:05:11,157] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 02:05:27,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 02:05:27,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 02:05:28,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 02:05:29,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 02:05:39,410] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 02:10:06,179] INFO : Step 21100/50000 rank2 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6632 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1574 | Classification: 0.1277 | Classification_binary: 0.2026 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0024 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0856 | Classification_binary: 0.2180 | Regression_cos_phi: 0.1176 | Regression_energy: 0.0691 | Regression_eta: 0.0726 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0859 | LR: 6.59e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2639s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7159 | max_loss: 0.1411 | max_backward: 1.5805 | max_optimizer_step: 0.3396 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8247.8 | uss_MB: 7584.2 | child_rss_MB: 5046.5 | allocated_MB: 343.8 | reserved_MB: 21372.0 | inactive_split_MB: 255.4 +rank2 [2026-10-29 02:10:38,255] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank2 [2026-10-29 02:10:38,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-29 02:14:15,483] INFO : Step 21200/50000 rank2 | Train Loss: 1.2354 | Train Loss Unweighted: 0.6366 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1422 | Classification: 0.1056 | Classification_binary: 0.1876 | HitGrouping: 0.2030 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0017 | Regression_pt: 0.1709 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0760 | Regression_eta: 0.0379 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0944 | LR: 6.56e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.1581s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7385 | max_loss: 0.2183 | max_backward: 1.4276 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.7483 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7137.9 | uss_MB: 6469.9 | child_rss_MB: 5019.5 | allocated_MB: 342.5 | reserved_MB: 23286.0 | inactive_split_MB: 273.0 +rank2 [2026-10-29 02:18:17,991] INFO : Step 21300/50000 rank2 | Train Loss: 1.1511 | Train Loss Unweighted: 0.5782 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1240 | Classification: 0.1021 | Classification_binary: 0.1706 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1505 | Regression_eta: 0.0017 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1519 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0716 | Regression_eta: 0.0718 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0536 | LR: 6.54e-04 | DataLoad Time: 0.0228s | Model Forward Time: 4.4920s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0290 | max_forward: 1.3238 | max_loss: 0.2105 | max_backward: 1.6672 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.6878 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7140.4 | uss_MB: 6475.6 | child_rss_MB: 5012.2 | allocated_MB: 343.8 | reserved_MB: 23286.0 | inactive_split_MB: 272.8 +rank2 [2026-10-29 02:22:14,590] INFO : Step 21400/50000 rank2 | Train Loss: 1.2993 | Train Loss Unweighted: 0.6458 | Train Loss Components: Auxiliary: 0.2724 | Cardinality: 0.1796 | Classification: 0.1204 | Classification_binary: 0.2158 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1518 | Regression_eta: 0.0018 | Regression_pt: 0.1542 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2334 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0764 | Regression_eta: 0.0556 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0632 | LR: 6.51e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.5459s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8696 | max_loss: 0.1572 | max_backward: 1.8002 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 0.6992 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7114.7 | uss_MB: 6081.0 | child_rss_MB: 5015.7 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 201.9 +rank2 [2026-10-29 02:26:14,445] INFO : Step 21500/50000 rank2 | Train Loss: 1.2839 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2693 | Cardinality: 0.1478 | Classification: 0.1058 | Classification_binary: 0.2162 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2148 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0689 | Regression_eta: 0.0680 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0820 | LR: 6.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7758s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.8684 | max_loss: 0.1577 | max_backward: 1.9355 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.7593 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0648 | Memory: rss_MB: 7121.8 | uss_MB: 6458.0 | child_rss_MB: 5014.4 | allocated_MB: 341.9 | reserved_MB: 25248.0 | inactive_split_MB: 284.1 +rank2 [2026-10-29 02:30:00,713] INFO : Step 21600/50000 rank2 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1290 | Classification: 0.1062 | Classification_binary: 0.1618 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1486 | Regression_eta: 0.0018 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.1571 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0788 | Regression_eta: 0.0533 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0684 | LR: 6.45e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.9657s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0061 | max_forward: 0.7663 | max_loss: 0.1883 | max_backward: 1.4257 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.4997 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7120.6 | uss_MB: 6456.9 | child_rss_MB: 5032.2 | allocated_MB: 340.5 | reserved_MB: 25248.0 | inactive_split_MB: 234.8 +rank2 [2026-10-29 02:33:57,426] INFO : Step 21700/50000 rank2 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6283 | Train Loss Components: Auxiliary: 0.2577 | Cardinality: 0.1575 | Classification: 0.1186 | Classification_binary: 0.1853 | HitGrouping: 0.2068 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1594 | Regression_eta: 0.0022 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1948 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0820 | Regression_eta: 0.0793 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0645 | LR: 6.43e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7320s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.6796 | max_loss: 0.1417 | max_backward: 1.3881 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7122.2 | uss_MB: 6458.0 | child_rss_MB: 5034.2 | allocated_MB: 343.8 | reserved_MB: 25248.0 | inactive_split_MB: 244.5 +rank2 [2026-10-29 02:37:54,449] INFO : Step 21800/50000 rank2 | Train Loss: 1.2260 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2691 | Cardinality: 0.1439 | Classification: 0.1105 | Classification_binary: 0.2043 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1576 | Regression_eta: 0.0016 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1840 | Regression_cos_phi: 0.0938 | Regression_energy: 0.0759 | Regression_eta: 0.0412 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0741 | LR: 6.40e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.9309s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 0.8043 | max_loss: 0.1895 | max_backward: 2.4027 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.9442 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7124.3 | uss_MB: 5827.3 | child_rss_MB: 5037.0 | allocated_MB: 344.5 | reserved_MB: 25248.0 | inactive_split_MB: 277.8 +rank2 [2026-10-29 02:41:42,512] INFO : Step 21900/50000 rank2 | Train Loss: 1.2119 | Train Loss Unweighted: 0.5952 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1861 | Classification: 0.1033 | Classification_binary: 0.1865 | HitGrouping: 0.1989 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0015 | Regression_pt: 0.1511 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0695 | Regression_eta: 0.0360 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0473 | LR: 6.37e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5504s | Phase Times: max_pre_step_barrier: 0.0174 | max_batch_to_device: 0.0163 | max_forward: 0.9684 | max_loss: 0.1448 | max_backward: 1.6354 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.8990 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 7124.6 | uss_MB: 6460.4 | child_rss_MB: 5033.8 | allocated_MB: 342.7 | reserved_MB: 25248.0 | inactive_split_MB: 192.3 +rank2 [2026-10-29 02:45:41,732] INFO : Step 22000/50000 rank2 | Train Loss: 1.1094 | Train Loss Unweighted: 0.5651 | Train Loss Components: Auxiliary: 0.2344 | Cardinality: 0.1248 | Classification: 0.1037 | Classification_binary: 0.1579 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1497 | Regression_eta: 0.0017 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1749 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0722 | Regression_eta: 0.0417 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0490 | LR: 6.34e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.4217s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.6825 | max_loss: 0.1653 | max_backward: 1.4396 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.0256 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 7124.1 | uss_MB: 6463.7 | child_rss_MB: 5009.1 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 247.3 +rank2 [2026-10-29 02:45:42,811] INFO : Running validation on rank2 +rank2 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 02:46:37,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:46:38,045] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 02:46:57,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 02:46:58,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 02:46:59,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 02:47:00,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 02:47:10,893] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 02:47:11,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:47:11,961] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 02:47:27,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 02:47:28,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 02:47:29,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 02:47:30,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 02:47:40,823] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 02:47:41,460] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:47:42,031] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 02:47:57,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 02:47:58,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 02:47:59,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 02:47:59,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 02:48:09,955] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 02:52:24,844] INFO : Step 22100/50000 rank2 | Train Loss: 1.2514 | Train Loss Unweighted: 0.6219 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1530 | Classification: 0.1140 | Classification_binary: 0.2076 | HitGrouping: 0.2031 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1470 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2147 | Regression_cos_phi: 0.0840 | Regression_energy: 0.0752 | Regression_eta: 0.0753 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0913 | LR: 6.32e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5403s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7250 | max_loss: 0.2514 | max_backward: 1.5956 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.8632 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0587 | Memory: rss_MB: 8032.9 | uss_MB: 7365.3 | child_rss_MB: 5019.4 | allocated_MB: 347.3 | reserved_MB: 23386.0 | inactive_split_MB: 278.6 +rank2 [2026-10-29 02:56:29,084] INFO : Step 22200/50000 rank2 | Train Loss: 1.2727 | Train Loss Unweighted: 0.6485 | Train Loss Components: Auxiliary: 0.2709 | Cardinality: 0.1498 | Classification: 0.1104 | Classification_binary: 0.2184 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1574 | Regression_eta: 0.0018 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.2116 | Regression_cos_phi: 0.1082 | Regression_energy: 0.0754 | Regression_eta: 0.0610 | Regression_pt: 0.0792 | Regression_sin_phi: 0.1075 | LR: 6.29e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.9305s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0074 | max_forward: 0.7776 | max_loss: 0.1472 | max_backward: 1.7604 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.3305 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7158.4 | uss_MB: 5092.5 | child_rss_MB: 5016.1 | allocated_MB: 342.3 | reserved_MB: 23578.0 | inactive_split_MB: 277.9 +rank2 [2026-10-29 03:00:35,125] INFO : Step 22300/50000 rank2 | Train Loss: 1.1242 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2347 | Cardinality: 0.1367 | Classification: 0.1013 | Classification_binary: 0.1688 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.1764 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0909 | Regression_eta: 0.0463 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0624 | LR: 6.26e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.7802s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7697 | max_loss: 0.1538 | max_backward: 1.3553 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.4896 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 7162.9 | uss_MB: 6498.9 | child_rss_MB: 4987.3 | allocated_MB: 341.2 | reserved_MB: 23578.0 | inactive_split_MB: 233.8 +rank2 [2026-10-29 03:04:32,773] INFO : Step 22400/50000 rank2 | Train Loss: 1.2357 | Train Loss Unweighted: 0.6255 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1493 | Classification: 0.1171 | Classification_binary: 0.1885 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1575 | Regression_eta: 0.0021 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1946 | Regression_cos_phi: 0.0321 | Regression_energy: 0.0617 | Regression_eta: 0.0542 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0871 | LR: 6.23e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9148s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0067 | max_forward: 0.6425 | max_loss: 0.1530 | max_backward: 1.4136 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.5876 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 7158.4 | uss_MB: 6496.8 | child_rss_MB: 5021.9 | allocated_MB: 342.5 | reserved_MB: 23578.0 | inactive_split_MB: 251.5 +rank2 [2026-10-29 03:08:26,684] INFO : Step 22500/50000 rank2 | Train Loss: 1.2633 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2638 | Cardinality: 0.1676 | Classification: 0.0954 | Classification_binary: 0.2181 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0020 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.2376 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0764 | Regression_eta: 0.0648 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0981 | LR: 6.20e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.4280s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1522 | max_backward: 1.5631 | max_optimizer_step: 0.3261 | max_tensorboard_logging: 0.7552 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 7160.2 | uss_MB: 6502.4 | child_rss_MB: 5011.7 | allocated_MB: 342.9 | reserved_MB: 24052.0 | inactive_split_MB: 295.1 +rank2 [2026-10-29 03:12:18,754] INFO : Step 22600/50000 rank2 | Train Loss: 1.1636 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1329 | Classification: 0.0997 | Classification_binary: 0.1889 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1493 | Regression_eta: 0.0020 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1771 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0759 | Regression_eta: 0.0450 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0616 | LR: 6.18e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9320s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7375 | max_loss: 0.2403 | max_backward: 1.4584 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7163.5 | uss_MB: 6503.3 | child_rss_MB: 5024.4 | allocated_MB: 344.2 | reserved_MB: 24590.0 | inactive_split_MB: 288.5 +rank2 [2026-10-29 03:16:20,064] INFO : Step 22700/50000 rank2 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1212 | Classification: 0.0976 | Classification_binary: 0.1566 | HitGrouping: 0.2101 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1455 | Regression_eta: 0.0015 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0864 | Regression_eta: 0.0409 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0431 | LR: 6.15e-04 | DataLoad Time: 0.0166s | Model Forward Time: 6.9686s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.6124 | max_loss: 0.1376 | max_backward: 1.1766 | max_optimizer_step: 0.2237 | max_tensorboard_logging: 3.8568 | max_tensorboard_flush: 0.0169 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 7164.2 | uss_MB: 6160.1 | child_rss_MB: 5007.8 | allocated_MB: 341.6 | reserved_MB: 25058.0 | inactive_split_MB: 256.4 +rank2 [2026-10-29 03:20:29,740] INFO : Step 22800/50000 rank2 | Train Loss: 1.1840 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2533 | Cardinality: 0.1443 | Classification: 0.1191 | Classification_binary: 0.1924 | HitGrouping: 0.1959 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1444 | Regression_eta: 0.0016 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1764 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0776 | Regression_eta: 0.0341 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0650 | LR: 6.12e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.4542s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7010 | max_loss: 0.2312 | max_backward: 1.6853 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8726 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 7164.4 | uss_MB: 6502.8 | child_rss_MB: 5021.2 | allocated_MB: 341.7 | reserved_MB: 25058.0 | inactive_split_MB: 281.3 +rank2 [2026-10-29 03:24:21,161] INFO : Step 22900/50000 rank2 | Train Loss: 1.2680 | Train Loss Unweighted: 0.6318 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1792 | Classification: 0.1020 | Classification_binary: 0.2074 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1580 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2279 | Regression_cos_phi: 0.0923 | Regression_energy: 0.0665 | Regression_eta: 0.0433 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0569 | LR: 6.09e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.9994s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 1.1637 | max_loss: 0.4031 | max_backward: 1.8442 | max_optimizer_step: 0.3320 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7165.6 | uss_MB: 6506.2 | child_rss_MB: 5023.4 | allocated_MB: 344.8 | reserved_MB: 25058.0 | inactive_split_MB: 300.1 +rank2 [2026-10-29 03:28:05,592] INFO : Step 23000/50000 rank2 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5484 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1129 | Classification: 0.0836 | Classification_binary: 0.1519 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1547 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1223 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0640 | Regression_eta: 0.0500 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0465 | LR: 6.06e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.7523s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0063 | max_forward: 0.6963 | max_loss: 0.2382 | max_backward: 1.2870 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 7162.6 | uss_MB: 6506.3 | child_rss_MB: 5008.7 | allocated_MB: 340.3 | reserved_MB: 26066.0 | inactive_split_MB: 218.4 +rank2 [2026-10-29 03:28:06,534] INFO : Running validation on rank2 +rank2 [2026-10-29 03:28:06,534] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 03:28:06,535] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 03:28:56,134] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:28:56,640] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 03:29:22,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 03:29:23,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 03:29:24,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 03:29:25,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 03:29:35,362] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 03:29:35,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:29:36,304] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 03:29:52,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 03:29:53,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 03:29:54,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 03:29:55,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 03:30:05,124] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 03:30:05,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:30:05,702] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 03:30:21,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 03:30:21,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 03:30:22,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 03:30:23,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 03:30:33,127] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 03:34:45,381] INFO : Step 23100/50000 rank2 | Train Loss: 1.2422 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1379 | Classification: 0.1246 | Classification_binary: 0.1795 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1482 | Regression_eta: 0.0023 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.1906 | Regression_cos_phi: 0.0881 | Regression_energy: 0.0612 | Regression_eta: 0.0639 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0787 | LR: 6.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8251s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7203 | max_loss: 0.1522 | max_backward: 1.1000 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.6183 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 8419.6 | uss_MB: 7758.7 | child_rss_MB: 5039.2 | allocated_MB: 344.7 | reserved_MB: 22598.0 | inactive_split_MB: 244.5 +rank2 [2026-10-29 03:38:33,525] INFO : Step 23200/50000 rank2 | Train Loss: 1.2507 | Train Loss Unweighted: 0.6359 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1400 | Classification: 0.1121 | Classification_binary: 0.2208 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1488 | Regression_eta: 0.0020 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2387 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0732 | Regression_eta: 0.0439 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0819 | LR: 6.01e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.5977s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0727 | max_forward: 0.9486 | max_loss: 0.1744 | max_backward: 1.7031 | max_optimizer_step: 0.2405 | max_tensorboard_logging: 0.6717 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1109 | Memory: rss_MB: 7225.3 | uss_MB: 6564.6 | child_rss_MB: 5046.8 | allocated_MB: 343.7 | reserved_MB: 23792.0 | inactive_split_MB: 223.2 +rank2 [2026-10-29 03:42:24,564] INFO : Step 23300/50000 rank2 | Train Loss: 1.1880 | Train Loss Unweighted: 0.5927 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1464 | Classification: 0.0987 | Classification_binary: 0.1851 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1524 | Regression_eta: 0.0019 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.1883 | Regression_cos_phi: 0.1193 | Regression_energy: 0.0816 | Regression_eta: 0.0919 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0854 | LR: 5.98e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8596s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7557 | max_loss: 0.1467 | max_backward: 1.4708 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.4854 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 7231.3 | uss_MB: 6572.4 | child_rss_MB: 5045.1 | allocated_MB: 342.6 | reserved_MB: 24534.0 | inactive_split_MB: 271.6 +rank2 [2026-10-29 03:46:11,296] INFO : Step 23400/50000 rank2 | Train Loss: 1.1136 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2402 | Cardinality: 0.1202 | Classification: 0.1070 | Classification_binary: 0.1497 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1644 | Regression_eta: 0.0017 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0749 | Regression_eta: 0.0303 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0732 | LR: 5.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.6666s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.5743 | max_loss: 0.2297 | max_backward: 1.0127 | max_optimizer_step: 0.3321 | max_tensorboard_logging: 0.6182 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 7232.3 | uss_MB: 6571.7 | child_rss_MB: 5046.9 | allocated_MB: 340.5 | reserved_MB: 24848.0 | inactive_split_MB: 241.3 +rank2 [2026-10-29 03:50:07,301] INFO : Step 23500/50000 rank2 | Train Loss: 1.1601 | Train Loss Unweighted: 0.5799 | Train Loss Components: Auxiliary: 0.2477 | Cardinality: 0.1161 | Classification: 0.1057 | Classification_binary: 0.1852 | HitGrouping: 0.2029 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1416 | Regression_eta: 0.0017 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2013 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0734 | Regression_eta: 0.0390 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0810 | LR: 5.92e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4418s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8864 | max_loss: 0.1573 | max_backward: 1.9036 | max_optimizer_step: 0.2294 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7232.1 | uss_MB: 6574.4 | child_rss_MB: 5036.9 | allocated_MB: 341.9 | reserved_MB: 24848.0 | inactive_split_MB: 233.9 +rank2 [2026-10-29 03:53:52,163] INFO : Step 23600/50000 rank2 | Train Loss: 1.2402 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1447 | Classification: 0.1136 | Classification_binary: 0.2049 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1534 | Regression_eta: 0.0020 | Regression_pt: 0.1550 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1947 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0745 | Regression_eta: 0.0592 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0685 | LR: 5.90e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.9442s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8397 | max_loss: 0.1470 | max_backward: 1.4395 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7234.1 | uss_MB: 6575.7 | child_rss_MB: 5045.6 | allocated_MB: 342.5 | reserved_MB: 24848.0 | inactive_split_MB: 215.5 +rank2 [2026-10-29 03:57:35,779] INFO : Step 23700/50000 rank2 | Train Loss: 1.1009 | Train Loss Unweighted: 0.5682 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1218 | Classification: 0.0954 | Classification_binary: 0.1599 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1528 | Regression_eta: 0.0014 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.1622 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0793 | Regression_eta: 0.0447 | Regression_pt: 0.0863 | Regression_sin_phi: 0.1202 | LR: 5.87e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6819s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6589 | max_loss: 0.1387 | max_backward: 1.2152 | max_optimizer_step: 0.2251 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7230.0 | uss_MB: 6574.3 | child_rss_MB: 5051.4 | allocated_MB: 342.5 | reserved_MB: 25508.0 | inactive_split_MB: 226.3 +rank2 [2026-10-29 04:01:35,999] INFO : Step 23800/50000 rank2 | Train Loss: 1.1390 | Train Loss Unweighted: 0.5882 | Train Loss Components: Auxiliary: 0.2427 | Cardinality: 0.1196 | Classification: 0.1035 | Classification_binary: 0.1735 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1525 | Regression_cos_phi: 0.1037 | Regression_energy: 0.0609 | Regression_eta: 0.0439 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0888 | LR: 5.84e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.7620s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.7283 | max_loss: 0.1547 | max_backward: 1.2415 | max_optimizer_step: 0.2259 | max_tensorboard_logging: 0.5270 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7231.0 | uss_MB: 6110.2 | child_rss_MB: 5050.1 | allocated_MB: 345.2 | reserved_MB: 25508.0 | inactive_split_MB: 241.8 +rank2 [2026-10-29 04:05:42,354] INFO : Step 23900/50000 rank2 | Train Loss: 1.2349 | Train Loss Unweighted: 0.6078 | Train Loss Components: Auxiliary: 0.2621 | Cardinality: 0.1596 | Classification: 0.1044 | Classification_binary: 0.2077 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1451 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2179 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0604 | Regression_eta: 0.0471 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0861 | LR: 5.81e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4870s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.9077 | max_loss: 0.1538 | max_backward: 1.9088 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7234.7 | uss_MB: 6579.6 | child_rss_MB: 5044.4 | allocated_MB: 342.1 | reserved_MB: 27474.0 | inactive_split_MB: 282.4 +rank2 [2026-10-29 04:09:40,154] INFO : Step 24000/50000 rank2 | Train Loss: 1.1074 | Train Loss Unweighted: 0.5714 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1170 | Classification: 0.1007 | Classification_binary: 0.1792 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1431 | Regression_eta: 0.0015 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1726 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0612 | Regression_eta: 0.0489 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0521 | LR: 5.78e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.2194s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 1.1428 | max_loss: 0.2613 | max_backward: 2.5767 | max_optimizer_step: 0.2298 | max_tensorboard_logging: 0.5998 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7236.7 | uss_MB: 6577.2 | child_rss_MB: 5038.1 | allocated_MB: 342.1 | reserved_MB: 27476.0 | inactive_split_MB: 218.6 +rank2 [2026-10-29 04:09:41,047] INFO : Running validation on rank2 +rank2 [2026-10-29 04:09:41,047] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 04:09:41,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 04:10:38,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:10:39,336] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 04:11:05,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 04:11:06,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 04:11:08,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 04:11:10,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 04:11:20,502] INFO : Time taken to make predictions on device 2 is: 0.29 min +rank2 [2026-10-29 04:11:20,671] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:11:21,254] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 04:11:36,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 04:11:37,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 04:11:38,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 04:11:39,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 04:11:49,640] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 04:11:50,532] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:11:51,026] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 04:12:06,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 04:12:07,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 04:12:08,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 04:12:08,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 04:12:18,700] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 04:16:38,288] INFO : Step 24100/50000 rank2 | Train Loss: 1.1244 | Train Loss Unweighted: 0.5584 | Train Loss Components: Auxiliary: 0.2373 | Cardinality: 0.1285 | Classification: 0.0978 | Classification_binary: 0.1606 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1723 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0674 | Regression_eta: 0.0537 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0830 | LR: 5.75e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1999s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0860 | max_forward: 0.7361 | max_loss: 0.1432 | max_backward: 1.4268 | max_optimizer_step: 0.3451 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 8228.7 | uss_MB: 7570.1 | child_rss_MB: 5001.0 | allocated_MB: 343.2 | reserved_MB: 21608.0 | inactive_split_MB: 269.9 +rank2 [2026-10-29 04:20:36,776] INFO : Step 24200/50000 rank2 | Train Loss: 1.3067 | Train Loss Unweighted: 0.6341 | Train Loss Components: Auxiliary: 0.2739 | Cardinality: 0.1760 | Classification: 0.1145 | Classification_binary: 0.2101 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0018 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2178 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0774 | Regression_eta: 0.0454 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0964 | LR: 5.73e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.0792s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1066 | max_forward: 1.0531 | max_loss: 0.2796 | max_backward: 2.3820 | max_optimizer_step: 0.2441 | max_tensorboard_logging: 0.6187 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 8233.6 | uss_MB: 7574.6 | child_rss_MB: 5010.3 | allocated_MB: 345.0 | reserved_MB: 23864.0 | inactive_split_MB: 203.0 +rank2 [2026-10-29 04:24:33,010] INFO : Step 24300/50000 rank2 | Train Loss: 1.2207 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1582 | Classification: 0.1151 | Classification_binary: 0.2013 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1450 | Regression_eta: 0.0017 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1974 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0735 | Regression_eta: 0.0422 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0834 | LR: 5.70e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3063s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7395 | max_loss: 0.2490 | max_backward: 1.4969 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.6555 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7355.3 | uss_MB: 6697.1 | child_rss_MB: 5007.1 | allocated_MB: 344.8 | reserved_MB: 23864.0 | inactive_split_MB: 284.8 +rank2 [2026-10-29 04:28:19,173] INFO : Step 24400/50000 rank2 | Train Loss: 1.1288 | Train Loss Unweighted: 0.5921 | Train Loss Components: Auxiliary: 0.2439 | Cardinality: 0.1229 | Classification: 0.1017 | Classification_binary: 0.1613 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1631 | Regression_eta: 0.0015 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.1422 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0713 | Regression_eta: 0.0312 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0571 | LR: 5.67e-04 | DataLoad Time: 0.0193s | Model Forward Time: 3.9871s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.6685 | max_loss: 0.1564 | max_backward: 1.4736 | max_optimizer_step: 0.2223 | max_tensorboard_logging: 0.5369 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7357.1 | uss_MB: 6698.7 | child_rss_MB: 5016.5 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 267.2 +rank2 [2026-10-29 04:32:03,440] INFO : Step 24500/50000 rank2 | Train Loss: 1.2319 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2586 | Cardinality: 0.1511 | Classification: 0.1107 | Classification_binary: 0.1876 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1540 | Regression_eta: 0.0018 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0663 | Regression_eta: 0.0446 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0616 | LR: 5.64e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1624s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0677 | max_forward: 0.7219 | max_loss: 0.1439 | max_backward: 1.4253 | max_optimizer_step: 0.3416 | max_tensorboard_logging: 0.5859 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 7356.5 | uss_MB: 6698.2 | child_rss_MB: 5010.5 | allocated_MB: 343.1 | reserved_MB: 23864.0 | inactive_split_MB: 275.8 +rank2 [2026-10-29 04:36:01,552] INFO : Step 24600/50000 rank2 | Train Loss: 1.2474 | Train Loss Unweighted: 0.6241 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1630 | Classification: 0.1074 | Classification_binary: 0.2102 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2198 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0686 | Regression_eta: 0.0553 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0940 | LR: 5.61e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5134s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7876 | max_loss: 0.1528 | max_backward: 1.7437 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.6204 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7358.2 | uss_MB: 6700.7 | child_rss_MB: 5006.8 | allocated_MB: 344.3 | reserved_MB: 23864.0 | inactive_split_MB: 285.7 +rank2 [2026-10-29 04:39:51,298] INFO : Step 24700/50000 rank2 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1193 | Classification: 0.0990 | Classification_binary: 0.1723 | HitGrouping: 0.2019 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1471 | Regression_eta: 0.0016 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1769 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0689 | Regression_eta: 0.0303 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0609 | LR: 5.58e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9516s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6978 | max_loss: 0.2493 | max_backward: 1.2925 | max_optimizer_step: 0.2314 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7361.3 | uss_MB: 6700.3 | child_rss_MB: 5009.8 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 223.9 +rank2 [2026-10-29 04:43:37,807] INFO : Step 24800/50000 rank2 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5554 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1264 | Classification: 0.0932 | Classification_binary: 0.1583 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0630 | Regression_eta: 0.0514 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0607 | LR: 5.56e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2768s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6582 | max_loss: 0.2533 | max_backward: 1.5858 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.6923 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1086 | Memory: rss_MB: 7365.2 | uss_MB: 6703.3 | child_rss_MB: 5005.7 | allocated_MB: 342.9 | reserved_MB: 23864.0 | inactive_split_MB: 278.6 +rank2 [2026-10-29 04:47:36,372] INFO : Step 24900/50000 rank2 | Train Loss: 1.1348 | Train Loss Unweighted: 0.5816 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1381 | Classification: 0.1126 | Classification_binary: 0.1845 | HitGrouping: 0.1937 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1403 | Regression_eta: 0.0015 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0655 | Regression_eta: 0.0369 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0583 | LR: 5.53e-04 | DataLoad Time: 0.0152s | Model Forward Time: 8.7038s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0067 | max_forward: 0.7276 | max_loss: 0.2411 | max_backward: 1.3958 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 5.1754 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1038 | Memory: rss_MB: 7366.2 | uss_MB: 6703.5 | child_rss_MB: 5007.2 | allocated_MB: 343.9 | reserved_MB: 23864.0 | inactive_split_MB: 239.7 +rank2 [2026-10-29 04:51:28,373] INFO : Step 25000/50000 rank2 | Train Loss: 1.1995 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1590 | Classification: 0.0891 | Classification_binary: 0.2009 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1421 | Regression_eta: 0.0018 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0623 | Regression_eta: 0.0393 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0708 | LR: 5.50e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2027s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7692 | max_loss: 0.2091 | max_backward: 1.3616 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6652 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1392 | Memory: rss_MB: 7366.4 | uss_MB: 6702.5 | child_rss_MB: 5014.2 | allocated_MB: 343.5 | reserved_MB: 23864.0 | inactive_split_MB: 293.8 +rank2 [2026-10-29 04:51:29,310] INFO : Running validation on rank2 +rank2 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 04:52:22,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:52:22,822] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 04:52:49,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 04:52:50,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 04:52:51,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 04:52:52,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 04:53:02,287] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 04:53:02,764] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:53:03,282] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 04:53:19,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 04:53:19,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 04:53:20,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 04:53:21,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 04:53:31,857] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 04:53:32,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:53:32,992] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 04:53:48,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 04:53:49,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 04:53:49,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 04:53:50,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 04:54:00,597] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 04:58:07,812] INFO : Step 25100/50000 rank2 | Train Loss: 1.0833 | Train Loss Unweighted: 0.5514 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1317 | Classification: 0.0920 | Classification_binary: 0.1568 | HitGrouping: 0.2008 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1491 | Regression_eta: 0.0016 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1538 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0801 | Regression_eta: 0.0471 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0649 | LR: 5.47e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7887s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7295 | max_loss: 0.1401 | max_backward: 1.2439 | max_optimizer_step: 0.2490 | max_tensorboard_logging: 0.4940 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8607.6 | uss_MB: 7944.9 | child_rss_MB: 5042.3 | allocated_MB: 342.3 | reserved_MB: 23608.0 | inactive_split_MB: 222.3 +rank2 [2026-10-29 05:01:53,205] INFO : Step 25200/50000 rank2 | Train Loss: 1.1886 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1523 | Classification: 0.1158 | Classification_binary: 0.1800 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1512 | Regression_eta: 0.0016 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2076 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0895 | Regression_eta: 0.0517 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0927 | LR: 5.44e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.7330s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8134 | max_loss: 0.1416 | max_backward: 1.5095 | max_optimizer_step: 0.3471 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 8606.1 | uss_MB: 7945.9 | child_rss_MB: 5039.5 | allocated_MB: 343.6 | reserved_MB: 24590.0 | inactive_split_MB: 260.9 +rank2 [2026-10-29 05:05:52,268] INFO : Step 25300/50000 rank2 | Train Loss: 1.2031 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2624 | Cardinality: 0.1455 | Classification: 0.1145 | Classification_binary: 0.1966 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1448 | Regression_eta: 0.0016 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.2253 | Regression_cos_phi: 0.0978 | Regression_energy: 0.0797 | Regression_eta: 0.0571 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0804 | LR: 5.42e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.5373s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.8219 | max_loss: 0.1491 | max_backward: 1.6406 | max_optimizer_step: 0.2735 | max_tensorboard_logging: 0.7204 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0488 | Memory: rss_MB: 7456.6 | uss_MB: 6796.5 | child_rss_MB: 5036.7 | allocated_MB: 341.8 | reserved_MB: 25572.0 | inactive_split_MB: 253.5 +rank2 [2026-10-29 05:09:44,971] INFO : Step 25400/50000 rank2 | Train Loss: 1.1194 | Train Loss Unweighted: 0.5655 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1346 | Classification: 0.0983 | Classification_binary: 0.1754 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1438 | Regression_eta: 0.0015 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0681 | Regression_eta: 0.0446 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0765 | LR: 5.39e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.1072s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.6986 | max_loss: 0.1530 | max_backward: 1.2606 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.8427 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7454.9 | uss_MB: 6795.9 | child_rss_MB: 5040.6 | allocated_MB: 345.2 | reserved_MB: 25572.0 | inactive_split_MB: 223.8 +rank2 [2026-10-29 05:13:34,055] INFO : Step 25500/50000 rank2 | Train Loss: 1.0289 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1068 | Classification: 0.1034 | Classification_binary: 0.1444 | HitGrouping: 0.1913 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1484 | Regression_eta: 0.0014 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.1639 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0754 | Regression_eta: 0.0464 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0436 | LR: 5.36e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1605s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0554 | max_forward: 0.7008 | max_loss: 0.1433 | max_backward: 1.4571 | max_optimizer_step: 0.3349 | max_tensorboard_logging: 0.6083 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 7460.4 | uss_MB: 6802.0 | child_rss_MB: 5038.5 | allocated_MB: 345.4 | reserved_MB: 25572.0 | inactive_split_MB: 240.6 +rank2 [2026-10-29 05:17:32,216] INFO : Step 25600/50000 rank2 | Train Loss: 1.2028 | Train Loss Unweighted: 0.5968 | Train Loss Components: Auxiliary: 0.2590 | Cardinality: 0.1607 | Classification: 0.1135 | Classification_binary: 0.1926 | HitGrouping: 0.1978 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1429 | Regression_eta: 0.0016 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.1865 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0694 | Regression_eta: 0.0411 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0718 | LR: 5.33e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6834s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8800 | max_loss: 0.1578 | max_backward: 1.8564 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 7459.9 | uss_MB: 6802.4 | child_rss_MB: 5035.4 | allocated_MB: 345.5 | reserved_MB: 26054.0 | inactive_split_MB: 284.5 +rank2 [2026-10-29 05:21:26,002] INFO : Step 25700/50000 rank2 | Train Loss: 1.1800 | Train Loss Unweighted: 0.5964 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1298 | Classification: 0.1086 | Classification_binary: 0.1961 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0019 | Regression_pt: 0.1444 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1845 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0606 | Regression_eta: 0.0462 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0704 | LR: 5.30e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.5217s | Phase Times: max_pre_step_barrier: 0.0086 | max_batch_to_device: 0.0137 | max_forward: 1.0340 | max_loss: 0.2483 | max_backward: 1.6704 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 0.7612 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7459.7 | uss_MB: 6801.4 | child_rss_MB: 5033.4 | allocated_MB: 343.5 | reserved_MB: 26054.0 | inactive_split_MB: 270.8 +rank2 [2026-10-29 05:25:16,514] INFO : Step 25800/50000 rank2 | Train Loss: 1.1337 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2388 | Cardinality: 0.1264 | Classification: 0.0898 | Classification_binary: 0.1617 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0019 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0809 | Regression_eta: 0.0466 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0588 | LR: 5.27e-04 | DataLoad Time: 0.0214s | Model Forward Time: 3.9359s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0986 | max_forward: 0.8129 | max_loss: 0.1662 | max_backward: 1.2979 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1103 | Memory: rss_MB: 7457.8 | uss_MB: 6799.8 | child_rss_MB: 5030.2 | allocated_MB: 341.8 | reserved_MB: 26054.0 | inactive_split_MB: 273.2 +rank2 [2026-10-29 05:29:08,212] INFO : Step 25900/50000 rank2 | Train Loss: 1.1019 | Train Loss Unweighted: 0.5481 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1165 | Classification: 0.0977 | Classification_binary: 0.1594 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1434 | Regression_eta: 0.0016 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0702 | Regression_eta: 0.0585 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0871 | LR: 5.25e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5198s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7034 | max_loss: 0.2469 | max_backward: 1.7130 | max_optimizer_step: 0.2363 | max_tensorboard_logging: 0.8382 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 7454.8 | uss_MB: 6800.6 | child_rss_MB: 5037.1 | allocated_MB: 343.0 | reserved_MB: 26054.0 | inactive_split_MB: 187.0 +rank2 [2026-10-29 05:33:07,468] INFO : Step 26000/50000 rank2 | Train Loss: 1.1708 | Train Loss Unweighted: 0.5680 | Train Loss Components: Auxiliary: 0.2498 | Cardinality: 0.1352 | Classification: 0.0999 | Classification_binary: 0.2021 | HitGrouping: 0.1994 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1304 | Regression_eta: 0.0021 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0722 | Regression_eta: 0.0736 | Regression_pt: 0.0735 | Regression_sin_phi: 0.1007 | LR: 5.22e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.2249s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.7877 | max_loss: 0.1637 | max_backward: 1.6550 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.4954 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 7454.4 | uss_MB: 6802.1 | child_rss_MB: 5032.6 | allocated_MB: 345.0 | reserved_MB: 27098.0 | inactive_split_MB: 285.6 +rank2 [2026-10-29 05:33:08,394] INFO : Running validation on rank2 +rank2 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 05:33:56,765] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:33:57,246] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 05:34:24,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 05:34:25,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 05:34:26,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 05:34:27,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 05:34:37,505] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 05:34:38,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:34:38,538] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 05:34:54,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 05:34:55,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 05:34:56,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 05:34:57,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 05:35:07,717] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 05:35:08,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:35:08,509] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 05:35:24,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 05:35:24,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 05:35:25,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 05:35:26,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 05:35:36,110] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 05:39:49,626] INFO : Step 26100/50000 rank2 | Train Loss: 1.0974 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2307 | Cardinality: 0.1119 | Classification: 0.0956 | Classification_binary: 0.1689 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1513 | Regression_eta: 0.0014 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1722 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0616 | Regression_eta: 0.0332 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0538 | LR: 5.19e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1600s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0117 | max_forward: 0.7200 | max_loss: 0.1501 | max_backward: 1.2286 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.8284 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 8630.6 | uss_MB: 7973.7 | child_rss_MB: 5008.3 | allocated_MB: 340.1 | reserved_MB: 21762.0 | inactive_split_MB: 215.7 +rank2 [2026-10-29 05:43:56,538] INFO : Step 26200/50000 rank2 | Train Loss: 1.0876 | Train Loss Unweighted: 0.5502 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1065 | Classification: 0.0968 | Classification_binary: 0.1527 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1476 | Regression_eta: 0.0018 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0660 | Regression_eta: 0.0501 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0602 | LR: 5.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3247s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6530 | max_loss: 0.1421 | max_backward: 1.3973 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.8695 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8630.6 | uss_MB: 7973.8 | child_rss_MB: 5031.7 | allocated_MB: 342.3 | reserved_MB: 22678.0 | inactive_split_MB: 242.5 +rank2 [2026-10-29 05:47:48,032] INFO : Step 26300/50000 rank2 | Train Loss: 1.2051 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1512 | Classification: 0.1193 | Classification_binary: 0.1877 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1458 | Regression_eta: 0.0017 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.1985 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0727 | Regression_eta: 0.0497 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0998 | LR: 5.13e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4705s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0072 | max_forward: 0.7601 | max_loss: 0.2267 | max_backward: 1.8902 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7522.0 | uss_MB: 6871.8 | child_rss_MB: 4996.1 | allocated_MB: 345.4 | reserved_MB: 22678.0 | inactive_split_MB: 270.9 +rank2 [2026-10-29 05:51:47,278] INFO : Step 26400/50000 rank2 | Train Loss: 1.2256 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1648 | Classification: 0.1020 | Classification_binary: 0.1981 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1520 | Regression_eta: 0.0015 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.2088 | Regression_cos_phi: 0.1131 | Regression_energy: 0.0630 | Regression_eta: 0.0536 | Regression_pt: 0.0669 | Regression_sin_phi: 0.1353 | LR: 5.10e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3847s | Phase Times: max_pre_step_barrier: 0.0351 | max_batch_to_device: 0.0070 | max_forward: 0.8129 | max_loss: 0.1477 | max_backward: 1.4015 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8384 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 7520.2 | uss_MB: 6869.2 | child_rss_MB: 4985.0 | allocated_MB: 342.8 | reserved_MB: 22678.0 | inactive_split_MB: 196.9 +rank2 [2026-10-29 05:55:40,759] INFO : Step 26500/50000 rank2 | Train Loss: 1.1110 | Train Loss Unweighted: 0.5356 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1488 | Classification: 0.0891 | Classification_binary: 0.1546 | HitGrouping: 0.2012 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0692 | Regression_eta: 0.0342 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0554 | LR: 5.08e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0221s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6841 | max_loss: 0.2421 | max_backward: 1.1616 | max_optimizer_step: 0.2383 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7505.4 | uss_MB: 6872.6 | child_rss_MB: 4957.6 | allocated_MB: 340.6 | reserved_MB: 22678.0 | inactive_split_MB: 269.9 +rank2 [2026-10-29 05:59:33,069] INFO : Step 26600/50000 rank2 | Train Loss: 1.0895 | Train Loss Unweighted: 0.5529 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1261 | Classification: 0.1083 | Classification_binary: 0.1603 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1386 | Regression_eta: 0.0015 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1345 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0697 | Regression_eta: 0.0362 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0588 | LR: 5.05e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.4536s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0696 | max_forward: 0.6926 | max_loss: 0.1459 | max_backward: 1.4149 | max_optimizer_step: 0.2385 | max_tensorboard_logging: 0.8927 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7477.7 | uss_MB: 6873.1 | child_rss_MB: 4831.1 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 232.0 +rank2 [2026-10-29 06:03:23,755] INFO : Step 26700/50000 rank2 | Train Loss: 1.2592 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1717 | Classification: 0.1100 | Classification_binary: 0.2137 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1420 | Regression_eta: 0.0017 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2144 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0728 | Regression_eta: 0.0458 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0848 | LR: 5.02e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.6732s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7750 | max_loss: 0.1526 | max_backward: 1.6427 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.8866 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7338.5 | uss_MB: 6135.7 | child_rss_MB: 4788.2 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 267.4 +rank2 [2026-10-29 06:07:16,644] INFO : Step 26800/50000 rank2 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5792 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1288 | Classification: 0.0911 | Classification_binary: 0.1633 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1600 | Regression_eta: 0.0015 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0293 | Classification_binary: 0.1328 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0743 | Regression_eta: 0.0280 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0419 | LR: 4.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0413s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7108 | max_loss: 0.1467 | max_backward: 1.2855 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7173 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 7253.0 | uss_MB: 6872.2 | child_rss_MB: 4809.0 | allocated_MB: 340.8 | reserved_MB: 22846.0 | inactive_split_MB: 253.2 +rank2 [2026-10-29 06:11:09,652] INFO : Step 26900/50000 rank2 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.0996 | Classification: 0.0867 | Classification_binary: 0.1428 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1483 | Regression_eta: 0.0018 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0718 | Regression_eta: 0.0511 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0763 | LR: 4.96e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8396s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0336 | max_forward: 0.6534 | max_loss: 0.1408 | max_backward: 1.3898 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0912 | Memory: rss_MB: 7214.0 | uss_MB: 6872.7 | child_rss_MB: 4790.0 | allocated_MB: 342.5 | reserved_MB: 22846.0 | inactive_split_MB: 233.5 +rank2 [2026-10-29 06:15:05,369] INFO : Step 27000/50000 rank2 | Train Loss: 1.0945 | Train Loss Unweighted: 0.5396 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1196 | Classification: 0.0957 | Classification_binary: 0.1912 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1247 | Regression_eta: 0.0015 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1906 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0523 | Regression_eta: 0.0425 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0509 | LR: 4.94e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.1553s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7531 | max_loss: 0.1689 | max_backward: 1.5554 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6017 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7214.1 | uss_MB: 6877.0 | child_rss_MB: 4792.2 | allocated_MB: 345.3 | reserved_MB: 23694.0 | inactive_split_MB: 251.5 +rank2 [2026-10-29 06:15:06,391] INFO : Running validation on rank2 +rank2 [2026-10-29 06:15:06,391] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 06:15:06,391] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 06:16:00,132] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:16:00,711] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 06:16:29,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 06:16:30,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 06:16:31,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 06:16:32,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 06:16:42,719] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 06:16:43,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:16:43,719] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 06:16:59,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 06:17:00,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 06:17:01,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 06:17:02,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 06:17:12,324] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 06:17:13,249] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:17:13,778] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 06:17:29,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 06:17:29,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 06:17:30,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 06:17:31,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 06:17:41,335] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 06:21:51,976] INFO : Step 27100/50000 rank2 | Train Loss: 1.1577 | Train Loss Unweighted: 0.5595 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1497 | Classification: 0.0884 | Classification_binary: 0.1879 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1399 | Regression_eta: 0.0018 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0718 | Regression_eta: 0.0361 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0478 | LR: 4.91e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.0509s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8543 | max_loss: 0.1477 | max_backward: 1.8762 | max_optimizer_step: 0.2540 | max_tensorboard_logging: 0.9462 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 8306.7 | uss_MB: 7966.7 | child_rss_MB: 4814.1 | allocated_MB: 342.9 | reserved_MB: 21640.0 | inactive_split_MB: 268.0 +rank2 [2026-10-29 06:25:39,974] INFO : Step 27200/50000 rank2 | Train Loss: 1.0028 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1065 | Classification: 0.0834 | Classification_binary: 0.1354 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1443 | Regression_eta: 0.0015 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0649 | Regression_eta: 0.0339 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0508 | LR: 4.88e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.0901s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6839 | max_loss: 0.1724 | max_backward: 1.0628 | max_optimizer_step: 0.2515 | max_tensorboard_logging: 0.9641 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0689 | Memory: rss_MB: 8305.7 | uss_MB: 7964.4 | child_rss_MB: 4815.8 | allocated_MB: 340.5 | reserved_MB: 22612.0 | inactive_split_MB: 267.2 +rank2 [2026-10-29 06:29:18,756] INFO : Step 27300/50000 rank2 | Train Loss: 1.1913 | Train Loss Unweighted: 0.5857 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1563 | Classification: 0.1169 | Classification_binary: 0.1769 | HitGrouping: 0.1965 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1426 | Regression_eta: 0.0019 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0672 | Regression_eta: 0.0522 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0752 | LR: 4.85e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.2518s | Phase Times: max_pre_step_barrier: 0.0244 | max_batch_to_device: 0.0064 | max_forward: 0.6813 | max_loss: 0.1454 | max_backward: 1.4618 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7699 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 7290.8 | uss_MB: 6952.0 | child_rss_MB: 4811.5 | allocated_MB: 345.6 | reserved_MB: 23586.0 | inactive_split_MB: 267.9 +rank2 [2026-10-29 06:33:03,428] INFO : Step 27400/50000 rank2 | Train Loss: 1.1723 | Train Loss Unweighted: 0.5888 | Train Loss Components: Auxiliary: 0.2572 | Cardinality: 0.1400 | Classification: 0.1088 | Classification_binary: 0.2016 | HitGrouping: 0.1987 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1370 | Regression_eta: 0.0017 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2099 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0609 | Regression_eta: 0.0501 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0676 | LR: 4.82e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6666s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8048 | max_loss: 0.1475 | max_backward: 1.6655 | max_optimizer_step: 0.2279 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7292.0 | uss_MB: 6951.9 | child_rss_MB: 4814.0 | allocated_MB: 344.5 | reserved_MB: 23586.0 | inactive_split_MB: 279.5 +rank2 [2026-10-29 06:37:01,482] INFO : Step 27500/50000 rank2 | Train Loss: 1.0739 | Train Loss Unweighted: 0.5273 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1251 | Classification: 0.0933 | Classification_binary: 0.1655 | HitGrouping: 0.1957 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0662 | Regression_eta: 0.0453 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0442 | LR: 4.80e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2214s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7686 | max_loss: 0.1542 | max_backward: 1.3750 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.8628 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 7288.8 | uss_MB: 6945.7 | child_rss_MB: 4804.4 | allocated_MB: 343.6 | reserved_MB: 23586.0 | inactive_split_MB: 241.4 +rank2 [2026-10-29 06:40:55,522] INFO : Step 27600/50000 rank2 | Train Loss: 1.0507 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1163 | Classification: 0.1024 | Classification_binary: 0.1467 | HitGrouping: 0.1992 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1211 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0600 | Regression_eta: 0.0329 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0617 | LR: 4.77e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4906s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0060 | max_forward: 0.6744 | max_loss: 0.2492 | max_backward: 1.6867 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.5257 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 7290.8 | uss_MB: 6853.7 | child_rss_MB: 4808.0 | allocated_MB: 341.2 | reserved_MB: 24480.0 | inactive_split_MB: 234.8 +rank2 [2026-10-29 06:44:47,870] INFO : Step 27700/50000 rank2 | Train Loss: 1.1881 | Train Loss Unweighted: 0.5834 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1544 | Classification: 0.1001 | Classification_binary: 0.1885 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1444 | Regression_eta: 0.0015 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1864 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0639 | Regression_eta: 0.0322 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0660 | LR: 4.74e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2913s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7716 | max_loss: 0.1527 | max_backward: 1.4941 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 0.7272 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7291.1 | uss_MB: 6952.7 | child_rss_MB: 4804.6 | allocated_MB: 344.5 | reserved_MB: 24480.0 | inactive_split_MB: 194.2 +rank2 [2026-10-29 06:48:38,928] INFO : Step 27800/50000 rank2 | Train Loss: 1.1903 | Train Loss Unweighted: 0.5862 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1435 | Classification: 0.1032 | Classification_binary: 0.2006 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1387 | Regression_eta: 0.0018 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0704 | Regression_eta: 0.0344 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0566 | LR: 4.71e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1888s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8137 | max_loss: 0.1509 | max_backward: 1.4748 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6041 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7291.1 | uss_MB: 6953.1 | child_rss_MB: 4808.7 | allocated_MB: 348.4 | reserved_MB: 24592.0 | inactive_split_MB: 226.8 +rank2 [2026-10-29 06:52:39,106] INFO : Step 27900/50000 rank2 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5521 | Train Loss Components: Auxiliary: 0.2284 | Cardinality: 0.1139 | Classification: 0.0902 | Classification_binary: 0.1544 | HitGrouping: 0.2057 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1519 | Regression_eta: 0.0017 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1817 | Regression_cos_phi: 0.1146 | Regression_energy: 0.0677 | Regression_eta: 0.0584 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0618 | LR: 4.68e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6327s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6397 | max_loss: 0.2456 | max_backward: 1.5234 | max_optimizer_step: 0.2276 | max_tensorboard_logging: 1.1626 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 7291.1 | uss_MB: 6955.2 | child_rss_MB: 4805.2 | allocated_MB: 342.4 | reserved_MB: 25612.0 | inactive_split_MB: 259.0 +rank2 [2026-10-29 06:56:30,760] INFO : Step 28000/50000 rank2 | Train Loss: 1.1111 | Train Loss Unweighted: 0.5401 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1274 | Classification: 0.1035 | Classification_binary: 0.1699 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.1878 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0588 | Regression_eta: 0.0526 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0586 | LR: 4.66e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.0967s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.7503 | max_loss: 0.1492 | max_backward: 1.7566 | max_optimizer_step: 0.2764 | max_tensorboard_logging: 1.2295 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 7291.2 | uss_MB: 5984.8 | child_rss_MB: 4794.8 | allocated_MB: 344.1 | reserved_MB: 25612.0 | inactive_split_MB: 243.2 +rank2 [2026-10-29 06:56:31,844] INFO : Running validation on rank2 +rank2 [2026-10-29 06:56:31,844] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 06:56:31,844] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 06:57:24,998] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:57:25,561] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 06:57:52,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 06:57:53,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 06:57:54,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 06:57:55,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 06:58:05,277] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 06:58:05,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:58:06,180] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 06:58:22,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 06:58:22,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 06:58:23,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 06:58:24,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 06:58:34,892] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 06:58:35,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:58:35,800] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 06:58:51,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 06:58:52,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 06:58:53,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 06:58:53,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 06:59:03,878] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 07:03:11,252] INFO : Step 28100/50000 rank2 | Train Loss: 1.1727 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1644 | Classification: 0.0934 | Classification_binary: 0.1929 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1365 | Regression_eta: 0.0015 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1771 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0577 | Regression_eta: 0.0372 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0476 | LR: 4.63e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.9505s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8425 | max_loss: 0.2571 | max_backward: 1.8157 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 1.0134 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 8340.2 | uss_MB: 8005.6 | child_rss_MB: 4802.9 | allocated_MB: 344.8 | reserved_MB: 22140.0 | inactive_split_MB: 293.2 +rank2 [2026-10-29 07:07:01,163] INFO : Step 28200/50000 rank2 | Train Loss: 1.0520 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2296 | Cardinality: 0.1203 | Classification: 0.0934 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1466 | Regression_eta: 0.0014 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0659 | Regression_eta: 0.0429 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0722 | LR: 4.60e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3563s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7943 | max_loss: 0.1496 | max_backward: 1.4503 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 0.7872 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 8343.2 | uss_MB: 8006.4 | child_rss_MB: 4796.0 | allocated_MB: 341.0 | reserved_MB: 24034.0 | inactive_split_MB: 210.4 +rank2 [2026-10-29 07:10:52,729] INFO : Step 28300/50000 rank2 | Train Loss: 1.0602 | Train Loss Unweighted: 0.5453 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1043 | Classification: 0.0923 | Classification_binary: 0.1501 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1496 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0674 | Regression_eta: 0.0421 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0614 | LR: 4.57e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2457s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6482 | max_loss: 0.2266 | max_backward: 1.5658 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.6870 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0836 | Memory: rss_MB: 7393.0 | uss_MB: 7059.5 | child_rss_MB: 4787.3 | allocated_MB: 342.7 | reserved_MB: 25016.0 | inactive_split_MB: 243.9 +rank2 [2026-10-29 07:14:40,574] INFO : Step 28400/50000 rank2 | Train Loss: 1.1507 | Train Loss Unweighted: 0.5391 | Train Loss Components: Auxiliary: 0.2436 | Cardinality: 0.1500 | Classification: 0.0965 | Classification_binary: 0.1896 | HitGrouping: 0.1911 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1241 | Regression_eta: 0.0018 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2083 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0607 | Regression_eta: 0.0401 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0452 | LR: 4.55e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6294s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7664 | max_loss: 0.1532 | max_backward: 1.7342 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.8455 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7394.0 | uss_MB: 7059.9 | child_rss_MB: 4785.1 | allocated_MB: 344.2 | reserved_MB: 25016.0 | inactive_split_MB: 278.9 +rank2 [2026-10-29 07:18:29,002] INFO : Step 28500/50000 rank2 | Train Loss: 1.1595 | Train Loss Unweighted: 0.5640 | Train Loss Components: Auxiliary: 0.2492 | Cardinality: 0.1556 | Classification: 0.0927 | Classification_binary: 0.1908 | HitGrouping: 0.1954 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1384 | Regression_eta: 0.0015 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1915 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0696 | Regression_eta: 0.0431 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0717 | LR: 4.52e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2725s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.8033 | max_loss: 0.1665 | max_backward: 1.5231 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.6178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0709 | Memory: rss_MB: 7393.5 | uss_MB: 7056.4 | child_rss_MB: 4808.7 | allocated_MB: 347.4 | reserved_MB: 26000.0 | inactive_split_MB: 283.2 +rank2 [2026-10-29 07:22:17,774] INFO : Step 28600/50000 rank2 | Train Loss: 1.0672 | Train Loss Unweighted: 0.5222 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1172 | Classification: 0.0893 | Classification_binary: 0.1544 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0623 | Regression_eta: 0.0473 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0511 | LR: 4.49e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0231s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6704 | max_loss: 0.1829 | max_backward: 1.2353 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.8074 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7396.5 | uss_MB: 7057.8 | child_rss_MB: 4793.1 | allocated_MB: 342.3 | reserved_MB: 26000.0 | inactive_split_MB: 243.7 +rank2 [2026-10-29 07:26:10,154] INFO : Step 28700/50000 rank2 | Train Loss: 1.1535 | Train Loss Unweighted: 0.5811 | Train Loss Components: Auxiliary: 0.2469 | Cardinality: 0.1254 | Classification: 0.1220 | Classification_binary: 0.1661 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1442 | Regression_eta: 0.0021 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0559 | Regression_eta: 0.0466 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0584 | LR: 4.46e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5817s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6905 | max_loss: 0.2072 | max_backward: 1.6876 | max_optimizer_step: 0.2353 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 7397.5 | uss_MB: 7064.1 | child_rss_MB: 4802.4 | allocated_MB: 343.5 | reserved_MB: 26000.0 | inactive_split_MB: 234.1 +rank2 [2026-10-29 07:30:12,149] INFO : Step 28800/50000 rank2 | Train Loss: 1.2131 | Train Loss Unweighted: 0.5739 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1788 | Classification: 0.1057 | Classification_binary: 0.2127 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.2065 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0582 | Regression_eta: 0.0333 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0686 | LR: 4.44e-04 | DataLoad Time: 0.0120s | Model Forward Time: 6.8519s | Phase Times: max_pre_step_barrier: 0.0263 | max_batch_to_device: 0.0149 | max_forward: 1.2928 | max_loss: 0.2713 | max_backward: 3.0161 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.8895 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 7398.3 | uss_MB: 5722.3 | child_rss_MB: 4796.9 | allocated_MB: 343.8 | reserved_MB: 26000.0 | inactive_split_MB: 273.3 +rank2 [2026-10-29 07:34:11,569] INFO : Step 28900/50000 rank2 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1252 | Classification: 0.0967 | Classification_binary: 0.1637 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1509 | Regression_eta: 0.0015 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1731 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0689 | Regression_eta: 0.0400 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0648 | LR: 4.41e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.1511s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7512 | max_loss: 0.2189 | max_backward: 1.6229 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.9330 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7399.6 | uss_MB: 5634.1 | child_rss_MB: 4800.1 | allocated_MB: 339.6 | reserved_MB: 26000.0 | inactive_split_MB: 223.0 +rank2 [2026-10-29 07:38:06,079] INFO : Step 29000/50000 rank2 | Train Loss: 1.0238 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1070 | Classification: 0.0861 | Classification_binary: 0.1439 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1389 | Regression_eta: 0.0014 | Regression_pt: 0.1414 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1223 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0697 | Regression_eta: 0.0604 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0547 | LR: 4.38e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9816s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.6636 | max_loss: 0.2429 | max_backward: 1.5394 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.5216 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7400.5 | uss_MB: 7067.2 | child_rss_MB: 4794.8 | allocated_MB: 345.2 | reserved_MB: 26984.0 | inactive_split_MB: 235.2 +rank2 [2026-10-29 07:38:07,029] INFO : Running validation on rank2 +rank2 [2026-10-29 07:38:07,030] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 07:38:07,030] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 07:38:57,929] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:38:58,451] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 07:39:24,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 07:39:25,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 07:39:26,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 07:39:27,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 07:39:37,728] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 07:39:38,026] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:39:38,576] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 07:39:54,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 07:39:55,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 07:39:56,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 07:39:57,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 07:40:07,177] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 07:40:07,779] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:40:08,278] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 07:40:24,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 07:40:24,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 07:40:25,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 07:40:26,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 07:40:36,328] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 07:45:01,997] INFO : Step 29100/50000 rank2 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5808 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1279 | Classification: 0.1113 | Classification_binary: 0.1973 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1339 | Regression_eta: 0.0017 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.1817 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0530 | Regression_eta: 0.0468 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0454 | LR: 4.35e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.8738s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.8060 | max_loss: 0.1503 | max_backward: 1.9210 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 8379.4 | uss_MB: 8043.2 | child_rss_MB: 4787.2 | allocated_MB: 344.6 | reserved_MB: 24404.0 | inactive_split_MB: 274.9 +rank2 [2026-10-29 07:48:58,054] INFO : Step 29200/50000 rank2 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5633 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1265 | Classification: 0.0969 | Classification_binary: 0.1883 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1368 | Regression_eta: 0.0014 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0608 | Regression_eta: 0.0363 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0479 | LR: 4.33e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.8377s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0742 | max_forward: 1.0303 | max_loss: 0.2183 | max_backward: 1.8745 | max_optimizer_step: 0.2501 | max_tensorboard_logging: 0.5570 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8379.4 | uss_MB: 8042.7 | child_rss_MB: 4798.0 | allocated_MB: 343.1 | reserved_MB: 24404.0 | inactive_split_MB: 228.1 +rank2 [2026-10-29 07:52:51,792] INFO : Step 29300/50000 rank2 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2330 | Cardinality: 0.1231 | Classification: 0.0898 | Classification_binary: 0.1516 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1428 | Regression_eta: 0.0017 | Regression_pt: 0.1453 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0799 | Regression_eta: 0.0445 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0756 | LR: 4.30e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.7357s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0067 | max_forward: 0.7139 | max_loss: 0.1578 | max_backward: 1.5096 | max_optimizer_step: 0.2586 | max_tensorboard_logging: 1.1324 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7473.7 | uss_MB: 6607.6 | child_rss_MB: 4797.1 | allocated_MB: 341.0 | reserved_MB: 24404.0 | inactive_split_MB: 247.0 +rank2 [2026-10-29 07:56:44,367] INFO : Step 29400/50000 rank2 | Train Loss: 1.1400 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1377 | Classification: 0.1135 | Classification_binary: 0.1661 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1386 | Regression_eta: 0.0019 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1585 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0682 | Regression_eta: 0.0388 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0782 | LR: 4.27e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2873s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.6285 | max_loss: 0.1485 | max_backward: 1.3787 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.9179 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7474.7 | uss_MB: 7139.2 | child_rss_MB: 4788.9 | allocated_MB: 343.3 | reserved_MB: 24404.0 | inactive_split_MB: 277.1 +rank2 [2026-10-29 08:00:58,645] INFO : Step 29500/50000 rank2 | Train Loss: 1.1987 | Train Loss Unweighted: 0.6106 | Train Loss Components: Auxiliary: 0.2551 | Cardinality: 0.1277 | Classification: 0.1201 | Classification_binary: 0.2013 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1419 | Regression_eta: 0.0019 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2065 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0736 | Regression_eta: 0.0412 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0508 | LR: 4.24e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.8983s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0749 | max_forward: 0.8153 | max_loss: 0.3465 | max_backward: 2.8565 | max_optimizer_step: 0.3442 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7477.7 | uss_MB: 7142.6 | child_rss_MB: 4792.1 | allocated_MB: 343.9 | reserved_MB: 24404.0 | inactive_split_MB: 271.5 +rank2 [2026-10-29 08:05:08,852] INFO : Step 29600/50000 rank2 | Train Loss: 1.1065 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2391 | Cardinality: 0.1179 | Classification: 0.0927 | Classification_binary: 0.1666 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0018 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0865 | Regression_eta: 0.0618 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0734 | LR: 4.22e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4752s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8025 | max_loss: 0.1485 | max_backward: 1.4674 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.8982 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 7476.5 | uss_MB: 7138.1 | child_rss_MB: 4790.4 | allocated_MB: 340.8 | reserved_MB: 24404.0 | inactive_split_MB: 290.7 +rank2 [2026-10-29 08:09:13,801] INFO : Step 29700/50000 rank2 | Train Loss: 1.0488 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2273 | Cardinality: 0.1414 | Classification: 0.0952 | Classification_binary: 0.1459 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1503 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0567 | Regression_eta: 0.0339 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0580 | LR: 4.19e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.5170s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0057 | max_forward: 0.7342 | max_loss: 0.1652 | max_backward: 1.4907 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.9743 | max_tensorboard_flush: 0.0140 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7477.5 | uss_MB: 6998.2 | child_rss_MB: 4799.0 | allocated_MB: 342.3 | reserved_MB: 24404.0 | inactive_split_MB: 236.6 +rank2 [2026-10-29 08:13:32,079] INFO : Step 29800/50000 rank2 | Train Loss: 1.1686 | Train Loss Unweighted: 0.5644 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1551 | Classification: 0.0952 | Classification_binary: 0.1922 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1357 | Regression_eta: 0.0014 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1708 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0572 | Regression_eta: 0.0354 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0573 | LR: 4.16e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6783s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7294 | max_loss: 0.1685 | max_backward: 1.6547 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9409 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7478.5 | uss_MB: 7143.5 | child_rss_MB: 4794.4 | allocated_MB: 345.0 | reserved_MB: 24404.0 | inactive_split_MB: 265.1 +rank2 [2026-10-29 08:17:37,375] INFO : Step 29900/50000 rank2 | Train Loss: 1.1305 | Train Loss Unweighted: 0.5666 | Train Loss Components: Auxiliary: 0.2471 | Cardinality: 0.1380 | Classification: 0.0944 | Classification_binary: 0.1750 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1464 | Regression_eta: 0.0015 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1683 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0628 | Regression_eta: 0.0449 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0487 | LR: 4.14e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3511s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0611 | max_forward: 0.8060 | max_loss: 0.1494 | max_backward: 1.3251 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 7479.5 | uss_MB: 7143.0 | child_rss_MB: 4800.7 | allocated_MB: 344.7 | reserved_MB: 26422.0 | inactive_split_MB: 298.4 +rank2 [2026-10-29 08:22:04,988] INFO : Step 30000/50000 rank2 | Train Loss: 1.0854 | Train Loss Unweighted: 0.5586 | Train Loss Components: Auxiliary: 0.2416 | Cardinality: 0.1337 | Classification: 0.1000 | Classification_binary: 0.1541 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1499 | Regression_eta: 0.0014 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1562 | Regression_cos_phi: 0.0917 | Regression_energy: 0.0541 | Regression_eta: 0.0720 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0479 | LR: 4.11e-04 | DataLoad Time: 0.0198s | Model Forward Time: 7.0746s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0172 | max_forward: 0.9685 | max_loss: 0.2166 | max_backward: 4.0112 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.8430 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7461.9 | uss_MB: 7142.5 | child_rss_MB: 4797.1 | allocated_MB: 338.0 | reserved_MB: 26422.0 | inactive_split_MB: 288.0 +rank2 [2026-10-29 08:22:05,952] INFO : Running validation on rank2 +rank2 [2026-10-29 08:22:05,952] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 08:22:05,953] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 08:22:57,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:22:58,359] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 08:23:24,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 08:23:25,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 08:23:26,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 08:23:28,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 08:23:38,158] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 08:23:38,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:23:38,901] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 08:23:54,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 08:23:55,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 08:23:56,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 08:23:57,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 08:24:07,642] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 08:24:08,113] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:24:08,615] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 08:24:24,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 08:24:24,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 08:24:25,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 08:24:26,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 08:24:36,531] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 08:29:00,406] INFO : Step 30100/50000 rank2 | Train Loss: 1.0762 | Train Loss Unweighted: 0.5358 | Train Loss Components: Auxiliary: 0.2378 | Cardinality: 0.1106 | Classification: 0.1097 | Classification_binary: 0.1679 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0663 | Regression_eta: 0.0340 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0494 | LR: 4.08e-04 | DataLoad Time: 0.0167s | Model Forward Time: 8.2916s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7352 | max_loss: 0.1443 | max_backward: 1.7375 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 1.7731 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.3497 | Memory: rss_MB: 7528.6 | uss_MB: 6331.6 | child_rss_MB: 4801.2 | allocated_MB: 344.1 | reserved_MB: 22922.0 | inactive_split_MB: 248.8 +rank2 [2026-10-29 08:33:16,461] INFO : Step 30200/50000 rank2 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5304 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1421 | Classification: 0.0934 | Classification_binary: 0.1839 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0016 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0570 | Regression_eta: 0.0509 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0512 | LR: 4.06e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5996s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8008 | max_loss: 0.1560 | max_backward: 1.6706 | max_optimizer_step: 0.2663 | max_tensorboard_logging: 0.7725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0555 | Memory: rss_MB: 7535.6 | uss_MB: 7212.9 | child_rss_MB: 4775.8 | allocated_MB: 344.1 | reserved_MB: 23882.0 | inactive_split_MB: 274.8 +rank2 [2026-10-29 08:37:30,482] INFO : Step 30300/50000 rank2 | Train Loss: 1.0429 | Train Loss Unweighted: 0.5252 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1223 | Classification: 0.0922 | Classification_binary: 0.1566 | HitGrouping: 0.1929 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0600 | Regression_eta: 0.0365 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0501 | LR: 4.03e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.6272s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0060 | max_forward: 1.1719 | max_loss: 0.2884 | max_backward: 2.4994 | max_optimizer_step: 0.2255 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7536.6 | uss_MB: 7213.9 | child_rss_MB: 4771.6 | allocated_MB: 341.8 | reserved_MB: 23882.0 | inactive_split_MB: 270.7 +rank2 [2026-10-29 08:41:40,106] INFO : Step 30400/50000 rank2 | Train Loss: 1.0543 | Train Loss Unweighted: 0.5182 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1318 | Classification: 0.0889 | Classification_binary: 0.1487 | HitGrouping: 0.1973 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1382 | Regression_eta: 0.0014 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0927 | Classification_binary: 0.1615 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0623 | Regression_eta: 0.0529 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0890 | LR: 4.00e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.0682s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0150 | max_forward: 0.8845 | max_loss: 0.2482 | max_backward: 2.1745 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.5112 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7452.2 | uss_MB: 7214.6 | child_rss_MB: 4475.9 | allocated_MB: 344.7 | reserved_MB: 24842.0 | inactive_split_MB: 245.3 +rank2 [2026-10-29 08:45:49,048] INFO : Step 30500/50000 rank2 | Train Loss: 1.1589 | Train Loss Unweighted: 0.5596 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1407 | Classification: 0.1085 | Classification_binary: 0.1952 | HitGrouping: 0.1960 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0512 | Regression_eta: 0.0397 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0456 | LR: 3.98e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.7066s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7791 | max_loss: 0.2528 | max_backward: 1.9433 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.5210 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0672 | Memory: rss_MB: 7432.7 | uss_MB: 7008.9 | child_rss_MB: 4374.7 | allocated_MB: 346.2 | reserved_MB: 26806.0 | inactive_split_MB: 227.8 +rank2 [2026-10-29 08:49:58,495] INFO : Step 30600/50000 rank2 | Train Loss: 1.1269 | Train Loss Unweighted: 0.5561 | Train Loss Components: Auxiliary: 0.2414 | Cardinality: 0.1456 | Classification: 0.0973 | Classification_binary: 0.1804 | HitGrouping: 0.1953 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1373 | Regression_eta: 0.0014 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0481 | Regression_eta: 0.0363 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0720 | LR: 3.95e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4772s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1011 | max_forward: 0.8824 | max_loss: 0.1534 | max_backward: 1.4258 | max_optimizer_step: 0.3067 | max_tensorboard_logging: 0.7925 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 7378.2 | uss_MB: 7215.1 | child_rss_MB: 4177.7 | allocated_MB: 345.8 | reserved_MB: 27800.0 | inactive_split_MB: 283.5 +rank2 [2026-10-29 08:54:17,062] INFO : Step 30700/50000 rank2 | Train Loss: 1.0989 | Train Loss Unweighted: 0.5515 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1377 | Classification: 0.0890 | Classification_binary: 0.1568 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0013 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1541 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0739 | Regression_eta: 0.0340 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0412 | LR: 3.92e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2944s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0056 | max_forward: 0.7546 | max_loss: 0.2396 | max_backward: 1.3389 | max_optimizer_step: 0.2508 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0572 | Memory: rss_MB: 7380.2 | uss_MB: 7216.5 | child_rss_MB: 4178.9 | allocated_MB: 340.3 | reserved_MB: 27800.0 | inactive_split_MB: 223.2 +rank2 [2026-10-29 08:58:31,929] INFO : Step 30800/50000 rank2 | Train Loss: 1.1241 | Train Loss Unweighted: 0.5488 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1377 | Classification: 0.1081 | Classification_binary: 0.1717 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1317 | Regression_eta: 0.0015 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1862 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0727 | Regression_eta: 0.0400 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0455 | LR: 3.90e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.8030s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7775 | max_loss: 0.2735 | max_backward: 1.9298 | max_optimizer_step: 0.3489 | max_tensorboard_logging: 0.8756 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0726 | Memory: rss_MB: 7357.4 | uss_MB: 5945.6 | child_rss_MB: 4116.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 275.1 +rank2 [2026-10-29 09:02:40,911] INFO : Step 30900/50000 rank2 | Train Loss: 1.1501 | Train Loss Unweighted: 0.5762 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1300 | Classification: 0.0976 | Classification_binary: 0.1996 | HitGrouping: 0.2000 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1374 | Regression_eta: 0.0017 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0845 | Classification_binary: 0.2197 | Regression_cos_phi: 0.0979 | Regression_energy: 0.0739 | Regression_eta: 0.0667 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0783 | LR: 3.87e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.5961s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.3511 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 1.0306 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 7360.4 | uss_MB: 7217.8 | child_rss_MB: 4118.5 | allocated_MB: 342.2 | reserved_MB: 27800.0 | inactive_split_MB: 198.3 +rank2 [2026-10-29 09:06:46,924] INFO : Step 31000/50000 rank2 | Train Loss: 1.1054 | Train Loss Unweighted: 0.5218 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1446 | Classification: 0.0888 | Classification_binary: 0.1693 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0016 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0806 | Regression_eta: 0.0482 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0686 | LR: 3.84e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.7146s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.8505 | max_loss: 0.2491 | max_backward: 1.6460 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9731 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7337.9 | uss_MB: 6674.8 | child_rss_MB: 4046.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 190.7 +rank2 [2026-10-29 09:06:47,903] INFO : Running validation on rank2 +rank2 [2026-10-29 09:06:47,903] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 09:06:47,903] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 09:07:39,008] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:07:39,556] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 09:08:08,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 09:08:09,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 09:08:10,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 09:08:11,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 09:08:21,501] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 09:08:22,017] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:08:22,479] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 09:08:39,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 09:08:40,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 09:08:41,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 09:08:42,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 09:08:52,469] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 09:08:52,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:08:53,433] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 09:09:08,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 09:09:09,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 09:09:10,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 09:09:10,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 09:09:20,880] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 09:13:45,041] INFO : Step 31100/50000 rank2 | Train Loss: 1.0306 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2182 | Cardinality: 0.1238 | Classification: 0.0932 | Classification_binary: 0.1415 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1695 | Regression_cos_phi: 0.1076 | Regression_energy: 0.0619 | Regression_eta: 0.0432 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0566 | LR: 3.82e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.5134s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1068 | max_forward: 0.8067 | max_loss: 0.1436 | max_backward: 1.1427 | max_optimizer_step: 0.2576 | max_tensorboard_logging: 1.1475 | max_tensorboard_flush: 0.0054 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 8196.3 | uss_MB: 5921.9 | child_rss_MB: 4032.2 | allocated_MB: 343.8 | reserved_MB: 24492.0 | inactive_split_MB: 293.3 +rank2 [2026-10-29 09:17:44,912] INFO : Step 31200/50000 rank2 | Train Loss: 1.1102 | Train Loss Unweighted: 0.5369 | Train Loss Components: Auxiliary: 0.2472 | Cardinality: 0.1495 | Classification: 0.0959 | Classification_binary: 0.1708 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1328 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1788 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0658 | Regression_eta: 0.0642 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0697 | LR: 3.79e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.1838s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7127 | max_loss: 0.1476 | max_backward: 1.5707 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7428.7 | uss_MB: 7313.8 | child_rss_MB: 4003.5 | allocated_MB: 344.5 | reserved_MB: 24648.0 | inactive_split_MB: 235.5 +rank2 [2026-10-29 09:21:47,087] INFO : Step 31300/50000 rank2 | Train Loss: 1.1865 | Train Loss Unweighted: 0.5472 | Train Loss Components: Auxiliary: 0.2511 | Cardinality: 0.1748 | Classification: 0.0979 | Classification_binary: 0.1851 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0019 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2057 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0630 | Regression_eta: 0.0413 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0616 | LR: 3.76e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.2289s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0064 | max_forward: 0.7815 | max_loss: 0.1495 | max_backward: 1.6330 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.5234 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7366.1 | uss_MB: 7313.9 | child_rss_MB: 3706.8 | allocated_MB: 344.4 | reserved_MB: 25640.0 | inactive_split_MB: 233.6 +rank2 [2026-10-29 09:25:42,888] INFO : Step 31400/50000 rank2 | Train Loss: 1.0272 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1191 | Classification: 0.0784 | Classification_binary: 0.1451 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1379 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0532 | Regression_eta: 0.0431 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0536 | LR: 3.74e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5260s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0063 | max_forward: 0.7252 | max_loss: 0.1478 | max_backward: 1.6173 | max_optimizer_step: 0.2404 | max_tensorboard_logging: 0.9541 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7335.5 | uss_MB: 7312.2 | child_rss_MB: 3625.5 | allocated_MB: 342.1 | reserved_MB: 25640.0 | inactive_split_MB: 296.4 +rank2 [2026-10-29 09:29:44,710] INFO : Step 31500/50000 rank2 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5247 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1206 | Classification: 0.1037 | Classification_binary: 0.1583 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1297 | Regression_eta: 0.0014 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0693 | Regression_eta: 0.0483 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0582 | LR: 3.71e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.3658s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6643 | max_loss: 0.2487 | max_backward: 1.5648 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.8248 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7336.0 | uss_MB: 7313.0 | child_rss_MB: 3638.4 | allocated_MB: 344.4 | reserved_MB: 26092.0 | inactive_split_MB: 238.2 +rank2 [2026-10-29 09:33:42,078] INFO : Step 31600/50000 rank2 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5471 | Train Loss Components: Auxiliary: 0.2424 | Cardinality: 0.1284 | Classification: 0.1033 | Classification_binary: 0.1828 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1282 | Regression_eta: 0.0017 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1815 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0713 | Regression_eta: 0.0452 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0638 | LR: 3.69e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.0249s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7603 | max_loss: 0.1471 | max_backward: 2.0475 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.9076 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7328.4 | uss_MB: 7314.4 | child_rss_MB: 3585.6 | allocated_MB: 341.5 | reserved_MB: 26640.0 | inactive_split_MB: 287.0 +rank2 [2026-10-29 09:37:47,826] INFO : Step 31700/50000 rank2 | Train Loss: 1.0533 | Train Loss Unweighted: 0.5009 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1416 | Classification: 0.0871 | Classification_binary: 0.1632 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0649 | Regression_eta: 0.0288 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0561 | LR: 3.66e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.1115s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7635 | max_loss: 0.1543 | max_backward: 1.2075 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7330.4 | uss_MB: 7312.9 | child_rss_MB: 3557.0 | allocated_MB: 341.9 | reserved_MB: 26640.0 | inactive_split_MB: 274.1 +rank2 [2026-10-29 09:45:40,561] INFO : Step 31800/50000 rank2 | Train Loss: 1.0385 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1148 | Classification: 0.0877 | Classification_binary: 0.1481 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1418 | Regression_eta: 0.0012 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0319 | Classification_binary: 0.1333 | Regression_cos_phi: 0.0401 | Regression_energy: 0.0604 | Regression_eta: 0.0246 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0398 | LR: 3.64e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.0348s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0072 | max_forward: 0.6606 | max_loss: 0.1610 | max_backward: 1.1159 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.8258 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1417 | Memory: rss_MB: 7328.9 | uss_MB: 7312.1 | child_rss_MB: 3561.3 | allocated_MB: 345.5 | reserved_MB: 27080.0 | inactive_split_MB: 260.5 +rank2 [2026-10-29 09:49:44,090] INFO : Step 31900/50000 rank2 | Train Loss: 1.1464 | Train Loss Unweighted: 0.5585 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1372 | Classification: 0.1078 | Classification_binary: 0.1815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0017 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0583 | Regression_eta: 0.0420 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0564 | LR: 3.61e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.5164s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7718 | max_loss: 0.1553 | max_backward: 1.7857 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.6721 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7333.9 | uss_MB: 7320.8 | child_rss_MB: 3569.3 | allocated_MB: 343.8 | reserved_MB: 27080.0 | inactive_split_MB: 280.2 +rank2 [2026-10-29 09:53:38,294] INFO : Step 32000/50000 rank2 | Train Loss: 1.1115 | Train Loss Unweighted: 0.5351 | Train Loss Components: Auxiliary: 0.2393 | Cardinality: 0.1406 | Classification: 0.0954 | Classification_binary: 0.1748 | HitGrouping: 0.1912 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1302 | Regression_eta: 0.0018 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.1847 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0730 | Regression_eta: 0.0448 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0841 | LR: 3.58e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.2455s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.9767 | max_loss: 0.6919 | max_backward: 2.1955 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.9277 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 7329.3 | uss_MB: 7313.9 | child_rss_MB: 3566.4 | allocated_MB: 344.2 | reserved_MB: 27080.0 | inactive_split_MB: 265.7 +rank2 [2026-10-29 09:53:39,335] INFO : Running validation on rank2 +rank2 [2026-10-29 09:53:39,335] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 09:53:39,336] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 09:54:28,722] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:54:29,292] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 09:54:57,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 09:54:58,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 09:54:59,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 09:55:00,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 09:55:10,533] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 09:55:10,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:55:11,414] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 09:55:26,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 09:55:27,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 09:55:28,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 09:55:29,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 09:55:39,892] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 09:55:41,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:55:41,778] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 09:55:57,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 09:55:58,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 09:55:58,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 09:55:59,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 09:56:09,474] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 10:00:19,240] INFO : Step 32100/50000 rank2 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1218 | Classification: 0.0914 | Classification_binary: 0.1509 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1333 | Regression_eta: 0.0016 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0665 | Regression_eta: 0.0425 | Regression_pt: 0.0684 | Regression_sin_phi: 0.1278 | LR: 3.56e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.3633s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7098 | max_loss: 0.1454 | max_backward: 1.1394 | max_optimizer_step: 0.2650 | max_tensorboard_logging: 0.9832 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 8404.9 | uss_MB: 8386.6 | child_rss_MB: 3573.9 | allocated_MB: 344.0 | reserved_MB: 22148.0 | inactive_split_MB: 276.8 +rank2 [2026-10-29 10:04:33,375] INFO : Step 32200/50000 rank2 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1241 | Classification: 0.1035 | Classification_binary: 0.1630 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1279 | Regression_eta: 0.0014 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.1778 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0546 | Regression_eta: 0.0402 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0596 | LR: 3.53e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6706s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0891 | max_forward: 0.7517 | max_loss: 0.1441 | max_backward: 1.7835 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.7808 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7418.2 | uss_MB: 7403.1 | child_rss_MB: 3576.2 | allocated_MB: 345.5 | reserved_MB: 23122.0 | inactive_split_MB: 233.4 +rank2 [2026-10-29 10:08:31,628] INFO : Step 32300/50000 rank2 | Train Loss: 1.1438 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2535 | Cardinality: 0.1335 | Classification: 0.1042 | Classification_binary: 0.1958 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1303 | Regression_eta: 0.0015 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.2156 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0496 | Regression_eta: 0.0417 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0623 | LR: 3.51e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7842s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7857 | max_loss: 0.1780 | max_backward: 1.7580 | max_optimizer_step: 0.2428 | max_tensorboard_logging: 0.8954 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 7421.3 | uss_MB: 7404.5 | child_rss_MB: 3576.8 | allocated_MB: 343.9 | reserved_MB: 24108.0 | inactive_split_MB: 280.1 +rank2 [2026-10-29 10:12:37,148] INFO : Step 32400/50000 rank2 | Train Loss: 1.0306 | Train Loss Unweighted: 0.5063 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1168 | Classification: 0.0867 | Classification_binary: 0.1599 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0013 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0550 | Regression_eta: 0.0394 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0479 | LR: 3.48e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.3217s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.1456 | max_backward: 1.3401 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.8752 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7420.4 | uss_MB: 7401.9 | child_rss_MB: 3574.4 | allocated_MB: 344.8 | reserved_MB: 24108.0 | inactive_split_MB: 271.0 +rank2 [2026-10-29 10:16:48,486] INFO : Step 32500/50000 rank2 | Train Loss: 1.0033 | Train Loss Unweighted: 0.5126 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.0995 | Classification: 0.1015 | Classification_binary: 0.1402 | HitGrouping: 0.1898 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1332 | Regression_eta: 0.0015 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1110 | Regression_cos_phi: 0.0260 | Regression_energy: 0.0555 | Regression_eta: 0.0209 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0342 | LR: 3.46e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9963s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6088 | max_loss: 0.1480 | max_backward: 1.0729 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.8045 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7420.4 | uss_MB: 7401.8 | child_rss_MB: 3575.0 | allocated_MB: 341.9 | reserved_MB: 25096.0 | inactive_split_MB: 268.1 +rank2 [2026-10-29 10:21:04,044] INFO : Step 32600/50000 rank2 | Train Loss: 1.0981 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1271 | Classification: 0.1053 | Classification_binary: 0.1773 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1306 | Regression_eta: 0.0014 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.1898 | Regression_cos_phi: 0.0916 | Regression_energy: 0.0674 | Regression_eta: 0.0505 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0701 | LR: 3.43e-04 | DataLoad Time: 0.0152s | Model Forward Time: 6.2757s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8529 | max_loss: 0.3369 | max_backward: 2.9831 | max_optimizer_step: 0.2386 | max_tensorboard_logging: 0.5557 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7420.7 | uss_MB: 5748.3 | child_rss_MB: 3585.6 | allocated_MB: 344.1 | reserved_MB: 25096.0 | inactive_split_MB: 287.9 +rank2 [2026-10-29 10:25:23,204] INFO : Step 32700/50000 rank2 | Train Loss: 1.0956 | Train Loss Unweighted: 0.5307 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.1282 | Classification: 0.1011 | Classification_binary: 0.1731 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0603 | Regression_eta: 0.0445 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0515 | LR: 3.41e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1108s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0062 | max_forward: 0.8233 | max_loss: 0.1478 | max_backward: 1.5120 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0564 | Memory: rss_MB: 7420.7 | uss_MB: 7403.4 | child_rss_MB: 3588.8 | allocated_MB: 342.9 | reserved_MB: 25096.0 | inactive_split_MB: 272.6 +rank2 [2026-10-29 10:29:23,500] INFO : Step 32800/50000 rank2 | Train Loss: 1.0857 | Train Loss Unweighted: 0.5241 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1299 | Classification: 0.1011 | Classification_binary: 0.1551 | HitGrouping: 0.1982 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1327 | Regression_eta: 0.0016 | Regression_pt: 0.1319 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0696 | Regression_eta: 0.0348 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0479 | LR: 3.38e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.2396s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0179 | max_forward: 0.9787 | max_loss: 0.1782 | max_backward: 1.5075 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 0.5808 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.1577 | Memory: rss_MB: 7419.5 | uss_MB: 7401.5 | child_rss_MB: 3585.9 | allocated_MB: 345.1 | reserved_MB: 25096.0 | inactive_split_MB: 263.9 +rank2 [2026-10-29 10:33:26,398] INFO : Step 32900/50000 rank2 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1116 | Classification: 0.1066 | Classification_binary: 0.1631 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1300 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1460 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0763 | Regression_eta: 0.0376 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0480 | LR: 3.36e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.7005s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 1.3284 | max_loss: 0.5826 | max_backward: 1.9391 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7423.7 | uss_MB: 7407.1 | child_rss_MB: 3589.0 | allocated_MB: 344.3 | reserved_MB: 26086.0 | inactive_split_MB: 212.1 +rank2 [2026-10-29 10:37:13,222] INFO : Step 33000/50000 rank2 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5373 | Train Loss Components: Auxiliary: 0.2310 | Cardinality: 0.1017 | Classification: 0.1099 | Classification_binary: 0.1844 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0521 | Regression_eta: 0.0399 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0426 | LR: 3.33e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.3113s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 0.7919 | max_loss: 0.2231 | max_backward: 1.5724 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.5025 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1340 | Memory: rss_MB: 7423.5 | uss_MB: 7406.2 | child_rss_MB: 3581.4 | allocated_MB: 344.2 | reserved_MB: 26086.0 | inactive_split_MB: 269.8 +rank2 [2026-10-29 10:37:14,208] INFO : Running validation on rank2 +rank2 [2026-10-29 10:37:14,209] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 10:37:14,209] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 10:38:03,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:38:04,389] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 10:38:31,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 10:38:33,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 10:38:34,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 10:38:35,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 10:38:45,264] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 10:38:45,833] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:38:46,361] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 10:39:02,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 10:39:03,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 10:39:04,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 10:39:05,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 10:39:15,133] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 10:39:15,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:39:16,281] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 10:39:32,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 10:39:32,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 10:39:33,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 10:39:34,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 10:39:44,128] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 10:43:56,310] INFO : Step 33100/50000 rank2 | Train Loss: 1.1040 | Train Loss Unweighted: 0.5268 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1427 | Classification: 0.0921 | Classification_binary: 0.1655 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1319 | Regression_eta: 0.0015 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1715 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0658 | Regression_eta: 0.0402 | Regression_pt: 0.0702 | Regression_sin_phi: 0.1020 | LR: 3.31e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3483s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0325 | max_forward: 0.7755 | max_loss: 0.1478 | max_backward: 1.3916 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.6610 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0760 | Memory: rss_MB: 8784.9 | uss_MB: 8766.5 | child_rss_MB: 3557.4 | allocated_MB: 343.8 | reserved_MB: 23356.0 | inactive_split_MB: 267.3 +rank2 [2026-10-29 10:47:56,313] INFO : Step 33200/50000 rank2 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4955 | Train Loss Components: Auxiliary: 0.2164 | Cardinality: 0.1253 | Classification: 0.1001 | Classification_binary: 0.1407 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0019 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0534 | Regression_eta: 0.0298 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0455 | LR: 3.28e-04 | DataLoad Time: 0.0224s | Model Forward Time: 5.0965s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0401 | max_forward: 0.9954 | max_loss: 0.3116 | max_backward: 2.2139 | max_optimizer_step: 0.2657 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7500.4 | uss_MB: 7483.0 | child_rss_MB: 3560.3 | allocated_MB: 344.1 | reserved_MB: 23356.0 | inactive_split_MB: 241.6 +rank2 [2026-10-29 10:51:55,085] INFO : Step 33300/50000 rank2 | Train Loss: 1.0288 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2302 | Cardinality: 0.1166 | Classification: 0.0975 | Classification_binary: 0.1653 | HitGrouping: 0.1885 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1220 | Regression_eta: 0.0012 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1688 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0704 | Regression_eta: 0.0399 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0693 | LR: 3.26e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5488s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7054 | max_loss: 0.2213 | max_backward: 1.6633 | max_optimizer_step: 0.2474 | max_tensorboard_logging: 0.8237 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0708 | Memory: rss_MB: 7500.4 | uss_MB: 7483.1 | child_rss_MB: 3556.5 | allocated_MB: 343.5 | reserved_MB: 23356.0 | inactive_split_MB: 261.9 +rank2 [2026-10-29 10:55:54,910] INFO : Step 33400/50000 rank2 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5337 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1249 | Classification: 0.0970 | Classification_binary: 0.1720 | HitGrouping: 0.1947 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0013 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1462 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0608 | Regression_eta: 0.0293 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0463 | LR: 3.23e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.4611s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0073 | max_forward: 1.2444 | max_loss: 0.2025 | max_backward: 2.5109 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7502.5 | uss_MB: 7485.1 | child_rss_MB: 3549.5 | allocated_MB: 347.6 | reserved_MB: 23358.0 | inactive_split_MB: 303.0 +rank2 [2026-10-29 10:59:53,596] INFO : Step 33500/50000 rank2 | Train Loss: 1.0164 | Train Loss Unweighted: 0.4956 | Train Loss Components: Auxiliary: 0.2212 | Cardinality: 0.1274 | Classification: 0.0875 | Classification_binary: 0.1421 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1309 | Regression_eta: 0.0013 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1231 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0642 | Regression_eta: 0.0419 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0518 | LR: 3.21e-04 | DataLoad Time: 0.0114s | Model Forward Time: 4.6354s | Phase Times: max_pre_step_barrier: 0.0271 | max_batch_to_device: 0.0558 | max_forward: 0.9437 | max_loss: 0.1437 | max_backward: 1.5429 | max_optimizer_step: 0.2447 | max_tensorboard_logging: 0.8833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7505.9 | uss_MB: 7486.7 | child_rss_MB: 3558.2 | allocated_MB: 341.2 | reserved_MB: 23358.0 | inactive_split_MB: 237.5 +rank2 [2026-10-29 11:05:22,802] INFO : Step 33600/50000 rank2 | Train Loss: 1.0074 | Train Loss Unweighted: 0.4988 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1109 | Classification: 0.0969 | Classification_binary: 0.1509 | HitGrouping: 0.1901 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0012 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0506 | Regression_eta: 0.0255 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0472 | LR: 3.19e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1887s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0970 | max_forward: 0.7667 | max_loss: 0.1478 | max_backward: 1.4192 | max_optimizer_step: 0.2426 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7500.7 | uss_MB: 7485.1 | child_rss_MB: 3550.7 | allocated_MB: 344.5 | reserved_MB: 24528.0 | inactive_split_MB: 238.9 +rank2 [2026-10-29 11:09:17,748] INFO : Step 33700/50000 rank2 | Train Loss: 1.1891 | Train Loss Unweighted: 0.5658 | Train Loss Components: Auxiliary: 0.2602 | Cardinality: 0.1587 | Classification: 0.1116 | Classification_binary: 0.1953 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1270 | Regression_eta: 0.0015 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1940 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0775 | Regression_eta: 0.0448 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0761 | LR: 3.16e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.8186s | Phase Times: max_pre_step_barrier: 0.0253 | max_batch_to_device: 0.0066 | max_forward: 1.3051 | max_loss: 0.1764 | max_backward: 1.7331 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.5706 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0895 | Memory: rss_MB: 7502.0 | uss_MB: 7487.9 | child_rss_MB: 3548.7 | allocated_MB: 347.7 | reserved_MB: 24528.0 | inactive_split_MB: 288.7 +rank2 [2026-10-29 11:13:14,784] INFO : Step 33800/50000 rank2 | Train Loss: 1.0442 | Train Loss Unweighted: 0.4970 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1421 | Classification: 0.0764 | Classification_binary: 0.1525 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1318 | Regression_eta: 0.0014 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0596 | Regression_eta: 0.0226 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0666 | LR: 3.14e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3005s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7370 | max_loss: 0.2458 | max_backward: 1.5902 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.6292 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1331 | Memory: rss_MB: 7496.1 | uss_MB: 7482.2 | child_rss_MB: 3539.4 | allocated_MB: 344.2 | reserved_MB: 24528.0 | inactive_split_MB: 271.5 +rank2 [2026-10-29 11:17:10,799] INFO : Step 33900/50000 rank2 | Train Loss: 1.0341 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1352 | Classification: 0.0865 | Classification_binary: 0.1441 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1254 | Regression_eta: 0.0016 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0657 | Regression_eta: 0.0390 | Regression_pt: 0.0692 | Regression_sin_phi: 0.1030 | LR: 3.11e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3836s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.6305 | max_loss: 0.1507 | max_backward: 1.4015 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9471 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0605 | Memory: rss_MB: 7498.1 | uss_MB: 7488.3 | child_rss_MB: 3543.5 | allocated_MB: 344.1 | reserved_MB: 24528.0 | inactive_split_MB: 268.7 +rank2 [2026-10-29 11:21:05,085] INFO : Step 34000/50000 rank2 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1309 | Classification: 0.0985 | Classification_binary: 0.1707 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1170 | Regression_eta: 0.0014 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1873 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0549 | Regression_eta: 0.0306 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0677 | LR: 3.09e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.6363s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7414 | max_loss: 0.1458 | max_backward: 1.7218 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 7499.1 | uss_MB: 7491.4 | child_rss_MB: 3548.4 | allocated_MB: 342.9 | reserved_MB: 24528.0 | inactive_split_MB: 277.5 +rank2 [2026-10-29 11:21:06,163] INFO : Running validation on rank2 +rank2 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 11:21:56,883] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:21:57,375] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 11:22:16,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 11:22:18,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 11:22:19,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 11:22:21,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 11:22:31,040] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 11:22:32,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:22:32,506] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 11:22:48,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 11:22:49,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 11:22:50,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 11:22:51,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 11:23:01,516] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 11:23:02,010] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:23:02,514] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 11:23:18,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 11:23:19,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 11:23:20,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 11:23:20,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 11:23:30,680] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 11:27:45,673] INFO : Step 34100/50000 rank2 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5582 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1263 | Classification: 0.1031 | Classification_binary: 0.1801 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1358 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1830 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0598 | Regression_eta: 0.0586 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0641 | LR: 3.06e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8615s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9025 | max_loss: 0.1870 | max_backward: 1.7732 | max_optimizer_step: 0.2826 | max_tensorboard_logging: 0.8215 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0738 | Memory: rss_MB: 9109.5 | uss_MB: 8140.4 | child_rss_MB: 3555.5 | allocated_MB: 342.7 | reserved_MB: 22124.0 | inactive_split_MB: 211.6 +rank2 [2026-10-29 11:31:40,876] INFO : Step 34200/50000 rank2 | Train Loss: 0.9931 | Train Loss Unweighted: 0.4794 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1232 | Classification: 0.0771 | Classification_binary: 0.1377 | HitGrouping: 0.1996 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1301 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0631 | Regression_eta: 0.0312 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0468 | LR: 3.04e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.7111s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.8809 | max_loss: 0.3036 | max_backward: 1.8297 | max_optimizer_step: 0.2897 | max_tensorboard_logging: 0.6835 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7608.6 | uss_MB: 6291.1 | child_rss_MB: 3559.3 | allocated_MB: 339.0 | reserved_MB: 22508.0 | inactive_split_MB: 220.4 +rank2 [2026-10-29 11:35:37,837] INFO : Step 34300/50000 rank2 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4920 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1192 | Classification: 0.0965 | Classification_binary: 0.1484 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0347 | Regression_energy: 0.0506 | Regression_eta: 0.0240 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0474 | LR: 3.02e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.3225s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7255 | max_loss: 0.1427 | max_backward: 1.3652 | max_optimizer_step: 0.2496 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7610.6 | uss_MB: 7596.5 | child_rss_MB: 3557.3 | allocated_MB: 345.0 | reserved_MB: 22508.0 | inactive_split_MB: 216.4 +rank2 [2026-10-29 11:39:28,026] INFO : Step 34400/50000 rank2 | Train Loss: 1.0662 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2383 | Cardinality: 0.1289 | Classification: 0.0974 | Classification_binary: 0.1749 | HitGrouping: 0.1864 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1281 | Regression_eta: 0.0014 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1597 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0472 | Regression_eta: 0.0336 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0425 | LR: 2.99e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.1262s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9438 | max_loss: 0.1560 | max_backward: 2.9359 | max_optimizer_step: 0.2526 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7613.7 | uss_MB: 6304.5 | child_rss_MB: 3550.6 | allocated_MB: 345.3 | reserved_MB: 23132.0 | inactive_split_MB: 282.7 +rank2 [2026-10-29 11:43:17,761] INFO : Step 34500/50000 rank2 | Train Loss: 1.0328 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1103 | Classification: 0.0867 | Classification_binary: 0.1518 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0014 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.1350 | Regression_cos_phi: 0.0913 | Regression_energy: 0.0591 | Regression_eta: 0.0380 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0646 | LR: 2.97e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.6256s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.8153 | max_loss: 0.1429 | max_backward: 1.8299 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.5315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7615.7 | uss_MB: 6643.2 | child_rss_MB: 3549.5 | allocated_MB: 341.4 | reserved_MB: 23132.0 | inactive_split_MB: 238.6 +rank2 [2026-10-29 11:47:05,944] INFO : Step 34600/50000 rank2 | Train Loss: 1.0317 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1201 | Classification: 0.0877 | Classification_binary: 0.1434 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1359 | Regression_eta: 0.0014 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0568 | Regression_eta: 0.0369 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0916 | LR: 2.95e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2932s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.6882 | max_loss: 0.1360 | max_backward: 1.3984 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5787 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7607.9 | uss_MB: 6553.3 | child_rss_MB: 3582.8 | allocated_MB: 343.8 | reserved_MB: 23132.0 | inactive_split_MB: 233.1 +rank2 [2026-10-29 11:51:12,922] INFO : Step 34700/50000 rank2 | Train Loss: 1.1152 | Train Loss Unweighted: 0.5285 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1612 | Classification: 0.0937 | Classification_binary: 0.1734 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1283 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1835 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0570 | Regression_eta: 0.0438 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0652 | LR: 2.92e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5244s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7859 | max_loss: 0.2571 | max_backward: 1.9119 | max_optimizer_step: 0.2516 | max_tensorboard_logging: 0.5032 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7610.6 | uss_MB: 7599.4 | child_rss_MB: 3559.6 | allocated_MB: 343.9 | reserved_MB: 23132.0 | inactive_split_MB: 446.8 +rank2 [2026-10-29 11:54:56,884] INFO : Step 34800/50000 rank2 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5142 | Train Loss Components: Auxiliary: 0.2371 | Cardinality: 0.1583 | Classification: 0.0857 | Classification_binary: 0.1750 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1255 | Regression_eta: 0.0013 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1863 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0636 | Regression_eta: 0.0413 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0677 | LR: 2.90e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2212s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0423 | max_forward: 0.8214 | max_loss: 0.1472 | max_backward: 1.4019 | max_optimizer_step: 0.3234 | max_tensorboard_logging: 0.5950 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7613.6 | uss_MB: 7602.4 | child_rss_MB: 3541.0 | allocated_MB: 344.5 | reserved_MB: 23132.0 | inactive_split_MB: 201.5 +rank2 [2026-10-29 11:58:48,855] INFO : Step 34900/50000 rank2 | Train Loss: 0.9512 | Train Loss Unweighted: 0.4678 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1120 | Classification: 0.0788 | Classification_binary: 0.1318 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1262 | Regression_eta: 0.0014 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1236 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0635 | Regression_eta: 0.0296 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0397 | LR: 2.88e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2611s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0986 | max_forward: 0.7397 | max_loss: 0.1421 | max_backward: 1.3918 | max_optimizer_step: 0.3560 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7614.6 | uss_MB: 7602.0 | child_rss_MB: 3543.9 | allocated_MB: 341.5 | reserved_MB: 23500.0 | inactive_split_MB: 262.9 +rank2 [2026-10-29 12:02:43,664] INFO : Step 35000/50000 rank2 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1401 | Classification: 0.0852 | Classification_binary: 0.1555 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0012 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1569 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0473 | Regression_eta: 0.0247 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0460 | LR: 2.85e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7045s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0159 | max_forward: 0.7961 | max_loss: 0.2615 | max_backward: 2.0104 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6382 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7618.3 | uss_MB: 7606.3 | child_rss_MB: 3544.7 | allocated_MB: 345.2 | reserved_MB: 23500.0 | inactive_split_MB: 284.5 +rank2 [2026-10-29 12:02:44,719] INFO : Running validation on rank2 +rank2 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 12:03:33,662] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 12:03:34,210] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 12:04:01,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 12:04:03,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 12:04:04,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 12:04:05,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 12:04:15,728] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 12:04:18,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 12:04:18,509] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 12:04:34,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 12:04:35,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 12:04:36,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 12:04:37,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 12:04:47,720] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 12:04:48,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 12:04:48,821] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 12:05:04,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 12:05:04,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 12:05:05,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 12:05:06,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 12:05:16,317] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 12:09:34,464] INFO : Step 35100/50000 rank2 | Train Loss: 1.1060 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1387 | Classification: 0.0926 | Classification_binary: 0.1848 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0014 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0581 | Regression_eta: 0.0400 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0721 | LR: 2.83e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.6641s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.9070 | max_loss: 0.1513 | max_backward: 2.1746 | max_optimizer_step: 0.2850 | max_tensorboard_logging: 1.1439 | max_tensorboard_flush: 0.0024 | max_post_step_barrier: 0.0768 | Memory: rss_MB: 9217.3 | uss_MB: 7564.1 | child_rss_MB: 3570.6 | allocated_MB: 344.3 | reserved_MB: 22622.0 | inactive_split_MB: 852.9 +rank2 [2026-10-29 12:13:24,279] INFO : Step 35200/50000 rank2 | Train Loss: 1.0386 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1293 | Classification: 0.0826 | Classification_binary: 0.1559 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1247 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.1558 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0608 | Regression_eta: 0.0488 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0730 | LR: 2.81e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.5311s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.8193 | max_loss: 0.1608 | max_backward: 1.3752 | max_optimizer_step: 0.2535 | max_tensorboard_logging: 0.9185 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0700 | Memory: rss_MB: 7738.0 | uss_MB: 7717.0 | child_rss_MB: 3546.7 | allocated_MB: 341.8 | reserved_MB: 25614.0 | inactive_split_MB: 192.0 +rank2 [2026-10-29 12:17:18,033] INFO : Step 35300/50000 rank2 | Train Loss: 0.9736 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1059 | Classification: 0.0970 | Classification_binary: 0.1371 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0016 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0539 | Regression_eta: 0.0365 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0418 | LR: 2.79e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6480s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6434 | max_loss: 0.2341 | max_backward: 1.6834 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0588 | Memory: rss_MB: 7740.0 | uss_MB: 7297.3 | child_rss_MB: 3543.2 | allocated_MB: 343.2 | reserved_MB: 25614.0 | inactive_split_MB: 226.8 +rank2 [2026-10-29 12:21:06,026] INFO : Step 35400/50000 rank2 | Train Loss: 1.1186 | Train Loss Unweighted: 0.5363 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1393 | Classification: 0.1069 | Classification_binary: 0.1767 | HitGrouping: 0.1943 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0016 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.2136 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0613 | Regression_eta: 0.0587 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0738 | LR: 2.76e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.8165s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0060 | max_forward: 0.7591 | max_loss: 0.1562 | max_backward: 1.6040 | max_optimizer_step: 0.2543 | max_tensorboard_logging: 0.9624 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.0563 | Memory: rss_MB: 7735.5 | uss_MB: 7248.0 | child_rss_MB: 3540.1 | allocated_MB: 345.3 | reserved_MB: 25762.0 | inactive_split_MB: 277.5 +rank2 [2026-10-29 12:24:54,118] INFO : Step 35500/50000 rank2 | Train Loss: 1.1216 | Train Loss Unweighted: 0.5466 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1527 | Classification: 0.0970 | Classification_binary: 0.1767 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1342 | Regression_eta: 0.0014 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1633 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0567 | Regression_eta: 0.0295 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0515 | LR: 2.74e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4075s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0648 | max_forward: 0.8748 | max_loss: 0.1544 | max_backward: 1.6387 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 0.5130 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0632 | Memory: rss_MB: 7738.5 | uss_MB: 7721.7 | child_rss_MB: 3543.4 | allocated_MB: 343.4 | reserved_MB: 25762.0 | inactive_split_MB: 284.4 +rank2 [2026-10-29 12:28:50,969] INFO : Step 35600/50000 rank2 | Train Loss: 1.0380 | Train Loss Unweighted: 0.5039 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1124 | Classification: 0.0922 | Classification_binary: 0.1405 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0828 | Regression_eta: 0.0345 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0483 | LR: 2.72e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8986s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0559 | max_forward: 0.7376 | max_loss: 0.1415 | max_backward: 1.1754 | max_optimizer_step: 0.2524 | max_tensorboard_logging: 0.5225 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1603 | Memory: rss_MB: 7739.4 | uss_MB: 7724.9 | child_rss_MB: 3557.9 | allocated_MB: 343.8 | reserved_MB: 25762.0 | inactive_split_MB: 233.6 +rank2 [2026-10-29 12:32:54,485] INFO : Step 35700/50000 rank2 | Train Loss: 1.0352 | Train Loss Unweighted: 0.5000 | Train Loss Components: Auxiliary: 0.2288 | Cardinality: 0.1199 | Classification: 0.1043 | Classification_binary: 0.1492 | HitGrouping: 0.1850 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1212 | Regression_eta: 0.0015 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0450 | Regression_eta: 0.0270 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0703 | LR: 2.70e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3753s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0360 | max_forward: 0.7358 | max_loss: 0.1419 | max_backward: 1.6761 | max_optimizer_step: 0.2531 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7741.1 | uss_MB: 7726.8 | child_rss_MB: 3543.5 | allocated_MB: 344.2 | reserved_MB: 26034.0 | inactive_split_MB: 271.8 +rank2 [2026-10-29 12:36:36,039] INFO : Step 35800/50000 rank2 | Train Loss: 1.0929 | Train Loss Unweighted: 0.5323 | Train Loss Components: Auxiliary: 0.2446 | Cardinality: 0.1241 | Classification: 0.0986 | Classification_binary: 0.1801 | HitGrouping: 0.1908 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0014 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0601 | Regression_eta: 0.0390 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0478 | LR: 2.68e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1519s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7732 | max_loss: 0.1506 | max_backward: 1.4048 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0581 | Memory: rss_MB: 7740.3 | uss_MB: 7206.4 | child_rss_MB: 3548.3 | allocated_MB: 344.1 | reserved_MB: 26034.0 | inactive_split_MB: 276.1 +rank2 [2026-10-29 14:48:22,173] INFO : Initializing model with seed=12345; process seed=12347 +rank2 [2026-10-29 14:48:22,183] INFO : configured dtype=torch.bfloat16 for autocast +rank2 [2026-10-29 14:48:45,046] INFO : Instantiating model +rank2 [2026-10-29 14:48:45,046] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank2 [2026-10-29 14:48:45,046] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank2 [2026-10-29 14:48:45,064] INFO : Input encoding initialization took 0.02s +rank2 [2026-10-29 14:48:45,064] INFO : nn0 parameters: 171776 +rank2 [2026-10-29 14:48:45,064] INFO : Initializing shared backbone layers, num_convs=6 +rank2 [2026-10-29 14:48:45,064] INFO : Initializing backbone layer 0 +rank2 [2026-10-29 14:48:45,064] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:48:45,068] INFO : Initializing backbone layer 1 +rank2 [2026-10-29 14:48:45,068] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:48:45,071] INFO : Initializing backbone layer 2 +rank2 [2026-10-29 14:48:45,071] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:48:45,074] INFO : Initializing backbone layer 3 +rank2 [2026-10-29 14:48:45,074] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:48:45,077] INFO : Initializing backbone layer 4 +rank2 [2026-10-29 14:48:45,077] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:48:45,080] INFO : Initializing backbone layer 5 +rank2 [2026-10-29 14:48:45,081] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank2 [2026-10-29 14:48:45,084] INFO : Backbone initialization took 0.02s +rank2 [2026-10-29 14:48:45,084] INFO : backbone parameters: 2374656 +rank2 [2026-10-29 14:48:45,084] INFO : Initializing output DNNs +rank2 [2026-10-29 14:48:45,133] INFO : Output DNNs initialization took 0.05s +rank2 [2026-10-29 14:48:45,133] INFO : backbone_mode=BackboneMode.SHARED +rank2 [2026-10-29 14:48:45,133] INFO : jagged_attention enabled=True +rank2 [2026-10-29 14:48:45,133] INFO : flash_attn_varlen enabled=True +rank2 [2026-10-29 14:48:45,133] INFO : task_queries enabled=False +rank2 [2026-10-29 14:48:45,133] INFO : classification_norm parameters: 0 +rank2 [2026-10-29 14:48:45,133] INFO : regression_norm parameters: 0 +rank2 [2026-10-29 14:48:45,133] INFO : classification_query parameters: 0 +rank2 [2026-10-29 14:48:45,133] INFO : regression_query parameters: 0 +rank2 [2026-10-29 14:48:45,133] INFO : classification_readout parameters: 0 +rank2 [2026-10-29 14:48:45,133] INFO : regression_readout parameters: 0 +rank2 [2026-10-29 14:48:45,133] INFO : output_mode=set +rank2 [2026-10-29 14:48:45,134] INFO : set_decoder parameters: 4713812 +rank2 [2026-10-29 14:48:45,134] INFO : nn_binary_particle parameters: 0 +rank2 [2026-10-29 14:48:45,134] INFO : nn_pid parameters: 0 +rank2 [2026-10-29 14:48:45,134] INFO : nn_pt parameters: 0 +rank2 [2026-10-29 14:48:45,134] INFO : nn_eta parameters: 0 +rank2 [2026-10-29 14:48:45,134] INFO : nn_sin_phi parameters: 0 +rank2 [2026-10-29 14:48:45,134] INFO : nn_cos_phi parameters: 0 +rank2 [2026-10-29 14:48:45,134] INFO : nn_energy parameters: 0 +rank2 [2026-10-29 14:48:45,134] INFO : Total MLPF parameters: 7260244 +rank2 [2026-10-29 14:48:45,134] INFO : MLPF __init__ done +rank2 [2026-10-29 14:48:45,136] INFO : Instantiated model +rank2 [2026-10-29 14:48:45,136] INFO : Moving model to device rank=2 +rank2 [2026-10-29 14:48:45,171] INFO : Moved model to device rank=2 +rank2 [2026-10-29 14:48:45,172] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank2 [2026-10-29 14:48:45,514] INFO : Restoring training from step 35001 +rank2 [2026-10-29 14:48:45,520] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth +rank2 [2026-10-29 14:48:45,520] INFO : Configured model for SyncBatchNorm rank=2 +rank2 [2026-10-29 14:48:45,764] INFO : Configured model for DistributedDataParallel rank=2 +rank2 [2026-10-29 14:48:45,876] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank2 [2026-10-29 14:48:45,876] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank2 [2026-10-29 14:48:45,878] INFO : Trainable parameters: 7260244 +rank2 [2026-10-29 14:48:45,878] INFO : Non-trainable parameters: 0 +rank2 [2026-10-29 14:48:45,878] INFO : Total parameters: 7260244 +rank2 [2026-10-29 14:48:45,880] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank2 [2026-10-29 14:48:45,881] INFO : Getting dataloaders +rank2 [2026-10-29 14:48:45,881] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:48:51,678] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:48:52,999] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:48:53,950] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-29 14:48:53,950] INFO : Creating WeightedMixIterator with 1 data loaders. +rank2 [2026-10-29 14:48:53,951] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:48:54,226] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:48:54,496] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:48:54,745] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-29 14:48:54,746] INFO : Creating InterleavedIterator with 1 data loaders. +rank2 [2026-10-29 14:49:08,219] INFO : Got dataloaders +rank2 [2026-10-29 14:49:08,219] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13888] of epochs [1] +rank2 [2026-10-29 14:49:08,219] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank2 [2026-10-29 14:49:08,219] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank2 [2026-10-29 14:49:08,219] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank2 [2026-10-29 14:49:09,957] INFO : loader split=train rank=2 len=21112 +rank2 [2026-10-29 14:49:09,957] INFO : loader split=valid rank=2 len=12 +rank2 [2026-10-29 14:49:09,957] INFO : Starting training from step 35001 to 50000 on rank 2 of 8 +rank2 [2026-10-29 14:49:09,957] INFO : Creating train_iterator +rank2 [2026-10-29 14:49:09,957] INFO : ResumableSampler physical:train loading state: start_index=222208 +rank2 [2026-10-29 14:49:24,146] INFO : Created train_iterator +rank2 [2026-10-29 14:53:12,252] INFO : Step 35100/50000 rank2 | Train Loss: 1.0826 | Train Loss Unweighted: 0.5185 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1393 | Classification: 0.0961 | Classification_binary: 0.1822 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2071 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0591 | Regression_eta: 0.0289 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0543 | LR: 2.83e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.4935s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.7340 | max_loss: 0.1651 | max_backward: 1.4152 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4761 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4625.8 | uss_MB: 3388.6 | child_rss_MB: 5578.9 | allocated_MB: 372.6 | reserved_MB: 22674.0 | inactive_split_MB: 823.1 +rank2 [2026-10-29 14:56:45,511] INFO : Step 35200/50000 rank2 | Train Loss: 1.0562 | Train Loss Unweighted: 0.5125 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1301 | Classification: 0.0878 | Classification_binary: 0.1540 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1331 | Regression_eta: 0.0017 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1516 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0614 | Regression_eta: 0.0389 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0832 | LR: 2.81e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.1439s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6042 | max_loss: 0.1411 | max_backward: 1.1580 | max_optimizer_step: 0.1447 | max_tensorboard_logging: 0.5108 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4625.0 | uss_MB: 3391.6 | child_rss_MB: 5591.1 | allocated_MB: 368.1 | reserved_MB: 25668.0 | inactive_split_MB: 192.7 +rank2 [2026-10-29 15:00:19,321] INFO : Step 35300/50000 rank2 | Train Loss: 0.9962 | Train Loss Unweighted: 0.4725 | Train Loss Components: Auxiliary: 0.2160 | Cardinality: 0.1172 | Classification: 0.0927 | Classification_binary: 0.1341 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0603 | Regression_eta: 0.0362 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0419 | LR: 2.79e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.0275s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.5940 | max_loss: 0.1358 | max_backward: 1.1911 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0027 | Memory: rss_MB: 4636.4 | uss_MB: 3402.6 | child_rss_MB: 5598.9 | allocated_MB: 371.5 | reserved_MB: 25668.0 | inactive_split_MB: 207.6 +rank2 [2026-10-29 15:03:53,852] INFO : Step 35400/50000 rank2 | Train Loss: 1.1270 | Train Loss Unweighted: 0.5445 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1356 | Classification: 0.0990 | Classification_binary: 0.1805 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1297 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.2261 | Regression_cos_phi: 0.0917 | Regression_energy: 0.0605 | Regression_eta: 0.0548 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0928 | LR: 2.76e-04 | DataLoad Time: 0.0149s | Model Forward Time: 3.6499s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1507 | max_backward: 1.5023 | max_optimizer_step: 0.2459 | max_tensorboard_logging: 0.5679 | max_tensorboard_flush: 0.0064 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4646.3 | uss_MB: 3410.8 | child_rss_MB: 5599.4 | allocated_MB: 371.4 | reserved_MB: 25816.0 | inactive_split_MB: 263.6 +rank2 [2026-10-29 15:07:33,750] INFO : Step 35500/50000 rank2 | Train Loss: 1.0502 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1302 | Classification: 0.0904 | Classification_binary: 0.1703 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1250 | Regression_eta: 0.0013 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1550 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0561 | Regression_eta: 0.0306 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0647 | LR: 2.74e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.6099s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0737 | max_forward: 0.7546 | max_loss: 0.1581 | max_backward: 1.4977 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4644.7 | uss_MB: 3412.9 | child_rss_MB: 5568.4 | allocated_MB: 371.1 | reserved_MB: 25816.0 | inactive_split_MB: 263.4 +rank2 [2026-10-29 15:11:19,389] INFO : Step 35600/50000 rank2 | Train Loss: 1.0258 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2226 | Cardinality: 0.1119 | Classification: 0.0922 | Classification_binary: 0.1418 | HitGrouping: 0.1950 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1286 | Regression_eta: 0.0015 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1545 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0676 | Regression_eta: 0.0378 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0460 | LR: 2.72e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.2537s | Phase Times: max_pre_step_barrier: 0.0929 | max_batch_to_device: 0.0064 | max_forward: 0.5696 | max_loss: 0.1430 | max_backward: 1.1627 | max_optimizer_step: 0.1475 | max_tensorboard_logging: 0.5805 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4646.9 | uss_MB: 3412.7 | child_rss_MB: 5602.7 | allocated_MB: 370.5 | reserved_MB: 25816.0 | inactive_split_MB: 212.3 +rank2 [2026-10-29 15:15:08,570] INFO : Step 35700/50000 rank2 | Train Loss: 1.0162 | Train Loss Unweighted: 0.5013 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1185 | Classification: 0.0946 | Classification_binary: 0.1537 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1248 | Regression_eta: 0.0015 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0549 | Regression_eta: 0.0360 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0390 | LR: 2.70e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5938s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7397 | max_loss: 0.1413 | max_backward: 1.6688 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4646.6 | uss_MB: 3411.7 | child_rss_MB: 5604.2 | allocated_MB: 372.2 | reserved_MB: 26084.0 | inactive_split_MB: 249.8 +rank2 [2026-10-29 15:18:49,408] INFO : Step 35800/50000 rank2 | Train Loss: 1.0884 | Train Loss Unweighted: 0.5265 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1296 | Classification: 0.0948 | Classification_binary: 0.1803 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1779 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0511 | Regression_eta: 0.0335 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0405 | LR: 2.68e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2013s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0199 | max_forward: 0.9402 | max_loss: 0.2389 | max_backward: 1.7889 | max_optimizer_step: 0.2466 | max_tensorboard_logging: 0.7071 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4643.4 | uss_MB: 3414.5 | child_rss_MB: 5602.6 | allocated_MB: 371.6 | reserved_MB: 26084.0 | inactive_split_MB: 257.5 +rank2 [2026-10-29 15:22:30,828] INFO : Step 35900/50000 rank2 | Train Loss: 1.0760 | Train Loss Unweighted: 0.5299 | Train Loss Components: Auxiliary: 0.2328 | Cardinality: 0.1277 | Classification: 0.0983 | Classification_binary: 0.1568 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1349 | Regression_eta: 0.0016 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0820 | Regression_eta: 0.0574 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0630 | LR: 2.65e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.2905s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6268 | max_loss: 0.2373 | max_backward: 1.2570 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.4934 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0016 | Memory: rss_MB: 4639.4 | uss_MB: 3412.6 | child_rss_MB: 5605.5 | allocated_MB: 369.7 | reserved_MB: 26800.0 | inactive_split_MB: 294.3 +rank2 [2026-10-29 15:26:13,158] INFO : Step 36000/50000 rank2 | Train Loss: 1.0831 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1276 | Classification: 0.1031 | Classification_binary: 0.1434 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1452 | Regression_eta: 0.0017 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1217 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0678 | Regression_eta: 0.0448 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0570 | LR: 2.63e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.2174s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6266 | max_loss: 0.1437 | max_backward: 1.3209 | max_optimizer_step: 0.1464 | max_tensorboard_logging: 0.4977 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4620.1 | uss_MB: 3411.5 | child_rss_MB: 5582.3 | allocated_MB: 368.7 | reserved_MB: 26800.0 | inactive_split_MB: 270.9 +rank2 [2026-10-29 15:26:14,074] INFO : Running validation on rank2 +rank2 [2026-10-29 15:26:14,074] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 15:26:14,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 15:27:03,341] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:27:03,871] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 15:27:29,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 15:27:30,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 15:27:31,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 15:27:32,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 15:27:42,617] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 15:27:42,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:27:43,314] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 15:27:59,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 15:28:00,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 15:28:01,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 15:28:01,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 15:28:11,968] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 15:28:12,651] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:28:13,125] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 15:28:28,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 15:28:29,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 15:28:30,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 15:28:30,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 15:28:40,707] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 15:32:50,525] INFO : Step 36100/50000 rank2 | Train Loss: 1.0137 | Train Loss Unweighted: 0.5026 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1079 | Classification: 0.1014 | Classification_binary: 0.1522 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1216 | Regression_eta: 0.0013 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0597 | Regression_eta: 0.0286 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0697 | LR: 2.61e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0475s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0065 | max_forward: 0.6968 | max_loss: 0.2782 | max_backward: 1.4278 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.9130 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6101.3 | uss_MB: 4907.2 | child_rss_MB: 5609.1 | allocated_MB: 372.5 | reserved_MB: 23174.0 | inactive_split_MB: 246.6 +rank2 [2026-10-29 15:36:33,883] INFO : Step 36200/50000 rank2 | Train Loss: 1.0439 | Train Loss Unweighted: 0.5147 | Train Loss Components: Auxiliary: 0.2338 | Cardinality: 0.1304 | Classification: 0.0909 | Classification_binary: 0.1697 | HitGrouping: 0.1917 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1255 | Regression_eta: 0.0011 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0407 | Regression_energy: 0.0483 | Regression_eta: 0.0256 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0467 | LR: 2.59e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5413s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7440 | max_loss: 0.1544 | max_backward: 1.4037 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 6101.4 | uss_MB: 4909.6 | child_rss_MB: 5587.9 | allocated_MB: 367.8 | reserved_MB: 23174.0 | inactive_split_MB: 282.8 +rank2 [2026-10-29 15:40:20,924] INFO : Step 36300/50000 rank2 | Train Loss: 1.0379 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1216 | Classification: 0.0886 | Classification_binary: 0.1447 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1266 | Regression_eta: 0.0016 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1545 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0594 | Regression_eta: 0.0388 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0599 | LR: 2.57e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.3304s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6106 | max_loss: 0.2412 | max_backward: 1.3407 | max_optimizer_step: 0.1535 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1063 | Memory: rss_MB: 5450.1 | uss_MB: 4269.0 | child_rss_MB: 5611.1 | allocated_MB: 370.5 | reserved_MB: 23174.0 | inactive_split_MB: 274.2 +rank2 [2026-10-29 15:44:06,295] INFO : Step 36400/50000 rank2 | Train Loss: 1.0182 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.0976 | Classification: 0.0939 | Classification_binary: 0.1520 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1167 | Regression_eta: 0.0016 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1625 | Regression_cos_phi: 0.0802 | Regression_energy: 0.0620 | Regression_eta: 0.0394 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0885 | LR: 2.55e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5215s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.6772 | max_loss: 0.1711 | max_backward: 1.4058 | max_optimizer_step: 0.1518 | max_tensorboard_logging: 0.5460 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0089 | Memory: rss_MB: 5451.9 | uss_MB: 4272.1 | child_rss_MB: 5615.2 | allocated_MB: 371.8 | reserved_MB: 23382.0 | inactive_split_MB: 230.6 +rank2 [2026-10-29 15:47:54,075] INFO : Step 36500/50000 rank2 | Train Loss: 1.0953 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1370 | Classification: 0.0895 | Classification_binary: 0.1838 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1828 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0629 | Regression_eta: 0.0418 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0673 | LR: 2.52e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9826s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7475 | max_loss: 0.2537 | max_backward: 1.7352 | max_optimizer_step: 0.1520 | max_tensorboard_logging: 0.6267 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5440.5 | uss_MB: 4262.4 | child_rss_MB: 5614.9 | allocated_MB: 372.3 | reserved_MB: 24160.0 | inactive_split_MB: 275.1 +rank2 [2026-10-29 15:51:40,079] INFO : Step 36600/50000 rank2 | Train Loss: 0.9967 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1165 | Classification: 0.0804 | Classification_binary: 0.1438 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1366 | Regression_eta: 0.0013 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0730 | Regression_eta: 0.0667 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0760 | LR: 2.50e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6289s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6592 | max_loss: 0.2501 | max_backward: 1.4213 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5953 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5450.1 | uss_MB: 4272.8 | child_rss_MB: 5617.4 | allocated_MB: 367.8 | reserved_MB: 27180.0 | inactive_split_MB: 225.4 +rank2 [2026-10-29 15:55:26,197] INFO : Step 36700/50000 rank2 | Train Loss: 0.9770 | Train Loss Unweighted: 0.4817 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1070 | Classification: 0.0901 | Classification_binary: 0.1347 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1164 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0587 | Regression_eta: 0.0274 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0508 | LR: 2.48e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.2750s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0062 | max_forward: 0.5676 | max_loss: 0.2381 | max_backward: 1.1343 | max_optimizer_step: 0.2417 | max_tensorboard_logging: 0.4693 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5448.5 | uss_MB: 4274.1 | child_rss_MB: 5620.1 | allocated_MB: 370.8 | reserved_MB: 27180.0 | inactive_split_MB: 235.2 +rank2 [2026-10-29 15:59:20,983] INFO : Step 36800/50000 rank2 | Train Loss: 1.0234 | Train Loss Unweighted: 0.4967 | Train Loss Components: Auxiliary: 0.2267 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1662 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1197 | Regression_eta: 0.0013 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0541 | Regression_eta: 0.0335 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0693 | LR: 2.46e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5128s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 1.1394 | max_loss: 0.1518 | max_backward: 2.3088 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5450.3 | uss_MB: 4278.9 | child_rss_MB: 5629.5 | allocated_MB: 371.9 | reserved_MB: 27180.0 | inactive_split_MB: 230.7 +rank2 [2026-10-29 16:03:11,411] INFO : Step 36900/50000 rank2 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2268 | Cardinality: 0.1211 | Classification: 0.0855 | Classification_binary: 0.1637 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1219 | Regression_eta: 0.0015 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1853 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0540 | Regression_eta: 0.0382 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0554 | LR: 2.44e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.5002s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0071 | max_forward: 0.7005 | max_loss: 0.1465 | max_backward: 1.2640 | max_optimizer_step: 0.1579 | max_tensorboard_logging: 0.4728 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5451.1 | uss_MB: 4280.1 | child_rss_MB: 5636.6 | allocated_MB: 372.3 | reserved_MB: 27180.0 | inactive_split_MB: 201.7 +rank2 [2026-10-29 16:07:15,145] INFO : Step 37000/50000 rank2 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2194 | Cardinality: 0.1268 | Classification: 0.0926 | Classification_binary: 0.1402 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0014 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1817 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0619 | Regression_eta: 0.0437 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0616 | LR: 2.42e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5220s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6075 | max_loss: 0.2475 | max_backward: 1.2956 | max_optimizer_step: 0.1585 | max_tensorboard_logging: 0.7613 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5445.1 | uss_MB: 4277.2 | child_rss_MB: 5637.5 | allocated_MB: 368.1 | reserved_MB: 27180.0 | inactive_split_MB: 226.3 +rank2 [2026-10-29 16:07:16,059] INFO : Running validation on rank2 +rank2 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 16:08:14,060] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:08:14,583] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 16:08:39,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 16:08:40,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 16:08:41,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 16:08:42,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 16:08:52,768] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 16:08:53,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:08:53,528] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 16:09:09,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 16:09:10,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 16:09:11,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 16:09:12,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 16:09:22,290] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 16:09:23,588] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:09:24,058] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 16:09:39,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 16:09:39,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 16:09:40,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 16:09:41,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 16:09:51,291] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 16:13:56,322] INFO : Step 37100/50000 rank2 | Train Loss: 1.1150 | Train Loss Unweighted: 0.5431 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1226 | Classification: 0.1105 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1340 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1749 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0600 | Regression_eta: 0.0320 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0712 | LR: 2.40e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8590s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7539 | max_loss: 0.3988 | max_backward: 1.7903 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4747 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6593.7 | uss_MB: 5426.2 | child_rss_MB: 5642.5 | allocated_MB: 372.5 | reserved_MB: 23160.0 | inactive_split_MB: 265.5 +rank2 [2026-10-29 16:17:46,448] INFO : Step 37200/50000 rank2 | Train Loss: 1.0752 | Train Loss Unweighted: 0.5181 | Train Loss Components: Auxiliary: 0.2406 | Cardinality: 0.1258 | Classification: 0.0924 | Classification_binary: 0.1812 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1200 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0518 | Regression_eta: 0.0304 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0347 | LR: 2.38e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.1718s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0549 | max_forward: 0.8058 | max_loss: 0.1510 | max_backward: 1.8476 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5280 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6591.2 | uss_MB: 5426.9 | child_rss_MB: 5644.6 | allocated_MB: 371.5 | reserved_MB: 24140.0 | inactive_split_MB: 271.0 +rank2 [2026-10-29 16:21:30,619] INFO : Step 37300/50000 rank2 | Train Loss: 1.0230 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1563 | HitGrouping: 0.1981 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1213 | Regression_eta: 0.0016 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0638 | Regression_eta: 0.0581 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0563 | LR: 2.36e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0301s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.6853 | max_loss: 0.1502 | max_backward: 1.3245 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 1.0657 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5580.1 | uss_MB: 4415.2 | child_rss_MB: 5668.9 | allocated_MB: 370.9 | reserved_MB: 24140.0 | inactive_split_MB: 261.7 +rank2 [2026-10-29 16:25:14,030] INFO : Step 37400/50000 rank2 | Train Loss: 1.0130 | Train Loss Unweighted: 0.4929 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1180 | Classification: 0.0961 | Classification_binary: 0.1390 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1273 | Regression_eta: 0.0015 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1538 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0581 | Regression_eta: 0.0603 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0627 | LR: 2.34e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5039s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.5424 | max_loss: 0.1464 | max_backward: 1.0792 | max_optimizer_step: 0.1675 | max_tensorboard_logging: 0.8668 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5585.6 | uss_MB: 4420.8 | child_rss_MB: 5670.0 | allocated_MB: 368.8 | reserved_MB: 25122.0 | inactive_split_MB: 232.7 +rank2 [2026-10-29 16:30:41,706] INFO : Step 37500/50000 rank2 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1315 | Classification: 0.0992 | Classification_binary: 0.1679 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1176 | Regression_eta: 0.0013 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0545 | Regression_eta: 0.0372 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0541 | LR: 2.32e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.1494s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.7240 | max_loss: 0.1472 | max_backward: 1.5381 | max_optimizer_step: 0.2707 | max_tensorboard_logging: 0.8554 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5585.8 | uss_MB: 4422.3 | child_rss_MB: 5642.8 | allocated_MB: 370.2 | reserved_MB: 26134.0 | inactive_split_MB: 179.7 +rank2 [2026-10-29 16:34:38,362] INFO : Step 37600/50000 rank2 | Train Loss: 1.0243 | Train Loss Unweighted: 0.4867 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1182 | Classification: 0.0808 | Classification_binary: 0.1619 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1447 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0461 | Regression_eta: 0.0436 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0524 | LR: 2.30e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1672s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.7069 | max_loss: 0.1543 | max_backward: 1.5139 | max_optimizer_step: 0.1704 | max_tensorboard_logging: 1.0159 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5587.9 | uss_MB: 4422.7 | child_rss_MB: 5672.0 | allocated_MB: 370.8 | reserved_MB: 26846.0 | inactive_split_MB: 273.3 +rank2 [2026-10-29 16:38:29,294] INFO : Step 37700/50000 rank2 | Train Loss: 0.9610 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2102 | Cardinality: 0.1049 | Classification: 0.0974 | Classification_binary: 0.1317 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0012 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1278 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0568 | Regression_eta: 0.0295 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0460 | LR: 2.28e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5168s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6233 | max_loss: 0.2433 | max_backward: 1.3306 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5644 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5591.0 | uss_MB: 4425.5 | child_rss_MB: 5643.2 | allocated_MB: 369.4 | reserved_MB: 28822.0 | inactive_split_MB: 231.5 +rank2 [2026-10-29 16:42:35,307] INFO : Step 37800/50000 rank2 | Train Loss: 0.9994 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1105 | Classification: 0.0952 | Classification_binary: 0.1458 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1161 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1767 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0561 | Regression_eta: 0.0404 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0692 | LR: 2.26e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.8912s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.6659 | max_loss: 0.1473 | max_backward: 1.3840 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.9558 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5593.5 | uss_MB: 4427.0 | child_rss_MB: 5671.1 | allocated_MB: 371.5 | reserved_MB: 28822.0 | inactive_split_MB: 170.1 +rank2 [2026-10-29 16:46:40,061] INFO : Step 37900/50000 rank2 | Train Loss: 1.0837 | Train Loss Unweighted: 0.5094 | Train Loss Components: Auxiliary: 0.2372 | Cardinality: 0.1419 | Classification: 0.0937 | Classification_binary: 0.1795 | HitGrouping: 0.1848 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1160 | Regression_eta: 0.0015 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0566 | Regression_eta: 0.0476 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0723 | LR: 2.24e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.2210s | Phase Times: max_pre_step_barrier: 0.1076 | max_batch_to_device: 0.0071 | max_forward: 0.7572 | max_loss: 0.1547 | max_backward: 1.4632 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 0.9205 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 5593.0 | uss_MB: 4429.3 | child_rss_MB: 5670.1 | allocated_MB: 370.2 | reserved_MB: 28822.0 | inactive_split_MB: 265.3 +rank2 [2026-10-29 16:50:31,520] INFO : Step 38000/50000 rank2 | Train Loss: 0.9811 | Train Loss Unweighted: 0.4782 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1213 | Classification: 0.0781 | Classification_binary: 0.1413 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1277 | Regression_eta: 0.0012 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1608 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0628 | Regression_eta: 0.0571 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0566 | LR: 2.22e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.5109s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.6120 | max_loss: 0.2171 | max_backward: 1.2526 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5836 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5590.4 | uss_MB: 4430.0 | child_rss_MB: 5667.2 | allocated_MB: 368.7 | reserved_MB: 28822.0 | inactive_split_MB: 223.4 +rank2 [2026-10-29 16:50:32,412] INFO : Running validation on rank2 +rank2 [2026-10-29 16:50:32,412] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 16:50:32,412] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 16:51:23,718] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:51:24,264] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 16:51:51,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 16:51:52,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 16:51:53,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 16:51:54,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 16:52:04,019] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 16:52:04,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:52:05,008] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 16:52:20,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 16:52:21,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 16:52:22,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 16:52:23,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 16:52:33,513] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 16:52:34,974] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:52:35,418] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 16:52:51,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 16:52:51,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 16:52:52,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 16:52:53,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 16:53:03,197] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 16:57:14,096] INFO : Step 38100/50000 rank2 | Train Loss: 1.0158 | Train Loss Unweighted: 0.4912 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1220 | Classification: 0.0982 | Classification_binary: 0.1455 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1224 | Regression_eta: 0.0013 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0492 | Regression_eta: 0.0388 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0636 | LR: 2.20e-04 | DataLoad Time: 0.0253s | Model Forward Time: 4.9412s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8406 | max_loss: 0.1820 | max_backward: 1.8536 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.3354 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0179 | Memory: rss_MB: 5714.6 | uss_MB: 4482.4 | child_rss_MB: 5669.8 | allocated_MB: 370.5 | reserved_MB: 24292.0 | inactive_split_MB: 257.8 +rank2 [2026-10-29 17:01:01,944] INFO : Step 38200/50000 rank2 | Train Loss: 1.0273 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2292 | Cardinality: 0.1060 | Classification: 0.0997 | Classification_binary: 0.1650 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2257 | Regression_cos_phi: 0.1098 | Regression_energy: 0.0662 | Regression_eta: 0.0531 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0802 | LR: 2.18e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.8905s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7758 | max_loss: 0.1794 | max_backward: 1.7969 | max_optimizer_step: 0.1713 | max_tensorboard_logging: 0.4698 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5717.4 | uss_MB: 4558.1 | child_rss_MB: 5664.1 | allocated_MB: 372.9 | reserved_MB: 26212.0 | inactive_split_MB: 221.1 +rank2 [2026-10-29 17:05:07,130] INFO : Step 38300/50000 rank2 | Train Loss: 1.0498 | Train Loss Unweighted: 0.5122 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1249 | Classification: 0.0913 | Classification_binary: 0.1583 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1294 | Regression_eta: 0.0013 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1588 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0681 | Regression_eta: 0.0291 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0750 | LR: 2.16e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7358s | Phase Times: max_pre_step_barrier: 0.1103 | max_batch_to_device: 0.0086 | max_forward: 0.6730 | max_loss: 0.1509 | max_backward: 1.3787 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.4782 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5715.4 | uss_MB: 4556.5 | child_rss_MB: 5665.1 | allocated_MB: 369.2 | reserved_MB: 26212.0 | inactive_split_MB: 171.8 +rank2 [2026-10-29 17:09:08,355] INFO : Step 38400/50000 rank2 | Train Loss: 0.9354 | Train Loss Unweighted: 0.4735 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.0968 | Classification: 0.0760 | Classification_binary: 0.1246 | HitGrouping: 0.1889 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1340 | Regression_eta: 0.0013 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1088 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0627 | Regression_eta: 0.0468 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0330 | LR: 2.14e-04 | DataLoad Time: 0.0223s | Model Forward Time: 3.6831s | Phase Times: max_pre_step_barrier: 0.1107 | max_batch_to_device: 0.0062 | max_forward: 0.6635 | max_loss: 0.1419 | max_backward: 1.4771 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 0.4619 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5721.1 | uss_MB: 4559.4 | child_rss_MB: 5662.2 | allocated_MB: 370.2 | reserved_MB: 26212.0 | inactive_split_MB: 267.8 +rank2 [2026-10-29 17:13:06,091] INFO : Step 38500/50000 rank2 | Train Loss: 1.0482 | Train Loss Unweighted: 0.5077 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1228 | Classification: 0.1147 | Classification_binary: 0.1473 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1208 | Regression_eta: 0.0016 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0502 | Regression_eta: 0.0316 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0577 | LR: 2.12e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.4601s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.6383 | max_loss: 0.1393 | max_backward: 1.3136 | max_optimizer_step: 0.1695 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 5717.0 | uss_MB: 4562.8 | child_rss_MB: 5658.1 | allocated_MB: 370.2 | reserved_MB: 27188.0 | inactive_split_MB: 210.0 +rank2 [2026-10-29 17:17:04,838] INFO : Step 38600/50000 rank2 | Train Loss: 1.0321 | Train Loss Unweighted: 0.4950 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1157 | Classification: 0.0910 | Classification_binary: 0.1735 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1130 | Regression_eta: 0.0014 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1767 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0504 | Regression_eta: 0.0365 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0677 | LR: 2.11e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0835s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0127 | max_forward: 0.7516 | max_loss: 0.1551 | max_backward: 1.6059 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.7250 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5714.3 | uss_MB: 4563.8 | child_rss_MB: 5662.6 | allocated_MB: 372.6 | reserved_MB: 28172.0 | inactive_split_MB: 283.9 +rank2 [2026-10-29 17:20:59,240] INFO : Step 38700/50000 rank2 | Train Loss: 0.9420 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1147 | Classification: 0.0828 | Classification_binary: 0.1348 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1155 | Regression_eta: 0.0013 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1210 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0562 | Regression_eta: 0.0350 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0494 | LR: 2.09e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.6826s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.8052 | max_loss: 0.2133 | max_backward: 1.3215 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5712.0 | uss_MB: 4410.4 | child_rss_MB: 5659.5 | allocated_MB: 367.4 | reserved_MB: 28172.0 | inactive_split_MB: 206.7 +rank2 [2026-10-29 17:25:08,219] INFO : Step 38800/50000 rank2 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4825 | Train Loss Components: Auxiliary: 0.2201 | Cardinality: 0.1186 | Classification: 0.0965 | Classification_binary: 0.1429 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1198 | Regression_eta: 0.0012 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0482 | Regression_eta: 0.0303 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0414 | LR: 2.07e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.6580s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.9011 | max_loss: 0.1880 | max_backward: 1.7487 | max_optimizer_step: 0.2166 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5710.9 | uss_MB: 4431.1 | child_rss_MB: 5666.7 | allocated_MB: 371.5 | reserved_MB: 28172.0 | inactive_split_MB: 220.5 +rank2 [2026-10-29 17:29:07,477] INFO : Step 38900/50000 rank2 | Train Loss: 0.9986 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2211 | Cardinality: 0.1078 | Classification: 0.0988 | Classification_binary: 0.1578 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1683 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0587 | Regression_eta: 0.0291 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0531 | LR: 2.05e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.4270s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0073 | max_forward: 0.9800 | max_loss: 0.2159 | max_backward: 2.2158 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0079 | Memory: rss_MB: 5714.2 | uss_MB: 3754.5 | child_rss_MB: 5665.7 | allocated_MB: 373.3 | reserved_MB: 29164.0 | inactive_split_MB: 189.2 +rank2 [2026-10-29 17:33:02,383] INFO : Step 39000/50000 rank2 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4758 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1227 | Classification: 0.0847 | Classification_binary: 0.1518 | HitGrouping: 0.1853 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1179 | Regression_eta: 0.0013 | Regression_pt: 0.1189 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0533 | Regression_eta: 0.0853 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0972 | LR: 2.03e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.7820s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0515 | max_forward: 0.7410 | max_loss: 0.1790 | max_backward: 1.4655 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5713.4 | uss_MB: 4569.9 | child_rss_MB: 5663.9 | allocated_MB: 370.5 | reserved_MB: 29164.0 | inactive_split_MB: 192.2 +rank2 [2026-10-29 17:33:03,293] INFO : Running validation on rank2 +rank2 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 17:33:56,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:33:56,829] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 17:34:24,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 17:34:25,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 17:34:26,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 17:34:27,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 17:34:37,642] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 17:34:38,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:34:38,630] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 17:34:54,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 17:34:55,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 17:34:56,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 17:34:57,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 17:35:07,397] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 17:35:07,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:35:08,263] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 17:35:23,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 17:35:24,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 17:35:25,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 17:35:25,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 17:35:35,826] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 17:39:50,704] INFO : Step 39100/50000 rank2 | Train Loss: 1.0043 | Train Loss Unweighted: 0.4886 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1149 | Classification: 0.0878 | Classification_binary: 0.1320 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1329 | Regression_eta: 0.0012 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1188 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0582 | Regression_eta: 0.0341 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0461 | LR: 2.01e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6392s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0736 | max_forward: 0.7590 | max_loss: 0.4062 | max_backward: 1.4330 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.4941 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6810.5 | uss_MB: 5664.9 | child_rss_MB: 5641.2 | allocated_MB: 371.1 | reserved_MB: 23656.0 | inactive_split_MB: 194.8 +rank2 [2026-10-29 17:43:44,583] INFO : Step 39200/50000 rank2 | Train Loss: 1.0325 | Train Loss Unweighted: 0.5174 | Train Loss Components: Auxiliary: 0.2315 | Cardinality: 0.1117 | Classification: 0.1086 | Classification_binary: 0.1481 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1483 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0507 | Regression_eta: 0.0387 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0957 | LR: 2.00e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6986s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0102 | max_forward: 0.7546 | max_loss: 0.4268 | max_backward: 1.4370 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5801.6 | uss_MB: 4660.4 | child_rss_MB: 5666.2 | allocated_MB: 371.2 | reserved_MB: 23934.0 | inactive_split_MB: 196.6 +rank2 [2026-10-29 17:47:41,749] INFO : Step 39300/50000 rank2 | Train Loss: 1.0373 | Train Loss Unweighted: 0.5050 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1241 | Classification: 0.0956 | Classification_binary: 0.1745 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1150 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0529 | Regression_eta: 0.0388 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0489 | LR: 1.98e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.2661s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0844 | max_forward: 0.8662 | max_loss: 0.2199 | max_backward: 1.7416 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5805.0 | uss_MB: 4500.0 | child_rss_MB: 5642.3 | allocated_MB: 373.1 | reserved_MB: 23934.0 | inactive_split_MB: 256.8 +rank2 [2026-10-29 17:51:43,103] INFO : Step 39400/50000 rank2 | Train Loss: 0.9741 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2189 | Cardinality: 0.1078 | Classification: 0.0808 | Classification_binary: 0.1461 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1284 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0669 | Regression_eta: 0.0319 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0509 | LR: 1.96e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5601s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.6778 | max_loss: 0.1515 | max_backward: 1.4156 | max_optimizer_step: 0.1740 | max_tensorboard_logging: 0.4653 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5807.5 | uss_MB: 4663.4 | child_rss_MB: 5665.4 | allocated_MB: 368.0 | reserved_MB: 23934.0 | inactive_split_MB: 199.1 +rank2 [2026-10-29 17:55:40,087] INFO : Step 39500/50000 rank2 | Train Loss: 0.9809 | Train Loss Unweighted: 0.4664 | Train Loss Components: Auxiliary: 0.2175 | Cardinality: 0.1105 | Classification: 0.0963 | Classification_binary: 0.1347 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1158 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0568 | Regression_eta: 0.0258 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0416 | LR: 1.94e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5626s | Phase Times: max_pre_step_barrier: 0.0259 | max_batch_to_device: 0.1050 | max_forward: 0.6813 | max_loss: 0.1541 | max_backward: 1.3449 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 0.5375 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0111 | Memory: rss_MB: 5806.7 | uss_MB: 4606.4 | child_rss_MB: 5664.0 | allocated_MB: 367.2 | reserved_MB: 24970.0 | inactive_split_MB: 230.9 +rank2 [2026-10-29 17:59:42,116] INFO : Step 39600/50000 rank2 | Train Loss: 1.0814 | Train Loss Unweighted: 0.5099 | Train Loss Components: Auxiliary: 0.2411 | Cardinality: 0.1344 | Classification: 0.0978 | Classification_binary: 0.1731 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0662 | Regression_eta: 0.0304 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0352 | LR: 1.93e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1501s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8839 | max_loss: 0.1464 | max_backward: 1.8057 | max_optimizer_step: 0.2568 | max_tensorboard_logging: 0.5748 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5805.3 | uss_MB: 4666.8 | child_rss_MB: 5646.1 | allocated_MB: 370.6 | reserved_MB: 24970.0 | inactive_split_MB: 243.4 +rank2 [2026-10-29 18:03:55,339] INFO : Step 39700/50000 rank2 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4877 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1185 | Classification: 0.0909 | Classification_binary: 0.1674 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1134 | Regression_eta: 0.0013 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1624 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0664 | Regression_eta: 0.0333 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0817 | LR: 1.91e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8042s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.6954 | max_loss: 0.2515 | max_backward: 1.5651 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.5599 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5806.9 | uss_MB: 4667.8 | child_rss_MB: 5665.2 | allocated_MB: 371.1 | reserved_MB: 24970.0 | inactive_split_MB: 244.9 +rank2 [2026-10-29 18:07:56,652] INFO : Step 39800/50000 rank2 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1022 | Classification: 0.0711 | Classification_binary: 0.1322 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1242 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0552 | Regression_eta: 0.0267 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0689 | LR: 1.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9400s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0063 | max_forward: 1.0012 | max_loss: 0.2859 | max_backward: 2.1063 | max_optimizer_step: 0.1800 | max_tensorboard_logging: 0.4881 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5810.9 | uss_MB: 3839.1 | child_rss_MB: 5666.9 | allocated_MB: 368.3 | reserved_MB: 24970.0 | inactive_split_MB: 241.7 +rank2 [2026-10-29 18:11:54,152] INFO : Step 39900/50000 rank2 | Train Loss: 0.9667 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.2188 | Cardinality: 0.1055 | Classification: 0.0897 | Classification_binary: 0.1526 | HitGrouping: 0.1883 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1101 | Regression_eta: 0.0012 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0490 | Regression_eta: 0.0345 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0598 | LR: 1.88e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.9942s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.3186 | max_loss: 0.4377 | max_backward: 3.6189 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.6123 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 5815.3 | uss_MB: 4217.6 | child_rss_MB: 5667.9 | allocated_MB: 370.5 | reserved_MB: 24970.0 | inactive_split_MB: 229.4 +rank2 [2026-10-29 18:15:59,744] INFO : Step 40000/50000 rank2 | Train Loss: 1.0232 | Train Loss Unweighted: 0.5131 | Train Loss Components: Auxiliary: 0.2348 | Cardinality: 0.1100 | Classification: 0.0962 | Classification_binary: 0.1712 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1206 | Regression_eta: 0.0013 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0518 | Regression_eta: 0.0369 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0440 | LR: 1.86e-04 | DataLoad Time: 0.0186s | Model Forward Time: 7.4577s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0077 | max_forward: 0.8168 | max_loss: 0.2161 | max_backward: 1.5709 | max_optimizer_step: 0.1733 | max_tensorboard_logging: 1.1681 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.1427 | Memory: rss_MB: 5816.6 | uss_MB: 3786.6 | child_rss_MB: 5665.4 | allocated_MB: 375.8 | reserved_MB: 24970.0 | inactive_split_MB: 259.2 +rank2 [2026-10-29 18:16:02,308] INFO : Running validation on rank2 +rank2 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 18:16:53,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:16:54,184] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 18:17:19,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 18:17:20,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 18:17:21,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 18:17:23,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 18:17:33,159] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 18:17:33,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:17:33,668] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 18:17:49,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 18:17:50,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 18:17:51,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 18:17:52,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 18:18:02,429] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 18:18:03,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:18:03,746] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 18:18:19,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 18:18:20,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 18:18:20,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 18:18:21,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 18:18:31,608] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 18:22:46,459] INFO : Step 40100/50000 rank2 | Train Loss: 0.9349 | Train Loss Unweighted: 0.4412 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1220 | Classification: 0.0762 | Classification_binary: 0.1328 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1148 | Regression_eta: 0.0012 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1389 | Regression_cos_phi: 0.0310 | Regression_energy: 0.0520 | Regression_eta: 0.0321 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0447 | LR: 1.84e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6890s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.7093 | max_loss: 0.3974 | max_backward: 1.3310 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.5555 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 7103.1 | uss_MB: 5953.7 | child_rss_MB: 5672.5 | allocated_MB: 367.6 | reserved_MB: 22178.0 | inactive_split_MB: 265.9 +rank2 [2026-10-29 18:26:52,186] INFO : Step 40200/50000 rank2 | Train Loss: 0.9570 | Train Loss Unweighted: 0.4556 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1119 | Classification: 0.0905 | Classification_binary: 0.1293 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0013 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1205 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0514 | Regression_eta: 0.0489 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0666 | LR: 1.83e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5900s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6287 | max_loss: 0.4129 | max_backward: 1.3901 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 0.5609 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5918.3 | uss_MB: 4768.5 | child_rss_MB: 5665.4 | allocated_MB: 370.8 | reserved_MB: 22294.0 | inactive_split_MB: 233.2 +rank2 [2026-10-29 18:30:59,536] INFO : Step 40300/50000 rank2 | Train Loss: 1.0209 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1232 | Classification: 0.0991 | Classification_binary: 0.1688 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1068 | Regression_eta: 0.0015 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1854 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0675 | Regression_eta: 0.0423 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0422 | LR: 1.81e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9660s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0749 | max_forward: 0.7563 | max_loss: 0.1523 | max_backward: 1.5452 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5632 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5917.0 | uss_MB: 4772.2 | child_rss_MB: 5652.3 | allocated_MB: 371.8 | reserved_MB: 23296.0 | inactive_split_MB: 234.2 +rank2 [2026-10-29 18:35:20,932] INFO : Step 40400/50000 rank2 | Train Loss: 1.0486 | Train Loss Unweighted: 0.5003 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1357 | Classification: 0.0888 | Classification_binary: 0.1690 | HitGrouping: 0.1879 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1196 | Regression_eta: 0.0015 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1639 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0573 | Regression_eta: 0.0371 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0579 | LR: 1.79e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.7346s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0072 | max_forward: 0.7368 | max_loss: 0.2443 | max_backward: 3.1596 | max_optimizer_step: 0.1851 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5913.1 | uss_MB: 3802.2 | child_rss_MB: 5663.8 | allocated_MB: 371.2 | reserved_MB: 23386.0 | inactive_split_MB: 254.8 +rank2 [2026-10-29 18:39:23,871] INFO : Step 40500/50000 rank2 | Train Loss: 0.9309 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.0955 | Classification: 0.0829 | Classification_binary: 0.1355 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0014 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1499 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0566 | Regression_eta: 0.0236 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0378 | LR: 1.78e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7232s | Phase Times: max_pre_step_barrier: 0.0186 | max_batch_to_device: 0.0076 | max_forward: 0.6863 | max_loss: 0.2393 | max_backward: 1.3065 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.7004 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5916.1 | uss_MB: 4771.6 | child_rss_MB: 5669.9 | allocated_MB: 369.2 | reserved_MB: 24370.0 | inactive_split_MB: 237.3 +rank2 [2026-10-29 18:43:23,922] INFO : Step 40600/50000 rank2 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1087 | Classification: 0.0887 | Classification_binary: 0.1521 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1173 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0467 | Regression_eta: 0.0356 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0552 | LR: 1.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3443s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.1102 | max_forward: 0.7412 | max_loss: 0.1464 | max_backward: 1.4782 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 1.0719 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5917.7 | uss_MB: 4772.7 | child_rss_MB: 5655.3 | allocated_MB: 372.3 | reserved_MB: 24370.0 | inactive_split_MB: 234.4 +rank2 [2026-10-29 18:47:14,843] INFO : Step 40700/50000 rank2 | Train Loss: 1.0611 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1374 | Classification: 0.0904 | Classification_binary: 0.1773 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1190 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0528 | Regression_eta: 0.0302 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0427 | LR: 1.75e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2476s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7422 | max_loss: 0.1589 | max_backward: 1.5298 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5918.6 | uss_MB: 4774.9 | child_rss_MB: 5665.5 | allocated_MB: 372.7 | reserved_MB: 24370.0 | inactive_split_MB: 255.3 +rank2 [2026-10-29 18:51:03,370] INFO : Step 40800/50000 rank2 | Train Loss: 0.9701 | Train Loss Unweighted: 0.4785 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.0983 | Classification: 0.0901 | Classification_binary: 0.1373 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.1455 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0563 | Regression_eta: 0.0653 | Regression_pt: 0.0582 | Regression_sin_phi: 0.1037 | LR: 1.73e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.7276s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5843 | max_loss: 0.1422 | max_backward: 1.0661 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.0389 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5915.8 | uss_MB: 4772.0 | child_rss_MB: 5661.2 | allocated_MB: 367.5 | reserved_MB: 24658.0 | inactive_split_MB: 205.4 +rank2 [2026-10-29 18:54:43,249] INFO : Step 40900/50000 rank2 | Train Loss: 0.9731 | Train Loss Unweighted: 0.4636 | Train Loss Components: Auxiliary: 0.2169 | Cardinality: 0.1073 | Classification: 0.0883 | Classification_binary: 0.1426 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1140 | Regression_eta: 0.0013 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0544 | Regression_eta: 0.0280 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0348 | LR: 1.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8718s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.1047 | max_forward: 0.7107 | max_loss: 0.1679 | max_backward: 1.3343 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.8664 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5919.0 | uss_MB: 4778.3 | child_rss_MB: 5681.4 | allocated_MB: 372.0 | reserved_MB: 25640.0 | inactive_split_MB: 260.9 +rank2 [2026-10-29 18:58:25,753] INFO : Step 41000/50000 rank2 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1031 | Classification: 0.0930 | Classification_binary: 0.1618 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1164 | Regression_eta: 0.0014 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0552 | Regression_eta: 0.0343 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0657 | LR: 1.70e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4155s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.7604 | max_loss: 0.2558 | max_backward: 1.7603 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.8399 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 5926.0 | uss_MB: 4782.0 | child_rss_MB: 5654.3 | allocated_MB: 371.9 | reserved_MB: 25640.0 | inactive_split_MB: 195.8 +rank2 [2026-10-29 18:58:26,765] INFO : Running validation on rank2 +rank2 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 18:59:14,710] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:59:15,274] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 18:59:40,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 18:59:41,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 18:59:42,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 18:59:43,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 18:59:53,495] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 18:59:53,543] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:59:54,025] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 19:00:09,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 19:00:10,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 19:00:11,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 19:00:12,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 19:00:22,797] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 19:00:23,208] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:00:23,683] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 19:00:39,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 19:00:40,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 19:00:40,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 19:00:41,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 19:00:51,348] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 19:04:58,255] INFO : Step 41100/50000 rank2 | Train Loss: 1.0684 | Train Loss Unweighted: 0.5053 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1438 | Classification: 0.0856 | Classification_binary: 0.1616 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1271 | Regression_eta: 0.0014 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1358 | Regression_cos_phi: 0.0339 | Regression_energy: 0.0513 | Regression_eta: 0.0246 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0332 | LR: 1.69e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3318s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0066 | max_forward: 0.8006 | max_loss: 0.1726 | max_backward: 1.6699 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.6214 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0566 | Memory: rss_MB: 7184.9 | uss_MB: 5901.7 | child_rss_MB: 5647.6 | allocated_MB: 371.5 | reserved_MB: 22898.0 | inactive_split_MB: 258.4 +rank2 [2026-10-29 19:08:41,623] INFO : Step 41200/50000 rank2 | Train Loss: 0.9455 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1027 | Classification: 0.0887 | Classification_binary: 0.1242 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1223 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1110 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0557 | Regression_eta: 0.0282 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0502 | LR: 1.67e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.4131s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0589 | max_forward: 0.6517 | max_loss: 0.1401 | max_backward: 1.1761 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5685 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5984.6 | uss_MB: 4847.1 | child_rss_MB: 5628.1 | allocated_MB: 370.2 | reserved_MB: 24878.0 | inactive_split_MB: 199.0 +rank2 [2026-10-29 19:12:28,248] INFO : Step 41300/50000 rank2 | Train Loss: 0.9373 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1011 | Classification: 0.0853 | Classification_binary: 0.1448 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0013 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0460 | Regression_eta: 0.0303 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0601 | LR: 1.66e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8982s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.8726 | max_loss: 0.1617 | max_backward: 1.6974 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5417 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5987.6 | uss_MB: 4852.4 | child_rss_MB: 5623.6 | allocated_MB: 370.7 | reserved_MB: 24878.0 | inactive_split_MB: 233.6 +rank2 [2026-10-29 19:16:15,464] INFO : Step 41400/50000 rank2 | Train Loss: 1.0355 | Train Loss Unweighted: 0.4965 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1363 | Classification: 0.0957 | Classification_binary: 0.1654 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1162 | Regression_eta: 0.0012 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0699 | Regression_eta: 0.0307 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0515 | LR: 1.64e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9186s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7129 | max_loss: 0.2460 | max_backward: 1.6340 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5967.8 | uss_MB: 4849.9 | child_rss_MB: 5563.5 | allocated_MB: 372.0 | reserved_MB: 24878.0 | inactive_split_MB: 305.4 +rank2 [2026-10-29 19:19:52,656] INFO : Step 41500/50000 rank2 | Train Loss: 0.9689 | Train Loss Unweighted: 0.4689 | Train Loss Components: Auxiliary: 0.2118 | Cardinality: 0.1040 | Classification: 0.0862 | Classification_binary: 0.1340 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1227 | Regression_eta: 0.0012 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1309 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0420 | Regression_eta: 0.0263 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0419 | LR: 1.63e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.4468s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.6415 | max_loss: 0.1502 | max_backward: 1.1845 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5935.8 | uss_MB: 4853.4 | child_rss_MB: 5403.5 | allocated_MB: 368.5 | reserved_MB: 24878.0 | inactive_split_MB: 173.5 +rank2 [2026-10-29 19:23:40,392] INFO : Step 41600/50000 rank2 | Train Loss: 0.9559 | Train Loss Unweighted: 0.4625 | Train Loss Components: Auxiliary: 0.2141 | Cardinality: 0.1097 | Classification: 0.0872 | Classification_binary: 0.1298 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1211 | Regression_eta: 0.0012 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0283 | Regression_energy: 0.0589 | Regression_eta: 0.0319 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0430 | LR: 1.61e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.5877s | Phase Times: max_pre_step_barrier: 0.0755 | max_batch_to_device: 0.0062 | max_forward: 0.6849 | max_loss: 0.2047 | max_backward: 1.1646 | max_optimizer_step: 0.2235 | max_tensorboard_logging: 0.5806 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5909.3 | uss_MB: 4827.6 | child_rss_MB: 5308.7 | allocated_MB: 370.4 | reserved_MB: 24878.0 | inactive_split_MB: 287.4 +rank2 [2026-10-29 19:27:23,742] INFO : Step 41700/50000 rank2 | Train Loss: 1.0204 | Train Loss Unweighted: 0.4905 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1165 | Classification: 0.0860 | Classification_binary: 0.1552 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0477 | Regression_eta: 0.0284 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0413 | LR: 1.60e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.0297s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0773 | max_forward: 0.7754 | max_loss: 0.1486 | max_backward: 1.5759 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 0.5773 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5907.3 | uss_MB: 4857.3 | child_rss_MB: 5302.2 | allocated_MB: 370.8 | reserved_MB: 24878.0 | inactive_split_MB: 238.3 +rank2 [2026-10-29 19:31:24,301] INFO : Step 41800/50000 rank2 | Train Loss: 1.0101 | Train Loss Unweighted: 0.4703 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1282 | Classification: 0.0818 | Classification_binary: 0.1618 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0498 | Regression_eta: 0.0321 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0583 | LR: 1.58e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9772s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8147 | max_loss: 0.4259 | max_backward: 1.4886 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1200 | Memory: rss_MB: 5899.0 | uss_MB: 4855.0 | child_rss_MB: 5276.1 | allocated_MB: 371.9 | reserved_MB: 25802.0 | inactive_split_MB: 266.5 +rank2 [2026-10-29 19:35:32,346] INFO : Step 41900/50000 rank2 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4672 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1185 | Classification: 0.0878 | Classification_binary: 0.1282 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1241 | Regression_eta: 0.0011 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0710 | Regression_eta: 0.0320 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0371 | LR: 1.57e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.4618s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0473 | max_forward: 0.6783 | max_loss: 0.1431 | max_backward: 1.2347 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.5634 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5868.0 | uss_MB: 4858.2 | child_rss_MB: 5156.0 | allocated_MB: 370.8 | reserved_MB: 25802.0 | inactive_split_MB: 166.5 +rank2 [2026-10-29 19:39:32,016] INFO : Step 42000/50000 rank2 | Train Loss: 0.9837 | Train Loss Unweighted: 0.4719 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.1093 | Classification: 0.1015 | Classification_binary: 0.1413 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1128 | Regression_eta: 0.0014 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0913 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0608 | Regression_eta: 0.0789 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0612 | LR: 1.56e-04 | DataLoad Time: 0.0215s | Model Forward Time: 3.6383s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6848 | max_loss: 0.2509 | max_backward: 1.3746 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5866.0 | uss_MB: 4860.4 | child_rss_MB: 5161.1 | allocated_MB: 370.2 | reserved_MB: 25802.0 | inactive_split_MB: 219.9 +rank2 [2026-10-29 19:39:32,940] INFO : Running validation on rank2 +rank2 [2026-10-29 19:39:32,941] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 19:39:32,941] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 19:40:23,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:40:23,892] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 19:40:52,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 19:40:53,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 19:40:54,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 19:40:55,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 19:41:05,899] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 19:41:06,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:41:06,794] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 19:41:23,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 19:41:24,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 19:41:26,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 19:41:27,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 19:41:37,162] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 19:41:38,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:41:38,675] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 19:41:54,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 19:41:54,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 19:41:55,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 19:41:56,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 19:42:06,095] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 19:46:29,431] INFO : Step 42100/50000 rank2 | Train Loss: 1.0639 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1434 | Classification: 0.0999 | Classification_binary: 0.1737 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1097 | Regression_eta: 0.0012 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0526 | Regression_eta: 0.0264 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0522 | LR: 1.54e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.0387s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.7424 | max_loss: 0.1586 | max_backward: 1.4434 | max_optimizer_step: 0.3014 | max_tensorboard_logging: 0.6771 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 7018.7 | uss_MB: 6010.1 | child_rss_MB: 5164.1 | allocated_MB: 370.8 | reserved_MB: 23298.0 | inactive_split_MB: 265.3 +rank2 [2026-10-29 19:50:21,662] INFO : Step 42200/50000 rank2 | Train Loss: 1.0152 | Train Loss Unweighted: 0.4831 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1169 | Classification: 0.0913 | Classification_binary: 0.1515 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1190 | Regression_eta: 0.0015 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0352 | Regression_energy: 0.0606 | Regression_eta: 0.0269 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0455 | LR: 1.53e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.5439s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.6567 | max_loss: 0.2000 | max_backward: 1.2379 | max_optimizer_step: 0.1803 | max_tensorboard_logging: 0.5782 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5943.7 | uss_MB: 4948.2 | child_rss_MB: 5171.4 | allocated_MB: 370.9 | reserved_MB: 24984.0 | inactive_split_MB: 272.4 +rank2 [2026-10-29 19:51:24,069] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank2 [2026-10-29 19:51:24,069] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-29 19:54:15,428] INFO : Step 42300/50000 rank2 | Train Loss: 1.0224 | Train Loss Unweighted: 0.4869 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.1261 | Classification: 0.0943 | Classification_binary: 0.1704 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0574 | Regression_eta: 0.0327 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0466 | LR: 1.52e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.7700s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7065 | max_loss: 0.2237 | max_backward: 1.4902 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.5353 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6010.9 | uss_MB: 5019.0 | child_rss_MB: 5170.2 | allocated_MB: 371.9 | reserved_MB: 24984.0 | inactive_split_MB: 183.6 +rank2 [2026-10-29 19:58:13,284] INFO : Step 42400/50000 rank2 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4895 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1253 | Classification: 0.0872 | Classification_binary: 0.1616 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1184 | Regression_eta: 0.0013 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0505 | Regression_eta: 0.0365 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0517 | LR: 1.50e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7984s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7436 | max_loss: 0.2477 | max_backward: 1.5279 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5169 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5933.8 | uss_MB: 4947.5 | child_rss_MB: 5164.9 | allocated_MB: 373.0 | reserved_MB: 24984.0 | inactive_split_MB: 275.6 +rank2 [2026-10-29 20:02:13,261] INFO : Step 42500/50000 rank2 | Train Loss: 0.9211 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1056 | Classification: 0.0789 | Classification_binary: 0.1296 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1162 | Regression_eta: 0.0011 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0608 | Regression_eta: 0.0317 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0513 | LR: 1.49e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.5552s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.6196 | max_loss: 0.2440 | max_backward: 1.3403 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 2.3828 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5926.8 | uss_MB: 4949.7 | child_rss_MB: 5158.5 | allocated_MB: 371.3 | reserved_MB: 25980.0 | inactive_split_MB: 274.9 +rank2 [2026-10-29 20:06:17,860] INFO : Step 42600/50000 rank2 | Train Loss: 0.9688 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1101 | Classification: 0.0874 | Classification_binary: 0.1472 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0012 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1356 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0630 | Regression_eta: 0.0342 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0466 | LR: 1.48e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9505s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6353 | max_loss: 0.2547 | max_backward: 1.3961 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 5918.3 | uss_MB: 4947.3 | child_rss_MB: 5167.3 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 272.5 +rank2 [2026-10-29 20:10:16,915] INFO : Step 42700/50000 rank2 | Train Loss: 1.0476 | Train Loss Unweighted: 0.4808 | Train Loss Components: Auxiliary: 0.2377 | Cardinality: 0.1284 | Classification: 0.0883 | Classification_binary: 0.1739 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1071 | Regression_eta: 0.0014 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1855 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0577 | Regression_eta: 0.0388 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0676 | LR: 1.47e-04 | DataLoad Time: 0.0110s | Model Forward Time: 5.7799s | Phase Times: max_pre_step_barrier: 0.0320 | max_batch_to_device: 0.0114 | max_forward: 2.1726 | max_loss: 0.5101 | max_backward: 2.7494 | max_optimizer_step: 0.2679 | max_tensorboard_logging: 0.6520 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5921.8 | uss_MB: 4952.4 | child_rss_MB: 5151.8 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 482.9 +rank2 [2026-10-29 20:14:17,050] INFO : Step 42800/50000 rank2 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4792 | Train Loss Components: Auxiliary: 0.2262 | Cardinality: 0.1277 | Classification: 0.0841 | Classification_binary: 0.1487 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1209 | Regression_eta: 0.0012 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0383 | Regression_energy: 0.0578 | Regression_eta: 0.0228 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0356 | LR: 1.45e-04 | DataLoad Time: 0.0212s | Model Forward Time: 6.2803s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0175 | max_forward: 1.3870 | max_loss: 0.5106 | max_backward: 1.9221 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 2.4017 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5924.3 | uss_MB: 4952.5 | child_rss_MB: 5165.0 | allocated_MB: 372.6 | reserved_MB: 25980.0 | inactive_split_MB: 234.4 +rank2 [2026-10-29 20:18:09,439] INFO : Step 42900/50000 rank2 | Train Loss: 0.8847 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.0993 | Classification: 0.0847 | Classification_binary: 0.1207 | HitGrouping: 0.1828 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1069 | Regression_eta: 0.0011 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1224 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0440 | Regression_eta: 0.0389 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0488 | LR: 1.44e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7391s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.6133 | max_loss: 0.1478 | max_backward: 1.3445 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.7551 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0762 | Memory: rss_MB: 5924.9 | uss_MB: 4957.1 | child_rss_MB: 5179.7 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 276.9 +rank2 [2026-10-29 20:22:02,172] INFO : Step 43000/50000 rank2 | Train Loss: 0.9800 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1096 | Classification: 0.0987 | Classification_binary: 0.1606 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0012 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1729 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0438 | Regression_eta: 0.0392 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0439 | LR: 1.43e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.9037s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0071 | max_forward: 0.7477 | max_loss: 0.1477 | max_backward: 1.5663 | max_optimizer_step: 0.1600 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 5929.7 | uss_MB: 4961.8 | child_rss_MB: 5171.4 | allocated_MB: 369.1 | reserved_MB: 25980.0 | inactive_split_MB: 209.4 +rank2 [2026-10-29 20:22:03,144] INFO : Running validation on rank2 +rank2 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 20:22:51,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:22:52,330] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 20:23:18,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 20:23:19,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 20:23:20,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 20:23:21,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 20:23:31,599] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 20:23:31,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:23:32,370] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 20:23:48,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 20:23:49,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 20:23:50,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 20:23:50,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 20:24:00,990] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 20:24:01,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:24:02,214] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 20:24:17,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 20:24:18,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 20:24:18,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 20:24:19,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 20:24:29,594] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 20:28:29,103] INFO : Step 43100/50000 rank2 | Train Loss: 0.9692 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1151 | Classification: 0.0766 | Classification_binary: 0.1586 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1062 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0484 | Regression_eta: 0.0384 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0559 | LR: 1.42e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7245s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.8051 | max_loss: 0.1438 | max_backward: 1.3224 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 7068.8 | uss_MB: 6101.3 | child_rss_MB: 5191.0 | allocated_MB: 370.5 | reserved_MB: 21096.0 | inactive_split_MB: 247.5 +rank2 [2026-10-29 20:32:30,813] INFO : Step 43200/50000 rank2 | Train Loss: 0.9431 | Train Loss Unweighted: 0.4458 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1175 | Classification: 0.0811 | Classification_binary: 0.1294 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0012 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1124 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0488 | Regression_eta: 0.0231 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0311 | LR: 1.40e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.5224s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0120 | max_forward: 0.6342 | max_loss: 0.1417 | max_backward: 1.2752 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6068.4 | uss_MB: 5101.7 | child_rss_MB: 5194.2 | allocated_MB: 370.4 | reserved_MB: 22140.0 | inactive_split_MB: 198.0 +rank2 [2026-10-29 20:36:16,428] INFO : Step 43300/50000 rank2 | Train Loss: 1.0382 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1221 | Classification: 0.1031 | Classification_binary: 0.1555 | HitGrouping: 0.1894 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1173 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0445 | Regression_eta: 0.0477 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0911 | LR: 1.39e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2395s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0266 | max_forward: 0.7527 | max_loss: 0.1476 | max_backward: 1.6633 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.7701 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6071.5 | uss_MB: 5107.2 | child_rss_MB: 5175.9 | allocated_MB: 371.1 | reserved_MB: 23082.0 | inactive_split_MB: 236.4 +rank2 [2026-10-29 20:40:05,496] INFO : Step 43400/50000 rank2 | Train Loss: 1.0532 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1292 | Classification: 0.0981 | Classification_binary: 0.1785 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0015 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.1976 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0518 | Regression_eta: 0.0698 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0742 | LR: 1.38e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2973s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0691 | max_forward: 0.8057 | max_loss: 0.1503 | max_backward: 1.8319 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.6703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6067.0 | uss_MB: 5102.4 | child_rss_MB: 5192.5 | allocated_MB: 371.5 | reserved_MB: 23082.0 | inactive_split_MB: 776.0 +rank2 [2026-10-29 20:43:57,009] INFO : Step 43500/50000 rank2 | Train Loss: 0.9308 | Train Loss Unweighted: 0.4531 | Train Loss Components: Auxiliary: 0.2084 | Cardinality: 0.0966 | Classification: 0.0846 | Classification_binary: 0.1384 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1135 | Regression_eta: 0.0013 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1313 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0554 | Regression_eta: 0.0544 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0418 | LR: 1.37e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6264s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0171 | max_forward: 0.6718 | max_loss: 0.1455 | max_backward: 1.4141 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 6070.4 | uss_MB: 5108.5 | child_rss_MB: 5191.0 | allocated_MB: 372.0 | reserved_MB: 23082.0 | inactive_split_MB: 228.0 +rank2 [2026-10-29 20:47:52,744] INFO : Step 43600/50000 rank2 | Train Loss: 0.9195 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.0926 | Classification: 0.0847 | Classification_binary: 0.1298 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1175 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0539 | Regression_eta: 0.0283 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0489 | LR: 1.36e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5158s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.6453 | max_loss: 0.1399 | max_backward: 1.3514 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0942 | Memory: rss_MB: 6067.9 | uss_MB: 5106.0 | child_rss_MB: 5190.1 | allocated_MB: 367.6 | reserved_MB: 23082.0 | inactive_split_MB: 270.4 +rank2 [2026-10-29 20:51:54,646] INFO : Step 43700/50000 rank2 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1181 | Classification: 0.0916 | Classification_binary: 0.1553 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1009 | Regression_eta: 0.0014 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0512 | Regression_eta: 0.0421 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0537 | LR: 1.35e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.0866s | Phase Times: max_pre_step_barrier: 0.0199 | max_batch_to_device: 0.0121 | max_forward: 0.8005 | max_loss: 0.2526 | max_backward: 1.9159 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 6075.4 | uss_MB: 5110.5 | child_rss_MB: 5168.5 | allocated_MB: 371.6 | reserved_MB: 23674.0 | inactive_split_MB: 256.4 +rank2 [2026-10-29 20:55:51,187] INFO : Step 43800/50000 rank2 | Train Loss: 1.0325 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1341 | Classification: 0.0892 | Classification_binary: 0.1542 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0012 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0647 | Regression_eta: 0.0273 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0350 | LR: 1.34e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6853s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7618 | max_loss: 0.1715 | max_backward: 1.3818 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6073.5 | uss_MB: 5110.6 | child_rss_MB: 5186.9 | allocated_MB: 371.7 | reserved_MB: 23674.0 | inactive_split_MB: 282.3 +rank2 [2026-10-29 20:59:46,232] INFO : Step 43900/50000 rank2 | Train Loss: 0.9104 | Train Loss Unweighted: 0.4325 | Train Loss Components: Auxiliary: 0.2009 | Cardinality: 0.1078 | Classification: 0.0725 | Classification_binary: 0.1272 | HitGrouping: 0.1855 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1146 | Regression_eta: 0.0011 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1013 | Regression_cos_phi: 0.0310 | Regression_energy: 0.0553 | Regression_eta: 0.0205 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0377 | LR: 1.33e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.3863s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.6145 | max_loss: 0.1412 | max_backward: 1.1415 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5764 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6071.8 | uss_MB: 5111.9 | child_rss_MB: 5178.4 | allocated_MB: 368.3 | reserved_MB: 23674.0 | inactive_split_MB: 193.1 +rank2 [2026-10-29 21:03:43,513] INFO : Step 44000/50000 rank2 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4573 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1078 | Classification: 0.0956 | Classification_binary: 0.1421 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1083 | Regression_eta: 0.0011 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.1592 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0503 | Regression_eta: 0.0435 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0568 | LR: 1.32e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.9494s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0061 | max_forward: 1.1697 | max_loss: 0.2455 | max_backward: 1.7366 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6074.9 | uss_MB: 5118.1 | child_rss_MB: 5179.2 | allocated_MB: 370.7 | reserved_MB: 23674.0 | inactive_split_MB: 207.3 +rank2 [2026-10-29 21:03:44,527] INFO : Running validation on rank2 +rank2 [2026-10-29 21:03:44,527] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 21:03:44,527] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 21:04:32,715] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:04:33,227] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 21:04:58,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 21:05:00,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 21:05:01,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 21:05:01,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 21:05:11,990] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 21:05:12,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:05:12,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 21:05:28,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 21:05:29,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 21:05:30,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 21:05:31,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 21:05:41,575] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 21:05:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:05:42,732] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 21:05:58,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 21:05:59,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 21:05:59,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 21:06:00,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 21:06:10,351] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 21:10:20,117] INFO : Step 44100/50000 rank2 | Train Loss: 1.0009 | Train Loss Unweighted: 0.4605 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1400 | Classification: 0.0841 | Classification_binary: 0.1644 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1044 | Regression_eta: 0.0011 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0490 | Regression_eta: 0.0258 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0459 | LR: 1.31e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.5842s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 1.2763 | max_loss: 0.5349 | max_backward: 2.2676 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 1.2755 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 7076.5 | uss_MB: 6129.8 | child_rss_MB: 5164.2 | allocated_MB: 372.3 | reserved_MB: 24224.0 | inactive_split_MB: 272.2 +rank2 [2026-10-29 21:14:15,311] INFO : Step 44200/50000 rank2 | Train Loss: 0.9801 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1146 | Classification: 0.0882 | Classification_binary: 0.1482 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1167 | Regression_eta: 0.0014 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0542 | Regression_eta: 0.0405 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0344 | LR: 1.30e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.0038s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0132 | max_forward: 0.6993 | max_loss: 0.1574 | max_backward: 1.3862 | max_optimizer_step: 0.2915 | max_tensorboard_logging: 0.7724 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6137.2 | uss_MB: 5191.5 | child_rss_MB: 5174.4 | allocated_MB: 370.2 | reserved_MB: 26182.0 | inactive_split_MB: 176.5 +rank2 [2026-10-29 21:17:58,583] INFO : Step 44300/50000 rank2 | Train Loss: 0.8787 | Train Loss Unweighted: 0.4332 | Train Loss Components: Auxiliary: 0.1952 | Cardinality: 0.0950 | Classification: 0.0869 | Classification_binary: 0.1153 | HitGrouping: 0.1807 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0011 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0597 | Regression_eta: 0.0236 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0634 | LR: 1.29e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.7432s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0142 | max_forward: 0.7699 | max_loss: 0.2088 | max_backward: 1.4049 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 0.4839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 6137.2 | uss_MB: 4195.8 | child_rss_MB: 5178.8 | allocated_MB: 367.2 | reserved_MB: 27204.0 | inactive_split_MB: 205.8 +rank2 [2026-10-29 21:21:59,795] INFO : Step 44400/50000 rank2 | Train Loss: 0.9558 | Train Loss Unweighted: 0.4680 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.0905 | Classification: 0.0911 | Classification_binary: 0.1587 | HitGrouping: 0.1858 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0438 | Regression_eta: 0.0348 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0618 | LR: 1.28e-04 | DataLoad Time: 0.0214s | Model Forward Time: 6.3285s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0370 | max_forward: 0.7829 | max_loss: 0.1727 | max_backward: 1.7862 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 2.7361 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6138.4 | uss_MB: 5206.0 | child_rss_MB: 5161.4 | allocated_MB: 373.1 | reserved_MB: 27586.0 | inactive_split_MB: 176.3 +rank2 [2026-10-29 21:26:11,867] INFO : Step 44500/50000 rank2 | Train Loss: 0.9739 | Train Loss Unweighted: 0.4610 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1139 | Classification: 0.0849 | Classification_binary: 0.1578 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1077 | Regression_eta: 0.0012 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.1788 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0515 | Regression_eta: 0.0352 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0746 | LR: 1.27e-04 | DataLoad Time: 0.0125s | Model Forward Time: 3.9306s | Phase Times: max_pre_step_barrier: 0.0250 | max_batch_to_device: 0.0065 | max_forward: 0.7930 | max_loss: 0.1668 | max_backward: 1.3363 | max_optimizer_step: 0.2871 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6136.3 | uss_MB: 5200.8 | child_rss_MB: 5177.3 | allocated_MB: 371.5 | reserved_MB: 27586.0 | inactive_split_MB: 195.8 +rank2 [2026-10-29 21:30:04,561] INFO : Step 44600/50000 rank2 | Train Loss: 0.8904 | Train Loss Unweighted: 0.4364 | Train Loss Components: Auxiliary: 0.2007 | Cardinality: 0.1041 | Classification: 0.0775 | Classification_binary: 0.1238 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1161 | Regression_eta: 0.0010 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1324 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0522 | Regression_eta: 0.0391 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0348 | LR: 1.26e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.5129s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.6364 | max_loss: 0.1431 | max_backward: 1.3712 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.4900 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6139.4 | uss_MB: 5205.0 | child_rss_MB: 5167.5 | allocated_MB: 368.4 | reserved_MB: 27586.0 | inactive_split_MB: 170.2 +rank2 [2026-10-29 21:33:51,597] INFO : Step 44700/50000 rank2 | Train Loss: 1.0003 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1252 | Classification: 0.0899 | Classification_binary: 0.1520 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1117 | Regression_eta: 0.0013 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0548 | Regression_eta: 0.0292 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0488 | LR: 1.25e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.6377s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6952 | max_loss: 0.1455 | max_backward: 1.5015 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6145.1 | uss_MB: 5206.5 | child_rss_MB: 5162.4 | allocated_MB: 371.9 | reserved_MB: 28574.0 | inactive_split_MB: 214.1 +rank2 [2026-10-29 21:37:55,987] INFO : Step 44800/50000 rank2 | Train Loss: 0.9665 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1115 | Classification: 0.0889 | Classification_binary: 0.1605 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1042 | Regression_eta: 0.0011 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1981 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0502 | Regression_eta: 0.0265 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0666 | LR: 1.24e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.8737s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7593 | max_loss: 0.1924 | max_backward: 2.8086 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.6087 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 6148.4 | uss_MB: 5210.8 | child_rss_MB: 5160.0 | allocated_MB: 370.8 | reserved_MB: 28574.0 | inactive_split_MB: 257.7 +rank2 [2026-10-29 21:41:36,915] INFO : Step 44900/50000 rank2 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.0886 | Classification_binary: 0.1466 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1121 | Regression_eta: 0.0013 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1260 | Regression_cos_phi: 0.0318 | Regression_energy: 0.0500 | Regression_eta: 0.0349 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0457 | LR: 1.23e-04 | DataLoad Time: 0.0144s | Model Forward Time: 3.5211s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0101 | max_forward: 0.6531 | max_loss: 0.1431 | max_backward: 1.3230 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6145.9 | uss_MB: 5207.3 | child_rss_MB: 5182.8 | allocated_MB: 368.1 | reserved_MB: 28574.0 | inactive_split_MB: 178.8 +rank2 [2026-10-29 21:45:50,735] INFO : Step 45000/50000 rank2 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4443 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1053 | Classification: 0.0832 | Classification_binary: 0.1311 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.1141 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0689 | Regression_eta: 0.0688 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0968 | LR: 1.22e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.1252s | Phase Times: max_pre_step_barrier: 0.0074 | max_batch_to_device: 0.0159 | max_forward: 1.1897 | max_loss: 0.2457 | max_backward: 1.2951 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.8342 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0919 | Memory: rss_MB: 6152.1 | uss_MB: 5213.7 | child_rss_MB: 5170.7 | allocated_MB: 368.8 | reserved_MB: 28576.0 | inactive_split_MB: 266.4 +rank2 [2026-10-29 21:45:51,739] INFO : Running validation on rank2 +rank2 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 21:46:40,561] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:46:41,116] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 21:47:08,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 21:47:09,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 21:47:11,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 21:47:12,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 21:47:22,412] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 21:47:23,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:47:23,688] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 21:47:39,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 21:47:40,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 21:47:41,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 21:47:42,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 21:47:52,580] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 21:47:53,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:47:53,966] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 21:48:10,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 21:48:10,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 21:48:11,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 21:48:12,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 21:48:22,175] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 21:52:56,452] INFO : Step 45100/50000 rank2 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1151 | Classification: 0.1010 | Classification_binary: 0.1562 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1118 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0340 | Regression_energy: 0.0635 | Regression_eta: 0.0234 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0325 | LR: 1.21e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.3061s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7307 | max_loss: 0.4409 | max_backward: 1.5823 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 7066.7 | uss_MB: 6122.4 | child_rss_MB: 5160.9 | allocated_MB: 373.1 | reserved_MB: 23280.0 | inactive_split_MB: 227.2 +rank2 [2026-10-29 21:56:57,436] INFO : Step 45200/50000 rank2 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1108 | Classification: 0.0815 | Classification_binary: 0.1593 | HitGrouping: 0.1884 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1587 | Regression_cos_phi: 0.0801 | Regression_energy: 0.0584 | Regression_eta: 0.0428 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0558 | LR: 1.20e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2220s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.9482 | max_loss: 0.4476 | max_backward: 1.7165 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.4656 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7064.1 | uss_MB: 6120.3 | child_rss_MB: 5157.7 | allocated_MB: 373.0 | reserved_MB: 23338.0 | inactive_split_MB: 277.4 +rank2 [2026-10-29 22:01:28,910] INFO : Step 45300/50000 rank2 | Train Loss: 0.9312 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1141 | Classification: 0.0882 | Classification_binary: 0.1299 | HitGrouping: 0.1891 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0010 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0550 | Regression_eta: 0.0299 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0426 | LR: 1.19e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.4279s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6106 | max_loss: 0.2394 | max_backward: 1.1660 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 6271.7 | uss_MB: 5329.4 | child_rss_MB: 5157.3 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 223.2 +rank2 [2026-10-29 22:05:37,387] INFO : Step 45400/50000 rank2 | Train Loss: 0.9251 | Train Loss Unweighted: 0.4554 | Train Loss Components: Auxiliary: 0.2112 | Cardinality: 0.1034 | Classification: 0.0967 | Classification_binary: 0.1405 | HitGrouping: 0.1801 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1072 | Regression_eta: 0.0012 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1023 | Regression_cos_phi: 0.0211 | Regression_energy: 0.0463 | Regression_eta: 0.0248 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0397 | LR: 1.19e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9541s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7158 | max_loss: 0.2520 | max_backward: 1.7727 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6276.9 | uss_MB: 5335.5 | child_rss_MB: 5162.7 | allocated_MB: 372.6 | reserved_MB: 23338.0 | inactive_split_MB: 212.6 +rank2 [2026-10-29 22:09:49,791] INFO : Step 45500/50000 rank2 | Train Loss: 1.0136 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1238 | Classification: 0.0902 | Classification_binary: 0.1661 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1060 | Regression_eta: 0.0013 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0414 | Regression_energy: 0.0492 | Regression_eta: 0.0322 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0463 | LR: 1.18e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.1606s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0147 | max_forward: 1.9023 | max_loss: 0.2979 | max_backward: 3.5478 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5661 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 6272.0 | uss_MB: 4281.7 | child_rss_MB: 5159.4 | allocated_MB: 372.4 | reserved_MB: 23338.0 | inactive_split_MB: 256.1 +rank2 [2026-10-29 22:14:16,023] INFO : Step 45600/50000 rank2 | Train Loss: 0.9675 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2172 | Cardinality: 0.1193 | Classification: 0.0849 | Classification_binary: 0.1400 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1218 | Regression_eta: 0.0010 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0551 | Regression_eta: 0.0322 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0670 | LR: 1.17e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.7205s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6781 | max_loss: 0.1576 | max_backward: 1.5089 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 0.5447 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6272.0 | uss_MB: 5337.5 | child_rss_MB: 5161.4 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 216.2 +rank2 [2026-10-29 22:18:15,630] INFO : Step 45700/50000 rank2 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1961 | Cardinality: 0.0908 | Classification: 0.0837 | Classification_binary: 0.1223 | HitGrouping: 0.1794 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1108 | Regression_eta: 0.0011 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1084 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0585 | Regression_eta: 0.0221 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0719 | LR: 1.16e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.4717s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6216 | max_loss: 0.1658 | max_backward: 1.1797 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6234.4 | uss_MB: 5336.4 | child_rss_MB: 5162.7 | allocated_MB: 370.8 | reserved_MB: 23338.0 | inactive_split_MB: 226.6 +rank2 [2026-10-29 22:22:19,536] INFO : Step 45800/50000 rank2 | Train Loss: 0.9372 | Train Loss Unweighted: 0.4491 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1073 | Classification: 0.0880 | Classification_binary: 0.1481 | HitGrouping: 0.1781 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1696 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0572 | Regression_eta: 0.0433 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0626 | LR: 1.16e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2366s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8017 | max_loss: 0.2428 | max_backward: 1.9917 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0978 | Memory: rss_MB: 6240.7 | uss_MB: 5347.0 | child_rss_MB: 5162.1 | allocated_MB: 375.2 | reserved_MB: 23338.0 | inactive_split_MB: 268.8 +rank2 [2026-10-29 22:26:26,931] INFO : Step 45900/50000 rank2 | Train Loss: 0.9600 | Train Loss Unweighted: 0.4592 | Train Loss Components: Auxiliary: 0.2179 | Cardinality: 0.1027 | Classification: 0.0816 | Classification_binary: 0.1590 | HitGrouping: 0.1875 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1600 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0610 | Regression_eta: 0.0378 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0401 | LR: 1.15e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.8434s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 1.3210 | max_loss: 0.4294 | max_backward: 3.2770 | max_optimizer_step: 0.2740 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 6235.6 | uss_MB: 5343.7 | child_rss_MB: 5161.3 | allocated_MB: 370.5 | reserved_MB: 23338.0 | inactive_split_MB: 259.6 +rank2 [2026-10-29 22:30:40,887] INFO : Step 46000/50000 rank2 | Train Loss: 0.8490 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.0938 | Classification: 0.0817 | Classification_binary: 0.1110 | HitGrouping: 0.1819 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0010 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1266 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0489 | Regression_eta: 0.0277 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0674 | LR: 1.14e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.5931s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.5955 | max_loss: 0.1430 | max_backward: 1.2483 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 0.6657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 6228.9 | uss_MB: 5344.9 | child_rss_MB: 5164.3 | allocated_MB: 365.7 | reserved_MB: 23338.0 | inactive_split_MB: 222.2 +rank2 [2026-10-29 22:30:41,828] INFO : Running validation on rank2 +rank2 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 22:31:32,964] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:31:33,519] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 22:31:59,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 22:32:01,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 22:32:03,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 22:32:05,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 22:32:15,127] INFO : Time taken to make predictions on device 2 is: 0.29 min +rank2 [2026-10-29 22:32:15,390] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:32:16,222] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 22:32:32,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 22:32:32,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 22:32:33,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 22:32:34,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 22:32:44,881] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 22:32:45,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:32:46,019] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 22:33:01,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 22:33:02,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 22:33:02,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 22:33:03,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 22:33:13,654] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-29 22:37:44,432] INFO : Step 46100/50000 rank2 | Train Loss: 0.9299 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1082 | Classification: 0.0899 | Classification_binary: 0.1357 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1081 | Regression_eta: 0.0010 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0435 | Regression_eta: 0.0405 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0620 | LR: 1.13e-04 | DataLoad Time: 0.0193s | Model Forward Time: 6.9699s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 2.5146 | max_loss: 0.4579 | max_backward: 2.4958 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1014 | Memory: rss_MB: 6980.2 | uss_MB: 6096.7 | child_rss_MB: 5172.0 | allocated_MB: 371.5 | reserved_MB: 22784.0 | inactive_split_MB: 245.0 +rank2 [2026-10-29 22:41:43,512] INFO : Step 46200/50000 rank2 | Train Loss: 0.9944 | Train Loss Unweighted: 0.4707 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1286 | Classification: 0.0940 | Classification_binary: 0.1623 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1052 | Regression_eta: 0.0011 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1825 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0583 | Regression_eta: 0.0302 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0547 | LR: 1.13e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2034s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7685 | max_loss: 0.2359 | max_backward: 1.6855 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1253 | Memory: rss_MB: 6976.0 | uss_MB: 6094.5 | child_rss_MB: 5141.0 | allocated_MB: 373.1 | reserved_MB: 22784.0 | inactive_split_MB: 264.3 +rank2 [2026-10-29 22:45:39,133] INFO : Step 46300/50000 rank2 | Train Loss: 0.9564 | Train Loss Unweighted: 0.4515 | Train Loss Components: Auxiliary: 0.2114 | Cardinality: 0.1124 | Classification: 0.0869 | Classification_binary: 0.1435 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1313 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0528 | Regression_eta: 0.0245 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0341 | LR: 1.12e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8631s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6709 | max_loss: 0.2410 | max_backward: 1.2189 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.8303 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6276.5 | uss_MB: 5413.4 | child_rss_MB: 5109.4 | allocated_MB: 372.3 | reserved_MB: 23784.0 | inactive_split_MB: 228.2 +rank2 [2026-10-29 22:49:34,649] INFO : Step 46400/50000 rank2 | Train Loss: 0.9514 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1147 | Classification: 0.0822 | Classification_binary: 0.1248 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0010 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1140 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0629 | Regression_eta: 0.0310 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0384 | LR: 1.11e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.9541s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.6748 | max_loss: 0.4371 | max_backward: 1.4077 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.8211 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6270.0 | uss_MB: 5413.5 | child_rss_MB: 5075.5 | allocated_MB: 374.2 | reserved_MB: 23988.0 | inactive_split_MB: 241.8 +rank2 [2026-10-29 22:53:41,743] INFO : Step 46500/50000 rank2 | Train Loss: 0.9517 | Train Loss Unweighted: 0.4566 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.0937 | Classification: 0.0928 | Classification_binary: 0.1533 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1031 | Regression_eta: 0.0013 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0521 | Regression_eta: 0.0539 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0820 | LR: 1.11e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9720s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0087 | max_forward: 0.7136 | max_loss: 0.1507 | max_backward: 1.5229 | max_optimizer_step: 0.2767 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6272.9 | uss_MB: 5416.3 | child_rss_MB: 5079.6 | allocated_MB: 372.0 | reserved_MB: 23988.0 | inactive_split_MB: 248.1 +rank2 [2026-10-29 22:57:41,615] INFO : Step 46600/50000 rank2 | Train Loss: 0.9541 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.2166 | Cardinality: 0.1078 | Classification: 0.0915 | Classification_binary: 0.1560 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1478 | Regression_cos_phi: 0.0397 | Regression_energy: 0.0503 | Regression_eta: 0.0259 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0588 | LR: 1.10e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5737s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7127 | max_loss: 0.1604 | max_backward: 1.3079 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4722 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 6261.6 | uss_MB: 5413.8 | child_rss_MB: 5045.9 | allocated_MB: 370.5 | reserved_MB: 23988.0 | inactive_split_MB: 179.8 +rank2 [2026-10-29 23:02:40,691] INFO : Step 46700/50000 rank2 | Train Loss: 0.9378 | Train Loss Unweighted: 0.4497 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.0977 | Classification: 0.0857 | Classification_binary: 0.1264 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0015 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1130 | Regression_cos_phi: 0.0371 | Regression_energy: 0.0554 | Regression_eta: 0.0256 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0387 | LR: 1.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7571s | Phase Times: max_pre_step_barrier: 0.0197 | max_batch_to_device: 0.0481 | max_forward: 0.6699 | max_loss: 0.2593 | max_backward: 1.4356 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5914 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6261.6 | uss_MB: 5415.0 | child_rss_MB: 5040.6 | allocated_MB: 368.9 | reserved_MB: 24784.0 | inactive_split_MB: 229.2 +rank2 [2026-10-29 23:06:36,578] INFO : Step 46800/50000 rank2 | Train Loss: 0.9152 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.2022 | Cardinality: 0.1011 | Classification: 0.0910 | Classification_binary: 0.1376 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0988 | Regression_eta: 0.0014 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1227 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0552 | Regression_eta: 0.0235 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0371 | LR: 1.09e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.6079s | Phase Times: max_pre_step_barrier: 0.1216 | max_batch_to_device: 0.0144 | max_forward: 1.0361 | max_loss: 0.2737 | max_backward: 1.7951 | max_optimizer_step: 0.2978 | max_tensorboard_logging: 0.5843 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6263.1 | uss_MB: 5417.5 | child_rss_MB: 5044.7 | allocated_MB: 368.9 | reserved_MB: 24994.0 | inactive_split_MB: 204.7 +rank2 [2026-10-29 23:10:45,824] INFO : Step 46900/50000 rank2 | Train Loss: 0.9278 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2149 | Cardinality: 0.1073 | Classification: 0.0827 | Classification_binary: 0.1607 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1027 | Regression_eta: 0.0010 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1669 | Regression_cos_phi: 0.0365 | Regression_energy: 0.0541 | Regression_eta: 0.0309 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0478 | LR: 1.09e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0940s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0446 | max_forward: 0.7815 | max_loss: 0.1553 | max_backward: 1.4732 | max_optimizer_step: 0.2929 | max_tensorboard_logging: 0.6668 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6256.7 | uss_MB: 5416.7 | child_rss_MB: 5041.4 | allocated_MB: 369.3 | reserved_MB: 24994.0 | inactive_split_MB: 274.7 +rank2 [2026-10-29 23:14:40,387] INFO : Step 47000/50000 rank2 | Train Loss: 0.9023 | Train Loss Unweighted: 0.4500 | Train Loss Components: Auxiliary: 0.2093 | Cardinality: 0.0953 | Classification: 0.0802 | Classification_binary: 0.1325 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1175 | Regression_eta: 0.0010 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1227 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0569 | Regression_eta: 0.0358 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0542 | LR: 1.08e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5092s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6624 | max_loss: 0.1440 | max_backward: 1.2028 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 6259.7 | uss_MB: 5421.0 | child_rss_MB: 5043.0 | allocated_MB: 369.4 | reserved_MB: 25784.0 | inactive_split_MB: 224.5 +rank2 [2026-10-29 23:14:41,333] INFO : Running validation on rank2 +rank2 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 23:15:32,857] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:15:33,149] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 23:16:02,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 23:16:03,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 23:16:04,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 23:16:05,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 23:16:15,859] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 23:16:16,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:16:16,905] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 23:16:32,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 23:16:33,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 23:16:34,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 23:16:36,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 23:16:46,173] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 23:16:47,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:16:47,863] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 23:17:03,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 23:17:04,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 23:17:04,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 23:17:05,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 23:17:15,394] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 23:21:36,067] INFO : Step 47100/50000 rank2 | Train Loss: 0.9138 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.2058 | Cardinality: 0.0938 | Classification: 0.0946 | Classification_binary: 0.1243 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1103 | Regression_eta: 0.0011 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1298 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0670 | Regression_eta: 0.0406 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0832 | LR: 1.07e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1764s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0063 | max_forward: 0.8048 | max_loss: 0.2433 | max_backward: 1.3159 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 1.0353 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 6952.3 | uss_MB: 6113.6 | child_rss_MB: 5037.2 | allocated_MB: 368.1 | reserved_MB: 22106.0 | inactive_split_MB: 269.0 +rank2 [2026-10-29 23:25:45,624] INFO : Step 47200/50000 rank2 | Train Loss: 0.9875 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2263 | Cardinality: 0.1259 | Classification: 0.0903 | Classification_binary: 0.1586 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1110 | Regression_eta: 0.0012 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0368 | Regression_energy: 0.0570 | Regression_eta: 0.0253 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0401 | LR: 1.07e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2839s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0486 | max_forward: 0.7581 | max_loss: 0.1506 | max_backward: 1.5914 | max_optimizer_step: 0.2300 | max_tensorboard_logging: 0.7953 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 6951.3 | uss_MB: 6117.8 | child_rss_MB: 5032.9 | allocated_MB: 370.5 | reserved_MB: 23104.0 | inactive_split_MB: 195.8 +rank2 [2026-10-29 23:29:53,763] INFO : Step 47300/50000 rank2 | Train Loss: 0.9350 | Train Loss Unweighted: 0.4467 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.0958 | Classification: 0.0857 | Classification_binary: 0.1512 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0620 | Regression_eta: 0.0389 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0529 | LR: 1.06e-04 | DataLoad Time: 0.0213s | Model Forward Time: 6.4014s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0184 | max_forward: 1.9890 | max_loss: 0.4771 | max_backward: 3.1656 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6309.1 | uss_MB: 5475.6 | child_rss_MB: 5028.4 | allocated_MB: 370.8 | reserved_MB: 23104.0 | inactive_split_MB: 271.1 +rank2 [2026-10-29 23:33:58,357] INFO : Step 47400/50000 rank2 | Train Loss: 0.8593 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.0807 | Classification: 0.0812 | Classification_binary: 0.1182 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1114 | Regression_eta: 0.0011 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1005 | Regression_cos_phi: 0.0280 | Regression_energy: 0.0413 | Regression_eta: 0.0280 | Regression_pt: 0.0418 | Regression_sin_phi: 0.0348 | LR: 1.06e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.0133s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0125 | max_forward: 0.7544 | max_loss: 0.1826 | max_backward: 1.4299 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8027 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6311.1 | uss_MB: 5475.9 | child_rss_MB: 5031.1 | allocated_MB: 369.9 | reserved_MB: 23104.0 | inactive_split_MB: 253.6 +rank2 [2026-10-29 23:37:53,327] INFO : Step 47500/50000 rank2 | Train Loss: 0.9089 | Train Loss Unweighted: 0.4235 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1000 | Classification: 0.0799 | Classification_binary: 0.1391 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1008 | Regression_eta: 0.0012 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0430 | Regression_eta: 0.0291 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0519 | LR: 1.06e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.2718s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0068 | max_forward: 0.7329 | max_loss: 0.1827 | max_backward: 1.5814 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9161 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6306.2 | uss_MB: 5478.3 | child_rss_MB: 5026.1 | allocated_MB: 371.7 | reserved_MB: 23104.0 | inactive_split_MB: 243.6 +rank2 [2026-10-29 23:41:48,171] INFO : Step 47600/50000 rank2 | Train Loss: 0.9781 | Train Loss Unweighted: 0.4635 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1226 | Classification: 0.0910 | Classification_binary: 0.1607 | HitGrouping: 0.1784 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0497 | Regression_eta: 0.0256 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0435 | LR: 1.05e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.4255s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8435 | max_loss: 0.1581 | max_backward: 1.8989 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5341 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6310.2 | uss_MB: 5483.0 | child_rss_MB: 5033.7 | allocated_MB: 370.6 | reserved_MB: 24122.0 | inactive_split_MB: 257.4 +rank2 [2026-10-29 23:46:00,212] INFO : Step 47700/50000 rank2 | Train Loss: 0.9311 | Train Loss Unweighted: 0.4523 | Train Loss Components: Auxiliary: 0.2105 | Cardinality: 0.0992 | Classification: 0.0889 | Classification_binary: 0.1321 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1138 | Regression_eta: 0.0012 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1587 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0551 | Regression_eta: 0.0386 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0551 | LR: 1.05e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.5066s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0068 | max_forward: 0.6708 | max_loss: 0.1613 | max_backward: 1.2326 | max_optimizer_step: 0.1946 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6306.7 | uss_MB: 5481.0 | child_rss_MB: 5031.8 | allocated_MB: 368.1 | reserved_MB: 24122.0 | inactive_split_MB: 276.0 +rank2 [2026-10-29 23:50:02,912] INFO : Step 47800/50000 rank2 | Train Loss: 0.9063 | Train Loss Unweighted: 0.4379 | Train Loss Components: Auxiliary: 0.2020 | Cardinality: 0.0932 | Classification: 0.0920 | Classification_binary: 0.1284 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0011 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0576 | Regression_eta: 0.0277 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0397 | LR: 1.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.8455s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0610 | max_forward: 0.6626 | max_loss: 0.1349 | max_backward: 1.5091 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6302.7 | uss_MB: 5485.7 | child_rss_MB: 5029.8 | allocated_MB: 369.5 | reserved_MB: 24122.0 | inactive_split_MB: 236.5 +rank2 [2026-10-29 23:54:07,644] INFO : Step 47900/50000 rank2 | Train Loss: 0.9591 | Train Loss Unweighted: 0.4478 | Train Loss Components: Auxiliary: 0.2152 | Cardinality: 0.1022 | Classification: 0.0882 | Classification_binary: 0.1545 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1003 | Regression_eta: 0.0014 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0537 | Regression_eta: 0.0640 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0574 | LR: 1.04e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.1695s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8744 | max_loss: 0.2241 | max_backward: 1.6370 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 6302.2 | uss_MB: 5483.9 | child_rss_MB: 5030.3 | allocated_MB: 370.5 | reserved_MB: 24122.0 | inactive_split_MB: 218.9 +rank2 [2026-10-30 00:00:28,988] INFO : Step 48000/50000 rank2 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1221 | Classification: 0.0823 | Classification_binary: 0.1525 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1079 | Regression_eta: 0.0017 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1531 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0574 | Regression_eta: 0.1555 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0628 | LR: 1.04e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1277s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0057 | max_forward: 0.7069 | max_loss: 0.2447 | max_backward: 1.4090 | max_optimizer_step: 0.1933 | max_tensorboard_logging: 0.8132 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1250 | Memory: rss_MB: 6301.2 | uss_MB: 5485.0 | child_rss_MB: 5024.5 | allocated_MB: 368.8 | reserved_MB: 24122.0 | inactive_split_MB: 269.6 +rank2 [2026-10-30 00:00:29,930] INFO : Running validation on rank2 +rank2 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 00:01:19,556] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:01:20,065] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 00:01:46,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 00:01:48,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 00:01:49,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 00:01:50,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 00:02:00,423] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-30 00:02:01,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:02:01,859] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 00:02:18,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 00:02:19,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 00:02:19,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 00:02:20,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 00:02:30,940] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-30 00:02:32,510] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:02:32,991] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 00:02:48,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 00:02:49,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 00:02:49,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 00:02:50,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 00:03:00,553] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-30 00:07:25,203] INFO : Step 48100/50000 rank2 | Train Loss: 0.9084 | Train Loss Unweighted: 0.4360 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1162 | Classification: 0.0769 | Classification_binary: 0.1204 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1183 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0240 | Classification_binary: 0.0993 | Regression_cos_phi: 0.0286 | Regression_energy: 0.0550 | Regression_eta: 0.0261 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0243 | LR: 1.03e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9237s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1372 | max_backward: 1.2024 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.9158 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7374.2 | uss_MB: 6557.5 | child_rss_MB: 5038.7 | allocated_MB: 367.7 | reserved_MB: 24830.0 | inactive_split_MB: 222.3 +rank2 [2026-10-30 00:11:33,472] INFO : Step 48200/50000 rank2 | Train Loss: 0.9585 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2151 | Cardinality: 0.1199 | Classification: 0.0839 | Classification_binary: 0.1503 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0472 | Regression_eta: 0.0323 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0342 | LR: 1.03e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4365s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7766 | max_loss: 0.1498 | max_backward: 1.7414 | max_optimizer_step: 0.3242 | max_tensorboard_logging: 0.7504 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 7374.2 | uss_MB: 6560.6 | child_rss_MB: 5031.5 | allocated_MB: 370.4 | reserved_MB: 24830.0 | inactive_split_MB: 179.3 +rank2 [2026-10-30 00:15:41,698] INFO : Step 48300/50000 rank2 | Train Loss: 1.0575 | Train Loss Unweighted: 0.4963 | Train Loss Components: Auxiliary: 0.2408 | Cardinality: 0.1341 | Classification: 0.0958 | Classification_binary: 0.1687 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1138 | Regression_eta: 0.0013 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0571 | Regression_eta: 0.0315 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0517 | LR: 1.03e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2032s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7497 | max_loss: 0.2201 | max_backward: 1.7233 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6378.2 | uss_MB: 5566.5 | child_rss_MB: 5032.3 | allocated_MB: 372.3 | reserved_MB: 24830.0 | inactive_split_MB: 269.6 +rank2 [2026-10-30 00:19:56,563] INFO : Step 48400/50000 rank2 | Train Loss: 0.9108 | Train Loss Unweighted: 0.4349 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1093 | Classification: 0.0691 | Classification_binary: 0.1311 | HitGrouping: 0.1830 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1153 | Regression_eta: 0.0013 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0301 | Classification_binary: 0.1192 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0436 | Regression_eta: 0.0212 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0230 | LR: 1.02e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.8544s | Phase Times: max_pre_step_barrier: 0.1141 | max_batch_to_device: 0.0064 | max_forward: 0.6489 | max_loss: 0.1390 | max_backward: 1.1801 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6705 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6380.2 | uss_MB: 5566.3 | child_rss_MB: 5030.1 | allocated_MB: 370.1 | reserved_MB: 24830.0 | inactive_split_MB: 261.5 +rank2 [2026-10-30 00:23:53,558] INFO : Step 48500/50000 rank2 | Train Loss: 0.9694 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1004 | Classification: 0.0844 | Classification_binary: 0.1367 | HitGrouping: 0.1928 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1194 | Regression_eta: 0.0012 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1403 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0625 | Regression_eta: 0.0364 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0614 | LR: 1.02e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.7577s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 1.7512 | max_loss: 0.2678 | max_backward: 1.9538 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.4804 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 6374.4 | uss_MB: 5565.1 | child_rss_MB: 5023.6 | allocated_MB: 371.5 | reserved_MB: 26842.0 | inactive_split_MB: 176.5 +rank2 [2026-10-30 00:27:51,108] INFO : Step 48600/50000 rank2 | Train Loss: 0.9724 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1148 | Classification: 0.0854 | Classification_binary: 0.1551 | HitGrouping: 0.1844 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1719 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0587 | Regression_eta: 0.0209 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0508 | LR: 1.02e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0078s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0086 | max_forward: 0.6984 | max_loss: 0.2525 | max_backward: 1.6611 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6375.4 | uss_MB: 5569.8 | child_rss_MB: 5026.2 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 234.8 +rank2 [2026-10-30 00:31:52,433] INFO : Step 48700/50000 rank2 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4863 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1212 | Classification: 0.0968 | Classification_binary: 0.1569 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1143 | Regression_eta: 0.0012 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1387 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0587 | Regression_eta: 0.0446 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0648 | LR: 1.02e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9646s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7410 | max_loss: 0.2472 | max_backward: 1.4647 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.6585 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6316.1 | uss_MB: 5565.7 | child_rss_MB: 4823.4 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 275.8 +rank2 [2026-10-30 00:35:40,395] INFO : Step 48800/50000 rank2 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1981 | Cardinality: 0.0898 | Classification: 0.0814 | Classification_binary: 0.1196 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1094 | Regression_eta: 0.0012 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.0876 | Regression_cos_phi: 0.0295 | Regression_energy: 0.0484 | Regression_eta: 0.0212 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0283 | LR: 1.01e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.7716s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.6122 | max_loss: 0.1775 | max_backward: 1.1932 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6316.1 | uss_MB: 5568.8 | child_rss_MB: 4813.9 | allocated_MB: 368.0 | reserved_MB: 26842.0 | inactive_split_MB: 263.7 +rank2 [2026-10-30 00:39:23,571] INFO : Step 48900/50000 rank2 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1005 | Classification: 0.0873 | Classification_binary: 0.1401 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0478 | Regression_eta: 0.0314 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0621 | LR: 1.01e-04 | DataLoad Time: 0.0193s | Model Forward Time: 3.9261s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0809 | max_forward: 0.7397 | max_loss: 0.1455 | max_backward: 1.5027 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6303.5 | uss_MB: 5491.8 | child_rss_MB: 4768.8 | allocated_MB: 373.0 | reserved_MB: 26842.0 | inactive_split_MB: 191.3 +rank2 [2026-10-30 00:43:08,795] INFO : Step 49000/50000 rank2 | Train Loss: 1.0019 | Train Loss Unweighted: 0.4657 | Train Loss Components: Auxiliary: 0.2334 | Cardinality: 0.1354 | Classification: 0.0893 | Classification_binary: 0.1587 | HitGrouping: 0.1838 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0010 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0545 | Regression_eta: 0.0208 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0321 | LR: 1.01e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4551s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1085 | max_forward: 0.8072 | max_loss: 0.1558 | max_backward: 1.7809 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.6082 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6305.0 | uss_MB: 5577.9 | child_rss_MB: 4749.4 | allocated_MB: 370.9 | reserved_MB: 26842.0 | inactive_split_MB: 271.3 +rank2 [2026-10-30 00:43:09,868] INFO : Running validation on rank2 +rank2 [2026-10-30 00:43:09,868] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 00:43:09,869] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 00:43:59,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:43:59,786] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 00:44:27,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 00:44:28,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 00:44:29,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 00:44:30,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 00:44:40,212] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 00:44:40,645] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:44:41,121] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 00:44:57,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 00:44:58,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 00:44:59,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 00:45:00,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 00:45:10,127] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-30 00:45:10,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:45:10,743] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 00:45:26,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 00:45:27,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 00:45:27,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 00:45:28,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 00:45:38,411] INFO : Time taken to make predictions on device 2 is: 0.22 min +rank2 [2026-10-30 00:49:42,470] INFO : Step 49100/50000 rank2 | Train Loss: 0.9797 | Train Loss Unweighted: 0.4561 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1125 | Classification: 0.0877 | Classification_binary: 0.1433 | HitGrouping: 0.1905 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0013 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1277 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0483 | Regression_eta: 0.0239 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0377 | LR: 1.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7311s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6869 | max_loss: 0.1436 | max_backward: 1.2639 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.4741 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0455 | Memory: rss_MB: 7470.2 | uss_MB: 6734.9 | child_rss_MB: 4778.3 | allocated_MB: 370.4 | reserved_MB: 22116.0 | inactive_split_MB: 274.9 +rank2 [2026-10-30 00:53:47,123] INFO : Step 49200/50000 rank2 | Train Loss: 0.8929 | Train Loss Unweighted: 0.4393 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1020 | Classification: 0.0903 | Classification_binary: 0.1250 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0009 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1010 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0581 | Regression_eta: 0.0192 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0247 | LR: 1.01e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.7549s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6890 | max_loss: 0.2430 | max_backward: 1.3427 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.6347 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7470.5 | uss_MB: 6738.6 | child_rss_MB: 4771.6 | allocated_MB: 368.3 | reserved_MB: 24092.0 | inactive_split_MB: 187.3 +rank2 [2026-10-30 00:57:33,941] INFO : Step 49300/50000 rank2 | Train Loss: 1.0122 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1210 | Classification: 0.0951 | Classification_binary: 0.1621 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1066 | Regression_eta: 0.0015 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0548 | Regression_eta: 0.0366 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0408 | LR: 1.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8977s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1495 | max_backward: 1.5906 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6415.4 | uss_MB: 5684.6 | child_rss_MB: 4746.9 | allocated_MB: 370.9 | reserved_MB: 24092.0 | inactive_split_MB: 226.9 +rank2 [2026-10-30 01:01:33,226] INFO : Step 49400/50000 rank2 | Train Loss: 0.9790 | Train Loss Unweighted: 0.4695 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1137 | Classification: 0.0883 | Classification_binary: 0.1513 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1131 | Regression_eta: 0.0013 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1466 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0619 | Regression_eta: 0.0357 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0450 | LR: 1.00e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.0370s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2433 | max_backward: 1.8823 | max_optimizer_step: 0.3081 | max_tensorboard_logging: 0.5951 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6413.4 | uss_MB: 5608.6 | child_rss_MB: 4765.4 | allocated_MB: 372.2 | reserved_MB: 24092.0 | inactive_split_MB: 278.9 +rank2 [2026-10-30 01:05:37,233] INFO : Step 49500/50000 rank2 | Train Loss: 0.9706 | Train Loss Unweighted: 0.4581 | Train Loss Components: Auxiliary: 0.2120 | Cardinality: 0.1158 | Classification: 0.0879 | Classification_binary: 0.1279 | HitGrouping: 0.1920 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0675 | Regression_eta: 0.0327 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0515 | LR: 1.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9624s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6307 | max_loss: 0.1505 | max_backward: 1.3337 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.8584 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6409.4 | uss_MB: 5683.7 | child_rss_MB: 4759.9 | allocated_MB: 370.4 | reserved_MB: 24614.0 | inactive_split_MB: 171.1 +rank2 [2026-10-30 01:09:49,676] INFO : Step 49600/50000 rank2 | Train Loss: 0.9965 | Train Loss Unweighted: 0.4752 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1098 | Classification: 0.1016 | Classification_binary: 0.1437 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1133 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0498 | Regression_eta: 0.0477 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0595 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1642s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.6933 | max_loss: 0.1496 | max_backward: 1.5639 | max_optimizer_step: 0.3077 | max_tensorboard_logging: 0.7702 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6412.2 | uss_MB: 5687.4 | child_rss_MB: 4755.4 | allocated_MB: 371.3 | reserved_MB: 24614.0 | inactive_split_MB: 250.0 +rank2 [2026-10-30 01:13:44,418] INFO : Step 49700/50000 rank2 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4775 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1108 | Classification: 0.1000 | Classification_binary: 0.1702 | HitGrouping: 0.1796 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1019 | Regression_eta: 0.0014 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0413 | Regression_eta: 0.0289 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0470 | LR: 1.00e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.2327s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7591 | max_loss: 0.1933 | max_backward: 1.7410 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 6407.4 | uss_MB: 5675.2 | child_rss_MB: 4703.7 | allocated_MB: 372.4 | reserved_MB: 24614.0 | inactive_split_MB: 259.5 +rank2 [2026-10-30 01:17:46,085] INFO : Step 49800/50000 rank2 | Train Loss: 0.9472 | Train Loss Unweighted: 0.4401 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1188 | Classification: 0.0821 | Classification_binary: 0.1427 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0521 | Regression_eta: 0.0584 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0504 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7725s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7430 | max_loss: 0.2153 | max_backward: 1.0846 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.7647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6388.3 | uss_MB: 5552.5 | child_rss_MB: 4630.2 | allocated_MB: 368.0 | reserved_MB: 24614.0 | inactive_split_MB: 273.7 +rank2 [2026-10-30 01:21:39,661] INFO : Step 49900/50000 rank2 | Train Loss: 0.9411 | Train Loss Unweighted: 0.4586 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.0996 | Classification: 0.0961 | Classification_binary: 0.1284 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1145 | Regression_eta: 0.0013 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0618 | Regression_eta: 0.0312 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0539 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0440s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.6992 | max_loss: 0.1527 | max_backward: 1.3836 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8576 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6363.3 | uss_MB: 5692.1 | child_rss_MB: 4528.3 | allocated_MB: 368.7 | reserved_MB: 24614.0 | inactive_split_MB: 241.3 +rank2 [2026-10-30 01:25:29,378] INFO : Step 50000/50000 rank2 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2297 | Cardinality: 0.1196 | Classification: 0.0902 | Classification_binary: 0.1583 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0550 | Regression_eta: 0.0240 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0505 | LR: 1.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2533s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.7146 | max_loss: 0.1481 | max_backward: 1.6207 | max_optimizer_step: 0.2721 | max_tensorboard_logging: 0.8195 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6364.5 | uss_MB: 5690.8 | child_rss_MB: 4519.4 | allocated_MB: 371.5 | reserved_MB: 25088.0 | inactive_split_MB: 268.9 +rank2 [2026-10-30 01:25:30,313] INFO : Running validation on rank2 +rank2 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 01:26:21,849] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:26:22,340] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 01:26:49,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 01:26:51,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 01:26:52,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 01:26:53,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 01:27:03,610] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-30 01:27:03,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:27:04,490] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 01:27:20,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 01:27:21,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 01:27:22,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 01:27:23,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 01:27:33,204] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-30 01:27:34,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:27:34,757] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 01:27:50,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 01:27:51,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 01:27:52,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 01:27:52,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 01:28:02,899] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 01:28:03,386] INFO : Training completed. Total time on device 2: 638.890min diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.3 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.3 new file mode 100644 index 0000000000000000000000000000000000000000..5dff6d2664e81d7312baa8b73347a10b4f06cb26 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.3 @@ -0,0 +1,2484 @@ +rank3 [2026-10-28 11:12:32,341] INFO : Initializing model with seed=12345; process seed=12348 +rank3 [2026-10-28 11:12:32,342] INFO : configured dtype=torch.bfloat16 for autocast +rank3 [2026-10-28 11:12:54,339] INFO : Instantiating model +rank3 [2026-10-28 11:12:54,339] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank3 [2026-10-28 11:12:54,339] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank3 [2026-10-28 11:12:54,358] INFO : Input encoding initialization took 0.02s +rank3 [2026-10-28 11:12:54,358] INFO : nn0 parameters: 171776 +rank3 [2026-10-28 11:12:54,358] INFO : Initializing shared backbone layers, num_convs=6 +rank3 [2026-10-28 11:12:54,358] INFO : Initializing backbone layer 0 +rank3 [2026-10-28 11:12:54,358] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:54,362] INFO : Initializing backbone layer 1 +rank3 [2026-10-28 11:12:54,362] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:54,365] INFO : Initializing backbone layer 2 +rank3 [2026-10-28 11:12:54,365] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:54,368] INFO : Initializing backbone layer 3 +rank3 [2026-10-28 11:12:54,368] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:54,371] INFO : Initializing backbone layer 4 +rank3 [2026-10-28 11:12:54,371] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:54,374] INFO : Initializing backbone layer 5 +rank3 [2026-10-28 11:12:54,374] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-28 11:12:54,377] INFO : Backbone initialization took 0.02s +rank3 [2026-10-28 11:12:54,377] INFO : backbone parameters: 2374656 +rank3 [2026-10-28 11:12:54,377] INFO : Initializing output DNNs +rank3 [2026-10-28 11:12:54,426] INFO : Output DNNs initialization took 0.05s +rank3 [2026-10-28 11:12:54,426] INFO : backbone_mode=BackboneMode.SHARED +rank3 [2026-10-28 11:12:54,426] INFO : jagged_attention enabled=True +rank3 [2026-10-28 11:12:54,426] INFO : flash_attn_varlen enabled=True +rank3 [2026-10-28 11:12:54,427] INFO : task_queries enabled=False +rank3 [2026-10-28 11:12:54,427] INFO : classification_norm parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : regression_norm parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : classification_query parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : regression_query parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : classification_readout parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : regression_readout parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : output_mode=set +rank3 [2026-10-28 11:12:54,427] INFO : set_decoder parameters: 4713812 +rank3 [2026-10-28 11:12:54,427] INFO : nn_binary_particle parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : nn_pid parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : nn_pt parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : nn_eta parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : nn_sin_phi parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : nn_cos_phi parameters: 0 +rank3 [2026-10-28 11:12:54,427] INFO : nn_energy parameters: 0 +rank3 [2026-10-28 11:12:54,428] INFO : Total MLPF parameters: 7260244 +rank3 [2026-10-28 11:12:54,428] INFO : MLPF __init__ done +rank3 [2026-10-28 11:12:54,430] INFO : Instantiated model +rank3 [2026-10-28 11:12:54,430] INFO : Moving model to device rank=3 +rank3 [2026-10-28 11:12:54,447] INFO : Moved model to device rank=3 +rank3 [2026-10-28 11:12:54,449] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank3 [2026-10-28 11:12:54,449] INFO : Configured model for SyncBatchNorm rank=3 +rank3 [2026-10-28 11:12:54,644] INFO : Configured model for DistributedDataParallel rank=3 +rank3 [2026-10-28 11:12:54,763] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank3 [2026-10-28 11:12:54,763] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank3 [2026-10-28 11:12:54,763] INFO : Trainable parameters: 7260244 +rank3 [2026-10-28 11:12:54,763] INFO : Non-trainable parameters: 0 +rank3 [2026-10-28 11:12:54,763] INFO : Total parameters: 7260244 +rank3 [2026-10-28 11:12:54,766] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank3 [2026-10-28 11:12:54,766] INFO : Getting dataloaders +rank3 [2026-10-28 11:12:54,766] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:03,741] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:04,811] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,144] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-28 11:13:06,144] INFO : Creating WeightedMixIterator with 1 data loaders. +rank3 [2026-10-28 11:13:06,144] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,413] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,673] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,928] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-28 11:13:06,929] INFO : Creating InterleavedIterator with 1 data loaders. +rank3 [2026-10-28 11:13:20,799] INFO : Got dataloaders +rank3 [2026-10-28 11:13:20,799] INFO : loader split=train rank=3 len=21112 +rank3 [2026-10-28 11:13:20,799] INFO : loader split=valid rank=3 len=12 +rank3 [2026-10-28 11:13:20,799] INFO : Starting training from step 1 to 50000 on rank 3 of 8 +rank3 [2026-10-28 11:13:20,799] INFO : Creating train_iterator +rank3 [2026-10-28 11:13:20,800] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-28 11:13:35,140] INFO : Created train_iterator +rank3 [2026-10-28 11:17:15,588] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5434697668626904, 'Classification': 0.9147019968181849, 'Regression_pt': 1.0888657218962907, 'Regression_eta': 0.019410275079426356, 'Regression_sin_phi': 0.00660347732307855, 'Regression_cos_phi': 0.006484371119877323, 'Regression_energy': 1.1263945941627025}, weights={'Classification_binary': 1.0, 'Classification': 0.5941495299339294, 'Regression_pt': 0.4991154968738556, 'Regression_eta': 27.99907684326172, 'Regression_sin_phi': 82.30054473876953, 'Regression_cos_phi': 83.812255859375, 'Regression_energy': 0.48248612880706787} +rank3 [2026-10-28 11:17:17,664] INFO : Step 100/50000 rank3 | Train Loss: 5.4941 | Train Loss Unweighted: 3.3573 | Train Loss Components: Auxiliary: 1.0155 | Cardinality: 0.6417 | Classification: 0.8373 | Classification_binary: 0.4757 | HitGrouping: 0.4797 | Regression_cos_phi: 0.0063 | Regression_energy: 1.0524 | Regression_eta: 0.0130 | Regression_pt: 0.9668 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8238 | Classification_binary: 0.4459 | Regression_cos_phi: 0.0055 | Regression_energy: 1.1109 | Regression_eta: 0.0098 | Regression_pt: 1.0205 | Regression_sin_phi: 0.0054 | LR: 1.00e-03 | DataLoad Time: 0.0145s | Model Forward Time: 3.0159s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.5597 | max_loss: 0.2615 | max_backward: 1.0157 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.4654 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4647.0 | uss_MB: 3401.7 | child_rss_MB: 5567.8 | allocated_MB: 341.0 | reserved_MB: 22914.0 | inactive_split_MB: 237.2 +rank3 [2026-10-28 11:20:50,236] INFO : Step 200/50000 rank3 | Train Loss: 4.4651 | Train Loss Unweighted: 2.7232 | Train Loss Components: Auxiliary: 0.8610 | Cardinality: 0.5153 | Classification: 0.6425 | Classification_binary: 0.5162 | HitGrouping: 0.4582 | Regression_cos_phi: 0.0043 | Regression_energy: 0.7694 | Regression_eta: 0.0086 | Regression_pt: 0.7776 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3746 | Classification_binary: 0.5071 | Regression_cos_phi: 0.3532 | Regression_energy: 0.3334 | Regression_eta: 0.2135 | Regression_pt: 0.3540 | Regression_sin_phi: 0.3569 | LR: 1.00e-03 | DataLoad Time: 0.0159s | Model Forward Time: 3.7251s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7368 | max_loss: 0.2463 | max_backward: 1.7032 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5468 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4651.2 | uss_MB: 3409.7 | child_rss_MB: 5571.3 | allocated_MB: 343.8 | reserved_MB: 23264.0 | inactive_split_MB: 284.1 +rank3 [2026-10-28 11:24:24,164] INFO : Step 300/50000 rank3 | Train Loss: 3.6383 | Train Loss Unweighted: 2.1819 | Train Loss Components: Auxiliary: 0.6931 | Cardinality: 0.3447 | Classification: 0.5089 | Classification_binary: 0.4824 | HitGrouping: 0.4305 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5908 | Regression_eta: 0.0074 | Regression_pt: 0.5852 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3252 | Classification_binary: 0.4882 | Regression_cos_phi: 0.2946 | Regression_energy: 0.2983 | Regression_eta: 0.2253 | Regression_pt: 0.3088 | Regression_sin_phi: 0.3279 | LR: 1.00e-03 | DataLoad Time: 0.0159s | Model Forward Time: 3.8727s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0315 | max_forward: 0.6855 | max_loss: 0.1568 | max_backward: 1.3632 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 1.0182 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4652.7 | uss_MB: 3415.2 | child_rss_MB: 5569.6 | allocated_MB: 341.4 | reserved_MB: 23916.0 | inactive_split_MB: 289.5 +rank3 [2026-10-28 11:27:58,396] INFO : Step 400/50000 rank3 | Train Loss: 3.5064 | Train Loss Unweighted: 1.9938 | Train Loss Components: Auxiliary: 0.6582 | Cardinality: 0.3738 | Classification: 0.4591 | Classification_binary: 0.4306 | HitGrouping: 0.4041 | Regression_cos_phi: 0.0039 | Regression_energy: 0.5433 | Regression_eta: 0.0080 | Regression_pt: 0.5456 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2786 | Classification_binary: 0.4442 | Regression_cos_phi: 0.3537 | Regression_energy: 0.2573 | Regression_eta: 0.2664 | Regression_pt: 0.2668 | Regression_sin_phi: 0.4131 | LR: 1.00e-03 | DataLoad Time: 0.0149s | Model Forward Time: 3.7434s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.6100 | max_loss: 0.1486 | max_backward: 1.2396 | max_optimizer_step: 0.1474 | max_tensorboard_logging: 1.0157 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4650.3 | uss_MB: 3416.8 | child_rss_MB: 5584.9 | allocated_MB: 342.9 | reserved_MB: 23982.0 | inactive_split_MB: 297.5 +rank3 [2026-10-28 11:31:35,335] INFO : Step 500/50000 rank3 | Train Loss: 3.8899 | Train Loss Unweighted: 2.1856 | Train Loss Components: Auxiliary: 0.6982 | Cardinality: 0.3682 | Classification: 0.4707 | Classification_binary: 0.4836 | HitGrouping: 0.4071 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6048 | Regression_eta: 0.0103 | Regression_pt: 0.6070 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2454 | Classification_binary: 0.4784 | Regression_cos_phi: 0.3479 | Regression_energy: 0.2707 | Regression_eta: 0.2450 | Regression_pt: 0.2840 | Regression_sin_phi: 0.3834 | LR: 1.00e-03 | DataLoad Time: 0.0154s | Model Forward Time: 4.1831s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.7460 | max_loss: 0.2622 | max_backward: 1.7372 | max_optimizer_step: 0.1478 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 4651.3 | uss_MB: 3418.2 | child_rss_MB: 5567.0 | allocated_MB: 344.2 | reserved_MB: 24988.0 | inactive_split_MB: 288.4 +rank3 [2026-10-28 11:35:12,859] INFO : Step 600/50000 rank3 | Train Loss: 3.5648 | Train Loss Unweighted: 2.0654 | Train Loss Components: Auxiliary: 0.6756 | Cardinality: 0.3628 | Classification: 0.4159 | Classification_binary: 0.5246 | HitGrouping: 0.3632 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5539 | Regression_eta: 0.0077 | Regression_pt: 0.5557 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2522 | Classification_binary: 0.5490 | Regression_cos_phi: 0.3668 | Regression_energy: 0.2607 | Regression_eta: 0.2480 | Regression_pt: 0.2708 | Regression_sin_phi: 0.3413 | LR: 1.00e-03 | DataLoad Time: 0.0159s | Model Forward Time: 4.1924s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0071 | max_forward: 0.7483 | max_loss: 0.2574 | max_backward: 1.6305 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.9709 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 4647.1 | uss_MB: 3418.2 | child_rss_MB: 5567.3 | allocated_MB: 344.6 | reserved_MB: 24988.0 | inactive_split_MB: 289.5 +rank3 [2026-10-28 11:38:47,057] INFO : Step 700/50000 rank3 | Train Loss: 3.0543 | Train Loss Unweighted: 1.8237 | Train Loss Components: Auxiliary: 0.6036 | Cardinality: 0.3094 | Classification: 0.3407 | Classification_binary: 0.4142 | HitGrouping: 0.3367 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5254 | Regression_eta: 0.0060 | Regression_pt: 0.5314 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2101 | Classification_binary: 0.3958 | Regression_cos_phi: 0.2152 | Regression_energy: 0.2378 | Regression_eta: 0.1575 | Regression_pt: 0.2555 | Regression_sin_phi: 0.2652 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 3.4880s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.6445 | max_loss: 0.2136 | max_backward: 1.2179 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.7893 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4647.0 | uss_MB: 3416.3 | child_rss_MB: 5577.3 | allocated_MB: 341.4 | reserved_MB: 24988.0 | inactive_split_MB: 289.7 +rank3 [2026-10-28 11:42:25,056] INFO : Step 800/50000 rank3 | Train Loss: 3.1854 | Train Loss Unweighted: 1.9126 | Train Loss Components: Auxiliary: 0.6154 | Cardinality: 0.2934 | Classification: 0.4077 | Classification_binary: 0.4086 | HitGrouping: 0.3317 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5404 | Regression_eta: 0.0067 | Regression_pt: 0.5423 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2472 | Classification_binary: 0.4082 | Regression_cos_phi: 0.3341 | Regression_energy: 0.2651 | Regression_eta: 0.1827 | Regression_pt: 0.2775 | Regression_sin_phi: 0.3378 | LR: 9.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.3178s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.6526 | max_loss: 0.2001 | max_backward: 1.2539 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5713 | max_tensorboard_flush: 0.0138 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 4642.6 | uss_MB: 3416.3 | child_rss_MB: 5579.0 | allocated_MB: 346.1 | reserved_MB: 24988.0 | inactive_split_MB: 222.7 +rank3 [2026-10-28 11:46:07,653] INFO : Step 900/50000 rank3 | Train Loss: 3.4950 | Train Loss Unweighted: 2.0678 | Train Loss Components: Auxiliary: 0.6566 | Cardinality: 0.3469 | Classification: 0.4160 | Classification_binary: 0.4953 | HitGrouping: 0.3295 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5663 | Regression_eta: 0.0083 | Regression_pt: 0.5742 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2456 | Classification_binary: 0.5328 | Regression_cos_phi: 0.3009 | Regression_energy: 0.2662 | Regression_eta: 0.2388 | Regression_pt: 0.2799 | Regression_sin_phi: 0.3088 | LR: 9.99e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.8992s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8131 | max_loss: 0.2548 | max_backward: 1.8396 | max_optimizer_step: 0.1492 | max_tensorboard_logging: 0.4938 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 4643.4 | uss_MB: 3417.5 | child_rss_MB: 5588.2 | allocated_MB: 344.2 | reserved_MB: 26006.0 | inactive_split_MB: 273.7 +rank3 [2026-10-28 11:49:58,412] INFO : Step 1000/50000 rank3 | Train Loss: 3.0581 | Train Loss Unweighted: 1.8801 | Train Loss Components: Auxiliary: 0.5942 | Cardinality: 0.2760 | Classification: 0.3603 | Classification_binary: 0.4232 | HitGrouping: 0.3177 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5411 | Regression_eta: 0.0066 | Regression_pt: 0.5426 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2281 | Classification_binary: 0.4191 | Regression_cos_phi: 0.2538 | Regression_energy: 0.2780 | Regression_eta: 0.1489 | Regression_pt: 0.2874 | Regression_sin_phi: 0.2084 | LR: 9.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2154s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1006 | max_forward: 0.7513 | max_loss: 0.1459 | max_backward: 1.3658 | max_optimizer_step: 0.1511 | max_tensorboard_logging: 1.2506 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4639.1 | uss_MB: 3415.3 | child_rss_MB: 5584.2 | allocated_MB: 344.4 | reserved_MB: 26006.0 | inactive_split_MB: 271.1 +rank3 [2026-10-28 11:49:59,498] INFO : Running validation on rank3 +rank3 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 11:50:52,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:50:53,226] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 11:51:19,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 11:51:20,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 11:51:21,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 11:51:22,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 11:51:32,448] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 11:51:32,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:51:33,551] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 11:51:49,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 11:51:50,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 11:51:51,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 11:51:52,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 11:52:02,604] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 11:52:02,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:52:03,476] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 11:52:18,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 11:52:19,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 11:52:19,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 11:52:20,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 11:52:30,660] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 11:56:35,453] INFO : Step 1100/50000 rank3 | Train Loss: 2.9181 | Train Loss Unweighted: 1.8556 | Train Loss Components: Auxiliary: 0.5873 | Cardinality: 0.2540 | Classification: 0.3370 | Classification_binary: 0.3789 | HitGrouping: 0.3247 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5584 | Regression_eta: 0.0064 | Regression_pt: 0.5695 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1941 | Classification_binary: 0.3816 | Regression_cos_phi: 0.3043 | Regression_energy: 0.2799 | Regression_eta: 0.2086 | Regression_pt: 0.2954 | Regression_sin_phi: 0.2739 | LR: 9.99e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.9459s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8528 | max_loss: 0.1390 | max_backward: 1.4220 | max_optimizer_step: 0.2689 | max_tensorboard_logging: 0.9135 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 6058.3 | uss_MB: 4835.8 | child_rss_MB: 5619.2 | allocated_MB: 344.1 | reserved_MB: 21080.0 | inactive_split_MB: 227.9 +rank3 [2026-10-28 12:00:25,221] INFO : Step 1200/50000 rank3 | Train Loss: 3.1885 | Train Loss Unweighted: 1.9202 | Train Loss Components: Auxiliary: 0.6265 | Cardinality: 0.3516 | Classification: 0.3718 | Classification_binary: 0.4294 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5507 | Regression_eta: 0.0065 | Regression_pt: 0.5556 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2259 | Classification_binary: 0.4137 | Regression_cos_phi: 0.2082 | Regression_energy: 0.2575 | Regression_eta: 0.1668 | Regression_pt: 0.2660 | Regression_sin_phi: 0.2698 | LR: 9.99e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.4095s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0076 | max_forward: 0.7370 | max_loss: 0.3921 | max_backward: 1.7945 | max_optimizer_step: 0.1639 | max_tensorboard_logging: 1.0479 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 6061.1 | uss_MB: 4838.4 | child_rss_MB: 5619.8 | allocated_MB: 344.9 | reserved_MB: 23662.0 | inactive_split_MB: 241.9 +rank3 [2026-10-28 12:04:16,198] INFO : Step 1300/50000 rank3 | Train Loss: 3.2694 | Train Loss Unweighted: 1.9280 | Train Loss Components: Auxiliary: 0.6193 | Cardinality: 0.3461 | Classification: 0.3601 | Classification_binary: 0.4731 | HitGrouping: 0.3156 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5359 | Regression_eta: 0.0069 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1886 | Classification_binary: 0.4911 | Regression_cos_phi: 0.3133 | Regression_energy: 0.2506 | Regression_eta: 0.1896 | Regression_pt: 0.2683 | Regression_sin_phi: 0.2918 | LR: 9.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.5220s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0540 | max_forward: 0.9909 | max_loss: 0.3234 | max_backward: 2.2964 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 1.1252 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5498.9 | uss_MB: 4193.7 | child_rss_MB: 5645.6 | allocated_MB: 344.9 | reserved_MB: 24052.0 | inactive_split_MB: 299.1 +rank3 [2026-10-28 12:07:55,471] INFO : Step 1400/50000 rank3 | Train Loss: 2.8491 | Train Loss Unweighted: 1.7889 | Train Loss Components: Auxiliary: 0.5645 | Cardinality: 0.2711 | Classification: 0.3226 | Classification_binary: 0.3768 | HitGrouping: 0.3109 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5376 | Regression_eta: 0.0061 | Regression_pt: 0.5407 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1915 | Classification_binary: 0.3329 | Regression_cos_phi: 0.2205 | Regression_energy: 0.2871 | Regression_eta: 0.1804 | Regression_pt: 0.3019 | Regression_sin_phi: 0.1999 | LR: 9.98e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7560s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0094 | max_forward: 0.6378 | max_loss: 0.4388 | max_backward: 1.2886 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 0.9631 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 5501.2 | uss_MB: 4277.0 | child_rss_MB: 5652.8 | allocated_MB: 350.4 | reserved_MB: 24916.0 | inactive_split_MB: 216.1 +rank3 [2026-10-28 12:11:42,090] INFO : Step 1500/50000 rank3 | Train Loss: 2.8928 | Train Loss Unweighted: 1.7749 | Train Loss Components: Auxiliary: 0.5690 | Cardinality: 0.2892 | Classification: 0.3566 | Classification_binary: 0.3704 | HitGrouping: 0.3078 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5147 | Regression_eta: 0.0058 | Regression_pt: 0.5216 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2639 | Classification_binary: 0.3760 | Regression_cos_phi: 0.2991 | Regression_energy: 0.2784 | Regression_eta: 0.2117 | Regression_pt: 0.2957 | Regression_sin_phi: 0.2698 | LR: 9.98e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5608s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.6094 | max_loss: 0.4244 | max_backward: 1.4479 | max_optimizer_step: 0.1602 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5488.8 | uss_MB: 4270.8 | child_rss_MB: 5621.5 | allocated_MB: 342.2 | reserved_MB: 24916.0 | inactive_split_MB: 182.9 +rank3 [2026-10-28 12:15:27,598] INFO : Step 1600/50000 rank3 | Train Loss: 2.9805 | Train Loss Unweighted: 1.8512 | Train Loss Components: Auxiliary: 0.5725 | Cardinality: 0.2336 | Classification: 0.3605 | Classification_binary: 0.4404 | HitGrouping: 0.3159 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5142 | Regression_eta: 0.0065 | Regression_pt: 0.5235 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1983 | Classification_binary: 0.4211 | Regression_cos_phi: 0.2418 | Regression_energy: 0.2879 | Regression_eta: 0.1508 | Regression_pt: 0.3034 | Regression_sin_phi: 0.2292 | LR: 9.98e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.7174s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7382 | max_loss: 0.1451 | max_backward: 1.5359 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.4679 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5499.3 | uss_MB: 4281.1 | child_rss_MB: 5637.2 | allocated_MB: 343.2 | reserved_MB: 25046.0 | inactive_split_MB: 260.8 +rank3 [2026-10-28 12:19:19,666] INFO : Step 1700/50000 rank3 | Train Loss: 2.9423 | Train Loss Unweighted: 1.7603 | Train Loss Components: Auxiliary: 0.5618 | Cardinality: 0.2834 | Classification: 0.3350 | Classification_binary: 0.4141 | HitGrouping: 0.3155 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4976 | Regression_eta: 0.0064 | Regression_pt: 0.5011 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1811 | Classification_binary: 0.4282 | Regression_cos_phi: 0.2730 | Regression_energy: 0.2584 | Regression_eta: 0.1534 | Regression_pt: 0.2721 | Regression_sin_phi: 0.2555 | LR: 9.97e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2303s | Phase Times: max_pre_step_barrier: 0.0070 | max_batch_to_device: 0.0065 | max_forward: 0.8351 | max_loss: 0.2466 | max_backward: 1.7292 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0857 | Memory: rss_MB: 5499.7 | uss_MB: 4254.7 | child_rss_MB: 5622.9 | allocated_MB: 342.0 | reserved_MB: 25046.0 | inactive_split_MB: 198.9 +rank3 [2026-10-28 12:23:11,816] INFO : Step 1800/50000 rank3 | Train Loss: 2.7182 | Train Loss Unweighted: 1.6765 | Train Loss Components: Auxiliary: 0.5305 | Cardinality: 0.2556 | Classification: 0.3337 | Classification_binary: 0.3613 | HitGrouping: 0.3103 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4855 | Regression_eta: 0.0054 | Regression_pt: 0.4854 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1863 | Classification_binary: 0.3617 | Regression_cos_phi: 0.2430 | Regression_energy: 0.2307 | Regression_eta: 0.1768 | Regression_pt: 0.2397 | Regression_sin_phi: 0.2723 | LR: 9.97e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0944s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6573 | max_loss: 0.2402 | max_backward: 1.4111 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 1.0385 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0959 | Memory: rss_MB: 5501.3 | uss_MB: 4283.5 | child_rss_MB: 5636.1 | allocated_MB: 340.6 | reserved_MB: 25046.0 | inactive_split_MB: 181.6 +rank3 [2026-10-28 12:26:53,741] INFO : Step 1900/50000 rank3 | Train Loss: 2.9893 | Train Loss Unweighted: 1.8606 | Train Loss Components: Auxiliary: 0.5839 | Cardinality: 0.2900 | Classification: 0.3568 | Classification_binary: 0.4041 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5399 | Regression_eta: 0.0063 | Regression_pt: 0.5477 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2375 | Classification_binary: 0.4001 | Regression_cos_phi: 0.2720 | Regression_energy: 0.2644 | Regression_eta: 0.1941 | Regression_pt: 0.2740 | Regression_sin_phi: 0.2784 | LR: 9.97e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.8710s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.6296 | max_loss: 0.1482 | max_backward: 1.2931 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 1.0537 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 5503.8 | uss_MB: 4286.6 | child_rss_MB: 5619.9 | allocated_MB: 344.8 | reserved_MB: 25048.0 | inactive_split_MB: 225.2 +rank3 [2026-10-28 12:30:53,516] INFO : Step 2000/50000 rank3 | Train Loss: 3.0007 | Train Loss Unweighted: 1.8040 | Train Loss Components: Auxiliary: 0.5673 | Cardinality: 0.3091 | Classification: 0.3519 | Classification_binary: 0.4482 | HitGrouping: 0.3137 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4922 | Regression_eta: 0.0062 | Regression_pt: 0.4996 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1883 | Classification_binary: 0.4354 | Regression_cos_phi: 0.2723 | Regression_energy: 0.2566 | Regression_eta: 0.1554 | Regression_pt: 0.2676 | Regression_sin_phi: 0.2067 | LR: 9.96e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1682s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6813 | max_loss: 0.2416 | max_backward: 1.5165 | max_optimizer_step: 0.1635 | max_tensorboard_logging: 0.9842 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5502.9 | uss_MB: 4287.2 | child_rss_MB: 5669.7 | allocated_MB: 346.7 | reserved_MB: 25696.0 | inactive_split_MB: 287.8 +rank3 [2026-10-28 12:30:54,413] INFO : Running validation on rank3 +rank3 [2026-10-28 12:30:54,413] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 12:30:54,413] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 12:31:46,081] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:31:46,628] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 12:32:13,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 12:32:14,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 12:32:15,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 12:32:16,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 12:32:26,877] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 12:32:27,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:32:27,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 12:32:43,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 12:32:44,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 12:32:45,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 12:32:46,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 12:32:56,564] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 12:32:57,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:32:57,797] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 12:33:13,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 12:33:14,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 12:33:15,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 12:33:15,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 12:33:25,732] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 12:37:37,593] INFO : Step 2100/50000 rank3 | Train Loss: 2.6299 | Train Loss Unweighted: 1.6429 | Train Loss Components: Auxiliary: 0.5172 | Cardinality: 0.2489 | Classification: 0.3119 | Classification_binary: 0.3591 | HitGrouping: 0.3047 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4785 | Regression_eta: 0.0053 | Regression_pt: 0.4834 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1739 | Classification_binary: 0.3337 | Regression_cos_phi: 0.2088 | Regression_energy: 0.2243 | Regression_eta: 0.0968 | Regression_pt: 0.2341 | Regression_sin_phi: 0.1137 | LR: 9.96e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0186s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0685 | max_forward: 0.7343 | max_loss: 0.1422 | max_backward: 1.3363 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.0015 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6447.9 | uss_MB: 5231.1 | child_rss_MB: 5629.3 | allocated_MB: 341.1 | reserved_MB: 22546.0 | inactive_split_MB: 269.0 +rank3 [2026-10-28 12:41:24,842] INFO : Step 2200/50000 rank3 | Train Loss: 2.8483 | Train Loss Unweighted: 1.7068 | Train Loss Components: Auxiliary: 0.5372 | Cardinality: 0.2547 | Classification: 0.3411 | Classification_binary: 0.3679 | HitGrouping: 0.3030 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4876 | Regression_eta: 0.0061 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1992 | Classification_binary: 0.3283 | Regression_cos_phi: 0.1847 | Regression_energy: 0.2190 | Regression_eta: 0.1447 | Regression_pt: 0.2314 | Regression_sin_phi: 0.2844 | LR: 9.96e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.4912s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6352 | max_loss: 0.1414 | max_backward: 1.3119 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.4703 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6448.9 | uss_MB: 5233.9 | child_rss_MB: 5636.2 | allocated_MB: 341.4 | reserved_MB: 22598.0 | inactive_split_MB: 313.4 +rank3 [2026-10-28 12:45:27,860] INFO : Step 2300/50000 rank3 | Train Loss: 3.1410 | Train Loss Unweighted: 1.8301 | Train Loss Components: Auxiliary: 0.5918 | Cardinality: 0.3583 | Classification: 0.3517 | Classification_binary: 0.4348 | HitGrouping: 0.3083 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5121 | Regression_eta: 0.0065 | Regression_pt: 0.5184 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1938 | Classification_binary: 0.4707 | Regression_cos_phi: 0.2456 | Regression_energy: 0.2491 | Regression_eta: 0.1585 | Regression_pt: 0.2624 | Regression_sin_phi: 0.2781 | LR: 9.95e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8600s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.7258 | max_loss: 0.1520 | max_backward: 1.6288 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.6027 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0166 | Memory: rss_MB: 5633.9 | uss_MB: 4422.4 | child_rss_MB: 5644.1 | allocated_MB: 346.5 | reserved_MB: 23596.0 | inactive_split_MB: 273.5 +rank3 [2026-10-28 12:49:22,411] INFO : Step 2400/50000 rank3 | Train Loss: 2.8063 | Train Loss Unweighted: 1.7109 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.2683 | Classification: 0.3276 | Classification_binary: 0.3929 | HitGrouping: 0.3025 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4873 | Regression_eta: 0.0054 | Regression_pt: 0.4919 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1663 | Classification_binary: 0.3711 | Regression_cos_phi: 0.2625 | Regression_energy: 0.2217 | Regression_eta: 0.1223 | Regression_pt: 0.2285 | Regression_sin_phi: 0.2388 | LR: 9.95e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.5882s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0314 | max_forward: 0.7331 | max_loss: 0.1502 | max_backward: 1.4174 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.5071 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0157 | Memory: rss_MB: 5635.8 | uss_MB: 4426.4 | child_rss_MB: 5648.9 | allocated_MB: 344.4 | reserved_MB: 23596.0 | inactive_split_MB: 276.8 +rank3 [2026-10-28 12:53:10,912] INFO : Step 2500/50000 rank3 | Train Loss: 2.5300 | Train Loss Unweighted: 1.6034 | Train Loss Components: Auxiliary: 0.5018 | Cardinality: 0.2387 | Classification: 0.2872 | Classification_binary: 0.3313 | HitGrouping: 0.3000 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4857 | Regression_eta: 0.0042 | Regression_pt: 0.4904 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1763 | Classification_binary: 0.3514 | Regression_cos_phi: 0.1905 | Regression_energy: 0.1817 | Regression_eta: 0.1186 | Regression_pt: 0.1891 | Regression_sin_phi: 0.1535 | LR: 9.94e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.0253s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0177 | max_forward: 0.6104 | max_loss: 0.1394 | max_backward: 1.2397 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 1.1029 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5639.0 | uss_MB: 4422.1 | child_rss_MB: 5639.4 | allocated_MB: 342.5 | reserved_MB: 23596.0 | inactive_split_MB: 302.8 +rank3 [2026-10-28 12:56:55,919] INFO : Step 2600/50000 rank3 | Train Loss: 2.8992 | Train Loss Unweighted: 1.7075 | Train Loss Components: Auxiliary: 0.5471 | Cardinality: 0.3219 | Classification: 0.3420 | Classification_binary: 0.4005 | HitGrouping: 0.3027 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4737 | Regression_eta: 0.0057 | Regression_pt: 0.4796 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1790 | Classification_binary: 0.3871 | Regression_cos_phi: 0.2552 | Regression_energy: 0.2296 | Regression_eta: 0.1439 | Regression_pt: 0.2416 | Regression_sin_phi: 0.2195 | LR: 9.94e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.7068s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8849 | max_loss: 0.1495 | max_backward: 1.9136 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1254 | Memory: rss_MB: 5638.3 | uss_MB: 4434.4 | child_rss_MB: 5644.3 | allocated_MB: 343.4 | reserved_MB: 23596.0 | inactive_split_MB: 290.6 +rank3 [2026-10-28 13:00:46,527] INFO : Step 2700/50000 rank3 | Train Loss: 2.8481 | Train Loss Unweighted: 1.7200 | Train Loss Components: Auxiliary: 0.5468 | Cardinality: 0.2918 | Classification: 0.3199 | Classification_binary: 0.4344 | HitGrouping: 0.3029 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4756 | Regression_eta: 0.0057 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1678 | Classification_binary: 0.4547 | Regression_cos_phi: 0.1876 | Regression_energy: 0.2250 | Regression_eta: 0.1367 | Regression_pt: 0.2360 | Regression_sin_phi: 0.1730 | LR: 9.94e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3248s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0064 | max_forward: 0.7554 | max_loss: 0.2027 | max_backward: 1.6333 | max_optimizer_step: 0.1633 | max_tensorboard_logging: 0.9978 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5637.9 | uss_MB: 4434.7 | child_rss_MB: 5643.6 | allocated_MB: 343.8 | reserved_MB: 23598.0 | inactive_split_MB: 284.2 +rank3 [2026-10-28 13:04:40,295] INFO : Step 2800/50000 rank3 | Train Loss: 2.6381 | Train Loss Unweighted: 1.5955 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2387 | Classification: 0.2848 | Classification_binary: 0.3420 | HitGrouping: 0.2974 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4746 | Regression_eta: 0.0056 | Regression_pt: 0.4829 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1611 | Classification_binary: 0.3224 | Regression_cos_phi: 0.1820 | Regression_energy: 0.2529 | Regression_eta: 0.1369 | Regression_pt: 0.2667 | Regression_sin_phi: 0.1753 | LR: 9.93e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.7383s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6805 | max_loss: 0.2368 | max_backward: 1.1707 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5635.7 | uss_MB: 4432.9 | child_rss_MB: 5638.4 | allocated_MB: 341.2 | reserved_MB: 24554.0 | inactive_split_MB: 308.0 +rank3 [2026-10-28 13:08:36,459] INFO : Step 2900/50000 rank3 | Train Loss: 2.7831 | Train Loss Unweighted: 1.6528 | Train Loss Components: Auxiliary: 0.5323 | Cardinality: 0.3031 | Classification: 0.3069 | Classification_binary: 0.3681 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4798 | Regression_eta: 0.0056 | Regression_pt: 0.4869 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1660 | Classification_binary: 0.3778 | Regression_cos_phi: 0.1687 | Regression_energy: 0.2358 | Regression_eta: 0.1272 | Regression_pt: 0.2476 | Regression_sin_phi: 0.2754 | LR: 9.93e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.7850s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6242 | max_loss: 0.1475 | max_backward: 1.3573 | max_optimizer_step: 0.2831 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5637.0 | uss_MB: 4433.8 | child_rss_MB: 5645.5 | allocated_MB: 341.4 | reserved_MB: 25570.0 | inactive_split_MB: 292.0 +rank3 [2026-10-28 13:12:15,921] INFO : Step 3000/50000 rank3 | Train Loss: 2.9336 | Train Loss Unweighted: 1.7260 | Train Loss Components: Auxiliary: 0.5466 | Cardinality: 0.2829 | Classification: 0.3524 | Classification_binary: 0.4205 | HitGrouping: 0.3085 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4673 | Regression_eta: 0.0061 | Regression_pt: 0.4733 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2089 | Classification_binary: 0.3863 | Regression_cos_phi: 0.2612 | Regression_energy: 0.2311 | Regression_eta: 0.1747 | Regression_pt: 0.2460 | Regression_sin_phi: 0.2767 | LR: 9.92e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1516s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0386 | max_forward: 0.8057 | max_loss: 0.1488 | max_backward: 1.9376 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5635.9 | uss_MB: 4440.1 | child_rss_MB: 5640.1 | allocated_MB: 346.5 | reserved_MB: 25570.0 | inactive_split_MB: 281.4 +rank3 [2026-10-28 13:12:16,814] INFO : Running validation on rank3 +rank3 [2026-10-28 13:12:16,814] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 13:12:16,814] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 13:13:09,908] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:13:10,420] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 13:13:30,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 13:13:31,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 13:13:32,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 13:13:33,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 13:13:43,108] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 13:13:43,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:13:44,015] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 13:14:00,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 13:14:01,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 13:14:02,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 13:14:03,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 13:14:13,563] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 13:14:13,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:14:14,219] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 13:14:29,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 13:14:29,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 13:14:30,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 13:14:31,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 13:14:41,260] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 13:18:52,946] INFO : Step 3100/50000 rank3 | Train Loss: 2.7478 | Train Loss Unweighted: 1.6734 | Train Loss Components: Auxiliary: 0.5216 | Cardinality: 0.2531 | Classification: 0.2981 | Classification_binary: 0.3781 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4881 | Regression_eta: 0.0057 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1977 | Classification_binary: 0.3954 | Regression_cos_phi: 0.2275 | Regression_energy: 0.2245 | Regression_eta: 0.1758 | Regression_pt: 0.2353 | Regression_sin_phi: 0.3190 | LR: 9.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5547s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.6511 | max_loss: 0.3988 | max_backward: 1.4306 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 1.2631 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5749.3 | uss_MB: 4552.2 | child_rss_MB: 5658.3 | allocated_MB: 342.5 | reserved_MB: 23120.0 | inactive_split_MB: 194.6 +rank3 [2026-10-28 13:22:38,495] INFO : Step 3200/50000 rank3 | Train Loss: 2.5254 | Train Loss Unweighted: 1.5710 | Train Loss Components: Auxiliary: 0.4888 | Cardinality: 0.2129 | Classification: 0.2845 | Classification_binary: 0.3271 | HitGrouping: 0.2964 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4770 | Regression_eta: 0.0050 | Regression_pt: 0.4723 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1837 | Classification_binary: 0.3245 | Regression_cos_phi: 0.2679 | Regression_energy: 0.2295 | Regression_eta: 0.1267 | Regression_pt: 0.2367 | Regression_sin_phi: 0.2443 | LR: 9.91e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5510s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6134 | max_loss: 0.2441 | max_backward: 1.2833 | max_optimizer_step: 0.2753 | max_tensorboard_logging: 0.5629 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5733.9 | uss_MB: 4558.8 | child_rss_MB: 5628.8 | allocated_MB: 343.6 | reserved_MB: 23340.0 | inactive_split_MB: 191.9 +rank3 [2026-10-28 13:26:30,566] INFO : Step 3300/50000 rank3 | Train Loss: 2.8037 | Train Loss Unweighted: 1.6635 | Train Loss Components: Auxiliary: 0.5345 | Cardinality: 0.2841 | Classification: 0.3310 | Classification_binary: 0.3857 | HitGrouping: 0.3002 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4633 | Regression_eta: 0.0058 | Regression_pt: 0.4718 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2043 | Classification_binary: 0.4252 | Regression_cos_phi: 0.2913 | Regression_energy: 0.2313 | Regression_eta: 0.2161 | Regression_pt: 0.2439 | Regression_sin_phi: 0.2570 | LR: 9.90e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2957s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9147 | max_loss: 0.1535 | max_backward: 1.9100 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.6631 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5735.1 | uss_MB: 4562.9 | child_rss_MB: 5661.7 | allocated_MB: 344.0 | reserved_MB: 23618.0 | inactive_split_MB: 250.0 +rank3 [2026-10-28 13:30:24,443] INFO : Step 3400/50000 rank3 | Train Loss: 2.8855 | Train Loss Unweighted: 1.7067 | Train Loss Components: Auxiliary: 0.5454 | Cardinality: 0.2859 | Classification: 0.3252 | Classification_binary: 0.4155 | HitGrouping: 0.2965 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4731 | Regression_eta: 0.0067 | Regression_pt: 0.4802 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1825 | Classification_binary: 0.4251 | Regression_cos_phi: 0.2256 | Regression_energy: 0.2278 | Regression_eta: 0.1859 | Regression_pt: 0.2397 | Regression_sin_phi: 0.2689 | LR: 9.90e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5646s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0438 | max_forward: 0.7271 | max_loss: 0.1506 | max_backward: 1.3643 | max_optimizer_step: 0.1716 | max_tensorboard_logging: 0.4827 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5729.2 | uss_MB: 4562.1 | child_rss_MB: 5658.8 | allocated_MB: 346.0 | reserved_MB: 23618.0 | inactive_split_MB: 235.6 +rank3 [2026-10-28 13:34:14,208] INFO : Step 3500/50000 rank3 | Train Loss: 2.4422 | Train Loss Unweighted: 1.4809 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.2120 | Classification: 0.2615 | Classification_binary: 0.3296 | HitGrouping: 0.2898 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4369 | Regression_eta: 0.0054 | Regression_pt: 0.4427 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1705 | Classification_binary: 0.3200 | Regression_cos_phi: 0.2238 | Regression_energy: 0.2067 | Regression_eta: 0.1453 | Regression_pt: 0.2157 | Regression_sin_phi: 0.1761 | LR: 9.89e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7303s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0058 | max_forward: 0.7056 | max_loss: 0.1971 | max_backward: 1.3359 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.6307 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5720.3 | uss_MB: 4563.4 | child_rss_MB: 5630.0 | allocated_MB: 339.0 | reserved_MB: 24610.0 | inactive_split_MB: 280.8 +rank3 [2026-10-28 13:38:02,722] INFO : Step 3600/50000 rank3 | Train Loss: 2.5780 | Train Loss Unweighted: 1.5250 | Train Loss Components: Auxiliary: 0.4892 | Cardinality: 0.2463 | Classification: 0.3079 | Classification_binary: 0.3446 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4295 | Regression_eta: 0.0052 | Regression_pt: 0.4324 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1685 | Classification_binary: 0.3608 | Regression_cos_phi: 0.1652 | Regression_energy: 0.1987 | Regression_eta: 0.1328 | Regression_pt: 0.2089 | Regression_sin_phi: 0.1835 | LR: 9.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.4652s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1453 | max_backward: 1.4106 | max_optimizer_step: 0.1681 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0126 | Memory: rss_MB: 5726.7 | uss_MB: 4567.6 | child_rss_MB: 5631.8 | allocated_MB: 343.9 | reserved_MB: 24610.0 | inactive_split_MB: 244.5 +rank3 [2026-10-28 13:41:50,385] INFO : Step 3700/50000 rank3 | Train Loss: 2.9213 | Train Loss Unweighted: 1.6952 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.3153 | Classification: 0.3418 | Classification_binary: 0.4227 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4570 | Regression_eta: 0.0063 | Regression_pt: 0.4612 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2073 | Classification_binary: 0.4279 | Regression_cos_phi: 0.2856 | Regression_energy: 0.2199 | Regression_eta: 0.1948 | Regression_pt: 0.2277 | Regression_sin_phi: 0.2692 | LR: 9.88e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2215s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7710 | max_loss: 0.1507 | max_backward: 1.6430 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.8888 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0145 | Memory: rss_MB: 5726.2 | uss_MB: 4569.4 | child_rss_MB: 5661.5 | allocated_MB: 345.4 | reserved_MB: 24610.0 | inactive_split_MB: 268.9 +rank3 [2026-10-28 13:45:46,300] INFO : Step 3800/50000 rank3 | Train Loss: 2.6063 | Train Loss Unweighted: 1.5289 | Train Loss Components: Auxiliary: 0.4933 | Cardinality: 0.2661 | Classification: 0.2839 | Classification_binary: 0.3652 | HitGrouping: 0.2879 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4319 | Regression_eta: 0.0058 | Regression_pt: 0.4369 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1723 | Classification_binary: 0.3832 | Regression_cos_phi: 0.2542 | Regression_energy: 0.2117 | Regression_eta: 0.1319 | Regression_pt: 0.2203 | Regression_sin_phi: 0.1639 | LR: 9.87e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1721s | Phase Times: max_pre_step_barrier: 0.1134 | max_batch_to_device: 0.0063 | max_forward: 0.6784 | max_loss: 0.1451 | max_backward: 1.4269 | max_optimizer_step: 0.2795 | max_tensorboard_logging: 0.9060 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5726.3 | uss_MB: 4569.5 | child_rss_MB: 5664.0 | allocated_MB: 343.8 | reserved_MB: 24610.0 | inactive_split_MB: 202.2 +rank3 [2026-10-28 13:49:32,924] INFO : Step 3900/50000 rank3 | Train Loss: 2.2761 | Train Loss Unweighted: 1.4165 | Train Loss Components: Auxiliary: 0.4517 | Cardinality: 0.1938 | Classification: 0.2553 | Classification_binary: 0.2930 | HitGrouping: 0.2882 | Regression_cos_phi: 0.0020 | Regression_energy: 0.4264 | Regression_eta: 0.0045 | Regression_pt: 0.4331 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1233 | Classification_binary: 0.2446 | Regression_cos_phi: 0.1335 | Regression_energy: 0.2109 | Regression_eta: 0.1041 | Regression_pt: 0.2184 | Regression_sin_phi: 0.1385 | LR: 9.87e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8910s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6197 | max_loss: 0.1501 | max_backward: 1.2810 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.0075 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 5727.5 | uss_MB: 3764.5 | child_rss_MB: 5661.9 | allocated_MB: 343.5 | reserved_MB: 24610.0 | inactive_split_MB: 286.6 +rank3 [2026-10-28 13:53:23,686] INFO : Step 4000/50000 rank3 | Train Loss: 2.6760 | Train Loss Unweighted: 1.5903 | Train Loss Components: Auxiliary: 0.5112 | Cardinality: 0.2445 | Classification: 0.3070 | Classification_binary: 0.3812 | HitGrouping: 0.2906 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4413 | Regression_eta: 0.0062 | Regression_pt: 0.4492 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1725 | Classification_binary: 0.3614 | Regression_cos_phi: 0.2483 | Regression_energy: 0.2177 | Regression_eta: 0.2105 | Regression_pt: 0.2299 | Regression_sin_phi: 0.2311 | LR: 9.86e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.0147s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0492 | max_forward: 0.7834 | max_loss: 0.1586 | max_backward: 1.6903 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.5589 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5731.5 | uss_MB: 4577.2 | child_rss_MB: 5661.6 | allocated_MB: 345.5 | reserved_MB: 24610.0 | inactive_split_MB: 252.6 +rank3 [2026-10-28 13:53:24,557] INFO : Running validation on rank3 +rank3 [2026-10-28 13:53:24,557] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 13:53:24,557] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 13:54:13,892] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:54:14,419] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 13:54:39,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 13:54:41,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 13:54:42,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 13:54:43,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 13:54:53,305] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 13:54:53,386] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:54:53,946] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 13:55:09,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 13:55:10,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 13:55:11,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 13:55:12,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 13:55:22,750] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 13:55:23,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:55:23,753] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 13:55:39,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 13:55:39,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 13:55:40,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 13:55:41,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 13:55:51,208] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 13:59:55,049] INFO : Step 4100/50000 rank3 | Train Loss: 2.6923 | Train Loss Unweighted: 1.5640 | Train Loss Components: Auxiliary: 0.5058 | Cardinality: 0.2671 | Classification: 0.2981 | Classification_binary: 0.4073 | HitGrouping: 0.2911 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4210 | Regression_eta: 0.0055 | Regression_pt: 0.4265 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1777 | Classification_binary: 0.4152 | Regression_cos_phi: 0.2961 | Regression_energy: 0.2206 | Regression_eta: 0.1618 | Regression_pt: 0.2307 | Regression_sin_phi: 0.2304 | LR: 9.85e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.1477s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0141 | max_forward: 0.7422 | max_loss: 0.3919 | max_backward: 1.3629 | max_optimizer_step: 0.1890 | max_tensorboard_logging: 0.9903 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6680.3 | uss_MB: 5525.0 | child_rss_MB: 5670.9 | allocated_MB: 345.2 | reserved_MB: 23430.0 | inactive_split_MB: 294.4 +rank3 [2026-10-28 14:03:47,899] INFO : Step 4200/50000 rank3 | Train Loss: 2.3415 | Train Loss Unweighted: 1.4424 | Train Loss Components: Auxiliary: 0.4542 | Cardinality: 0.2058 | Classification: 0.2540 | Classification_binary: 0.3143 | HitGrouping: 0.2846 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4310 | Regression_eta: 0.0050 | Regression_pt: 0.4337 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1025 | Classification_binary: 0.2962 | Regression_cos_phi: 0.1602 | Regression_energy: 0.2024 | Regression_eta: 0.1483 | Regression_pt: 0.2120 | Regression_sin_phi: 0.1136 | LR: 9.84e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.7395s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0168 | max_forward: 0.8335 | max_loss: 0.2425 | max_backward: 1.5586 | max_optimizer_step: 0.1744 | max_tensorboard_logging: 0.4781 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5904.9 | uss_MB: 4751.1 | child_rss_MB: 5672.4 | allocated_MB: 341.2 | reserved_MB: 24406.0 | inactive_split_MB: 273.9 +rank3 [2026-10-28 14:07:35,176] INFO : Step 4300/50000 rank3 | Train Loss: 2.5102 | Train Loss Unweighted: 1.4857 | Train Loss Components: Auxiliary: 0.4757 | Cardinality: 0.2309 | Classification: 0.2786 | Classification_binary: 0.3543 | HitGrouping: 0.2938 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4177 | Regression_eta: 0.0054 | Regression_pt: 0.4246 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1663 | Classification_binary: 0.3486 | Regression_cos_phi: 0.1752 | Regression_energy: 0.2065 | Regression_eta: 0.1302 | Regression_pt: 0.2138 | Regression_sin_phi: 0.1732 | LR: 9.84e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5986s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6520 | max_loss: 0.2453 | max_backward: 1.4084 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.5540 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5910.3 | uss_MB: 4757.7 | child_rss_MB: 5668.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 239.6 +rank3 [2026-10-28 14:11:23,917] INFO : Step 4400/50000 rank3 | Train Loss: 2.8258 | Train Loss Unweighted: 1.5691 | Train Loss Components: Auxiliary: 0.5223 | Cardinality: 0.3313 | Classification: 0.2877 | Classification_binary: 0.4120 | HitGrouping: 0.2902 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4255 | Regression_eta: 0.0063 | Regression_pt: 0.4316 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1857 | Classification_binary: 0.3865 | Regression_cos_phi: 0.2081 | Regression_energy: 0.2156 | Regression_eta: 0.1322 | Regression_pt: 0.2277 | Regression_sin_phi: 0.1890 | LR: 9.83e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.0425s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.7635 | max_loss: 0.2580 | max_backward: 1.7456 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.5221 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 5909.1 | uss_MB: 4757.0 | child_rss_MB: 5674.0 | allocated_MB: 344.6 | reserved_MB: 24406.0 | inactive_split_MB: 261.0 +rank3 [2026-10-28 14:15:17,053] INFO : Step 4500/50000 rank3 | Train Loss: 2.5622 | Train Loss Unweighted: 1.4664 | Train Loss Components: Auxiliary: 0.4942 | Cardinality: 0.3132 | Classification: 0.2685 | Classification_binary: 0.3695 | HitGrouping: 0.2876 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4070 | Regression_eta: 0.0048 | Regression_pt: 0.4117 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1655 | Classification_binary: 0.3459 | Regression_cos_phi: 0.1865 | Regression_energy: 0.1918 | Regression_eta: 0.1277 | Regression_pt: 0.2008 | Regression_sin_phi: 0.2077 | LR: 9.82e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.2142s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0462 | max_forward: 1.1330 | max_loss: 0.3526 | max_backward: 2.1744 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5571 | max_tensorboard_flush: 0.0016 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 5912.8 | uss_MB: 4736.1 | child_rss_MB: 5674.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 288.8 +rank3 [2026-10-28 14:19:09,439] INFO : Step 4600/50000 rank3 | Train Loss: 2.2516 | Train Loss Unweighted: 1.3548 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2151 | Classification: 0.2428 | Classification_binary: 0.2886 | HitGrouping: 0.2811 | Regression_cos_phi: 0.0019 | Regression_energy: 0.4059 | Regression_eta: 0.0042 | Regression_pt: 0.4088 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1285 | Classification_binary: 0.3100 | Regression_cos_phi: 0.1659 | Regression_energy: 0.1841 | Regression_eta: 0.1248 | Regression_pt: 0.1901 | Regression_sin_phi: 0.2042 | LR: 9.81e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6996s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0445 | max_forward: 0.8897 | max_loss: 0.2214 | max_backward: 1.8748 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5913.2 | uss_MB: 4741.7 | child_rss_MB: 5653.3 | allocated_MB: 341.6 | reserved_MB: 25392.0 | inactive_split_MB: 298.4 +rank3 [2026-10-28 14:23:11,223] INFO : Step 4700/50000 rank3 | Train Loss: 2.5991 | Train Loss Unweighted: 1.5142 | Train Loss Components: Auxiliary: 0.4914 | Cardinality: 0.2679 | Classification: 0.2997 | Classification_binary: 0.3514 | HitGrouping: 0.2867 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4228 | Regression_eta: 0.0061 | Regression_pt: 0.4290 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1878 | Classification_binary: 0.3066 | Regression_cos_phi: 0.2299 | Regression_energy: 0.2034 | Regression_eta: 0.1394 | Regression_pt: 0.2150 | Regression_sin_phi: 0.2113 | LR: 9.81e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8890s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7362 | max_loss: 0.2547 | max_backward: 1.8331 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5918.9 | uss_MB: 4764.8 | child_rss_MB: 5675.1 | allocated_MB: 343.1 | reserved_MB: 26384.0 | inactive_split_MB: 292.4 +rank3 [2026-10-28 14:27:13,058] INFO : Step 4800/50000 rank3 | Train Loss: 2.7077 | Train Loss Unweighted: 1.5200 | Train Loss Components: Auxiliary: 0.5118 | Cardinality: 0.3360 | Classification: 0.2606 | Classification_binary: 0.3918 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4264 | Regression_eta: 0.0053 | Regression_pt: 0.4305 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1610 | Classification_binary: 0.3947 | Regression_cos_phi: 0.2374 | Regression_energy: 0.2113 | Regression_eta: 0.1661 | Regression_pt: 0.2166 | Regression_sin_phi: 0.2505 | LR: 9.80e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.2497s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7154 | max_loss: 0.1552 | max_backward: 1.4398 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.8979 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0854 | Memory: rss_MB: 5917.3 | uss_MB: 4226.4 | child_rss_MB: 5678.8 | allocated_MB: 346.1 | reserved_MB: 26384.0 | inactive_split_MB: 284.0 +rank3 [2026-10-28 14:31:09,414] INFO : Step 4900/50000 rank3 | Train Loss: 2.3682 | Train Loss Unweighted: 1.4242 | Train Loss Components: Auxiliary: 0.4616 | Cardinality: 0.2176 | Classification: 0.2570 | Classification_binary: 0.3149 | HitGrouping: 0.2900 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4191 | Regression_eta: 0.0047 | Regression_pt: 0.4239 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1871 | Classification_binary: 0.3468 | Regression_cos_phi: 0.2970 | Regression_energy: 0.2036 | Regression_eta: 0.1693 | Regression_pt: 0.2078 | Regression_sin_phi: 0.2404 | LR: 9.79e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.4639s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0516 | max_forward: 0.6608 | max_loss: 0.1430 | max_backward: 1.2657 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5916.2 | uss_MB: 4765.4 | child_rss_MB: 5677.6 | allocated_MB: 343.8 | reserved_MB: 26384.0 | inactive_split_MB: 201.9 +rank3 [2026-10-28 14:35:19,045] INFO : Step 5000/50000 rank3 | Train Loss: 2.4060 | Train Loss Unweighted: 1.4130 | Train Loss Components: Auxiliary: 0.4756 | Cardinality: 0.2573 | Classification: 0.2748 | Classification_binary: 0.3198 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4020 | Regression_eta: 0.0043 | Regression_pt: 0.4074 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1689 | Classification_binary: 0.2946 | Regression_cos_phi: 0.1684 | Regression_energy: 0.1964 | Regression_eta: 0.0978 | Regression_pt: 0.2058 | Regression_sin_phi: 0.2316 | LR: 9.78e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8088s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6153 | max_loss: 0.1441 | max_backward: 1.2119 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.9741 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5914.8 | uss_MB: 4767.1 | child_rss_MB: 5669.5 | allocated_MB: 342.5 | reserved_MB: 27388.0 | inactive_split_MB: 218.4 +rank3 [2026-10-28 14:35:19,962] INFO : Running validation on rank3 +rank3 [2026-10-28 14:35:19,962] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 14:35:19,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 14:36:10,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:36:11,318] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 14:36:36,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 14:36:37,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 14:36:38,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 14:36:39,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 14:36:49,693] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 14:36:50,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:36:50,780] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 14:37:07,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 14:37:08,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 14:37:09,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 14:37:10,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 14:37:20,052] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 14:37:20,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:37:20,582] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 14:37:36,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 14:37:36,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 14:37:37,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 14:37:38,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 14:37:48,104] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 14:42:02,659] INFO : Step 5100/50000 rank3 | Train Loss: 2.7059 | Train Loss Unweighted: 1.5631 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2507 | Classification: 0.3068 | Classification_binary: 0.4110 | HitGrouping: 0.2884 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4148 | Regression_eta: 0.0060 | Regression_pt: 0.4186 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2119 | Classification_binary: 0.4320 | Regression_cos_phi: 0.2881 | Regression_energy: 0.2137 | Regression_eta: 0.1750 | Regression_pt: 0.2164 | Regression_sin_phi: 0.2331 | LR: 9.77e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4417s | Phase Times: max_pre_step_barrier: 0.1071 | max_batch_to_device: 0.0070 | max_forward: 0.7192 | max_loss: 0.2509 | max_backward: 1.6066 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.8859 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7032.2 | uss_MB: 5882.9 | child_rss_MB: 5662.0 | allocated_MB: 343.2 | reserved_MB: 23542.0 | inactive_split_MB: 236.2 +rank3 [2026-10-28 14:45:56,261] INFO : Step 5200/50000 rank3 | Train Loss: 2.4117 | Train Loss Unweighted: 1.4393 | Train Loss Components: Auxiliary: 0.4667 | Cardinality: 0.2215 | Classification: 0.2426 | Classification_binary: 0.3524 | HitGrouping: 0.2783 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4149 | Regression_eta: 0.0050 | Regression_pt: 0.4196 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1500 | Classification_binary: 0.3680 | Regression_cos_phi: 0.2514 | Regression_energy: 0.2030 | Regression_eta: 0.1747 | Regression_pt: 0.2159 | Regression_sin_phi: 0.2804 | LR: 9.76e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.9021s | Phase Times: max_pre_step_barrier: 0.1151 | max_batch_to_device: 0.0079 | max_forward: 0.8909 | max_loss: 0.1450 | max_backward: 1.3654 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.5791 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 6020.7 | uss_MB: 4875.4 | child_rss_MB: 5666.4 | allocated_MB: 346.1 | reserved_MB: 23690.0 | inactive_split_MB: 197.9 +rank3 [2026-10-28 14:50:03,850] INFO : Step 5300/50000 rank3 | Train Loss: 2.2417 | Train Loss Unweighted: 1.3238 | Train Loss Components: Auxiliary: 0.4353 | Cardinality: 0.2226 | Classification: 0.2387 | Classification_binary: 0.3027 | HitGrouping: 0.2808 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3844 | Regression_eta: 0.0041 | Regression_pt: 0.3895 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1360 | Classification_binary: 0.2910 | Regression_cos_phi: 0.1875 | Regression_energy: 0.1809 | Regression_eta: 0.0915 | Regression_pt: 0.1900 | Regression_sin_phi: 0.1540 | LR: 9.75e-04 | DataLoad Time: 0.0235s | Model Forward Time: 4.1969s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0155 | max_forward: 1.3549 | max_loss: 0.4272 | max_backward: 1.7517 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 6024.3 | uss_MB: 4878.1 | child_rss_MB: 5664.8 | allocated_MB: 341.7 | reserved_MB: 25496.0 | inactive_split_MB: 195.4 +rank3 [2026-10-28 14:54:26,616] INFO : Step 5400/50000 rank3 | Train Loss: 2.5857 | Train Loss Unweighted: 1.4820 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2715 | Classification: 0.2898 | Classification_binary: 0.3613 | HitGrouping: 0.2829 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4076 | Regression_eta: 0.0053 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1894 | Classification_binary: 0.3824 | Regression_cos_phi: 0.2227 | Regression_energy: 0.2013 | Regression_eta: 0.1204 | Regression_pt: 0.2094 | Regression_sin_phi: 0.2368 | LR: 9.74e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.8622s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7472 | max_loss: 0.1664 | max_backward: 1.6216 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4956 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6028.8 | uss_MB: 4880.2 | child_rss_MB: 5668.3 | allocated_MB: 345.8 | reserved_MB: 25496.0 | inactive_split_MB: 226.8 +rank3 [2026-10-28 14:58:29,775] INFO : Step 5500/50000 rank3 | Train Loss: 2.4383 | Train Loss Unweighted: 1.4361 | Train Loss Components: Auxiliary: 0.4636 | Cardinality: 0.2531 | Classification: 0.2547 | Classification_binary: 0.3846 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3900 | Regression_eta: 0.0045 | Regression_pt: 0.3974 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1300 | Classification_binary: 0.3570 | Regression_cos_phi: 0.1795 | Regression_energy: 0.1811 | Regression_eta: 0.0929 | Regression_pt: 0.1879 | Regression_sin_phi: 0.1563 | LR: 9.73e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3515s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7547 | max_loss: 0.2915 | max_backward: 1.8143 | max_optimizer_step: 0.2819 | max_tensorboard_logging: 0.5958 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6029.1 | uss_MB: 4882.8 | child_rss_MB: 5667.9 | allocated_MB: 342.5 | reserved_MB: 26496.0 | inactive_split_MB: 471.5 +rank3 [2026-10-28 15:02:37,794] INFO : Step 5600/50000 rank3 | Train Loss: 2.2622 | Train Loss Unweighted: 1.3177 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.1990 | Classification: 0.2257 | Classification_binary: 0.3268 | HitGrouping: 0.2800 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3758 | Regression_eta: 0.0051 | Regression_pt: 0.3799 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1701 | Classification_binary: 0.3238 | Regression_cos_phi: 0.1928 | Regression_energy: 0.1744 | Regression_eta: 0.1681 | Regression_pt: 0.1829 | Regression_sin_phi: 0.2150 | LR: 9.72e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.1700s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0347 | max_forward: 1.1910 | max_loss: 0.2555 | max_backward: 1.5431 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 6024.4 | uss_MB: 4885.9 | child_rss_MB: 5669.8 | allocated_MB: 340.8 | reserved_MB: 26496.0 | inactive_split_MB: 191.2 +rank3 [2026-10-28 15:06:24,089] INFO : Step 5700/50000 rank3 | Train Loss: 2.4588 | Train Loss Unweighted: 1.4026 | Train Loss Components: Auxiliary: 0.4666 | Cardinality: 0.2348 | Classification: 0.2768 | Classification_binary: 0.3365 | HitGrouping: 0.2916 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3864 | Regression_eta: 0.0055 | Regression_pt: 0.3923 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1738 | Classification_binary: 0.3191 | Regression_cos_phi: 0.2255 | Regression_energy: 0.1789 | Regression_eta: 0.1452 | Regression_pt: 0.1862 | Regression_sin_phi: 0.2623 | LR: 9.71e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5055s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6547 | max_loss: 0.1874 | max_backward: 1.2748 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6027.1 | uss_MB: 4889.2 | child_rss_MB: 5674.0 | allocated_MB: 344.4 | reserved_MB: 26496.0 | inactive_split_MB: 224.5 +rank3 [2026-10-28 15:10:14,851] INFO : Step 5800/50000 rank3 | Train Loss: 2.4947 | Train Loss Unweighted: 1.4616 | Train Loss Components: Auxiliary: 0.4752 | Cardinality: 0.2715 | Classification: 0.2623 | Classification_binary: 0.3729 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4032 | Regression_eta: 0.0047 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1565 | Classification_binary: 0.3762 | Regression_cos_phi: 0.1882 | Regression_energy: 0.1828 | Regression_eta: 0.1276 | Regression_pt: 0.1914 | Regression_sin_phi: 0.1862 | LR: 9.70e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7382s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.9768 | max_loss: 0.2512 | max_backward: 2.0573 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.6978 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6026.0 | uss_MB: 4048.8 | child_rss_MB: 5650.8 | allocated_MB: 343.4 | reserved_MB: 27494.0 | inactive_split_MB: 198.3 +rank3 [2026-10-28 15:14:12,312] INFO : Step 5900/50000 rank3 | Train Loss: 2.3800 | Train Loss Unweighted: 1.3838 | Train Loss Components: Auxiliary: 0.4549 | Cardinality: 0.2203 | Classification: 0.2438 | Classification_binary: 0.3435 | HitGrouping: 0.2757 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3900 | Regression_eta: 0.0050 | Regression_pt: 0.3965 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1326 | Classification_binary: 0.3573 | Regression_cos_phi: 0.2012 | Regression_energy: 0.1727 | Regression_eta: 0.1081 | Regression_pt: 0.1847 | Regression_sin_phi: 0.2227 | LR: 9.69e-04 | DataLoad Time: 0.0132s | Model Forward Time: 4.7252s | Phase Times: max_pre_step_barrier: 0.0158 | max_batch_to_device: 0.0069 | max_forward: 1.0355 | max_loss: 0.2311 | max_backward: 2.2749 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5131 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 6029.8 | uss_MB: 3868.6 | child_rss_MB: 5658.5 | allocated_MB: 342.1 | reserved_MB: 27494.0 | inactive_split_MB: 281.2 +rank3 [2026-10-28 15:18:13,895] INFO : Step 6000/50000 rank3 | Train Loss: 2.1372 | Train Loss Unweighted: 1.3015 | Train Loss Components: Auxiliary: 0.4179 | Cardinality: 0.1916 | Classification: 0.2091 | Classification_binary: 0.2848 | HitGrouping: 0.2746 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3966 | Regression_eta: 0.0041 | Regression_pt: 0.4029 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0998 | Classification_binary: 0.2805 | Regression_cos_phi: 0.1326 | Regression_energy: 0.1787 | Regression_eta: 0.0950 | Regression_pt: 0.1850 | Regression_sin_phi: 0.1222 | LR: 9.68e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6791s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6457 | max_loss: 0.2157 | max_backward: 1.4984 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.5590 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6033.8 | uss_MB: 4891.5 | child_rss_MB: 5658.2 | allocated_MB: 343.1 | reserved_MB: 27494.0 | inactive_split_MB: 260.7 +rank3 [2026-10-28 15:18:14,994] INFO : Running validation on rank3 +rank3 [2026-10-28 15:18:14,995] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 15:18:14,995] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 15:19:17,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:19:17,849] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 15:19:43,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 15:19:44,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 15:19:45,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 15:19:46,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 15:19:56,973] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 15:19:57,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:19:58,135] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 15:20:14,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 15:20:15,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 15:20:16,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 15:20:17,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 15:20:27,151] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 15:20:27,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:20:28,077] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 15:20:43,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 15:20:43,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 15:20:44,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 15:20:45,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 15:20:55,513] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 15:25:29,841] INFO : Step 6100/50000 rank3 | Train Loss: 2.4040 | Train Loss Unweighted: 1.3907 | Train Loss Components: Auxiliary: 0.4661 | Cardinality: 0.2610 | Classification: 0.2427 | Classification_binary: 0.3401 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3968 | Regression_eta: 0.0047 | Regression_pt: 0.4016 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1704 | Classification_binary: 0.3774 | Regression_cos_phi: 0.1938 | Regression_energy: 0.2139 | Regression_eta: 0.1240 | Regression_pt: 0.2241 | Regression_sin_phi: 0.1768 | LR: 9.67e-04 | DataLoad Time: 0.0203s | Model Forward Time: 6.3872s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0073 | max_forward: 1.0567 | max_loss: 0.2452 | max_backward: 2.4106 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.9836 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 7214.6 | uss_MB: 5292.3 | child_rss_MB: 5654.8 | allocated_MB: 344.4 | reserved_MB: 23648.0 | inactive_split_MB: 194.8 +rank3 [2026-10-28 15:29:38,612] INFO : Step 6200/50000 rank3 | Train Loss: 2.4942 | Train Loss Unweighted: 1.4287 | Train Loss Components: Auxiliary: 0.4744 | Cardinality: 0.2537 | Classification: 0.2580 | Classification_binary: 0.3817 | HitGrouping: 0.2748 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3867 | Regression_eta: 0.0050 | Regression_pt: 0.3920 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1794 | Classification_binary: 0.4170 | Regression_cos_phi: 0.2564 | Regression_energy: 0.2069 | Regression_eta: 0.1520 | Regression_pt: 0.2167 | Regression_sin_phi: 0.2569 | LR: 9.66e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0643s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7838 | max_loss: 0.1532 | max_backward: 1.6314 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6093.8 | uss_MB: 4537.3 | child_rss_MB: 5656.7 | allocated_MB: 343.8 | reserved_MB: 23830.0 | inactive_split_MB: 215.7 +rank3 [2026-10-28 15:34:14,250] INFO : Step 6300/50000 rank3 | Train Loss: 2.2204 | Train Loss Unweighted: 1.2763 | Train Loss Components: Auxiliary: 0.4285 | Cardinality: 0.2506 | Classification: 0.2258 | Classification_binary: 0.2983 | HitGrouping: 0.2710 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3704 | Regression_eta: 0.0045 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1329 | Classification_binary: 0.3070 | Regression_cos_phi: 0.1859 | Regression_energy: 0.1764 | Regression_eta: 0.1282 | Regression_pt: 0.1846 | Regression_sin_phi: 0.1714 | LR: 9.65e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6712s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.5854 | max_loss: 0.1923 | max_backward: 1.0848 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.8586 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6096.2 | uss_MB: 4957.2 | child_rss_MB: 5651.4 | allocated_MB: 339.7 | reserved_MB: 24826.0 | inactive_split_MB: 251.6 +rank3 [2026-10-28 15:38:39,607] INFO : Step 6400/50000 rank3 | Train Loss: 2.2882 | Train Loss Unweighted: 1.3014 | Train Loss Components: Auxiliary: 0.4348 | Cardinality: 0.2271 | Classification: 0.2431 | Classification_binary: 0.3058 | HitGrouping: 0.2702 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3679 | Regression_eta: 0.0046 | Regression_pt: 0.3751 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1133 | Classification_binary: 0.2644 | Regression_cos_phi: 0.2089 | Regression_energy: 0.1731 | Regression_eta: 0.1071 | Regression_pt: 0.1821 | Regression_sin_phi: 0.1868 | LR: 9.64e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.8574s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7489 | max_loss: 0.3608 | max_backward: 2.5327 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6095.6 | uss_MB: 4594.4 | child_rss_MB: 5649.8 | allocated_MB: 345.8 | reserved_MB: 24826.0 | inactive_split_MB: 253.0 +rank3 [2026-10-28 15:42:45,601] INFO : Step 6500/50000 rank3 | Train Loss: 2.4626 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.3022 | Classification: 0.2393 | Classification_binary: 0.3805 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3783 | Regression_eta: 0.0049 | Regression_pt: 0.3831 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1620 | Classification_binary: 0.3960 | Regression_cos_phi: 0.2147 | Regression_energy: 0.1750 | Regression_eta: 0.1410 | Regression_pt: 0.1832 | Regression_sin_phi: 0.1805 | LR: 9.63e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.1909s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7600 | max_loss: 0.1555 | max_backward: 1.3910 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6094.6 | uss_MB: 4960.0 | child_rss_MB: 5656.1 | allocated_MB: 344.4 | reserved_MB: 24826.0 | inactive_split_MB: 269.0 +rank3 [2026-10-28 15:46:47,732] INFO : Step 6600/50000 rank3 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3617 | Train Loss Components: Auxiliary: 0.4493 | Cardinality: 0.2405 | Classification: 0.2289 | Classification_binary: 0.3388 | HitGrouping: 0.2650 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3914 | Regression_eta: 0.0045 | Regression_pt: 0.3935 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1522 | Classification_binary: 0.3348 | Regression_cos_phi: 0.1804 | Regression_energy: 0.1871 | Regression_eta: 0.1458 | Regression_pt: 0.1939 | Regression_sin_phi: 0.1669 | LR: 9.62e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7765s | Phase Times: max_pre_step_barrier: 0.1106 | max_batch_to_device: 0.0061 | max_forward: 0.7587 | max_loss: 0.2869 | max_backward: 2.2641 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1321 | Memory: rss_MB: 6092.2 | uss_MB: 4956.2 | child_rss_MB: 5651.6 | allocated_MB: 343.8 | reserved_MB: 25846.0 | inactive_split_MB: 267.9 +rank3 [2026-10-28 15:51:24,435] INFO : Step 6700/50000 rank3 | Train Loss: 2.1225 | Train Loss Unweighted: 1.2385 | Train Loss Components: Auxiliary: 0.4092 | Cardinality: 0.2100 | Classification: 0.2079 | Classification_binary: 0.2916 | HitGrouping: 0.2751 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3623 | Regression_eta: 0.0041 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1045 | Classification_binary: 0.2307 | Regression_cos_phi: 0.1023 | Regression_energy: 0.1920 | Regression_eta: 0.0925 | Regression_pt: 0.2012 | Regression_sin_phi: 0.1300 | LR: 9.61e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.9901s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.6452 | max_loss: 0.1453 | max_backward: 1.2984 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.0269 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6098.1 | uss_MB: 4965.1 | child_rss_MB: 5657.8 | allocated_MB: 345.0 | reserved_MB: 25846.0 | inactive_split_MB: 258.8 +rank3 [2026-10-28 15:55:32,973] INFO : Step 6800/50000 rank3 | Train Loss: 2.4601 | Train Loss Unweighted: 1.3772 | Train Loss Components: Auxiliary: 0.4589 | Cardinality: 0.2427 | Classification: 0.2631 | Classification_binary: 0.3490 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3747 | Regression_eta: 0.0054 | Regression_pt: 0.3793 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1364 | Classification_binary: 0.3282 | Regression_cos_phi: 0.2709 | Regression_energy: 0.1811 | Regression_eta: 0.1274 | Regression_pt: 0.1868 | Regression_sin_phi: 0.3346 | LR: 9.60e-04 | DataLoad Time: 0.0241s | Model Forward Time: 5.6150s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0662 | max_forward: 1.6558 | max_loss: 0.3855 | max_backward: 2.1309 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 1.0709 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6092.6 | uss_MB: 4961.3 | child_rss_MB: 5647.9 | allocated_MB: 345.8 | reserved_MB: 26622.0 | inactive_split_MB: 194.2 +rank3 [2026-10-28 15:59:35,210] INFO : Step 6900/50000 rank3 | Train Loss: 2.3793 | Train Loss Unweighted: 1.3337 | Train Loss Components: Auxiliary: 0.4495 | Cardinality: 0.2757 | Classification: 0.2300 | Classification_binary: 0.3711 | HitGrouping: 0.2647 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3605 | Regression_eta: 0.0045 | Regression_pt: 0.3628 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1504 | Classification_binary: 0.4060 | Regression_cos_phi: 0.3182 | Regression_energy: 0.1961 | Regression_eta: 0.1503 | Regression_pt: 0.2053 | Regression_sin_phi: 0.2368 | LR: 9.58e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2957s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1120 | max_forward: 0.8190 | max_loss: 0.1496 | max_backward: 1.4356 | max_optimizer_step: 0.2847 | max_tensorboard_logging: 0.8440 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6096.9 | uss_MB: 4962.9 | child_rss_MB: 5656.8 | allocated_MB: 345.4 | reserved_MB: 26622.0 | inactive_split_MB: 300.5 +rank3 [2026-10-28 16:03:56,962] INFO : Step 7000/50000 rank3 | Train Loss: 2.2410 | Train Loss Unweighted: 1.2707 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2746 | Classification: 0.1970 | Classification_binary: 0.3002 | HitGrouping: 0.2655 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3795 | Regression_eta: 0.0045 | Regression_pt: 0.3852 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1000 | Classification_binary: 0.2675 | Regression_cos_phi: 0.1435 | Regression_energy: 0.1946 | Regression_eta: 0.1181 | Regression_pt: 0.2064 | Regression_sin_phi: 0.1536 | LR: 9.57e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.3866s | Phase Times: max_pre_step_barrier: 0.0229 | max_batch_to_device: 0.0154 | max_forward: 1.0393 | max_loss: 0.2251 | max_backward: 2.0451 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.9425 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6097.4 | uss_MB: 4965.6 | child_rss_MB: 5656.9 | allocated_MB: 345.5 | reserved_MB: 26622.0 | inactive_split_MB: 296.5 +rank3 [2026-10-28 16:03:58,348] INFO : Running validation on rank3 +rank3 [2026-10-28 16:03:58,349] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 16:03:58,349] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 16:04:51,887] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:04:52,382] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 16:05:18,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 16:05:19,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 16:05:20,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 16:05:21,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 16:05:31,564] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 16:05:32,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:05:32,551] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 16:05:48,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 16:05:49,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 16:05:50,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 16:05:51,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 16:06:01,524] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 16:06:02,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:06:02,706] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 16:06:18,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 16:06:18,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 16:06:19,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 16:06:20,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 16:06:30,286] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 16:10:49,042] INFO : Step 7100/50000 rank3 | Train Loss: 2.1086 | Train Loss Unweighted: 1.2525 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.1821 | Classification: 0.2186 | Classification_binary: 0.2903 | HitGrouping: 0.2613 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3655 | Regression_eta: 0.0041 | Regression_pt: 0.3696 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1029 | Classification_binary: 0.2523 | Regression_cos_phi: 0.1769 | Regression_energy: 0.1589 | Regression_eta: 0.0782 | Regression_pt: 0.1634 | Regression_sin_phi: 0.1634 | LR: 9.56e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.9387s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6376 | max_loss: 0.1424 | max_backward: 1.3436 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8454 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 7191.1 | uss_MB: 6057.4 | child_rss_MB: 5686.3 | allocated_MB: 345.6 | reserved_MB: 22618.0 | inactive_split_MB: 206.0 +rank3 [2026-10-28 16:15:05,704] INFO : Step 7200/50000 rank3 | Train Loss: 2.4589 | Train Loss Unweighted: 1.3873 | Train Loss Components: Auxiliary: 0.4658 | Cardinality: 0.2721 | Classification: 0.2648 | Classification_binary: 0.3806 | HitGrouping: 0.2633 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3642 | Regression_eta: 0.0051 | Regression_pt: 0.3675 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1500 | Classification_binary: 0.3661 | Regression_cos_phi: 0.2241 | Regression_energy: 0.1587 | Regression_eta: 0.1452 | Regression_pt: 0.1682 | Regression_sin_phi: 0.2209 | LR: 9.55e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3054s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 1.0191 | max_loss: 0.1674 | max_backward: 1.7813 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6183.3 | uss_MB: 5050.5 | child_rss_MB: 5662.0 | allocated_MB: 343.5 | reserved_MB: 22618.0 | inactive_split_MB: 315.3 +rank3 [2026-10-28 16:19:17,627] INFO : Step 7300/50000 rank3 | Train Loss: 2.1617 | Train Loss Unweighted: 1.2404 | Train Loss Components: Auxiliary: 0.4141 | Cardinality: 0.2258 | Classification: 0.1979 | Classification_binary: 0.3107 | HitGrouping: 0.2671 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3613 | Regression_eta: 0.0041 | Regression_pt: 0.3621 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1226 | Classification_binary: 0.3422 | Regression_cos_phi: 0.1947 | Regression_energy: 0.1632 | Regression_eta: 0.1053 | Regression_pt: 0.1707 | Regression_sin_phi: 0.2067 | LR: 9.53e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.7556s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0084 | max_forward: 0.7227 | max_loss: 0.1491 | max_backward: 1.4232 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.5902 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6180.9 | uss_MB: 5051.2 | child_rss_MB: 5696.9 | allocated_MB: 345.0 | reserved_MB: 23608.0 | inactive_split_MB: 295.0 +rank3 [2026-10-28 16:23:25,991] INFO : Step 7400/50000 rank3 | Train Loss: 2.0741 | Train Loss Unweighted: 1.2014 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2287 | Classification: 0.2035 | Classification_binary: 0.2781 | HitGrouping: 0.2646 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3549 | Regression_eta: 0.0040 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1301 | Classification_binary: 0.3026 | Regression_cos_phi: 0.1487 | Regression_energy: 0.1770 | Regression_eta: 0.0998 | Regression_pt: 0.1876 | Regression_sin_phi: 0.1425 | LR: 9.52e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.6545s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6129 | max_loss: 0.2127 | max_backward: 1.2944 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6450 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6180.9 | uss_MB: 5049.5 | child_rss_MB: 5695.6 | allocated_MB: 343.8 | reserved_MB: 23608.0 | inactive_split_MB: 229.5 +rank3 [2026-10-28 16:27:18,559] INFO : Step 7500/50000 rank3 | Train Loss: 2.4640 | Train Loss Unweighted: 1.3686 | Train Loss Components: Auxiliary: 0.4637 | Cardinality: 0.2640 | Classification: 0.2570 | Classification_binary: 0.3594 | HitGrouping: 0.2687 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3684 | Regression_eta: 0.0054 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1431 | Classification_binary: 0.3524 | Regression_cos_phi: 0.2069 | Regression_energy: 0.1690 | Regression_eta: 0.1762 | Regression_pt: 0.1773 | Regression_sin_phi: 0.2261 | LR: 9.51e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1736s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0887 | max_forward: 0.7717 | max_loss: 0.1508 | max_backward: 1.3590 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.9634 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6179.4 | uss_MB: 5053.3 | child_rss_MB: 5684.8 | allocated_MB: 348.6 | reserved_MB: 23608.0 | inactive_split_MB: 283.7 +rank3 [2026-10-28 16:31:22,304] INFO : Step 7600/50000 rank3 | Train Loss: 2.3745 | Train Loss Unweighted: 1.3447 | Train Loss Components: Auxiliary: 0.4509 | Cardinality: 0.2616 | Classification: 0.2519 | Classification_binary: 0.3743 | HitGrouping: 0.2627 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3532 | Regression_eta: 0.0050 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1636 | Classification_binary: 0.3736 | Regression_cos_phi: 0.1948 | Regression_energy: 0.1530 | Regression_eta: 0.1679 | Regression_pt: 0.1594 | Regression_sin_phi: 0.2145 | LR: 9.50e-04 | DataLoad Time: 0.0167s | Model Forward Time: 16.0028s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.7345 | max_loss: 0.1617 | max_backward: 1.4897 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 12.6867 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6179.5 | uss_MB: 5051.2 | child_rss_MB: 5695.8 | allocated_MB: 345.3 | reserved_MB: 23608.0 | inactive_split_MB: 284.8 +rank3 [2026-10-28 16:35:23,036] INFO : Step 7700/50000 rank3 | Train Loss: 2.0023 | Train Loss Unweighted: 1.1475 | Train Loss Components: Auxiliary: 0.3866 | Cardinality: 0.2005 | Classification: 0.1968 | Classification_binary: 0.2784 | HitGrouping: 0.2562 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3315 | Regression_eta: 0.0040 | Regression_pt: 0.3329 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1125 | Classification_binary: 0.2443 | Regression_cos_phi: 0.1309 | Regression_energy: 0.1526 | Regression_eta: 0.0944 | Regression_pt: 0.1582 | Regression_sin_phi: 0.1568 | LR: 9.48e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.0201s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7891 | max_loss: 0.1927 | max_backward: 1.4291 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6182.9 | uss_MB: 5055.0 | child_rss_MB: 5702.0 | allocated_MB: 343.9 | reserved_MB: 23608.0 | inactive_split_MB: 279.7 +rank3 [2026-10-28 16:39:23,043] INFO : Step 7800/50000 rank3 | Train Loss: 2.0473 | Train Loss Unweighted: 1.2037 | Train Loss Components: Auxiliary: 0.4013 | Cardinality: 0.1891 | Classification: 0.2102 | Classification_binary: 0.2760 | HitGrouping: 0.2604 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3526 | Regression_eta: 0.0038 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1437 | Classification_binary: 0.2603 | Regression_cos_phi: 0.1610 | Regression_energy: 0.1651 | Regression_eta: 0.0956 | Regression_pt: 0.1720 | Regression_sin_phi: 0.1442 | LR: 9.47e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4348s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.7326 | max_loss: 0.2464 | max_backward: 1.8124 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9395 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6182.4 | uss_MB: 5059.4 | child_rss_MB: 5675.6 | allocated_MB: 343.5 | reserved_MB: 24612.0 | inactive_split_MB: 224.8 +rank3 [2026-10-28 16:43:21,637] INFO : Step 7900/50000 rank3 | Train Loss: 2.3872 | Train Loss Unweighted: 1.3279 | Train Loss Components: Auxiliary: 0.4525 | Cardinality: 0.2722 | Classification: 0.2401 | Classification_binary: 0.3580 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3573 | Regression_eta: 0.0051 | Regression_pt: 0.3627 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1221 | Classification_binary: 0.3693 | Regression_cos_phi: 0.1959 | Regression_energy: 0.1748 | Regression_eta: 0.1510 | Regression_pt: 0.1841 | Regression_sin_phi: 0.2191 | LR: 9.46e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.7845s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7032 | max_loss: 0.2052 | max_backward: 1.5183 | max_optimizer_step: 0.1922 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6185.2 | uss_MB: 5060.7 | child_rss_MB: 5695.7 | allocated_MB: 347.1 | reserved_MB: 24612.0 | inactive_split_MB: 279.0 +rank3 [2026-10-28 16:47:21,752] INFO : Step 8000/50000 rank3 | Train Loss: 2.1303 | Train Loss Unweighted: 1.2019 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2451 | Classification: 0.2031 | Classification_binary: 0.3087 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3382 | Regression_eta: 0.0041 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1026 | Classification_binary: 0.3094 | Regression_cos_phi: 0.1149 | Regression_energy: 0.1674 | Regression_eta: 0.1100 | Regression_pt: 0.1771 | Regression_sin_phi: 0.1402 | LR: 9.44e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.7969s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0554 | max_forward: 0.7608 | max_loss: 0.1617 | max_backward: 1.3810 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 6186.5 | uss_MB: 4730.8 | child_rss_MB: 5692.5 | allocated_MB: 344.2 | reserved_MB: 24612.0 | inactive_split_MB: 224.1 +rank3 [2026-10-28 16:47:22,744] INFO : Running validation on rank3 +rank3 [2026-10-28 16:47:22,744] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 16:47:22,745] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 16:48:13,809] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:48:14,391] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 16:48:40,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 16:48:41,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 16:48:42,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 16:48:43,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 16:48:53,479] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 16:48:53,945] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:48:54,468] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 16:49:10,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 16:49:11,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 16:49:12,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 16:49:13,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 16:49:23,603] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 16:49:24,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:49:24,964] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 16:49:40,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 16:49:40,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 16:49:41,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 16:49:42,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 16:49:52,091] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 16:54:09,126] INFO : Step 8100/50000 rank3 | Train Loss: 2.0320 | Train Loss Unweighted: 1.1744 | Train Loss Components: Auxiliary: 0.3893 | Cardinality: 0.1986 | Classification: 0.1938 | Classification_binary: 0.2691 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3510 | Regression_eta: 0.0039 | Regression_pt: 0.3525 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1138 | Classification_binary: 0.2685 | Regression_cos_phi: 0.1675 | Regression_energy: 0.1523 | Regression_eta: 0.1026 | Regression_pt: 0.1596 | Regression_sin_phi: 0.1495 | LR: 9.43e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0435s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6251 | max_loss: 0.2465 | max_backward: 1.3564 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8507 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7244.9 | uss_MB: 6118.4 | child_rss_MB: 5660.8 | allocated_MB: 342.9 | reserved_MB: 23510.0 | inactive_split_MB: 258.1 +rank3 [2026-10-28 16:58:00,709] INFO : Step 8200/50000 rank3 | Train Loss: 2.1917 | Train Loss Unweighted: 1.2329 | Train Loss Components: Auxiliary: 0.4200 | Cardinality: 0.2405 | Classification: 0.2359 | Classification_binary: 0.3328 | HitGrouping: 0.2582 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3251 | Regression_eta: 0.0043 | Regression_pt: 0.3305 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1390 | Classification_binary: 0.3527 | Regression_cos_phi: 0.1653 | Regression_energy: 0.1403 | Regression_eta: 0.1154 | Regression_pt: 0.1482 | Regression_sin_phi: 0.1484 | LR: 9.42e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.2791s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0064 | max_forward: 0.7679 | max_loss: 0.2484 | max_backward: 1.6620 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.8049 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1289 | Memory: rss_MB: 6232.6 | uss_MB: 5106.1 | child_rss_MB: 5673.2 | allocated_MB: 343.2 | reserved_MB: 23510.0 | inactive_split_MB: 292.4 +rank3 [2026-10-28 17:02:03,650] INFO : Step 8300/50000 rank3 | Train Loss: 2.1924 | Train Loss Unweighted: 1.2399 | Train Loss Components: Auxiliary: 0.4122 | Cardinality: 0.2039 | Classification: 0.2083 | Classification_binary: 0.3661 | HitGrouping: 0.2599 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3243 | Regression_eta: 0.0043 | Regression_pt: 0.3323 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1090 | Classification_binary: 0.3715 | Regression_cos_phi: 0.1847 | Regression_energy: 0.1577 | Regression_eta: 0.1177 | Regression_pt: 0.1689 | Regression_sin_phi: 0.1832 | LR: 9.40e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.8410s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0061 | max_forward: 0.9407 | max_loss: 0.2296 | max_backward: 1.9622 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6228.1 | uss_MB: 5055.3 | child_rss_MB: 5669.2 | allocated_MB: 344.6 | reserved_MB: 24580.0 | inactive_split_MB: 267.5 +rank3 [2026-10-28 17:06:07,748] INFO : Step 8400/50000 rank3 | Train Loss: 1.9542 | Train Loss Unweighted: 1.1189 | Train Loss Components: Auxiliary: 0.3756 | Cardinality: 0.1979 | Classification: 0.1926 | Classification_binary: 0.2783 | HitGrouping: 0.2585 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3198 | Regression_eta: 0.0035 | Regression_pt: 0.3209 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1121 | Classification_binary: 0.2695 | Regression_cos_phi: 0.2167 | Regression_energy: 0.1450 | Regression_eta: 0.0807 | Regression_pt: 0.1511 | Regression_sin_phi: 0.1752 | LR: 9.39e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2383s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6939 | max_loss: 0.1474 | max_backward: 1.5357 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 0.9892 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 6216.9 | uss_MB: 5103.3 | child_rss_MB: 5675.6 | allocated_MB: 341.1 | reserved_MB: 24580.0 | inactive_split_MB: 178.9 +rank3 [2026-10-28 17:10:17,598] INFO : Step 8500/50000 rank3 | Train Loss: 2.0642 | Train Loss Unweighted: 1.1724 | Train Loss Components: Auxiliary: 0.3964 | Cardinality: 0.2023 | Classification: 0.2152 | Classification_binary: 0.2897 | HitGrouping: 0.2607 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3289 | Regression_eta: 0.0041 | Regression_pt: 0.3302 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1396 | Classification_binary: 0.2629 | Regression_cos_phi: 0.2261 | Regression_energy: 0.1498 | Regression_eta: 0.1317 | Regression_pt: 0.1565 | Regression_sin_phi: 0.1981 | LR: 9.37e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.7400s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0069 | max_forward: 0.6424 | max_loss: 0.1972 | max_backward: 1.2711 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.7327 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6218.9 | uss_MB: 5108.3 | child_rss_MB: 5667.6 | allocated_MB: 344.4 | reserved_MB: 25586.0 | inactive_split_MB: 317.0 +rank3 [2026-10-28 17:14:24,635] INFO : Step 8600/50000 rank3 | Train Loss: 2.2820 | Train Loss Unweighted: 1.2960 | Train Loss Components: Auxiliary: 0.4378 | Cardinality: 0.2546 | Classification: 0.2294 | Classification_binary: 0.3478 | HitGrouping: 0.2584 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3521 | Regression_eta: 0.0046 | Regression_pt: 0.3576 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1343 | Classification_binary: 0.3901 | Regression_cos_phi: 0.2035 | Regression_energy: 0.1849 | Regression_eta: 0.1445 | Regression_pt: 0.1958 | Regression_sin_phi: 0.2064 | LR: 9.36e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.9614s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0069 | max_forward: 0.7396 | max_loss: 0.1527 | max_backward: 1.6088 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5007 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6218.9 | uss_MB: 5112.5 | child_rss_MB: 5664.7 | allocated_MB: 344.8 | reserved_MB: 25586.0 | inactive_split_MB: 283.2 +rank3 [2026-10-28 17:18:21,030] INFO : Step 8700/50000 rank3 | Train Loss: 2.0581 | Train Loss Unweighted: 1.1705 | Train Loss Components: Auxiliary: 0.4038 | Cardinality: 0.2463 | Classification: 0.2066 | Classification_binary: 0.3017 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3267 | Regression_eta: 0.0035 | Regression_pt: 0.3282 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1272 | Classification_binary: 0.2921 | Regression_cos_phi: 0.1454 | Regression_energy: 0.1575 | Regression_eta: 0.0961 | Regression_pt: 0.1634 | Regression_sin_phi: 0.1730 | LR: 9.34e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.7142s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6873 | max_loss: 0.2444 | max_backward: 1.2985 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6214.6 | uss_MB: 5113.2 | child_rss_MB: 5650.1 | allocated_MB: 343.7 | reserved_MB: 25586.0 | inactive_split_MB: 260.4 +rank3 [2026-10-28 17:22:21,713] INFO : Step 8800/50000 rank3 | Train Loss: 1.8415 | Train Loss Unweighted: 1.0591 | Train Loss Components: Auxiliary: 0.3597 | Cardinality: 0.1811 | Classification: 0.1734 | Classification_binary: 0.2490 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3144 | Regression_eta: 0.0034 | Regression_pt: 0.3153 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1160 | Classification_binary: 0.2580 | Regression_cos_phi: 0.2015 | Regression_energy: 0.1353 | Regression_eta: 0.0850 | Regression_pt: 0.1384 | Regression_sin_phi: 0.1944 | LR: 9.33e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5721s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6737 | max_loss: 0.2385 | max_backward: 1.3073 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.5819 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6215.6 | uss_MB: 5114.6 | child_rss_MB: 5649.0 | allocated_MB: 344.1 | reserved_MB: 25586.0 | inactive_split_MB: 205.9 +rank3 [2026-10-28 17:26:43,764] INFO : Step 8900/50000 rank3 | Train Loss: 2.1706 | Train Loss Unweighted: 1.1768 | Train Loss Components: Auxiliary: 0.4167 | Cardinality: 0.3000 | Classification: 0.2064 | Classification_binary: 0.3037 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3284 | Regression_eta: 0.0039 | Regression_pt: 0.3304 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1183 | Classification_binary: 0.2968 | Regression_cos_phi: 0.1437 | Regression_energy: 0.1459 | Regression_eta: 0.0964 | Regression_pt: 0.1520 | Regression_sin_phi: 0.1736 | LR: 9.31e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.8177s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0073 | max_forward: 0.9973 | max_loss: 0.4040 | max_backward: 3.4126 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6194.6 | uss_MB: 5119.9 | child_rss_MB: 5654.4 | allocated_MB: 347.1 | reserved_MB: 25586.0 | inactive_split_MB: 275.1 +rank3 [2026-10-28 17:30:46,803] INFO : Step 9000/50000 rank3 | Train Loss: 2.1357 | Train Loss Unweighted: 1.1883 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.2351 | Classification: 0.2003 | Classification_binary: 0.3467 | HitGrouping: 0.2532 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3143 | Regression_eta: 0.0041 | Regression_pt: 0.3186 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1316 | Classification_binary: 0.3615 | Regression_cos_phi: 0.1662 | Regression_energy: 0.1508 | Regression_eta: 0.1044 | Regression_pt: 0.1577 | Regression_sin_phi: 0.1763 | LR: 9.30e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.7200s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.7873 | max_loss: 0.2178 | max_backward: 1.5407 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.3125 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 6192.6 | uss_MB: 5121.2 | child_rss_MB: 5660.4 | allocated_MB: 344.6 | reserved_MB: 25586.0 | inactive_split_MB: 289.9 +rank3 [2026-10-28 17:30:47,949] INFO : Running validation on rank3 +rank3 [2026-10-28 17:30:47,949] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 17:30:47,949] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 17:31:40,963] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:31:41,477] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 17:32:08,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 17:32:09,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 17:32:11,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 17:32:11,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 17:32:21,965] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 17:32:22,656] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:32:23,202] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 17:32:39,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 17:32:40,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 17:32:41,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 17:32:42,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 17:32:52,254] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 17:32:52,551] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:32:53,101] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 17:33:08,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 17:33:09,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 17:33:09,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 17:33:10,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 17:33:20,565] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 17:37:41,207] INFO : Step 9100/50000 rank3 | Train Loss: 1.8448 | Train Loss Unweighted: 1.0695 | Train Loss Components: Auxiliary: 0.3538 | Cardinality: 0.1722 | Classification: 0.1878 | Classification_binary: 0.2661 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3012 | Regression_eta: 0.0033 | Regression_pt: 0.3075 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1086 | Classification_binary: 0.2672 | Regression_cos_phi: 0.1163 | Regression_energy: 0.1628 | Regression_eta: 0.0723 | Regression_pt: 0.1733 | Regression_sin_phi: 0.1224 | LR: 9.28e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.4379s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0153 | max_forward: 1.2776 | max_loss: 0.3881 | max_backward: 1.3802 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9203 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7188.5 | uss_MB: 6128.3 | child_rss_MB: 5636.7 | allocated_MB: 345.3 | reserved_MB: 22582.0 | inactive_split_MB: 275.2 +rank3 [2026-10-28 17:41:50,228] INFO : Step 9200/50000 rank3 | Train Loss: 1.9883 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3829 | Cardinality: 0.2086 | Classification: 0.1812 | Classification_binary: 0.2802 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3207 | Regression_eta: 0.0035 | Regression_pt: 0.3257 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1184 | Classification_binary: 0.2889 | Regression_cos_phi: 0.2489 | Regression_energy: 0.1685 | Regression_eta: 0.0928 | Regression_pt: 0.1763 | Regression_sin_phi: 0.2106 | LR: 9.27e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5141s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0073 | max_forward: 0.6839 | max_loss: 0.2414 | max_backward: 1.6336 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 1.0636 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6214.5 | uss_MB: 5161.4 | child_rss_MB: 5636.3 | allocated_MB: 345.6 | reserved_MB: 23224.0 | inactive_split_MB: 190.1 +rank3 [2026-10-28 17:45:52,430] INFO : Step 9300/50000 rank3 | Train Loss: 2.0749 | Train Loss Unweighted: 1.1591 | Train Loss Components: Auxiliary: 0.4049 | Cardinality: 0.2573 | Classification: 0.2065 | Classification_binary: 0.3180 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3111 | Regression_eta: 0.0036 | Regression_pt: 0.3160 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1386 | Classification_binary: 0.3655 | Regression_cos_phi: 0.1844 | Regression_energy: 0.1543 | Regression_eta: 0.1134 | Regression_pt: 0.1627 | Regression_sin_phi: 0.1985 | LR: 9.25e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2183s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8329 | max_loss: 0.1499 | max_backward: 1.4059 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.8766 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6214.5 | uss_MB: 5164.3 | child_rss_MB: 5654.0 | allocated_MB: 344.8 | reserved_MB: 24586.0 | inactive_split_MB: 287.2 +rank3 [2026-10-28 17:50:16,942] INFO : Step 9400/50000 rank3 | Train Loss: 1.8467 | Train Loss Unweighted: 1.0417 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1520 | Classification: 0.1837 | Classification_binary: 0.2977 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2742 | Regression_eta: 0.0040 | Regression_pt: 0.2784 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1241 | Classification_binary: 0.2974 | Regression_cos_phi: 0.1662 | Regression_energy: 0.1221 | Regression_eta: 0.1008 | Regression_pt: 0.1301 | Regression_sin_phi: 0.1583 | LR: 9.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.6034s | Phase Times: max_pre_step_barrier: 0.0180 | max_batch_to_device: 0.0062 | max_forward: 1.5076 | max_loss: 0.3748 | max_backward: 2.6247 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.5247 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6206.4 | uss_MB: 4649.5 | child_rss_MB: 5650.0 | allocated_MB: 343.9 | reserved_MB: 24586.0 | inactive_split_MB: 267.3 +rank3 [2026-10-28 17:54:18,159] INFO : Step 9500/50000 rank3 | Train Loss: 1.8199 | Train Loss Unweighted: 0.9995 | Train Loss Components: Auxiliary: 0.3464 | Cardinality: 0.1875 | Classification: 0.1651 | Classification_binary: 0.2558 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2851 | Regression_eta: 0.0035 | Regression_pt: 0.2864 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2634 | Regression_cos_phi: 0.1386 | Regression_energy: 0.1561 | Regression_eta: 0.0961 | Regression_pt: 0.1617 | Regression_sin_phi: 0.1511 | LR: 9.22e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.8027s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6532 | max_loss: 0.2385 | max_backward: 1.0043 | max_optimizer_step: 0.3001 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6205.2 | uss_MB: 5165.5 | child_rss_MB: 5654.5 | allocated_MB: 341.2 | reserved_MB: 24586.0 | inactive_split_MB: 180.8 +rank3 [2026-10-28 17:58:23,634] INFO : Step 9600/50000 rank3 | Train Loss: 2.0900 | Train Loss Unweighted: 1.1491 | Train Loss Components: Auxiliary: 0.4030 | Cardinality: 0.2464 | Classification: 0.2089 | Classification_binary: 0.3189 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3034 | Regression_eta: 0.0042 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1144 | Classification_binary: 0.2985 | Regression_cos_phi: 0.1497 | Regression_energy: 0.1479 | Regression_eta: 0.1301 | Regression_pt: 0.1551 | Regression_sin_phi: 0.1581 | LR: 9.21e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8834s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1083 | max_forward: 0.7381 | max_loss: 0.1457 | max_backward: 1.4890 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4970 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6206.2 | uss_MB: 5167.2 | child_rss_MB: 5637.2 | allocated_MB: 345.2 | reserved_MB: 25594.0 | inactive_split_MB: 190.6 +rank3 [2026-10-28 18:02:35,091] INFO : Step 9700/50000 rank3 | Train Loss: 2.1397 | Train Loss Unweighted: 1.1700 | Train Loss Components: Auxiliary: 0.4082 | Cardinality: 0.2377 | Classification: 0.1942 | Classification_binary: 0.3460 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3095 | Regression_eta: 0.0043 | Regression_pt: 0.3118 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0996 | Classification_binary: 0.3438 | Regression_cos_phi: 0.1777 | Regression_energy: 0.1541 | Regression_eta: 0.1063 | Regression_pt: 0.1593 | Regression_sin_phi: 0.1642 | LR: 9.19e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1033s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7744 | max_loss: 0.1497 | max_backward: 1.6846 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4710 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 6209.3 | uss_MB: 5170.1 | child_rss_MB: 5650.1 | allocated_MB: 345.9 | reserved_MB: 25594.0 | inactive_split_MB: 288.1 +rank3 [2026-10-28 18:06:42,228] INFO : Step 9800/50000 rank3 | Train Loss: 1.8589 | Train Loss Unweighted: 1.0245 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2213 | Classification: 0.1590 | Classification_binary: 0.2751 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2908 | Regression_eta: 0.0031 | Regression_pt: 0.2931 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0857 | Classification_binary: 0.2678 | Regression_cos_phi: 0.1598 | Regression_energy: 0.1413 | Regression_eta: 0.0734 | Regression_pt: 0.1496 | Regression_sin_phi: 0.1583 | LR: 9.17e-04 | DataLoad Time: 0.0243s | Model Forward Time: 4.1202s | Phase Times: max_pre_step_barrier: 0.1099 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.1713 | max_backward: 1.7200 | max_optimizer_step: 0.1898 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6188.3 | uss_MB: 5169.7 | child_rss_MB: 5642.5 | allocated_MB: 342.5 | reserved_MB: 26610.0 | inactive_split_MB: 264.5 +rank3 [2026-10-28 18:10:58,313] INFO : Step 9900/50000 rank3 | Train Loss: 1.8366 | Train Loss Unweighted: 1.0077 | Train Loss Components: Auxiliary: 0.3557 | Cardinality: 0.2091 | Classification: 0.1810 | Classification_binary: 0.2609 | HitGrouping: 0.2462 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2775 | Regression_eta: 0.0035 | Regression_pt: 0.2814 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1194 | Classification_binary: 0.2729 | Regression_cos_phi: 0.1592 | Regression_energy: 0.1497 | Regression_eta: 0.1034 | Regression_pt: 0.1535 | Regression_sin_phi: 0.1302 | LR: 9.16e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5799s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6346 | max_loss: 0.1692 | max_backward: 1.3542 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6188.4 | uss_MB: 5170.9 | child_rss_MB: 5645.5 | allocated_MB: 343.7 | reserved_MB: 26610.0 | inactive_split_MB: 205.7 +rank3 [2026-10-28 18:15:06,635] INFO : Step 10000/50000 rank3 | Train Loss: 2.0706 | Train Loss Unweighted: 1.1295 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2648 | Classification: 0.1819 | Classification_binary: 0.3225 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3077 | Regression_eta: 0.0037 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1257 | Classification_binary: 0.3282 | Regression_cos_phi: 0.1619 | Regression_energy: 0.1538 | Regression_eta: 0.1019 | Regression_pt: 0.1606 | Regression_sin_phi: 0.1343 | LR: 9.14e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.7120s | Phase Times: max_pre_step_barrier: 0.0136 | max_batch_to_device: 0.0708 | max_forward: 1.8842 | max_loss: 0.5484 | max_backward: 2.6620 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.5314 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6190.4 | uss_MB: 5174.2 | child_rss_MB: 5642.5 | allocated_MB: 345.2 | reserved_MB: 26610.0 | inactive_split_MB: 288.8 +rank3 [2026-10-28 18:15:07,626] INFO : Running validation on rank3 +rank3 [2026-10-28 18:15:07,626] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 18:15:07,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 18:15:55,924] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:15:56,538] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 18:16:22,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 18:16:23,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 18:16:24,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 18:16:26,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 18:16:36,118] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 18:16:36,561] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:16:37,079] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 18:16:52,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 18:16:54,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 18:16:55,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 18:16:57,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 18:17:07,253] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 18:17:09,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:17:10,078] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 18:17:25,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 18:17:26,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 18:17:26,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 18:17:27,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 18:17:37,436] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 18:22:01,598] INFO : Step 10100/50000 rank3 | Train Loss: 1.8115 | Train Loss Unweighted: 1.0221 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.1999 | Classification: 0.1589 | Classification_binary: 0.2815 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2868 | Regression_eta: 0.0032 | Regression_pt: 0.2886 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1094 | Classification_binary: 0.3274 | Regression_cos_phi: 0.1474 | Regression_energy: 0.1449 | Regression_eta: 0.0872 | Regression_pt: 0.1519 | Regression_sin_phi: 0.1386 | LR: 9.12e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9772s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7090 | max_loss: 0.1507 | max_backward: 1.2179 | max_optimizer_step: 0.3146 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 7166.8 | uss_MB: 6150.2 | child_rss_MB: 5645.8 | allocated_MB: 343.1 | reserved_MB: 21794.0 | inactive_split_MB: 243.8 +rank3 [2026-10-28 18:26:06,876] INFO : Step 10200/50000 rank3 | Train Loss: 1.6749 | Train Loss Unweighted: 0.9660 | Train Loss Components: Auxiliary: 0.3310 | Cardinality: 0.1773 | Classification: 0.1544 | Classification_binary: 0.2377 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2827 | Regression_eta: 0.0027 | Regression_pt: 0.2857 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0823 | Classification_binary: 0.2158 | Regression_cos_phi: 0.1055 | Regression_energy: 0.1427 | Regression_eta: 0.0892 | Regression_pt: 0.1504 | Regression_sin_phi: 0.1271 | LR: 9.11e-04 | DataLoad Time: 0.0108s | Model Forward Time: 4.0245s | Phase Times: max_pre_step_barrier: 0.0352 | max_batch_to_device: 0.0241 | max_forward: 0.6625 | max_loss: 0.1442 | max_backward: 1.3835 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7009 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1256 | Memory: rss_MB: 7164.8 | uss_MB: 6150.1 | child_rss_MB: 5645.0 | allocated_MB: 345.9 | reserved_MB: 21794.0 | inactive_split_MB: 292.4 +rank3 [2026-10-28 18:30:14,208] INFO : Step 10300/50000 rank3 | Train Loss: 1.9113 | Train Loss Unweighted: 1.0360 | Train Loss Components: Auxiliary: 0.3660 | Cardinality: 0.2108 | Classification: 0.1830 | Classification_binary: 0.2931 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0033 | Regression_pt: 0.2786 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0991 | Classification_binary: 0.2742 | Regression_cos_phi: 0.1113 | Regression_energy: 0.1274 | Regression_eta: 0.0766 | Regression_pt: 0.1345 | Regression_sin_phi: 0.1150 | LR: 9.09e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9105s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7205 | max_loss: 0.1496 | max_backward: 1.6083 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 6287.2 | uss_MB: 5273.5 | child_rss_MB: 5650.9 | allocated_MB: 343.8 | reserved_MB: 22764.0 | inactive_split_MB: 257.9 +rank3 [2026-10-28 18:34:19,879] INFO : Step 10400/50000 rank3 | Train Loss: 1.9303 | Train Loss Unweighted: 1.0569 | Train Loss Components: Auxiliary: 0.3739 | Cardinality: 0.2318 | Classification: 0.1704 | Classification_binary: 0.3342 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2718 | Regression_eta: 0.0033 | Regression_pt: 0.2738 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0827 | Classification_binary: 0.3548 | Regression_cos_phi: 0.1471 | Regression_energy: 0.1375 | Regression_eta: 0.1002 | Regression_pt: 0.1436 | Regression_sin_phi: 0.1684 | LR: 9.07e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7366s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7465 | max_loss: 0.1499 | max_backward: 1.4212 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.4712 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6287.2 | uss_MB: 5272.0 | child_rss_MB: 5660.4 | allocated_MB: 345.6 | reserved_MB: 22792.0 | inactive_split_MB: 259.8 +rank3 [2026-10-28 18:38:24,242] INFO : Step 10500/50000 rank3 | Train Loss: 1.6601 | Train Loss Unweighted: 0.9268 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1892 | Classification: 0.1410 | Classification_binary: 0.2342 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2693 | Regression_eta: 0.0029 | Regression_pt: 0.2766 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2107 | Regression_cos_phi: 0.0756 | Regression_energy: 0.1506 | Regression_eta: 0.0621 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0905 | LR: 9.06e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.5607s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 0.6364 | max_loss: 0.2260 | max_backward: 1.1629 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6452 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6284.6 | uss_MB: 5269.3 | child_rss_MB: 5635.7 | allocated_MB: 340.4 | reserved_MB: 22792.0 | inactive_split_MB: 230.3 +rank3 [2026-10-28 18:42:22,546] INFO : Step 10600/50000 rank3 | Train Loss: 1.9388 | Train Loss Unweighted: 1.0719 | Train Loss Components: Auxiliary: 0.3737 | Cardinality: 0.2008 | Classification: 0.1863 | Classification_binary: 0.2709 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3047 | Regression_eta: 0.0042 | Regression_pt: 0.3019 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1104 | Classification_binary: 0.2876 | Regression_cos_phi: 0.2069 | Regression_energy: 0.1429 | Regression_eta: 0.1256 | Regression_pt: 0.1475 | Regression_sin_phi: 0.1617 | LR: 9.04e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9681s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6880 | max_loss: 0.2458 | max_backward: 1.4354 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7481 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 6285.6 | uss_MB: 5273.7 | child_rss_MB: 5653.3 | allocated_MB: 344.6 | reserved_MB: 22798.0 | inactive_split_MB: 286.4 +rank3 [2026-10-28 18:46:28,872] INFO : Step 10700/50000 rank3 | Train Loss: 1.8955 | Train Loss Unweighted: 1.0230 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.2139 | Classification: 0.1752 | Classification_binary: 0.3186 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2598 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1025 | Classification_binary: 0.3266 | Regression_cos_phi: 0.1762 | Regression_energy: 0.1168 | Regression_eta: 0.0978 | Regression_pt: 0.1234 | Regression_sin_phi: 0.1625 | LR: 9.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.1541s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0072 | max_forward: 1.0363 | max_loss: 0.1757 | max_backward: 2.3255 | max_optimizer_step: 0.1834 | max_tensorboard_logging: 0.7232 | max_tensorboard_flush: 0.0051 | max_post_step_barrier: 0.1576 | Memory: rss_MB: 6287.7 | uss_MB: 5274.9 | child_rss_MB: 5649.7 | allocated_MB: 342.1 | reserved_MB: 22798.0 | inactive_split_MB: 282.9 +rank3 [2026-10-28 18:50:28,328] INFO : Step 10800/50000 rank3 | Train Loss: 1.7679 | Train Loss Unweighted: 0.9563 | Train Loss Components: Auxiliary: 0.3446 | Cardinality: 0.2074 | Classification: 0.1518 | Classification_binary: 0.2766 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2590 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0888 | Classification_binary: 0.2908 | Regression_cos_phi: 0.1163 | Regression_energy: 0.1290 | Regression_eta: 0.1283 | Regression_pt: 0.1318 | Regression_sin_phi: 0.2016 | LR: 9.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7798s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0995 | max_forward: 0.7884 | max_loss: 0.2225 | max_backward: 2.0144 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.5230 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6288.7 | uss_MB: 5271.1 | child_rss_MB: 5654.7 | allocated_MB: 343.6 | reserved_MB: 23784.0 | inactive_split_MB: 286.4 +rank3 [2026-10-28 18:54:31,737] INFO : Step 10900/50000 rank3 | Train Loss: 1.6025 | Train Loss Unweighted: 0.8934 | Train Loss Components: Auxiliary: 0.3208 | Cardinality: 0.1729 | Classification: 0.1446 | Classification_binary: 0.2284 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2574 | Regression_eta: 0.0027 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.1864 | Regression_cos_phi: 0.0991 | Regression_energy: 0.1307 | Regression_eta: 0.0813 | Regression_pt: 0.1358 | Regression_sin_phi: 0.1018 | LR: 8.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.4465s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.5765 | max_loss: 0.1490 | max_backward: 1.1882 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6288.0 | uss_MB: 5275.5 | child_rss_MB: 5639.7 | allocated_MB: 342.1 | reserved_MB: 23796.0 | inactive_split_MB: 272.7 +rank3 [2026-10-28 18:58:45,418] INFO : Step 11000/50000 rank3 | Train Loss: 1.9315 | Train Loss Unweighted: 1.0466 | Train Loss Components: Auxiliary: 0.3721 | Cardinality: 0.2078 | Classification: 0.1806 | Classification_binary: 0.3087 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0037 | Regression_pt: 0.2758 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1268 | Classification_binary: 0.3397 | Regression_cos_phi: 0.1857 | Regression_energy: 0.1397 | Regression_eta: 0.1272 | Regression_pt: 0.1444 | Regression_sin_phi: 0.1823 | LR: 8.97e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.1701s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7424 | max_loss: 0.1704 | max_backward: 1.5866 | max_optimizer_step: 0.2906 | max_tensorboard_logging: 0.6287 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6277.0 | uss_MB: 5281.8 | child_rss_MB: 5663.8 | allocated_MB: 347.8 | reserved_MB: 23796.0 | inactive_split_MB: 262.8 +rank3 [2026-10-28 18:58:46,359] INFO : Running validation on rank3 +rank3 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 18:59:40,372] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:59:40,901] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 19:00:07,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 19:00:09,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 19:00:10,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 19:00:11,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 19:00:21,277] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 19:00:21,678] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:00:22,257] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 19:00:37,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 19:00:38,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 19:00:39,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 19:00:40,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 19:00:50,816] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 19:00:52,166] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:00:52,895] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 19:01:08,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 19:01:08,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 19:01:09,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 19:01:10,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 19:01:20,175] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 19:05:41,392] INFO : Step 11100/50000 rank3 | Train Loss: 1.9118 | Train Loss Unweighted: 1.0064 | Train Loss Components: Auxiliary: 0.3723 | Cardinality: 0.2793 | Classification: 0.1563 | Classification_binary: 0.3082 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2660 | Regression_eta: 0.0034 | Regression_pt: 0.2693 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0999 | Classification_binary: 0.3272 | Regression_cos_phi: 0.1226 | Regression_energy: 0.1374 | Regression_eta: 0.0909 | Regression_pt: 0.1428 | Regression_sin_phi: 0.1417 | LR: 8.95e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0404s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.9217 | max_loss: 0.2304 | max_backward: 1.5570 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 7169.9 | uss_MB: 6179.3 | child_rss_MB: 5664.2 | allocated_MB: 343.2 | reserved_MB: 22146.0 | inactive_split_MB: 281.7 +rank3 [2026-10-28 19:09:39,137] INFO : Step 11200/50000 rank3 | Train Loss: 1.7366 | Train Loss Unweighted: 0.9397 | Train Loss Components: Auxiliary: 0.3529 | Cardinality: 0.2466 | Classification: 0.1384 | Classification_binary: 0.2548 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2689 | Regression_eta: 0.0027 | Regression_pt: 0.2722 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2116 | Regression_cos_phi: 0.0866 | Regression_energy: 0.1206 | Regression_eta: 0.0562 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0498 | LR: 8.93e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7175s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7793 | max_loss: 0.1825 | max_backward: 1.3743 | max_optimizer_step: 0.2128 | max_tensorboard_logging: 0.4676 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7167.9 | uss_MB: 6179.2 | child_rss_MB: 5641.7 | allocated_MB: 344.6 | reserved_MB: 22972.0 | inactive_split_MB: 273.4 +rank3 [2026-10-28 19:13:41,796] INFO : Step 11300/50000 rank3 | Train Loss: 1.6876 | Train Loss Unweighted: 0.9423 | Train Loss Components: Auxiliary: 0.3334 | Cardinality: 0.1862 | Classification: 0.1581 | Classification_binary: 0.2713 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2524 | Regression_eta: 0.0027 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1010 | Classification_binary: 0.2915 | Regression_cos_phi: 0.1131 | Regression_energy: 0.1158 | Regression_eta: 0.0782 | Regression_pt: 0.1223 | Regression_sin_phi: 0.1015 | LR: 8.91e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6812s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0223 | max_forward: 0.9315 | max_loss: 0.2308 | max_backward: 1.9859 | max_optimizer_step: 0.1928 | max_tensorboard_logging: 0.5681 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6388.7 | uss_MB: 5273.7 | child_rss_MB: 5658.4 | allocated_MB: 345.5 | reserved_MB: 24368.0 | inactive_split_MB: 269.6 +rank3 [2026-10-28 19:17:45,333] INFO : Step 11400/50000 rank3 | Train Loss: 1.8563 | Train Loss Unweighted: 0.9980 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.2376 | Classification: 0.1627 | Classification_binary: 0.3164 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2551 | Regression_eta: 0.0029 | Regression_pt: 0.2577 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1226 | Classification_binary: 0.3623 | Regression_cos_phi: 0.1837 | Regression_energy: 0.1386 | Regression_eta: 0.1016 | Regression_pt: 0.1427 | Regression_sin_phi: 0.1598 | LR: 8.89e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.0887s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.7684 | max_loss: 0.1511 | max_backward: 1.6966 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5034 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 6391.7 | uss_MB: 5402.6 | child_rss_MB: 5664.5 | allocated_MB: 343.2 | reserved_MB: 25642.0 | inactive_split_MB: 251.6 +rank3 [2026-10-28 19:21:55,043] INFO : Step 11500/50000 rank3 | Train Loss: 1.6323 | Train Loss Unweighted: 0.8976 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1755 | Classification: 0.1375 | Classification_binary: 0.2717 | HitGrouping: 0.2346 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2408 | Regression_eta: 0.0028 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.2764 | Regression_cos_phi: 0.1349 | Regression_energy: 0.1266 | Regression_eta: 0.0741 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0977 | LR: 8.88e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.6938s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7232 | max_loss: 0.1557 | max_backward: 1.4770 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6390.5 | uss_MB: 5401.3 | child_rss_MB: 5644.3 | allocated_MB: 345.3 | reserved_MB: 25642.0 | inactive_split_MB: 274.7 +rank3 [2026-10-28 19:25:54,594] INFO : Step 11600/50000 rank3 | Train Loss: 1.6133 | Train Loss Unweighted: 0.9062 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1718 | Classification: 0.1502 | Classification_binary: 0.2356 | HitGrouping: 0.2377 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2569 | Regression_eta: 0.0025 | Regression_pt: 0.2581 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0827 | Classification_binary: 0.2530 | Regression_cos_phi: 0.1165 | Regression_energy: 0.1268 | Regression_eta: 0.0855 | Regression_pt: 0.1335 | Regression_sin_phi: 0.1129 | LR: 8.86e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.6498s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6646 | max_loss: 0.1710 | max_backward: 1.3208 | max_optimizer_step: 0.1963 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6391.5 | uss_MB: 5404.2 | child_rss_MB: 5665.3 | allocated_MB: 344.1 | reserved_MB: 26630.0 | inactive_split_MB: 209.9 +rank3 [2026-10-28 19:29:50,732] INFO : Step 11700/50000 rank3 | Train Loss: 1.7739 | Train Loss Unweighted: 0.9706 | Train Loss Components: Auxiliary: 0.3531 | Cardinality: 0.2418 | Classification: 0.1518 | Classification_binary: 0.2821 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2638 | Regression_eta: 0.0028 | Regression_pt: 0.2671 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0922 | Classification_binary: 0.2677 | Regression_cos_phi: 0.1025 | Regression_energy: 0.1434 | Regression_eta: 0.0640 | Regression_pt: 0.1508 | Regression_sin_phi: 0.1136 | LR: 8.84e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.4593s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8789 | max_loss: 0.2057 | max_backward: 2.3846 | max_optimizer_step: 0.1875 | max_tensorboard_logging: 1.0403 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6389.5 | uss_MB: 4921.9 | child_rss_MB: 5640.9 | allocated_MB: 343.8 | reserved_MB: 27624.0 | inactive_split_MB: 186.6 +rank3 [2026-10-28 19:33:54,004] INFO : Step 11800/50000 rank3 | Train Loss: 1.8223 | Train Loss Unweighted: 0.9745 | Train Loss Components: Auxiliary: 0.3609 | Cardinality: 0.2329 | Classification: 0.1589 | Classification_binary: 0.3100 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2473 | Regression_eta: 0.0028 | Regression_pt: 0.2523 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1074 | Classification_binary: 0.3119 | Regression_cos_phi: 0.1496 | Regression_energy: 0.1186 | Regression_eta: 0.0960 | Regression_pt: 0.1263 | Regression_sin_phi: 0.1263 | LR: 8.82e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.3890s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0070 | max_forward: 1.2302 | max_loss: 0.2767 | max_backward: 2.3745 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6390.5 | uss_MB: 5402.7 | child_rss_MB: 5661.5 | allocated_MB: 346.0 | reserved_MB: 27624.0 | inactive_split_MB: 280.5 +rank3 [2026-10-28 19:37:48,966] INFO : Step 11900/50000 rank3 | Train Loss: 1.5797 | Train Loss Unweighted: 0.8638 | Train Loss Components: Auxiliary: 0.3225 | Cardinality: 0.1935 | Classification: 0.1358 | Classification_binary: 0.2351 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2427 | Regression_eta: 0.0025 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1056 | Classification_binary: 0.2280 | Regression_cos_phi: 0.0965 | Regression_energy: 0.1171 | Regression_eta: 0.0816 | Regression_pt: 0.1219 | Regression_sin_phi: 0.1373 | LR: 8.80e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.1369s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0787 | max_forward: 0.7011 | max_loss: 0.1484 | max_backward: 1.2002 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.9463 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 6390.3 | uss_MB: 5402.5 | child_rss_MB: 5642.2 | allocated_MB: 340.1 | reserved_MB: 28620.0 | inactive_split_MB: 264.1 +rank3 [2026-10-28 19:41:43,167] INFO : Step 12000/50000 rank3 | Train Loss: 1.6036 | Train Loss Unweighted: 0.8706 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1972 | Classification: 0.1481 | Classification_binary: 0.2459 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2330 | Regression_eta: 0.0025 | Regression_pt: 0.2385 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0953 | Classification_binary: 0.2149 | Regression_cos_phi: 0.0925 | Regression_energy: 0.1166 | Regression_eta: 0.0720 | Regression_pt: 0.1222 | Regression_sin_phi: 0.1090 | LR: 8.78e-04 | DataLoad Time: 0.0147s | Model Forward Time: 3.6980s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0072 | max_forward: 0.6194 | max_loss: 0.1546 | max_backward: 1.4628 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5143 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6383.3 | uss_MB: 5409.4 | child_rss_MB: 5646.2 | allocated_MB: 342.9 | reserved_MB: 28620.0 | inactive_split_MB: 215.8 +rank3 [2026-10-28 19:41:44,122] INFO : Running validation on rank3 +rank3 [2026-10-28 19:41:44,123] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 19:41:44,123] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 19:42:35,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:42:35,720] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 19:43:02,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 19:43:04,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 19:43:05,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 19:43:06,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 19:43:16,513] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 19:43:17,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:43:17,650] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 19:43:33,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 19:43:35,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 19:43:36,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 19:43:36,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 19:43:46,973] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 19:43:48,085] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:43:48,652] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 19:44:04,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 19:44:04,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 19:44:05,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 19:44:06,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 19:44:16,307] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 19:48:34,083] INFO : Step 12100/50000 rank3 | Train Loss: 1.8149 | Train Loss Unweighted: 0.9793 | Train Loss Components: Auxiliary: 0.3632 | Cardinality: 0.2324 | Classification: 0.1506 | Classification_binary: 0.3149 | HitGrouping: 0.2337 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2518 | Regression_eta: 0.0029 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.3301 | Regression_cos_phi: 0.1086 | Regression_energy: 0.1050 | Regression_eta: 0.0770 | Regression_pt: 0.1106 | Regression_sin_phi: 0.1347 | LR: 8.76e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5277s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8305 | max_loss: 0.1678 | max_backward: 1.6498 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8610 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7131.3 | uss_MB: 6150.6 | child_rss_MB: 5646.8 | allocated_MB: 345.8 | reserved_MB: 23512.0 | inactive_split_MB: 299.6 +rank3 [2026-10-28 19:52:54,660] INFO : Step 12200/50000 rank3 | Train Loss: 1.6619 | Train Loss Unweighted: 0.9043 | Train Loss Components: Auxiliary: 0.3290 | Cardinality: 0.1906 | Classification: 0.1445 | Classification_binary: 0.2674 | HitGrouping: 0.2303 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2435 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0794 | Classification_binary: 0.2266 | Regression_cos_phi: 0.1109 | Regression_energy: 0.1133 | Regression_eta: 0.0716 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0774 | LR: 8.74e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8234s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.7862 | max_loss: 0.1438 | max_backward: 1.5991 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.8843 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7133.3 | uss_MB: 5953.1 | child_rss_MB: 5653.1 | allocated_MB: 341.2 | reserved_MB: 23512.0 | inactive_split_MB: 206.8 +rank3 [2026-10-28 19:56:47,338] INFO : Step 12300/50000 rank3 | Train Loss: 1.4699 | Train Loss Unweighted: 0.8181 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1474 | Classification: 0.1404 | Classification_binary: 0.2280 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2222 | Regression_eta: 0.0028 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0866 | Classification_binary: 0.2310 | Regression_cos_phi: 0.1168 | Regression_energy: 0.1085 | Regression_eta: 0.1048 | Regression_pt: 0.1119 | Regression_sin_phi: 0.1150 | LR: 8.72e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.7229s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6177 | max_loss: 0.2232 | max_backward: 1.1179 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 0.7974 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6497.9 | uss_MB: 5527.4 | child_rss_MB: 5654.1 | allocated_MB: 343.2 | reserved_MB: 24286.0 | inactive_split_MB: 290.2 +rank3 [2026-10-28 20:00:36,190] INFO : Step 12400/50000 rank3 | Train Loss: 1.6135 | Train Loss Unweighted: 0.8782 | Train Loss Components: Auxiliary: 0.3265 | Cardinality: 0.2056 | Classification: 0.1546 | Classification_binary: 0.2540 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2308 | Regression_eta: 0.0025 | Regression_pt: 0.2337 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0821 | Classification_binary: 0.2566 | Regression_cos_phi: 0.0991 | Regression_energy: 0.1320 | Regression_eta: 0.0809 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0776 | LR: 8.70e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.9944s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6928 | max_loss: 0.1486 | max_backward: 1.3860 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6497.9 | uss_MB: 5528.1 | child_rss_MB: 5671.7 | allocated_MB: 344.8 | reserved_MB: 24510.0 | inactive_split_MB: 295.5 +rank3 [2026-10-28 20:04:30,060] INFO : Step 12500/50000 rank3 | Train Loss: 1.6527 | Train Loss Unweighted: 0.8797 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1890 | Classification: 0.1391 | Classification_binary: 0.2779 | HitGrouping: 0.2315 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2273 | Regression_eta: 0.0030 | Regression_pt: 0.2295 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0938 | Classification_binary: 0.3014 | Regression_cos_phi: 0.1488 | Regression_energy: 0.1055 | Regression_eta: 0.0731 | Regression_pt: 0.1103 | Regression_sin_phi: 0.1489 | LR: 8.68e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1706s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0535 | max_forward: 0.7747 | max_loss: 0.1534 | max_backward: 1.3339 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6494.1 | uss_MB: 5528.8 | child_rss_MB: 5649.7 | allocated_MB: 347.2 | reserved_MB: 24510.0 | inactive_split_MB: 456.8 +rank3 [2026-10-28 20:08:30,520] INFO : Step 12600/50000 rank3 | Train Loss: 1.4969 | Train Loss Unweighted: 0.8185 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1595 | Classification: 0.1210 | Classification_binary: 0.2357 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2279 | Regression_eta: 0.0024 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0870 | Classification_binary: 0.2557 | Regression_cos_phi: 0.1144 | Regression_energy: 0.1069 | Regression_eta: 0.0671 | Regression_pt: 0.1122 | Regression_sin_phi: 0.1185 | LR: 8.66e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7599s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0062 | max_forward: 0.6571 | max_loss: 0.1429 | max_backward: 1.4364 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6482.1 | uss_MB: 5531.3 | child_rss_MB: 5638.2 | allocated_MB: 342.9 | reserved_MB: 25526.0 | inactive_split_MB: 234.6 +rank3 [2026-10-28 20:12:34,583] INFO : Step 12700/50000 rank3 | Train Loss: 1.6234 | Train Loss Unweighted: 0.8661 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.2004 | Classification: 0.1459 | Classification_binary: 0.2387 | HitGrouping: 0.2403 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2360 | Regression_eta: 0.0027 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0866 | Classification_binary: 0.2185 | Regression_cos_phi: 0.0871 | Regression_energy: 0.1205 | Regression_eta: 0.0688 | Regression_pt: 0.1249 | Regression_sin_phi: 0.1043 | LR: 8.64e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.4790s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6221 | max_loss: 0.1846 | max_backward: 1.1554 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6469.4 | uss_MB: 5531.3 | child_rss_MB: 5648.7 | allocated_MB: 345.9 | reserved_MB: 26286.0 | inactive_split_MB: 241.7 +rank3 [2026-10-28 20:16:22,943] INFO : Step 12800/50000 rank3 | Train Loss: 1.7877 | Train Loss Unweighted: 0.9398 | Train Loss Components: Auxiliary: 0.3651 | Cardinality: 0.2327 | Classification: 0.1495 | Classification_binary: 0.3094 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2357 | Regression_eta: 0.0027 | Regression_pt: 0.2394 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1001 | Classification_binary: 0.3179 | Regression_cos_phi: 0.1327 | Regression_energy: 0.1174 | Regression_eta: 0.0914 | Regression_pt: 0.1236 | Regression_sin_phi: 0.1514 | LR: 8.62e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.2572s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.8468 | max_loss: 0.1534 | max_backward: 1.7420 | max_optimizer_step: 0.3013 | max_tensorboard_logging: 0.5725 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6458.4 | uss_MB: 5535.9 | child_rss_MB: 5651.5 | allocated_MB: 345.6 | reserved_MB: 26286.0 | inactive_split_MB: 284.4 +rank3 [2026-10-28 20:20:13,786] INFO : Step 12900/50000 rank3 | Train Loss: 1.5881 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.3191 | Cardinality: 0.2029 | Classification: 0.1304 | Classification_binary: 0.2564 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2291 | Regression_eta: 0.0023 | Regression_pt: 0.2312 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2396 | Regression_cos_phi: 0.0962 | Regression_energy: 0.1061 | Regression_eta: 0.0666 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0971 | LR: 8.60e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9611s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7180 | max_loss: 0.2380 | max_backward: 1.4488 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6932 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6444.6 | uss_MB: 5531.5 | child_rss_MB: 5646.1 | allocated_MB: 341.8 | reserved_MB: 26286.0 | inactive_split_MB: 252.2 +rank3 [2026-10-28 20:24:06,549] INFO : Step 13000/50000 rank3 | Train Loss: 1.5217 | Train Loss Unweighted: 0.8547 | Train Loss Components: Auxiliary: 0.3141 | Cardinality: 0.1693 | Classification: 0.1382 | Classification_binary: 0.2135 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2470 | Regression_eta: 0.0024 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.1930 | Regression_cos_phi: 0.1089 | Regression_energy: 0.1148 | Regression_eta: 0.0544 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0586 | LR: 8.58e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8291s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7101 | max_loss: 0.1763 | max_backward: 1.1845 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 0.8282 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6429.6 | uss_MB: 5536.9 | child_rss_MB: 5653.9 | allocated_MB: 339.8 | reserved_MB: 26286.0 | inactive_split_MB: 213.4 +rank3 [2026-10-28 20:24:07,449] INFO : Running validation on rank3 +rank3 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 20:24:58,459] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:24:59,078] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 20:25:24,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 20:25:25,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 20:25:26,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 20:25:27,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 20:25:37,631] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 20:25:38,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:25:38,652] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 20:25:55,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 20:25:56,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 20:25:57,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 20:25:58,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 20:26:08,037] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 20:26:08,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:26:09,008] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 20:26:24,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 20:26:25,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 20:26:25,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 20:26:26,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 20:26:36,430] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 20:30:47,339] INFO : Step 13100/50000 rank3 | Train Loss: 1.6168 | Train Loss Unweighted: 0.8649 | Train Loss Components: Auxiliary: 0.3213 | Cardinality: 0.1869 | Classification: 0.1467 | Classification_binary: 0.2696 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2210 | Regression_eta: 0.0027 | Regression_pt: 0.2221 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.2538 | Regression_cos_phi: 0.1047 | Regression_energy: 0.1351 | Regression_eta: 0.0798 | Regression_pt: 0.1429 | Regression_sin_phi: 0.0992 | LR: 8.56e-04 | DataLoad Time: 0.0076s | Model Forward Time: 6.3902s | Phase Times: max_pre_step_barrier: 0.0337 | max_batch_to_device: 0.0136 | max_forward: 1.2842 | max_loss: 0.2583 | max_backward: 3.1281 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.9677 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7302.9 | uss_MB: 6406.9 | child_rss_MB: 5646.9 | allocated_MB: 341.8 | reserved_MB: 22940.0 | inactive_split_MB: 270.2 +rank3 [2026-10-28 20:34:56,292] INFO : Step 13200/50000 rank3 | Train Loss: 1.6996 | Train Loss Unweighted: 0.8855 | Train Loss Components: Auxiliary: 0.3386 | Cardinality: 0.2474 | Classification: 0.1410 | Classification_binary: 0.2769 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2299 | Regression_eta: 0.0027 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2586 | Regression_cos_phi: 0.1445 | Regression_energy: 0.1153 | Regression_eta: 0.0852 | Regression_pt: 0.1187 | Regression_sin_phi: 0.1169 | LR: 8.54e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1536s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0072 | max_forward: 0.7543 | max_loss: 0.1532 | max_backward: 1.3317 | max_optimizer_step: 0.3180 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7300.0 | uss_MB: 6407.2 | child_rss_MB: 5649.5 | allocated_MB: 343.4 | reserved_MB: 23942.0 | inactive_split_MB: 249.6 +rank3 [2026-10-28 20:38:53,364] INFO : Step 13300/50000 rank3 | Train Loss: 1.4876 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2970 | Cardinality: 0.2018 | Classification: 0.1100 | Classification_binary: 0.2192 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2205 | Regression_eta: 0.0026 | Regression_pt: 0.2230 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.2372 | Regression_cos_phi: 0.0890 | Regression_energy: 0.1325 | Regression_eta: 0.0637 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0714 | LR: 8.52e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.6361s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0259 | max_forward: 0.9208 | max_loss: 0.2245 | max_backward: 1.9523 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6473.0 | uss_MB: 4937.1 | child_rss_MB: 5535.0 | allocated_MB: 341.7 | reserved_MB: 24474.0 | inactive_split_MB: 265.4 +rank3 [2026-10-28 20:42:52,943] INFO : Step 13400/50000 rank3 | Train Loss: 1.4253 | Train Loss Unweighted: 0.7944 | Train Loss Components: Auxiliary: 0.2889 | Cardinality: 0.1585 | Classification: 0.1406 | Classification_binary: 0.2158 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2158 | Regression_eta: 0.0022 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2178 | Regression_cos_phi: 0.1097 | Regression_energy: 0.0996 | Regression_eta: 0.0574 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0715 | LR: 8.50e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.7382s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6533 | max_loss: 0.2500 | max_backward: 1.4458 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.5454 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6474.0 | uss_MB: 5610.2 | child_rss_MB: 5538.3 | allocated_MB: 341.6 | reserved_MB: 25334.0 | inactive_split_MB: 266.6 +rank3 [2026-10-28 20:47:05,650] INFO : Step 13500/50000 rank3 | Train Loss: 1.6021 | Train Loss Unweighted: 0.8749 | Train Loss Components: Auxiliary: 0.3241 | Cardinality: 0.1832 | Classification: 0.1473 | Classification_binary: 0.2801 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2200 | Regression_eta: 0.0024 | Regression_pt: 0.2225 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0825 | Classification_binary: 0.2673 | Regression_cos_phi: 0.0926 | Regression_energy: 0.1132 | Regression_eta: 0.0571 | Regression_pt: 0.1193 | Regression_sin_phi: 0.1013 | LR: 8.48e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.1040s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.7859 | max_loss: 0.3303 | max_backward: 2.5111 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4974 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6412.7 | uss_MB: 5395.8 | child_rss_MB: 5343.5 | allocated_MB: 344.3 | reserved_MB: 25574.0 | inactive_split_MB: 284.1 +rank3 [2026-10-28 20:51:04,210] INFO : Step 13600/50000 rank3 | Train Loss: 1.5296 | Train Loss Unweighted: 0.8114 | Train Loss Components: Auxiliary: 0.3050 | Cardinality: 0.1872 | Classification: 0.1291 | Classification_binary: 0.2450 | HitGrouping: 0.2218 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2158 | Regression_eta: 0.0025 | Regression_pt: 0.2165 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.2527 | Regression_cos_phi: 0.1358 | Regression_energy: 0.1025 | Regression_eta: 0.0653 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0918 | LR: 8.45e-04 | DataLoad Time: 0.0217s | Model Forward Time: 3.6638s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.7508 | max_loss: 0.1507 | max_backward: 1.2891 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6404.7 | uss_MB: 5608.6 | child_rss_MB: 5283.3 | allocated_MB: 345.0 | reserved_MB: 26608.0 | inactive_split_MB: 257.3 +rank3 [2026-10-28 20:55:12,337] INFO : Step 13700/50000 rank3 | Train Loss: 1.4515 | Train Loss Unweighted: 0.7895 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1575 | Classification: 0.1261 | Classification_binary: 0.2121 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2219 | Regression_eta: 0.0023 | Regression_pt: 0.2245 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2032 | Regression_cos_phi: 0.1190 | Regression_energy: 0.0906 | Regression_eta: 0.0857 | Regression_pt: 0.0976 | Regression_sin_phi: 0.1434 | LR: 8.43e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.8519s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.6804 | max_loss: 0.2128 | max_backward: 1.1331 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6398.1 | uss_MB: 5609.6 | child_rss_MB: 5245.6 | allocated_MB: 342.5 | reserved_MB: 26608.0 | inactive_split_MB: 241.7 +rank3 [2026-10-28 20:59:19,101] INFO : Step 13800/50000 rank3 | Train Loss: 1.5060 | Train Loss Unweighted: 0.8320 | Train Loss Components: Auxiliary: 0.3084 | Cardinality: 0.1762 | Classification: 0.1368 | Classification_binary: 0.2567 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2156 | Regression_eta: 0.0023 | Regression_pt: 0.2184 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1016 | Classification_binary: 0.2772 | Regression_cos_phi: 0.1013 | Regression_energy: 0.1099 | Regression_eta: 0.0600 | Regression_pt: 0.1129 | Regression_sin_phi: 0.1103 | LR: 8.41e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4116s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7967 | max_loss: 0.1491 | max_backward: 1.5976 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6392.1 | uss_MB: 5613.3 | child_rss_MB: 5216.1 | allocated_MB: 342.1 | reserved_MB: 26608.0 | inactive_split_MB: 241.3 +rank3 [2026-10-28 21:03:21,798] INFO : Step 13900/50000 rank3 | Train Loss: 1.6108 | Train Loss Unweighted: 0.8469 | Train Loss Components: Auxiliary: 0.3204 | Cardinality: 0.1942 | Classification: 0.1303 | Classification_binary: 0.2751 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2168 | Regression_eta: 0.0028 | Regression_pt: 0.2192 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.3011 | Regression_cos_phi: 0.1264 | Regression_energy: 0.1164 | Regression_eta: 0.0918 | Regression_pt: 0.1237 | Regression_sin_phi: 0.1092 | LR: 8.39e-04 | DataLoad Time: 0.0204s | Model Forward Time: 6.7324s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8010 | max_loss: 0.1552 | max_backward: 2.5526 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 1.2862 | max_tensorboard_flush: 0.0041 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 6389.1 | uss_MB: 5603.5 | child_rss_MB: 5217.8 | allocated_MB: 345.6 | reserved_MB: 26608.0 | inactive_split_MB: 292.8 +rank3 [2026-10-28 21:07:29,378] INFO : Step 14000/50000 rank3 | Train Loss: 1.3659 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1707 | Classification: 0.0974 | Classification_binary: 0.1997 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2149 | Regression_eta: 0.0023 | Regression_pt: 0.2181 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1935 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0849 | Regression_eta: 0.0627 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0645 | LR: 8.37e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.4515s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7574 | max_loss: 0.1908 | max_backward: 1.5870 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6393.2 | uss_MB: 4632.1 | child_rss_MB: 5217.4 | allocated_MB: 342.3 | reserved_MB: 26608.0 | inactive_split_MB: 238.4 +rank3 [2026-10-28 21:07:30,351] INFO : Running validation on rank3 +rank3 [2026-10-28 21:07:30,351] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 21:07:30,351] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 21:08:21,200] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:08:21,767] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 21:08:48,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 21:08:50,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 21:08:51,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 21:08:52,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 21:09:02,129] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 21:09:05,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:09:06,443] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 21:09:22,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 21:09:23,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 21:09:24,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 21:09:25,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 21:09:35,633] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 21:09:36,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:09:36,854] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 21:09:52,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 21:09:52,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 21:09:53,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 21:09:54,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 21:10:04,322] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 21:14:16,742] INFO : Step 14100/50000 rank3 | Train Loss: 1.5416 | Train Loss Unweighted: 0.8176 | Train Loss Components: Auxiliary: 0.3012 | Cardinality: 0.1751 | Classification: 0.1363 | Classification_binary: 0.2297 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2177 | Regression_eta: 0.0028 | Regression_pt: 0.2283 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0987 | Classification_binary: 0.2629 | Regression_cos_phi: 0.1494 | Regression_energy: 0.1200 | Regression_eta: 0.1648 | Regression_pt: 0.1307 | Regression_sin_phi: 0.1495 | LR: 8.35e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2014s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7718 | max_loss: 0.1710 | max_backward: 1.6376 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 7412.7 | uss_MB: 6170.5 | child_rss_MB: 5197.6 | allocated_MB: 347.1 | reserved_MB: 23970.0 | inactive_split_MB: 244.9 +rank3 [2026-10-28 21:18:17,875] INFO : Step 14200/50000 rank3 | Train Loss: 1.6164 | Train Loss Unweighted: 0.8542 | Train Loss Components: Auxiliary: 0.3190 | Cardinality: 0.1950 | Classification: 0.1478 | Classification_binary: 0.2794 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2094 | Regression_eta: 0.0027 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0991 | Classification_binary: 0.2887 | Regression_cos_phi: 0.0971 | Regression_energy: 0.0898 | Regression_eta: 0.0776 | Regression_pt: 0.0972 | Regression_sin_phi: 0.1186 | LR: 8.32e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.3704s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1153 | max_forward: 0.8788 | max_loss: 0.1615 | max_backward: 1.5286 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7413.4 | uss_MB: 6634.1 | child_rss_MB: 5195.0 | allocated_MB: 344.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.9 +rank3 [2026-10-28 21:22:21,672] INFO : Step 14300/50000 rank3 | Train Loss: 1.4246 | Train Loss Unweighted: 0.7744 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1685 | Classification: 0.1198 | Classification_binary: 0.2357 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2084 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2241 | Regression_cos_phi: 0.0784 | Regression_energy: 0.1033 | Regression_eta: 0.0608 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0976 | LR: 8.30e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.8784s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7183 | max_loss: 0.1494 | max_backward: 1.3371 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6454.4 | uss_MB: 5677.1 | child_rss_MB: 5184.2 | allocated_MB: 341.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.1 +rank3 [2026-10-28 21:26:25,336] INFO : Step 14400/50000 rank3 | Train Loss: 1.3539 | Train Loss Unweighted: 0.7267 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1402 | Classification: 0.1303 | Classification_binary: 0.2015 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1957 | Regression_eta: 0.0022 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0780 | Classification_binary: 0.1985 | Regression_cos_phi: 0.1155 | Regression_energy: 0.1010 | Regression_eta: 0.1009 | Regression_pt: 0.1036 | Regression_sin_phi: 0.1012 | LR: 8.28e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6676s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6350 | max_loss: 0.2071 | max_backward: 1.2597 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6458.7 | uss_MB: 5681.9 | child_rss_MB: 5188.8 | allocated_MB: 346.1 | reserved_MB: 24962.0 | inactive_split_MB: 231.6 +rank3 [2026-10-28 21:30:23,755] INFO : Step 14500/50000 rank3 | Train Loss: 1.5651 | Train Loss Unweighted: 0.7967 | Train Loss Components: Auxiliary: 0.3070 | Cardinality: 0.2312 | Classification: 0.1364 | Classification_binary: 0.2565 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1984 | Regression_eta: 0.0026 | Regression_pt: 0.2005 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0853 | Classification_binary: 0.2795 | Regression_cos_phi: 0.0938 | Regression_energy: 0.0901 | Regression_eta: 0.0539 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0761 | LR: 8.26e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.4643s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0069 | max_forward: 0.8745 | max_loss: 0.1717 | max_backward: 1.8485 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 1.4922 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6443.5 | uss_MB: 5682.8 | child_rss_MB: 5113.1 | allocated_MB: 344.2 | reserved_MB: 25374.0 | inactive_split_MB: 206.9 +rank3 [2026-10-28 21:34:34,947] INFO : Step 14600/50000 rank3 | Train Loss: 1.6485 | Train Loss Unweighted: 0.8413 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.2375 | Classification: 0.1249 | Classification_binary: 0.2902 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2104 | Regression_eta: 0.0023 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0894 | Classification_binary: 0.3295 | Regression_cos_phi: 0.1018 | Regression_energy: 0.1000 | Regression_eta: 0.0977 | Regression_pt: 0.1048 | Regression_sin_phi: 0.1316 | LR: 8.24e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.3662s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0068 | max_forward: 0.7866 | max_loss: 0.1515 | max_backward: 1.5469 | max_optimizer_step: 0.3128 | max_tensorboard_logging: 0.7882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6427.1 | uss_MB: 5683.8 | child_rss_MB: 5049.1 | allocated_MB: 345.7 | reserved_MB: 25374.0 | inactive_split_MB: 292.3 +rank3 [2026-10-28 21:38:39,703] INFO : Step 14700/50000 rank3 | Train Loss: 1.3645 | Train Loss Unweighted: 0.7326 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1507 | Classification: 0.1080 | Classification_binary: 0.2034 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0644 | Regression_energy: 0.1015 | Regression_eta: 0.0509 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0990 | LR: 8.21e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9098s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6571 | max_loss: 0.1494 | max_backward: 1.2978 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7751 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0922 | Memory: rss_MB: 6423.6 | uss_MB: 5684.9 | child_rss_MB: 5040.9 | allocated_MB: 341.1 | reserved_MB: 25374.0 | inactive_split_MB: 264.9 +rank3 [2026-10-28 21:42:53,741] INFO : Step 14800/50000 rank3 | Train Loss: 1.3987 | Train Loss Unweighted: 0.7335 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1827 | Classification: 0.1136 | Classification_binary: 0.2244 | HitGrouping: 0.2204 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1949 | Regression_eta: 0.0022 | Regression_pt: 0.1963 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.2469 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0840 | Regression_eta: 0.0697 | Regression_pt: 0.0876 | Regression_sin_phi: 0.1484 | LR: 8.19e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.2151s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.1679 | max_loss: 0.2171 | max_backward: 2.8051 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 6420.4 | uss_MB: 4434.9 | child_rss_MB: 5026.4 | allocated_MB: 345.2 | reserved_MB: 25374.0 | inactive_split_MB: 222.5 +rank3 [2026-10-28 21:47:06,021] INFO : Step 14900/50000 rank3 | Train Loss: 1.5046 | Train Loss Unweighted: 0.7991 | Train Loss Components: Auxiliary: 0.3055 | Cardinality: 0.1834 | Classification: 0.1311 | Classification_binary: 0.2643 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1982 | Regression_eta: 0.0023 | Regression_pt: 0.2008 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.2613 | Regression_cos_phi: 0.1122 | Regression_energy: 0.1104 | Regression_eta: 0.0591 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0974 | LR: 8.17e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.0339s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0071 | max_forward: 0.7642 | max_loss: 0.1561 | max_backward: 1.5975 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6421.6 | uss_MB: 5685.9 | child_rss_MB: 5030.2 | allocated_MB: 345.4 | reserved_MB: 25374.0 | inactive_split_MB: 289.7 +rank3 [2026-10-28 21:51:16,624] INFO : Step 15000/50000 rank3 | Train Loss: 1.3836 | Train Loss Unweighted: 0.7341 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1614 | Classification: 0.1182 | Classification_binary: 0.2252 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1930 | Regression_eta: 0.0023 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0841 | Regression_energy: 0.1086 | Regression_eta: 0.0449 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0625 | LR: 8.15e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.7138s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0066 | max_forward: 0.6828 | max_loss: 0.1453 | max_backward: 1.3064 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1064 | Memory: rss_MB: 6423.8 | uss_MB: 5686.3 | child_rss_MB: 5021.1 | allocated_MB: 342.6 | reserved_MB: 25374.0 | inactive_split_MB: 262.7 +rank3 [2026-10-28 21:51:17,568] INFO : Running validation on rank3 +rank3 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 21:52:06,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:52:07,026] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 21:52:33,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 21:52:34,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 21:52:35,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 21:52:36,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 21:52:46,457] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 21:52:46,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:52:47,272] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 21:53:03,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 21:53:04,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 21:53:05,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 21:53:06,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 21:53:16,200] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 21:53:16,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:53:17,339] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 21:53:32,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 21:53:32,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 21:53:33,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 21:53:34,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 21:53:44,324] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 21:57:58,526] INFO : Step 15100/50000 rank3 | Train Loss: 1.3706 | Train Loss Unweighted: 0.7159 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1535 | Classification: 0.1271 | Classification_binary: 0.1954 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1940 | Regression_eta: 0.0022 | Regression_pt: 0.1948 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0889 | Classification_binary: 0.1737 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0994 | Regression_eta: 0.0429 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0703 | LR: 8.12e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.0262s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.6658 | max_loss: 0.2401 | max_backward: 1.3105 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8277 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7654.5 | uss_MB: 6910.5 | child_rss_MB: 5047.2 | allocated_MB: 341.1 | reserved_MB: 22096.0 | inactive_split_MB: 215.5 +rank3 [2026-10-28 22:01:59,131] INFO : Step 15200/50000 rank3 | Train Loss: 1.5024 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3072 | Cardinality: 0.2080 | Classification: 0.1273 | Classification_binary: 0.2385 | HitGrouping: 0.2164 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1989 | Regression_eta: 0.0022 | Regression_pt: 0.2007 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.2312 | Regression_cos_phi: 0.1041 | Regression_energy: 0.0832 | Regression_eta: 0.0673 | Regression_pt: 0.0876 | Regression_sin_phi: 0.1125 | LR: 8.10e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.5456s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7502 | max_loss: 0.1534 | max_backward: 1.8002 | max_optimizer_step: 0.2270 | max_tensorboard_logging: 0.8682 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 7649.1 | uss_MB: 6910.3 | child_rss_MB: 5045.3 | allocated_MB: 345.6 | reserved_MB: 22420.0 | inactive_split_MB: 257.7 +rank3 [2026-10-28 22:05:42,232] INFO : Step 15300/50000 rank3 | Train Loss: 1.4920 | Train Loss Unweighted: 0.7880 | Train Loss Components: Auxiliary: 0.3065 | Cardinality: 0.1970 | Classification: 0.1161 | Classification_binary: 0.2589 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2026 | Regression_eta: 0.0022 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2708 | Regression_cos_phi: 0.0955 | Regression_energy: 0.0894 | Regression_eta: 0.0690 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0916 | LR: 8.08e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5775s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0192 | max_forward: 0.7642 | max_loss: 0.1530 | max_backward: 1.6292 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 0.9125 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6524.0 | uss_MB: 5797.3 | child_rss_MB: 5044.3 | allocated_MB: 341.2 | reserved_MB: 23096.0 | inactive_split_MB: 271.7 +rank3 [2026-10-28 22:09:28,150] INFO : Step 15400/50000 rank3 | Train Loss: 1.3549 | Train Loss Unweighted: 0.7210 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1572 | Classification: 0.1148 | Classification_binary: 0.2052 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1983 | Regression_eta: 0.0022 | Regression_pt: 0.1984 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0910 | Regression_eta: 0.0447 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0729 | LR: 8.05e-04 | DataLoad Time: 0.0223s | Model Forward Time: 3.7020s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6594 | max_loss: 0.2185 | max_backward: 1.2416 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6511.0 | uss_MB: 5790.6 | child_rss_MB: 5047.2 | allocated_MB: 340.8 | reserved_MB: 23394.0 | inactive_split_MB: 166.6 +rank3 [2026-10-28 22:13:12,490] INFO : Step 15500/50000 rank3 | Train Loss: 1.4313 | Train Loss Unweighted: 0.7527 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1666 | Classification: 0.1321 | Classification_binary: 0.2182 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1979 | Regression_eta: 0.0025 | Regression_pt: 0.1996 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0802 | Classification_binary: 0.2293 | Regression_cos_phi: 0.1084 | Regression_energy: 0.0993 | Regression_eta: 0.0847 | Regression_pt: 0.1027 | Regression_sin_phi: 0.1284 | LR: 8.03e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2542s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0792 | max_forward: 0.7874 | max_loss: 0.1514 | max_backward: 1.8676 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5380 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6511.0 | uss_MB: 5805.3 | child_rss_MB: 5052.2 | allocated_MB: 345.6 | reserved_MB: 24092.0 | inactive_split_MB: 260.5 +rank3 [2026-10-28 22:17:01,248] INFO : Step 15600/50000 rank3 | Train Loss: 1.5305 | Train Loss Unweighted: 0.8109 | Train Loss Components: Auxiliary: 0.3137 | Cardinality: 0.1790 | Classification: 0.1304 | Classification_binary: 0.2721 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2011 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0857 | Classification_binary: 0.2590 | Regression_cos_phi: 0.1085 | Regression_energy: 0.0977 | Regression_eta: 0.0822 | Regression_pt: 0.1015 | Regression_sin_phi: 0.1212 | LR: 8.01e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2096s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7645 | max_loss: 0.2230 | max_backward: 1.7015 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1398 | Memory: rss_MB: 6508.8 | uss_MB: 5803.4 | child_rss_MB: 5052.3 | allocated_MB: 344.1 | reserved_MB: 24402.0 | inactive_split_MB: 287.9 +rank3 [2026-10-28 22:20:46,546] INFO : Step 15700/50000 rank3 | Train Loss: 1.4083 | Train Loss Unweighted: 0.7205 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1775 | Classification: 0.1238 | Classification_binary: 0.2322 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1785 | Regression_eta: 0.0024 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.2405 | Regression_cos_phi: 0.0984 | Regression_energy: 0.0732 | Regression_eta: 0.0608 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0815 | LR: 7.98e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.0308s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2392 | max_backward: 1.6606 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6506.8 | uss_MB: 5802.2 | child_rss_MB: 5055.5 | allocated_MB: 344.4 | reserved_MB: 25380.0 | inactive_split_MB: 228.2 +rank3 [2026-10-28 22:24:39,044] INFO : Step 15800/50000 rank3 | Train Loss: 1.2988 | Train Loss Unweighted: 0.6968 | Train Loss Components: Auxiliary: 0.2671 | Cardinality: 0.1436 | Classification: 0.1112 | Classification_binary: 0.1916 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1933 | Regression_eta: 0.0020 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1612 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0851 | Regression_eta: 0.0504 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0689 | LR: 7.96e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7276s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6109 | max_loss: 0.2015 | max_backward: 1.1291 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6506.5 | uss_MB: 5803.6 | child_rss_MB: 5051.3 | allocated_MB: 341.6 | reserved_MB: 25380.0 | inactive_split_MB: 263.9 +rank3 [2026-10-28 22:28:35,066] INFO : Step 15900/50000 rank3 | Train Loss: 1.4020 | Train Loss Unweighted: 0.7322 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1800 | Classification: 0.1295 | Classification_binary: 0.2346 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1816 | Regression_eta: 0.0023 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.2385 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0844 | Regression_eta: 0.0560 | Regression_pt: 0.0870 | Regression_sin_phi: 0.1007 | LR: 7.94e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.2706s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0933 | max_forward: 0.7823 | max_loss: 0.1445 | max_backward: 1.4740 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6508.5 | uss_MB: 5807.0 | child_rss_MB: 5047.2 | allocated_MB: 342.9 | reserved_MB: 25380.0 | inactive_split_MB: 283.5 +rank3 [2026-10-28 22:32:40,676] INFO : Step 16000/50000 rank3 | Train Loss: 1.4978 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3126 | Cardinality: 0.2093 | Classification: 0.1190 | Classification_binary: 0.2580 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1920 | Regression_eta: 0.0021 | Regression_pt: 0.1969 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0769 | Classification_binary: 0.2429 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0946 | Regression_eta: 0.0588 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0971 | LR: 7.91e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4906s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.7474 | max_loss: 0.1892 | max_backward: 1.4114 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.7981 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6509.3 | uss_MB: 5675.5 | child_rss_MB: 5048.3 | allocated_MB: 342.7 | reserved_MB: 25380.0 | inactive_split_MB: 292.9 +rank3 [2026-10-28 22:32:41,702] INFO : Running validation on rank3 +rank3 [2026-10-28 22:32:41,702] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 22:32:41,702] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 22:33:34,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:33:35,151] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 22:34:00,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 22:34:02,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 22:34:03,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 22:34:03,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 22:34:14,004] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 22:34:14,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:34:14,938] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 22:34:30,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 22:34:31,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 22:34:32,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 22:34:33,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 22:34:43,470] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 22:34:44,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:34:44,852] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 22:35:00,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 22:35:00,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 22:35:01,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 22:35:02,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 22:35:12,161] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 22:39:22,135] INFO : Step 16100/50000 rank3 | Train Loss: 1.3441 | Train Loss Unweighted: 0.7192 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1631 | Classification: 0.1197 | Classification_binary: 0.2058 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1931 | Regression_eta: 0.0020 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0836 | Regression_eta: 0.0680 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0748 | LR: 7.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1522s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0584 | max_forward: 0.7257 | max_loss: 0.1421 | max_backward: 1.2590 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.0197 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6577.1 | uss_MB: 5871.7 | child_rss_MB: 5053.6 | allocated_MB: 341.3 | reserved_MB: 23342.0 | inactive_split_MB: 221.6 +rank3 [2026-10-28 22:43:09,259] INFO : Step 16200/50000 rank3 | Train Loss: 1.3737 | Train Loss Unweighted: 0.7229 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1695 | Classification: 0.1213 | Classification_binary: 0.2167 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2373 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0947 | Regression_eta: 0.0522 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0871 | LR: 7.86e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0515s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.1062 | max_forward: 0.7366 | max_loss: 0.1469 | max_backward: 1.4999 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.6015 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6580.7 | uss_MB: 5882.6 | child_rss_MB: 5053.8 | allocated_MB: 342.5 | reserved_MB: 24304.0 | inactive_split_MB: 205.9 +rank3 [2026-10-28 22:46:54,662] INFO : Step 16300/50000 rank3 | Train Loss: 1.4671 | Train Loss Unweighted: 0.7592 | Train Loss Components: Auxiliary: 0.3008 | Cardinality: 0.1775 | Classification: 0.1342 | Classification_binary: 0.2702 | HitGrouping: 0.2148 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0023 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2807 | Regression_cos_phi: 0.0876 | Regression_energy: 0.0714 | Regression_eta: 0.0464 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0698 | LR: 7.84e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.4612s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7718 | max_loss: 0.1628 | max_backward: 1.7017 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.8513 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6574.7 | uss_MB: 5880.8 | child_rss_MB: 5053.1 | allocated_MB: 345.8 | reserved_MB: 24304.0 | inactive_split_MB: 281.0 +rank3 [2026-10-28 22:50:39,547] INFO : Step 16400/50000 rank3 | Train Loss: 1.4200 | Train Loss Unweighted: 0.7161 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.2086 | Classification: 0.1206 | Classification_binary: 0.2158 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0024 | Regression_pt: 0.1885 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.2037 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0893 | Regression_eta: 0.0487 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0744 | LR: 7.81e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.8025s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7570 | max_loss: 0.1546 | max_backward: 1.2568 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5199 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1436 | Memory: rss_MB: 6571.7 | uss_MB: 5879.8 | child_rss_MB: 5049.7 | allocated_MB: 342.7 | reserved_MB: 25268.0 | inactive_split_MB: 213.6 +rank3 [2026-10-28 22:54:25,074] INFO : Step 16500/50000 rank3 | Train Loss: 1.2201 | Train Loss Unweighted: 0.6592 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1195 | Classification: 0.1029 | Classification_binary: 0.1835 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1836 | Regression_eta: 0.0021 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1978 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0881 | Regression_eta: 0.0836 | Regression_pt: 0.0929 | Regression_sin_phi: 0.1107 | LR: 7.79e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.3259s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0128 | max_forward: 0.7866 | max_loss: 0.2200 | max_backward: 1.9590 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.4913 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6572.2 | uss_MB: 5885.7 | child_rss_MB: 5046.8 | allocated_MB: 343.6 | reserved_MB: 26238.0 | inactive_split_MB: 248.8 +rank3 [2026-10-28 22:58:25,288] INFO : Step 16600/50000 rank3 | Train Loss: 1.4335 | Train Loss Unweighted: 0.7657 | Train Loss Components: Auxiliary: 0.2949 | Cardinality: 0.1698 | Classification: 0.1386 | Classification_binary: 0.2312 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1941 | Regression_eta: 0.0021 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0981 | Classification_binary: 0.2534 | Regression_cos_phi: 0.1365 | Regression_energy: 0.1087 | Regression_eta: 0.0776 | Regression_pt: 0.1146 | Regression_sin_phi: 0.1007 | LR: 7.77e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.9964s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0139 | max_forward: 1.0329 | max_loss: 0.3279 | max_backward: 2.5489 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.5244 | max_tensorboard_flush: 0.0014 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6575.3 | uss_MB: 4925.2 | child_rss_MB: 5045.0 | allocated_MB: 344.2 | reserved_MB: 27228.0 | inactive_split_MB: 219.6 +rank3 [2026-10-28 23:02:38,763] INFO : Step 16700/50000 rank3 | Train Loss: 1.4062 | Train Loss Unweighted: 0.7378 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1840 | Classification: 0.1203 | Classification_binary: 0.2439 | HitGrouping: 0.2108 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1839 | Regression_eta: 0.0020 | Regression_pt: 0.1855 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2450 | Regression_cos_phi: 0.0947 | Regression_energy: 0.0886 | Regression_eta: 0.0531 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0713 | LR: 7.74e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.0479s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7336 | max_loss: 0.2460 | max_backward: 1.6747 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6577.4 | uss_MB: 5888.0 | child_rss_MB: 5049.7 | allocated_MB: 344.6 | reserved_MB: 27228.0 | inactive_split_MB: 279.4 +rank3 [2026-10-28 23:06:47,051] INFO : Step 16800/50000 rank3 | Train Loss: 1.3172 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1533 | Classification: 0.1173 | Classification_binary: 0.1949 | HitGrouping: 0.2175 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1819 | Regression_eta: 0.0027 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2018 | Regression_cos_phi: 0.0969 | Regression_energy: 0.0843 | Regression_eta: 0.1132 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0761 | LR: 7.72e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.6835s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0962 | max_forward: 0.7772 | max_loss: 0.1423 | max_backward: 1.3121 | max_optimizer_step: 0.2117 | max_tensorboard_logging: 0.4865 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6580.1 | uss_MB: 5886.7 | child_rss_MB: 5043.1 | allocated_MB: 342.1 | reserved_MB: 27250.0 | inactive_split_MB: 200.8 +rank3 [2026-10-28 23:11:05,116] INFO : Step 16900/50000 rank3 | Train Loss: 1.3132 | Train Loss Unweighted: 0.6868 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1474 | Classification: 0.1260 | Classification_binary: 0.1988 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1779 | Regression_eta: 0.0021 | Regression_pt: 0.1798 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.1971 | Regression_cos_phi: 0.0917 | Regression_energy: 0.0839 | Regression_eta: 0.0563 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0942 | LR: 7.69e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.7607s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7203 | max_loss: 0.1660 | max_backward: 1.1458 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6581.3 | uss_MB: 5889.6 | child_rss_MB: 5044.4 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 193.9 +rank3 [2026-10-28 23:15:09,202] INFO : Step 17000/50000 rank3 | Train Loss: 1.3575 | Train Loss Unweighted: 0.7115 | Train Loss Components: Auxiliary: 0.2906 | Cardinality: 0.1781 | Classification: 0.1179 | Classification_binary: 0.2325 | HitGrouping: 0.2089 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0018 | Regression_pt: 0.1792 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2441 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0891 | Regression_eta: 0.0543 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0701 | LR: 7.67e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4168s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.9008 | max_loss: 0.1474 | max_backward: 1.9229 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6575.6 | uss_MB: 5894.0 | child_rss_MB: 5025.6 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 278.9 +rank3 [2026-10-28 23:15:10,040] INFO : Running validation on rank3 +rank3 [2026-10-28 23:15:10,040] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 23:16:00,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:16:01,059] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 23:16:27,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 23:16:28,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 23:16:29,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 23:16:30,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 23:16:41,010] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 23:16:42,017] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:16:42,520] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 23:16:58,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 23:16:59,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 23:17:00,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 23:17:01,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 23:17:11,188] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 23:17:11,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:17:12,025] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 23:17:27,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 23:17:28,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 23:17:28,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 23:17:29,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 23:17:39,481] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-28 23:21:59,503] INFO : Step 17100/50000 rank3 | Train Loss: 1.3790 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1704 | Classification: 0.1078 | Classification_binary: 0.2095 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1968 | Regression_eta: 0.0020 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1879 | Regression_cos_phi: 0.0972 | Regression_energy: 0.0839 | Regression_eta: 0.0489 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0518 | LR: 7.64e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.9066s | Phase Times: max_pre_step_barrier: 0.0234 | max_batch_to_device: 0.0065 | max_forward: 0.8292 | max_loss: 0.1455 | max_backward: 1.4833 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6666.8 | uss_MB: 5307.9 | child_rss_MB: 5009.6 | allocated_MB: 343.8 | reserved_MB: 22688.0 | inactive_split_MB: 254.6 +rank3 [2026-10-28 23:26:06,863] INFO : Step 17200/50000 rank3 | Train Loss: 1.2786 | Train Loss Unweighted: 0.6598 | Train Loss Components: Auxiliary: 0.2576 | Cardinality: 0.1378 | Classification: 0.1161 | Classification_binary: 0.1910 | HitGrouping: 0.2191 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0025 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0845 | Classification_binary: 0.2191 | Regression_cos_phi: 0.1167 | Regression_energy: 0.0804 | Regression_eta: 0.1042 | Regression_pt: 0.0819 | Regression_sin_phi: 0.1362 | LR: 7.62e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8627s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6470 | max_loss: 0.2505 | max_backward: 1.4380 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6320 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6664.1 | uss_MB: 5982.7 | child_rss_MB: 5028.0 | allocated_MB: 344.1 | reserved_MB: 22688.0 | inactive_split_MB: 295.9 +rank3 [2026-10-28 23:30:05,520] INFO : Step 17300/50000 rank3 | Train Loss: 1.3571 | Train Loss Unweighted: 0.6976 | Train Loss Components: Auxiliary: 0.2789 | Cardinality: 0.1665 | Classification: 0.1193 | Classification_binary: 0.2287 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1724 | Regression_eta: 0.0021 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2392 | Regression_cos_phi: 0.1024 | Regression_energy: 0.0692 | Regression_eta: 0.0593 | Regression_pt: 0.0721 | Regression_sin_phi: 0.1154 | LR: 7.59e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5420s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7276 | max_loss: 0.2520 | max_backward: 1.8579 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8525 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6666.1 | uss_MB: 5121.5 | child_rss_MB: 5026.9 | allocated_MB: 344.2 | reserved_MB: 23616.0 | inactive_split_MB: 266.5 +rank3 [2026-10-28 23:34:25,425] INFO : Step 17400/50000 rank3 | Train Loss: 1.3668 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2811 | Cardinality: 0.1760 | Classification: 0.1195 | Classification_binary: 0.2351 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1732 | Regression_eta: 0.0019 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2482 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0806 | Regression_eta: 0.0543 | Regression_pt: 0.0863 | Regression_sin_phi: 0.0722 | LR: 7.57e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4909s | Phase Times: max_pre_step_barrier: 0.1038 | max_batch_to_device: 0.0306 | max_forward: 0.7782 | max_loss: 0.2429 | max_backward: 1.3743 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.8348 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0757 | Memory: rss_MB: 6667.1 | uss_MB: 5984.6 | child_rss_MB: 5026.4 | allocated_MB: 343.3 | reserved_MB: 24604.0 | inactive_split_MB: 262.4 +rank3 [2026-10-28 23:38:51,400] INFO : Step 17500/50000 rank3 | Train Loss: 1.3114 | Train Loss Unweighted: 0.6526 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.1481 | Classification: 0.1145 | Classification_binary: 0.1884 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1714 | Regression_eta: 0.0024 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1952 | Regression_cos_phi: 0.1051 | Regression_energy: 0.0852 | Regression_eta: 0.0516 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0815 | LR: 7.54e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.9152s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.6394 | max_loss: 0.1427 | max_backward: 1.2079 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.8375 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 6667.1 | uss_MB: 5986.2 | child_rss_MB: 5023.0 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 240.0 +rank3 [2026-10-28 23:43:03,145] INFO : Step 17600/50000 rank3 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2688 | Cardinality: 0.1362 | Classification: 0.1273 | Classification_binary: 0.2065 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0019 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0731 | Regression_eta: 0.0557 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0899 | LR: 7.52e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6101s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.1120 | max_forward: 1.2883 | max_loss: 0.2624 | max_backward: 1.5380 | max_optimizer_step: 0.2147 | max_tensorboard_logging: 0.7644 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 6665.9 | uss_MB: 5985.5 | child_rss_MB: 5028.5 | allocated_MB: 344.2 | reserved_MB: 25596.0 | inactive_split_MB: 237.9 +rank3 [2026-10-28 23:47:02,149] INFO : Step 17700/50000 rank3 | Train Loss: 1.3491 | Train Loss Unweighted: 0.6939 | Train Loss Components: Auxiliary: 0.2835 | Cardinality: 0.1622 | Classification: 0.1186 | Classification_binary: 0.2310 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0022 | Regression_pt: 0.1701 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.2539 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0750 | Regression_eta: 0.0514 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0692 | LR: 7.49e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6561s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0704 | max_forward: 0.9841 | max_loss: 0.1994 | max_backward: 1.8911 | max_optimizer_step: 0.2227 | max_tensorboard_logging: 0.5849 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6667.9 | uss_MB: 5989.0 | child_rss_MB: 5031.2 | allocated_MB: 345.5 | reserved_MB: 25596.0 | inactive_split_MB: 287.6 +rank3 [2026-10-28 23:51:17,533] INFO : Step 17800/50000 rank3 | Train Loss: 1.3063 | Train Loss Unweighted: 0.6692 | Train Loss Components: Auxiliary: 0.2664 | Cardinality: 0.1579 | Classification: 0.1042 | Classification_binary: 0.2130 | HitGrouping: 0.2150 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1741 | Regression_eta: 0.0019 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0847 | Classification_binary: 0.2310 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0819 | Regression_eta: 0.0670 | Regression_pt: 0.0850 | Regression_sin_phi: 0.1125 | LR: 7.47e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8693s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.7351 | max_loss: 0.1686 | max_backward: 1.4616 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5023 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 6666.0 | uss_MB: 5988.8 | child_rss_MB: 5032.9 | allocated_MB: 341.0 | reserved_MB: 25596.0 | inactive_split_MB: 197.8 +rank3 [2026-10-28 23:55:22,790] INFO : Step 17900/50000 rank3 | Train Loss: 1.2213 | Train Loss Unweighted: 0.6433 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1427 | Classification: 0.1147 | Classification_binary: 0.1725 | HitGrouping: 0.2070 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0946 | Classification_binary: 0.1808 | Regression_cos_phi: 0.1148 | Regression_energy: 0.0833 | Regression_eta: 0.0682 | Regression_pt: 0.0868 | Regression_sin_phi: 0.1201 | LR: 7.44e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8170s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7249 | max_loss: 0.1579 | max_backward: 1.1981 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6654.3 | uss_MB: 5990.9 | child_rss_MB: 5033.3 | allocated_MB: 341.6 | reserved_MB: 25596.0 | inactive_split_MB: 297.9 +rank3 [2026-10-28 23:59:20,281] INFO : Step 18000/50000 rank3 | Train Loss: 1.3597 | Train Loss Unweighted: 0.7083 | Train Loss Components: Auxiliary: 0.2781 | Cardinality: 0.1599 | Classification: 0.1247 | Classification_binary: 0.2275 | HitGrouping: 0.2122 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0021 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.1917 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0798 | Regression_eta: 0.0589 | Regression_pt: 0.0843 | Regression_sin_phi: 0.1099 | LR: 7.42e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.6432s | Phase Times: max_pre_step_barrier: 0.0816 | max_batch_to_device: 0.0068 | max_forward: 0.8980 | max_loss: 0.1586 | max_backward: 1.9185 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.7546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6654.3 | uss_MB: 5995.0 | child_rss_MB: 5035.3 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 254.8 +rank3 [2026-10-28 23:59:21,238] INFO : Running validation on rank3 +rank3 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 00:00:12,212] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:00:12,738] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 00:00:39,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 00:00:40,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 00:00:41,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 00:00:42,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 00:00:52,365] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 00:00:52,802] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:00:53,360] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 00:01:09,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 00:01:10,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 00:01:11,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 00:01:12,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 00:01:22,116] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 00:01:22,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:01:23,290] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 00:01:38,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 00:01:39,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 00:01:40,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 00:01:40,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 00:01:50,888] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 00:05:57,253] INFO : Step 18100/50000 rank3 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7216 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1666 | Classification: 0.1199 | Classification_binary: 0.2340 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1807 | Regression_eta: 0.0024 | Regression_pt: 0.1824 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2492 | Regression_cos_phi: 0.0936 | Regression_energy: 0.0800 | Regression_eta: 0.0708 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0883 | LR: 7.39e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5266s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.9127 | max_loss: 0.4546 | max_backward: 1.7406 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.7463 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7789.9 | uss_MB: 7130.3 | child_rss_MB: 5031.5 | allocated_MB: 344.6 | reserved_MB: 22256.0 | inactive_split_MB: 291.4 +rank3 [2026-10-29 00:09:47,528] INFO : Step 18200/50000 rank3 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6468 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1359 | Classification: 0.1208 | Classification_binary: 0.1803 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1700 | Regression_eta: 0.0020 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1806 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0819 | Regression_eta: 0.0377 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0686 | LR: 7.36e-04 | DataLoad Time: 0.0148s | Model Forward Time: 3.7610s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7114 | max_loss: 0.2369 | max_backward: 1.1051 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6784.1 | uss_MB: 6119.0 | child_rss_MB: 5031.9 | allocated_MB: 341.2 | reserved_MB: 23772.0 | inactive_split_MB: 170.7 +rank3 [2026-10-29 00:13:32,738] INFO : Step 18300/50000 rank3 | Train Loss: 1.3100 | Train Loss Unweighted: 0.6891 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1522 | Classification: 0.1230 | Classification_binary: 0.1938 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1834 | Regression_eta: 0.0019 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0896 | Regression_eta: 0.0435 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0784 | LR: 7.34e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1455s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7219 | max_loss: 0.1491 | max_backward: 1.5825 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.7390 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 6786.1 | uss_MB: 6124.1 | child_rss_MB: 5031.6 | allocated_MB: 344.4 | reserved_MB: 24778.0 | inactive_split_MB: 234.4 +rank3 [2026-10-29 00:17:16,845] INFO : Step 18400/50000 rank3 | Train Loss: 1.3667 | Train Loss Unweighted: 0.7337 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1448 | Classification: 0.1277 | Classification_binary: 0.2393 | HitGrouping: 0.2095 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1805 | Regression_eta: 0.0022 | Regression_pt: 0.1818 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.2221 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0870 | Regression_eta: 0.0885 | Regression_pt: 0.0906 | Regression_sin_phi: 0.1137 | LR: 7.31e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4857s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1744 | max_backward: 1.7308 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6782.0 | uss_MB: 6123.4 | child_rss_MB: 5027.3 | allocated_MB: 344.6 | reserved_MB: 24778.0 | inactive_split_MB: 281.4 +rank3 [2026-10-29 00:21:03,776] INFO : Step 18500/50000 rank3 | Train Loss: 1.2528 | Train Loss Unweighted: 0.6816 | Train Loss Components: Auxiliary: 0.2631 | Cardinality: 0.1350 | Classification: 0.1140 | Classification_binary: 0.1973 | HitGrouping: 0.2066 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1821 | Regression_eta: 0.0017 | Regression_pt: 0.1847 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0846 | Classification_binary: 0.2191 | Regression_cos_phi: 0.1192 | Regression_energy: 0.0964 | Regression_eta: 0.0677 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0973 | LR: 7.29e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2641s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0113 | max_forward: 0.8281 | max_loss: 0.2418 | max_backward: 1.4683 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7561 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1431 | Memory: rss_MB: 6783.0 | uss_MB: 6122.8 | child_rss_MB: 5026.7 | allocated_MB: 343.3 | reserved_MB: 24778.0 | inactive_split_MB: 226.5 +rank3 [2026-10-29 00:24:44,690] INFO : Step 18600/50000 rank3 | Train Loss: 1.1881 | Train Loss Unweighted: 0.6335 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1331 | Classification: 0.1084 | Classification_binary: 0.1723 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1746 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1594 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0872 | Regression_eta: 0.0386 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0604 | LR: 7.26e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8518s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7299 | max_loss: 0.2279 | max_backward: 1.0799 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8080 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6782.4 | uss_MB: 6120.8 | child_rss_MB: 5022.4 | allocated_MB: 341.0 | reserved_MB: 24778.0 | inactive_split_MB: 229.3 +rank3 [2026-10-29 00:28:24,928] INFO : Step 18700/50000 rank3 | Train Loss: 1.3122 | Train Loss Unweighted: 0.6534 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1756 | Classification: 0.1225 | Classification_binary: 0.2098 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1571 | Regression_eta: 0.0020 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.1965 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0683 | Regression_eta: 0.0327 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0653 | LR: 7.24e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6245s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9129 | max_loss: 0.1606 | max_backward: 1.9397 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6785.3 | uss_MB: 6128.9 | child_rss_MB: 5021.8 | allocated_MB: 344.8 | reserved_MB: 24778.0 | inactive_split_MB: 239.8 +rank3 [2026-10-29 00:32:05,525] INFO : Step 18800/50000 rank3 | Train Loss: 1.2696 | Train Loss Unweighted: 0.6643 | Train Loss Components: Auxiliary: 0.2674 | Cardinality: 0.1602 | Classification: 0.1070 | Classification_binary: 0.2200 | HitGrouping: 0.2058 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1652 | Regression_eta: 0.0018 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2190 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0909 | Regression_eta: 0.0716 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0749 | LR: 7.21e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6466s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7444 | max_loss: 0.2527 | max_backward: 1.7167 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.9324 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 6785.1 | uss_MB: 6124.1 | child_rss_MB: 5021.8 | allocated_MB: 345.4 | reserved_MB: 25044.0 | inactive_split_MB: 284.4 +rank3 [2026-10-29 00:35:49,544] INFO : Step 18900/50000 rank3 | Train Loss: 1.1718 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1382 | Classification: 0.0996 | Classification_binary: 0.1741 | HitGrouping: 0.2036 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1652 | Regression_eta: 0.0017 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0877 | Regression_eta: 0.0418 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0747 | LR: 7.18e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0756s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0056 | max_forward: 0.6830 | max_loss: 0.1421 | max_backward: 1.2812 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.8931 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6786.1 | uss_MB: 6126.4 | child_rss_MB: 5006.2 | allocated_MB: 338.7 | reserved_MB: 25044.0 | inactive_split_MB: 232.6 +rank3 [2026-10-29 00:39:31,233] INFO : Step 19000/50000 rank3 | Train Loss: 1.2397 | Train Loss Unweighted: 0.6317 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1277 | Classification: 0.1188 | Classification_binary: 0.1866 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1600 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0715 | Regression_eta: 0.0552 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0524 | LR: 7.16e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2157s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6593 | max_loss: 0.2467 | max_backward: 1.6203 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7732 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6787.7 | uss_MB: 6128.3 | child_rss_MB: 5026.8 | allocated_MB: 342.2 | reserved_MB: 25790.0 | inactive_split_MB: 229.6 +rank3 [2026-10-29 00:39:32,174] INFO : Running validation on rank3 +rank3 [2026-10-29 00:39:32,174] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 00:39:32,174] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 00:40:23,808] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:40:24,368] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 00:40:50,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 00:40:51,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 00:40:52,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 00:40:53,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 00:41:03,113] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 00:41:03,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:41:04,089] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 00:41:19,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 00:41:20,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 00:41:21,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 00:41:22,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 00:41:32,837] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 00:41:33,291] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:41:33,791] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 00:41:49,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 00:41:50,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 00:41:50,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 00:41:51,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 00:42:01,437] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 00:46:01,995] INFO : Step 19100/50000 rank3 | Train Loss: 1.3562 | Train Loss Unweighted: 0.6933 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1644 | Classification: 0.1204 | Classification_binary: 0.2426 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1627 | Regression_eta: 0.0023 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2250 | Regression_cos_phi: 0.0933 | Regression_energy: 0.0849 | Regression_eta: 0.0609 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0814 | LR: 7.13e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.5896s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7796 | max_loss: 0.1634 | max_backward: 1.6942 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.8232 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 8027.6 | uss_MB: 7365.9 | child_rss_MB: 5027.2 | allocated_MB: 345.2 | reserved_MB: 23900.0 | inactive_split_MB: 240.8 +rank3 [2026-10-29 00:49:48,721] INFO : Step 19200/50000 rank3 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5946 | Train Loss Components: Auxiliary: 0.2384 | Cardinality: 0.1391 | Classification: 0.0965 | Classification_binary: 0.1984 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1472 | Regression_eta: 0.0018 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2233 | Regression_cos_phi: 0.0955 | Regression_energy: 0.0766 | Regression_eta: 0.0601 | Regression_pt: 0.0795 | Regression_sin_phi: 0.1175 | LR: 7.10e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9568s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7138 | max_loss: 0.2391 | max_backward: 1.2379 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.7395 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6876.1 | uss_MB: 6216.1 | child_rss_MB: 5025.0 | allocated_MB: 341.3 | reserved_MB: 23900.0 | inactive_split_MB: 206.8 +rank3 [2026-10-29 00:53:30,059] INFO : Step 19300/50000 rank3 | Train Loss: 1.1768 | Train Loss Unweighted: 0.6083 | Train Loss Components: Auxiliary: 0.2422 | Cardinality: 0.1345 | Classification: 0.1035 | Classification_binary: 0.1709 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0018 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.1721 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0724 | Regression_eta: 0.0460 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0528 | LR: 7.08e-04 | DataLoad Time: 0.0218s | Model Forward Time: 3.8954s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0503 | max_forward: 0.7163 | max_loss: 0.1442 | max_backward: 1.3745 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6878.1 | uss_MB: 6217.4 | child_rss_MB: 5020.2 | allocated_MB: 341.0 | reserved_MB: 23900.0 | inactive_split_MB: 218.1 +rank3 [2026-10-29 00:57:11,120] INFO : Step 19400/50000 rank3 | Train Loss: 1.4590 | Train Loss Unweighted: 0.7109 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.2328 | Classification: 0.1237 | Classification_binary: 0.2265 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1770 | Regression_eta: 0.0021 | Regression_pt: 0.1794 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.2344 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0891 | Regression_eta: 0.0685 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0744 | LR: 7.05e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.4236s | Phase Times: max_pre_step_barrier: 0.1118 | max_batch_to_device: 0.0070 | max_forward: 0.7578 | max_loss: 0.2459 | max_backward: 1.7975 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6876.3 | uss_MB: 6217.1 | child_rss_MB: 5016.1 | allocated_MB: 347.7 | reserved_MB: 24186.0 | inactive_split_MB: 276.3 +rank3 [2026-10-29 01:00:48,549] INFO : Step 19500/50000 rank3 | Train Loss: 1.3753 | Train Loss Unweighted: 0.6794 | Train Loss Components: Auxiliary: 0.2810 | Cardinality: 0.1865 | Classification: 0.1153 | Classification_binary: 0.2310 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1637 | Regression_eta: 0.0023 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.2542 | Regression_cos_phi: 0.0975 | Regression_energy: 0.0841 | Regression_eta: 0.0563 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0820 | LR: 7.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8972s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7197 | max_loss: 0.2164 | max_backward: 1.3937 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.4987 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6876.3 | uss_MB: 6219.5 | child_rss_MB: 5017.7 | allocated_MB: 345.2 | reserved_MB: 24186.0 | inactive_split_MB: 275.2 +rank3 [2026-10-29 01:04:42,199] INFO : Step 19600/50000 rank3 | Train Loss: 1.2086 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1557 | Classification: 0.0942 | Classification_binary: 0.1768 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1735 | Regression_eta: 0.0015 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0913 | Regression_eta: 0.0376 | Regression_pt: 0.0952 | Regression_sin_phi: 0.0620 | LR: 7.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.7332s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6937 | max_loss: 0.1942 | max_backward: 1.2727 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6877.2 | uss_MB: 6219.0 | child_rss_MB: 5004.9 | allocated_MB: 340.6 | reserved_MB: 24186.0 | inactive_split_MB: 277.9 +rank3 [2026-10-29 01:08:29,168] INFO : Step 19700/50000 rank3 | Train Loss: 1.2115 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1478 | Classification: 0.1094 | Classification_binary: 0.1919 | HitGrouping: 0.2027 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.2059 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0712 | Regression_eta: 0.0515 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0678 | LR: 6.97e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.8681s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0914 | max_forward: 0.7091 | max_loss: 0.1429 | max_backward: 1.4681 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0652 | Memory: rss_MB: 6879.2 | uss_MB: 6222.3 | child_rss_MB: 5016.1 | allocated_MB: 344.0 | reserved_MB: 24630.0 | inactive_split_MB: 143.8 +rank3 [2026-10-29 01:12:36,832] INFO : Step 19800/50000 rank3 | Train Loss: 1.2919 | Train Loss Unweighted: 0.6584 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1453 | Classification: 0.1132 | Classification_binary: 0.2375 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0020 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2728 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0765 | Regression_eta: 0.0805 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0978 | LR: 6.94e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4552s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0074 | max_forward: 0.7828 | max_loss: 0.2508 | max_backward: 1.8604 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6879.1 | uss_MB: 6223.6 | child_rss_MB: 5015.7 | allocated_MB: 348.0 | reserved_MB: 25634.0 | inactive_split_MB: 277.9 +rank3 [2026-10-29 01:16:27,497] INFO : Step 19900/50000 rank3 | Train Loss: 1.2010 | Train Loss Unweighted: 0.6167 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1346 | Classification: 0.0885 | Classification_binary: 0.1961 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1633 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1881 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0721 | Regression_eta: 0.0518 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0583 | LR: 6.92e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9734s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6671 | max_loss: 0.2548 | max_backward: 1.5862 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6881.1 | uss_MB: 6224.6 | child_rss_MB: 5016.9 | allocated_MB: 340.3 | reserved_MB: 25634.0 | inactive_split_MB: 201.7 +rank3 [2026-10-29 01:20:16,835] INFO : Step 20000/50000 rank3 | Train Loss: 1.2077 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1487 | Classification: 0.1030 | Classification_binary: 0.1729 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1639 | Regression_eta: 0.0019 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1815 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0712 | Regression_eta: 0.0549 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0768 | LR: 6.89e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8147s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.6570 | max_loss: 0.1550 | max_backward: 1.4848 | max_optimizer_step: 0.2205 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6884.1 | uss_MB: 6226.8 | child_rss_MB: 5009.1 | allocated_MB: 341.8 | reserved_MB: 26642.0 | inactive_split_MB: 237.7 +rank3 [2026-10-29 01:20:17,835] INFO : Running validation on rank3 +rank3 [2026-10-29 01:20:17,835] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 01:20:17,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 01:21:07,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:21:08,251] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 01:21:34,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 01:21:35,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 01:21:36,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 01:21:37,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 01:21:47,824] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 01:21:48,205] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:21:48,733] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 01:22:04,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 01:22:05,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 01:22:06,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 01:22:07,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 01:22:17,260] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 01:22:18,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:22:18,871] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 01:22:34,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 01:22:35,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 01:22:35,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 01:22:36,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 01:22:46,425] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 01:26:57,040] INFO : Step 20100/50000 rank3 | Train Loss: 1.3825 | Train Loss Unweighted: 0.6739 | Train Loss Components: Auxiliary: 0.2830 | Cardinality: 0.1854 | Classification: 0.1189 | Classification_binary: 0.2349 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1561 | Regression_eta: 0.0022 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0799 | Classification_binary: 0.2334 | Regression_cos_phi: 0.1132 | Regression_energy: 0.0648 | Regression_eta: 0.0847 | Regression_pt: 0.0683 | Regression_sin_phi: 0.1271 | LR: 6.86e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2451s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.9496 | max_loss: 0.1965 | max_backward: 1.6425 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.5117 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1294 | Memory: rss_MB: 8120.6 | uss_MB: 7458.3 | child_rss_MB: 5010.7 | allocated_MB: 344.8 | reserved_MB: 21328.0 | inactive_split_MB: 198.7 +rank3 [2026-10-29 01:30:57,229] INFO : Step 20200/50000 rank3 | Train Loss: 1.3220 | Train Loss Unweighted: 0.6624 | Train Loss Components: Auxiliary: 0.2728 | Cardinality: 0.1545 | Classification: 0.1109 | Classification_binary: 0.2232 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1618 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.2202 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0906 | Regression_eta: 0.0710 | Regression_pt: 0.0926 | Regression_sin_phi: 0.1046 | LR: 6.84e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8619s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.9337 | max_loss: 0.2287 | max_backward: 1.4619 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6969.7 | uss_MB: 6305.7 | child_rss_MB: 5014.0 | allocated_MB: 344.4 | reserved_MB: 21328.0 | inactive_split_MB: 287.4 +rank3 [2026-10-29 01:34:46,195] INFO : Step 20300/50000 rank3 | Train Loss: 1.2742 | Train Loss Unweighted: 0.6629 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1615 | Classification: 0.1132 | Classification_binary: 0.1937 | HitGrouping: 0.2129 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1761 | Regression_eta: 0.0018 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1639 | Regression_cos_phi: 0.0933 | Regression_energy: 0.0719 | Regression_eta: 0.0430 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0702 | LR: 6.81e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8201s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6857 | max_loss: 0.2423 | max_backward: 1.2822 | max_optimizer_step: 0.2837 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0419 | Memory: rss_MB: 6967.2 | uss_MB: 6303.7 | child_rss_MB: 5019.2 | allocated_MB: 340.8 | reserved_MB: 21826.0 | inactive_split_MB: 239.4 +rank3 [2026-10-29 01:38:43,603] INFO : Step 20400/50000 rank3 | Train Loss: 1.2743 | Train Loss Unweighted: 0.6521 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1507 | Classification: 0.1279 | Classification_binary: 0.1990 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1604 | Regression_eta: 0.0019 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0932 | Classification_binary: 0.2010 | Regression_cos_phi: 0.0887 | Regression_energy: 0.0695 | Regression_eta: 0.0578 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0753 | LR: 6.78e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.7466s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0283 | max_forward: 1.3648 | max_loss: 0.1816 | max_backward: 1.7875 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 6969.3 | uss_MB: 6306.4 | child_rss_MB: 5010.2 | allocated_MB: 343.0 | reserved_MB: 22154.0 | inactive_split_MB: 243.5 +rank3 [2026-10-29 01:42:48,352] INFO : Step 20500/50000 rank3 | Train Loss: 1.2712 | Train Loss Unweighted: 0.6509 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1409 | Classification: 0.1147 | Classification_binary: 0.2202 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1558 | Regression_eta: 0.0019 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2010 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0768 | Regression_eta: 0.0470 | Regression_pt: 0.0796 | Regression_sin_phi: 0.1105 | LR: 6.76e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2552s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7205 | max_loss: 0.2283 | max_backward: 1.6623 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.6254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0704 | Memory: rss_MB: 6969.0 | uss_MB: 6306.9 | child_rss_MB: 5000.2 | allocated_MB: 345.0 | reserved_MB: 22154.0 | inactive_split_MB: 208.2 +rank3 [2026-10-29 01:46:45,622] INFO : Step 20600/50000 rank3 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6338 | Train Loss Components: Auxiliary: 0.2625 | Cardinality: 0.1623 | Classification: 0.1101 | Classification_binary: 0.1900 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0018 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0762 | Regression_eta: 0.0368 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0650 | LR: 6.73e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.9332s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 1.1520 | max_loss: 0.2849 | max_backward: 2.1471 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.6141 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6963.6 | uss_MB: 6301.3 | child_rss_MB: 5002.5 | allocated_MB: 341.4 | reserved_MB: 22154.0 | inactive_split_MB: 183.5 +rank3 [2026-10-29 01:50:54,179] INFO : Step 20700/50000 rank3 | Train Loss: 1.1613 | Train Loss Unweighted: 0.6066 | Train Loss Components: Auxiliary: 0.2395 | Cardinality: 0.1327 | Classification: 0.1061 | Classification_binary: 0.1677 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0016 | Regression_pt: 0.1666 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0775 | Regression_eta: 0.0447 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0894 | LR: 6.70e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5592s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6276 | max_loss: 0.2330 | max_backward: 1.4954 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6965.5 | uss_MB: 5936.8 | child_rss_MB: 5004.9 | allocated_MB: 341.0 | reserved_MB: 22154.0 | inactive_split_MB: 285.0 +rank3 [2026-10-29 01:55:01,769] INFO : Step 20800/50000 rank3 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1493 | Classification: 0.1205 | Classification_binary: 0.2020 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0761 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0617 | Regression_eta: 0.0377 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0788 | LR: 6.67e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.2087s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.7679 | max_loss: 0.1604 | max_backward: 1.7694 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.4914 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6972.0 | uss_MB: 6311.7 | child_rss_MB: 5027.1 | allocated_MB: 344.1 | reserved_MB: 22154.0 | inactive_split_MB: 258.7 +rank3 [2026-10-29 01:59:06,393] INFO : Step 20900/50000 rank3 | Train Loss: 1.2934 | Train Loss Unweighted: 0.6525 | Train Loss Components: Auxiliary: 0.2719 | Cardinality: 0.1620 | Classification: 0.1167 | Classification_binary: 0.2121 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1591 | Regression_eta: 0.0021 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0927 | Regression_energy: 0.0732 | Regression_eta: 0.0523 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0803 | LR: 6.65e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1334s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7787 | max_loss: 0.1747 | max_backward: 1.6678 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 6974.0 | uss_MB: 5772.6 | child_rss_MB: 5037.2 | allocated_MB: 343.6 | reserved_MB: 23208.0 | inactive_split_MB: 296.4 +rank3 [2026-10-29 02:03:08,876] INFO : Step 21000/50000 rank3 | Train Loss: 1.1402 | Train Loss Unweighted: 0.5874 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1236 | Classification: 0.0967 | Classification_binary: 0.1701 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1583 | Regression_eta: 0.0017 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1958 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0692 | Regression_eta: 0.0463 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0602 | LR: 6.62e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.8975s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6629 | max_loss: 0.1833 | max_backward: 1.4689 | max_optimizer_step: 0.2206 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6973.2 | uss_MB: 6313.8 | child_rss_MB: 5014.5 | allocated_MB: 340.4 | reserved_MB: 23208.0 | inactive_split_MB: 264.1 +rank3 [2026-10-29 02:03:09,906] INFO : Running validation on rank3 +rank3 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 02:04:00,487] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:04:01,073] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 02:04:27,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 02:04:28,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 02:04:29,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 02:04:30,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 02:04:40,196] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 02:04:40,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:04:41,351] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 02:04:57,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 02:04:58,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 02:04:58,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 02:04:59,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 02:05:09,923] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 02:05:10,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:05:11,302] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 02:05:26,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 02:05:27,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 02:05:28,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 02:05:29,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 02:05:39,140] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 02:10:06,179] INFO : Step 21100/50000 rank3 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6632 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1574 | Classification: 0.1277 | Classification_binary: 0.2026 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0024 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.2236 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0830 | Regression_eta: 0.0674 | Regression_pt: 0.0874 | Regression_sin_phi: 0.1178 | LR: 6.59e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2650s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7159 | max_loss: 0.1411 | max_backward: 1.5805 | max_optimizer_step: 0.3396 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8247.8 | uss_MB: 7584.2 | child_rss_MB: 5046.5 | allocated_MB: 343.8 | reserved_MB: 21372.0 | inactive_split_MB: 255.4 +rank3 [2026-10-29 02:10:38,255] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank3 [2026-10-29 02:10:38,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-29 02:14:15,482] INFO : Step 21200/50000 rank3 | Train Loss: 1.2354 | Train Loss Unweighted: 0.6366 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1422 | Classification: 0.1056 | Classification_binary: 0.1876 | HitGrouping: 0.2030 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0017 | Regression_pt: 0.1709 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1722 | Regression_cos_phi: 0.1149 | Regression_energy: 0.0681 | Regression_eta: 0.0365 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0801 | LR: 6.56e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.1583s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7385 | max_loss: 0.2183 | max_backward: 1.4276 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.7483 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7137.9 | uss_MB: 6469.9 | child_rss_MB: 5019.5 | allocated_MB: 342.5 | reserved_MB: 23286.0 | inactive_split_MB: 273.0 +rank3 [2026-10-29 02:18:17,990] INFO : Step 21300/50000 rank3 | Train Loss: 1.1511 | Train Loss Unweighted: 0.5782 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1240 | Classification: 0.1021 | Classification_binary: 0.1706 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1505 | Regression_eta: 0.0017 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1820 | Regression_cos_phi: 0.0883 | Regression_energy: 0.0655 | Regression_eta: 0.0480 | Regression_pt: 0.0692 | Regression_sin_phi: 0.1203 | LR: 6.54e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.4914s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0290 | max_forward: 1.3238 | max_loss: 0.2105 | max_backward: 1.6672 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.6878 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7140.4 | uss_MB: 6475.6 | child_rss_MB: 5012.2 | allocated_MB: 343.8 | reserved_MB: 23286.0 | inactive_split_MB: 272.8 +rank3 [2026-10-29 02:22:14,589] INFO : Step 21400/50000 rank3 | Train Loss: 1.2993 | Train Loss Unweighted: 0.6458 | Train Loss Components: Auxiliary: 0.2724 | Cardinality: 0.1796 | Classification: 0.1204 | Classification_binary: 0.2158 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1518 | Regression_eta: 0.0018 | Regression_pt: 0.1542 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0769 | Classification_binary: 0.2185 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0763 | Regression_eta: 0.0420 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0736 | LR: 6.51e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5455s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8696 | max_loss: 0.1572 | max_backward: 1.8002 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 0.6992 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7114.7 | uss_MB: 6081.0 | child_rss_MB: 5015.7 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 201.9 +rank3 [2026-10-29 02:26:14,444] INFO : Step 21500/50000 rank3 | Train Loss: 1.2839 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2693 | Cardinality: 0.1478 | Classification: 0.1058 | Classification_binary: 0.2162 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.2404 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0804 | Regression_eta: 0.0569 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0919 | LR: 6.48e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.7752s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.8684 | max_loss: 0.1577 | max_backward: 1.9355 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.7593 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0648 | Memory: rss_MB: 7121.8 | uss_MB: 6458.0 | child_rss_MB: 5014.4 | allocated_MB: 341.9 | reserved_MB: 25248.0 | inactive_split_MB: 284.1 +rank3 [2026-10-29 02:30:00,722] INFO : Step 21600/50000 rank3 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1290 | Classification: 0.1062 | Classification_binary: 0.1618 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1486 | Regression_eta: 0.0018 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1436 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0616 | Regression_eta: 0.0532 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0873 | LR: 6.45e-04 | DataLoad Time: 0.0187s | Model Forward Time: 3.9666s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0061 | max_forward: 0.7663 | max_loss: 0.1883 | max_backward: 1.4257 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.4997 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7120.6 | uss_MB: 6456.9 | child_rss_MB: 5032.2 | allocated_MB: 340.5 | reserved_MB: 25248.0 | inactive_split_MB: 234.8 +rank3 [2026-10-29 02:33:57,426] INFO : Step 21700/50000 rank3 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6283 | Train Loss Components: Auxiliary: 0.2577 | Cardinality: 0.1575 | Classification: 0.1186 | Classification_binary: 0.1853 | HitGrouping: 0.2068 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1594 | Regression_eta: 0.0022 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0892 | Classification_binary: 0.1821 | Regression_cos_phi: 0.1088 | Regression_energy: 0.0764 | Regression_eta: 0.0572 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0991 | LR: 6.43e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7316s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.6796 | max_loss: 0.1417 | max_backward: 1.3881 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7122.2 | uss_MB: 6458.0 | child_rss_MB: 5034.2 | allocated_MB: 343.8 | reserved_MB: 25248.0 | inactive_split_MB: 244.5 +rank3 [2026-10-29 02:37:54,439] INFO : Step 21800/50000 rank3 | Train Loss: 1.2260 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2691 | Cardinality: 0.1439 | Classification: 0.1105 | Classification_binary: 0.2043 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1576 | Regression_eta: 0.0016 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2123 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0839 | Regression_eta: 0.0478 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0638 | LR: 6.40e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.9228s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 0.8043 | max_loss: 0.1895 | max_backward: 2.4027 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.9442 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7124.3 | uss_MB: 5827.3 | child_rss_MB: 5037.0 | allocated_MB: 344.5 | reserved_MB: 25248.0 | inactive_split_MB: 277.8 +rank3 [2026-10-29 02:41:42,512] INFO : Step 21900/50000 rank3 | Train Loss: 1.2119 | Train Loss Unweighted: 0.5952 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1861 | Classification: 0.1033 | Classification_binary: 0.1865 | HitGrouping: 0.1989 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0015 | Regression_pt: 0.1511 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1933 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0776 | Regression_eta: 0.0370 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0644 | LR: 6.37e-04 | DataLoad Time: 0.0135s | Model Forward Time: 4.5639s | Phase Times: max_pre_step_barrier: 0.0174 | max_batch_to_device: 0.0163 | max_forward: 0.9684 | max_loss: 0.1448 | max_backward: 1.6354 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.8990 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 7124.6 | uss_MB: 6460.4 | child_rss_MB: 5033.8 | allocated_MB: 342.7 | reserved_MB: 25248.0 | inactive_split_MB: 192.3 +rank3 [2026-10-29 02:45:41,732] INFO : Step 22000/50000 rank3 | Train Loss: 1.1094 | Train Loss Unweighted: 0.5651 | Train Loss Components: Auxiliary: 0.2344 | Cardinality: 0.1248 | Classification: 0.1037 | Classification_binary: 0.1579 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1497 | Regression_eta: 0.0017 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0592 | Regression_eta: 0.0500 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0603 | LR: 6.34e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.4231s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.6825 | max_loss: 0.1653 | max_backward: 1.4396 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.0256 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 7124.1 | uss_MB: 6463.7 | child_rss_MB: 5009.1 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 247.3 +rank3 [2026-10-29 02:45:42,810] INFO : Running validation on rank3 +rank3 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 02:46:37,521] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:46:38,045] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 02:46:57,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 02:46:58,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 02:46:59,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 02:47:00,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 02:47:10,960] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 02:47:11,416] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:47:11,958] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 02:47:28,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 02:47:29,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 02:47:29,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 02:47:30,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 02:47:40,903] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 02:47:41,459] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:47:42,030] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 02:47:57,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 02:47:58,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 02:47:58,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 02:47:59,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 02:48:09,580] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 02:52:24,843] INFO : Step 22100/50000 rank3 | Train Loss: 1.2514 | Train Loss Unweighted: 0.6219 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1530 | Classification: 0.1140 | Classification_binary: 0.2076 | HitGrouping: 0.2031 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1470 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0791 | Classification_binary: 0.2242 | Regression_cos_phi: 0.1257 | Regression_energy: 0.0726 | Regression_eta: 0.0488 | Regression_pt: 0.0773 | Regression_sin_phi: 0.1066 | LR: 6.32e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5398s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7250 | max_loss: 0.2514 | max_backward: 1.5956 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.8632 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0587 | Memory: rss_MB: 8032.9 | uss_MB: 7365.3 | child_rss_MB: 5019.4 | allocated_MB: 347.3 | reserved_MB: 23386.0 | inactive_split_MB: 278.6 +rank3 [2026-10-29 02:56:29,083] INFO : Step 22200/50000 rank3 | Train Loss: 1.2727 | Train Loss Unweighted: 0.6485 | Train Loss Components: Auxiliary: 0.2709 | Cardinality: 0.1498 | Classification: 0.1104 | Classification_binary: 0.2184 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1574 | Regression_eta: 0.0018 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2266 | Regression_cos_phi: 0.0948 | Regression_energy: 0.0714 | Regression_eta: 0.0474 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0772 | LR: 6.29e-04 | DataLoad Time: 0.0158s | Model Forward Time: 6.9305s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0074 | max_forward: 0.7776 | max_loss: 0.1472 | max_backward: 1.7604 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.3305 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7158.4 | uss_MB: 5092.5 | child_rss_MB: 5016.1 | allocated_MB: 342.3 | reserved_MB: 23578.0 | inactive_split_MB: 277.9 +rank3 [2026-10-29 03:00:35,124] INFO : Step 22300/50000 rank3 | Train Loss: 1.1242 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2347 | Cardinality: 0.1367 | Classification: 0.1013 | Classification_binary: 0.1688 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0786 | Regression_eta: 0.0400 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0567 | LR: 6.26e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.7790s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7697 | max_loss: 0.1538 | max_backward: 1.3553 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.4896 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 7162.9 | uss_MB: 6498.9 | child_rss_MB: 4987.3 | allocated_MB: 341.2 | reserved_MB: 23578.0 | inactive_split_MB: 233.8 +rank3 [2026-10-29 03:04:32,773] INFO : Step 22400/50000 rank3 | Train Loss: 1.2357 | Train Loss Unweighted: 0.6255 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1493 | Classification: 0.1171 | Classification_binary: 0.1885 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1575 | Regression_eta: 0.0021 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1622 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0684 | Regression_eta: 0.0438 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0700 | LR: 6.23e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9151s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0067 | max_forward: 0.6425 | max_loss: 0.1530 | max_backward: 1.4136 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.5876 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 7158.4 | uss_MB: 6496.8 | child_rss_MB: 5021.9 | allocated_MB: 342.5 | reserved_MB: 23578.0 | inactive_split_MB: 251.5 +rank3 [2026-10-29 03:08:26,684] INFO : Step 22500/50000 rank3 | Train Loss: 1.2633 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2638 | Cardinality: 0.1676 | Classification: 0.0954 | Classification_binary: 0.2181 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0020 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.2555 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0756 | Regression_eta: 0.0432 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0620 | LR: 6.20e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4295s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1522 | max_backward: 1.5631 | max_optimizer_step: 0.3261 | max_tensorboard_logging: 0.7552 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 7160.2 | uss_MB: 6502.4 | child_rss_MB: 5011.7 | allocated_MB: 342.9 | reserved_MB: 24052.0 | inactive_split_MB: 295.1 +rank3 [2026-10-29 03:12:18,754] INFO : Step 22600/50000 rank3 | Train Loss: 1.1636 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1329 | Classification: 0.0997 | Classification_binary: 0.1889 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1493 | Regression_eta: 0.0020 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0697 | Regression_eta: 0.0321 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0876 | LR: 6.18e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9324s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7375 | max_loss: 0.2403 | max_backward: 1.4584 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7163.5 | uss_MB: 6503.3 | child_rss_MB: 5024.4 | allocated_MB: 344.2 | reserved_MB: 24590.0 | inactive_split_MB: 288.5 +rank3 [2026-10-29 03:16:20,063] INFO : Step 22700/50000 rank3 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1212 | Classification: 0.0976 | Classification_binary: 0.1566 | HitGrouping: 0.2101 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1455 | Regression_eta: 0.0015 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1673 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0560 | Regression_eta: 0.0273 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0513 | LR: 6.15e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.9683s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.6124 | max_loss: 0.1376 | max_backward: 1.1766 | max_optimizer_step: 0.2237 | max_tensorboard_logging: 3.8568 | max_tensorboard_flush: 0.0169 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 7164.2 | uss_MB: 6160.1 | child_rss_MB: 5007.8 | allocated_MB: 341.6 | reserved_MB: 25058.0 | inactive_split_MB: 256.4 +rank3 [2026-10-29 03:20:29,740] INFO : Step 22800/50000 rank3 | Train Loss: 1.1840 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2533 | Cardinality: 0.1443 | Classification: 0.1191 | Classification_binary: 0.1924 | HitGrouping: 0.1959 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1444 | Regression_eta: 0.0016 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1964 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0731 | Regression_eta: 0.0534 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0896 | LR: 6.12e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4527s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7010 | max_loss: 0.2312 | max_backward: 1.6853 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8726 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 7164.4 | uss_MB: 6502.8 | child_rss_MB: 5021.2 | allocated_MB: 341.7 | reserved_MB: 25058.0 | inactive_split_MB: 281.3 +rank3 [2026-10-29 03:24:21,160] INFO : Step 22900/50000 rank3 | Train Loss: 1.2680 | Train Loss Unweighted: 0.6318 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1792 | Classification: 0.1020 | Classification_binary: 0.2074 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1580 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2338 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0690 | Regression_eta: 0.0597 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0718 | LR: 6.09e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.0005s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 1.1637 | max_loss: 0.4031 | max_backward: 1.8442 | max_optimizer_step: 0.3320 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7165.6 | uss_MB: 6506.2 | child_rss_MB: 5023.4 | allocated_MB: 344.8 | reserved_MB: 25058.0 | inactive_split_MB: 300.1 +rank3 [2026-10-29 03:28:05,592] INFO : Step 23000/50000 rank3 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5484 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1129 | Classification: 0.0836 | Classification_binary: 0.1519 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1547 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0771 | Regression_eta: 0.0651 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0641 | LR: 6.06e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.7519s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0063 | max_forward: 0.6963 | max_loss: 0.2382 | max_backward: 1.2870 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 7162.6 | uss_MB: 6506.3 | child_rss_MB: 5008.7 | allocated_MB: 340.3 | reserved_MB: 26066.0 | inactive_split_MB: 218.4 +rank3 [2026-10-29 03:28:06,534] INFO : Running validation on rank3 +rank3 [2026-10-29 03:28:06,534] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 03:28:06,534] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 03:28:56,135] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:28:56,639] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 03:29:22,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 03:29:23,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 03:29:24,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 03:29:25,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 03:29:35,248] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 03:29:35,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:29:36,301] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 03:29:52,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 03:29:53,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 03:29:54,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 03:29:55,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 03:30:05,161] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 03:30:05,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:30:05,702] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 03:30:21,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 03:30:21,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 03:30:22,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 03:30:23,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 03:30:33,084] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 03:34:45,381] INFO : Step 23100/50000 rank3 | Train Loss: 1.2422 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1379 | Classification: 0.1246 | Classification_binary: 0.1795 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1482 | Regression_eta: 0.0023 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.1756 | Regression_cos_phi: 0.1025 | Regression_energy: 0.0832 | Regression_eta: 0.0445 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0724 | LR: 6.04e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8247s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7203 | max_loss: 0.1522 | max_backward: 1.1000 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.6183 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 8419.6 | uss_MB: 7758.7 | child_rss_MB: 5039.2 | allocated_MB: 344.7 | reserved_MB: 22598.0 | inactive_split_MB: 244.5 +rank3 [2026-10-29 03:38:33,524] INFO : Step 23200/50000 rank3 | Train Loss: 1.2507 | Train Loss Unweighted: 0.6359 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1400 | Classification: 0.1121 | Classification_binary: 0.2208 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1488 | Regression_eta: 0.0020 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2240 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0735 | Regression_eta: 0.0408 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0601 | LR: 6.01e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.5959s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0727 | max_forward: 0.9486 | max_loss: 0.1744 | max_backward: 1.7031 | max_optimizer_step: 0.2405 | max_tensorboard_logging: 0.6717 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1109 | Memory: rss_MB: 7225.3 | uss_MB: 6564.6 | child_rss_MB: 5046.8 | allocated_MB: 343.7 | reserved_MB: 23792.0 | inactive_split_MB: 223.2 +rank3 [2026-10-29 03:42:24,564] INFO : Step 23300/50000 rank3 | Train Loss: 1.1880 | Train Loss Unweighted: 0.5927 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1464 | Classification: 0.0987 | Classification_binary: 0.1851 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1524 | Regression_eta: 0.0019 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0739 | Regression_eta: 0.0338 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0504 | LR: 5.98e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.8604s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7557 | max_loss: 0.1467 | max_backward: 1.4708 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.4854 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 7231.3 | uss_MB: 6572.4 | child_rss_MB: 5045.1 | allocated_MB: 342.6 | reserved_MB: 24534.0 | inactive_split_MB: 271.6 +rank3 [2026-10-29 03:46:11,297] INFO : Step 23400/50000 rank3 | Train Loss: 1.1136 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2402 | Cardinality: 0.1202 | Classification: 0.1070 | Classification_binary: 0.1497 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1644 | Regression_eta: 0.0017 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0801 | Regression_energy: 0.0844 | Regression_eta: 0.0623 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0661 | LR: 5.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6670s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.5743 | max_loss: 0.2297 | max_backward: 1.0127 | max_optimizer_step: 0.3321 | max_tensorboard_logging: 0.6182 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 7232.3 | uss_MB: 6571.7 | child_rss_MB: 5046.9 | allocated_MB: 340.5 | reserved_MB: 24848.0 | inactive_split_MB: 241.3 +rank3 [2026-10-29 03:50:07,300] INFO : Step 23500/50000 rank3 | Train Loss: 1.1601 | Train Loss Unweighted: 0.5799 | Train Loss Components: Auxiliary: 0.2477 | Cardinality: 0.1161 | Classification: 0.1057 | Classification_binary: 0.1852 | HitGrouping: 0.2029 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1416 | Regression_eta: 0.0017 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1780 | Regression_cos_phi: 0.0986 | Regression_energy: 0.0599 | Regression_eta: 0.0747 | Regression_pt: 0.0617 | Regression_sin_phi: 0.1123 | LR: 5.92e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4408s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8864 | max_loss: 0.1573 | max_backward: 1.9036 | max_optimizer_step: 0.2294 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7232.1 | uss_MB: 6574.4 | child_rss_MB: 5036.9 | allocated_MB: 341.9 | reserved_MB: 24848.0 | inactive_split_MB: 233.9 +rank3 [2026-10-29 03:53:52,162] INFO : Step 23600/50000 rank3 | Train Loss: 1.2402 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1447 | Classification: 0.1136 | Classification_binary: 0.2049 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1534 | Regression_eta: 0.0020 | Regression_pt: 0.1550 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0786 | Classification_binary: 0.2049 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0668 | Regression_eta: 0.0429 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0721 | LR: 5.90e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.9441s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8397 | max_loss: 0.1470 | max_backward: 1.4395 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7234.1 | uss_MB: 6575.7 | child_rss_MB: 5045.6 | allocated_MB: 342.5 | reserved_MB: 24848.0 | inactive_split_MB: 215.5 +rank3 [2026-10-29 03:57:35,778] INFO : Step 23700/50000 rank3 | Train Loss: 1.1009 | Train Loss Unweighted: 0.5682 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1218 | Classification: 0.0954 | Classification_binary: 0.1599 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1528 | Regression_eta: 0.0014 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0601 | Regression_eta: 0.0335 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0370 | LR: 5.87e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.6832s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6589 | max_loss: 0.1387 | max_backward: 1.2152 | max_optimizer_step: 0.2251 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7230.0 | uss_MB: 6574.3 | child_rss_MB: 5051.4 | allocated_MB: 342.5 | reserved_MB: 25508.0 | inactive_split_MB: 226.3 +rank3 [2026-10-29 04:01:35,999] INFO : Step 23800/50000 rank3 | Train Loss: 1.1390 | Train Loss Unweighted: 0.5882 | Train Loss Components: Auxiliary: 0.2427 | Cardinality: 0.1196 | Classification: 0.1035 | Classification_binary: 0.1735 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1698 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0686 | Regression_eta: 0.0427 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0798 | LR: 5.84e-04 | DataLoad Time: 0.0226s | Model Forward Time: 3.7617s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.7283 | max_loss: 0.1547 | max_backward: 1.2415 | max_optimizer_step: 0.2259 | max_tensorboard_logging: 0.5270 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7231.0 | uss_MB: 6110.2 | child_rss_MB: 5050.1 | allocated_MB: 345.2 | reserved_MB: 25508.0 | inactive_split_MB: 241.8 +rank3 [2026-10-29 04:05:42,354] INFO : Step 23900/50000 rank3 | Train Loss: 1.2349 | Train Loss Unweighted: 0.6078 | Train Loss Components: Auxiliary: 0.2621 | Cardinality: 0.1596 | Classification: 0.1044 | Classification_binary: 0.2077 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1451 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0686 | Regression_eta: 0.0498 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0715 | LR: 5.81e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4872s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.9077 | max_loss: 0.1538 | max_backward: 1.9088 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7234.7 | uss_MB: 6579.6 | child_rss_MB: 5044.4 | allocated_MB: 342.1 | reserved_MB: 27474.0 | inactive_split_MB: 282.4 +rank3 [2026-10-29 04:09:40,154] INFO : Step 24000/50000 rank3 | Train Loss: 1.1074 | Train Loss Unweighted: 0.5714 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1170 | Classification: 0.1007 | Classification_binary: 0.1792 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1431 | Regression_eta: 0.0015 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.2023 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0668 | Regression_eta: 0.0418 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0641 | LR: 5.78e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.2198s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 1.1428 | max_loss: 0.2613 | max_backward: 2.5767 | max_optimizer_step: 0.2298 | max_tensorboard_logging: 0.5998 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7236.7 | uss_MB: 6577.2 | child_rss_MB: 5038.1 | allocated_MB: 342.1 | reserved_MB: 27476.0 | inactive_split_MB: 218.6 +rank3 [2026-10-29 04:09:41,046] INFO : Running validation on rank3 +rank3 [2026-10-29 04:09:41,046] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 04:09:41,046] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 04:10:38,743] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:10:39,335] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 04:11:05,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 04:11:06,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 04:11:07,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 04:11:10,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 04:11:20,178] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 04:11:20,671] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:11:21,255] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 04:11:37,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 04:11:38,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 04:11:39,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 04:11:40,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 04:11:50,033] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 04:11:50,532] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:11:51,029] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 04:12:06,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 04:12:07,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 04:12:07,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 04:12:08,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 04:12:18,476] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 04:16:38,288] INFO : Step 24100/50000 rank3 | Train Loss: 1.1244 | Train Loss Unweighted: 0.5584 | Train Loss Components: Auxiliary: 0.2373 | Cardinality: 0.1285 | Classification: 0.0978 | Classification_binary: 0.1606 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0652 | Regression_eta: 0.0812 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0732 | LR: 5.75e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1995s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0860 | max_forward: 0.7361 | max_loss: 0.1432 | max_backward: 1.4268 | max_optimizer_step: 0.3451 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 8228.7 | uss_MB: 7570.1 | child_rss_MB: 5001.0 | allocated_MB: 343.2 | reserved_MB: 21608.0 | inactive_split_MB: 269.9 +rank3 [2026-10-29 04:20:36,776] INFO : Step 24200/50000 rank3 | Train Loss: 1.3067 | Train Loss Unweighted: 0.6341 | Train Loss Components: Auxiliary: 0.2739 | Cardinality: 0.1760 | Classification: 0.1145 | Classification_binary: 0.2101 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0018 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.2193 | Regression_cos_phi: 0.0922 | Regression_energy: 0.0807 | Regression_eta: 0.0459 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0885 | LR: 5.73e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.0785s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1066 | max_forward: 1.0531 | max_loss: 0.2796 | max_backward: 2.3820 | max_optimizer_step: 0.2441 | max_tensorboard_logging: 0.6187 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 8233.6 | uss_MB: 7574.6 | child_rss_MB: 5010.3 | allocated_MB: 345.0 | reserved_MB: 23864.0 | inactive_split_MB: 203.0 +rank3 [2026-10-29 04:24:33,009] INFO : Step 24300/50000 rank3 | Train Loss: 1.2207 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1582 | Classification: 0.1151 | Classification_binary: 0.2013 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1450 | Regression_eta: 0.0017 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.1944 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0755 | Regression_eta: 0.0608 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0949 | LR: 5.70e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3051s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7395 | max_loss: 0.2490 | max_backward: 1.4969 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.6555 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7355.3 | uss_MB: 6697.1 | child_rss_MB: 5007.1 | allocated_MB: 344.8 | reserved_MB: 23864.0 | inactive_split_MB: 284.8 +rank3 [2026-10-29 04:28:19,172] INFO : Step 24400/50000 rank3 | Train Loss: 1.1288 | Train Loss Unweighted: 0.5921 | Train Loss Components: Auxiliary: 0.2439 | Cardinality: 0.1229 | Classification: 0.1017 | Classification_binary: 0.1613 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1631 | Regression_eta: 0.0015 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0934 | Regression_energy: 0.0868 | Regression_eta: 0.0490 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0701 | LR: 5.67e-04 | DataLoad Time: 0.0184s | Model Forward Time: 3.9872s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.6685 | max_loss: 0.1564 | max_backward: 1.4736 | max_optimizer_step: 0.2223 | max_tensorboard_logging: 0.5369 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7357.1 | uss_MB: 6698.7 | child_rss_MB: 5016.5 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 267.2 +rank3 [2026-10-29 04:32:03,440] INFO : Step 24500/50000 rank3 | Train Loss: 1.2319 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2586 | Cardinality: 0.1511 | Classification: 0.1107 | Classification_binary: 0.1876 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1540 | Regression_eta: 0.0018 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0899 | Classification_binary: 0.2029 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0723 | Regression_eta: 0.0631 | Regression_pt: 0.0751 | Regression_sin_phi: 0.1420 | LR: 5.64e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1619s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0677 | max_forward: 0.7219 | max_loss: 0.1439 | max_backward: 1.4253 | max_optimizer_step: 0.3416 | max_tensorboard_logging: 0.5859 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 7356.5 | uss_MB: 6698.2 | child_rss_MB: 5010.5 | allocated_MB: 343.1 | reserved_MB: 23864.0 | inactive_split_MB: 275.8 +rank3 [2026-10-29 04:36:01,551] INFO : Step 24600/50000 rank3 | Train Loss: 1.2474 | Train Loss Unweighted: 0.6241 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1630 | Classification: 0.1074 | Classification_binary: 0.2102 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0719 | Regression_eta: 0.0341 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0637 | LR: 5.61e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5133s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7876 | max_loss: 0.1528 | max_backward: 1.7437 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.6204 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7358.2 | uss_MB: 6700.7 | child_rss_MB: 5006.8 | allocated_MB: 344.3 | reserved_MB: 23864.0 | inactive_split_MB: 285.7 +rank3 [2026-10-29 04:39:51,298] INFO : Step 24700/50000 rank3 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1193 | Classification: 0.0990 | Classification_binary: 0.1723 | HitGrouping: 0.2019 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1471 | Regression_eta: 0.0016 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1795 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0656 | Regression_eta: 0.0362 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0665 | LR: 5.58e-04 | DataLoad Time: 0.0143s | Model Forward Time: 3.9536s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6978 | max_loss: 0.2493 | max_backward: 1.2925 | max_optimizer_step: 0.2314 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7361.3 | uss_MB: 6700.3 | child_rss_MB: 5009.8 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 223.9 +rank3 [2026-10-29 04:43:37,808] INFO : Step 24800/50000 rank3 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5554 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1264 | Classification: 0.0932 | Classification_binary: 0.1583 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1493 | Regression_cos_phi: 0.0837 | Regression_energy: 0.0762 | Regression_eta: 0.0511 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0763 | LR: 5.56e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2775s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6582 | max_loss: 0.2533 | max_backward: 1.5858 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.6923 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1086 | Memory: rss_MB: 7365.2 | uss_MB: 6703.3 | child_rss_MB: 5005.7 | allocated_MB: 342.9 | reserved_MB: 23864.0 | inactive_split_MB: 278.6 +rank3 [2026-10-29 04:47:36,372] INFO : Step 24900/50000 rank3 | Train Loss: 1.1348 | Train Loss Unweighted: 0.5816 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1381 | Classification: 0.1126 | Classification_binary: 0.1845 | HitGrouping: 0.1937 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1403 | Regression_eta: 0.0015 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0820 | Classification_binary: 0.2189 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0790 | Regression_eta: 0.0525 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0812 | LR: 5.53e-04 | DataLoad Time: 0.0148s | Model Forward Time: 8.7038s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0067 | max_forward: 0.7276 | max_loss: 0.2411 | max_backward: 1.3958 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 5.1754 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1038 | Memory: rss_MB: 7366.2 | uss_MB: 6703.5 | child_rss_MB: 5007.2 | allocated_MB: 343.9 | reserved_MB: 23864.0 | inactive_split_MB: 239.7 +rank3 [2026-10-29 04:51:28,372] INFO : Step 25000/50000 rank3 | Train Loss: 1.1995 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1590 | Classification: 0.0891 | Classification_binary: 0.2009 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1421 | Regression_eta: 0.0018 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0629 | Regression_eta: 0.0355 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0592 | LR: 5.50e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2038s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7692 | max_loss: 0.2091 | max_backward: 1.3616 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6652 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1392 | Memory: rss_MB: 7366.4 | uss_MB: 6702.5 | child_rss_MB: 5014.2 | allocated_MB: 343.5 | reserved_MB: 23864.0 | inactive_split_MB: 293.8 +rank3 [2026-10-29 04:51:29,309] INFO : Running validation on rank3 +rank3 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 04:52:22,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:52:22,822] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 04:52:49,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 04:52:50,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 04:52:51,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 04:52:52,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 04:53:02,210] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 04:53:02,763] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:53:03,282] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 04:53:19,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 04:53:20,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 04:53:21,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 04:53:21,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 04:53:31,985] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 04:53:32,527] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:53:32,995] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 04:53:48,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 04:53:49,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 04:53:50,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 04:53:50,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 04:54:00,923] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 04:58:07,811] INFO : Step 25100/50000 rank3 | Train Loss: 1.0833 | Train Loss Unweighted: 0.5514 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1317 | Classification: 0.0920 | Classification_binary: 0.1568 | HitGrouping: 0.2008 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1491 | Regression_eta: 0.0016 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.1764 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0636 | Regression_eta: 0.0400 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0857 | LR: 5.47e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7882s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7295 | max_loss: 0.1401 | max_backward: 1.2439 | max_optimizer_step: 0.2490 | max_tensorboard_logging: 0.4940 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8607.6 | uss_MB: 7944.9 | child_rss_MB: 5042.3 | allocated_MB: 342.3 | reserved_MB: 23608.0 | inactive_split_MB: 222.3 +rank3 [2026-10-29 05:01:53,204] INFO : Step 25200/50000 rank3 | Train Loss: 1.1886 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1523 | Classification: 0.1158 | Classification_binary: 0.1800 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1512 | Regression_eta: 0.0016 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0669 | Regression_eta: 0.0288 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0471 | LR: 5.44e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.7324s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8134 | max_loss: 0.1416 | max_backward: 1.5095 | max_optimizer_step: 0.3471 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 8606.1 | uss_MB: 7945.9 | child_rss_MB: 5039.5 | allocated_MB: 343.6 | reserved_MB: 24590.0 | inactive_split_MB: 260.9 +rank3 [2026-10-29 05:05:52,266] INFO : Step 25300/50000 rank3 | Train Loss: 1.2031 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2624 | Cardinality: 0.1455 | Classification: 0.1145 | Classification_binary: 0.1966 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1448 | Regression_eta: 0.0016 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1971 | Regression_cos_phi: 0.0887 | Regression_energy: 0.0651 | Regression_eta: 0.0521 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0674 | LR: 5.42e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.5359s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.8219 | max_loss: 0.1491 | max_backward: 1.6406 | max_optimizer_step: 0.2735 | max_tensorboard_logging: 0.7204 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0488 | Memory: rss_MB: 7456.6 | uss_MB: 6796.5 | child_rss_MB: 5036.7 | allocated_MB: 341.8 | reserved_MB: 25572.0 | inactive_split_MB: 253.5 +rank3 [2026-10-29 05:09:44,971] INFO : Step 25400/50000 rank3 | Train Loss: 1.1194 | Train Loss Unweighted: 0.5655 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1346 | Classification: 0.0983 | Classification_binary: 0.1754 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1438 | Regression_eta: 0.0015 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1816 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0751 | Regression_eta: 0.0336 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0777 | LR: 5.39e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1071s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.6986 | max_loss: 0.1530 | max_backward: 1.2606 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.8427 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7454.9 | uss_MB: 6795.9 | child_rss_MB: 5040.6 | allocated_MB: 345.2 | reserved_MB: 25572.0 | inactive_split_MB: 223.8 +rank3 [2026-10-29 05:13:34,055] INFO : Step 25500/50000 rank3 | Train Loss: 1.0289 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1068 | Classification: 0.1034 | Classification_binary: 0.1444 | HitGrouping: 0.1913 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1484 | Regression_eta: 0.0014 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1343 | Regression_cos_phi: 0.0338 | Regression_energy: 0.0754 | Regression_eta: 0.0354 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0530 | LR: 5.36e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1585s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0554 | max_forward: 0.7008 | max_loss: 0.1433 | max_backward: 1.4571 | max_optimizer_step: 0.3349 | max_tensorboard_logging: 0.6083 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 7460.4 | uss_MB: 6802.0 | child_rss_MB: 5038.5 | allocated_MB: 345.4 | reserved_MB: 25572.0 | inactive_split_MB: 240.6 +rank3 [2026-10-29 05:17:32,216] INFO : Step 25600/50000 rank3 | Train Loss: 1.2028 | Train Loss Unweighted: 0.5968 | Train Loss Components: Auxiliary: 0.2590 | Cardinality: 0.1607 | Classification: 0.1135 | Classification_binary: 0.1926 | HitGrouping: 0.1978 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1429 | Regression_eta: 0.0016 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0783 | Classification_binary: 0.1888 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0713 | Regression_eta: 0.0405 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0552 | LR: 5.33e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6843s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8800 | max_loss: 0.1578 | max_backward: 1.8564 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 7459.9 | uss_MB: 6802.4 | child_rss_MB: 5035.4 | allocated_MB: 345.5 | reserved_MB: 26054.0 | inactive_split_MB: 284.5 +rank3 [2026-10-29 05:21:26,001] INFO : Step 25700/50000 rank3 | Train Loss: 1.1800 | Train Loss Unweighted: 0.5964 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1298 | Classification: 0.1086 | Classification_binary: 0.1961 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0019 | Regression_pt: 0.1444 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1717 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0663 | Regression_eta: 0.0567 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0792 | LR: 5.30e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.5206s | Phase Times: max_pre_step_barrier: 0.0086 | max_batch_to_device: 0.0137 | max_forward: 1.0340 | max_loss: 0.2483 | max_backward: 1.6704 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 0.7612 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7459.7 | uss_MB: 6801.4 | child_rss_MB: 5033.4 | allocated_MB: 343.5 | reserved_MB: 26054.0 | inactive_split_MB: 270.8 +rank3 [2026-10-29 05:25:16,515] INFO : Step 25800/50000 rank3 | Train Loss: 1.1337 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2388 | Cardinality: 0.1264 | Classification: 0.0898 | Classification_binary: 0.1617 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0019 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0640 | Regression_eta: 0.0548 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0582 | LR: 5.27e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.9380s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0986 | max_forward: 0.8129 | max_loss: 0.1662 | max_backward: 1.2979 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1103 | Memory: rss_MB: 7457.8 | uss_MB: 6799.8 | child_rss_MB: 5030.2 | allocated_MB: 341.8 | reserved_MB: 26054.0 | inactive_split_MB: 273.2 +rank3 [2026-10-29 05:29:08,212] INFO : Step 25900/50000 rank3 | Train Loss: 1.1019 | Train Loss Unweighted: 0.5481 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1165 | Classification: 0.0977 | Classification_binary: 0.1594 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1434 | Regression_eta: 0.0016 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0646 | Regression_eta: 0.0451 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0611 | LR: 5.25e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5193s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7034 | max_loss: 0.2469 | max_backward: 1.7130 | max_optimizer_step: 0.2363 | max_tensorboard_logging: 0.8382 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 7454.8 | uss_MB: 6800.6 | child_rss_MB: 5037.1 | allocated_MB: 343.0 | reserved_MB: 26054.0 | inactive_split_MB: 187.0 +rank3 [2026-10-29 05:33:07,468] INFO : Step 26000/50000 rank3 | Train Loss: 1.1708 | Train Loss Unweighted: 0.5680 | Train Loss Components: Auxiliary: 0.2498 | Cardinality: 0.1352 | Classification: 0.0999 | Classification_binary: 0.2021 | HitGrouping: 0.1994 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1304 | Regression_eta: 0.0021 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2367 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0693 | Regression_eta: 0.0585 | Regression_pt: 0.0735 | Regression_sin_phi: 0.1034 | LR: 5.22e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2239s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.7877 | max_loss: 0.1637 | max_backward: 1.6550 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.4954 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 7454.4 | uss_MB: 6802.1 | child_rss_MB: 5032.6 | allocated_MB: 345.0 | reserved_MB: 27098.0 | inactive_split_MB: 285.6 +rank3 [2026-10-29 05:33:08,393] INFO : Running validation on rank3 +rank3 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 05:33:56,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:33:57,245] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 05:34:24,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 05:34:25,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 05:34:26,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 05:34:27,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 05:34:37,369] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 05:34:38,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:34:38,538] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 05:34:54,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 05:34:55,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 05:34:56,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 05:34:57,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 05:35:07,612] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 05:35:08,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:35:08,514] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 05:35:24,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 05:35:24,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 05:35:25,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 05:35:26,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 05:35:36,300] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 05:39:49,626] INFO : Step 26100/50000 rank3 | Train Loss: 1.0974 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2307 | Cardinality: 0.1119 | Classification: 0.0956 | Classification_binary: 0.1689 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1513 | Regression_eta: 0.0014 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1809 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0752 | Regression_eta: 0.0293 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0920 | LR: 5.19e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1598s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0117 | max_forward: 0.7200 | max_loss: 0.1501 | max_backward: 1.2286 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.8284 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 8630.6 | uss_MB: 7973.7 | child_rss_MB: 5008.3 | allocated_MB: 340.1 | reserved_MB: 21762.0 | inactive_split_MB: 215.7 +rank3 [2026-10-29 05:43:56,538] INFO : Step 26200/50000 rank3 | Train Loss: 1.0876 | Train Loss Unweighted: 0.5502 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1065 | Classification: 0.0968 | Classification_binary: 0.1527 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1476 | Regression_eta: 0.0018 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1572 | Regression_cos_phi: 0.1316 | Regression_energy: 0.0750 | Regression_eta: 0.0662 | Regression_pt: 0.0789 | Regression_sin_phi: 0.1044 | LR: 5.16e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3242s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6530 | max_loss: 0.1421 | max_backward: 1.3973 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.8695 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8630.6 | uss_MB: 7973.8 | child_rss_MB: 5031.7 | allocated_MB: 342.3 | reserved_MB: 22678.0 | inactive_split_MB: 242.5 +rank3 [2026-10-29 05:47:48,030] INFO : Step 26300/50000 rank3 | Train Loss: 1.2051 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1512 | Classification: 0.1193 | Classification_binary: 0.1877 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1458 | Regression_eta: 0.0017 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1534 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0610 | Regression_eta: 0.0329 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0611 | LR: 5.13e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.4696s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0072 | max_forward: 0.7601 | max_loss: 0.2267 | max_backward: 1.8902 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7522.0 | uss_MB: 6871.8 | child_rss_MB: 4996.1 | allocated_MB: 345.4 | reserved_MB: 22678.0 | inactive_split_MB: 270.9 +rank3 [2026-10-29 05:51:47,278] INFO : Step 26400/50000 rank3 | Train Loss: 1.2256 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1648 | Classification: 0.1020 | Classification_binary: 0.1981 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1520 | Regression_eta: 0.0015 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0799 | Classification_binary: 0.2040 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0787 | Regression_eta: 0.0474 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0779 | LR: 5.10e-04 | DataLoad Time: 0.0102s | Model Forward Time: 4.4007s | Phase Times: max_pre_step_barrier: 0.0351 | max_batch_to_device: 0.0070 | max_forward: 0.8129 | max_loss: 0.1477 | max_backward: 1.4015 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8384 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 7520.2 | uss_MB: 6869.2 | child_rss_MB: 4985.0 | allocated_MB: 342.8 | reserved_MB: 22678.0 | inactive_split_MB: 196.9 +rank3 [2026-10-29 05:55:40,758] INFO : Step 26500/50000 rank3 | Train Loss: 1.1110 | Train Loss Unweighted: 0.5356 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1488 | Classification: 0.0891 | Classification_binary: 0.1546 | HitGrouping: 0.2012 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0605 | Regression_eta: 0.0456 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0540 | LR: 5.08e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0222s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6841 | max_loss: 0.2421 | max_backward: 1.1616 | max_optimizer_step: 0.2383 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7505.4 | uss_MB: 6872.6 | child_rss_MB: 4957.6 | allocated_MB: 340.6 | reserved_MB: 22678.0 | inactive_split_MB: 269.9 +rank3 [2026-10-29 05:59:33,068] INFO : Step 26600/50000 rank3 | Train Loss: 1.0895 | Train Loss Unweighted: 0.5529 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1261 | Classification: 0.1083 | Classification_binary: 0.1603 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1386 | Regression_eta: 0.0015 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0620 | Regression_eta: 0.0497 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0933 | LR: 5.05e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.4529s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0696 | max_forward: 0.6926 | max_loss: 0.1459 | max_backward: 1.4149 | max_optimizer_step: 0.2385 | max_tensorboard_logging: 0.8927 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7477.7 | uss_MB: 6873.1 | child_rss_MB: 4831.1 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 232.0 +rank3 [2026-10-29 06:03:23,755] INFO : Step 26700/50000 rank3 | Train Loss: 1.2592 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1717 | Classification: 0.1100 | Classification_binary: 0.2137 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1420 | Regression_eta: 0.0017 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1870 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0623 | Regression_eta: 0.0276 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0598 | LR: 5.02e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6738s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7750 | max_loss: 0.1526 | max_backward: 1.6427 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.8866 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7338.5 | uss_MB: 6135.7 | child_rss_MB: 4788.2 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 267.4 +rank3 [2026-10-29 06:07:16,644] INFO : Step 26800/50000 rank3 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5792 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1288 | Classification: 0.0911 | Classification_binary: 0.1633 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1600 | Regression_eta: 0.0015 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0792 | Regression_eta: 0.0304 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0597 | LR: 4.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0415s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7108 | max_loss: 0.1467 | max_backward: 1.2855 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7173 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 7253.0 | uss_MB: 6872.2 | child_rss_MB: 4809.0 | allocated_MB: 340.8 | reserved_MB: 22846.0 | inactive_split_MB: 253.2 +rank3 [2026-10-29 06:11:09,652] INFO : Step 26900/50000 rank3 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.0996 | Classification: 0.0867 | Classification_binary: 0.1428 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1483 | Regression_eta: 0.0018 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0754 | Regression_eta: 0.0727 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0715 | LR: 4.96e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8403s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0336 | max_forward: 0.6534 | max_loss: 0.1408 | max_backward: 1.3898 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0912 | Memory: rss_MB: 7214.0 | uss_MB: 6872.7 | child_rss_MB: 4790.0 | allocated_MB: 342.5 | reserved_MB: 22846.0 | inactive_split_MB: 233.5 +rank3 [2026-10-29 06:15:05,369] INFO : Step 27000/50000 rank3 | Train Loss: 1.0945 | Train Loss Unweighted: 0.5396 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1196 | Classification: 0.0957 | Classification_binary: 0.1912 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1247 | Regression_eta: 0.0015 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0521 | Regression_eta: 0.0222 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0576 | LR: 4.94e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1550s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7531 | max_loss: 0.1689 | max_backward: 1.5554 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6017 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7214.1 | uss_MB: 6877.0 | child_rss_MB: 4792.2 | allocated_MB: 345.3 | reserved_MB: 23694.0 | inactive_split_MB: 251.5 +rank3 [2026-10-29 06:15:06,390] INFO : Running validation on rank3 +rank3 [2026-10-29 06:15:06,391] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 06:15:06,391] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 06:16:00,135] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:16:00,712] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 06:16:29,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 06:16:30,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 06:16:31,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 06:16:32,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 06:16:42,814] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 06:16:43,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:16:43,719] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 06:16:59,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 06:17:00,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 06:17:01,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 06:17:02,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 06:17:12,500] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 06:17:13,248] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:17:13,778] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 06:17:29,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 06:17:29,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 06:17:30,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 06:17:31,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 06:17:41,345] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 06:21:51,976] INFO : Step 27100/50000 rank3 | Train Loss: 1.1577 | Train Loss Unweighted: 0.5595 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1497 | Classification: 0.0884 | Classification_binary: 0.1879 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1399 | Regression_eta: 0.0018 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.2102 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0641 | Regression_eta: 0.0786 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0847 | LR: 4.91e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0513s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8543 | max_loss: 0.1477 | max_backward: 1.8762 | max_optimizer_step: 0.2540 | max_tensorboard_logging: 0.9462 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 8306.7 | uss_MB: 7966.7 | child_rss_MB: 4814.1 | allocated_MB: 342.9 | reserved_MB: 21640.0 | inactive_split_MB: 268.0 +rank3 [2026-10-29 06:25:39,973] INFO : Step 27200/50000 rank3 | Train Loss: 1.0028 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1065 | Classification: 0.0834 | Classification_binary: 0.1354 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1443 | Regression_eta: 0.0015 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1171 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0825 | Regression_eta: 0.0514 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0615 | LR: 4.88e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.0909s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6839 | max_loss: 0.1724 | max_backward: 1.0628 | max_optimizer_step: 0.2515 | max_tensorboard_logging: 0.9641 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0689 | Memory: rss_MB: 8305.7 | uss_MB: 7964.4 | child_rss_MB: 4815.8 | allocated_MB: 340.5 | reserved_MB: 22612.0 | inactive_split_MB: 267.2 +rank3 [2026-10-29 06:29:18,756] INFO : Step 27300/50000 rank3 | Train Loss: 1.1913 | Train Loss Unweighted: 0.5857 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1563 | Classification: 0.1169 | Classification_binary: 0.1769 | HitGrouping: 0.1965 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1426 | Regression_eta: 0.0019 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1734 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0812 | Regression_eta: 0.0439 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0937 | LR: 4.85e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.2502s | Phase Times: max_pre_step_barrier: 0.0244 | max_batch_to_device: 0.0064 | max_forward: 0.6813 | max_loss: 0.1454 | max_backward: 1.4618 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7699 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 7290.8 | uss_MB: 6952.0 | child_rss_MB: 4811.5 | allocated_MB: 345.6 | reserved_MB: 23586.0 | inactive_split_MB: 267.9 +rank3 [2026-10-29 06:33:03,428] INFO : Step 27400/50000 rank3 | Train Loss: 1.1723 | Train Loss Unweighted: 0.5888 | Train Loss Components: Auxiliary: 0.2572 | Cardinality: 0.1400 | Classification: 0.1088 | Classification_binary: 0.2016 | HitGrouping: 0.1987 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1370 | Regression_eta: 0.0017 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.2175 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0640 | Regression_eta: 0.0521 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0518 | LR: 4.82e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6657s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8048 | max_loss: 0.1475 | max_backward: 1.6655 | max_optimizer_step: 0.2279 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7292.0 | uss_MB: 6951.9 | child_rss_MB: 4814.0 | allocated_MB: 344.5 | reserved_MB: 23586.0 | inactive_split_MB: 279.5 +rank3 [2026-10-29 06:37:01,482] INFO : Step 27500/50000 rank3 | Train Loss: 1.0739 | Train Loss Unweighted: 0.5273 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1251 | Classification: 0.0933 | Classification_binary: 0.1655 | HitGrouping: 0.1957 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1953 | Regression_cos_phi: 0.0853 | Regression_energy: 0.0637 | Regression_eta: 0.0594 | Regression_pt: 0.0663 | Regression_sin_phi: 0.1235 | LR: 4.80e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2222s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7686 | max_loss: 0.1542 | max_backward: 1.3750 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.8628 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 7288.8 | uss_MB: 6945.7 | child_rss_MB: 4804.4 | allocated_MB: 343.6 | reserved_MB: 23586.0 | inactive_split_MB: 241.4 +rank3 [2026-10-29 06:40:55,521] INFO : Step 27600/50000 rank3 | Train Loss: 1.0507 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1163 | Classification: 0.1024 | Classification_binary: 0.1467 | HitGrouping: 0.1992 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.1736 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0688 | Regression_eta: 0.0375 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0533 | LR: 4.77e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4900s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0060 | max_forward: 0.6744 | max_loss: 0.2492 | max_backward: 1.6867 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.5257 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 7290.8 | uss_MB: 6853.7 | child_rss_MB: 4808.0 | allocated_MB: 341.2 | reserved_MB: 24480.0 | inactive_split_MB: 234.8 +rank3 [2026-10-29 06:44:47,870] INFO : Step 27700/50000 rank3 | Train Loss: 1.1881 | Train Loss Unweighted: 0.5834 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1544 | Classification: 0.1001 | Classification_binary: 0.1885 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1444 | Regression_eta: 0.0015 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1737 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0895 | Regression_eta: 0.0354 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0567 | LR: 4.74e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.2918s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7716 | max_loss: 0.1527 | max_backward: 1.4941 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 0.7272 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7291.1 | uss_MB: 6952.7 | child_rss_MB: 4804.6 | allocated_MB: 344.5 | reserved_MB: 24480.0 | inactive_split_MB: 194.2 +rank3 [2026-10-29 06:48:38,928] INFO : Step 27800/50000 rank3 | Train Loss: 1.1903 | Train Loss Unweighted: 0.5862 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1435 | Classification: 0.1032 | Classification_binary: 0.2006 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1387 | Regression_eta: 0.0018 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.2006 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0600 | Regression_eta: 0.0390 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0556 | LR: 4.71e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1868s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8137 | max_loss: 0.1509 | max_backward: 1.4748 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6041 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7291.1 | uss_MB: 6953.1 | child_rss_MB: 4808.7 | allocated_MB: 348.4 | reserved_MB: 24592.0 | inactive_split_MB: 226.8 +rank3 [2026-10-29 06:52:39,106] INFO : Step 27900/50000 rank3 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5521 | Train Loss Components: Auxiliary: 0.2284 | Cardinality: 0.1139 | Classification: 0.0902 | Classification_binary: 0.1544 | HitGrouping: 0.2057 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1519 | Regression_eta: 0.0017 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1556 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0842 | Regression_eta: 0.0710 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0650 | LR: 4.68e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6323s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6397 | max_loss: 0.2456 | max_backward: 1.5234 | max_optimizer_step: 0.2276 | max_tensorboard_logging: 1.1626 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 7291.1 | uss_MB: 6955.2 | child_rss_MB: 4805.2 | allocated_MB: 342.4 | reserved_MB: 25612.0 | inactive_split_MB: 259.0 +rank3 [2026-10-29 06:56:30,759] INFO : Step 28000/50000 rank3 | Train Loss: 1.1111 | Train Loss Unweighted: 0.5401 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1274 | Classification: 0.1035 | Classification_binary: 0.1699 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0484 | Regression_eta: 0.0395 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0512 | LR: 4.66e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.0948s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.7503 | max_loss: 0.1492 | max_backward: 1.7566 | max_optimizer_step: 0.2764 | max_tensorboard_logging: 1.2295 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 7291.2 | uss_MB: 5984.8 | child_rss_MB: 4794.8 | allocated_MB: 344.1 | reserved_MB: 25612.0 | inactive_split_MB: 243.2 +rank3 [2026-10-29 06:56:31,843] INFO : Running validation on rank3 +rank3 [2026-10-29 06:56:31,844] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 06:56:31,844] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 06:57:25,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:57:25,561] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 06:57:52,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 06:57:53,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 06:57:54,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 06:57:55,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 06:58:05,303] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 06:58:05,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:58:06,180] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 06:58:22,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 06:58:23,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 06:58:23,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 06:58:24,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 06:58:34,936] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 06:58:35,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:58:35,800] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 06:58:51,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 06:58:51,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 06:58:52,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 06:58:53,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 06:59:03,324] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 07:03:11,251] INFO : Step 28100/50000 rank3 | Train Loss: 1.1727 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1644 | Classification: 0.0934 | Classification_binary: 0.1929 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1365 | Regression_eta: 0.0015 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1946 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0680 | Regression_eta: 0.0414 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0862 | LR: 4.63e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9499s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8425 | max_loss: 0.2571 | max_backward: 1.8157 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 1.0134 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 8340.2 | uss_MB: 8005.6 | child_rss_MB: 4802.9 | allocated_MB: 344.8 | reserved_MB: 22140.0 | inactive_split_MB: 293.2 +rank3 [2026-10-29 07:07:01,162] INFO : Step 28200/50000 rank3 | Train Loss: 1.0520 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2296 | Cardinality: 0.1203 | Classification: 0.0934 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1466 | Regression_eta: 0.0014 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0832 | Regression_eta: 0.0368 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0402 | LR: 4.60e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3562s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7943 | max_loss: 0.1496 | max_backward: 1.4503 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 0.7872 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 8343.2 | uss_MB: 8006.4 | child_rss_MB: 4796.0 | allocated_MB: 341.0 | reserved_MB: 24034.0 | inactive_split_MB: 210.4 +rank3 [2026-10-29 07:10:52,728] INFO : Step 28300/50000 rank3 | Train Loss: 1.0602 | Train Loss Unweighted: 0.5453 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1043 | Classification: 0.0923 | Classification_binary: 0.1501 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1496 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0774 | Regression_eta: 0.0468 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0457 | LR: 4.57e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2458s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6482 | max_loss: 0.2266 | max_backward: 1.5658 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.6870 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0836 | Memory: rss_MB: 7393.0 | uss_MB: 7059.5 | child_rss_MB: 4787.3 | allocated_MB: 342.7 | reserved_MB: 25016.0 | inactive_split_MB: 243.9 +rank3 [2026-10-29 07:14:40,573] INFO : Step 28400/50000 rank3 | Train Loss: 1.1507 | Train Loss Unweighted: 0.5391 | Train Loss Components: Auxiliary: 0.2436 | Cardinality: 0.1500 | Classification: 0.0965 | Classification_binary: 0.1896 | HitGrouping: 0.1911 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1241 | Regression_eta: 0.0018 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.2057 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0719 | Regression_eta: 0.0384 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0804 | LR: 4.55e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6298s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7664 | max_loss: 0.1532 | max_backward: 1.7342 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.8455 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7394.0 | uss_MB: 7059.9 | child_rss_MB: 4785.1 | allocated_MB: 344.2 | reserved_MB: 25016.0 | inactive_split_MB: 278.9 +rank3 [2026-10-29 07:18:29,002] INFO : Step 28500/50000 rank3 | Train Loss: 1.1595 | Train Loss Unweighted: 0.5640 | Train Loss Components: Auxiliary: 0.2492 | Cardinality: 0.1556 | Classification: 0.0927 | Classification_binary: 0.1908 | HitGrouping: 0.1954 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1384 | Regression_eta: 0.0015 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1791 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0655 | Regression_eta: 0.0370 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0546 | LR: 4.52e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2727s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.8033 | max_loss: 0.1665 | max_backward: 1.5231 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.6178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0709 | Memory: rss_MB: 7393.5 | uss_MB: 7056.4 | child_rss_MB: 4808.7 | allocated_MB: 347.4 | reserved_MB: 26000.0 | inactive_split_MB: 283.2 +rank3 [2026-10-29 07:22:17,774] INFO : Step 28600/50000 rank3 | Train Loss: 1.0672 | Train Loss Unweighted: 0.5222 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1172 | Classification: 0.0893 | Classification_binary: 0.1544 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0332 | Classification_binary: 0.1273 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0574 | Regression_eta: 0.0335 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0593 | LR: 4.49e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0230s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6704 | max_loss: 0.1829 | max_backward: 1.2353 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.8074 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7396.5 | uss_MB: 7057.8 | child_rss_MB: 4793.1 | allocated_MB: 342.3 | reserved_MB: 26000.0 | inactive_split_MB: 243.7 +rank3 [2026-10-29 07:26:10,154] INFO : Step 28700/50000 rank3 | Train Loss: 1.1535 | Train Loss Unweighted: 0.5811 | Train Loss Components: Auxiliary: 0.2469 | Cardinality: 0.1254 | Classification: 0.1220 | Classification_binary: 0.1661 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1442 | Regression_eta: 0.0021 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0794 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0862 | Regression_energy: 0.0598 | Regression_eta: 0.1003 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0562 | LR: 4.46e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5814s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6905 | max_loss: 0.2072 | max_backward: 1.6876 | max_optimizer_step: 0.2353 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 7397.5 | uss_MB: 7064.1 | child_rss_MB: 4802.4 | allocated_MB: 343.5 | reserved_MB: 26000.0 | inactive_split_MB: 234.1 +rank3 [2026-10-29 07:30:12,160] INFO : Step 28800/50000 rank3 | Train Loss: 1.2131 | Train Loss Unweighted: 0.5739 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1788 | Classification: 0.1057 | Classification_binary: 0.2127 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2357 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0616 | Regression_eta: 0.0395 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0519 | LR: 4.44e-04 | DataLoad Time: 0.0119s | Model Forward Time: 6.8525s | Phase Times: max_pre_step_barrier: 0.0263 | max_batch_to_device: 0.0149 | max_forward: 1.2928 | max_loss: 0.2713 | max_backward: 3.0161 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.8895 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 7398.3 | uss_MB: 5722.3 | child_rss_MB: 4796.9 | allocated_MB: 343.8 | reserved_MB: 26000.0 | inactive_split_MB: 273.3 +rank3 [2026-10-29 07:34:11,570] INFO : Step 28900/50000 rank3 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1252 | Classification: 0.0967 | Classification_binary: 0.1637 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1509 | Regression_eta: 0.0015 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0797 | Regression_eta: 0.0545 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0715 | LR: 4.41e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.1519s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7512 | max_loss: 0.2189 | max_backward: 1.6229 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.9330 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7399.6 | uss_MB: 5634.1 | child_rss_MB: 4800.1 | allocated_MB: 339.6 | reserved_MB: 26000.0 | inactive_split_MB: 223.0 +rank3 [2026-10-29 07:38:06,079] INFO : Step 29000/50000 rank3 | Train Loss: 1.0238 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1070 | Classification: 0.0861 | Classification_binary: 0.1439 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1389 | Regression_eta: 0.0014 | Regression_pt: 0.1414 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1468 | Regression_cos_phi: 0.1070 | Regression_energy: 0.0569 | Regression_eta: 0.0277 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0397 | LR: 4.38e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9819s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.6636 | max_loss: 0.2429 | max_backward: 1.5394 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.5216 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7400.5 | uss_MB: 7067.2 | child_rss_MB: 4794.8 | allocated_MB: 345.2 | reserved_MB: 26984.0 | inactive_split_MB: 235.2 +rank3 [2026-10-29 07:38:07,029] INFO : Running validation on rank3 +rank3 [2026-10-29 07:38:07,029] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 07:38:07,029] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 07:38:57,932] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:38:58,450] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 07:39:24,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 07:39:25,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 07:39:26,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 07:39:27,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 07:39:37,617] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 07:39:38,025] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:39:38,577] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 07:39:54,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 07:39:55,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 07:39:56,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 07:39:56,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 07:40:06,981] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 07:40:07,779] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:40:08,276] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 07:40:23,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 07:40:24,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 07:40:25,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 07:40:25,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 07:40:35,911] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 07:45:01,997] INFO : Step 29100/50000 rank3 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5808 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1279 | Classification: 0.1113 | Classification_binary: 0.1973 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1339 | Regression_eta: 0.0017 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1929 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0566 | Regression_eta: 0.0392 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0629 | LR: 4.35e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8724s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.8060 | max_loss: 0.1503 | max_backward: 1.9210 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 8379.4 | uss_MB: 8043.2 | child_rss_MB: 4787.2 | allocated_MB: 344.6 | reserved_MB: 24404.0 | inactive_split_MB: 274.9 +rank3 [2026-10-29 07:48:58,054] INFO : Step 29200/50000 rank3 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5633 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1265 | Classification: 0.0969 | Classification_binary: 0.1883 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1368 | Regression_eta: 0.0014 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1776 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0664 | Regression_eta: 0.0284 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0498 | LR: 4.33e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.8367s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0742 | max_forward: 1.0303 | max_loss: 0.2183 | max_backward: 1.8745 | max_optimizer_step: 0.2501 | max_tensorboard_logging: 0.5570 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8379.4 | uss_MB: 8042.7 | child_rss_MB: 4798.0 | allocated_MB: 343.1 | reserved_MB: 24404.0 | inactive_split_MB: 228.1 +rank3 [2026-10-29 07:52:51,791] INFO : Step 29300/50000 rank3 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2330 | Cardinality: 0.1231 | Classification: 0.0898 | Classification_binary: 0.1516 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1428 | Regression_eta: 0.0017 | Regression_pt: 0.1453 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1683 | Regression_cos_phi: 0.0840 | Regression_energy: 0.0681 | Regression_eta: 0.0474 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0613 | LR: 4.30e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7351s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0067 | max_forward: 0.7139 | max_loss: 0.1578 | max_backward: 1.5096 | max_optimizer_step: 0.2586 | max_tensorboard_logging: 1.1324 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7473.7 | uss_MB: 6607.6 | child_rss_MB: 4797.1 | allocated_MB: 341.0 | reserved_MB: 24404.0 | inactive_split_MB: 247.0 +rank3 [2026-10-29 07:56:44,367] INFO : Step 29400/50000 rank3 | Train Loss: 1.1400 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1377 | Classification: 0.1135 | Classification_binary: 0.1661 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1386 | Regression_eta: 0.0019 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1692 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0504 | Regression_eta: 0.0325 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0427 | LR: 4.27e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.2878s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.6285 | max_loss: 0.1485 | max_backward: 1.3787 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.9179 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7474.7 | uss_MB: 7139.2 | child_rss_MB: 4788.9 | allocated_MB: 343.3 | reserved_MB: 24404.0 | inactive_split_MB: 277.1 +rank3 [2026-10-29 08:00:58,644] INFO : Step 29500/50000 rank3 | Train Loss: 1.1987 | Train Loss Unweighted: 0.6106 | Train Loss Components: Auxiliary: 0.2551 | Cardinality: 0.1277 | Classification: 0.1201 | Classification_binary: 0.2013 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1419 | Regression_eta: 0.0019 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1761 | Regression_cos_phi: 0.0566 | Regression_energy: 0.0614 | Regression_eta: 0.0385 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0485 | LR: 4.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.8972s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0749 | max_forward: 0.8153 | max_loss: 0.3465 | max_backward: 2.8565 | max_optimizer_step: 0.3442 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7477.7 | uss_MB: 7142.6 | child_rss_MB: 4792.1 | allocated_MB: 343.9 | reserved_MB: 24404.0 | inactive_split_MB: 271.5 +rank3 [2026-10-29 08:05:08,851] INFO : Step 29600/50000 rank3 | Train Loss: 1.1065 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2391 | Cardinality: 0.1179 | Classification: 0.0927 | Classification_binary: 0.1666 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0018 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1446 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0642 | Regression_eta: 0.0540 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0424 | LR: 4.22e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4740s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8025 | max_loss: 0.1485 | max_backward: 1.4674 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.8982 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 7476.5 | uss_MB: 7138.1 | child_rss_MB: 4790.4 | allocated_MB: 340.8 | reserved_MB: 24404.0 | inactive_split_MB: 290.7 +rank3 [2026-10-29 08:09:13,804] INFO : Step 29700/50000 rank3 | Train Loss: 1.0488 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2273 | Cardinality: 0.1414 | Classification: 0.0952 | Classification_binary: 0.1459 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0701 | Regression_eta: 0.0470 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0818 | LR: 4.19e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5175s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0057 | max_forward: 0.7342 | max_loss: 0.1652 | max_backward: 1.4907 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.9743 | max_tensorboard_flush: 0.0140 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7477.5 | uss_MB: 6998.2 | child_rss_MB: 4799.0 | allocated_MB: 342.3 | reserved_MB: 24404.0 | inactive_split_MB: 236.6 +rank3 [2026-10-29 08:13:32,079] INFO : Step 29800/50000 rank3 | Train Loss: 1.1686 | Train Loss Unweighted: 0.5644 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1551 | Classification: 0.0952 | Classification_binary: 0.1922 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1357 | Regression_eta: 0.0014 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.2111 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0687 | Regression_eta: 0.0523 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0877 | LR: 4.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6770s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7294 | max_loss: 0.1685 | max_backward: 1.6547 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9409 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7478.5 | uss_MB: 7143.5 | child_rss_MB: 4794.4 | allocated_MB: 345.0 | reserved_MB: 24404.0 | inactive_split_MB: 265.1 +rank3 [2026-10-29 08:17:37,375] INFO : Step 29900/50000 rank3 | Train Loss: 1.1305 | Train Loss Unweighted: 0.5666 | Train Loss Components: Auxiliary: 0.2471 | Cardinality: 0.1380 | Classification: 0.0944 | Classification_binary: 0.1750 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1464 | Regression_eta: 0.0015 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0968 | Regression_energy: 0.0712 | Regression_eta: 0.0373 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0960 | LR: 4.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3510s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0611 | max_forward: 0.8060 | max_loss: 0.1494 | max_backward: 1.3251 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 7479.5 | uss_MB: 7143.0 | child_rss_MB: 4800.7 | allocated_MB: 344.7 | reserved_MB: 26422.0 | inactive_split_MB: 298.4 +rank3 [2026-10-29 08:22:04,989] INFO : Step 30000/50000 rank3 | Train Loss: 1.0854 | Train Loss Unweighted: 0.5586 | Train Loss Components: Auxiliary: 0.2416 | Cardinality: 0.1337 | Classification: 0.1000 | Classification_binary: 0.1541 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1499 | Regression_eta: 0.0014 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0760 | Regression_eta: 0.0371 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0446 | LR: 4.11e-04 | DataLoad Time: 0.0208s | Model Forward Time: 7.0742s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0172 | max_forward: 0.9685 | max_loss: 0.2166 | max_backward: 4.0112 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.8430 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7461.9 | uss_MB: 7142.5 | child_rss_MB: 4797.1 | allocated_MB: 338.0 | reserved_MB: 26422.0 | inactive_split_MB: 288.0 +rank3 [2026-10-29 08:22:05,952] INFO : Running validation on rank3 +rank3 [2026-10-29 08:22:05,952] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 08:22:05,952] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 08:22:57,876] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:22:58,358] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 08:23:24,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 08:23:25,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 08:23:26,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 08:23:27,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 08:23:37,981] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 08:23:38,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:23:38,901] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 08:23:54,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 08:23:55,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 08:23:56,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 08:23:57,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 08:24:07,621] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 08:24:08,113] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:24:08,619] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 08:24:24,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 08:24:24,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 08:24:25,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 08:24:26,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 08:24:36,307] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 08:29:00,406] INFO : Step 30100/50000 rank3 | Train Loss: 1.0762 | Train Loss Unweighted: 0.5358 | Train Loss Components: Auxiliary: 0.2378 | Cardinality: 0.1106 | Classification: 0.1097 | Classification_binary: 0.1679 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1842 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0562 | Regression_eta: 0.0385 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0573 | LR: 4.08e-04 | DataLoad Time: 0.0165s | Model Forward Time: 8.2917s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7352 | max_loss: 0.1443 | max_backward: 1.7375 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 1.7731 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.3497 | Memory: rss_MB: 7528.6 | uss_MB: 6331.6 | child_rss_MB: 4801.2 | allocated_MB: 344.1 | reserved_MB: 22922.0 | inactive_split_MB: 248.8 +rank3 [2026-10-29 08:33:16,461] INFO : Step 30200/50000 rank3 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5304 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1421 | Classification: 0.0934 | Classification_binary: 0.1839 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0016 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1737 | Regression_cos_phi: 0.0981 | Regression_energy: 0.0721 | Regression_eta: 0.0416 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0602 | LR: 4.06e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6001s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8008 | max_loss: 0.1560 | max_backward: 1.6706 | max_optimizer_step: 0.2663 | max_tensorboard_logging: 0.7725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0555 | Memory: rss_MB: 7535.6 | uss_MB: 7212.9 | child_rss_MB: 4775.8 | allocated_MB: 344.1 | reserved_MB: 23882.0 | inactive_split_MB: 274.8 +rank3 [2026-10-29 08:37:30,482] INFO : Step 30300/50000 rank3 | Train Loss: 1.0429 | Train Loss Unweighted: 0.5252 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1223 | Classification: 0.0922 | Classification_binary: 0.1566 | HitGrouping: 0.1929 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0681 | Regression_eta: 0.0288 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0601 | LR: 4.03e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.6252s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0060 | max_forward: 1.1719 | max_loss: 0.2884 | max_backward: 2.4994 | max_optimizer_step: 0.2255 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7536.6 | uss_MB: 7213.9 | child_rss_MB: 4771.6 | allocated_MB: 341.8 | reserved_MB: 23882.0 | inactive_split_MB: 270.7 +rank3 [2026-10-29 08:41:40,106] INFO : Step 30400/50000 rank3 | Train Loss: 1.0543 | Train Loss Unweighted: 0.5182 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1318 | Classification: 0.0889 | Classification_binary: 0.1487 | HitGrouping: 0.1973 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1382 | Regression_eta: 0.0014 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1507 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0602 | Regression_eta: 0.0431 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0726 | LR: 4.00e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.0783s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0150 | max_forward: 0.8845 | max_loss: 0.2482 | max_backward: 2.1745 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.5112 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7452.2 | uss_MB: 7214.6 | child_rss_MB: 4475.9 | allocated_MB: 344.7 | reserved_MB: 24842.0 | inactive_split_MB: 245.3 +rank3 [2026-10-29 08:45:49,048] INFO : Step 30500/50000 rank3 | Train Loss: 1.1589 | Train Loss Unweighted: 0.5596 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1407 | Classification: 0.1085 | Classification_binary: 0.1952 | HitGrouping: 0.1960 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1909 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0557 | Regression_eta: 0.0573 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0807 | LR: 3.98e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.7056s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7791 | max_loss: 0.2528 | max_backward: 1.9433 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.5210 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0672 | Memory: rss_MB: 7432.7 | uss_MB: 7008.9 | child_rss_MB: 4374.7 | allocated_MB: 346.2 | reserved_MB: 26806.0 | inactive_split_MB: 227.8 +rank3 [2026-10-29 08:49:58,495] INFO : Step 30600/50000 rank3 | Train Loss: 1.1269 | Train Loss Unweighted: 0.5561 | Train Loss Components: Auxiliary: 0.2414 | Cardinality: 0.1456 | Classification: 0.0973 | Classification_binary: 0.1804 | HitGrouping: 0.1953 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1373 | Regression_eta: 0.0014 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.2030 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0801 | Regression_eta: 0.0317 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0615 | LR: 3.95e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4778s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1011 | max_forward: 0.8824 | max_loss: 0.1534 | max_backward: 1.4258 | max_optimizer_step: 0.3067 | max_tensorboard_logging: 0.7925 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 7378.2 | uss_MB: 7215.1 | child_rss_MB: 4177.7 | allocated_MB: 345.8 | reserved_MB: 27800.0 | inactive_split_MB: 283.5 +rank3 [2026-10-29 08:54:17,062] INFO : Step 30700/50000 rank3 | Train Loss: 1.0989 | Train Loss Unweighted: 0.5515 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1377 | Classification: 0.0890 | Classification_binary: 0.1568 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0013 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.1964 | Regression_cos_phi: 0.0890 | Regression_energy: 0.0612 | Regression_eta: 0.0431 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0915 | LR: 3.92e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2940s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0056 | max_forward: 0.7546 | max_loss: 0.2396 | max_backward: 1.3389 | max_optimizer_step: 0.2508 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0572 | Memory: rss_MB: 7380.2 | uss_MB: 7216.5 | child_rss_MB: 4178.9 | allocated_MB: 340.3 | reserved_MB: 27800.0 | inactive_split_MB: 223.2 +rank3 [2026-10-29 08:58:31,924] INFO : Step 30800/50000 rank3 | Train Loss: 1.1241 | Train Loss Unweighted: 0.5488 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1377 | Classification: 0.1081 | Classification_binary: 0.1717 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1317 | Regression_eta: 0.0015 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0624 | Regression_eta: 0.0223 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0628 | LR: 3.90e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.8021s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7775 | max_loss: 0.2735 | max_backward: 1.9298 | max_optimizer_step: 0.3489 | max_tensorboard_logging: 0.8756 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0726 | Memory: rss_MB: 7357.4 | uss_MB: 5945.6 | child_rss_MB: 4116.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 275.1 +rank3 [2026-10-29 09:02:40,911] INFO : Step 30900/50000 rank3 | Train Loss: 1.1501 | Train Loss Unweighted: 0.5762 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1300 | Classification: 0.0976 | Classification_binary: 0.1996 | HitGrouping: 0.2000 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1374 | Regression_eta: 0.0017 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1965 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0616 | Regression_eta: 0.0370 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0522 | LR: 3.87e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.5943s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.3511 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 1.0306 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 7360.4 | uss_MB: 7217.8 | child_rss_MB: 4118.5 | allocated_MB: 342.2 | reserved_MB: 27800.0 | inactive_split_MB: 198.3 +rank3 [2026-10-29 09:06:46,925] INFO : Step 31000/50000 rank3 | Train Loss: 1.1054 | Train Loss Unweighted: 0.5218 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1446 | Classification: 0.0888 | Classification_binary: 0.1693 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0016 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1678 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0507 | Regression_eta: 0.0489 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0851 | LR: 3.84e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.7153s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.8505 | max_loss: 0.2491 | max_backward: 1.6460 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9731 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7337.9 | uss_MB: 6674.8 | child_rss_MB: 4046.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 190.7 +rank3 [2026-10-29 09:06:47,902] INFO : Running validation on rank3 +rank3 [2026-10-29 09:06:47,902] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 09:06:47,902] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 09:07:39,009] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:07:39,555] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 09:08:08,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 09:08:09,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 09:08:10,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 09:08:11,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 09:08:21,521] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 09:08:22,017] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:08:22,479] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 09:08:38,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 09:08:40,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 09:08:41,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 09:08:42,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 09:08:52,950] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 09:08:52,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:08:53,432] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 09:09:08,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 09:09:09,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 09:09:09,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 09:09:10,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 09:09:20,508] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 09:13:45,030] INFO : Step 31100/50000 rank3 | Train Loss: 1.0306 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2182 | Cardinality: 0.1238 | Classification: 0.0932 | Classification_binary: 0.1415 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1071 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0662 | Regression_eta: 0.0248 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0288 | LR: 3.82e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.5114s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1068 | max_forward: 0.8067 | max_loss: 0.1436 | max_backward: 1.1427 | max_optimizer_step: 0.2576 | max_tensorboard_logging: 1.1475 | max_tensorboard_flush: 0.0054 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 8196.3 | uss_MB: 5921.9 | child_rss_MB: 4032.2 | allocated_MB: 343.8 | reserved_MB: 24492.0 | inactive_split_MB: 293.3 +rank3 [2026-10-29 09:17:44,912] INFO : Step 31200/50000 rank3 | Train Loss: 1.1102 | Train Loss Unweighted: 0.5369 | Train Loss Components: Auxiliary: 0.2472 | Cardinality: 0.1495 | Classification: 0.0959 | Classification_binary: 0.1708 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1328 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1849 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0691 | Regression_eta: 0.0378 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0587 | LR: 3.79e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1833s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7127 | max_loss: 0.1476 | max_backward: 1.5707 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7428.7 | uss_MB: 7313.8 | child_rss_MB: 4003.5 | allocated_MB: 344.5 | reserved_MB: 24648.0 | inactive_split_MB: 235.5 +rank3 [2026-10-29 09:21:47,087] INFO : Step 31300/50000 rank3 | Train Loss: 1.1865 | Train Loss Unweighted: 0.5472 | Train Loss Components: Auxiliary: 0.2511 | Cardinality: 0.1748 | Classification: 0.0979 | Classification_binary: 0.1851 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0019 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0574 | Regression_eta: 0.0484 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0468 | LR: 3.76e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.2291s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0064 | max_forward: 0.7815 | max_loss: 0.1495 | max_backward: 1.6330 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.5234 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7366.1 | uss_MB: 7313.9 | child_rss_MB: 3706.8 | allocated_MB: 344.4 | reserved_MB: 25640.0 | inactive_split_MB: 233.6 +rank3 [2026-10-29 09:25:42,887] INFO : Step 31400/50000 rank3 | Train Loss: 1.0272 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1191 | Classification: 0.0784 | Classification_binary: 0.1451 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.1296 | Regression_cos_phi: 0.1171 | Regression_energy: 0.0626 | Regression_eta: 0.0556 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0797 | LR: 3.74e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5252s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0063 | max_forward: 0.7252 | max_loss: 0.1478 | max_backward: 1.6173 | max_optimizer_step: 0.2404 | max_tensorboard_logging: 0.9541 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7335.5 | uss_MB: 7312.2 | child_rss_MB: 3625.5 | allocated_MB: 342.1 | reserved_MB: 25640.0 | inactive_split_MB: 296.4 +rank3 [2026-10-29 09:29:44,709] INFO : Step 31500/50000 rank3 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5247 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1206 | Classification: 0.1037 | Classification_binary: 0.1583 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1297 | Regression_eta: 0.0014 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0517 | Regression_eta: 0.0253 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0392 | LR: 3.71e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.3662s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6643 | max_loss: 0.2487 | max_backward: 1.5648 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.8248 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7336.0 | uss_MB: 7313.0 | child_rss_MB: 3638.4 | allocated_MB: 344.4 | reserved_MB: 26092.0 | inactive_split_MB: 238.2 +rank3 [2026-10-29 09:33:42,077] INFO : Step 31600/50000 rank3 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5471 | Train Loss Components: Auxiliary: 0.2424 | Cardinality: 0.1284 | Classification: 0.1033 | Classification_binary: 0.1828 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1282 | Regression_eta: 0.0017 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2005 | Regression_cos_phi: 0.0780 | Regression_energy: 0.0742 | Regression_eta: 0.0853 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0876 | LR: 3.69e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.0246s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7603 | max_loss: 0.1471 | max_backward: 2.0475 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.9076 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7328.4 | uss_MB: 7314.4 | child_rss_MB: 3585.6 | allocated_MB: 341.5 | reserved_MB: 26640.0 | inactive_split_MB: 287.0 +rank3 [2026-10-29 09:37:47,825] INFO : Step 31700/50000 rank3 | Train Loss: 1.0533 | Train Loss Unweighted: 0.5009 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1416 | Classification: 0.0871 | Classification_binary: 0.1632 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0371 | Regression_energy: 0.0529 | Regression_eta: 0.0318 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0617 | LR: 3.66e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1118s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7635 | max_loss: 0.1543 | max_backward: 1.2075 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7330.4 | uss_MB: 7312.9 | child_rss_MB: 3557.0 | allocated_MB: 341.9 | reserved_MB: 26640.0 | inactive_split_MB: 274.1 +rank3 [2026-10-29 09:45:40,560] INFO : Step 31800/50000 rank3 | Train Loss: 1.0385 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1148 | Classification: 0.0877 | Classification_binary: 0.1481 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1418 | Regression_eta: 0.0012 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1433 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0779 | Regression_eta: 0.0430 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0625 | LR: 3.64e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.0341s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0072 | max_forward: 0.6606 | max_loss: 0.1610 | max_backward: 1.1159 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.8258 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1417 | Memory: rss_MB: 7328.9 | uss_MB: 7312.1 | child_rss_MB: 3561.3 | allocated_MB: 345.5 | reserved_MB: 27080.0 | inactive_split_MB: 260.5 +rank3 [2026-10-29 09:49:44,089] INFO : Step 31900/50000 rank3 | Train Loss: 1.1464 | Train Loss Unweighted: 0.5585 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1372 | Classification: 0.1078 | Classification_binary: 0.1815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0017 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.2025 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0656 | Regression_eta: 0.0618 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0700 | LR: 3.61e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.5151s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7718 | max_loss: 0.1553 | max_backward: 1.7857 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.6721 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7333.9 | uss_MB: 7320.8 | child_rss_MB: 3569.3 | allocated_MB: 343.8 | reserved_MB: 27080.0 | inactive_split_MB: 280.2 +rank3 [2026-10-29 09:53:38,294] INFO : Step 32000/50000 rank3 | Train Loss: 1.1115 | Train Loss Unweighted: 0.5351 | Train Loss Components: Auxiliary: 0.2393 | Cardinality: 0.1406 | Classification: 0.0954 | Classification_binary: 0.1748 | HitGrouping: 0.1912 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1302 | Regression_eta: 0.0018 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0588 | Regression_eta: 0.0252 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0460 | LR: 3.58e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.2462s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.9767 | max_loss: 0.6919 | max_backward: 2.1955 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.9277 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 7329.3 | uss_MB: 7313.9 | child_rss_MB: 3566.4 | allocated_MB: 344.2 | reserved_MB: 27080.0 | inactive_split_MB: 265.7 +rank3 [2026-10-29 09:53:39,334] INFO : Running validation on rank3 +rank3 [2026-10-29 09:53:39,335] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 09:53:39,335] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 09:54:28,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:54:29,292] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 09:54:57,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 09:54:58,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 09:54:59,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 09:55:00,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 09:55:10,501] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 09:55:10,899] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:55:11,415] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 09:55:27,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 09:55:28,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 09:55:29,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 09:55:30,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 09:55:40,394] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 09:55:41,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:55:41,777] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 09:55:56,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 09:55:57,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 09:55:58,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 09:55:58,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 09:56:08,893] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 10:00:19,239] INFO : Step 32100/50000 rank3 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1218 | Classification: 0.0914 | Classification_binary: 0.1509 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1333 | Regression_eta: 0.0016 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1529 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0754 | Regression_eta: 0.0452 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0507 | LR: 3.56e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.3629s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7098 | max_loss: 0.1454 | max_backward: 1.1394 | max_optimizer_step: 0.2650 | max_tensorboard_logging: 0.9832 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 8404.9 | uss_MB: 8386.6 | child_rss_MB: 3573.9 | allocated_MB: 344.0 | reserved_MB: 22148.0 | inactive_split_MB: 276.8 +rank3 [2026-10-29 10:04:33,375] INFO : Step 32200/50000 rank3 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1241 | Classification: 0.1035 | Classification_binary: 0.1630 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1279 | Regression_eta: 0.0014 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1853 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0592 | Regression_eta: 0.0562 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0634 | LR: 3.53e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6699s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0891 | max_forward: 0.7517 | max_loss: 0.1441 | max_backward: 1.7835 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.7808 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7418.2 | uss_MB: 7403.1 | child_rss_MB: 3576.2 | allocated_MB: 345.5 | reserved_MB: 23122.0 | inactive_split_MB: 233.4 +rank3 [2026-10-29 10:08:31,626] INFO : Step 32300/50000 rank3 | Train Loss: 1.1438 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2535 | Cardinality: 0.1335 | Classification: 0.1042 | Classification_binary: 0.1958 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1303 | Regression_eta: 0.0015 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0679 | Regression_eta: 0.0283 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0424 | LR: 3.51e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.7844s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7857 | max_loss: 0.1780 | max_backward: 1.7580 | max_optimizer_step: 0.2428 | max_tensorboard_logging: 0.8954 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 7421.3 | uss_MB: 7404.5 | child_rss_MB: 3576.8 | allocated_MB: 343.9 | reserved_MB: 24108.0 | inactive_split_MB: 280.1 +rank3 [2026-10-29 10:12:37,147] INFO : Step 32400/50000 rank3 | Train Loss: 1.0306 | Train Loss Unweighted: 0.5063 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1168 | Classification: 0.0867 | Classification_binary: 0.1599 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0013 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1688 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0572 | Regression_eta: 0.0338 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0767 | LR: 3.48e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.3212s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.1456 | max_backward: 1.3401 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.8752 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7420.4 | uss_MB: 7401.9 | child_rss_MB: 3574.4 | allocated_MB: 344.8 | reserved_MB: 24108.0 | inactive_split_MB: 271.0 +rank3 [2026-10-29 10:16:48,486] INFO : Step 32500/50000 rank3 | Train Loss: 1.0033 | Train Loss Unweighted: 0.5126 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.0995 | Classification: 0.1015 | Classification_binary: 0.1402 | HitGrouping: 0.1898 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1332 | Regression_eta: 0.0015 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0559 | Regression_eta: 0.0307 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0725 | LR: 3.46e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9959s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6088 | max_loss: 0.1480 | max_backward: 1.0729 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.8045 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7420.4 | uss_MB: 7401.8 | child_rss_MB: 3575.0 | allocated_MB: 341.9 | reserved_MB: 25096.0 | inactive_split_MB: 268.1 +rank3 [2026-10-29 10:21:04,044] INFO : Step 32600/50000 rank3 | Train Loss: 1.0981 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1271 | Classification: 0.1053 | Classification_binary: 0.1773 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1306 | Regression_eta: 0.0014 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0636 | Regression_eta: 0.0295 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0522 | LR: 3.43e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.2744s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8529 | max_loss: 0.3369 | max_backward: 2.9831 | max_optimizer_step: 0.2386 | max_tensorboard_logging: 0.5557 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7420.7 | uss_MB: 5748.3 | child_rss_MB: 3585.6 | allocated_MB: 344.1 | reserved_MB: 25096.0 | inactive_split_MB: 287.9 +rank3 [2026-10-29 10:25:23,204] INFO : Step 32700/50000 rank3 | Train Loss: 1.0956 | Train Loss Unweighted: 0.5307 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.1282 | Classification: 0.1011 | Classification_binary: 0.1731 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0558 | Regression_eta: 0.0366 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0625 | LR: 3.41e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.1095s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0062 | max_forward: 0.8233 | max_loss: 0.1478 | max_backward: 1.5120 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0564 | Memory: rss_MB: 7420.7 | uss_MB: 7403.4 | child_rss_MB: 3588.8 | allocated_MB: 342.9 | reserved_MB: 25096.0 | inactive_split_MB: 272.6 +rank3 [2026-10-29 10:29:23,500] INFO : Step 32800/50000 rank3 | Train Loss: 1.0857 | Train Loss Unweighted: 0.5241 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1299 | Classification: 0.1011 | Classification_binary: 0.1551 | HitGrouping: 0.1982 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1327 | Regression_eta: 0.0016 | Regression_pt: 0.1319 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1710 | Regression_cos_phi: 0.0916 | Regression_energy: 0.0698 | Regression_eta: 0.0335 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0621 | LR: 3.38e-04 | DataLoad Time: 0.0253s | Model Forward Time: 4.2369s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0179 | max_forward: 0.9787 | max_loss: 0.1782 | max_backward: 1.5075 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 0.5808 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.1577 | Memory: rss_MB: 7419.5 | uss_MB: 7401.5 | child_rss_MB: 3585.9 | allocated_MB: 345.1 | reserved_MB: 25096.0 | inactive_split_MB: 263.9 +rank3 [2026-10-29 10:33:26,398] INFO : Step 32900/50000 rank3 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1116 | Classification: 0.1066 | Classification_binary: 0.1631 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1300 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0665 | Regression_eta: 0.0510 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0557 | LR: 3.36e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.6990s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 1.3284 | max_loss: 0.5826 | max_backward: 1.9391 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7423.7 | uss_MB: 7407.1 | child_rss_MB: 3589.0 | allocated_MB: 344.3 | reserved_MB: 26086.0 | inactive_split_MB: 212.1 +rank3 [2026-10-29 10:37:13,221] INFO : Step 33000/50000 rank3 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5373 | Train Loss Components: Auxiliary: 0.2310 | Cardinality: 0.1017 | Classification: 0.1099 | Classification_binary: 0.1844 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0713 | Classification_binary: 0.1866 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0570 | Regression_eta: 0.0370 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0555 | LR: 3.33e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.3108s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 0.7919 | max_loss: 0.2231 | max_backward: 1.5724 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.5025 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1340 | Memory: rss_MB: 7423.5 | uss_MB: 7406.2 | child_rss_MB: 3581.4 | allocated_MB: 344.2 | reserved_MB: 26086.0 | inactive_split_MB: 269.8 +rank3 [2026-10-29 10:37:14,208] INFO : Running validation on rank3 +rank3 [2026-10-29 10:37:14,208] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 10:37:14,208] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 10:38:03,872] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:38:04,390] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 10:38:31,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 10:38:33,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 10:38:34,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 10:38:35,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 10:38:45,297] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 10:38:45,832] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:38:46,354] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 10:39:02,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 10:39:03,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 10:39:04,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 10:39:05,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 10:39:15,581] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 10:39:15,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:39:16,280] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 10:39:31,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 10:39:32,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 10:39:33,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 10:39:33,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 10:39:43,909] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 10:43:56,312] INFO : Step 33100/50000 rank3 | Train Loss: 1.1040 | Train Loss Unweighted: 0.5268 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1427 | Classification: 0.0921 | Classification_binary: 0.1655 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1319 | Regression_eta: 0.0015 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0613 | Regression_eta: 0.0310 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0554 | LR: 3.31e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3499s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0325 | max_forward: 0.7755 | max_loss: 0.1478 | max_backward: 1.3916 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.6610 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0760 | Memory: rss_MB: 8784.9 | uss_MB: 8766.5 | child_rss_MB: 3557.4 | allocated_MB: 343.8 | reserved_MB: 23356.0 | inactive_split_MB: 267.3 +rank3 [2026-10-29 10:47:56,312] INFO : Step 33200/50000 rank3 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4955 | Train Loss Components: Auxiliary: 0.2164 | Cardinality: 0.1253 | Classification: 0.1001 | Classification_binary: 0.1407 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0019 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0671 | Regression_eta: 0.0404 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0629 | LR: 3.28e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.0976s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0401 | max_forward: 0.9954 | max_loss: 0.3116 | max_backward: 2.2139 | max_optimizer_step: 0.2657 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7500.4 | uss_MB: 7483.0 | child_rss_MB: 3560.3 | allocated_MB: 344.1 | reserved_MB: 23356.0 | inactive_split_MB: 241.6 +rank3 [2026-10-29 10:51:55,086] INFO : Step 33300/50000 rank3 | Train Loss: 1.0288 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2302 | Cardinality: 0.1166 | Classification: 0.0975 | Classification_binary: 0.1653 | HitGrouping: 0.1885 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1220 | Regression_eta: 0.0012 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0742 | Regression_eta: 0.0371 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0458 | LR: 3.26e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.5503s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7054 | max_loss: 0.2213 | max_backward: 1.6633 | max_optimizer_step: 0.2474 | max_tensorboard_logging: 0.8237 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0708 | Memory: rss_MB: 7500.4 | uss_MB: 7483.1 | child_rss_MB: 3556.5 | allocated_MB: 343.5 | reserved_MB: 23356.0 | inactive_split_MB: 261.9 +rank3 [2026-10-29 10:55:54,910] INFO : Step 33400/50000 rank3 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5337 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1249 | Classification: 0.0970 | Classification_binary: 0.1720 | HitGrouping: 0.1947 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0013 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.1617 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0740 | Regression_eta: 0.0350 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0678 | LR: 3.23e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.4611s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0073 | max_forward: 1.2444 | max_loss: 0.2025 | max_backward: 2.5109 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7502.5 | uss_MB: 7485.1 | child_rss_MB: 3549.5 | allocated_MB: 347.6 | reserved_MB: 23358.0 | inactive_split_MB: 303.0 +rank3 [2026-10-29 10:59:53,594] INFO : Step 33500/50000 rank3 | Train Loss: 1.0164 | Train Loss Unweighted: 0.4956 | Train Loss Components: Auxiliary: 0.2212 | Cardinality: 0.1274 | Classification: 0.0875 | Classification_binary: 0.1421 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1309 | Regression_eta: 0.0013 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1455 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0685 | Regression_eta: 0.0393 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0617 | LR: 3.21e-04 | DataLoad Time: 0.0123s | Model Forward Time: 4.6372s | Phase Times: max_pre_step_barrier: 0.0271 | max_batch_to_device: 0.0558 | max_forward: 0.9437 | max_loss: 0.1437 | max_backward: 1.5429 | max_optimizer_step: 0.2447 | max_tensorboard_logging: 0.8833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7505.9 | uss_MB: 7486.7 | child_rss_MB: 3558.2 | allocated_MB: 341.2 | reserved_MB: 23358.0 | inactive_split_MB: 237.5 +rank3 [2026-10-29 11:05:22,801] INFO : Step 33600/50000 rank3 | Train Loss: 1.0074 | Train Loss Unweighted: 0.4988 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1109 | Classification: 0.0969 | Classification_binary: 0.1509 | HitGrouping: 0.1901 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0012 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1366 | Regression_cos_phi: 0.0349 | Regression_energy: 0.0477 | Regression_eta: 0.0226 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0516 | LR: 3.19e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1877s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0970 | max_forward: 0.7667 | max_loss: 0.1478 | max_backward: 1.4192 | max_optimizer_step: 0.2426 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7500.7 | uss_MB: 7485.1 | child_rss_MB: 3550.7 | allocated_MB: 344.5 | reserved_MB: 24528.0 | inactive_split_MB: 238.9 +rank3 [2026-10-29 11:09:17,747] INFO : Step 33700/50000 rank3 | Train Loss: 1.1891 | Train Loss Unweighted: 0.5658 | Train Loss Components: Auxiliary: 0.2602 | Cardinality: 0.1587 | Classification: 0.1116 | Classification_binary: 0.1953 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1270 | Regression_eta: 0.0015 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1907 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0585 | Regression_eta: 0.0447 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0685 | LR: 3.16e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8180s | Phase Times: max_pre_step_barrier: 0.0253 | max_batch_to_device: 0.0066 | max_forward: 1.3051 | max_loss: 0.1764 | max_backward: 1.7331 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.5706 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0895 | Memory: rss_MB: 7502.0 | uss_MB: 7487.9 | child_rss_MB: 3548.7 | allocated_MB: 347.7 | reserved_MB: 24528.0 | inactive_split_MB: 288.7 +rank3 [2026-10-29 11:13:14,785] INFO : Step 33800/50000 rank3 | Train Loss: 1.0442 | Train Loss Unweighted: 0.4970 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1421 | Classification: 0.0764 | Classification_binary: 0.1525 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1318 | Regression_eta: 0.0014 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0698 | Regression_eta: 0.0291 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0442 | LR: 3.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2998s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7370 | max_loss: 0.2458 | max_backward: 1.5902 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.6292 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1331 | Memory: rss_MB: 7496.1 | uss_MB: 7482.2 | child_rss_MB: 3539.4 | allocated_MB: 344.2 | reserved_MB: 24528.0 | inactive_split_MB: 271.5 +rank3 [2026-10-29 11:17:10,800] INFO : Step 33900/50000 rank3 | Train Loss: 1.0341 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1352 | Classification: 0.0865 | Classification_binary: 0.1441 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1254 | Regression_eta: 0.0016 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1275 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0475 | Regression_eta: 0.0576 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0415 | LR: 3.11e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3847s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.6305 | max_loss: 0.1507 | max_backward: 1.4015 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9471 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0605 | Memory: rss_MB: 7498.1 | uss_MB: 7488.3 | child_rss_MB: 3543.5 | allocated_MB: 344.1 | reserved_MB: 24528.0 | inactive_split_MB: 268.7 +rank3 [2026-10-29 11:21:05,086] INFO : Step 34000/50000 rank3 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1309 | Classification: 0.0985 | Classification_binary: 0.1707 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1170 | Regression_eta: 0.0014 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0589 | Regression_eta: 0.0390 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0596 | LR: 3.09e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6365s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7414 | max_loss: 0.1458 | max_backward: 1.7218 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 7499.1 | uss_MB: 7491.4 | child_rss_MB: 3548.4 | allocated_MB: 342.9 | reserved_MB: 24528.0 | inactive_split_MB: 277.5 +rank3 [2026-10-29 11:21:06,162] INFO : Running validation on rank3 +rank3 [2026-10-29 11:21:06,162] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 11:21:56,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:21:57,375] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 11:22:17,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 11:22:18,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 11:22:19,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 11:22:21,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 11:22:31,356] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 11:22:32,014] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:22:32,507] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 11:22:48,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 11:22:49,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 11:22:50,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 11:22:51,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 11:23:01,588] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 11:23:02,010] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:23:02,514] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 11:23:18,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 11:23:19,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 11:23:19,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 11:23:20,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 11:23:30,680] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 11:27:45,673] INFO : Step 34100/50000 rank3 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5582 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1263 | Classification: 0.1031 | Classification_binary: 0.1801 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1358 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1778 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0734 | Regression_eta: 0.0364 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0507 | LR: 3.06e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8610s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9025 | max_loss: 0.1870 | max_backward: 1.7732 | max_optimizer_step: 0.2826 | max_tensorboard_logging: 0.8215 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0738 | Memory: rss_MB: 9109.5 | uss_MB: 8140.4 | child_rss_MB: 3555.5 | allocated_MB: 342.7 | reserved_MB: 22124.0 | inactive_split_MB: 211.6 +rank3 [2026-10-29 11:31:40,881] INFO : Step 34200/50000 rank3 | Train Loss: 0.9931 | Train Loss Unweighted: 0.4794 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1232 | Classification: 0.0771 | Classification_binary: 0.1377 | HitGrouping: 0.1996 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1301 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0804 | Regression_eta: 0.0354 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0759 | LR: 3.04e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.7103s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.8809 | max_loss: 0.3036 | max_backward: 1.8297 | max_optimizer_step: 0.2897 | max_tensorboard_logging: 0.6835 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7608.6 | uss_MB: 6291.1 | child_rss_MB: 3559.3 | allocated_MB: 339.0 | reserved_MB: 22508.0 | inactive_split_MB: 220.4 +rank3 [2026-10-29 11:35:37,831] INFO : Step 34300/50000 rank3 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4920 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1192 | Classification: 0.0965 | Classification_binary: 0.1484 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0512 | Regression_eta: 0.0481 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0454 | LR: 3.02e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.3203s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7255 | max_loss: 0.1427 | max_backward: 1.3652 | max_optimizer_step: 0.2496 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7610.6 | uss_MB: 7596.5 | child_rss_MB: 3557.3 | allocated_MB: 345.0 | reserved_MB: 22508.0 | inactive_split_MB: 216.4 +rank3 [2026-10-29 11:39:28,026] INFO : Step 34400/50000 rank3 | Train Loss: 1.0662 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2383 | Cardinality: 0.1289 | Classification: 0.0974 | Classification_binary: 0.1749 | HitGrouping: 0.1864 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1281 | Regression_eta: 0.0014 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0684 | Regression_eta: 0.0383 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0785 | LR: 2.99e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.1262s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9438 | max_loss: 0.1560 | max_backward: 2.9359 | max_optimizer_step: 0.2526 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7613.7 | uss_MB: 6304.5 | child_rss_MB: 3550.6 | allocated_MB: 345.3 | reserved_MB: 23132.0 | inactive_split_MB: 282.7 +rank3 [2026-10-29 11:43:17,761] INFO : Step 34500/50000 rank3 | Train Loss: 1.0328 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1103 | Classification: 0.0867 | Classification_binary: 0.1518 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0014 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0753 | Regression_eta: 0.0459 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0533 | LR: 2.97e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.6243s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.8153 | max_loss: 0.1429 | max_backward: 1.8299 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.5315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7615.7 | uss_MB: 6643.2 | child_rss_MB: 3549.5 | allocated_MB: 341.4 | reserved_MB: 23132.0 | inactive_split_MB: 238.6 +rank3 [2026-10-29 11:47:05,943] INFO : Step 34600/50000 rank3 | Train Loss: 1.0317 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1201 | Classification: 0.0877 | Classification_binary: 0.1434 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1359 | Regression_eta: 0.0014 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1424 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0611 | Regression_eta: 0.0389 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0578 | LR: 2.95e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2917s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.6882 | max_loss: 0.1360 | max_backward: 1.3984 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5787 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7607.9 | uss_MB: 6553.3 | child_rss_MB: 3582.8 | allocated_MB: 343.8 | reserved_MB: 23132.0 | inactive_split_MB: 233.1 +rank3 [2026-10-29 11:51:12,922] INFO : Step 34700/50000 rank3 | Train Loss: 1.1152 | Train Loss Unweighted: 0.5285 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1612 | Classification: 0.0937 | Classification_binary: 0.1734 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1283 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1738 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0636 | Regression_eta: 0.0383 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0706 | LR: 2.92e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5262s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7859 | max_loss: 0.2571 | max_backward: 1.9119 | max_optimizer_step: 0.2516 | max_tensorboard_logging: 0.5032 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7610.6 | uss_MB: 7599.4 | child_rss_MB: 3559.6 | allocated_MB: 343.9 | reserved_MB: 23132.0 | inactive_split_MB: 446.8 +rank3 [2026-10-29 11:54:56,883] INFO : Step 34800/50000 rank3 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5142 | Train Loss Components: Auxiliary: 0.2371 | Cardinality: 0.1583 | Classification: 0.0857 | Classification_binary: 0.1750 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1255 | Regression_eta: 0.0013 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1700 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0584 | Regression_eta: 0.0271 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0441 | LR: 2.90e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2203s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0423 | max_forward: 0.8214 | max_loss: 0.1472 | max_backward: 1.4019 | max_optimizer_step: 0.3234 | max_tensorboard_logging: 0.5950 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7613.6 | uss_MB: 7602.4 | child_rss_MB: 3541.0 | allocated_MB: 344.5 | reserved_MB: 23132.0 | inactive_split_MB: 201.5 +rank3 [2026-10-29 11:58:48,855] INFO : Step 34900/50000 rank3 | Train Loss: 0.9512 | Train Loss Unweighted: 0.4678 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1120 | Classification: 0.0788 | Classification_binary: 0.1318 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1262 | Regression_eta: 0.0014 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0610 | Regression_eta: 0.0486 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0535 | LR: 2.88e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2614s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0986 | max_forward: 0.7397 | max_loss: 0.1421 | max_backward: 1.3918 | max_optimizer_step: 0.3560 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7614.6 | uss_MB: 7602.0 | child_rss_MB: 3543.9 | allocated_MB: 341.5 | reserved_MB: 23500.0 | inactive_split_MB: 262.9 +rank3 [2026-10-29 12:02:43,664] INFO : Step 35000/50000 rank3 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1401 | Classification: 0.0852 | Classification_binary: 0.1555 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0012 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0567 | Regression_eta: 0.0256 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0422 | LR: 2.85e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.7036s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0159 | max_forward: 0.7961 | max_loss: 0.2615 | max_backward: 2.0104 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6382 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7618.3 | uss_MB: 7606.3 | child_rss_MB: 3544.7 | allocated_MB: 345.2 | reserved_MB: 23500.0 | inactive_split_MB: 284.5 +rank3 [2026-10-29 12:02:44,718] INFO : Running validation on rank3 +rank3 [2026-10-29 12:02:44,718] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 12:03:33,664] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 12:03:34,206] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 12:04:01,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 12:04:03,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 12:04:04,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 12:04:05,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 12:04:15,557] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 12:04:18,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 12:04:18,509] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 12:04:34,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 12:04:35,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 12:04:36,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 12:04:37,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 12:04:47,240] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 12:04:48,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 12:04:48,822] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 12:05:04,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 12:05:04,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 12:05:05,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 12:05:06,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 12:05:16,377] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 12:09:34,457] INFO : Step 35100/50000 rank3 | Train Loss: 1.1060 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1387 | Classification: 0.0926 | Classification_binary: 0.1848 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0014 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1499 | Regression_cos_phi: 0.0332 | Regression_energy: 0.0558 | Regression_eta: 0.0235 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0327 | LR: 2.83e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.6631s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.9070 | max_loss: 0.1513 | max_backward: 2.1746 | max_optimizer_step: 0.2850 | max_tensorboard_logging: 1.1439 | max_tensorboard_flush: 0.0024 | max_post_step_barrier: 0.0768 | Memory: rss_MB: 9217.3 | uss_MB: 7564.1 | child_rss_MB: 3570.6 | allocated_MB: 344.3 | reserved_MB: 22622.0 | inactive_split_MB: 852.9 +rank3 [2026-10-29 12:13:24,279] INFO : Step 35200/50000 rank3 | Train Loss: 1.0386 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1293 | Classification: 0.0826 | Classification_binary: 0.1559 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1247 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1618 | Regression_cos_phi: 0.0348 | Regression_energy: 0.0639 | Regression_eta: 0.0288 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0429 | LR: 2.81e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5304s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.8193 | max_loss: 0.1608 | max_backward: 1.3752 | max_optimizer_step: 0.2535 | max_tensorboard_logging: 0.9185 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0700 | Memory: rss_MB: 7738.0 | uss_MB: 7717.0 | child_rss_MB: 3546.7 | allocated_MB: 341.8 | reserved_MB: 25614.0 | inactive_split_MB: 192.0 +rank3 [2026-10-29 12:17:18,033] INFO : Step 35300/50000 rank3 | Train Loss: 0.9736 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1059 | Classification: 0.0970 | Classification_binary: 0.1371 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0016 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0615 | Regression_eta: 0.0652 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0562 | LR: 2.79e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6480s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6434 | max_loss: 0.2341 | max_backward: 1.6834 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0588 | Memory: rss_MB: 7740.0 | uss_MB: 7297.3 | child_rss_MB: 3543.2 | allocated_MB: 343.2 | reserved_MB: 25614.0 | inactive_split_MB: 226.8 +rank3 [2026-10-29 12:21:06,024] INFO : Step 35400/50000 rank3 | Train Loss: 1.1186 | Train Loss Unweighted: 0.5363 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1393 | Classification: 0.1069 | Classification_binary: 0.1767 | HitGrouping: 0.1943 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0016 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0931 | Regression_energy: 0.0560 | Regression_eta: 0.0463 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0584 | LR: 2.76e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.8141s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0060 | max_forward: 0.7591 | max_loss: 0.1562 | max_backward: 1.6040 | max_optimizer_step: 0.2543 | max_tensorboard_logging: 0.9624 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.0563 | Memory: rss_MB: 7735.5 | uss_MB: 7248.0 | child_rss_MB: 3540.1 | allocated_MB: 345.3 | reserved_MB: 25762.0 | inactive_split_MB: 277.5 +rank3 [2026-10-29 12:24:54,118] INFO : Step 35500/50000 rank3 | Train Loss: 1.1216 | Train Loss Unweighted: 0.5466 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1527 | Classification: 0.0970 | Classification_binary: 0.1767 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1342 | Regression_eta: 0.0014 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0357 | Regression_energy: 0.0706 | Regression_eta: 0.0279 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0270 | LR: 2.74e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4080s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0648 | max_forward: 0.8748 | max_loss: 0.1544 | max_backward: 1.6387 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 0.5130 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0632 | Memory: rss_MB: 7738.5 | uss_MB: 7721.7 | child_rss_MB: 3543.4 | allocated_MB: 343.4 | reserved_MB: 25762.0 | inactive_split_MB: 284.4 +rank3 [2026-10-29 12:28:50,969] INFO : Step 35600/50000 rank3 | Train Loss: 1.0380 | Train Loss Unweighted: 0.5039 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1124 | Classification: 0.0922 | Classification_binary: 0.1405 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0309 | Classification_binary: 0.1224 | Regression_cos_phi: 0.0375 | Regression_energy: 0.0597 | Regression_eta: 0.0365 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0414 | LR: 2.72e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8983s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0559 | max_forward: 0.7376 | max_loss: 0.1415 | max_backward: 1.1754 | max_optimizer_step: 0.2524 | max_tensorboard_logging: 0.5225 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1603 | Memory: rss_MB: 7739.4 | uss_MB: 7724.9 | child_rss_MB: 3557.9 | allocated_MB: 343.8 | reserved_MB: 25762.0 | inactive_split_MB: 233.6 +rank3 [2026-10-29 12:32:54,485] INFO : Step 35700/50000 rank3 | Train Loss: 1.0352 | Train Loss Unweighted: 0.5000 | Train Loss Components: Auxiliary: 0.2288 | Cardinality: 0.1199 | Classification: 0.1043 | Classification_binary: 0.1492 | HitGrouping: 0.1850 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1212 | Regression_eta: 0.0015 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0544 | Regression_eta: 0.0539 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0705 | LR: 2.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3739s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0360 | max_forward: 0.7358 | max_loss: 0.1419 | max_backward: 1.6761 | max_optimizer_step: 0.2531 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7741.1 | uss_MB: 7726.8 | child_rss_MB: 3543.5 | allocated_MB: 344.2 | reserved_MB: 26034.0 | inactive_split_MB: 271.8 +rank3 [2026-10-29 12:36:36,039] INFO : Step 35800/50000 rank3 | Train Loss: 1.0929 | Train Loss Unweighted: 0.5323 | Train Loss Components: Auxiliary: 0.2446 | Cardinality: 0.1241 | Classification: 0.0986 | Classification_binary: 0.1801 | HitGrouping: 0.1908 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0014 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1761 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0620 | Regression_eta: 0.0512 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0395 | LR: 2.68e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1524s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7732 | max_loss: 0.1506 | max_backward: 1.4048 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0581 | Memory: rss_MB: 7740.3 | uss_MB: 7206.4 | child_rss_MB: 3548.3 | allocated_MB: 344.1 | reserved_MB: 26034.0 | inactive_split_MB: 276.1 +rank3 [2026-10-29 14:48:21,987] INFO : Initializing model with seed=12345; process seed=12348 +rank3 [2026-10-29 14:48:21,996] INFO : configured dtype=torch.bfloat16 for autocast +rank3 [2026-10-29 14:48:45,046] INFO : Instantiating model +rank3 [2026-10-29 14:48:45,046] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank3 [2026-10-29 14:48:45,046] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank3 [2026-10-29 14:48:45,065] INFO : Input encoding initialization took 0.02s +rank3 [2026-10-29 14:48:45,065] INFO : nn0 parameters: 171776 +rank3 [2026-10-29 14:48:45,065] INFO : Initializing shared backbone layers, num_convs=6 +rank3 [2026-10-29 14:48:45,065] INFO : Initializing backbone layer 0 +rank3 [2026-10-29 14:48:45,065] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:48:45,068] INFO : Initializing backbone layer 1 +rank3 [2026-10-29 14:48:45,069] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:48:45,072] INFO : Initializing backbone layer 2 +rank3 [2026-10-29 14:48:45,072] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:48:45,075] INFO : Initializing backbone layer 3 +rank3 [2026-10-29 14:48:45,075] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:48:45,078] INFO : Initializing backbone layer 4 +rank3 [2026-10-29 14:48:45,078] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:48:45,081] INFO : Initializing backbone layer 5 +rank3 [2026-10-29 14:48:45,081] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank3 [2026-10-29 14:48:45,084] INFO : Backbone initialization took 0.02s +rank3 [2026-10-29 14:48:45,085] INFO : backbone parameters: 2374656 +rank3 [2026-10-29 14:48:45,085] INFO : Initializing output DNNs +rank3 [2026-10-29 14:48:45,133] INFO : Output DNNs initialization took 0.05s +rank3 [2026-10-29 14:48:45,133] INFO : backbone_mode=BackboneMode.SHARED +rank3 [2026-10-29 14:48:45,133] INFO : jagged_attention enabled=True +rank3 [2026-10-29 14:48:45,133] INFO : flash_attn_varlen enabled=True +rank3 [2026-10-29 14:48:45,133] INFO : task_queries enabled=False +rank3 [2026-10-29 14:48:45,134] INFO : classification_norm parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : regression_norm parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : classification_query parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : regression_query parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : classification_readout parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : regression_readout parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : output_mode=set +rank3 [2026-10-29 14:48:45,134] INFO : set_decoder parameters: 4713812 +rank3 [2026-10-29 14:48:45,134] INFO : nn_binary_particle parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : nn_pid parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : nn_pt parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : nn_eta parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : nn_sin_phi parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : nn_cos_phi parameters: 0 +rank3 [2026-10-29 14:48:45,134] INFO : nn_energy parameters: 0 +rank3 [2026-10-29 14:48:45,135] INFO : Total MLPF parameters: 7260244 +rank3 [2026-10-29 14:48:45,135] INFO : MLPF __init__ done +rank3 [2026-10-29 14:48:45,136] INFO : Instantiated model +rank3 [2026-10-29 14:48:45,136] INFO : Moving model to device rank=3 +rank3 [2026-10-29 14:48:45,171] INFO : Moved model to device rank=3 +rank3 [2026-10-29 14:48:45,172] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank3 [2026-10-29 14:48:45,513] INFO : Restoring training from step 35001 +rank3 [2026-10-29 14:48:45,519] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth +rank3 [2026-10-29 14:48:45,520] INFO : Configured model for SyncBatchNorm rank=3 +rank3 [2026-10-29 14:48:45,764] INFO : Configured model for DistributedDataParallel rank=3 +rank3 [2026-10-29 14:48:45,874] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank3 [2026-10-29 14:48:45,875] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank3 [2026-10-29 14:48:45,876] INFO : Trainable parameters: 7260244 +rank3 [2026-10-29 14:48:45,876] INFO : Non-trainable parameters: 0 +rank3 [2026-10-29 14:48:45,876] INFO : Total parameters: 7260244 +rank3 [2026-10-29 14:48:45,878] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank3 [2026-10-29 14:48:45,879] INFO : Getting dataloaders +rank3 [2026-10-29 14:48:45,879] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:48:51,716] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:48:52,723] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:48:53,924] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-29 14:48:53,925] INFO : Creating WeightedMixIterator with 1 data loaders. +rank3 [2026-10-29 14:48:53,925] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:48:54,208] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:48:54,480] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:48:54,733] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-29 14:48:54,733] INFO : Creating InterleavedIterator with 1 data loaders. +rank3 [2026-10-29 14:49:08,289] INFO : Got dataloaders +rank3 [2026-10-29 14:49:08,290] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13888] of epochs [1] +rank3 [2026-10-29 14:49:08,290] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank3 [2026-10-29 14:49:08,290] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank3 [2026-10-29 14:49:08,290] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank3 [2026-10-29 14:49:09,987] INFO : loader split=train rank=3 len=21112 +rank3 [2026-10-29 14:49:09,987] INFO : loader split=valid rank=3 len=12 +rank3 [2026-10-29 14:49:09,987] INFO : Starting training from step 35001 to 50000 on rank 3 of 8 +rank3 [2026-10-29 14:49:09,987] INFO : Creating train_iterator +rank3 [2026-10-29 14:49:09,987] INFO : ResumableSampler physical:train loading state: start_index=222208 +rank3 [2026-10-29 14:49:24,168] INFO : Created train_iterator +rank3 [2026-10-29 14:53:12,252] INFO : Step 35100/50000 rank3 | Train Loss: 1.0826 | Train Loss Unweighted: 0.5185 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1393 | Classification: 0.0961 | Classification_binary: 0.1822 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1400 | Regression_cos_phi: 0.0358 | Regression_energy: 0.0460 | Regression_eta: 0.0214 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0635 | LR: 2.83e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.4932s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.7340 | max_loss: 0.1651 | max_backward: 1.4152 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4761 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4625.8 | uss_MB: 3388.6 | child_rss_MB: 5578.9 | allocated_MB: 372.6 | reserved_MB: 22674.0 | inactive_split_MB: 823.1 +rank3 [2026-10-29 14:56:45,511] INFO : Step 35200/50000 rank3 | Train Loss: 1.0562 | Train Loss Unweighted: 0.5125 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1301 | Classification: 0.0878 | Classification_binary: 0.1540 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1331 | Regression_eta: 0.0017 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0831 | Regression_eta: 0.0353 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0614 | LR: 2.81e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.1434s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6042 | max_loss: 0.1411 | max_backward: 1.1580 | max_optimizer_step: 0.1447 | max_tensorboard_logging: 0.5108 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4625.0 | uss_MB: 3391.6 | child_rss_MB: 5591.1 | allocated_MB: 368.1 | reserved_MB: 25668.0 | inactive_split_MB: 192.7 +rank3 [2026-10-29 15:00:19,321] INFO : Step 35300/50000 rank3 | Train Loss: 0.9962 | Train Loss Unweighted: 0.4725 | Train Loss Components: Auxiliary: 0.2160 | Cardinality: 0.1172 | Classification: 0.0927 | Classification_binary: 0.1341 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1372 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0663 | Regression_eta: 0.0593 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0413 | LR: 2.79e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.0275s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.5940 | max_loss: 0.1358 | max_backward: 1.1911 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0027 | Memory: rss_MB: 4636.4 | uss_MB: 3402.6 | child_rss_MB: 5598.9 | allocated_MB: 371.5 | reserved_MB: 25668.0 | inactive_split_MB: 207.6 +rank3 [2026-10-29 15:03:53,851] INFO : Step 35400/50000 rank3 | Train Loss: 1.1270 | Train Loss Unweighted: 0.5445 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1356 | Classification: 0.0990 | Classification_binary: 0.1805 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1297 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1612 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0575 | Regression_eta: 0.0421 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0964 | LR: 2.76e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.6487s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1507 | max_backward: 1.5023 | max_optimizer_step: 0.2459 | max_tensorboard_logging: 0.5679 | max_tensorboard_flush: 0.0064 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4646.3 | uss_MB: 3410.8 | child_rss_MB: 5599.4 | allocated_MB: 371.4 | reserved_MB: 25816.0 | inactive_split_MB: 263.6 +rank3 [2026-10-29 15:07:33,750] INFO : Step 35500/50000 rank3 | Train Loss: 1.0502 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1302 | Classification: 0.0904 | Classification_binary: 0.1703 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1250 | Regression_eta: 0.0013 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1351 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0688 | Regression_eta: 0.0214 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0319 | LR: 2.74e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.6099s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0737 | max_forward: 0.7546 | max_loss: 0.1581 | max_backward: 1.4977 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4644.7 | uss_MB: 3412.9 | child_rss_MB: 5568.4 | allocated_MB: 371.1 | reserved_MB: 25816.0 | inactive_split_MB: 263.4 +rank3 [2026-10-29 15:11:19,389] INFO : Step 35600/50000 rank3 | Train Loss: 1.0258 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2226 | Cardinality: 0.1119 | Classification: 0.0922 | Classification_binary: 0.1418 | HitGrouping: 0.1950 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1286 | Regression_eta: 0.0015 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1238 | Regression_cos_phi: 0.0329 | Regression_energy: 0.0559 | Regression_eta: 0.0319 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0338 | LR: 2.72e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.2533s | Phase Times: max_pre_step_barrier: 0.0929 | max_batch_to_device: 0.0064 | max_forward: 0.5696 | max_loss: 0.1430 | max_backward: 1.1627 | max_optimizer_step: 0.1475 | max_tensorboard_logging: 0.5805 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4646.9 | uss_MB: 3412.7 | child_rss_MB: 5602.7 | allocated_MB: 370.5 | reserved_MB: 25816.0 | inactive_split_MB: 212.3 +rank3 [2026-10-29 15:15:08,570] INFO : Step 35700/50000 rank3 | Train Loss: 1.0162 | Train Loss Unweighted: 0.5013 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1185 | Classification: 0.0946 | Classification_binary: 0.1537 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1248 | Regression_eta: 0.0015 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1596 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0543 | Regression_eta: 0.0317 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0442 | LR: 2.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5927s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7397 | max_loss: 0.1413 | max_backward: 1.6688 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4646.6 | uss_MB: 3411.7 | child_rss_MB: 5604.2 | allocated_MB: 372.2 | reserved_MB: 26084.0 | inactive_split_MB: 249.8 +rank3 [2026-10-29 15:18:49,408] INFO : Step 35800/50000 rank3 | Train Loss: 1.0884 | Train Loss Unweighted: 0.5265 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1296 | Classification: 0.0948 | Classification_binary: 0.1803 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1796 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0564 | Regression_eta: 0.0468 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0728 | LR: 2.68e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.2019s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0199 | max_forward: 0.9402 | max_loss: 0.2389 | max_backward: 1.7889 | max_optimizer_step: 0.2466 | max_tensorboard_logging: 0.7071 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4643.4 | uss_MB: 3414.5 | child_rss_MB: 5602.6 | allocated_MB: 371.6 | reserved_MB: 26084.0 | inactive_split_MB: 257.5 +rank3 [2026-10-29 15:22:30,827] INFO : Step 35900/50000 rank3 | Train Loss: 1.0760 | Train Loss Unweighted: 0.5299 | Train Loss Components: Auxiliary: 0.2328 | Cardinality: 0.1277 | Classification: 0.0983 | Classification_binary: 0.1568 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1349 | Regression_eta: 0.0016 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0483 | Regression_eta: 0.0343 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0536 | LR: 2.65e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.2893s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6268 | max_loss: 0.2373 | max_backward: 1.2570 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.4934 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0016 | Memory: rss_MB: 4639.4 | uss_MB: 3412.6 | child_rss_MB: 5605.5 | allocated_MB: 369.7 | reserved_MB: 26800.0 | inactive_split_MB: 294.3 +rank3 [2026-10-29 15:26:13,158] INFO : Step 36000/50000 rank3 | Train Loss: 1.0831 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1276 | Classification: 0.1031 | Classification_binary: 0.1434 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1452 | Regression_eta: 0.0017 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0893 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0632 | Regression_eta: 0.0744 | Regression_pt: 0.0666 | Regression_sin_phi: 0.1025 | LR: 2.63e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.2175s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6266 | max_loss: 0.1437 | max_backward: 1.3209 | max_optimizer_step: 0.1464 | max_tensorboard_logging: 0.4977 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4620.1 | uss_MB: 3411.5 | child_rss_MB: 5582.3 | allocated_MB: 368.7 | reserved_MB: 26800.0 | inactive_split_MB: 270.9 +rank3 [2026-10-29 15:26:14,074] INFO : Running validation on rank3 +rank3 [2026-10-29 15:26:14,074] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 15:26:14,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 15:27:03,342] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:27:03,874] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 15:27:29,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 15:27:30,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 15:27:31,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 15:27:32,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 15:27:42,415] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 15:27:42,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:27:43,314] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 15:27:58,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 15:27:59,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 15:28:00,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 15:28:01,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 15:28:11,575] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 15:28:12,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:28:13,125] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 15:28:28,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 15:28:29,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 15:28:29,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 15:28:30,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 15:28:40,555] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 15:32:50,531] INFO : Step 36100/50000 rank3 | Train Loss: 1.0137 | Train Loss Unweighted: 0.5026 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1079 | Classification: 0.1014 | Classification_binary: 0.1522 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1216 | Regression_eta: 0.0013 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0535 | Regression_eta: 0.0368 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0644 | LR: 2.61e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0473s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0065 | max_forward: 0.6968 | max_loss: 0.2782 | max_backward: 1.4278 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.9130 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6101.3 | uss_MB: 4907.2 | child_rss_MB: 5609.1 | allocated_MB: 372.5 | reserved_MB: 23174.0 | inactive_split_MB: 246.6 +rank3 [2026-10-29 15:36:33,882] INFO : Step 36200/50000 rank3 | Train Loss: 1.0439 | Train Loss Unweighted: 0.5147 | Train Loss Components: Auxiliary: 0.2338 | Cardinality: 0.1304 | Classification: 0.0909 | Classification_binary: 0.1697 | HitGrouping: 0.1917 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1255 | Regression_eta: 0.0011 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1817 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0659 | Regression_eta: 0.0337 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0590 | LR: 2.59e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5409s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7440 | max_loss: 0.1544 | max_backward: 1.4037 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 6101.4 | uss_MB: 4909.6 | child_rss_MB: 5587.9 | allocated_MB: 367.8 | reserved_MB: 23174.0 | inactive_split_MB: 282.8 +rank3 [2026-10-29 15:40:20,925] INFO : Step 36300/50000 rank3 | Train Loss: 1.0379 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1216 | Classification: 0.0886 | Classification_binary: 0.1447 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1266 | Regression_eta: 0.0016 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0938 | Regression_energy: 0.0688 | Regression_eta: 0.0717 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0736 | LR: 2.57e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.3307s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6106 | max_loss: 0.2412 | max_backward: 1.3407 | max_optimizer_step: 0.1535 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1063 | Memory: rss_MB: 5450.1 | uss_MB: 4269.0 | child_rss_MB: 5611.1 | allocated_MB: 370.5 | reserved_MB: 23174.0 | inactive_split_MB: 274.2 +rank3 [2026-10-29 15:44:06,295] INFO : Step 36400/50000 rank3 | Train Loss: 1.0182 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.0976 | Classification: 0.0939 | Classification_binary: 0.1520 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1167 | Regression_eta: 0.0016 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0523 | Regression_eta: 0.0349 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0518 | LR: 2.55e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.5218s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.6772 | max_loss: 0.1711 | max_backward: 1.4058 | max_optimizer_step: 0.1518 | max_tensorboard_logging: 0.5460 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0089 | Memory: rss_MB: 5451.9 | uss_MB: 4272.1 | child_rss_MB: 5615.2 | allocated_MB: 371.8 | reserved_MB: 23382.0 | inactive_split_MB: 230.6 +rank3 [2026-10-29 15:47:54,064] INFO : Step 36500/50000 rank3 | Train Loss: 1.0953 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1370 | Classification: 0.0895 | Classification_binary: 0.1838 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0581 | Regression_eta: 0.0324 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0496 | LR: 2.52e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9815s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7475 | max_loss: 0.2537 | max_backward: 1.7352 | max_optimizer_step: 0.1520 | max_tensorboard_logging: 0.6267 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5440.5 | uss_MB: 4262.4 | child_rss_MB: 5614.9 | allocated_MB: 372.3 | reserved_MB: 24160.0 | inactive_split_MB: 275.1 +rank3 [2026-10-29 15:51:40,079] INFO : Step 36600/50000 rank3 | Train Loss: 0.9967 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1165 | Classification: 0.0804 | Classification_binary: 0.1438 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1366 | Regression_eta: 0.0013 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0557 | Regression_eta: 0.0267 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0458 | LR: 2.50e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.6283s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6592 | max_loss: 0.2501 | max_backward: 1.4213 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5953 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5450.1 | uss_MB: 4272.8 | child_rss_MB: 5617.4 | allocated_MB: 367.8 | reserved_MB: 27180.0 | inactive_split_MB: 225.4 +rank3 [2026-10-29 15:55:26,196] INFO : Step 36700/50000 rank3 | Train Loss: 0.9770 | Train Loss Unweighted: 0.4817 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1070 | Classification: 0.0901 | Classification_binary: 0.1347 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1471 | Regression_cos_phi: 0.0289 | Regression_energy: 0.0593 | Regression_eta: 0.0389 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0535 | LR: 2.48e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.2747s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0062 | max_forward: 0.5676 | max_loss: 0.2381 | max_backward: 1.1343 | max_optimizer_step: 0.2417 | max_tensorboard_logging: 0.4693 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5448.5 | uss_MB: 4274.1 | child_rss_MB: 5620.1 | allocated_MB: 370.8 | reserved_MB: 27180.0 | inactive_split_MB: 235.2 +rank3 [2026-10-29 15:59:20,982] INFO : Step 36800/50000 rank3 | Train Loss: 1.0234 | Train Loss Unweighted: 0.4967 | Train Loss Components: Auxiliary: 0.2267 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1662 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1197 | Regression_eta: 0.0013 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1372 | Regression_cos_phi: 0.0283 | Regression_energy: 0.0516 | Regression_eta: 0.0234 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0343 | LR: 2.46e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5112s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 1.1394 | max_loss: 0.1518 | max_backward: 2.3088 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5450.3 | uss_MB: 4278.9 | child_rss_MB: 5629.5 | allocated_MB: 371.9 | reserved_MB: 27180.0 | inactive_split_MB: 230.7 +rank3 [2026-10-29 16:03:11,410] INFO : Step 36900/50000 rank3 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2268 | Cardinality: 0.1211 | Classification: 0.0855 | Classification_binary: 0.1637 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1219 | Regression_eta: 0.0015 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0609 | Regression_eta: 0.0524 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0524 | LR: 2.44e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.4990s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0071 | max_forward: 0.7005 | max_loss: 0.1465 | max_backward: 1.2640 | max_optimizer_step: 0.1579 | max_tensorboard_logging: 0.4728 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5451.1 | uss_MB: 4280.1 | child_rss_MB: 5636.6 | allocated_MB: 372.3 | reserved_MB: 27180.0 | inactive_split_MB: 201.7 +rank3 [2026-10-29 16:07:15,145] INFO : Step 37000/50000 rank3 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2194 | Cardinality: 0.1268 | Classification: 0.0926 | Classification_binary: 0.1402 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0014 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1454 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0673 | Regression_eta: 0.0446 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0410 | LR: 2.42e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5224s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6075 | max_loss: 0.2475 | max_backward: 1.2956 | max_optimizer_step: 0.1585 | max_tensorboard_logging: 0.7613 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5445.1 | uss_MB: 4277.2 | child_rss_MB: 5637.5 | allocated_MB: 368.1 | reserved_MB: 27180.0 | inactive_split_MB: 226.3 +rank3 [2026-10-29 16:07:16,059] INFO : Running validation on rank3 +rank3 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 16:08:14,062] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:08:14,583] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 16:08:39,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 16:08:40,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 16:08:41,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 16:08:42,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 16:08:52,655] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 16:08:53,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:08:53,528] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 16:09:09,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 16:09:10,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 16:09:11,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 16:09:12,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 16:09:22,700] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 16:09:23,587] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:09:24,058] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 16:09:39,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 16:09:40,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 16:09:40,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 16:09:41,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 16:09:51,468] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 16:13:56,322] INFO : Step 37100/50000 rank3 | Train Loss: 1.1150 | Train Loss Unweighted: 0.5431 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1226 | Classification: 0.1105 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1340 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0651 | Regression_eta: 0.0472 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0654 | LR: 2.40e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8595s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7539 | max_loss: 0.3988 | max_backward: 1.7903 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4747 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6593.7 | uss_MB: 5426.2 | child_rss_MB: 5642.5 | allocated_MB: 372.5 | reserved_MB: 23160.0 | inactive_split_MB: 265.5 +rank3 [2026-10-29 16:17:46,448] INFO : Step 37200/50000 rank3 | Train Loss: 1.0752 | Train Loss Unweighted: 0.5181 | Train Loss Components: Auxiliary: 0.2406 | Cardinality: 0.1258 | Classification: 0.0924 | Classification_binary: 0.1812 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1200 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0508 | Regression_eta: 0.0406 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0690 | LR: 2.38e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1702s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0549 | max_forward: 0.8058 | max_loss: 0.1510 | max_backward: 1.8476 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5280 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6591.2 | uss_MB: 5426.9 | child_rss_MB: 5644.6 | allocated_MB: 371.5 | reserved_MB: 24140.0 | inactive_split_MB: 271.0 +rank3 [2026-10-29 16:21:30,618] INFO : Step 37300/50000 rank3 | Train Loss: 1.0230 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1563 | HitGrouping: 0.1981 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1213 | Regression_eta: 0.0016 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0569 | Regression_eta: 0.0279 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0452 | LR: 2.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0291s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.6853 | max_loss: 0.1502 | max_backward: 1.3245 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 1.0657 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5580.1 | uss_MB: 4415.2 | child_rss_MB: 5668.9 | allocated_MB: 370.9 | reserved_MB: 24140.0 | inactive_split_MB: 261.7 +rank3 [2026-10-29 16:25:14,031] INFO : Step 37400/50000 rank3 | Train Loss: 1.0130 | Train Loss Unweighted: 0.4929 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1180 | Classification: 0.0961 | Classification_binary: 0.1390 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1273 | Regression_eta: 0.0015 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1290 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0606 | Regression_eta: 0.0634 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0508 | LR: 2.34e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5052s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.5424 | max_loss: 0.1464 | max_backward: 1.0792 | max_optimizer_step: 0.1675 | max_tensorboard_logging: 0.8668 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5585.6 | uss_MB: 4420.8 | child_rss_MB: 5670.0 | allocated_MB: 368.8 | reserved_MB: 25122.0 | inactive_split_MB: 232.7 +rank3 [2026-10-29 16:30:41,705] INFO : Step 37500/50000 rank3 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1315 | Classification: 0.0992 | Classification_binary: 0.1679 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1176 | Regression_eta: 0.0013 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0513 | Regression_eta: 0.0301 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0489 | LR: 2.32e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1476s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.7240 | max_loss: 0.1472 | max_backward: 1.5381 | max_optimizer_step: 0.2707 | max_tensorboard_logging: 0.8554 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5585.8 | uss_MB: 4422.3 | child_rss_MB: 5642.8 | allocated_MB: 370.2 | reserved_MB: 26134.0 | inactive_split_MB: 179.7 +rank3 [2026-10-29 16:34:38,361] INFO : Step 37600/50000 rank3 | Train Loss: 1.0243 | Train Loss Unweighted: 0.4867 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1182 | Classification: 0.0808 | Classification_binary: 0.1619 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0534 | Regression_eta: 0.0501 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0386 | LR: 2.30e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1656s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.7069 | max_loss: 0.1543 | max_backward: 1.5139 | max_optimizer_step: 0.1704 | max_tensorboard_logging: 1.0159 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5587.9 | uss_MB: 4422.7 | child_rss_MB: 5672.0 | allocated_MB: 370.8 | reserved_MB: 26846.0 | inactive_split_MB: 273.3 +rank3 [2026-10-29 16:38:29,293] INFO : Step 37700/50000 rank3 | Train Loss: 0.9610 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2102 | Cardinality: 0.1049 | Classification: 0.0974 | Classification_binary: 0.1317 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0012 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0494 | Regression_eta: 0.0533 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0614 | LR: 2.28e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5152s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6233 | max_loss: 0.2433 | max_backward: 1.3306 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5644 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5591.0 | uss_MB: 4425.5 | child_rss_MB: 5643.2 | allocated_MB: 369.4 | reserved_MB: 28822.0 | inactive_split_MB: 231.5 +rank3 [2026-10-29 16:42:35,306] INFO : Step 37800/50000 rank3 | Train Loss: 0.9994 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1105 | Classification: 0.0952 | Classification_binary: 0.1458 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1161 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1374 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0563 | Regression_eta: 0.0328 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0846 | LR: 2.26e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8882s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.6659 | max_loss: 0.1473 | max_backward: 1.3840 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.9558 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5593.5 | uss_MB: 4427.0 | child_rss_MB: 5671.1 | allocated_MB: 371.5 | reserved_MB: 28822.0 | inactive_split_MB: 170.1 +rank3 [2026-10-29 16:46:40,060] INFO : Step 37900/50000 rank3 | Train Loss: 1.0837 | Train Loss Unweighted: 0.5094 | Train Loss Components: Auxiliary: 0.2372 | Cardinality: 0.1419 | Classification: 0.0937 | Classification_binary: 0.1795 | HitGrouping: 0.1848 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1160 | Regression_eta: 0.0015 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1707 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0566 | Regression_eta: 0.0505 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0554 | LR: 2.24e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.2206s | Phase Times: max_pre_step_barrier: 0.1076 | max_batch_to_device: 0.0071 | max_forward: 0.7572 | max_loss: 0.1547 | max_backward: 1.4632 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 0.9205 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 5593.0 | uss_MB: 4429.3 | child_rss_MB: 5670.1 | allocated_MB: 370.2 | reserved_MB: 28822.0 | inactive_split_MB: 265.3 +rank3 [2026-10-29 16:50:31,519] INFO : Step 38000/50000 rank3 | Train Loss: 0.9811 | Train Loss Unweighted: 0.4782 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1213 | Classification: 0.0781 | Classification_binary: 0.1413 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1277 | Regression_eta: 0.0012 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0307 | Classification_binary: 0.1440 | Regression_cos_phi: 0.0777 | Regression_energy: 0.0677 | Regression_eta: 0.0290 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0261 | LR: 2.22e-04 | DataLoad Time: 0.0224s | Model Forward Time: 3.5073s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.6120 | max_loss: 0.2171 | max_backward: 1.2526 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5836 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5590.4 | uss_MB: 4430.0 | child_rss_MB: 5667.2 | allocated_MB: 368.7 | reserved_MB: 28822.0 | inactive_split_MB: 223.4 +rank3 [2026-10-29 16:50:32,412] INFO : Running validation on rank3 +rank3 [2026-10-29 16:50:32,412] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 16:50:32,412] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 16:51:23,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:51:24,264] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 16:51:50,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 16:51:51,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 16:51:52,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 16:51:53,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 16:52:03,807] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 16:52:04,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:52:05,008] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 16:52:20,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 16:52:21,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 16:52:22,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 16:52:23,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 16:52:33,685] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 16:52:34,974] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:52:35,419] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 16:52:50,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 16:52:51,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 16:52:52,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 16:52:52,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 16:53:02,948] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 16:57:14,095] INFO : Step 38100/50000 rank3 | Train Loss: 1.0158 | Train Loss Unweighted: 0.4912 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1220 | Classification: 0.0982 | Classification_binary: 0.1455 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1224 | Regression_eta: 0.0013 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1400 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0661 | Regression_eta: 0.0260 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0322 | LR: 2.20e-04 | DataLoad Time: 0.0251s | Model Forward Time: 4.9406s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8406 | max_loss: 0.1820 | max_backward: 1.8536 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.3354 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0179 | Memory: rss_MB: 5714.6 | uss_MB: 4482.4 | child_rss_MB: 5669.8 | allocated_MB: 370.5 | reserved_MB: 24292.0 | inactive_split_MB: 257.8 +rank3 [2026-10-29 17:01:01,944] INFO : Step 38200/50000 rank3 | Train Loss: 1.0273 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2292 | Cardinality: 0.1060 | Classification: 0.0997 | Classification_binary: 0.1650 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0617 | Regression_eta: 0.0409 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0725 | LR: 2.18e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.8895s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7758 | max_loss: 0.1794 | max_backward: 1.7969 | max_optimizer_step: 0.1713 | max_tensorboard_logging: 0.4698 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5717.4 | uss_MB: 4558.1 | child_rss_MB: 5664.1 | allocated_MB: 372.9 | reserved_MB: 26212.0 | inactive_split_MB: 221.1 +rank3 [2026-10-29 17:05:07,130] INFO : Step 38300/50000 rank3 | Train Loss: 1.0498 | Train Loss Unweighted: 0.5122 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1249 | Classification: 0.0913 | Classification_binary: 0.1583 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1294 | Regression_eta: 0.0013 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0310 | Regression_energy: 0.0625 | Regression_eta: 0.0286 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0510 | LR: 2.16e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7354s | Phase Times: max_pre_step_barrier: 0.1103 | max_batch_to_device: 0.0086 | max_forward: 0.6730 | max_loss: 0.1509 | max_backward: 1.3787 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.4782 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5715.4 | uss_MB: 4556.5 | child_rss_MB: 5665.1 | allocated_MB: 369.2 | reserved_MB: 26212.0 | inactive_split_MB: 171.8 +rank3 [2026-10-29 17:09:08,354] INFO : Step 38400/50000 rank3 | Train Loss: 0.9354 | Train Loss Unweighted: 0.4735 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.0968 | Classification: 0.0760 | Classification_binary: 0.1246 | HitGrouping: 0.1889 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1340 | Regression_eta: 0.0013 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1203 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0698 | Regression_eta: 0.0361 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0541 | LR: 2.14e-04 | DataLoad Time: 0.0223s | Model Forward Time: 3.6824s | Phase Times: max_pre_step_barrier: 0.1107 | max_batch_to_device: 0.0062 | max_forward: 0.6635 | max_loss: 0.1419 | max_backward: 1.4771 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 0.4619 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5721.1 | uss_MB: 4559.4 | child_rss_MB: 5662.2 | allocated_MB: 370.2 | reserved_MB: 26212.0 | inactive_split_MB: 267.8 +rank3 [2026-10-29 17:13:06,090] INFO : Step 38500/50000 rank3 | Train Loss: 1.0482 | Train Loss Unweighted: 0.5077 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1228 | Classification: 0.1147 | Classification_binary: 0.1473 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1208 | Regression_eta: 0.0016 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0780 | Classification_binary: 0.1726 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0761 | Regression_eta: 0.0952 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0888 | LR: 2.12e-04 | DataLoad Time: 0.0183s | Model Forward Time: 3.4610s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.6383 | max_loss: 0.1393 | max_backward: 1.3136 | max_optimizer_step: 0.1695 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 5717.0 | uss_MB: 4562.8 | child_rss_MB: 5658.1 | allocated_MB: 370.2 | reserved_MB: 27188.0 | inactive_split_MB: 210.0 +rank3 [2026-10-29 17:17:04,838] INFO : Step 38600/50000 rank3 | Train Loss: 1.0321 | Train Loss Unweighted: 0.4950 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1157 | Classification: 0.0910 | Classification_binary: 0.1735 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1130 | Regression_eta: 0.0014 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1871 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0527 | Regression_eta: 0.0363 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0625 | LR: 2.11e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.0842s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0127 | max_forward: 0.7516 | max_loss: 0.1551 | max_backward: 1.6059 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.7250 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5714.3 | uss_MB: 4563.8 | child_rss_MB: 5662.6 | allocated_MB: 372.6 | reserved_MB: 28172.0 | inactive_split_MB: 283.9 +rank3 [2026-10-29 17:20:59,239] INFO : Step 38700/50000 rank3 | Train Loss: 0.9420 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1147 | Classification: 0.0828 | Classification_binary: 0.1348 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1155 | Regression_eta: 0.0013 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1238 | Regression_cos_phi: 0.0343 | Regression_energy: 0.0425 | Regression_eta: 0.0306 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0473 | LR: 2.09e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.6818s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.8052 | max_loss: 0.2133 | max_backward: 1.3215 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5712.0 | uss_MB: 4410.4 | child_rss_MB: 5659.5 | allocated_MB: 367.4 | reserved_MB: 28172.0 | inactive_split_MB: 206.7 +rank3 [2026-10-29 17:25:08,218] INFO : Step 38800/50000 rank3 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4825 | Train Loss Components: Auxiliary: 0.2201 | Cardinality: 0.1186 | Classification: 0.0965 | Classification_binary: 0.1429 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1198 | Regression_eta: 0.0012 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0645 | Regression_eta: 0.0382 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0716 | LR: 2.07e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6544s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.9011 | max_loss: 0.1880 | max_backward: 1.7487 | max_optimizer_step: 0.2166 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5710.9 | uss_MB: 4431.1 | child_rss_MB: 5666.7 | allocated_MB: 371.5 | reserved_MB: 28172.0 | inactive_split_MB: 220.5 +rank3 [2026-10-29 17:29:07,476] INFO : Step 38900/50000 rank3 | Train Loss: 0.9986 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2211 | Cardinality: 0.1078 | Classification: 0.0988 | Classification_binary: 0.1578 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0525 | Regression_eta: 0.0380 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0688 | LR: 2.05e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.4275s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0073 | max_forward: 0.9800 | max_loss: 0.2159 | max_backward: 2.2158 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0079 | Memory: rss_MB: 5714.2 | uss_MB: 3754.5 | child_rss_MB: 5665.7 | allocated_MB: 373.3 | reserved_MB: 29164.0 | inactive_split_MB: 189.2 +rank3 [2026-10-29 17:33:02,383] INFO : Step 39000/50000 rank3 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4758 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1227 | Classification: 0.0847 | Classification_binary: 0.1518 | HitGrouping: 0.1853 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1179 | Regression_eta: 0.0013 | Regression_pt: 0.1189 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1865 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0569 | Regression_eta: 0.0433 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0560 | LR: 2.03e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.7820s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0515 | max_forward: 0.7410 | max_loss: 0.1790 | max_backward: 1.4655 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5713.4 | uss_MB: 4569.9 | child_rss_MB: 5663.9 | allocated_MB: 370.5 | reserved_MB: 29164.0 | inactive_split_MB: 192.2 +rank3 [2026-10-29 17:33:03,293] INFO : Running validation on rank3 +rank3 [2026-10-29 17:33:03,293] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 17:33:56,306] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:33:56,829] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 17:34:24,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 17:34:25,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 17:34:26,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 17:34:27,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 17:34:37,640] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 17:34:38,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:34:38,631] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 17:34:54,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 17:34:55,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 17:34:56,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 17:34:57,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 17:35:07,103] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 17:35:07,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:35:08,264] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 17:35:23,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 17:35:24,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 17:35:24,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 17:35:25,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 17:35:35,487] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 17:39:50,703] INFO : Step 39100/50000 rank3 | Train Loss: 1.0043 | Train Loss Unweighted: 0.4886 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1149 | Classification: 0.0878 | Classification_binary: 0.1320 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1329 | Regression_eta: 0.0012 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1090 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0661 | Regression_eta: 0.0234 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0369 | LR: 2.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6395s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0736 | max_forward: 0.7590 | max_loss: 0.4062 | max_backward: 1.4330 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.4941 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6810.5 | uss_MB: 5664.9 | child_rss_MB: 5641.2 | allocated_MB: 371.1 | reserved_MB: 23656.0 | inactive_split_MB: 194.8 +rank3 [2026-10-29 17:43:44,583] INFO : Step 39200/50000 rank3 | Train Loss: 1.0325 | Train Loss Unweighted: 0.5174 | Train Loss Components: Auxiliary: 0.2315 | Cardinality: 0.1117 | Classification: 0.1086 | Classification_binary: 0.1481 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0546 | Regression_eta: 0.0415 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0588 | LR: 2.00e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.6965s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0102 | max_forward: 0.7546 | max_loss: 0.4268 | max_backward: 1.4370 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5801.6 | uss_MB: 4660.4 | child_rss_MB: 5666.2 | allocated_MB: 371.2 | reserved_MB: 23934.0 | inactive_split_MB: 196.6 +rank3 [2026-10-29 17:47:41,748] INFO : Step 39300/50000 rank3 | Train Loss: 1.0373 | Train Loss Unweighted: 0.5050 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1241 | Classification: 0.0956 | Classification_binary: 0.1745 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1150 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1794 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0584 | Regression_eta: 0.0274 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0574 | LR: 1.98e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.2670s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0844 | max_forward: 0.8662 | max_loss: 0.2199 | max_backward: 1.7416 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5805.0 | uss_MB: 4500.0 | child_rss_MB: 5642.3 | allocated_MB: 373.1 | reserved_MB: 23934.0 | inactive_split_MB: 256.8 +rank3 [2026-10-29 17:51:43,102] INFO : Step 39400/50000 rank3 | Train Loss: 0.9741 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2189 | Cardinality: 0.1078 | Classification: 0.0808 | Classification_binary: 0.1461 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1317 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0599 | Regression_eta: 0.0277 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0349 | LR: 1.96e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5592s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.6778 | max_loss: 0.1515 | max_backward: 1.4156 | max_optimizer_step: 0.1740 | max_tensorboard_logging: 0.4653 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5807.5 | uss_MB: 4663.4 | child_rss_MB: 5665.4 | allocated_MB: 368.0 | reserved_MB: 23934.0 | inactive_split_MB: 199.1 +rank3 [2026-10-29 17:55:40,086] INFO : Step 39500/50000 rank3 | Train Loss: 0.9809 | Train Loss Unweighted: 0.4664 | Train Loss Components: Auxiliary: 0.2175 | Cardinality: 0.1105 | Classification: 0.0963 | Classification_binary: 0.1347 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1158 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1182 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0559 | Regression_eta: 0.0579 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0436 | LR: 1.94e-04 | DataLoad Time: 0.0149s | Model Forward Time: 3.5634s | Phase Times: max_pre_step_barrier: 0.0259 | max_batch_to_device: 0.1050 | max_forward: 0.6813 | max_loss: 0.1541 | max_backward: 1.3449 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 0.5375 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0111 | Memory: rss_MB: 5806.7 | uss_MB: 4606.4 | child_rss_MB: 5664.0 | allocated_MB: 367.2 | reserved_MB: 24970.0 | inactive_split_MB: 230.9 +rank3 [2026-10-29 17:59:42,115] INFO : Step 39600/50000 rank3 | Train Loss: 1.0814 | Train Loss Unweighted: 0.5099 | Train Loss Components: Auxiliary: 0.2411 | Cardinality: 0.1344 | Classification: 0.0978 | Classification_binary: 0.1731 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0540 | Regression_eta: 0.0345 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0696 | LR: 1.93e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1490s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8839 | max_loss: 0.1464 | max_backward: 1.8057 | max_optimizer_step: 0.2568 | max_tensorboard_logging: 0.5748 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5805.3 | uss_MB: 4666.8 | child_rss_MB: 5646.1 | allocated_MB: 370.6 | reserved_MB: 24970.0 | inactive_split_MB: 243.4 +rank3 [2026-10-29 18:03:55,338] INFO : Step 39700/50000 rank3 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4877 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1185 | Classification: 0.0909 | Classification_binary: 0.1674 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1134 | Regression_eta: 0.0013 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0523 | Regression_eta: 0.0349 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0621 | LR: 1.91e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.8051s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.6954 | max_loss: 0.2515 | max_backward: 1.5651 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.5599 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5806.9 | uss_MB: 4667.8 | child_rss_MB: 5665.2 | allocated_MB: 371.1 | reserved_MB: 24970.0 | inactive_split_MB: 244.9 +rank3 [2026-10-29 18:07:56,646] INFO : Step 39800/50000 rank3 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1022 | Classification: 0.0711 | Classification_binary: 0.1322 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0923 | Regression_energy: 0.0511 | Regression_eta: 0.0284 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0668 | LR: 1.89e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9402s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0063 | max_forward: 1.0012 | max_loss: 0.2859 | max_backward: 2.1063 | max_optimizer_step: 0.1800 | max_tensorboard_logging: 0.4881 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5810.9 | uss_MB: 3839.1 | child_rss_MB: 5666.9 | allocated_MB: 368.3 | reserved_MB: 24970.0 | inactive_split_MB: 241.7 +rank3 [2026-10-29 18:11:54,159] INFO : Step 39900/50000 rank3 | Train Loss: 0.9667 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.2188 | Cardinality: 0.1055 | Classification: 0.0897 | Classification_binary: 0.1526 | HitGrouping: 0.1883 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1101 | Regression_eta: 0.0012 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0484 | Regression_eta: 0.0302 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0447 | LR: 1.88e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.9942s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.3186 | max_loss: 0.4377 | max_backward: 3.6189 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.6123 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 5815.3 | uss_MB: 4217.6 | child_rss_MB: 5667.9 | allocated_MB: 370.5 | reserved_MB: 24970.0 | inactive_split_MB: 229.4 +rank3 [2026-10-29 18:15:59,744] INFO : Step 40000/50000 rank3 | Train Loss: 1.0232 | Train Loss Unweighted: 0.5131 | Train Loss Components: Auxiliary: 0.2348 | Cardinality: 0.1100 | Classification: 0.0962 | Classification_binary: 0.1712 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1206 | Regression_eta: 0.0013 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0518 | Regression_eta: 0.0280 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0472 | LR: 1.86e-04 | DataLoad Time: 0.0197s | Model Forward Time: 7.4557s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0077 | max_forward: 0.8168 | max_loss: 0.2161 | max_backward: 1.5709 | max_optimizer_step: 0.1733 | max_tensorboard_logging: 1.1681 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.1427 | Memory: rss_MB: 5816.6 | uss_MB: 3786.6 | child_rss_MB: 5665.4 | allocated_MB: 375.8 | reserved_MB: 24970.0 | inactive_split_MB: 259.2 +rank3 [2026-10-29 18:16:02,308] INFO : Running validation on rank3 +rank3 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 18:16:53,651] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:16:54,179] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 18:17:18,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 18:17:20,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 18:17:21,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 18:17:22,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 18:17:32,339] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 18:17:33,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:17:33,664] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 18:17:49,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 18:17:50,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 18:17:51,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 18:17:52,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 18:18:02,615] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 18:18:03,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:18:03,744] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 18:18:19,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 18:18:19,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 18:18:20,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 18:18:21,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 18:18:31,254] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 18:22:46,460] INFO : Step 40100/50000 rank3 | Train Loss: 0.9349 | Train Loss Unweighted: 0.4412 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1220 | Classification: 0.0762 | Classification_binary: 0.1328 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1148 | Regression_eta: 0.0012 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1203 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0533 | Regression_eta: 0.0405 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0318 | LR: 1.84e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.6901s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.7093 | max_loss: 0.3974 | max_backward: 1.3310 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.5555 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 7103.1 | uss_MB: 5953.7 | child_rss_MB: 5672.5 | allocated_MB: 367.6 | reserved_MB: 22178.0 | inactive_split_MB: 265.9 +rank3 [2026-10-29 18:26:52,185] INFO : Step 40200/50000 rank3 | Train Loss: 0.9570 | Train Loss Unweighted: 0.4556 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1119 | Classification: 0.0905 | Classification_binary: 0.1293 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0013 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0645 | Regression_eta: 0.0276 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0472 | LR: 1.83e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.5892s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6287 | max_loss: 0.4129 | max_backward: 1.3901 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 0.5609 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5918.3 | uss_MB: 4768.5 | child_rss_MB: 5665.4 | allocated_MB: 370.8 | reserved_MB: 22294.0 | inactive_split_MB: 233.2 +rank3 [2026-10-29 18:30:59,534] INFO : Step 40300/50000 rank3 | Train Loss: 1.0209 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1232 | Classification: 0.0991 | Classification_binary: 0.1688 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1068 | Regression_eta: 0.0015 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0509 | Regression_eta: 0.0547 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0836 | LR: 1.81e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9648s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0749 | max_forward: 0.7563 | max_loss: 0.1523 | max_backward: 1.5452 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5632 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5917.0 | uss_MB: 4772.2 | child_rss_MB: 5652.3 | allocated_MB: 371.8 | reserved_MB: 23296.0 | inactive_split_MB: 234.2 +rank3 [2026-10-29 18:35:20,932] INFO : Step 40400/50000 rank3 | Train Loss: 1.0486 | Train Loss Unweighted: 0.5003 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1357 | Classification: 0.0888 | Classification_binary: 0.1690 | HitGrouping: 0.1879 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1196 | Regression_eta: 0.0015 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0563 | Regression_eta: 0.0480 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0524 | LR: 1.79e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.7351s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0072 | max_forward: 0.7368 | max_loss: 0.2443 | max_backward: 3.1596 | max_optimizer_step: 0.1851 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5913.1 | uss_MB: 3802.2 | child_rss_MB: 5663.8 | allocated_MB: 371.2 | reserved_MB: 23386.0 | inactive_split_MB: 254.8 +rank3 [2026-10-29 18:39:23,871] INFO : Step 40500/50000 rank3 | Train Loss: 0.9309 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.0955 | Classification: 0.0829 | Classification_binary: 0.1355 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0014 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1227 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0541 | Regression_eta: 0.0363 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0406 | LR: 1.78e-04 | DataLoad Time: 0.0142s | Model Forward Time: 3.7328s | Phase Times: max_pre_step_barrier: 0.0186 | max_batch_to_device: 0.0076 | max_forward: 0.6863 | max_loss: 0.2393 | max_backward: 1.3065 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.7004 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5916.1 | uss_MB: 4771.6 | child_rss_MB: 5669.9 | allocated_MB: 369.2 | reserved_MB: 24370.0 | inactive_split_MB: 237.3 +rank3 [2026-10-29 18:43:23,923] INFO : Step 40600/50000 rank3 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1087 | Classification: 0.0887 | Classification_binary: 0.1521 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0801 | Regression_energy: 0.0669 | Regression_eta: 0.0392 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0506 | LR: 1.76e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3446s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.1102 | max_forward: 0.7412 | max_loss: 0.1464 | max_backward: 1.4782 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 1.0719 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5917.7 | uss_MB: 4772.7 | child_rss_MB: 5655.3 | allocated_MB: 372.3 | reserved_MB: 24370.0 | inactive_split_MB: 234.4 +rank3 [2026-10-29 18:47:14,843] INFO : Step 40700/50000 rank3 | Train Loss: 1.0611 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1374 | Classification: 0.0904 | Classification_binary: 0.1773 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1190 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1773 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0560 | Regression_eta: 0.0293 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0497 | LR: 1.75e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2480s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7422 | max_loss: 0.1589 | max_backward: 1.5298 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5918.6 | uss_MB: 4774.9 | child_rss_MB: 5665.5 | allocated_MB: 372.7 | reserved_MB: 24370.0 | inactive_split_MB: 255.3 +rank3 [2026-10-29 18:51:03,371] INFO : Step 40800/50000 rank3 | Train Loss: 0.9701 | Train Loss Unweighted: 0.4785 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.0983 | Classification: 0.0901 | Classification_binary: 0.1373 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0604 | Regression_eta: 0.0423 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0733 | LR: 1.73e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.7292s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5843 | max_loss: 0.1422 | max_backward: 1.0661 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.0389 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5915.8 | uss_MB: 4772.0 | child_rss_MB: 5661.2 | allocated_MB: 367.5 | reserved_MB: 24658.0 | inactive_split_MB: 205.4 +rank3 [2026-10-29 18:54:43,248] INFO : Step 40900/50000 rank3 | Train Loss: 0.9731 | Train Loss Unweighted: 0.4636 | Train Loss Components: Auxiliary: 0.2169 | Cardinality: 0.1073 | Classification: 0.0883 | Classification_binary: 0.1426 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1140 | Regression_eta: 0.0013 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1657 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0599 | Regression_eta: 0.0410 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0873 | LR: 1.72e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8719s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.1047 | max_forward: 0.7107 | max_loss: 0.1679 | max_backward: 1.3343 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.8664 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5919.0 | uss_MB: 4778.3 | child_rss_MB: 5681.4 | allocated_MB: 372.0 | reserved_MB: 25640.0 | inactive_split_MB: 260.9 +rank3 [2026-10-29 18:58:25,753] INFO : Step 41000/50000 rank3 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1031 | Classification: 0.0930 | Classification_binary: 0.1618 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1164 | Regression_eta: 0.0014 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0540 | Regression_eta: 0.0530 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0660 | LR: 1.70e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4154s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.7604 | max_loss: 0.2558 | max_backward: 1.7603 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.8399 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 5926.0 | uss_MB: 4782.0 | child_rss_MB: 5654.3 | allocated_MB: 371.9 | reserved_MB: 25640.0 | inactive_split_MB: 195.8 +rank3 [2026-10-29 18:58:26,765] INFO : Running validation on rank3 +rank3 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 18:59:14,712] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:59:15,275] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 18:59:40,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 18:59:41,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 18:59:42,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 18:59:43,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 18:59:53,366] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 18:59:53,543] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:59:54,025] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 19:00:09,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 19:00:10,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 19:00:11,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 19:00:12,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 19:00:22,751] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 19:00:23,207] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:00:23,684] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 19:00:39,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 19:00:39,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 19:00:40,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 19:00:41,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 19:00:51,103] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 19:04:58,256] INFO : Step 41100/50000 rank3 | Train Loss: 1.0684 | Train Loss Unweighted: 0.5053 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1438 | Classification: 0.0856 | Classification_binary: 0.1616 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1271 | Regression_eta: 0.0014 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0605 | Regression_eta: 0.0502 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0766 | LR: 1.69e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.3309s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0066 | max_forward: 0.8006 | max_loss: 0.1726 | max_backward: 1.6699 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.6214 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0566 | Memory: rss_MB: 7184.9 | uss_MB: 5901.7 | child_rss_MB: 5647.6 | allocated_MB: 371.5 | reserved_MB: 22898.0 | inactive_split_MB: 258.4 +rank3 [2026-10-29 19:08:41,622] INFO : Step 41200/50000 rank3 | Train Loss: 0.9455 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1027 | Classification: 0.0887 | Classification_binary: 0.1242 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1223 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1397 | Regression_cos_phi: 0.0802 | Regression_energy: 0.0667 | Regression_eta: 0.0442 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0373 | LR: 1.67e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.4115s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0589 | max_forward: 0.6517 | max_loss: 0.1401 | max_backward: 1.1761 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5685 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5984.6 | uss_MB: 4847.1 | child_rss_MB: 5628.1 | allocated_MB: 370.2 | reserved_MB: 24878.0 | inactive_split_MB: 199.0 +rank3 [2026-10-29 19:12:28,247] INFO : Step 41300/50000 rank3 | Train Loss: 0.9373 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1011 | Classification: 0.0853 | Classification_binary: 0.1448 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0013 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0576 | Regression_eta: 0.0338 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0600 | LR: 1.66e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8971s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.8726 | max_loss: 0.1617 | max_backward: 1.6974 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5417 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5987.6 | uss_MB: 4852.4 | child_rss_MB: 5623.6 | allocated_MB: 370.7 | reserved_MB: 24878.0 | inactive_split_MB: 233.6 +rank3 [2026-10-29 19:16:15,463] INFO : Step 41400/50000 rank3 | Train Loss: 1.0355 | Train Loss Unweighted: 0.4965 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1363 | Classification: 0.0957 | Classification_binary: 0.1654 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1162 | Regression_eta: 0.0012 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1925 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0592 | Regression_eta: 0.0493 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0892 | LR: 1.64e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9189s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7129 | max_loss: 0.2460 | max_backward: 1.6340 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5967.8 | uss_MB: 4849.9 | child_rss_MB: 5563.5 | allocated_MB: 372.0 | reserved_MB: 24878.0 | inactive_split_MB: 305.4 +rank3 [2026-10-29 19:19:52,654] INFO : Step 41500/50000 rank3 | Train Loss: 0.9689 | Train Loss Unweighted: 0.4689 | Train Loss Components: Auxiliary: 0.2118 | Cardinality: 0.1040 | Classification: 0.0862 | Classification_binary: 0.1340 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1227 | Regression_eta: 0.0012 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1167 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0646 | Regression_eta: 0.0363 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0423 | LR: 1.63e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.4446s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.6415 | max_loss: 0.1502 | max_backward: 1.1845 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5935.8 | uss_MB: 4853.4 | child_rss_MB: 5403.5 | allocated_MB: 368.5 | reserved_MB: 24878.0 | inactive_split_MB: 173.5 +rank3 [2026-10-29 19:23:40,393] INFO : Step 41600/50000 rank3 | Train Loss: 0.9559 | Train Loss Unweighted: 0.4625 | Train Loss Components: Auxiliary: 0.2141 | Cardinality: 0.1097 | Classification: 0.0872 | Classification_binary: 0.1298 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1211 | Regression_eta: 0.0012 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1155 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0458 | Regression_eta: 0.0239 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0339 | LR: 1.61e-04 | DataLoad Time: 0.0193s | Model Forward Time: 3.5884s | Phase Times: max_pre_step_barrier: 0.0755 | max_batch_to_device: 0.0062 | max_forward: 0.6849 | max_loss: 0.2047 | max_backward: 1.1646 | max_optimizer_step: 0.2235 | max_tensorboard_logging: 0.5806 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5909.3 | uss_MB: 4827.6 | child_rss_MB: 5308.7 | allocated_MB: 370.4 | reserved_MB: 24878.0 | inactive_split_MB: 287.4 +rank3 [2026-10-29 19:27:23,742] INFO : Step 41700/50000 rank3 | Train Loss: 1.0204 | Train Loss Unweighted: 0.4905 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1165 | Classification: 0.0860 | Classification_binary: 0.1552 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0503 | Regression_eta: 0.0318 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0813 | LR: 1.60e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.0300s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0773 | max_forward: 0.7754 | max_loss: 0.1486 | max_backward: 1.5759 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 0.5773 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5907.3 | uss_MB: 4857.3 | child_rss_MB: 5302.2 | allocated_MB: 370.8 | reserved_MB: 24878.0 | inactive_split_MB: 238.3 +rank3 [2026-10-29 19:31:24,299] INFO : Step 41800/50000 rank3 | Train Loss: 1.0101 | Train Loss Unweighted: 0.4703 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1282 | Classification: 0.0818 | Classification_binary: 0.1618 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0522 | Regression_eta: 0.0272 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0480 | LR: 1.58e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9752s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8147 | max_loss: 0.4259 | max_backward: 1.4886 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1200 | Memory: rss_MB: 5899.0 | uss_MB: 4855.0 | child_rss_MB: 5276.1 | allocated_MB: 371.9 | reserved_MB: 25802.0 | inactive_split_MB: 266.5 +rank3 [2026-10-29 19:35:32,346] INFO : Step 41900/50000 rank3 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4672 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1185 | Classification: 0.0878 | Classification_binary: 0.1282 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1241 | Regression_eta: 0.0011 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1317 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0730 | Regression_eta: 0.0372 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0252 | LR: 1.57e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.4637s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0473 | max_forward: 0.6783 | max_loss: 0.1431 | max_backward: 1.2347 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.5634 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5868.0 | uss_MB: 4858.2 | child_rss_MB: 5156.0 | allocated_MB: 370.8 | reserved_MB: 25802.0 | inactive_split_MB: 166.5 +rank3 [2026-10-29 19:39:32,016] INFO : Step 42000/50000 rank3 | Train Loss: 0.9837 | Train Loss Unweighted: 0.4719 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.1093 | Classification: 0.1015 | Classification_binary: 0.1413 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1128 | Regression_eta: 0.0014 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1415 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0554 | Regression_eta: 0.0329 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0411 | LR: 1.56e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.6386s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6848 | max_loss: 0.2509 | max_backward: 1.3746 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5866.0 | uss_MB: 4860.4 | child_rss_MB: 5161.1 | allocated_MB: 370.2 | reserved_MB: 25802.0 | inactive_split_MB: 219.9 +rank3 [2026-10-29 19:39:32,940] INFO : Running validation on rank3 +rank3 [2026-10-29 19:39:32,940] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 19:39:32,940] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 19:40:23,395] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:40:23,887] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 19:40:52,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 19:40:53,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 19:40:54,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 19:40:55,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 19:41:05,769] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 19:41:06,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:41:06,794] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 19:41:22,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 19:41:24,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 19:41:25,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 19:41:26,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 19:41:36,869] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 19:41:38,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:41:38,674] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 19:41:54,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 19:41:54,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 19:41:55,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 19:41:56,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 19:42:06,356] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 19:46:29,430] INFO : Step 42100/50000 rank3 | Train Loss: 1.0639 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1434 | Classification: 0.0999 | Classification_binary: 0.1737 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1097 | Regression_eta: 0.0012 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1788 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0524 | Regression_eta: 0.0338 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0632 | LR: 1.54e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0366s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.7424 | max_loss: 0.1586 | max_backward: 1.4434 | max_optimizer_step: 0.3014 | max_tensorboard_logging: 0.6771 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 7018.7 | uss_MB: 6010.1 | child_rss_MB: 5164.1 | allocated_MB: 370.8 | reserved_MB: 23298.0 | inactive_split_MB: 265.3 +rank3 [2026-10-29 19:50:21,662] INFO : Step 42200/50000 rank3 | Train Loss: 1.0152 | Train Loss Unweighted: 0.4831 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1169 | Classification: 0.0913 | Classification_binary: 0.1515 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1190 | Regression_eta: 0.0015 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1818 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0580 | Regression_eta: 0.0453 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0673 | LR: 1.53e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.5439s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.6567 | max_loss: 0.2000 | max_backward: 1.2379 | max_optimizer_step: 0.1803 | max_tensorboard_logging: 0.5782 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5943.7 | uss_MB: 4948.2 | child_rss_MB: 5171.4 | allocated_MB: 370.9 | reserved_MB: 24984.0 | inactive_split_MB: 272.4 +rank3 [2026-10-29 19:51:24,069] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank3 [2026-10-29 19:51:24,069] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-29 19:54:15,428] INFO : Step 42300/50000 rank3 | Train Loss: 1.0224 | Train Loss Unweighted: 0.4869 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.1261 | Classification: 0.0943 | Classification_binary: 0.1704 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0502 | Regression_eta: 0.0602 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0605 | LR: 1.52e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7698s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7065 | max_loss: 0.2237 | max_backward: 1.4902 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.5353 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6010.9 | uss_MB: 5019.0 | child_rss_MB: 5170.2 | allocated_MB: 371.9 | reserved_MB: 24984.0 | inactive_split_MB: 183.6 +rank3 [2026-10-29 19:58:13,284] INFO : Step 42400/50000 rank3 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4895 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1253 | Classification: 0.0872 | Classification_binary: 0.1616 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1184 | Regression_eta: 0.0013 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1598 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0622 | Regression_eta: 0.0344 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0634 | LR: 1.50e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.7984s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7436 | max_loss: 0.2477 | max_backward: 1.5279 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5169 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5933.8 | uss_MB: 4947.5 | child_rss_MB: 5164.9 | allocated_MB: 373.0 | reserved_MB: 24984.0 | inactive_split_MB: 275.6 +rank3 [2026-10-29 20:02:13,260] INFO : Step 42500/50000 rank3 | Train Loss: 0.9211 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1056 | Classification: 0.0789 | Classification_binary: 0.1296 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1162 | Regression_eta: 0.0011 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0693 | Regression_eta: 0.0405 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0466 | LR: 1.49e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.5533s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.6196 | max_loss: 0.2440 | max_backward: 1.3403 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 2.3828 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5926.8 | uss_MB: 4949.7 | child_rss_MB: 5158.5 | allocated_MB: 371.3 | reserved_MB: 25980.0 | inactive_split_MB: 274.9 +rank3 [2026-10-29 20:06:17,860] INFO : Step 42600/50000 rank3 | Train Loss: 0.9688 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1101 | Classification: 0.0874 | Classification_binary: 0.1472 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0012 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1267 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0497 | Regression_eta: 0.0401 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0433 | LR: 1.48e-04 | DataLoad Time: 0.0148s | Model Forward Time: 3.9516s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6353 | max_loss: 0.2547 | max_backward: 1.3961 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 5918.3 | uss_MB: 4947.3 | child_rss_MB: 5167.3 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 272.5 +rank3 [2026-10-29 20:10:16,915] INFO : Step 42700/50000 rank3 | Train Loss: 1.0476 | Train Loss Unweighted: 0.4808 | Train Loss Components: Auxiliary: 0.2377 | Cardinality: 0.1284 | Classification: 0.0883 | Classification_binary: 0.1739 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1071 | Regression_eta: 0.0014 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1581 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0638 | Regression_eta: 0.0487 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0589 | LR: 1.47e-04 | DataLoad Time: 0.0129s | Model Forward Time: 5.7851s | Phase Times: max_pre_step_barrier: 0.0320 | max_batch_to_device: 0.0114 | max_forward: 2.1726 | max_loss: 0.5101 | max_backward: 2.7494 | max_optimizer_step: 0.2679 | max_tensorboard_logging: 0.6520 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5921.8 | uss_MB: 4952.4 | child_rss_MB: 5151.8 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 482.9 +rank3 [2026-10-29 20:14:17,050] INFO : Step 42800/50000 rank3 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4792 | Train Loss Components: Auxiliary: 0.2262 | Cardinality: 0.1277 | Classification: 0.0841 | Classification_binary: 0.1487 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1209 | Regression_eta: 0.0012 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1414 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0667 | Regression_eta: 0.0380 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0669 | LR: 1.45e-04 | DataLoad Time: 0.0212s | Model Forward Time: 6.2808s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0175 | max_forward: 1.3870 | max_loss: 0.5106 | max_backward: 1.9221 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 2.4017 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5924.3 | uss_MB: 4952.5 | child_rss_MB: 5165.0 | allocated_MB: 372.6 | reserved_MB: 25980.0 | inactive_split_MB: 234.4 +rank3 [2026-10-29 20:18:09,439] INFO : Step 42900/50000 rank3 | Train Loss: 0.8847 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.0993 | Classification: 0.0847 | Classification_binary: 0.1207 | HitGrouping: 0.1828 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1069 | Regression_eta: 0.0011 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1128 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0543 | Regression_eta: 0.0292 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0526 | LR: 1.44e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7386s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.6133 | max_loss: 0.1478 | max_backward: 1.3445 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.7551 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0762 | Memory: rss_MB: 5924.9 | uss_MB: 4957.1 | child_rss_MB: 5179.7 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 276.9 +rank3 [2026-10-29 20:22:02,171] INFO : Step 43000/50000 rank3 | Train Loss: 0.9800 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1096 | Classification: 0.0987 | Classification_binary: 0.1606 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0012 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1507 | Regression_cos_phi: 0.0359 | Regression_energy: 0.0430 | Regression_eta: 0.0362 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0443 | LR: 1.43e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.9021s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0071 | max_forward: 0.7477 | max_loss: 0.1477 | max_backward: 1.5663 | max_optimizer_step: 0.1600 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 5929.7 | uss_MB: 4961.8 | child_rss_MB: 5171.4 | allocated_MB: 369.1 | reserved_MB: 25980.0 | inactive_split_MB: 209.4 +rank3 [2026-10-29 20:22:03,144] INFO : Running validation on rank3 +rank3 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 20:22:51,802] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:22:52,330] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 20:23:18,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 20:23:19,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 20:23:20,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 20:23:21,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 20:23:31,620] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 20:23:31,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:23:32,370] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 20:23:48,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 20:23:49,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 20:23:49,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 20:23:50,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 20:24:00,931] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 20:24:01,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:24:02,214] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 20:24:17,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 20:24:18,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 20:24:18,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 20:24:19,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 20:24:29,431] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 20:28:29,103] INFO : Step 43100/50000 rank3 | Train Loss: 0.9692 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1151 | Classification: 0.0766 | Classification_binary: 0.1586 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1062 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1758 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0628 | Regression_eta: 0.0409 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0663 | LR: 1.42e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.7258s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.8051 | max_loss: 0.1438 | max_backward: 1.3224 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 7068.8 | uss_MB: 6101.3 | child_rss_MB: 5191.0 | allocated_MB: 370.5 | reserved_MB: 21096.0 | inactive_split_MB: 247.5 +rank3 [2026-10-29 20:32:30,812] INFO : Step 43200/50000 rank3 | Train Loss: 0.9431 | Train Loss Unweighted: 0.4458 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1175 | Classification: 0.0811 | Classification_binary: 0.1294 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0012 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1238 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0611 | Regression_eta: 0.0179 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0451 | LR: 1.40e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.5217s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0120 | max_forward: 0.6342 | max_loss: 0.1417 | max_backward: 1.2752 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6068.4 | uss_MB: 5101.7 | child_rss_MB: 5194.2 | allocated_MB: 370.4 | reserved_MB: 22140.0 | inactive_split_MB: 198.0 +rank3 [2026-10-29 20:36:16,428] INFO : Step 43300/50000 rank3 | Train Loss: 1.0382 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1221 | Classification: 0.1031 | Classification_binary: 0.1555 | HitGrouping: 0.1894 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1173 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0507 | Regression_eta: 0.0437 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0835 | LR: 1.39e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2392s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0266 | max_forward: 0.7527 | max_loss: 0.1476 | max_backward: 1.6633 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.7701 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6071.5 | uss_MB: 5107.2 | child_rss_MB: 5175.9 | allocated_MB: 371.1 | reserved_MB: 23082.0 | inactive_split_MB: 236.4 +rank3 [2026-10-29 20:40:05,495] INFO : Step 43400/50000 rank3 | Train Loss: 1.0532 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1292 | Classification: 0.0981 | Classification_binary: 0.1785 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0015 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.2174 | Regression_cos_phi: 0.0918 | Regression_energy: 0.0571 | Regression_eta: 0.0653 | Regression_pt: 0.0600 | Regression_sin_phi: 0.1018 | LR: 1.38e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2968s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0691 | max_forward: 0.8057 | max_loss: 0.1503 | max_backward: 1.8319 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.6703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6067.0 | uss_MB: 5102.4 | child_rss_MB: 5192.5 | allocated_MB: 371.5 | reserved_MB: 23082.0 | inactive_split_MB: 776.0 +rank3 [2026-10-29 20:43:57,010] INFO : Step 43500/50000 rank3 | Train Loss: 0.9308 | Train Loss Unweighted: 0.4531 | Train Loss Components: Auxiliary: 0.2084 | Cardinality: 0.0966 | Classification: 0.0846 | Classification_binary: 0.1384 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1135 | Regression_eta: 0.0013 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1292 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0656 | Regression_eta: 0.0298 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0563 | LR: 1.37e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.6284s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0171 | max_forward: 0.6718 | max_loss: 0.1455 | max_backward: 1.4141 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 6070.4 | uss_MB: 5108.5 | child_rss_MB: 5191.0 | allocated_MB: 372.0 | reserved_MB: 23082.0 | inactive_split_MB: 228.0 +rank3 [2026-10-29 20:47:52,744] INFO : Step 43600/50000 rank3 | Train Loss: 0.9195 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.0926 | Classification: 0.0847 | Classification_binary: 0.1298 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1175 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0506 | Regression_eta: 0.0450 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0318 | LR: 1.36e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5151s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.6453 | max_loss: 0.1399 | max_backward: 1.3514 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0942 | Memory: rss_MB: 6067.9 | uss_MB: 5106.0 | child_rss_MB: 5190.1 | allocated_MB: 367.6 | reserved_MB: 23082.0 | inactive_split_MB: 270.4 +rank3 [2026-10-29 20:51:54,645] INFO : Step 43700/50000 rank3 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1181 | Classification: 0.0916 | Classification_binary: 0.1553 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1009 | Regression_eta: 0.0014 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0519 | Regression_eta: 0.0357 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0409 | LR: 1.35e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.0860s | Phase Times: max_pre_step_barrier: 0.0199 | max_batch_to_device: 0.0121 | max_forward: 0.8005 | max_loss: 0.2526 | max_backward: 1.9159 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 6075.4 | uss_MB: 5110.5 | child_rss_MB: 5168.5 | allocated_MB: 371.6 | reserved_MB: 23674.0 | inactive_split_MB: 256.4 +rank3 [2026-10-29 20:55:51,186] INFO : Step 43800/50000 rank3 | Train Loss: 1.0325 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1341 | Classification: 0.0892 | Classification_binary: 0.1542 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0012 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0549 | Regression_eta: 0.0382 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0593 | LR: 1.34e-04 | DataLoad Time: 0.0185s | Model Forward Time: 3.6827s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7618 | max_loss: 0.1715 | max_backward: 1.3818 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6073.5 | uss_MB: 5110.6 | child_rss_MB: 5186.9 | allocated_MB: 371.7 | reserved_MB: 23674.0 | inactive_split_MB: 282.3 +rank3 [2026-10-29 20:59:46,231] INFO : Step 43900/50000 rank3 | Train Loss: 0.9104 | Train Loss Unweighted: 0.4325 | Train Loss Components: Auxiliary: 0.2009 | Cardinality: 0.1078 | Classification: 0.0725 | Classification_binary: 0.1272 | HitGrouping: 0.1855 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1146 | Regression_eta: 0.0011 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1383 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0605 | Regression_eta: 0.0329 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0509 | LR: 1.33e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.3845s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.6145 | max_loss: 0.1412 | max_backward: 1.1415 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5764 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6071.8 | uss_MB: 5111.9 | child_rss_MB: 5178.4 | allocated_MB: 368.3 | reserved_MB: 23674.0 | inactive_split_MB: 193.1 +rank3 [2026-10-29 21:03:43,514] INFO : Step 44000/50000 rank3 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4573 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1078 | Classification: 0.0956 | Classification_binary: 0.1421 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1083 | Regression_eta: 0.0011 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0511 | Regression_eta: 0.0355 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0665 | LR: 1.32e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.9489s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0061 | max_forward: 1.1697 | max_loss: 0.2455 | max_backward: 1.7366 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6074.9 | uss_MB: 5118.1 | child_rss_MB: 5179.2 | allocated_MB: 370.7 | reserved_MB: 23674.0 | inactive_split_MB: 207.3 +rank3 [2026-10-29 21:03:44,527] INFO : Running validation on rank3 +rank3 [2026-10-29 21:03:44,527] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 21:03:44,527] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 21:04:32,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:04:33,228] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 21:04:58,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 21:05:00,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 21:05:01,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 21:05:02,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 21:05:12,277] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 21:05:12,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:05:12,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 21:05:28,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 21:05:29,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 21:05:30,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 21:05:31,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 21:05:41,682] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 21:05:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:05:42,731] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 21:05:58,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 21:05:58,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 21:05:59,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 21:06:00,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 21:06:10,379] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 21:10:20,117] INFO : Step 44100/50000 rank3 | Train Loss: 1.0009 | Train Loss Unweighted: 0.4605 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1400 | Classification: 0.0841 | Classification_binary: 0.1644 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1044 | Regression_eta: 0.0011 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0507 | Regression_eta: 0.0326 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0664 | LR: 1.31e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.5843s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 1.2763 | max_loss: 0.5349 | max_backward: 2.2676 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 1.2755 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 7076.5 | uss_MB: 6129.8 | child_rss_MB: 5164.2 | allocated_MB: 372.3 | reserved_MB: 24224.0 | inactive_split_MB: 272.2 +rank3 [2026-10-29 21:14:15,311] INFO : Step 44200/50000 rank3 | Train Loss: 0.9801 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1146 | Classification: 0.0882 | Classification_binary: 0.1482 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1167 | Regression_eta: 0.0014 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0625 | Regression_eta: 0.0304 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0534 | LR: 1.30e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.0040s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0132 | max_forward: 0.6993 | max_loss: 0.1574 | max_backward: 1.3862 | max_optimizer_step: 0.2915 | max_tensorboard_logging: 0.7724 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6137.2 | uss_MB: 5191.5 | child_rss_MB: 5174.4 | allocated_MB: 370.2 | reserved_MB: 26182.0 | inactive_split_MB: 176.5 +rank3 [2026-10-29 21:17:58,578] INFO : Step 44300/50000 rank3 | Train Loss: 0.8787 | Train Loss Unweighted: 0.4332 | Train Loss Components: Auxiliary: 0.1952 | Cardinality: 0.0950 | Classification: 0.0869 | Classification_binary: 0.1153 | HitGrouping: 0.1807 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0011 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1116 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0534 | Regression_eta: 0.0492 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0540 | LR: 1.29e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.7427s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0142 | max_forward: 0.7699 | max_loss: 0.2088 | max_backward: 1.4049 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 0.4839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 6137.2 | uss_MB: 4195.8 | child_rss_MB: 5178.8 | allocated_MB: 367.2 | reserved_MB: 27204.0 | inactive_split_MB: 205.8 +rank3 [2026-10-29 21:21:59,793] INFO : Step 44400/50000 rank3 | Train Loss: 0.9558 | Train Loss Unweighted: 0.4680 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.0905 | Classification: 0.0911 | Classification_binary: 0.1587 | HitGrouping: 0.1858 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0531 | Regression_eta: 0.0364 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0477 | LR: 1.28e-04 | DataLoad Time: 0.0214s | Model Forward Time: 6.3265s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0370 | max_forward: 0.7829 | max_loss: 0.1727 | max_backward: 1.7862 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 2.7361 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6138.4 | uss_MB: 5206.0 | child_rss_MB: 5161.4 | allocated_MB: 373.1 | reserved_MB: 27586.0 | inactive_split_MB: 176.3 +rank3 [2026-10-29 21:26:11,867] INFO : Step 44500/50000 rank3 | Train Loss: 0.9739 | Train Loss Unweighted: 0.4610 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1139 | Classification: 0.0849 | Classification_binary: 0.1578 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1077 | Regression_eta: 0.0012 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0550 | Regression_eta: 0.0225 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0395 | LR: 1.27e-04 | DataLoad Time: 0.0138s | Model Forward Time: 3.9324s | Phase Times: max_pre_step_barrier: 0.0250 | max_batch_to_device: 0.0065 | max_forward: 0.7930 | max_loss: 0.1668 | max_backward: 1.3363 | max_optimizer_step: 0.2871 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6136.3 | uss_MB: 5200.8 | child_rss_MB: 5177.3 | allocated_MB: 371.5 | reserved_MB: 27586.0 | inactive_split_MB: 195.8 +rank3 [2026-10-29 21:30:04,561] INFO : Step 44600/50000 rank3 | Train Loss: 0.8904 | Train Loss Unweighted: 0.4364 | Train Loss Components: Auxiliary: 0.2007 | Cardinality: 0.1041 | Classification: 0.0775 | Classification_binary: 0.1238 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1161 | Regression_eta: 0.0010 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0277 | Regression_energy: 0.0503 | Regression_eta: 0.0254 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0389 | LR: 1.26e-04 | DataLoad Time: 0.0186s | Model Forward Time: 3.5120s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.6364 | max_loss: 0.1431 | max_backward: 1.3712 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.4900 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6139.4 | uss_MB: 5205.0 | child_rss_MB: 5167.5 | allocated_MB: 368.4 | reserved_MB: 27586.0 | inactive_split_MB: 170.2 +rank3 [2026-10-29 21:33:51,597] INFO : Step 44700/50000 rank3 | Train Loss: 1.0003 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1252 | Classification: 0.0899 | Classification_binary: 0.1520 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1117 | Regression_eta: 0.0013 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1887 | Regression_cos_phi: 0.0912 | Regression_energy: 0.0614 | Regression_eta: 0.0371 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0699 | LR: 1.25e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6377s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6952 | max_loss: 0.1455 | max_backward: 1.5015 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6145.1 | uss_MB: 5206.5 | child_rss_MB: 5162.4 | allocated_MB: 371.9 | reserved_MB: 28574.0 | inactive_split_MB: 214.1 +rank3 [2026-10-29 21:37:55,986] INFO : Step 44800/50000 rank3 | Train Loss: 0.9665 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1115 | Classification: 0.0889 | Classification_binary: 0.1605 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1042 | Regression_eta: 0.0011 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0558 | Regression_eta: 0.0351 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0668 | LR: 1.24e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.8728s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7593 | max_loss: 0.1924 | max_backward: 2.8086 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.6087 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 6148.4 | uss_MB: 5210.8 | child_rss_MB: 5160.0 | allocated_MB: 370.8 | reserved_MB: 28574.0 | inactive_split_MB: 257.7 +rank3 [2026-10-29 21:41:36,914] INFO : Step 44900/50000 rank3 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.0886 | Classification_binary: 0.1466 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1121 | Regression_eta: 0.0013 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1527 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0527 | Regression_eta: 0.0537 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0618 | LR: 1.23e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.5192s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0101 | max_forward: 0.6531 | max_loss: 0.1431 | max_backward: 1.3230 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6145.9 | uss_MB: 5207.3 | child_rss_MB: 5182.8 | allocated_MB: 368.1 | reserved_MB: 28574.0 | inactive_split_MB: 178.8 +rank3 [2026-10-29 21:45:50,735] INFO : Step 45000/50000 rank3 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4443 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1053 | Classification: 0.0832 | Classification_binary: 0.1311 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1295 | Regression_cos_phi: 0.0262 | Regression_energy: 0.0542 | Regression_eta: 0.0239 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0343 | LR: 1.22e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1241s | Phase Times: max_pre_step_barrier: 0.0074 | max_batch_to_device: 0.0159 | max_forward: 1.1897 | max_loss: 0.2457 | max_backward: 1.2951 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.8342 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0919 | Memory: rss_MB: 6152.1 | uss_MB: 5213.7 | child_rss_MB: 5170.7 | allocated_MB: 368.8 | reserved_MB: 28576.0 | inactive_split_MB: 266.4 +rank3 [2026-10-29 21:45:51,739] INFO : Running validation on rank3 +rank3 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 21:46:40,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:46:41,116] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 21:47:08,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 21:47:09,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 21:47:11,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 21:47:12,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 21:47:22,592] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 21:47:23,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:47:23,695] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 21:47:39,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 21:47:40,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 21:47:41,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 21:47:42,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 21:47:52,811] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 21:47:53,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:47:53,967] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 21:48:10,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 21:48:11,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 21:48:11,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 21:48:12,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 21:48:22,695] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 21:52:56,452] INFO : Step 45100/50000 rank3 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1151 | Classification: 0.1010 | Classification_binary: 0.1562 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1118 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0516 | Regression_eta: 0.0344 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0498 | LR: 1.21e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.3049s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7307 | max_loss: 0.4409 | max_backward: 1.5823 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 7066.7 | uss_MB: 6122.4 | child_rss_MB: 5160.9 | allocated_MB: 373.1 | reserved_MB: 23280.0 | inactive_split_MB: 227.2 +rank3 [2026-10-29 21:56:57,437] INFO : Step 45200/50000 rank3 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1108 | Classification: 0.0815 | Classification_binary: 0.1593 | HitGrouping: 0.1884 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1669 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0503 | Regression_eta: 0.0374 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0481 | LR: 1.20e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2224s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.9482 | max_loss: 0.4476 | max_backward: 1.7165 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.4656 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7064.1 | uss_MB: 6120.3 | child_rss_MB: 5157.7 | allocated_MB: 373.0 | reserved_MB: 23338.0 | inactive_split_MB: 277.4 +rank3 [2026-10-29 22:01:28,910] INFO : Step 45300/50000 rank3 | Train Loss: 0.9312 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1141 | Classification: 0.0882 | Classification_binary: 0.1299 | HitGrouping: 0.1891 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0010 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1175 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0498 | Regression_eta: 0.0268 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0537 | LR: 1.19e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.4286s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6106 | max_loss: 0.2394 | max_backward: 1.1660 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 6271.7 | uss_MB: 5329.4 | child_rss_MB: 5157.3 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 223.2 +rank3 [2026-10-29 22:05:37,388] INFO : Step 45400/50000 rank3 | Train Loss: 0.9251 | Train Loss Unweighted: 0.4554 | Train Loss Components: Auxiliary: 0.2112 | Cardinality: 0.1034 | Classification: 0.0967 | Classification_binary: 0.1405 | HitGrouping: 0.1801 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1072 | Regression_eta: 0.0012 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1756 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0556 | Regression_eta: 0.0427 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0632 | LR: 1.19e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9545s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7158 | max_loss: 0.2520 | max_backward: 1.7727 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6276.9 | uss_MB: 5335.5 | child_rss_MB: 5162.7 | allocated_MB: 372.6 | reserved_MB: 23338.0 | inactive_split_MB: 212.6 +rank3 [2026-10-29 22:09:49,791] INFO : Step 45500/50000 rank3 | Train Loss: 1.0136 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1238 | Classification: 0.0902 | Classification_binary: 0.1661 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1060 | Regression_eta: 0.0013 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0579 | Regression_eta: 0.0453 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0783 | LR: 1.18e-04 | DataLoad Time: 0.0178s | Model Forward Time: 6.1592s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0147 | max_forward: 1.9023 | max_loss: 0.2979 | max_backward: 3.5478 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5661 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 6272.0 | uss_MB: 4281.7 | child_rss_MB: 5159.4 | allocated_MB: 372.4 | reserved_MB: 23338.0 | inactive_split_MB: 256.1 +rank3 [2026-10-29 22:14:16,023] INFO : Step 45600/50000 rank3 | Train Loss: 0.9675 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2172 | Cardinality: 0.1193 | Classification: 0.0849 | Classification_binary: 0.1400 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1218 | Regression_eta: 0.0010 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1400 | Regression_cos_phi: 0.0423 | Regression_energy: 0.0599 | Regression_eta: 0.0177 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0389 | LR: 1.17e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.7204s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6781 | max_loss: 0.1576 | max_backward: 1.5089 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 0.5447 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6272.0 | uss_MB: 5337.5 | child_rss_MB: 5161.4 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 216.2 +rank3 [2026-10-29 22:18:15,631] INFO : Step 45700/50000 rank3 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1961 | Cardinality: 0.0908 | Classification: 0.0837 | Classification_binary: 0.1223 | HitGrouping: 0.1794 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1108 | Regression_eta: 0.0011 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0553 | Regression_eta: 0.0383 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0345 | LR: 1.16e-04 | DataLoad Time: 0.0218s | Model Forward Time: 3.4725s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6216 | max_loss: 0.1658 | max_backward: 1.1797 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6234.4 | uss_MB: 5336.4 | child_rss_MB: 5162.7 | allocated_MB: 370.8 | reserved_MB: 23338.0 | inactive_split_MB: 226.6 +rank3 [2026-10-29 22:22:19,535] INFO : Step 45800/50000 rank3 | Train Loss: 0.9372 | Train Loss Unweighted: 0.4491 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1073 | Classification: 0.0880 | Classification_binary: 0.1481 | HitGrouping: 0.1781 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0475 | Regression_eta: 0.0288 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0469 | LR: 1.16e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2362s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8017 | max_loss: 0.2428 | max_backward: 1.9917 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0978 | Memory: rss_MB: 6240.7 | uss_MB: 5347.0 | child_rss_MB: 5162.1 | allocated_MB: 375.2 | reserved_MB: 23338.0 | inactive_split_MB: 268.8 +rank3 [2026-10-29 22:26:26,931] INFO : Step 45900/50000 rank3 | Train Loss: 0.9600 | Train Loss Unweighted: 0.4592 | Train Loss Components: Auxiliary: 0.2179 | Cardinality: 0.1027 | Classification: 0.0816 | Classification_binary: 0.1590 | HitGrouping: 0.1875 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1438 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0542 | Regression_eta: 0.0246 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0451 | LR: 1.15e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.8437s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 1.3210 | max_loss: 0.4294 | max_backward: 3.2770 | max_optimizer_step: 0.2740 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 6235.6 | uss_MB: 5343.7 | child_rss_MB: 5161.3 | allocated_MB: 370.5 | reserved_MB: 23338.0 | inactive_split_MB: 259.6 +rank3 [2026-10-29 22:30:40,886] INFO : Step 46000/50000 rank3 | Train Loss: 0.8490 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.0938 | Classification: 0.0817 | Classification_binary: 0.1110 | HitGrouping: 0.1819 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0010 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.1009 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0435 | Regression_eta: 0.0357 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0817 | LR: 1.14e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.5929s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.5955 | max_loss: 0.1430 | max_backward: 1.2483 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 0.6657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 6228.9 | uss_MB: 5344.9 | child_rss_MB: 5164.3 | allocated_MB: 365.7 | reserved_MB: 23338.0 | inactive_split_MB: 222.2 +rank3 [2026-10-29 22:30:41,828] INFO : Running validation on rank3 +rank3 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 22:31:32,966] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:31:33,519] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 22:31:59,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 22:32:01,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 22:32:03,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 22:32:04,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 22:32:14,959] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 22:32:15,389] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:32:16,222] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 22:32:31,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 22:32:32,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 22:32:33,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 22:32:34,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 22:32:44,719] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 22:32:45,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:32:46,015] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 22:33:01,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 22:33:01,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 22:33:02,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 22:33:03,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 22:33:13,343] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 22:37:44,432] INFO : Step 46100/50000 rank3 | Train Loss: 0.9299 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1082 | Classification: 0.0899 | Classification_binary: 0.1357 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1081 | Regression_eta: 0.0010 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1292 | Regression_cos_phi: 0.0284 | Regression_energy: 0.0579 | Regression_eta: 0.0229 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0591 | LR: 1.13e-04 | DataLoad Time: 0.0198s | Model Forward Time: 6.9689s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 2.5146 | max_loss: 0.4579 | max_backward: 2.4958 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1014 | Memory: rss_MB: 6980.2 | uss_MB: 6096.7 | child_rss_MB: 5172.0 | allocated_MB: 371.5 | reserved_MB: 22784.0 | inactive_split_MB: 245.0 +rank3 [2026-10-29 22:41:43,512] INFO : Step 46200/50000 rank3 | Train Loss: 0.9944 | Train Loss Unweighted: 0.4707 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1286 | Classification: 0.0940 | Classification_binary: 0.1623 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1052 | Regression_eta: 0.0011 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1710 | Regression_cos_phi: 0.0448 | Regression_energy: 0.0476 | Regression_eta: 0.0343 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0472 | LR: 1.13e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2039s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7685 | max_loss: 0.2359 | max_backward: 1.6855 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1253 | Memory: rss_MB: 6976.0 | uss_MB: 6094.5 | child_rss_MB: 5141.0 | allocated_MB: 373.1 | reserved_MB: 22784.0 | inactive_split_MB: 264.3 +rank3 [2026-10-29 22:45:39,133] INFO : Step 46300/50000 rank3 | Train Loss: 0.9564 | Train Loss Unweighted: 0.4515 | Train Loss Components: Auxiliary: 0.2114 | Cardinality: 0.1124 | Classification: 0.0869 | Classification_binary: 0.1435 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1422 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0635 | Regression_eta: 0.0357 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0562 | LR: 1.12e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8635s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6709 | max_loss: 0.2410 | max_backward: 1.2189 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.8303 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6276.5 | uss_MB: 5413.4 | child_rss_MB: 5109.4 | allocated_MB: 372.3 | reserved_MB: 23784.0 | inactive_split_MB: 228.2 +rank3 [2026-10-29 22:49:34,648] INFO : Step 46400/50000 rank3 | Train Loss: 0.9514 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1147 | Classification: 0.0822 | Classification_binary: 0.1248 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0010 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1039 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0492 | Regression_eta: 0.0317 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0851 | LR: 1.11e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.9530s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.6748 | max_loss: 0.4371 | max_backward: 1.4077 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.8211 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6270.0 | uss_MB: 5413.5 | child_rss_MB: 5075.5 | allocated_MB: 374.2 | reserved_MB: 23988.0 | inactive_split_MB: 241.8 +rank3 [2026-10-29 22:53:41,742] INFO : Step 46500/50000 rank3 | Train Loss: 0.9517 | Train Loss Unweighted: 0.4566 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.0937 | Classification: 0.0928 | Classification_binary: 0.1533 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1031 | Regression_eta: 0.0013 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1723 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0427 | Regression_eta: 0.0341 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0427 | LR: 1.11e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9721s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0087 | max_forward: 0.7136 | max_loss: 0.1507 | max_backward: 1.5229 | max_optimizer_step: 0.2767 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6272.9 | uss_MB: 5416.3 | child_rss_MB: 5079.6 | allocated_MB: 372.0 | reserved_MB: 23988.0 | inactive_split_MB: 248.1 +rank3 [2026-10-29 22:57:41,614] INFO : Step 46600/50000 rank3 | Train Loss: 0.9541 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.2166 | Cardinality: 0.1078 | Classification: 0.0915 | Classification_binary: 0.1560 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0490 | Regression_eta: 0.0279 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0479 | LR: 1.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5722s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7127 | max_loss: 0.1604 | max_backward: 1.3079 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4722 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 6261.6 | uss_MB: 5413.8 | child_rss_MB: 5045.9 | allocated_MB: 370.5 | reserved_MB: 23988.0 | inactive_split_MB: 179.8 +rank3 [2026-10-29 23:02:40,690] INFO : Step 46700/50000 rank3 | Train Loss: 0.9378 | Train Loss Unweighted: 0.4497 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.0977 | Classification: 0.0857 | Classification_binary: 0.1264 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0015 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1070 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0577 | Regression_eta: 0.0271 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0386 | LR: 1.10e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7555s | Phase Times: max_pre_step_barrier: 0.0197 | max_batch_to_device: 0.0481 | max_forward: 0.6699 | max_loss: 0.2593 | max_backward: 1.4356 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5914 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6261.6 | uss_MB: 5415.0 | child_rss_MB: 5040.6 | allocated_MB: 368.9 | reserved_MB: 24784.0 | inactive_split_MB: 229.2 +rank3 [2026-10-29 23:06:36,577] INFO : Step 46800/50000 rank3 | Train Loss: 0.9152 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.2022 | Cardinality: 0.1011 | Classification: 0.0910 | Classification_binary: 0.1376 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0988 | Regression_eta: 0.0014 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1593 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0451 | Regression_eta: 0.0405 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0475 | LR: 1.09e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.6074s | Phase Times: max_pre_step_barrier: 0.1216 | max_batch_to_device: 0.0144 | max_forward: 1.0361 | max_loss: 0.2737 | max_backward: 1.7951 | max_optimizer_step: 0.2978 | max_tensorboard_logging: 0.5843 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6263.1 | uss_MB: 5417.5 | child_rss_MB: 5044.7 | allocated_MB: 368.9 | reserved_MB: 24994.0 | inactive_split_MB: 204.7 +rank3 [2026-10-29 23:10:45,824] INFO : Step 46900/50000 rank3 | Train Loss: 0.9278 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2149 | Cardinality: 0.1073 | Classification: 0.0827 | Classification_binary: 0.1607 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1027 | Regression_eta: 0.0010 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1827 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0515 | Regression_eta: 0.0410 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0412 | LR: 1.09e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.0948s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0446 | max_forward: 0.7815 | max_loss: 0.1553 | max_backward: 1.4732 | max_optimizer_step: 0.2929 | max_tensorboard_logging: 0.6668 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6256.7 | uss_MB: 5416.7 | child_rss_MB: 5041.4 | allocated_MB: 369.3 | reserved_MB: 24994.0 | inactive_split_MB: 274.7 +rank3 [2026-10-29 23:14:40,387] INFO : Step 47000/50000 rank3 | Train Loss: 0.9023 | Train Loss Unweighted: 0.4500 | Train Loss Components: Auxiliary: 0.2093 | Cardinality: 0.0953 | Classification: 0.0802 | Classification_binary: 0.1325 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1175 | Regression_eta: 0.0010 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0276 | Regression_energy: 0.0555 | Regression_eta: 0.0255 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0393 | LR: 1.08e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5084s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6624 | max_loss: 0.1440 | max_backward: 1.2028 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 6259.7 | uss_MB: 5421.0 | child_rss_MB: 5043.0 | allocated_MB: 369.4 | reserved_MB: 25784.0 | inactive_split_MB: 224.5 +rank3 [2026-10-29 23:14:41,333] INFO : Running validation on rank3 +rank3 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 23:15:32,860] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:15:33,157] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 23:16:02,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 23:16:03,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 23:16:04,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 23:16:05,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 23:16:15,895] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 23:16:16,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:16:16,905] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 23:16:33,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 23:16:35,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 23:16:36,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 23:16:37,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 23:16:47,359] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 23:16:47,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:16:47,862] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 23:17:03,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 23:17:04,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 23:17:04,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 23:17:05,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 23:17:15,427] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-29 23:21:36,066] INFO : Step 47100/50000 rank3 | Train Loss: 0.9138 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.2058 | Cardinality: 0.0938 | Classification: 0.0946 | Classification_binary: 0.1243 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1103 | Regression_eta: 0.0011 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1211 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0617 | Regression_eta: 0.0230 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0420 | LR: 1.07e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1755s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0063 | max_forward: 0.8048 | max_loss: 0.2433 | max_backward: 1.3159 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 1.0353 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 6952.3 | uss_MB: 6113.6 | child_rss_MB: 5037.2 | allocated_MB: 368.1 | reserved_MB: 22106.0 | inactive_split_MB: 269.0 +rank3 [2026-10-29 23:25:45,623] INFO : Step 47200/50000 rank3 | Train Loss: 0.9875 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2263 | Cardinality: 0.1259 | Classification: 0.0903 | Classification_binary: 0.1586 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1110 | Regression_eta: 0.0012 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.1779 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0680 | Regression_eta: 0.0413 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0553 | LR: 1.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2828s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0486 | max_forward: 0.7581 | max_loss: 0.1506 | max_backward: 1.5914 | max_optimizer_step: 0.2300 | max_tensorboard_logging: 0.7953 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 6951.3 | uss_MB: 6117.8 | child_rss_MB: 5032.9 | allocated_MB: 370.5 | reserved_MB: 23104.0 | inactive_split_MB: 195.8 +rank3 [2026-10-29 23:29:53,763] INFO : Step 47300/50000 rank3 | Train Loss: 0.9350 | Train Loss Unweighted: 0.4467 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.0958 | Classification: 0.0857 | Classification_binary: 0.1512 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0433 | Regression_eta: 0.0418 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0482 | LR: 1.06e-04 | DataLoad Time: 0.0214s | Model Forward Time: 6.4014s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0184 | max_forward: 1.9890 | max_loss: 0.4771 | max_backward: 3.1656 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6309.1 | uss_MB: 5475.6 | child_rss_MB: 5028.4 | allocated_MB: 370.8 | reserved_MB: 23104.0 | inactive_split_MB: 271.1 +rank3 [2026-10-29 23:33:58,356] INFO : Step 47400/50000 rank3 | Train Loss: 0.8593 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.0807 | Classification: 0.0812 | Classification_binary: 0.1182 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1114 | Regression_eta: 0.0011 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1134 | Regression_cos_phi: 0.0279 | Regression_energy: 0.0572 | Regression_eta: 0.0245 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0325 | LR: 1.06e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.0137s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0125 | max_forward: 0.7544 | max_loss: 0.1826 | max_backward: 1.4299 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8027 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6311.1 | uss_MB: 5475.9 | child_rss_MB: 5031.1 | allocated_MB: 369.9 | reserved_MB: 23104.0 | inactive_split_MB: 253.6 +rank3 [2026-10-29 23:37:53,326] INFO : Step 47500/50000 rank3 | Train Loss: 0.9089 | Train Loss Unweighted: 0.4235 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1000 | Classification: 0.0799 | Classification_binary: 0.1391 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1008 | Regression_eta: 0.0012 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0524 | Regression_eta: 0.0321 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0555 | LR: 1.06e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2728s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0068 | max_forward: 0.7329 | max_loss: 0.1827 | max_backward: 1.5814 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9161 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6306.2 | uss_MB: 5478.3 | child_rss_MB: 5026.1 | allocated_MB: 371.7 | reserved_MB: 23104.0 | inactive_split_MB: 243.6 +rank3 [2026-10-29 23:41:48,170] INFO : Step 47600/50000 rank3 | Train Loss: 0.9781 | Train Loss Unweighted: 0.4635 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1226 | Classification: 0.0910 | Classification_binary: 0.1607 | HitGrouping: 0.1784 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0578 | Regression_eta: 0.0230 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0600 | LR: 1.05e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.4264s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8435 | max_loss: 0.1581 | max_backward: 1.8989 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5341 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6310.2 | uss_MB: 5483.0 | child_rss_MB: 5033.7 | allocated_MB: 370.6 | reserved_MB: 24122.0 | inactive_split_MB: 257.4 +rank3 [2026-10-29 23:46:00,211] INFO : Step 47700/50000 rank3 | Train Loss: 0.9311 | Train Loss Unweighted: 0.4523 | Train Loss Components: Auxiliary: 0.2105 | Cardinality: 0.0992 | Classification: 0.0889 | Classification_binary: 0.1321 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1138 | Regression_eta: 0.0012 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.0978 | Regression_cos_phi: 0.0342 | Regression_energy: 0.0561 | Regression_eta: 0.0214 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0548 | LR: 1.05e-04 | DataLoad Time: 0.0185s | Model Forward Time: 3.5072s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0068 | max_forward: 0.6708 | max_loss: 0.1613 | max_backward: 1.2326 | max_optimizer_step: 0.1946 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6306.7 | uss_MB: 5481.0 | child_rss_MB: 5031.8 | allocated_MB: 368.1 | reserved_MB: 24122.0 | inactive_split_MB: 276.0 +rank3 [2026-10-29 23:50:02,911] INFO : Step 47800/50000 rank3 | Train Loss: 0.9063 | Train Loss Unweighted: 0.4379 | Train Loss Components: Auxiliary: 0.2020 | Cardinality: 0.0932 | Classification: 0.0920 | Classification_binary: 0.1284 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0011 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0547 | Regression_eta: 0.0448 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0790 | LR: 1.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8434s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0610 | max_forward: 0.6626 | max_loss: 0.1349 | max_backward: 1.5091 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6302.7 | uss_MB: 5485.7 | child_rss_MB: 5029.8 | allocated_MB: 369.5 | reserved_MB: 24122.0 | inactive_split_MB: 236.5 +rank3 [2026-10-29 23:54:07,643] INFO : Step 47900/50000 rank3 | Train Loss: 0.9591 | Train Loss Unweighted: 0.4478 | Train Loss Components: Auxiliary: 0.2152 | Cardinality: 0.1022 | Classification: 0.0882 | Classification_binary: 0.1545 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1003 | Regression_eta: 0.0014 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0543 | Regression_eta: 0.0365 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0572 | LR: 1.04e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.1694s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8744 | max_loss: 0.2241 | max_backward: 1.6370 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 6302.2 | uss_MB: 5483.9 | child_rss_MB: 5030.3 | allocated_MB: 370.5 | reserved_MB: 24122.0 | inactive_split_MB: 218.9 +rank3 [2026-10-30 00:00:28,988] INFO : Step 48000/50000 rank3 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1221 | Classification: 0.0823 | Classification_binary: 0.1525 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1079 | Regression_eta: 0.0017 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0549 | Regression_eta: 0.0605 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0665 | LR: 1.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.1275s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0057 | max_forward: 0.7069 | max_loss: 0.2447 | max_backward: 1.4090 | max_optimizer_step: 0.1933 | max_tensorboard_logging: 0.8132 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1250 | Memory: rss_MB: 6301.2 | uss_MB: 5485.0 | child_rss_MB: 5024.5 | allocated_MB: 368.8 | reserved_MB: 24122.0 | inactive_split_MB: 269.6 +rank3 [2026-10-30 00:00:29,930] INFO : Running validation on rank3 +rank3 [2026-10-30 00:00:29,930] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 00:01:19,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:01:20,066] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 00:01:46,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 00:01:48,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 00:01:49,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 00:01:50,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 00:02:00,564] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-30 00:02:01,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:02:01,859] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 00:02:17,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 00:02:18,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 00:02:19,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 00:02:20,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 00:02:30,399] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 00:02:32,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:02:32,991] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 00:02:48,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 00:02:49,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 00:02:49,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 00:02:50,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 00:03:00,624] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-30 00:07:25,203] INFO : Step 48100/50000 rank3 | Train Loss: 0.9084 | Train Loss Unweighted: 0.4360 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1162 | Classification: 0.0769 | Classification_binary: 0.1204 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1183 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1194 | Regression_cos_phi: 0.0342 | Regression_energy: 0.0634 | Regression_eta: 0.0367 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0305 | LR: 1.03e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9224s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1372 | max_backward: 1.2024 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.9158 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7374.2 | uss_MB: 6557.5 | child_rss_MB: 5038.7 | allocated_MB: 367.7 | reserved_MB: 24830.0 | inactive_split_MB: 222.3 +rank3 [2026-10-30 00:11:33,471] INFO : Step 48200/50000 rank3 | Train Loss: 0.9585 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2151 | Cardinality: 0.1199 | Classification: 0.0839 | Classification_binary: 0.1503 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0503 | Regression_eta: 0.0281 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0760 | LR: 1.03e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4363s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7766 | max_loss: 0.1498 | max_backward: 1.7414 | max_optimizer_step: 0.3242 | max_tensorboard_logging: 0.7504 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 7374.2 | uss_MB: 6560.6 | child_rss_MB: 5031.5 | allocated_MB: 370.4 | reserved_MB: 24830.0 | inactive_split_MB: 179.3 +rank3 [2026-10-30 00:15:41,698] INFO : Step 48300/50000 rank3 | Train Loss: 1.0575 | Train Loss Unweighted: 0.4963 | Train Loss Components: Auxiliary: 0.2408 | Cardinality: 0.1341 | Classification: 0.0958 | Classification_binary: 0.1687 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1138 | Regression_eta: 0.0013 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0698 | Regression_eta: 0.0495 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0673 | LR: 1.03e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2037s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7497 | max_loss: 0.2201 | max_backward: 1.7233 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6378.2 | uss_MB: 5566.5 | child_rss_MB: 5032.3 | allocated_MB: 372.3 | reserved_MB: 24830.0 | inactive_split_MB: 269.6 +rank3 [2026-10-30 00:19:56,563] INFO : Step 48400/50000 rank3 | Train Loss: 0.9108 | Train Loss Unweighted: 0.4349 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1093 | Classification: 0.0691 | Classification_binary: 0.1311 | HitGrouping: 0.1830 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1153 | Regression_eta: 0.0013 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1367 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0588 | Regression_eta: 0.0401 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0428 | LR: 1.02e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.8539s | Phase Times: max_pre_step_barrier: 0.1141 | max_batch_to_device: 0.0064 | max_forward: 0.6489 | max_loss: 0.1390 | max_backward: 1.1801 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6705 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6380.2 | uss_MB: 5566.3 | child_rss_MB: 5030.1 | allocated_MB: 370.1 | reserved_MB: 24830.0 | inactive_split_MB: 261.5 +rank3 [2026-10-30 00:23:53,558] INFO : Step 48500/50000 rank3 | Train Loss: 0.9694 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1004 | Classification: 0.0844 | Classification_binary: 0.1367 | HitGrouping: 0.1928 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1194 | Regression_eta: 0.0012 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0595 | Regression_eta: 0.0334 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0639 | LR: 1.02e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.7562s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 1.7512 | max_loss: 0.2678 | max_backward: 1.9538 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.4804 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 6374.4 | uss_MB: 5565.1 | child_rss_MB: 5023.6 | allocated_MB: 371.5 | reserved_MB: 26842.0 | inactive_split_MB: 176.5 +rank3 [2026-10-30 00:27:51,107] INFO : Step 48600/50000 rank3 | Train Loss: 0.9724 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1148 | Classification: 0.0854 | Classification_binary: 0.1551 | HitGrouping: 0.1844 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0469 | Regression_eta: 0.0258 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0481 | LR: 1.02e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0069s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0086 | max_forward: 0.6984 | max_loss: 0.2525 | max_backward: 1.6611 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6375.4 | uss_MB: 5569.8 | child_rss_MB: 5026.2 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 234.8 +rank3 [2026-10-30 00:31:52,432] INFO : Step 48700/50000 rank3 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4863 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1212 | Classification: 0.0968 | Classification_binary: 0.1569 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1143 | Regression_eta: 0.0012 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0512 | Regression_eta: 0.0309 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0790 | LR: 1.02e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9641s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7410 | max_loss: 0.2472 | max_backward: 1.4647 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.6585 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6316.1 | uss_MB: 5565.7 | child_rss_MB: 4823.4 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 275.8 +rank3 [2026-10-30 00:35:40,394] INFO : Step 48800/50000 rank3 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1981 | Cardinality: 0.0898 | Classification: 0.0814 | Classification_binary: 0.1196 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1094 | Regression_eta: 0.0012 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1214 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0496 | Regression_eta: 0.0466 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0315 | LR: 1.01e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7706s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.6122 | max_loss: 0.1775 | max_backward: 1.1932 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6316.1 | uss_MB: 5568.8 | child_rss_MB: 4813.9 | allocated_MB: 368.0 | reserved_MB: 26842.0 | inactive_split_MB: 263.7 +rank3 [2026-10-30 00:39:23,570] INFO : Step 48900/50000 rank3 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1005 | Classification: 0.0873 | Classification_binary: 0.1401 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0576 | Regression_eta: 0.0278 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0781 | LR: 1.01e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.9258s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0809 | max_forward: 0.7397 | max_loss: 0.1455 | max_backward: 1.5027 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6303.5 | uss_MB: 5491.8 | child_rss_MB: 4768.8 | allocated_MB: 373.0 | reserved_MB: 26842.0 | inactive_split_MB: 191.3 +rank3 [2026-10-30 00:43:08,795] INFO : Step 49000/50000 rank3 | Train Loss: 1.0019 | Train Loss Unweighted: 0.4657 | Train Loss Components: Auxiliary: 0.2334 | Cardinality: 0.1354 | Classification: 0.0893 | Classification_binary: 0.1587 | HitGrouping: 0.1838 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0010 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1965 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0593 | Regression_eta: 0.0305 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0473 | LR: 1.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4539s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1085 | max_forward: 0.8072 | max_loss: 0.1558 | max_backward: 1.7809 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.6082 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6305.0 | uss_MB: 5577.9 | child_rss_MB: 4749.4 | allocated_MB: 370.9 | reserved_MB: 26842.0 | inactive_split_MB: 271.3 +rank3 [2026-10-30 00:43:09,868] INFO : Running validation on rank3 +rank3 [2026-10-30 00:43:09,868] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 00:43:09,868] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 00:43:59,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:43:59,787] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 00:44:27,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 00:44:28,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 00:44:29,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 00:44:30,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 00:44:40,232] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 00:44:40,645] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:44:41,122] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 00:44:56,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 00:44:57,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 00:44:58,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 00:44:59,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 00:45:09,771] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 00:45:10,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:45:10,743] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 00:45:26,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 00:45:27,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 00:45:27,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 00:45:28,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 00:45:38,483] INFO : Time taken to make predictions on device 3 is: 0.22 min +rank3 [2026-10-30 00:49:42,469] INFO : Step 49100/50000 rank3 | Train Loss: 0.9797 | Train Loss Unweighted: 0.4561 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1125 | Classification: 0.0877 | Classification_binary: 0.1433 | HitGrouping: 0.1905 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0013 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0531 | Regression_eta: 0.0397 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0669 | LR: 1.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7299s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6869 | max_loss: 0.1436 | max_backward: 1.2639 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.4741 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0455 | Memory: rss_MB: 7470.2 | uss_MB: 6734.9 | child_rss_MB: 4778.3 | allocated_MB: 370.4 | reserved_MB: 22116.0 | inactive_split_MB: 274.9 +rank3 [2026-10-30 00:53:47,121] INFO : Step 49200/50000 rank3 | Train Loss: 0.8929 | Train Loss Unweighted: 0.4393 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1020 | Classification: 0.0903 | Classification_binary: 0.1250 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0009 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1219 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0522 | Regression_eta: 0.0264 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0463 | LR: 1.01e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.7526s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6890 | max_loss: 0.2430 | max_backward: 1.3427 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.6347 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7470.5 | uss_MB: 6738.6 | child_rss_MB: 4771.6 | allocated_MB: 368.3 | reserved_MB: 24092.0 | inactive_split_MB: 187.3 +rank3 [2026-10-30 00:57:33,940] INFO : Step 49300/50000 rank3 | Train Loss: 1.0122 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1210 | Classification: 0.0951 | Classification_binary: 0.1621 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1066 | Regression_eta: 0.0015 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1843 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0469 | Regression_eta: 0.0362 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0501 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8964s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1495 | max_backward: 1.5906 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6415.4 | uss_MB: 5684.6 | child_rss_MB: 4746.9 | allocated_MB: 370.9 | reserved_MB: 24092.0 | inactive_split_MB: 226.9 +rank3 [2026-10-30 01:01:33,225] INFO : Step 49400/50000 rank3 | Train Loss: 0.9790 | Train Loss Unweighted: 0.4695 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1137 | Classification: 0.0883 | Classification_binary: 0.1513 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1131 | Regression_eta: 0.0013 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0544 | Regression_eta: 0.0458 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0728 | LR: 1.00e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.0360s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2433 | max_backward: 1.8823 | max_optimizer_step: 0.3081 | max_tensorboard_logging: 0.5951 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6413.4 | uss_MB: 5608.6 | child_rss_MB: 4765.4 | allocated_MB: 372.2 | reserved_MB: 24092.0 | inactive_split_MB: 278.9 +rank3 [2026-10-30 01:05:37,233] INFO : Step 49500/50000 rank3 | Train Loss: 0.9706 | Train Loss Unweighted: 0.4581 | Train Loss Components: Auxiliary: 0.2120 | Cardinality: 0.1158 | Classification: 0.0879 | Classification_binary: 0.1279 | HitGrouping: 0.1920 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0293 | Classification_binary: 0.0945 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0523 | Regression_eta: 0.0273 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0727 | LR: 1.00e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.9617s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6307 | max_loss: 0.1505 | max_backward: 1.3337 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.8584 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6409.4 | uss_MB: 5683.7 | child_rss_MB: 4759.9 | allocated_MB: 370.4 | reserved_MB: 24614.0 | inactive_split_MB: 171.1 +rank3 [2026-10-30 01:09:49,675] INFO : Step 49600/50000 rank3 | Train Loss: 0.9965 | Train Loss Unweighted: 0.4752 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1098 | Classification: 0.1016 | Classification_binary: 0.1437 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1133 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1238 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0498 | Regression_eta: 0.0256 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0457 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1634s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.6933 | max_loss: 0.1496 | max_backward: 1.5639 | max_optimizer_step: 0.3077 | max_tensorboard_logging: 0.7702 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6412.2 | uss_MB: 5687.4 | child_rss_MB: 4755.4 | allocated_MB: 371.3 | reserved_MB: 24614.0 | inactive_split_MB: 250.0 +rank3 [2026-10-30 01:13:44,418] INFO : Step 49700/50000 rank3 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4775 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1108 | Classification: 0.1000 | Classification_binary: 0.1702 | HitGrouping: 0.1796 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1019 | Regression_eta: 0.0014 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1784 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0493 | Regression_eta: 0.0373 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0876 | LR: 1.00e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.2334s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7591 | max_loss: 0.1933 | max_backward: 1.7410 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 6407.4 | uss_MB: 5675.2 | child_rss_MB: 4703.7 | allocated_MB: 372.4 | reserved_MB: 24614.0 | inactive_split_MB: 259.5 +rank3 [2026-10-30 01:17:46,084] INFO : Step 49800/50000 rank3 | Train Loss: 0.9472 | Train Loss Unweighted: 0.4401 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1188 | Classification: 0.0821 | Classification_binary: 0.1427 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1211 | Regression_cos_phi: 0.0341 | Regression_energy: 0.0516 | Regression_eta: 0.0348 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0389 | LR: 1.00e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.7717s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7430 | max_loss: 0.2153 | max_backward: 1.0846 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.7647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6388.3 | uss_MB: 5552.5 | child_rss_MB: 4630.2 | allocated_MB: 368.0 | reserved_MB: 24614.0 | inactive_split_MB: 273.7 +rank3 [2026-10-30 01:21:39,660] INFO : Step 49900/50000 rank3 | Train Loss: 0.9411 | Train Loss Unweighted: 0.4586 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.0996 | Classification: 0.0961 | Classification_binary: 0.1284 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1145 | Regression_eta: 0.0013 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1016 | Regression_cos_phi: 0.0377 | Regression_energy: 0.0546 | Regression_eta: 0.0210 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0381 | LR: 1.00e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0426s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.6992 | max_loss: 0.1527 | max_backward: 1.3836 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8576 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6363.3 | uss_MB: 5692.1 | child_rss_MB: 4528.3 | allocated_MB: 368.7 | reserved_MB: 24614.0 | inactive_split_MB: 241.3 +rank3 [2026-10-30 01:25:29,378] INFO : Step 50000/50000 rank3 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2297 | Cardinality: 0.1196 | Classification: 0.0902 | Classification_binary: 0.1583 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0981 | Regression_energy: 0.0518 | Regression_eta: 0.0571 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0612 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2531s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.7146 | max_loss: 0.1481 | max_backward: 1.6207 | max_optimizer_step: 0.2721 | max_tensorboard_logging: 0.8195 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6364.5 | uss_MB: 5690.8 | child_rss_MB: 4519.4 | allocated_MB: 371.5 | reserved_MB: 25088.0 | inactive_split_MB: 268.9 +rank3 [2026-10-30 01:25:30,313] INFO : Running validation on rank3 +rank3 [2026-10-30 01:25:30,313] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 01:26:21,850] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:26:22,340] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 01:26:49,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 01:26:51,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 01:26:52,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 01:26:53,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 01:27:03,496] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-30 01:27:03,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:27:04,490] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 01:27:20,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 01:27:21,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 01:27:22,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 01:27:23,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 01:27:33,050] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 01:27:34,256] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:27:34,756] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 01:27:51,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 01:27:51,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 01:27:52,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 01:27:53,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 01:28:03,057] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 01:28:03,372] INFO : Training completed. Total time on device 3: 638.890min diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.4 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.4 new file mode 100644 index 0000000000000000000000000000000000000000..1ac3666c0639ce4fe3fcb7da795fb8734a797bde --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.4 @@ -0,0 +1,2484 @@ +rank4 [2026-10-28 11:12:33,122] INFO : Initializing model with seed=12345; process seed=12349 +rank4 [2026-10-28 11:12:33,123] INFO : configured dtype=torch.bfloat16 for autocast +rank4 [2026-10-28 11:12:54,339] INFO : Instantiating model +rank4 [2026-10-28 11:12:54,339] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank4 [2026-10-28 11:12:54,339] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank4 [2026-10-28 11:12:54,358] INFO : Input encoding initialization took 0.02s +rank4 [2026-10-28 11:12:54,358] INFO : nn0 parameters: 171776 +rank4 [2026-10-28 11:12:54,358] INFO : Initializing shared backbone layers, num_convs=6 +rank4 [2026-10-28 11:12:54,358] INFO : Initializing backbone layer 0 +rank4 [2026-10-28 11:12:54,358] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:54,362] INFO : Initializing backbone layer 1 +rank4 [2026-10-28 11:12:54,362] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:54,365] INFO : Initializing backbone layer 2 +rank4 [2026-10-28 11:12:54,365] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:54,368] INFO : Initializing backbone layer 3 +rank4 [2026-10-28 11:12:54,368] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:54,371] INFO : Initializing backbone layer 4 +rank4 [2026-10-28 11:12:54,371] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:54,374] INFO : Initializing backbone layer 5 +rank4 [2026-10-28 11:12:54,374] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-28 11:12:54,377] INFO : Backbone initialization took 0.02s +rank4 [2026-10-28 11:12:54,377] INFO : backbone parameters: 2374656 +rank4 [2026-10-28 11:12:54,377] INFO : Initializing output DNNs +rank4 [2026-10-28 11:12:54,427] INFO : Output DNNs initialization took 0.05s +rank4 [2026-10-28 11:12:54,427] INFO : backbone_mode=BackboneMode.SHARED +rank4 [2026-10-28 11:12:54,427] INFO : jagged_attention enabled=True +rank4 [2026-10-28 11:12:54,427] INFO : flash_attn_varlen enabled=True +rank4 [2026-10-28 11:12:54,427] INFO : task_queries enabled=False +rank4 [2026-10-28 11:12:54,427] INFO : classification_norm parameters: 0 +rank4 [2026-10-28 11:12:54,427] INFO : regression_norm parameters: 0 +rank4 [2026-10-28 11:12:54,427] INFO : classification_query parameters: 0 +rank4 [2026-10-28 11:12:54,427] INFO : regression_query parameters: 0 +rank4 [2026-10-28 11:12:54,427] INFO : classification_readout parameters: 0 +rank4 [2026-10-28 11:12:54,427] INFO : regression_readout parameters: 0 +rank4 [2026-10-28 11:12:54,427] INFO : output_mode=set +rank4 [2026-10-28 11:12:54,427] INFO : set_decoder parameters: 4713812 +rank4 [2026-10-28 11:12:54,427] INFO : nn_binary_particle parameters: 0 +rank4 [2026-10-28 11:12:54,427] INFO : nn_pid parameters: 0 +rank4 [2026-10-28 11:12:54,428] INFO : nn_pt parameters: 0 +rank4 [2026-10-28 11:12:54,428] INFO : nn_eta parameters: 0 +rank4 [2026-10-28 11:12:54,428] INFO : nn_sin_phi parameters: 0 +rank4 [2026-10-28 11:12:54,428] INFO : nn_cos_phi parameters: 0 +rank4 [2026-10-28 11:12:54,428] INFO : nn_energy parameters: 0 +rank4 [2026-10-28 11:12:54,428] INFO : Total MLPF parameters: 7260244 +rank4 [2026-10-28 11:12:54,428] INFO : MLPF __init__ done +rank4 [2026-10-28 11:12:54,430] INFO : Instantiated model +rank4 [2026-10-28 11:12:54,430] INFO : Moving model to device rank=4 +rank4 [2026-10-28 11:12:54,447] INFO : Moved model to device rank=4 +rank4 [2026-10-28 11:12:54,449] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank4 [2026-10-28 11:12:54,449] INFO : Configured model for SyncBatchNorm rank=4 +rank4 [2026-10-28 11:12:54,644] INFO : Configured model for DistributedDataParallel rank=4 +rank4 [2026-10-28 11:12:54,779] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank4 [2026-10-28 11:12:54,779] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank4 [2026-10-28 11:12:54,780] INFO : Trainable parameters: 7260244 +rank4 [2026-10-28 11:12:54,780] INFO : Non-trainable parameters: 0 +rank4 [2026-10-28 11:12:54,780] INFO : Total parameters: 7260244 +rank4 [2026-10-28 11:12:54,783] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank4 [2026-10-28 11:12:54,783] INFO : Getting dataloaders +rank4 [2026-10-28 11:12:54,783] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:03,739] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:04,811] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,142] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-28 11:13:06,143] INFO : Creating WeightedMixIterator with 1 data loaders. +rank4 [2026-10-28 11:13:06,143] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,414] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,692] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,951] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-28 11:13:06,951] INFO : Creating InterleavedIterator with 1 data loaders. +rank4 [2026-10-28 11:13:21,046] INFO : Got dataloaders +rank4 [2026-10-28 11:13:21,046] INFO : loader split=train rank=4 len=21112 +rank4 [2026-10-28 11:13:21,046] INFO : loader split=valid rank=4 len=12 +rank4 [2026-10-28 11:13:21,046] INFO : Starting training from step 1 to 50000 on rank 4 of 8 +rank4 [2026-10-28 11:13:21,046] INFO : Creating train_iterator +rank4 [2026-10-28 11:13:21,046] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-28 11:13:35,563] INFO : Created train_iterator +rank4 [2026-10-28 11:17:15,587] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5434697668626904, 'Classification': 0.9147019968181849, 'Regression_pt': 1.0888657218962907, 'Regression_eta': 0.019410275079426356, 'Regression_sin_phi': 0.00660347732307855, 'Regression_cos_phi': 0.006484371119877323, 'Regression_energy': 1.1263945941627025}, weights={'Classification_binary': 1.0, 'Classification': 0.5941495299339294, 'Regression_pt': 0.4991154968738556, 'Regression_eta': 27.99907684326172, 'Regression_sin_phi': 82.30054473876953, 'Regression_cos_phi': 83.812255859375, 'Regression_energy': 0.48248612880706787} +rank4 [2026-10-28 11:17:17,664] INFO : Step 100/50000 rank4 | Train Loss: 5.4941 | Train Loss Unweighted: 3.3573 | Train Loss Components: Auxiliary: 1.0155 | Cardinality: 0.6417 | Classification: 0.8373 | Classification_binary: 0.4757 | HitGrouping: 0.4797 | Regression_cos_phi: 0.0063 | Regression_energy: 1.0524 | Regression_eta: 0.0130 | Regression_pt: 0.9668 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.7893 | Classification_binary: 0.4633 | Regression_cos_phi: 0.0058 | Regression_energy: 1.1054 | Regression_eta: 0.0150 | Regression_pt: 1.0130 | Regression_sin_phi: 0.0052 | LR: 1.00e-03 | DataLoad Time: 0.0155s | Model Forward Time: 3.0149s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.5597 | max_loss: 0.2615 | max_backward: 1.0157 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.4654 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4647.0 | uss_MB: 3401.7 | child_rss_MB: 5567.8 | allocated_MB: 341.0 | reserved_MB: 22914.0 | inactive_split_MB: 237.2 +rank4 [2026-10-28 11:20:50,235] INFO : Step 200/50000 rank4 | Train Loss: 4.4651 | Train Loss Unweighted: 2.7232 | Train Loss Components: Auxiliary: 0.8610 | Cardinality: 0.5153 | Classification: 0.6425 | Classification_binary: 0.5162 | HitGrouping: 0.4582 | Regression_cos_phi: 0.0043 | Regression_energy: 0.7694 | Regression_eta: 0.0086 | Regression_pt: 0.7776 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3787 | Classification_binary: 0.5397 | Regression_cos_phi: 0.4140 | Regression_energy: 0.3491 | Regression_eta: 0.2727 | Regression_pt: 0.3657 | Regression_sin_phi: 0.3973 | LR: 1.00e-03 | DataLoad Time: 0.0148s | Model Forward Time: 3.7251s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7368 | max_loss: 0.2463 | max_backward: 1.7032 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5468 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4651.2 | uss_MB: 3409.7 | child_rss_MB: 5571.3 | allocated_MB: 343.8 | reserved_MB: 23264.0 | inactive_split_MB: 284.1 +rank4 [2026-10-28 11:24:24,163] INFO : Step 300/50000 rank4 | Train Loss: 3.6383 | Train Loss Unweighted: 2.1819 | Train Loss Components: Auxiliary: 0.6931 | Cardinality: 0.3447 | Classification: 0.5089 | Classification_binary: 0.4824 | HitGrouping: 0.4305 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5908 | Regression_eta: 0.0074 | Regression_pt: 0.5852 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2760 | Classification_binary: 0.4867 | Regression_cos_phi: 0.2798 | Regression_energy: 0.2420 | Regression_eta: 0.1911 | Regression_pt: 0.2494 | Regression_sin_phi: 0.3133 | LR: 1.00e-03 | DataLoad Time: 0.0150s | Model Forward Time: 3.8725s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0315 | max_forward: 0.6855 | max_loss: 0.1568 | max_backward: 1.3632 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 1.0182 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4652.7 | uss_MB: 3415.2 | child_rss_MB: 5569.6 | allocated_MB: 341.4 | reserved_MB: 23916.0 | inactive_split_MB: 289.5 +rank4 [2026-10-28 11:27:58,397] INFO : Step 400/50000 rank4 | Train Loss: 3.5064 | Train Loss Unweighted: 1.9938 | Train Loss Components: Auxiliary: 0.6582 | Cardinality: 0.3738 | Classification: 0.4591 | Classification_binary: 0.4306 | HitGrouping: 0.4041 | Regression_cos_phi: 0.0039 | Regression_energy: 0.5433 | Regression_eta: 0.0080 | Regression_pt: 0.5456 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2602 | Classification_binary: 0.4381 | Regression_cos_phi: 0.3695 | Regression_energy: 0.3211 | Regression_eta: 0.3099 | Regression_pt: 0.3359 | Regression_sin_phi: 0.3370 | LR: 1.00e-03 | DataLoad Time: 0.0147s | Model Forward Time: 3.7436s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.6100 | max_loss: 0.1486 | max_backward: 1.2396 | max_optimizer_step: 0.1474 | max_tensorboard_logging: 1.0157 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4650.3 | uss_MB: 3416.8 | child_rss_MB: 5584.9 | allocated_MB: 342.9 | reserved_MB: 23982.0 | inactive_split_MB: 297.5 +rank4 [2026-10-28 11:31:35,335] INFO : Step 500/50000 rank4 | Train Loss: 3.8899 | Train Loss Unweighted: 2.1856 | Train Loss Components: Auxiliary: 0.6982 | Cardinality: 0.3682 | Classification: 0.4707 | Classification_binary: 0.4836 | HitGrouping: 0.4071 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6048 | Regression_eta: 0.0103 | Regression_pt: 0.6070 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2719 | Classification_binary: 0.4664 | Regression_cos_phi: 0.3686 | Regression_energy: 0.3162 | Regression_eta: 0.3159 | Regression_pt: 0.3182 | Regression_sin_phi: 0.3008 | LR: 1.00e-03 | DataLoad Time: 0.0158s | Model Forward Time: 4.1824s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.7460 | max_loss: 0.2622 | max_backward: 1.7372 | max_optimizer_step: 0.1478 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 4651.3 | uss_MB: 3418.2 | child_rss_MB: 5567.0 | allocated_MB: 344.2 | reserved_MB: 24988.0 | inactive_split_MB: 288.4 +rank4 [2026-10-28 11:35:12,859] INFO : Step 600/50000 rank4 | Train Loss: 3.5648 | Train Loss Unweighted: 2.0654 | Train Loss Components: Auxiliary: 0.6756 | Cardinality: 0.3628 | Classification: 0.4159 | Classification_binary: 0.5246 | HitGrouping: 0.3632 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5539 | Regression_eta: 0.0077 | Regression_pt: 0.5557 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2257 | Classification_binary: 0.5081 | Regression_cos_phi: 0.2645 | Regression_energy: 0.2663 | Regression_eta: 0.1897 | Regression_pt: 0.2834 | Regression_sin_phi: 0.2946 | LR: 1.00e-03 | DataLoad Time: 0.0154s | Model Forward Time: 4.1927s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0071 | max_forward: 0.7483 | max_loss: 0.2574 | max_backward: 1.6305 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.9709 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 4647.1 | uss_MB: 3418.2 | child_rss_MB: 5567.3 | allocated_MB: 344.6 | reserved_MB: 24988.0 | inactive_split_MB: 289.5 +rank4 [2026-10-28 11:38:47,058] INFO : Step 700/50000 rank4 | Train Loss: 3.0543 | Train Loss Unweighted: 1.8237 | Train Loss Components: Auxiliary: 0.6036 | Cardinality: 0.3094 | Classification: 0.3407 | Classification_binary: 0.4142 | HitGrouping: 0.3367 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5254 | Regression_eta: 0.0060 | Regression_pt: 0.5314 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1842 | Classification_binary: 0.3998 | Regression_cos_phi: 0.2784 | Regression_energy: 0.2555 | Regression_eta: 0.1470 | Regression_pt: 0.2686 | Regression_sin_phi: 0.1737 | LR: 1.00e-03 | DataLoad Time: 0.0170s | Model Forward Time: 3.4879s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.6445 | max_loss: 0.2136 | max_backward: 1.2179 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.7893 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4647.0 | uss_MB: 3416.3 | child_rss_MB: 5577.3 | allocated_MB: 341.4 | reserved_MB: 24988.0 | inactive_split_MB: 289.7 +rank4 [2026-10-28 11:42:25,056] INFO : Step 800/50000 rank4 | Train Loss: 3.1854 | Train Loss Unweighted: 1.9126 | Train Loss Components: Auxiliary: 0.6154 | Cardinality: 0.2934 | Classification: 0.4077 | Classification_binary: 0.4086 | HitGrouping: 0.3317 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5404 | Regression_eta: 0.0067 | Regression_pt: 0.5423 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2337 | Classification_binary: 0.3515 | Regression_cos_phi: 0.3065 | Regression_energy: 0.2802 | Regression_eta: 0.1756 | Regression_pt: 0.2881 | Regression_sin_phi: 0.4379 | LR: 9.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.3172s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.6526 | max_loss: 0.2001 | max_backward: 1.2539 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5713 | max_tensorboard_flush: 0.0138 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 4642.6 | uss_MB: 3416.3 | child_rss_MB: 5579.0 | allocated_MB: 346.1 | reserved_MB: 24988.0 | inactive_split_MB: 222.7 +rank4 [2026-10-28 11:46:07,652] INFO : Step 900/50000 rank4 | Train Loss: 3.4950 | Train Loss Unweighted: 2.0678 | Train Loss Components: Auxiliary: 0.6566 | Cardinality: 0.3469 | Classification: 0.4160 | Classification_binary: 0.4953 | HitGrouping: 0.3295 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5663 | Regression_eta: 0.0083 | Regression_pt: 0.5742 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2676 | Classification_binary: 0.5015 | Regression_cos_phi: 0.3041 | Regression_energy: 0.2783 | Regression_eta: 0.2429 | Regression_pt: 0.2986 | Regression_sin_phi: 0.3233 | LR: 9.99e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.8985s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8131 | max_loss: 0.2548 | max_backward: 1.8396 | max_optimizer_step: 0.1492 | max_tensorboard_logging: 0.4938 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 4643.4 | uss_MB: 3417.5 | child_rss_MB: 5588.2 | allocated_MB: 344.2 | reserved_MB: 26006.0 | inactive_split_MB: 273.7 +rank4 [2026-10-28 11:49:58,411] INFO : Step 1000/50000 rank4 | Train Loss: 3.0581 | Train Loss Unweighted: 1.8801 | Train Loss Components: Auxiliary: 0.5942 | Cardinality: 0.2760 | Classification: 0.3603 | Classification_binary: 0.4232 | HitGrouping: 0.3177 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5411 | Regression_eta: 0.0066 | Regression_pt: 0.5426 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2232 | Classification_binary: 0.4880 | Regression_cos_phi: 0.3073 | Regression_energy: 0.2384 | Regression_eta: 0.2167 | Regression_pt: 0.2470 | Regression_sin_phi: 0.3247 | LR: 9.99e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2161s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1006 | max_forward: 0.7513 | max_loss: 0.1459 | max_backward: 1.3658 | max_optimizer_step: 0.1511 | max_tensorboard_logging: 1.2506 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4639.1 | uss_MB: 3415.3 | child_rss_MB: 5584.2 | allocated_MB: 344.4 | reserved_MB: 26006.0 | inactive_split_MB: 271.1 +rank4 [2026-10-28 11:49:59,499] INFO : Running validation on rank4 +rank4 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 11:50:52,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:50:53,226] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 11:51:19,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 11:51:20,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 11:51:21,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 11:51:22,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 11:51:32,256] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 11:51:32,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:51:33,321] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 11:51:49,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 11:51:50,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 11:51:51,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 11:51:52,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 11:52:02,229] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 11:52:02,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:52:03,755] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 11:52:19,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 11:52:19,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 11:52:20,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 11:52:21,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 11:52:31,293] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 11:56:35,453] INFO : Step 1100/50000 rank4 | Train Loss: 2.9181 | Train Loss Unweighted: 1.8556 | Train Loss Components: Auxiliary: 0.5873 | Cardinality: 0.2540 | Classification: 0.3370 | Classification_binary: 0.3789 | HitGrouping: 0.3247 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5584 | Regression_eta: 0.0064 | Regression_pt: 0.5695 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1904 | Classification_binary: 0.3642 | Regression_cos_phi: 0.2847 | Regression_energy: 0.2966 | Regression_eta: 0.2113 | Regression_pt: 0.3152 | Regression_sin_phi: 0.2311 | LR: 9.99e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9464s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8528 | max_loss: 0.1390 | max_backward: 1.4220 | max_optimizer_step: 0.2689 | max_tensorboard_logging: 0.9135 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 6058.3 | uss_MB: 4835.8 | child_rss_MB: 5619.2 | allocated_MB: 344.1 | reserved_MB: 21080.0 | inactive_split_MB: 227.9 +rank4 [2026-10-28 12:00:25,220] INFO : Step 1200/50000 rank4 | Train Loss: 3.1885 | Train Loss Unweighted: 1.9202 | Train Loss Components: Auxiliary: 0.6265 | Cardinality: 0.3516 | Classification: 0.3718 | Classification_binary: 0.4294 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5507 | Regression_eta: 0.0065 | Regression_pt: 0.5556 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2191 | Classification_binary: 0.4544 | Regression_cos_phi: 0.2338 | Regression_energy: 0.2401 | Regression_eta: 0.1737 | Regression_pt: 0.2479 | Regression_sin_phi: 0.2794 | LR: 9.99e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.4087s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0076 | max_forward: 0.7370 | max_loss: 0.3921 | max_backward: 1.7945 | max_optimizer_step: 0.1639 | max_tensorboard_logging: 1.0479 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 6061.1 | uss_MB: 4838.4 | child_rss_MB: 5619.8 | allocated_MB: 344.9 | reserved_MB: 23662.0 | inactive_split_MB: 241.9 +rank4 [2026-10-28 12:04:16,198] INFO : Step 1300/50000 rank4 | Train Loss: 3.2694 | Train Loss Unweighted: 1.9280 | Train Loss Components: Auxiliary: 0.6193 | Cardinality: 0.3461 | Classification: 0.3601 | Classification_binary: 0.4731 | HitGrouping: 0.3156 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5359 | Regression_eta: 0.0069 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2189 | Classification_binary: 0.4560 | Regression_cos_phi: 0.2264 | Regression_energy: 0.2765 | Regression_eta: 0.1402 | Regression_pt: 0.2903 | Regression_sin_phi: 0.2435 | LR: 9.98e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.5238s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0540 | max_forward: 0.9909 | max_loss: 0.3234 | max_backward: 2.2964 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 1.1252 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5498.9 | uss_MB: 4193.7 | child_rss_MB: 5645.6 | allocated_MB: 344.9 | reserved_MB: 24052.0 | inactive_split_MB: 299.1 +rank4 [2026-10-28 12:07:55,472] INFO : Step 1400/50000 rank4 | Train Loss: 2.8491 | Train Loss Unweighted: 1.7889 | Train Loss Components: Auxiliary: 0.5645 | Cardinality: 0.2711 | Classification: 0.3226 | Classification_binary: 0.3768 | HitGrouping: 0.3109 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5376 | Regression_eta: 0.0061 | Regression_pt: 0.5407 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1883 | Classification_binary: 0.3758 | Regression_cos_phi: 0.3063 | Regression_energy: 0.2240 | Regression_eta: 0.2601 | Regression_pt: 0.2342 | Regression_sin_phi: 0.3090 | LR: 9.98e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.7575s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0094 | max_forward: 0.6378 | max_loss: 0.4388 | max_backward: 1.2886 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 0.9631 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 5501.2 | uss_MB: 4277.0 | child_rss_MB: 5652.8 | allocated_MB: 350.4 | reserved_MB: 24916.0 | inactive_split_MB: 216.1 +rank4 [2026-10-28 12:11:42,090] INFO : Step 1500/50000 rank4 | Train Loss: 2.8928 | Train Loss Unweighted: 1.7749 | Train Loss Components: Auxiliary: 0.5690 | Cardinality: 0.2892 | Classification: 0.3566 | Classification_binary: 0.3704 | HitGrouping: 0.3078 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5147 | Regression_eta: 0.0058 | Regression_pt: 0.5216 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2015 | Classification_binary: 0.3692 | Regression_cos_phi: 0.2419 | Regression_energy: 0.2578 | Regression_eta: 0.1682 | Regression_pt: 0.2665 | Regression_sin_phi: 0.2129 | LR: 9.98e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5604s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.6094 | max_loss: 0.4244 | max_backward: 1.4479 | max_optimizer_step: 0.1602 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5488.8 | uss_MB: 4270.8 | child_rss_MB: 5621.5 | allocated_MB: 342.2 | reserved_MB: 24916.0 | inactive_split_MB: 182.9 +rank4 [2026-10-28 12:15:27,599] INFO : Step 1600/50000 rank4 | Train Loss: 2.9805 | Train Loss Unweighted: 1.8512 | Train Loss Components: Auxiliary: 0.5725 | Cardinality: 0.2336 | Classification: 0.3605 | Classification_binary: 0.4404 | HitGrouping: 0.3159 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5142 | Regression_eta: 0.0065 | Regression_pt: 0.5235 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1873 | Classification_binary: 0.4279 | Regression_cos_phi: 0.2153 | Regression_energy: 0.2436 | Regression_eta: 0.1516 | Regression_pt: 0.2527 | Regression_sin_phi: 0.2138 | LR: 9.98e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7167s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7382 | max_loss: 0.1451 | max_backward: 1.5359 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.4679 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5499.3 | uss_MB: 4281.1 | child_rss_MB: 5637.2 | allocated_MB: 343.2 | reserved_MB: 25046.0 | inactive_split_MB: 260.8 +rank4 [2026-10-28 12:19:19,672] INFO : Step 1700/50000 rank4 | Train Loss: 2.9423 | Train Loss Unweighted: 1.7603 | Train Loss Components: Auxiliary: 0.5618 | Cardinality: 0.2834 | Classification: 0.3350 | Classification_binary: 0.4141 | HitGrouping: 0.3155 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4976 | Regression_eta: 0.0064 | Regression_pt: 0.5011 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2202 | Classification_binary: 0.4037 | Regression_cos_phi: 0.2643 | Regression_energy: 0.2185 | Regression_eta: 0.2411 | Regression_pt: 0.2254 | Regression_sin_phi: 0.2207 | LR: 9.97e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2289s | Phase Times: max_pre_step_barrier: 0.0070 | max_batch_to_device: 0.0065 | max_forward: 0.8351 | max_loss: 0.2466 | max_backward: 1.7292 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0857 | Memory: rss_MB: 5499.7 | uss_MB: 4254.7 | child_rss_MB: 5622.9 | allocated_MB: 342.0 | reserved_MB: 25046.0 | inactive_split_MB: 198.9 +rank4 [2026-10-28 12:23:11,815] INFO : Step 1800/50000 rank4 | Train Loss: 2.7182 | Train Loss Unweighted: 1.6765 | Train Loss Components: Auxiliary: 0.5305 | Cardinality: 0.2556 | Classification: 0.3337 | Classification_binary: 0.3613 | HitGrouping: 0.3103 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4855 | Regression_eta: 0.0054 | Regression_pt: 0.4854 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2114 | Classification_binary: 0.3598 | Regression_cos_phi: 0.1798 | Regression_energy: 0.2302 | Regression_eta: 0.1186 | Regression_pt: 0.2354 | Regression_sin_phi: 0.2219 | LR: 9.97e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0931s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6573 | max_loss: 0.2402 | max_backward: 1.4111 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 1.0385 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0959 | Memory: rss_MB: 5501.3 | uss_MB: 4283.5 | child_rss_MB: 5636.1 | allocated_MB: 340.6 | reserved_MB: 25046.0 | inactive_split_MB: 181.6 +rank4 [2026-10-28 12:26:53,740] INFO : Step 1900/50000 rank4 | Train Loss: 2.9893 | Train Loss Unweighted: 1.8606 | Train Loss Components: Auxiliary: 0.5839 | Cardinality: 0.2900 | Classification: 0.3568 | Classification_binary: 0.4041 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5399 | Regression_eta: 0.0063 | Regression_pt: 0.5477 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1959 | Classification_binary: 0.3667 | Regression_cos_phi: 0.2388 | Regression_energy: 0.2685 | Regression_eta: 0.1757 | Regression_pt: 0.2854 | Regression_sin_phi: 0.2469 | LR: 9.97e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8698s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.6296 | max_loss: 0.1482 | max_backward: 1.2931 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 1.0537 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 5503.8 | uss_MB: 4286.6 | child_rss_MB: 5619.9 | allocated_MB: 344.8 | reserved_MB: 25048.0 | inactive_split_MB: 225.2 +rank4 [2026-10-28 12:30:53,516] INFO : Step 2000/50000 rank4 | Train Loss: 3.0007 | Train Loss Unweighted: 1.8040 | Train Loss Components: Auxiliary: 0.5673 | Cardinality: 0.3091 | Classification: 0.3519 | Classification_binary: 0.4482 | HitGrouping: 0.3137 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4922 | Regression_eta: 0.0062 | Regression_pt: 0.4996 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2211 | Classification_binary: 0.4691 | Regression_cos_phi: 0.2881 | Regression_energy: 0.2352 | Regression_eta: 0.2223 | Regression_pt: 0.2507 | Regression_sin_phi: 0.2658 | LR: 9.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1674s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6813 | max_loss: 0.2416 | max_backward: 1.5165 | max_optimizer_step: 0.1635 | max_tensorboard_logging: 0.9842 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5502.9 | uss_MB: 4287.2 | child_rss_MB: 5669.7 | allocated_MB: 346.7 | reserved_MB: 25696.0 | inactive_split_MB: 287.8 +rank4 [2026-10-28 12:30:54,414] INFO : Running validation on rank4 +rank4 [2026-10-28 12:30:54,414] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 12:30:54,414] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 12:31:46,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:31:46,628] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 12:32:13,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 12:32:14,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 12:32:15,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 12:32:16,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 12:32:26,327] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 12:32:27,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:32:27,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 12:32:44,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 12:32:44,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 12:32:45,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 12:32:46,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 12:32:56,686] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 12:32:57,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:32:57,629] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 12:33:13,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 12:33:14,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 12:33:15,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 12:33:15,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 12:33:25,924] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 12:37:37,593] INFO : Step 2100/50000 rank4 | Train Loss: 2.6299 | Train Loss Unweighted: 1.6429 | Train Loss Components: Auxiliary: 0.5172 | Cardinality: 0.2489 | Classification: 0.3119 | Classification_binary: 0.3591 | HitGrouping: 0.3047 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4785 | Regression_eta: 0.0053 | Regression_pt: 0.4834 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1925 | Classification_binary: 0.3462 | Regression_cos_phi: 0.2272 | Regression_energy: 0.1956 | Regression_eta: 0.1830 | Regression_pt: 0.2017 | Regression_sin_phi: 0.2641 | LR: 9.96e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0184s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0685 | max_forward: 0.7343 | max_loss: 0.1422 | max_backward: 1.3363 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.0015 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6447.9 | uss_MB: 5231.1 | child_rss_MB: 5629.3 | allocated_MB: 341.1 | reserved_MB: 22546.0 | inactive_split_MB: 269.0 +rank4 [2026-10-28 12:41:24,842] INFO : Step 2200/50000 rank4 | Train Loss: 2.8483 | Train Loss Unweighted: 1.7068 | Train Loss Components: Auxiliary: 0.5372 | Cardinality: 0.2547 | Classification: 0.3411 | Classification_binary: 0.3679 | HitGrouping: 0.3030 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4876 | Regression_eta: 0.0061 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2244 | Classification_binary: 0.4003 | Regression_cos_phi: 0.2842 | Regression_energy: 0.2469 | Regression_eta: 0.1521 | Regression_pt: 0.2627 | Regression_sin_phi: 0.2022 | LR: 9.96e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.4920s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6352 | max_loss: 0.1414 | max_backward: 1.3119 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.4703 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6448.9 | uss_MB: 5233.9 | child_rss_MB: 5636.2 | allocated_MB: 341.4 | reserved_MB: 22598.0 | inactive_split_MB: 313.4 +rank4 [2026-10-28 12:45:27,860] INFO : Step 2300/50000 rank4 | Train Loss: 3.1410 | Train Loss Unweighted: 1.8301 | Train Loss Components: Auxiliary: 0.5918 | Cardinality: 0.3583 | Classification: 0.3517 | Classification_binary: 0.4348 | HitGrouping: 0.3083 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5121 | Regression_eta: 0.0065 | Regression_pt: 0.5184 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2091 | Classification_binary: 0.4536 | Regression_cos_phi: 0.2936 | Regression_energy: 0.2325 | Regression_eta: 0.1691 | Regression_pt: 0.2420 | Regression_sin_phi: 0.2395 | LR: 9.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8600s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.7258 | max_loss: 0.1520 | max_backward: 1.6288 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.6027 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0166 | Memory: rss_MB: 5633.9 | uss_MB: 4422.4 | child_rss_MB: 5644.1 | allocated_MB: 346.5 | reserved_MB: 23596.0 | inactive_split_MB: 273.5 +rank4 [2026-10-28 12:49:22,411] INFO : Step 2400/50000 rank4 | Train Loss: 2.8063 | Train Loss Unweighted: 1.7109 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.2683 | Classification: 0.3276 | Classification_binary: 0.3929 | HitGrouping: 0.3025 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4873 | Regression_eta: 0.0054 | Regression_pt: 0.4919 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1953 | Classification_binary: 0.4077 | Regression_cos_phi: 0.2026 | Regression_energy: 0.2299 | Regression_eta: 0.1692 | Regression_pt: 0.2379 | Regression_sin_phi: 0.2780 | LR: 9.95e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.5893s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0314 | max_forward: 0.7331 | max_loss: 0.1502 | max_backward: 1.4174 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.5071 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0157 | Memory: rss_MB: 5635.8 | uss_MB: 4426.4 | child_rss_MB: 5648.9 | allocated_MB: 344.4 | reserved_MB: 23596.0 | inactive_split_MB: 276.8 +rank4 [2026-10-28 12:53:10,912] INFO : Step 2500/50000 rank4 | Train Loss: 2.5300 | Train Loss Unweighted: 1.6034 | Train Loss Components: Auxiliary: 0.5018 | Cardinality: 0.2387 | Classification: 0.2872 | Classification_binary: 0.3313 | HitGrouping: 0.3000 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4857 | Regression_eta: 0.0042 | Regression_pt: 0.4904 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1320 | Classification_binary: 0.2900 | Regression_cos_phi: 0.1621 | Regression_energy: 0.2738 | Regression_eta: 0.0888 | Regression_pt: 0.2872 | Regression_sin_phi: 0.1688 | LR: 9.94e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.0263s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0177 | max_forward: 0.6104 | max_loss: 0.1394 | max_backward: 1.2397 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 1.1029 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5639.0 | uss_MB: 4422.1 | child_rss_MB: 5639.4 | allocated_MB: 342.5 | reserved_MB: 23596.0 | inactive_split_MB: 302.8 +rank4 [2026-10-28 12:56:55,928] INFO : Step 2600/50000 rank4 | Train Loss: 2.8992 | Train Loss Unweighted: 1.7075 | Train Loss Components: Auxiliary: 0.5471 | Cardinality: 0.3219 | Classification: 0.3420 | Classification_binary: 0.4005 | HitGrouping: 0.3027 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4737 | Regression_eta: 0.0057 | Regression_pt: 0.4796 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1862 | Classification_binary: 0.3513 | Regression_cos_phi: 0.2530 | Regression_energy: 0.2383 | Regression_eta: 0.1439 | Regression_pt: 0.2460 | Regression_sin_phi: 0.2022 | LR: 9.94e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.7061s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8849 | max_loss: 0.1495 | max_backward: 1.9136 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1254 | Memory: rss_MB: 5638.3 | uss_MB: 4434.4 | child_rss_MB: 5644.3 | allocated_MB: 343.4 | reserved_MB: 23596.0 | inactive_split_MB: 290.6 +rank4 [2026-10-28 13:00:46,528] INFO : Step 2700/50000 rank4 | Train Loss: 2.8481 | Train Loss Unweighted: 1.7200 | Train Loss Components: Auxiliary: 0.5468 | Cardinality: 0.2918 | Classification: 0.3199 | Classification_binary: 0.4344 | HitGrouping: 0.3029 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4756 | Regression_eta: 0.0057 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2090 | Classification_binary: 0.4368 | Regression_cos_phi: 0.2353 | Regression_energy: 0.2345 | Regression_eta: 0.1966 | Regression_pt: 0.2433 | Regression_sin_phi: 0.2070 | LR: 9.94e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.3224s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0064 | max_forward: 0.7554 | max_loss: 0.2027 | max_backward: 1.6333 | max_optimizer_step: 0.1633 | max_tensorboard_logging: 0.9978 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5637.9 | uss_MB: 4434.7 | child_rss_MB: 5643.6 | allocated_MB: 343.8 | reserved_MB: 23598.0 | inactive_split_MB: 284.2 +rank4 [2026-10-28 13:04:40,297] INFO : Step 2800/50000 rank4 | Train Loss: 2.6381 | Train Loss Unweighted: 1.5955 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2387 | Classification: 0.2848 | Classification_binary: 0.3420 | HitGrouping: 0.2974 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4746 | Regression_eta: 0.0056 | Regression_pt: 0.4829 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1863 | Classification_binary: 0.3260 | Regression_cos_phi: 0.2714 | Regression_energy: 0.2590 | Regression_eta: 0.1502 | Regression_pt: 0.2694 | Regression_sin_phi: 0.2554 | LR: 9.93e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.7401s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6805 | max_loss: 0.2368 | max_backward: 1.1707 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5635.7 | uss_MB: 4432.9 | child_rss_MB: 5638.4 | allocated_MB: 341.2 | reserved_MB: 24554.0 | inactive_split_MB: 308.0 +rank4 [2026-10-28 13:08:36,459] INFO : Step 2900/50000 rank4 | Train Loss: 2.7831 | Train Loss Unweighted: 1.6528 | Train Loss Components: Auxiliary: 0.5323 | Cardinality: 0.3031 | Classification: 0.3069 | Classification_binary: 0.3681 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4798 | Regression_eta: 0.0056 | Regression_pt: 0.4869 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2018 | Classification_binary: 0.3831 | Regression_cos_phi: 0.2387 | Regression_energy: 0.2244 | Regression_eta: 0.1474 | Regression_pt: 0.2386 | Regression_sin_phi: 0.1873 | LR: 9.93e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7869s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6242 | max_loss: 0.1475 | max_backward: 1.3573 | max_optimizer_step: 0.2831 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5637.0 | uss_MB: 4433.8 | child_rss_MB: 5645.5 | allocated_MB: 341.4 | reserved_MB: 25570.0 | inactive_split_MB: 292.0 +rank4 [2026-10-28 13:12:15,921] INFO : Step 3000/50000 rank4 | Train Loss: 2.9336 | Train Loss Unweighted: 1.7260 | Train Loss Components: Auxiliary: 0.5466 | Cardinality: 0.2829 | Classification: 0.3524 | Classification_binary: 0.4205 | HitGrouping: 0.3085 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4673 | Regression_eta: 0.0061 | Regression_pt: 0.4733 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1858 | Classification_binary: 0.4044 | Regression_cos_phi: 0.2307 | Regression_energy: 0.2365 | Regression_eta: 0.1927 | Regression_pt: 0.2471 | Regression_sin_phi: 0.2589 | LR: 9.92e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1526s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0386 | max_forward: 0.8057 | max_loss: 0.1488 | max_backward: 1.9376 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5635.9 | uss_MB: 4440.1 | child_rss_MB: 5640.1 | allocated_MB: 346.5 | reserved_MB: 25570.0 | inactive_split_MB: 281.4 +rank4 [2026-10-28 13:12:16,814] INFO : Running validation on rank4 +rank4 [2026-10-28 13:12:16,814] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 13:12:16,814] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 13:13:09,907] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:13:10,420] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 13:13:29,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 13:13:30,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 13:13:31,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 13:13:32,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 13:13:42,735] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 13:13:43,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:13:44,016] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 13:13:59,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 13:14:00,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 13:14:01,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 13:14:02,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 13:14:12,943] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 13:14:13,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:14:14,219] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 13:14:29,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 13:14:30,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 13:14:30,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 13:14:31,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 13:14:41,373] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 13:18:52,946] INFO : Step 3100/50000 rank4 | Train Loss: 2.7478 | Train Loss Unweighted: 1.6734 | Train Loss Components: Auxiliary: 0.5216 | Cardinality: 0.2531 | Classification: 0.2981 | Classification_binary: 0.3781 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4881 | Regression_eta: 0.0057 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1710 | Classification_binary: 0.3925 | Regression_cos_phi: 0.2946 | Regression_energy: 0.2291 | Regression_eta: 0.1664 | Regression_pt: 0.2426 | Regression_sin_phi: 0.3101 | LR: 9.91e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5552s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.6511 | max_loss: 0.3988 | max_backward: 1.4306 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 1.2631 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5749.3 | uss_MB: 4552.2 | child_rss_MB: 5658.3 | allocated_MB: 342.5 | reserved_MB: 23120.0 | inactive_split_MB: 194.6 +rank4 [2026-10-28 13:22:38,494] INFO : Step 3200/50000 rank4 | Train Loss: 2.5254 | Train Loss Unweighted: 1.5710 | Train Loss Components: Auxiliary: 0.4888 | Cardinality: 0.2129 | Classification: 0.2845 | Classification_binary: 0.3271 | HitGrouping: 0.2964 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4770 | Regression_eta: 0.0050 | Regression_pt: 0.4723 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1932 | Classification_binary: 0.3615 | Regression_cos_phi: 0.2540 | Regression_energy: 0.2047 | Regression_eta: 0.1600 | Regression_pt: 0.2102 | Regression_sin_phi: 0.2449 | LR: 9.91e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.5495s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6134 | max_loss: 0.2441 | max_backward: 1.2833 | max_optimizer_step: 0.2753 | max_tensorboard_logging: 0.5629 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5733.9 | uss_MB: 4558.8 | child_rss_MB: 5628.8 | allocated_MB: 343.6 | reserved_MB: 23340.0 | inactive_split_MB: 191.9 +rank4 [2026-10-28 13:26:30,566] INFO : Step 3300/50000 rank4 | Train Loss: 2.8037 | Train Loss Unweighted: 1.6635 | Train Loss Components: Auxiliary: 0.5345 | Cardinality: 0.2841 | Classification: 0.3310 | Classification_binary: 0.3857 | HitGrouping: 0.3002 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4633 | Regression_eta: 0.0058 | Regression_pt: 0.4718 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1898 | Classification_binary: 0.3629 | Regression_cos_phi: 0.1719 | Regression_energy: 0.2174 | Regression_eta: 0.1233 | Regression_pt: 0.2320 | Regression_sin_phi: 0.1599 | LR: 9.90e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.2971s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9147 | max_loss: 0.1535 | max_backward: 1.9100 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.6631 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5735.1 | uss_MB: 4562.9 | child_rss_MB: 5661.7 | allocated_MB: 344.0 | reserved_MB: 23618.0 | inactive_split_MB: 250.0 +rank4 [2026-10-28 13:30:24,442] INFO : Step 3400/50000 rank4 | Train Loss: 2.8855 | Train Loss Unweighted: 1.7067 | Train Loss Components: Auxiliary: 0.5454 | Cardinality: 0.2859 | Classification: 0.3252 | Classification_binary: 0.4155 | HitGrouping: 0.2965 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4731 | Regression_eta: 0.0067 | Regression_pt: 0.4802 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1989 | Classification_binary: 0.4168 | Regression_cos_phi: 0.2743 | Regression_energy: 0.2402 | Regression_eta: 0.1911 | Regression_pt: 0.2551 | Regression_sin_phi: 0.2386 | LR: 9.90e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5635s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0438 | max_forward: 0.7271 | max_loss: 0.1506 | max_backward: 1.3643 | max_optimizer_step: 0.1716 | max_tensorboard_logging: 0.4827 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5729.2 | uss_MB: 4562.1 | child_rss_MB: 5658.8 | allocated_MB: 346.0 | reserved_MB: 23618.0 | inactive_split_MB: 235.6 +rank4 [2026-10-28 13:34:14,209] INFO : Step 3500/50000 rank4 | Train Loss: 2.4422 | Train Loss Unweighted: 1.4809 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.2120 | Classification: 0.2615 | Classification_binary: 0.3296 | HitGrouping: 0.2898 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4369 | Regression_eta: 0.0054 | Regression_pt: 0.4427 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1515 | Classification_binary: 0.3495 | Regression_cos_phi: 0.2445 | Regression_energy: 0.2019 | Regression_eta: 0.1856 | Regression_pt: 0.2135 | Regression_sin_phi: 0.2178 | LR: 9.89e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.7314s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0058 | max_forward: 0.7056 | max_loss: 0.1971 | max_backward: 1.3359 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.6307 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5720.3 | uss_MB: 4563.4 | child_rss_MB: 5630.0 | allocated_MB: 339.0 | reserved_MB: 24610.0 | inactive_split_MB: 280.8 +rank4 [2026-10-28 13:38:02,721] INFO : Step 3600/50000 rank4 | Train Loss: 2.5780 | Train Loss Unweighted: 1.5250 | Train Loss Components: Auxiliary: 0.4892 | Cardinality: 0.2463 | Classification: 0.3079 | Classification_binary: 0.3446 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4295 | Regression_eta: 0.0052 | Regression_pt: 0.4324 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1401 | Classification_binary: 0.2864 | Regression_cos_phi: 0.1854 | Regression_energy: 0.2168 | Regression_eta: 0.1251 | Regression_pt: 0.2253 | Regression_sin_phi: 0.1400 | LR: 9.89e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.4638s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1453 | max_backward: 1.4106 | max_optimizer_step: 0.1681 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0126 | Memory: rss_MB: 5726.7 | uss_MB: 4567.6 | child_rss_MB: 5631.8 | allocated_MB: 343.9 | reserved_MB: 24610.0 | inactive_split_MB: 244.5 +rank4 [2026-10-28 13:41:50,384] INFO : Step 3700/50000 rank4 | Train Loss: 2.9213 | Train Loss Unweighted: 1.6952 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.3153 | Classification: 0.3418 | Classification_binary: 0.4227 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4570 | Regression_eta: 0.0063 | Regression_pt: 0.4612 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2068 | Classification_binary: 0.4048 | Regression_cos_phi: 0.2227 | Regression_energy: 0.2243 | Regression_eta: 0.1825 | Regression_pt: 0.2378 | Regression_sin_phi: 0.2528 | LR: 9.88e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2202s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7710 | max_loss: 0.1507 | max_backward: 1.6430 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.8888 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0145 | Memory: rss_MB: 5726.2 | uss_MB: 4569.4 | child_rss_MB: 5661.5 | allocated_MB: 345.4 | reserved_MB: 24610.0 | inactive_split_MB: 268.9 +rank4 [2026-10-28 13:45:46,299] INFO : Step 3800/50000 rank4 | Train Loss: 2.6063 | Train Loss Unweighted: 1.5289 | Train Loss Components: Auxiliary: 0.4933 | Cardinality: 0.2661 | Classification: 0.2839 | Classification_binary: 0.3652 | HitGrouping: 0.2879 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4319 | Regression_eta: 0.0058 | Regression_pt: 0.4369 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1673 | Classification_binary: 0.3879 | Regression_cos_phi: 0.2267 | Regression_energy: 0.2025 | Regression_eta: 0.1265 | Regression_pt: 0.2138 | Regression_sin_phi: 0.1959 | LR: 9.87e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1723s | Phase Times: max_pre_step_barrier: 0.1134 | max_batch_to_device: 0.0063 | max_forward: 0.6784 | max_loss: 0.1451 | max_backward: 1.4269 | max_optimizer_step: 0.2795 | max_tensorboard_logging: 0.9060 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5726.3 | uss_MB: 4569.5 | child_rss_MB: 5664.0 | allocated_MB: 343.8 | reserved_MB: 24610.0 | inactive_split_MB: 202.2 +rank4 [2026-10-28 13:49:32,923] INFO : Step 3900/50000 rank4 | Train Loss: 2.2761 | Train Loss Unweighted: 1.4165 | Train Loss Components: Auxiliary: 0.4517 | Cardinality: 0.1938 | Classification: 0.2553 | Classification_binary: 0.2930 | HitGrouping: 0.2882 | Regression_cos_phi: 0.0020 | Regression_energy: 0.4264 | Regression_eta: 0.0045 | Regression_pt: 0.4331 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1294 | Classification_binary: 0.2426 | Regression_cos_phi: 0.1377 | Regression_energy: 0.2123 | Regression_eta: 0.0769 | Regression_pt: 0.2233 | Regression_sin_phi: 0.1232 | LR: 9.87e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8905s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6197 | max_loss: 0.1501 | max_backward: 1.2810 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.0075 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 5727.5 | uss_MB: 3764.5 | child_rss_MB: 5661.9 | allocated_MB: 343.5 | reserved_MB: 24610.0 | inactive_split_MB: 286.6 +rank4 [2026-10-28 13:53:23,686] INFO : Step 4000/50000 rank4 | Train Loss: 2.6760 | Train Loss Unweighted: 1.5903 | Train Loss Components: Auxiliary: 0.5112 | Cardinality: 0.2445 | Classification: 0.3070 | Classification_binary: 0.3812 | HitGrouping: 0.2906 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4413 | Regression_eta: 0.0062 | Regression_pt: 0.4492 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1921 | Classification_binary: 0.3844 | Regression_cos_phi: 0.2152 | Regression_energy: 0.2129 | Regression_eta: 0.1574 | Regression_pt: 0.2244 | Regression_sin_phi: 0.2011 | LR: 9.86e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.0137s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0492 | max_forward: 0.7834 | max_loss: 0.1586 | max_backward: 1.6903 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.5589 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5731.5 | uss_MB: 4577.2 | child_rss_MB: 5661.6 | allocated_MB: 345.5 | reserved_MB: 24610.0 | inactive_split_MB: 252.6 +rank4 [2026-10-28 13:53:24,557] INFO : Running validation on rank4 +rank4 [2026-10-28 13:53:24,557] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 13:53:24,557] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 13:54:13,891] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:54:14,418] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 13:54:39,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 13:54:41,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 13:54:41,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 13:54:42,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 13:54:52,976] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 13:54:53,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:54:53,946] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 13:55:10,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 13:55:10,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 13:55:11,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 13:55:12,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 13:55:22,750] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 13:55:23,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:55:23,757] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 13:55:39,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 13:55:39,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 13:55:40,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 13:55:41,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 13:55:51,160] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 13:59:55,049] INFO : Step 4100/50000 rank4 | Train Loss: 2.6923 | Train Loss Unweighted: 1.5640 | Train Loss Components: Auxiliary: 0.5058 | Cardinality: 0.2671 | Classification: 0.2981 | Classification_binary: 0.4073 | HitGrouping: 0.2911 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4210 | Regression_eta: 0.0055 | Regression_pt: 0.4265 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1770 | Classification_binary: 0.4558 | Regression_cos_phi: 0.2207 | Regression_energy: 0.1848 | Regression_eta: 0.1494 | Regression_pt: 0.1927 | Regression_sin_phi: 0.2681 | LR: 9.85e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1494s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0141 | max_forward: 0.7422 | max_loss: 0.3919 | max_backward: 1.3629 | max_optimizer_step: 0.1890 | max_tensorboard_logging: 0.9903 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6680.3 | uss_MB: 5525.0 | child_rss_MB: 5670.9 | allocated_MB: 345.2 | reserved_MB: 23430.0 | inactive_split_MB: 294.4 +rank4 [2026-10-28 14:03:47,899] INFO : Step 4200/50000 rank4 | Train Loss: 2.3415 | Train Loss Unweighted: 1.4424 | Train Loss Components: Auxiliary: 0.4542 | Cardinality: 0.2058 | Classification: 0.2540 | Classification_binary: 0.3143 | HitGrouping: 0.2846 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4310 | Regression_eta: 0.0050 | Regression_pt: 0.4337 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1610 | Classification_binary: 0.2687 | Regression_cos_phi: 0.1461 | Regression_energy: 0.1879 | Regression_eta: 0.1114 | Regression_pt: 0.1973 | Regression_sin_phi: 0.1733 | LR: 9.84e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.7409s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0168 | max_forward: 0.8335 | max_loss: 0.2425 | max_backward: 1.5586 | max_optimizer_step: 0.1744 | max_tensorboard_logging: 0.4781 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5904.9 | uss_MB: 4751.1 | child_rss_MB: 5672.4 | allocated_MB: 341.2 | reserved_MB: 24406.0 | inactive_split_MB: 273.9 +rank4 [2026-10-28 14:07:35,177] INFO : Step 4300/50000 rank4 | Train Loss: 2.5102 | Train Loss Unweighted: 1.4857 | Train Loss Components: Auxiliary: 0.4757 | Cardinality: 0.2309 | Classification: 0.2786 | Classification_binary: 0.3543 | HitGrouping: 0.2938 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4177 | Regression_eta: 0.0054 | Regression_pt: 0.4246 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1775 | Classification_binary: 0.3486 | Regression_cos_phi: 0.1688 | Regression_energy: 0.2269 | Regression_eta: 0.1579 | Regression_pt: 0.2375 | Regression_sin_phi: 0.2389 | LR: 9.84e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.6000s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6520 | max_loss: 0.2453 | max_backward: 1.4084 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.5540 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5910.3 | uss_MB: 4757.7 | child_rss_MB: 5668.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 239.6 +rank4 [2026-10-28 14:11:23,917] INFO : Step 4400/50000 rank4 | Train Loss: 2.8258 | Train Loss Unweighted: 1.5691 | Train Loss Components: Auxiliary: 0.5223 | Cardinality: 0.3313 | Classification: 0.2877 | Classification_binary: 0.4120 | HitGrouping: 0.2902 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4255 | Regression_eta: 0.0063 | Regression_pt: 0.4316 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1980 | Classification_binary: 0.4503 | Regression_cos_phi: 0.3835 | Regression_energy: 0.2138 | Regression_eta: 0.2301 | Regression_pt: 0.2208 | Regression_sin_phi: 0.2341 | LR: 9.83e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.0438s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.7635 | max_loss: 0.2580 | max_backward: 1.7456 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.5221 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 5909.1 | uss_MB: 4757.0 | child_rss_MB: 5674.0 | allocated_MB: 344.6 | reserved_MB: 24406.0 | inactive_split_MB: 261.0 +rank4 [2026-10-28 14:15:17,054] INFO : Step 4500/50000 rank4 | Train Loss: 2.5622 | Train Loss Unweighted: 1.4664 | Train Loss Components: Auxiliary: 0.4942 | Cardinality: 0.3132 | Classification: 0.2685 | Classification_binary: 0.3695 | HitGrouping: 0.2876 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4070 | Regression_eta: 0.0048 | Regression_pt: 0.4117 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1721 | Classification_binary: 0.3887 | Regression_cos_phi: 0.2119 | Regression_energy: 0.2179 | Regression_eta: 0.1760 | Regression_pt: 0.2267 | Regression_sin_phi: 0.2519 | LR: 9.82e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.2150s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0462 | max_forward: 1.1330 | max_loss: 0.3526 | max_backward: 2.1744 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5571 | max_tensorboard_flush: 0.0016 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 5912.8 | uss_MB: 4736.1 | child_rss_MB: 5674.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 288.8 +rank4 [2026-10-28 14:19:09,445] INFO : Step 4600/50000 rank4 | Train Loss: 2.2516 | Train Loss Unweighted: 1.3548 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2151 | Classification: 0.2428 | Classification_binary: 0.2886 | HitGrouping: 0.2811 | Regression_cos_phi: 0.0019 | Regression_energy: 0.4059 | Regression_eta: 0.0042 | Regression_pt: 0.4088 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1556 | Classification_binary: 0.2977 | Regression_cos_phi: 0.1649 | Regression_energy: 0.1869 | Regression_eta: 0.1127 | Regression_pt: 0.1968 | Regression_sin_phi: 0.2134 | LR: 9.81e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7002s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0445 | max_forward: 0.8897 | max_loss: 0.2214 | max_backward: 1.8748 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5913.2 | uss_MB: 4741.7 | child_rss_MB: 5653.3 | allocated_MB: 341.6 | reserved_MB: 25392.0 | inactive_split_MB: 298.4 +rank4 [2026-10-28 14:23:11,224] INFO : Step 4700/50000 rank4 | Train Loss: 2.5991 | Train Loss Unweighted: 1.5142 | Train Loss Components: Auxiliary: 0.4914 | Cardinality: 0.2679 | Classification: 0.2997 | Classification_binary: 0.3514 | HitGrouping: 0.2867 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4228 | Regression_eta: 0.0061 | Regression_pt: 0.4290 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1755 | Classification_binary: 0.3323 | Regression_cos_phi: 0.2566 | Regression_energy: 0.1845 | Regression_eta: 0.1866 | Regression_pt: 0.1921 | Regression_sin_phi: 0.2484 | LR: 9.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8895s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7362 | max_loss: 0.2547 | max_backward: 1.8331 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5918.9 | uss_MB: 4764.8 | child_rss_MB: 5675.1 | allocated_MB: 343.1 | reserved_MB: 26384.0 | inactive_split_MB: 292.4 +rank4 [2026-10-28 14:27:13,049] INFO : Step 4800/50000 rank4 | Train Loss: 2.7077 | Train Loss Unweighted: 1.5200 | Train Loss Components: Auxiliary: 0.5118 | Cardinality: 0.3360 | Classification: 0.2606 | Classification_binary: 0.3918 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4264 | Regression_eta: 0.0053 | Regression_pt: 0.4305 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1506 | Classification_binary: 0.4012 | Regression_cos_phi: 0.2467 | Regression_energy: 0.1971 | Regression_eta: 0.1637 | Regression_pt: 0.2058 | Regression_sin_phi: 0.2410 | LR: 9.80e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.2490s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7154 | max_loss: 0.1552 | max_backward: 1.4398 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.8979 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0854 | Memory: rss_MB: 5917.3 | uss_MB: 4226.4 | child_rss_MB: 5678.8 | allocated_MB: 346.1 | reserved_MB: 26384.0 | inactive_split_MB: 284.0 +rank4 [2026-10-28 14:31:09,414] INFO : Step 4900/50000 rank4 | Train Loss: 2.3682 | Train Loss Unweighted: 1.4242 | Train Loss Components: Auxiliary: 0.4616 | Cardinality: 0.2176 | Classification: 0.2570 | Classification_binary: 0.3149 | HitGrouping: 0.2900 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4191 | Regression_eta: 0.0047 | Regression_pt: 0.4239 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1445 | Classification_binary: 0.3340 | Regression_cos_phi: 0.2164 | Regression_energy: 0.1966 | Regression_eta: 0.1159 | Regression_pt: 0.2053 | Regression_sin_phi: 0.1799 | LR: 9.79e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.4642s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0516 | max_forward: 0.6608 | max_loss: 0.1430 | max_backward: 1.2657 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5916.2 | uss_MB: 4765.4 | child_rss_MB: 5677.6 | allocated_MB: 343.8 | reserved_MB: 26384.0 | inactive_split_MB: 201.9 +rank4 [2026-10-28 14:35:19,046] INFO : Step 5000/50000 rank4 | Train Loss: 2.4060 | Train Loss Unweighted: 1.4130 | Train Loss Components: Auxiliary: 0.4756 | Cardinality: 0.2573 | Classification: 0.2748 | Classification_binary: 0.3198 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4020 | Regression_eta: 0.0043 | Regression_pt: 0.4074 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1643 | Classification_binary: 0.3325 | Regression_cos_phi: 0.1656 | Regression_energy: 0.1671 | Regression_eta: 0.0952 | Regression_pt: 0.1754 | Regression_sin_phi: 0.1628 | LR: 9.78e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8092s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6153 | max_loss: 0.1441 | max_backward: 1.2119 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.9741 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5914.8 | uss_MB: 4767.1 | child_rss_MB: 5669.5 | allocated_MB: 342.5 | reserved_MB: 27388.0 | inactive_split_MB: 218.4 +rank4 [2026-10-28 14:35:19,962] INFO : Running validation on rank4 +rank4 [2026-10-28 14:35:19,963] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 14:35:19,963] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 14:36:10,799] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:36:11,318] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 14:36:36,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 14:36:37,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 14:36:38,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 14:36:39,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 14:36:49,593] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 14:36:50,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:36:50,780] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 14:37:06,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 14:37:07,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 14:37:08,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 14:37:09,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 14:37:19,331] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 14:37:20,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:37:20,582] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 14:37:36,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 14:37:36,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 14:37:37,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 14:37:38,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 14:37:48,189] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 14:42:02,659] INFO : Step 5100/50000 rank4 | Train Loss: 2.7059 | Train Loss Unweighted: 1.5631 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2507 | Classification: 0.3068 | Classification_binary: 0.4110 | HitGrouping: 0.2884 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4148 | Regression_eta: 0.0060 | Regression_pt: 0.4186 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1611 | Classification_binary: 0.3962 | Regression_cos_phi: 0.1971 | Regression_energy: 0.2206 | Regression_eta: 0.1476 | Regression_pt: 0.2293 | Regression_sin_phi: 0.2935 | LR: 9.77e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4428s | Phase Times: max_pre_step_barrier: 0.1071 | max_batch_to_device: 0.0070 | max_forward: 0.7192 | max_loss: 0.2509 | max_backward: 1.6066 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.8859 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7032.2 | uss_MB: 5882.9 | child_rss_MB: 5662.0 | allocated_MB: 343.2 | reserved_MB: 23542.0 | inactive_split_MB: 236.2 +rank4 [2026-10-28 14:45:56,261] INFO : Step 5200/50000 rank4 | Train Loss: 2.4117 | Train Loss Unweighted: 1.4393 | Train Loss Components: Auxiliary: 0.4667 | Cardinality: 0.2215 | Classification: 0.2426 | Classification_binary: 0.3524 | HitGrouping: 0.2783 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4149 | Regression_eta: 0.0050 | Regression_pt: 0.4196 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1308 | Classification_binary: 0.3163 | Regression_cos_phi: 0.1549 | Regression_energy: 0.2055 | Regression_eta: 0.1296 | Regression_pt: 0.2135 | Regression_sin_phi: 0.2230 | LR: 9.76e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.9019s | Phase Times: max_pre_step_barrier: 0.1151 | max_batch_to_device: 0.0079 | max_forward: 0.8909 | max_loss: 0.1450 | max_backward: 1.3654 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.5791 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 6020.7 | uss_MB: 4875.4 | child_rss_MB: 5666.4 | allocated_MB: 346.1 | reserved_MB: 23690.0 | inactive_split_MB: 197.9 +rank4 [2026-10-28 14:50:03,850] INFO : Step 5300/50000 rank4 | Train Loss: 2.2417 | Train Loss Unweighted: 1.3238 | Train Loss Components: Auxiliary: 0.4353 | Cardinality: 0.2226 | Classification: 0.2387 | Classification_binary: 0.3027 | HitGrouping: 0.2808 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3844 | Regression_eta: 0.0041 | Regression_pt: 0.3895 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1453 | Classification_binary: 0.3125 | Regression_cos_phi: 0.1332 | Regression_energy: 0.1733 | Regression_eta: 0.1075 | Regression_pt: 0.1798 | Regression_sin_phi: 0.1754 | LR: 9.75e-04 | DataLoad Time: 0.0235s | Model Forward Time: 4.1968s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0155 | max_forward: 1.3549 | max_loss: 0.4272 | max_backward: 1.7517 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 6024.3 | uss_MB: 4878.1 | child_rss_MB: 5664.8 | allocated_MB: 341.7 | reserved_MB: 25496.0 | inactive_split_MB: 195.4 +rank4 [2026-10-28 14:54:26,617] INFO : Step 5400/50000 rank4 | Train Loss: 2.5857 | Train Loss Unweighted: 1.4820 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2715 | Classification: 0.2898 | Classification_binary: 0.3613 | HitGrouping: 0.2829 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4076 | Regression_eta: 0.0053 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1495 | Classification_binary: 0.3470 | Regression_cos_phi: 0.1439 | Regression_energy: 0.1850 | Regression_eta: 0.1423 | Regression_pt: 0.1927 | Regression_sin_phi: 0.2211 | LR: 9.74e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.8621s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7472 | max_loss: 0.1664 | max_backward: 1.6216 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4956 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6028.8 | uss_MB: 4880.2 | child_rss_MB: 5668.3 | allocated_MB: 345.8 | reserved_MB: 25496.0 | inactive_split_MB: 226.8 +rank4 [2026-10-28 14:58:29,776] INFO : Step 5500/50000 rank4 | Train Loss: 2.4383 | Train Loss Unweighted: 1.4361 | Train Loss Components: Auxiliary: 0.4636 | Cardinality: 0.2531 | Classification: 0.2547 | Classification_binary: 0.3846 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3900 | Regression_eta: 0.0045 | Regression_pt: 0.3974 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1784 | Classification_binary: 0.4539 | Regression_cos_phi: 0.2157 | Regression_energy: 0.1816 | Regression_eta: 0.1574 | Regression_pt: 0.1911 | Regression_sin_phi: 0.2149 | LR: 9.73e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3517s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7547 | max_loss: 0.2915 | max_backward: 1.8143 | max_optimizer_step: 0.2819 | max_tensorboard_logging: 0.5958 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6029.1 | uss_MB: 4882.8 | child_rss_MB: 5667.9 | allocated_MB: 342.5 | reserved_MB: 26496.0 | inactive_split_MB: 471.5 +rank4 [2026-10-28 15:02:37,793] INFO : Step 5600/50000 rank4 | Train Loss: 2.2622 | Train Loss Unweighted: 1.3177 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.1990 | Classification: 0.2257 | Classification_binary: 0.3268 | HitGrouping: 0.2800 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3758 | Regression_eta: 0.0051 | Regression_pt: 0.3799 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0960 | Classification_binary: 0.3474 | Regression_cos_phi: 0.1477 | Regression_energy: 0.1658 | Regression_eta: 0.1100 | Regression_pt: 0.1741 | Regression_sin_phi: 0.2106 | LR: 9.72e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.1701s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0347 | max_forward: 1.1910 | max_loss: 0.2555 | max_backward: 1.5431 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 6024.4 | uss_MB: 4885.9 | child_rss_MB: 5669.8 | allocated_MB: 340.8 | reserved_MB: 26496.0 | inactive_split_MB: 191.2 +rank4 [2026-10-28 15:06:24,089] INFO : Step 5700/50000 rank4 | Train Loss: 2.4588 | Train Loss Unweighted: 1.4026 | Train Loss Components: Auxiliary: 0.4666 | Cardinality: 0.2348 | Classification: 0.2768 | Classification_binary: 0.3365 | HitGrouping: 0.2916 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3864 | Regression_eta: 0.0055 | Regression_pt: 0.3923 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1713 | Classification_binary: 0.3600 | Regression_cos_phi: 0.2094 | Regression_energy: 0.1901 | Regression_eta: 0.1652 | Regression_pt: 0.2009 | Regression_sin_phi: 0.2231 | LR: 9.71e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5067s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6547 | max_loss: 0.1874 | max_backward: 1.2748 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6027.1 | uss_MB: 4889.2 | child_rss_MB: 5674.0 | allocated_MB: 344.4 | reserved_MB: 26496.0 | inactive_split_MB: 224.5 +rank4 [2026-10-28 15:10:14,851] INFO : Step 5800/50000 rank4 | Train Loss: 2.4947 | Train Loss Unweighted: 1.4616 | Train Loss Components: Auxiliary: 0.4752 | Cardinality: 0.2715 | Classification: 0.2623 | Classification_binary: 0.3729 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4032 | Regression_eta: 0.0047 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1445 | Classification_binary: 0.3758 | Regression_cos_phi: 0.2017 | Regression_energy: 0.1868 | Regression_eta: 0.1050 | Regression_pt: 0.1999 | Regression_sin_phi: 0.1817 | LR: 9.70e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.7386s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.9768 | max_loss: 0.2512 | max_backward: 2.0573 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.6978 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6026.0 | uss_MB: 4048.8 | child_rss_MB: 5650.8 | allocated_MB: 343.4 | reserved_MB: 27494.0 | inactive_split_MB: 198.3 +rank4 [2026-10-28 15:14:12,312] INFO : Step 5900/50000 rank4 | Train Loss: 2.3800 | Train Loss Unweighted: 1.3838 | Train Loss Components: Auxiliary: 0.4549 | Cardinality: 0.2203 | Classification: 0.2438 | Classification_binary: 0.3435 | HitGrouping: 0.2757 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3900 | Regression_eta: 0.0050 | Regression_pt: 0.3965 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1285 | Classification_binary: 0.3253 | Regression_cos_phi: 0.1856 | Regression_energy: 0.1711 | Regression_eta: 0.1373 | Regression_pt: 0.1794 | Regression_sin_phi: 0.1564 | LR: 9.69e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.7150s | Phase Times: max_pre_step_barrier: 0.0158 | max_batch_to_device: 0.0069 | max_forward: 1.0355 | max_loss: 0.2311 | max_backward: 2.2749 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5131 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 6029.8 | uss_MB: 3868.6 | child_rss_MB: 5658.5 | allocated_MB: 342.1 | reserved_MB: 27494.0 | inactive_split_MB: 281.2 +rank4 [2026-10-28 15:18:13,895] INFO : Step 6000/50000 rank4 | Train Loss: 2.1372 | Train Loss Unweighted: 1.3015 | Train Loss Components: Auxiliary: 0.4179 | Cardinality: 0.1916 | Classification: 0.2091 | Classification_binary: 0.2848 | HitGrouping: 0.2746 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3966 | Regression_eta: 0.0041 | Regression_pt: 0.4029 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1253 | Classification_binary: 0.2956 | Regression_cos_phi: 0.1781 | Regression_energy: 0.1929 | Regression_eta: 0.1122 | Regression_pt: 0.2025 | Regression_sin_phi: 0.1852 | LR: 9.68e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6794s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6457 | max_loss: 0.2157 | max_backward: 1.4984 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.5590 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6033.8 | uss_MB: 4891.5 | child_rss_MB: 5658.2 | allocated_MB: 343.1 | reserved_MB: 27494.0 | inactive_split_MB: 260.7 +rank4 [2026-10-28 15:18:14,997] INFO : Running validation on rank4 +rank4 [2026-10-28 15:18:14,997] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 15:18:14,998] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 15:19:17,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:19:17,849] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 15:19:43,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 15:19:44,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 15:19:45,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 15:19:46,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 15:19:56,756] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 15:19:57,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:19:58,136] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 15:20:14,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 15:20:14,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 15:20:15,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 15:20:16,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 15:20:26,778] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 15:20:27,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:20:28,076] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 15:20:43,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 15:20:44,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 15:20:44,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 15:20:45,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 15:20:55,673] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 15:25:29,842] INFO : Step 6100/50000 rank4 | Train Loss: 2.4040 | Train Loss Unweighted: 1.3907 | Train Loss Components: Auxiliary: 0.4661 | Cardinality: 0.2610 | Classification: 0.2427 | Classification_binary: 0.3401 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3968 | Regression_eta: 0.0047 | Regression_pt: 0.4016 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1472 | Classification_binary: 0.3173 | Regression_cos_phi: 0.2096 | Regression_energy: 0.2002 | Regression_eta: 0.1273 | Regression_pt: 0.2073 | Regression_sin_phi: 0.1938 | LR: 9.67e-04 | DataLoad Time: 0.0209s | Model Forward Time: 6.3952s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0073 | max_forward: 1.0567 | max_loss: 0.2452 | max_backward: 2.4106 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.9836 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 7214.6 | uss_MB: 5292.3 | child_rss_MB: 5654.8 | allocated_MB: 344.4 | reserved_MB: 23648.0 | inactive_split_MB: 194.8 +rank4 [2026-10-28 15:29:38,612] INFO : Step 6200/50000 rank4 | Train Loss: 2.4942 | Train Loss Unweighted: 1.4287 | Train Loss Components: Auxiliary: 0.4744 | Cardinality: 0.2537 | Classification: 0.2580 | Classification_binary: 0.3817 | HitGrouping: 0.2748 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3867 | Regression_eta: 0.0050 | Regression_pt: 0.3920 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1633 | Classification_binary: 0.3828 | Regression_cos_phi: 0.2593 | Regression_energy: 0.1910 | Regression_eta: 0.1420 | Regression_pt: 0.1986 | Regression_sin_phi: 0.2356 | LR: 9.66e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.0656s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7838 | max_loss: 0.1532 | max_backward: 1.6314 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6093.8 | uss_MB: 4537.3 | child_rss_MB: 5656.7 | allocated_MB: 343.8 | reserved_MB: 23830.0 | inactive_split_MB: 215.7 +rank4 [2026-10-28 15:34:14,249] INFO : Step 6300/50000 rank4 | Train Loss: 2.2204 | Train Loss Unweighted: 1.2763 | Train Loss Components: Auxiliary: 0.4285 | Cardinality: 0.2506 | Classification: 0.2258 | Classification_binary: 0.2983 | HitGrouping: 0.2710 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3704 | Regression_eta: 0.0045 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1632 | Classification_binary: 0.2861 | Regression_cos_phi: 0.1573 | Regression_energy: 0.2051 | Regression_eta: 0.1152 | Regression_pt: 0.2077 | Regression_sin_phi: 0.1797 | LR: 9.65e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6715s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.5854 | max_loss: 0.1923 | max_backward: 1.0848 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.8586 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6096.2 | uss_MB: 4957.2 | child_rss_MB: 5651.4 | allocated_MB: 339.7 | reserved_MB: 24826.0 | inactive_split_MB: 251.6 +rank4 [2026-10-28 15:38:39,608] INFO : Step 6400/50000 rank4 | Train Loss: 2.2882 | Train Loss Unweighted: 1.3014 | Train Loss Components: Auxiliary: 0.4348 | Cardinality: 0.2271 | Classification: 0.2431 | Classification_binary: 0.3058 | HitGrouping: 0.2702 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3679 | Regression_eta: 0.0046 | Regression_pt: 0.3751 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1071 | Classification_binary: 0.2512 | Regression_cos_phi: 0.1231 | Regression_energy: 0.1755 | Regression_eta: 0.0739 | Regression_pt: 0.1847 | Regression_sin_phi: 0.1281 | LR: 9.64e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.8574s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7489 | max_loss: 0.3608 | max_backward: 2.5327 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6095.6 | uss_MB: 4594.4 | child_rss_MB: 5649.8 | allocated_MB: 345.8 | reserved_MB: 24826.0 | inactive_split_MB: 253.0 +rank4 [2026-10-28 15:42:45,601] INFO : Step 6500/50000 rank4 | Train Loss: 2.4626 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.3022 | Classification: 0.2393 | Classification_binary: 0.3805 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3783 | Regression_eta: 0.0049 | Regression_pt: 0.3831 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1183 | Classification_binary: 0.3587 | Regression_cos_phi: 0.1864 | Regression_energy: 0.1952 | Regression_eta: 0.1451 | Regression_pt: 0.2038 | Regression_sin_phi: 0.1565 | LR: 9.63e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1895s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7600 | max_loss: 0.1555 | max_backward: 1.3910 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6094.6 | uss_MB: 4960.0 | child_rss_MB: 5656.1 | allocated_MB: 344.4 | reserved_MB: 24826.0 | inactive_split_MB: 269.0 +rank4 [2026-10-28 15:46:47,733] INFO : Step 6600/50000 rank4 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3617 | Train Loss Components: Auxiliary: 0.4493 | Cardinality: 0.2405 | Classification: 0.2289 | Classification_binary: 0.3388 | HitGrouping: 0.2650 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3914 | Regression_eta: 0.0045 | Regression_pt: 0.3935 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1474 | Classification_binary: 0.3779 | Regression_cos_phi: 0.2480 | Regression_energy: 0.1818 | Regression_eta: 0.1291 | Regression_pt: 0.1886 | Regression_sin_phi: 0.1829 | LR: 9.62e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7770s | Phase Times: max_pre_step_barrier: 0.1106 | max_batch_to_device: 0.0061 | max_forward: 0.7587 | max_loss: 0.2869 | max_backward: 2.2641 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1321 | Memory: rss_MB: 6092.2 | uss_MB: 4956.2 | child_rss_MB: 5651.6 | allocated_MB: 343.8 | reserved_MB: 25846.0 | inactive_split_MB: 267.9 +rank4 [2026-10-28 15:51:24,436] INFO : Step 6700/50000 rank4 | Train Loss: 2.1225 | Train Loss Unweighted: 1.2385 | Train Loss Components: Auxiliary: 0.4092 | Cardinality: 0.2100 | Classification: 0.2079 | Classification_binary: 0.2916 | HitGrouping: 0.2751 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3623 | Regression_eta: 0.0041 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1085 | Classification_binary: 0.3205 | Regression_cos_phi: 0.1815 | Regression_energy: 0.1787 | Regression_eta: 0.1144 | Regression_pt: 0.1862 | Regression_sin_phi: 0.2127 | LR: 9.61e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9923s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.6452 | max_loss: 0.1453 | max_backward: 1.2984 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.0269 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6098.1 | uss_MB: 4965.1 | child_rss_MB: 5657.8 | allocated_MB: 345.0 | reserved_MB: 25846.0 | inactive_split_MB: 258.8 +rank4 [2026-10-28 15:55:32,973] INFO : Step 6800/50000 rank4 | Train Loss: 2.4601 | Train Loss Unweighted: 1.3772 | Train Loss Components: Auxiliary: 0.4589 | Cardinality: 0.2427 | Classification: 0.2631 | Classification_binary: 0.3490 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3747 | Regression_eta: 0.0054 | Regression_pt: 0.3793 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1650 | Classification_binary: 0.3720 | Regression_cos_phi: 0.1763 | Regression_energy: 0.1895 | Regression_eta: 0.1516 | Regression_pt: 0.2016 | Regression_sin_phi: 0.1850 | LR: 9.60e-04 | DataLoad Time: 0.0249s | Model Forward Time: 5.6148s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0662 | max_forward: 1.6558 | max_loss: 0.3855 | max_backward: 2.1309 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 1.0709 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6092.6 | uss_MB: 4961.3 | child_rss_MB: 5647.9 | allocated_MB: 345.8 | reserved_MB: 26622.0 | inactive_split_MB: 194.2 +rank4 [2026-10-28 15:59:35,211] INFO : Step 6900/50000 rank4 | Train Loss: 2.3793 | Train Loss Unweighted: 1.3337 | Train Loss Components: Auxiliary: 0.4495 | Cardinality: 0.2757 | Classification: 0.2300 | Classification_binary: 0.3711 | HitGrouping: 0.2647 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3605 | Regression_eta: 0.0045 | Regression_pt: 0.3628 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1356 | Classification_binary: 0.3609 | Regression_cos_phi: 0.2106 | Regression_energy: 0.1711 | Regression_eta: 0.1133 | Regression_pt: 0.1762 | Regression_sin_phi: 0.1577 | LR: 9.58e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2976s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1120 | max_forward: 0.8190 | max_loss: 0.1496 | max_backward: 1.4356 | max_optimizer_step: 0.2847 | max_tensorboard_logging: 0.8440 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6096.9 | uss_MB: 4962.9 | child_rss_MB: 5656.8 | allocated_MB: 345.4 | reserved_MB: 26622.0 | inactive_split_MB: 300.5 +rank4 [2026-10-28 16:03:56,953] INFO : Step 7000/50000 rank4 | Train Loss: 2.2410 | Train Loss Unweighted: 1.2707 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2746 | Classification: 0.1970 | Classification_binary: 0.3002 | HitGrouping: 0.2655 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3795 | Regression_eta: 0.0045 | Regression_pt: 0.3852 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1226 | Classification_binary: 0.2957 | Regression_cos_phi: 0.1982 | Regression_energy: 0.1923 | Regression_eta: 0.1463 | Regression_pt: 0.1998 | Regression_sin_phi: 0.1975 | LR: 9.57e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.3864s | Phase Times: max_pre_step_barrier: 0.0229 | max_batch_to_device: 0.0154 | max_forward: 1.0393 | max_loss: 0.2251 | max_backward: 2.0451 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.9425 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6097.4 | uss_MB: 4965.6 | child_rss_MB: 5656.9 | allocated_MB: 345.5 | reserved_MB: 26622.0 | inactive_split_MB: 296.5 +rank4 [2026-10-28 16:03:58,353] INFO : Running validation on rank4 +rank4 [2026-10-28 16:03:58,353] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 16:03:58,353] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 16:04:51,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:04:52,382] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 16:05:18,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 16:05:19,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 16:05:20,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 16:05:21,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 16:05:31,420] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 16:05:32,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:05:32,551] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 16:05:48,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 16:05:49,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 16:05:50,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 16:05:51,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 16:06:01,640] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 16:06:02,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:06:02,706] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 16:06:18,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 16:06:19,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 16:06:19,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 16:06:20,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 16:06:30,657] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 16:10:49,043] INFO : Step 7100/50000 rank4 | Train Loss: 2.1086 | Train Loss Unweighted: 1.2525 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.1821 | Classification: 0.2186 | Classification_binary: 0.2903 | HitGrouping: 0.2613 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3655 | Regression_eta: 0.0041 | Regression_pt: 0.3696 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1556 | Classification_binary: 0.3421 | Regression_cos_phi: 0.2275 | Regression_energy: 0.1694 | Regression_eta: 0.1368 | Regression_pt: 0.1785 | Regression_sin_phi: 0.1967 | LR: 9.56e-04 | DataLoad Time: 0.0183s | Model Forward Time: 3.9400s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6376 | max_loss: 0.1424 | max_backward: 1.3436 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8454 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 7191.1 | uss_MB: 6057.4 | child_rss_MB: 5686.3 | allocated_MB: 345.6 | reserved_MB: 22618.0 | inactive_split_MB: 206.0 +rank4 [2026-10-28 16:15:05,704] INFO : Step 7200/50000 rank4 | Train Loss: 2.4589 | Train Loss Unweighted: 1.3873 | Train Loss Components: Auxiliary: 0.4658 | Cardinality: 0.2721 | Classification: 0.2648 | Classification_binary: 0.3806 | HitGrouping: 0.2633 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3642 | Regression_eta: 0.0051 | Regression_pt: 0.3675 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1553 | Classification_binary: 0.3579 | Regression_cos_phi: 0.1669 | Regression_energy: 0.1963 | Regression_eta: 0.1362 | Regression_pt: 0.2032 | Regression_sin_phi: 0.1898 | LR: 9.55e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.3049s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 1.0191 | max_loss: 0.1674 | max_backward: 1.7813 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6183.3 | uss_MB: 5050.5 | child_rss_MB: 5662.0 | allocated_MB: 343.5 | reserved_MB: 22618.0 | inactive_split_MB: 315.3 +rank4 [2026-10-28 16:19:17,626] INFO : Step 7300/50000 rank4 | Train Loss: 2.1617 | Train Loss Unweighted: 1.2404 | Train Loss Components: Auxiliary: 0.4141 | Cardinality: 0.2258 | Classification: 0.1979 | Classification_binary: 0.3107 | HitGrouping: 0.2671 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3613 | Regression_eta: 0.0041 | Regression_pt: 0.3621 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1109 | Classification_binary: 0.2703 | Regression_cos_phi: 0.1547 | Regression_energy: 0.1719 | Regression_eta: 0.0984 | Regression_pt: 0.1749 | Regression_sin_phi: 0.1496 | LR: 9.53e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.7527s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0084 | max_forward: 0.7227 | max_loss: 0.1491 | max_backward: 1.4232 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.5902 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6180.9 | uss_MB: 5051.2 | child_rss_MB: 5696.9 | allocated_MB: 345.0 | reserved_MB: 23608.0 | inactive_split_MB: 295.0 +rank4 [2026-10-28 16:23:25,991] INFO : Step 7400/50000 rank4 | Train Loss: 2.0741 | Train Loss Unweighted: 1.2014 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2287 | Classification: 0.2035 | Classification_binary: 0.2781 | HitGrouping: 0.2646 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3549 | Regression_eta: 0.0040 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1196 | Classification_binary: 0.2817 | Regression_cos_phi: 0.1996 | Regression_energy: 0.1711 | Regression_eta: 0.1105 | Regression_pt: 0.1829 | Regression_sin_phi: 0.1609 | LR: 9.52e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6564s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6129 | max_loss: 0.2127 | max_backward: 1.2944 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6450 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6180.9 | uss_MB: 5049.5 | child_rss_MB: 5695.6 | allocated_MB: 343.8 | reserved_MB: 23608.0 | inactive_split_MB: 229.5 +rank4 [2026-10-28 16:27:18,561] INFO : Step 7500/50000 rank4 | Train Loss: 2.4640 | Train Loss Unweighted: 1.3686 | Train Loss Components: Auxiliary: 0.4637 | Cardinality: 0.2640 | Classification: 0.2570 | Classification_binary: 0.3594 | HitGrouping: 0.2687 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3684 | Regression_eta: 0.0054 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1206 | Classification_binary: 0.3182 | Regression_cos_phi: 0.1718 | Regression_energy: 0.1783 | Regression_eta: 0.1181 | Regression_pt: 0.1845 | Regression_sin_phi: 0.1455 | LR: 9.51e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1756s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0887 | max_forward: 0.7717 | max_loss: 0.1508 | max_backward: 1.3590 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.9634 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6179.4 | uss_MB: 5053.3 | child_rss_MB: 5684.8 | allocated_MB: 348.6 | reserved_MB: 23608.0 | inactive_split_MB: 283.7 +rank4 [2026-10-28 16:31:22,305] INFO : Step 7600/50000 rank4 | Train Loss: 2.3745 | Train Loss Unweighted: 1.3447 | Train Loss Components: Auxiliary: 0.4509 | Cardinality: 0.2616 | Classification: 0.2519 | Classification_binary: 0.3743 | HitGrouping: 0.2627 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3532 | Regression_eta: 0.0050 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1463 | Classification_binary: 0.4211 | Regression_cos_phi: 0.2193 | Regression_energy: 0.1791 | Regression_eta: 0.1437 | Regression_pt: 0.1877 | Regression_sin_phi: 0.1583 | LR: 9.50e-04 | DataLoad Time: 0.0155s | Model Forward Time: 16.0041s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.7345 | max_loss: 0.1617 | max_backward: 1.4897 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 12.6867 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6179.5 | uss_MB: 5051.2 | child_rss_MB: 5695.8 | allocated_MB: 345.3 | reserved_MB: 23608.0 | inactive_split_MB: 284.8 +rank4 [2026-10-28 16:35:23,036] INFO : Step 7700/50000 rank4 | Train Loss: 2.0023 | Train Loss Unweighted: 1.1475 | Train Loss Components: Auxiliary: 0.3866 | Cardinality: 0.2005 | Classification: 0.1968 | Classification_binary: 0.2784 | HitGrouping: 0.2562 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3315 | Regression_eta: 0.0040 | Regression_pt: 0.3329 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2437 | Regression_cos_phi: 0.1280 | Regression_energy: 0.1580 | Regression_eta: 0.0916 | Regression_pt: 0.1626 | Regression_sin_phi: 0.1556 | LR: 9.48e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.0199s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7891 | max_loss: 0.1927 | max_backward: 1.4291 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6182.9 | uss_MB: 5055.0 | child_rss_MB: 5702.0 | allocated_MB: 343.9 | reserved_MB: 23608.0 | inactive_split_MB: 279.7 +rank4 [2026-10-28 16:39:23,042] INFO : Step 7800/50000 rank4 | Train Loss: 2.0473 | Train Loss Unweighted: 1.2037 | Train Loss Components: Auxiliary: 0.4013 | Cardinality: 0.1891 | Classification: 0.2102 | Classification_binary: 0.2760 | HitGrouping: 0.2604 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3526 | Regression_eta: 0.0038 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1017 | Classification_binary: 0.2445 | Regression_cos_phi: 0.1071 | Regression_energy: 0.1617 | Regression_eta: 0.0893 | Regression_pt: 0.1655 | Regression_sin_phi: 0.1711 | LR: 9.47e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4333s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.7326 | max_loss: 0.2464 | max_backward: 1.8124 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9395 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6182.4 | uss_MB: 5059.4 | child_rss_MB: 5675.6 | allocated_MB: 343.5 | reserved_MB: 24612.0 | inactive_split_MB: 224.8 +rank4 [2026-10-28 16:43:21,636] INFO : Step 7900/50000 rank4 | Train Loss: 2.3872 | Train Loss Unweighted: 1.3279 | Train Loss Components: Auxiliary: 0.4525 | Cardinality: 0.2722 | Classification: 0.2401 | Classification_binary: 0.3580 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3573 | Regression_eta: 0.0051 | Regression_pt: 0.3627 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1007 | Classification_binary: 0.3273 | Regression_cos_phi: 0.1718 | Regression_energy: 0.1674 | Regression_eta: 0.1065 | Regression_pt: 0.1742 | Regression_sin_phi: 0.1585 | LR: 9.46e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7832s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7032 | max_loss: 0.2052 | max_backward: 1.5183 | max_optimizer_step: 0.1922 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6185.2 | uss_MB: 5060.7 | child_rss_MB: 5695.7 | allocated_MB: 347.1 | reserved_MB: 24612.0 | inactive_split_MB: 279.0 +rank4 [2026-10-28 16:47:21,752] INFO : Step 8000/50000 rank4 | Train Loss: 2.1303 | Train Loss Unweighted: 1.2019 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2451 | Classification: 0.2031 | Classification_binary: 0.3087 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3382 | Regression_eta: 0.0041 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1269 | Classification_binary: 0.3077 | Regression_cos_phi: 0.2396 | Regression_energy: 0.1561 | Regression_eta: 0.1094 | Regression_pt: 0.1624 | Regression_sin_phi: 0.1826 | LR: 9.44e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.7976s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0554 | max_forward: 0.7608 | max_loss: 0.1617 | max_backward: 1.3810 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 6186.5 | uss_MB: 4730.8 | child_rss_MB: 5692.5 | allocated_MB: 344.2 | reserved_MB: 24612.0 | inactive_split_MB: 224.1 +rank4 [2026-10-28 16:47:22,744] INFO : Running validation on rank4 +rank4 [2026-10-28 16:47:22,744] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 16:47:22,745] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 16:48:13,809] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:48:14,393] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 16:48:40,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 16:48:41,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 16:48:42,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 16:48:43,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 16:48:53,272] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 16:48:53,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:48:54,468] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 16:49:10,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 16:49:11,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 16:49:12,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 16:49:13,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 16:49:23,411] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 16:49:24,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:49:24,963] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 16:49:40,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 16:49:41,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 16:49:41,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 16:49:42,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 16:49:52,535] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 16:54:09,126] INFO : Step 8100/50000 rank4 | Train Loss: 2.0320 | Train Loss Unweighted: 1.1744 | Train Loss Components: Auxiliary: 0.3893 | Cardinality: 0.1986 | Classification: 0.1938 | Classification_binary: 0.2691 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3510 | Regression_eta: 0.0039 | Regression_pt: 0.3525 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1008 | Classification_binary: 0.2621 | Regression_cos_phi: 0.1309 | Regression_energy: 0.1769 | Regression_eta: 0.0894 | Regression_pt: 0.1819 | Regression_sin_phi: 0.1248 | LR: 9.43e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0450s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6251 | max_loss: 0.2465 | max_backward: 1.3564 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8507 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7244.9 | uss_MB: 6118.4 | child_rss_MB: 5660.8 | allocated_MB: 342.9 | reserved_MB: 23510.0 | inactive_split_MB: 258.1 +rank4 [2026-10-28 16:58:00,708] INFO : Step 8200/50000 rank4 | Train Loss: 2.1917 | Train Loss Unweighted: 1.2329 | Train Loss Components: Auxiliary: 0.4200 | Cardinality: 0.2405 | Classification: 0.2359 | Classification_binary: 0.3328 | HitGrouping: 0.2582 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3251 | Regression_eta: 0.0043 | Regression_pt: 0.3305 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1218 | Classification_binary: 0.2846 | Regression_cos_phi: 0.1551 | Regression_energy: 0.1639 | Regression_eta: 0.1164 | Regression_pt: 0.1685 | Regression_sin_phi: 0.1283 | LR: 9.42e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2770s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0064 | max_forward: 0.7679 | max_loss: 0.2484 | max_backward: 1.6620 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.8049 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1289 | Memory: rss_MB: 6232.6 | uss_MB: 5106.1 | child_rss_MB: 5673.2 | allocated_MB: 343.2 | reserved_MB: 23510.0 | inactive_split_MB: 292.4 +rank4 [2026-10-28 17:02:03,650] INFO : Step 8300/50000 rank4 | Train Loss: 2.1924 | Train Loss Unweighted: 1.2399 | Train Loss Components: Auxiliary: 0.4122 | Cardinality: 0.2039 | Classification: 0.2083 | Classification_binary: 0.3661 | HitGrouping: 0.2599 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3243 | Regression_eta: 0.0043 | Regression_pt: 0.3323 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1277 | Classification_binary: 0.3845 | Regression_cos_phi: 0.1924 | Regression_energy: 0.1523 | Regression_eta: 0.1316 | Regression_pt: 0.1620 | Regression_sin_phi: 0.2124 | LR: 9.40e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.8407s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0061 | max_forward: 0.9407 | max_loss: 0.2296 | max_backward: 1.9622 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6228.1 | uss_MB: 5055.3 | child_rss_MB: 5669.2 | allocated_MB: 344.6 | reserved_MB: 24580.0 | inactive_split_MB: 267.5 +rank4 [2026-10-28 17:06:07,749] INFO : Step 8400/50000 rank4 | Train Loss: 1.9542 | Train Loss Unweighted: 1.1189 | Train Loss Components: Auxiliary: 0.3756 | Cardinality: 0.1979 | Classification: 0.1926 | Classification_binary: 0.2783 | HitGrouping: 0.2585 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3198 | Regression_eta: 0.0035 | Regression_pt: 0.3209 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1341 | Classification_binary: 0.2816 | Regression_cos_phi: 0.1481 | Regression_energy: 0.1609 | Regression_eta: 0.1016 | Regression_pt: 0.1663 | Regression_sin_phi: 0.1533 | LR: 9.39e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2393s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6939 | max_loss: 0.1474 | max_backward: 1.5357 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 0.9892 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 6216.9 | uss_MB: 5103.3 | child_rss_MB: 5675.6 | allocated_MB: 341.1 | reserved_MB: 24580.0 | inactive_split_MB: 178.9 +rank4 [2026-10-28 17:10:17,599] INFO : Step 8500/50000 rank4 | Train Loss: 2.0642 | Train Loss Unweighted: 1.1724 | Train Loss Components: Auxiliary: 0.3964 | Cardinality: 0.2023 | Classification: 0.2152 | Classification_binary: 0.2897 | HitGrouping: 0.2607 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3289 | Regression_eta: 0.0041 | Regression_pt: 0.3302 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1171 | Classification_binary: 0.3004 | Regression_cos_phi: 0.1490 | Regression_energy: 0.1973 | Regression_eta: 0.1069 | Regression_pt: 0.2003 | Regression_sin_phi: 0.1615 | LR: 9.37e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.7384s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0069 | max_forward: 0.6424 | max_loss: 0.1972 | max_backward: 1.2711 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.7327 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6218.9 | uss_MB: 5108.3 | child_rss_MB: 5667.6 | allocated_MB: 344.4 | reserved_MB: 25586.0 | inactive_split_MB: 317.0 +rank4 [2026-10-28 17:14:24,635] INFO : Step 8600/50000 rank4 | Train Loss: 2.2820 | Train Loss Unweighted: 1.2960 | Train Loss Components: Auxiliary: 0.4378 | Cardinality: 0.2546 | Classification: 0.2294 | Classification_binary: 0.3478 | HitGrouping: 0.2584 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3521 | Regression_eta: 0.0046 | Regression_pt: 0.3576 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1324 | Classification_binary: 0.3329 | Regression_cos_phi: 0.1790 | Regression_energy: 0.1600 | Regression_eta: 0.1019 | Regression_pt: 0.1685 | Regression_sin_phi: 0.1316 | LR: 9.36e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.9616s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0069 | max_forward: 0.7396 | max_loss: 0.1527 | max_backward: 1.6088 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5007 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6218.9 | uss_MB: 5112.5 | child_rss_MB: 5664.7 | allocated_MB: 344.8 | reserved_MB: 25586.0 | inactive_split_MB: 283.2 +rank4 [2026-10-28 17:18:21,031] INFO : Step 8700/50000 rank4 | Train Loss: 2.0581 | Train Loss Unweighted: 1.1705 | Train Loss Components: Auxiliary: 0.4038 | Cardinality: 0.2463 | Classification: 0.2066 | Classification_binary: 0.3017 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3267 | Regression_eta: 0.0035 | Regression_pt: 0.3282 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1275 | Classification_binary: 0.2993 | Regression_cos_phi: 0.1337 | Regression_energy: 0.1526 | Regression_eta: 0.0897 | Regression_pt: 0.1584 | Regression_sin_phi: 0.1654 | LR: 9.34e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.7151s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6873 | max_loss: 0.2444 | max_backward: 1.2985 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6214.6 | uss_MB: 5113.2 | child_rss_MB: 5650.1 | allocated_MB: 343.7 | reserved_MB: 25586.0 | inactive_split_MB: 260.4 +rank4 [2026-10-28 17:22:21,714] INFO : Step 8800/50000 rank4 | Train Loss: 1.8415 | Train Loss Unweighted: 1.0591 | Train Loss Components: Auxiliary: 0.3597 | Cardinality: 0.1811 | Classification: 0.1734 | Classification_binary: 0.2490 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3144 | Regression_eta: 0.0034 | Regression_pt: 0.3153 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1239 | Classification_binary: 0.2989 | Regression_cos_phi: 0.1830 | Regression_energy: 0.1666 | Regression_eta: 0.1123 | Regression_pt: 0.1738 | Regression_sin_phi: 0.1460 | LR: 9.33e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5732s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6737 | max_loss: 0.2385 | max_backward: 1.3073 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.5819 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6215.6 | uss_MB: 5114.6 | child_rss_MB: 5649.0 | allocated_MB: 344.1 | reserved_MB: 25586.0 | inactive_split_MB: 205.9 +rank4 [2026-10-28 17:26:43,764] INFO : Step 8900/50000 rank4 | Train Loss: 2.1706 | Train Loss Unweighted: 1.1768 | Train Loss Components: Auxiliary: 0.4167 | Cardinality: 0.3000 | Classification: 0.2064 | Classification_binary: 0.3037 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3284 | Regression_eta: 0.0039 | Regression_pt: 0.3304 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1625 | Classification_binary: 0.3093 | Regression_cos_phi: 0.2034 | Regression_energy: 0.1512 | Regression_eta: 0.1213 | Regression_pt: 0.1599 | Regression_sin_phi: 0.1512 | LR: 9.31e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.8177s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0073 | max_forward: 0.9973 | max_loss: 0.4040 | max_backward: 3.4126 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6194.6 | uss_MB: 5119.9 | child_rss_MB: 5654.4 | allocated_MB: 347.1 | reserved_MB: 25586.0 | inactive_split_MB: 275.1 +rank4 [2026-10-28 17:30:46,804] INFO : Step 9000/50000 rank4 | Train Loss: 2.1357 | Train Loss Unweighted: 1.1883 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.2351 | Classification: 0.2003 | Classification_binary: 0.3467 | HitGrouping: 0.2532 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3143 | Regression_eta: 0.0041 | Regression_pt: 0.3186 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0963 | Classification_binary: 0.3277 | Regression_cos_phi: 0.1595 | Regression_energy: 0.1629 | Regression_eta: 0.1097 | Regression_pt: 0.1726 | Regression_sin_phi: 0.1356 | LR: 9.30e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7218s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.7873 | max_loss: 0.2178 | max_backward: 1.5407 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.3125 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 6192.6 | uss_MB: 5121.2 | child_rss_MB: 5660.4 | allocated_MB: 344.6 | reserved_MB: 25586.0 | inactive_split_MB: 289.9 +rank4 [2026-10-28 17:30:47,949] INFO : Running validation on rank4 +rank4 [2026-10-28 17:30:47,949] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 17:30:47,950] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 17:31:40,963] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:31:41,476] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 17:32:08,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 17:32:09,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 17:32:10,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 17:32:11,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 17:32:21,660] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 17:32:22,657] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:32:23,201] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 17:32:39,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 17:32:40,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 17:32:41,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 17:32:42,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 17:32:52,248] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 17:32:52,552] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:32:53,102] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 17:33:08,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 17:33:09,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 17:33:10,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 17:33:10,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 17:33:20,706] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 17:37:41,206] INFO : Step 9100/50000 rank4 | Train Loss: 1.8448 | Train Loss Unweighted: 1.0695 | Train Loss Components: Auxiliary: 0.3538 | Cardinality: 0.1722 | Classification: 0.1878 | Classification_binary: 0.2661 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3012 | Regression_eta: 0.0033 | Regression_pt: 0.3075 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1003 | Classification_binary: 0.2752 | Regression_cos_phi: 0.1464 | Regression_energy: 0.1310 | Regression_eta: 0.0800 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0997 | LR: 9.28e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4385s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0153 | max_forward: 1.2776 | max_loss: 0.3881 | max_backward: 1.3802 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9203 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7188.5 | uss_MB: 6128.3 | child_rss_MB: 5636.7 | allocated_MB: 345.3 | reserved_MB: 22582.0 | inactive_split_MB: 275.2 +rank4 [2026-10-28 17:41:50,229] INFO : Step 9200/50000 rank4 | Train Loss: 1.9883 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3829 | Cardinality: 0.2086 | Classification: 0.1812 | Classification_binary: 0.2802 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3207 | Regression_eta: 0.0035 | Regression_pt: 0.3257 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1091 | Classification_binary: 0.2797 | Regression_cos_phi: 0.1816 | Regression_energy: 0.1502 | Regression_eta: 0.1092 | Regression_pt: 0.1578 | Regression_sin_phi: 0.1688 | LR: 9.27e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5154s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0073 | max_forward: 0.6839 | max_loss: 0.2414 | max_backward: 1.6336 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 1.0636 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6214.5 | uss_MB: 5161.4 | child_rss_MB: 5636.3 | allocated_MB: 345.6 | reserved_MB: 23224.0 | inactive_split_MB: 190.1 +rank4 [2026-10-28 17:45:52,429] INFO : Step 9300/50000 rank4 | Train Loss: 2.0749 | Train Loss Unweighted: 1.1591 | Train Loss Components: Auxiliary: 0.4049 | Cardinality: 0.2573 | Classification: 0.2065 | Classification_binary: 0.3180 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3111 | Regression_eta: 0.0036 | Regression_pt: 0.3160 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1174 | Classification_binary: 0.3091 | Regression_cos_phi: 0.1302 | Regression_energy: 0.1298 | Regression_eta: 0.0944 | Regression_pt: 0.1348 | Regression_sin_phi: 0.1913 | LR: 9.25e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2171s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8329 | max_loss: 0.1499 | max_backward: 1.4059 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.8766 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6214.5 | uss_MB: 5164.3 | child_rss_MB: 5654.0 | allocated_MB: 344.8 | reserved_MB: 24586.0 | inactive_split_MB: 287.2 +rank4 [2026-10-28 17:50:16,942] INFO : Step 9400/50000 rank4 | Train Loss: 1.8467 | Train Loss Unweighted: 1.0417 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1520 | Classification: 0.1837 | Classification_binary: 0.2977 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2742 | Regression_eta: 0.0040 | Regression_pt: 0.2784 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1133 | Classification_binary: 0.2971 | Regression_cos_phi: 0.1502 | Regression_energy: 0.1318 | Regression_eta: 0.1644 | Regression_pt: 0.1381 | Regression_sin_phi: 0.1622 | LR: 9.24e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.5888s | Phase Times: max_pre_step_barrier: 0.0180 | max_batch_to_device: 0.0062 | max_forward: 1.5076 | max_loss: 0.3748 | max_backward: 2.6247 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.5247 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6206.4 | uss_MB: 4649.5 | child_rss_MB: 5650.0 | allocated_MB: 343.9 | reserved_MB: 24586.0 | inactive_split_MB: 267.3 +rank4 [2026-10-28 17:54:18,166] INFO : Step 9500/50000 rank4 | Train Loss: 1.8199 | Train Loss Unweighted: 0.9995 | Train Loss Components: Auxiliary: 0.3464 | Cardinality: 0.1875 | Classification: 0.1651 | Classification_binary: 0.2558 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2851 | Regression_eta: 0.0035 | Regression_pt: 0.2864 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0899 | Classification_binary: 0.2343 | Regression_cos_phi: 0.1738 | Regression_energy: 0.1267 | Regression_eta: 0.0852 | Regression_pt: 0.1319 | Regression_sin_phi: 0.1297 | LR: 9.22e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.8034s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6532 | max_loss: 0.2385 | max_backward: 1.0043 | max_optimizer_step: 0.3001 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6205.2 | uss_MB: 5165.5 | child_rss_MB: 5654.5 | allocated_MB: 341.2 | reserved_MB: 24586.0 | inactive_split_MB: 180.8 +rank4 [2026-10-28 17:58:23,634] INFO : Step 9600/50000 rank4 | Train Loss: 2.0900 | Train Loss Unweighted: 1.1491 | Train Loss Components: Auxiliary: 0.4030 | Cardinality: 0.2464 | Classification: 0.2089 | Classification_binary: 0.3189 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3034 | Regression_eta: 0.0042 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1378 | Classification_binary: 0.3304 | Regression_cos_phi: 0.1331 | Regression_energy: 0.1615 | Regression_eta: 0.1033 | Regression_pt: 0.1702 | Regression_sin_phi: 0.1751 | LR: 9.21e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8845s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1083 | max_forward: 0.7381 | max_loss: 0.1457 | max_backward: 1.4890 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4970 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6206.2 | uss_MB: 5167.2 | child_rss_MB: 5637.2 | allocated_MB: 345.2 | reserved_MB: 25594.0 | inactive_split_MB: 190.6 +rank4 [2026-10-28 18:02:35,091] INFO : Step 9700/50000 rank4 | Train Loss: 2.1397 | Train Loss Unweighted: 1.1700 | Train Loss Components: Auxiliary: 0.4082 | Cardinality: 0.2377 | Classification: 0.1942 | Classification_binary: 0.3460 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3095 | Regression_eta: 0.0043 | Regression_pt: 0.3118 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1314 | Classification_binary: 0.3423 | Regression_cos_phi: 0.1816 | Regression_energy: 0.1402 | Regression_eta: 0.1241 | Regression_pt: 0.1443 | Regression_sin_phi: 0.2061 | LR: 9.19e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.1044s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7744 | max_loss: 0.1497 | max_backward: 1.6846 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4710 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 6209.3 | uss_MB: 5170.1 | child_rss_MB: 5650.1 | allocated_MB: 345.9 | reserved_MB: 25594.0 | inactive_split_MB: 288.1 +rank4 [2026-10-28 18:06:42,229] INFO : Step 9800/50000 rank4 | Train Loss: 1.8589 | Train Loss Unweighted: 1.0245 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2213 | Classification: 0.1590 | Classification_binary: 0.2751 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2908 | Regression_eta: 0.0031 | Regression_pt: 0.2931 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1027 | Classification_binary: 0.2713 | Regression_cos_phi: 0.1249 | Regression_energy: 0.1480 | Regression_eta: 0.0810 | Regression_pt: 0.1501 | Regression_sin_phi: 0.1225 | LR: 9.17e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.1234s | Phase Times: max_pre_step_barrier: 0.1099 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.1713 | max_backward: 1.7200 | max_optimizer_step: 0.1898 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6188.3 | uss_MB: 5169.7 | child_rss_MB: 5642.5 | allocated_MB: 342.5 | reserved_MB: 26610.0 | inactive_split_MB: 264.5 +rank4 [2026-10-28 18:10:58,312] INFO : Step 9900/50000 rank4 | Train Loss: 1.8366 | Train Loss Unweighted: 1.0077 | Train Loss Components: Auxiliary: 0.3557 | Cardinality: 0.2091 | Classification: 0.1810 | Classification_binary: 0.2609 | HitGrouping: 0.2462 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2775 | Regression_eta: 0.0035 | Regression_pt: 0.2814 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0899 | Classification_binary: 0.2082 | Regression_cos_phi: 0.1351 | Regression_energy: 0.1208 | Regression_eta: 0.0663 | Regression_pt: 0.1285 | Regression_sin_phi: 0.1021 | LR: 9.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5785s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6346 | max_loss: 0.1692 | max_backward: 1.3542 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6188.4 | uss_MB: 5170.9 | child_rss_MB: 5645.5 | allocated_MB: 343.7 | reserved_MB: 26610.0 | inactive_split_MB: 205.7 +rank4 [2026-10-28 18:15:06,636] INFO : Step 10000/50000 rank4 | Train Loss: 2.0706 | Train Loss Unweighted: 1.1295 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2648 | Classification: 0.1819 | Classification_binary: 0.3225 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3077 | Regression_eta: 0.0037 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1077 | Classification_binary: 0.3595 | Regression_cos_phi: 0.1870 | Regression_energy: 0.1594 | Regression_eta: 0.1232 | Regression_pt: 0.1671 | Regression_sin_phi: 0.1978 | LR: 9.14e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.7118s | Phase Times: max_pre_step_barrier: 0.0136 | max_batch_to_device: 0.0708 | max_forward: 1.8842 | max_loss: 0.5484 | max_backward: 2.6620 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.5314 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6190.4 | uss_MB: 5174.2 | child_rss_MB: 5642.5 | allocated_MB: 345.2 | reserved_MB: 26610.0 | inactive_split_MB: 288.8 +rank4 [2026-10-28 18:15:07,627] INFO : Running validation on rank4 +rank4 [2026-10-28 18:15:07,627] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 18:15:07,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 18:15:55,923] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:15:56,539] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 18:16:22,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 18:16:23,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 18:16:24,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 18:16:25,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 18:16:35,447] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 18:16:36,561] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:16:37,045] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 18:16:53,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 18:16:53,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 18:16:55,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 18:16:57,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 18:17:07,018] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 18:17:09,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:17:10,083] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 18:17:25,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 18:17:26,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 18:17:27,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 18:17:27,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 18:17:37,719] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 18:22:01,597] INFO : Step 10100/50000 rank4 | Train Loss: 1.8115 | Train Loss Unweighted: 1.0221 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.1999 | Classification: 0.1589 | Classification_binary: 0.2815 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2868 | Regression_eta: 0.0032 | Regression_pt: 0.2886 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0861 | Classification_binary: 0.2736 | Regression_cos_phi: 0.0883 | Regression_energy: 0.1489 | Regression_eta: 0.0742 | Regression_pt: 0.1546 | Regression_sin_phi: 0.0930 | LR: 9.12e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9757s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7090 | max_loss: 0.1507 | max_backward: 1.2179 | max_optimizer_step: 0.3146 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 7166.8 | uss_MB: 6150.2 | child_rss_MB: 5645.8 | allocated_MB: 343.1 | reserved_MB: 21794.0 | inactive_split_MB: 243.8 +rank4 [2026-10-28 18:26:06,876] INFO : Step 10200/50000 rank4 | Train Loss: 1.6749 | Train Loss Unweighted: 0.9660 | Train Loss Components: Auxiliary: 0.3310 | Cardinality: 0.1773 | Classification: 0.1544 | Classification_binary: 0.2377 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2827 | Regression_eta: 0.0027 | Regression_pt: 0.2857 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2445 | Regression_cos_phi: 0.0948 | Regression_energy: 0.1337 | Regression_eta: 0.0615 | Regression_pt: 0.1410 | Regression_sin_phi: 0.0978 | LR: 9.11e-04 | DataLoad Time: 0.0115s | Model Forward Time: 4.0156s | Phase Times: max_pre_step_barrier: 0.0352 | max_batch_to_device: 0.0241 | max_forward: 0.6625 | max_loss: 0.1442 | max_backward: 1.3835 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7009 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1256 | Memory: rss_MB: 7164.8 | uss_MB: 6150.1 | child_rss_MB: 5645.0 | allocated_MB: 345.9 | reserved_MB: 21794.0 | inactive_split_MB: 292.4 +rank4 [2026-10-28 18:30:14,208] INFO : Step 10300/50000 rank4 | Train Loss: 1.9113 | Train Loss Unweighted: 1.0360 | Train Loss Components: Auxiliary: 0.3660 | Cardinality: 0.2108 | Classification: 0.1830 | Classification_binary: 0.2931 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0033 | Regression_pt: 0.2786 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1088 | Classification_binary: 0.2829 | Regression_cos_phi: 0.1556 | Regression_energy: 0.1394 | Regression_eta: 0.0861 | Regression_pt: 0.1454 | Regression_sin_phi: 0.1439 | LR: 9.09e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.9107s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7205 | max_loss: 0.1496 | max_backward: 1.6083 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 6287.2 | uss_MB: 5273.5 | child_rss_MB: 5650.9 | allocated_MB: 343.8 | reserved_MB: 22764.0 | inactive_split_MB: 257.9 +rank4 [2026-10-28 18:34:19,880] INFO : Step 10400/50000 rank4 | Train Loss: 1.9303 | Train Loss Unweighted: 1.0569 | Train Loss Components: Auxiliary: 0.3739 | Cardinality: 0.2318 | Classification: 0.1704 | Classification_binary: 0.3342 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2718 | Regression_eta: 0.0033 | Regression_pt: 0.2738 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0819 | Classification_binary: 0.3168 | Regression_cos_phi: 0.1427 | Regression_energy: 0.1310 | Regression_eta: 0.0781 | Regression_pt: 0.1360 | Regression_sin_phi: 0.1286 | LR: 9.07e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.7382s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7465 | max_loss: 0.1499 | max_backward: 1.4212 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.4712 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6287.2 | uss_MB: 5272.0 | child_rss_MB: 5660.4 | allocated_MB: 345.6 | reserved_MB: 22792.0 | inactive_split_MB: 259.8 +rank4 [2026-10-28 18:38:24,242] INFO : Step 10500/50000 rank4 | Train Loss: 1.6601 | Train Loss Unweighted: 0.9268 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1892 | Classification: 0.1410 | Classification_binary: 0.2342 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2693 | Regression_eta: 0.0029 | Regression_pt: 0.2766 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0936 | Classification_binary: 0.2407 | Regression_cos_phi: 0.1345 | Regression_energy: 0.1385 | Regression_eta: 0.0954 | Regression_pt: 0.1484 | Regression_sin_phi: 0.1497 | LR: 9.06e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.5618s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 0.6364 | max_loss: 0.2260 | max_backward: 1.1629 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6452 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6284.6 | uss_MB: 5269.3 | child_rss_MB: 5635.7 | allocated_MB: 340.4 | reserved_MB: 22792.0 | inactive_split_MB: 230.3 +rank4 [2026-10-28 18:42:22,547] INFO : Step 10600/50000 rank4 | Train Loss: 1.9388 | Train Loss Unweighted: 1.0719 | Train Loss Components: Auxiliary: 0.3737 | Cardinality: 0.2008 | Classification: 0.1863 | Classification_binary: 0.2709 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3047 | Regression_eta: 0.0042 | Regression_pt: 0.3019 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1006 | Classification_binary: 0.2440 | Regression_cos_phi: 0.1826 | Regression_energy: 0.1356 | Regression_eta: 0.1265 | Regression_pt: 0.1386 | Regression_sin_phi: 0.1406 | LR: 9.04e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9682s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6880 | max_loss: 0.2458 | max_backward: 1.4354 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7481 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 6285.6 | uss_MB: 5273.7 | child_rss_MB: 5653.3 | allocated_MB: 344.6 | reserved_MB: 22798.0 | inactive_split_MB: 286.4 +rank4 [2026-10-28 18:46:28,872] INFO : Step 10700/50000 rank4 | Train Loss: 1.8955 | Train Loss Unweighted: 1.0230 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.2139 | Classification: 0.1752 | Classification_binary: 0.3186 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2598 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0995 | Classification_binary: 0.2910 | Regression_cos_phi: 0.1289 | Regression_energy: 0.1108 | Regression_eta: 0.0943 | Regression_pt: 0.1150 | Regression_sin_phi: 0.1122 | LR: 9.02e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.1519s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0072 | max_forward: 1.0363 | max_loss: 0.1757 | max_backward: 2.3255 | max_optimizer_step: 0.1834 | max_tensorboard_logging: 0.7232 | max_tensorboard_flush: 0.0051 | max_post_step_barrier: 0.1576 | Memory: rss_MB: 6287.7 | uss_MB: 5274.9 | child_rss_MB: 5649.7 | allocated_MB: 342.1 | reserved_MB: 22798.0 | inactive_split_MB: 282.9 +rank4 [2026-10-28 18:50:28,323] INFO : Step 10800/50000 rank4 | Train Loss: 1.7679 | Train Loss Unweighted: 0.9563 | Train Loss Components: Auxiliary: 0.3446 | Cardinality: 0.2074 | Classification: 0.1518 | Classification_binary: 0.2766 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2590 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0879 | Classification_binary: 0.2303 | Regression_cos_phi: 0.1254 | Regression_energy: 0.1255 | Regression_eta: 0.0917 | Regression_pt: 0.1337 | Regression_sin_phi: 0.1079 | LR: 9.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7807s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0995 | max_forward: 0.7884 | max_loss: 0.2225 | max_backward: 2.0144 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.5230 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6288.7 | uss_MB: 5271.1 | child_rss_MB: 5654.7 | allocated_MB: 343.6 | reserved_MB: 23784.0 | inactive_split_MB: 286.4 +rank4 [2026-10-28 18:54:31,738] INFO : Step 10900/50000 rank4 | Train Loss: 1.6025 | Train Loss Unweighted: 0.8934 | Train Loss Components: Auxiliary: 0.3208 | Cardinality: 0.1729 | Classification: 0.1446 | Classification_binary: 0.2284 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2574 | Regression_eta: 0.0027 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0959 | Classification_binary: 0.2372 | Regression_cos_phi: 0.1417 | Regression_energy: 0.1146 | Regression_eta: 0.0847 | Regression_pt: 0.1185 | Regression_sin_phi: 0.1230 | LR: 8.99e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.4467s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.5765 | max_loss: 0.1490 | max_backward: 1.1882 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6288.0 | uss_MB: 5275.5 | child_rss_MB: 5639.7 | allocated_MB: 342.1 | reserved_MB: 23796.0 | inactive_split_MB: 272.7 +rank4 [2026-10-28 18:58:45,418] INFO : Step 11000/50000 rank4 | Train Loss: 1.9315 | Train Loss Unweighted: 1.0466 | Train Loss Components: Auxiliary: 0.3721 | Cardinality: 0.2078 | Classification: 0.1806 | Classification_binary: 0.3087 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0037 | Regression_pt: 0.2758 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0995 | Classification_binary: 0.2745 | Regression_cos_phi: 0.1513 | Regression_energy: 0.1318 | Regression_eta: 0.0981 | Regression_pt: 0.1381 | Regression_sin_phi: 0.1593 | LR: 8.97e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.1708s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7424 | max_loss: 0.1704 | max_backward: 1.5866 | max_optimizer_step: 0.2906 | max_tensorboard_logging: 0.6287 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6277.0 | uss_MB: 5281.8 | child_rss_MB: 5663.8 | allocated_MB: 347.8 | reserved_MB: 23796.0 | inactive_split_MB: 262.8 +rank4 [2026-10-28 18:58:46,360] INFO : Running validation on rank4 +rank4 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 18:59:40,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:59:40,901] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 19:00:07,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 19:00:08,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 19:00:10,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 19:00:11,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 19:00:21,173] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-28 19:00:21,678] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:00:22,257] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 19:00:38,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 19:00:39,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 19:00:40,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 19:00:41,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 19:00:51,196] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 19:00:52,166] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:00:52,896] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 19:01:08,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 19:01:09,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 19:01:09,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 19:01:10,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 19:01:20,580] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 19:05:41,392] INFO : Step 11100/50000 rank4 | Train Loss: 1.9118 | Train Loss Unweighted: 1.0064 | Train Loss Components: Auxiliary: 0.3723 | Cardinality: 0.2793 | Classification: 0.1563 | Classification_binary: 0.3082 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2660 | Regression_eta: 0.0034 | Regression_pt: 0.2693 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1006 | Classification_binary: 0.3497 | Regression_cos_phi: 0.1347 | Regression_energy: 0.1322 | Regression_eta: 0.0937 | Regression_pt: 0.1372 | Regression_sin_phi: 0.1663 | LR: 8.95e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0410s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.9217 | max_loss: 0.2304 | max_backward: 1.5570 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 7169.9 | uss_MB: 6179.3 | child_rss_MB: 5664.2 | allocated_MB: 343.2 | reserved_MB: 22146.0 | inactive_split_MB: 281.7 +rank4 [2026-10-28 19:09:39,138] INFO : Step 11200/50000 rank4 | Train Loss: 1.7366 | Train Loss Unweighted: 0.9397 | Train Loss Components: Auxiliary: 0.3529 | Cardinality: 0.2466 | Classification: 0.1384 | Classification_binary: 0.2548 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2689 | Regression_eta: 0.0027 | Regression_pt: 0.2722 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0903 | Classification_binary: 0.2907 | Regression_cos_phi: 0.1198 | Regression_energy: 0.1457 | Regression_eta: 0.0775 | Regression_pt: 0.1483 | Regression_sin_phi: 0.1446 | LR: 8.93e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7181s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7793 | max_loss: 0.1825 | max_backward: 1.3743 | max_optimizer_step: 0.2128 | max_tensorboard_logging: 0.4676 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7167.9 | uss_MB: 6179.2 | child_rss_MB: 5641.7 | allocated_MB: 344.6 | reserved_MB: 22972.0 | inactive_split_MB: 273.4 +rank4 [2026-10-28 19:13:41,796] INFO : Step 11300/50000 rank4 | Train Loss: 1.6876 | Train Loss Unweighted: 0.9423 | Train Loss Components: Auxiliary: 0.3334 | Cardinality: 0.1862 | Classification: 0.1581 | Classification_binary: 0.2713 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2524 | Regression_eta: 0.0027 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0902 | Classification_binary: 0.2191 | Regression_cos_phi: 0.1290 | Regression_energy: 0.1190 | Regression_eta: 0.0871 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0920 | LR: 8.91e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6813s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0223 | max_forward: 0.9315 | max_loss: 0.2308 | max_backward: 1.9859 | max_optimizer_step: 0.1928 | max_tensorboard_logging: 0.5681 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6388.7 | uss_MB: 5273.7 | child_rss_MB: 5658.4 | allocated_MB: 345.5 | reserved_MB: 24368.0 | inactive_split_MB: 269.6 +rank4 [2026-10-28 19:17:45,333] INFO : Step 11400/50000 rank4 | Train Loss: 1.8563 | Train Loss Unweighted: 0.9980 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.2376 | Classification: 0.1627 | Classification_binary: 0.3164 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2551 | Regression_eta: 0.0029 | Regression_pt: 0.2577 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.2884 | Regression_cos_phi: 0.1233 | Regression_energy: 0.1122 | Regression_eta: 0.0902 | Regression_pt: 0.1176 | Regression_sin_phi: 0.1096 | LR: 8.89e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.0881s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.7684 | max_loss: 0.1511 | max_backward: 1.6966 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5034 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 6391.7 | uss_MB: 5402.6 | child_rss_MB: 5664.5 | allocated_MB: 343.2 | reserved_MB: 25642.0 | inactive_split_MB: 251.6 +rank4 [2026-10-28 19:21:55,044] INFO : Step 11500/50000 rank4 | Train Loss: 1.6323 | Train Loss Unweighted: 0.8976 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1755 | Classification: 0.1375 | Classification_binary: 0.2717 | HitGrouping: 0.2346 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2408 | Regression_eta: 0.0028 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2859 | Regression_cos_phi: 0.1095 | Regression_energy: 0.1024 | Regression_eta: 0.0810 | Regression_pt: 0.1070 | Regression_sin_phi: 0.1142 | LR: 8.88e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.6955s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7232 | max_loss: 0.1557 | max_backward: 1.4770 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6390.5 | uss_MB: 5401.3 | child_rss_MB: 5644.3 | allocated_MB: 345.3 | reserved_MB: 25642.0 | inactive_split_MB: 274.7 +rank4 [2026-10-28 19:25:54,594] INFO : Step 11600/50000 rank4 | Train Loss: 1.6133 | Train Loss Unweighted: 0.9062 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1718 | Classification: 0.1502 | Classification_binary: 0.2356 | HitGrouping: 0.2377 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2569 | Regression_eta: 0.0025 | Regression_pt: 0.2581 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.2520 | Regression_cos_phi: 0.1108 | Regression_energy: 0.1103 | Regression_eta: 0.0741 | Regression_pt: 0.1145 | Regression_sin_phi: 0.1082 | LR: 8.86e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6488s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6646 | max_loss: 0.1710 | max_backward: 1.3208 | max_optimizer_step: 0.1963 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6391.5 | uss_MB: 5404.2 | child_rss_MB: 5665.3 | allocated_MB: 344.1 | reserved_MB: 26630.0 | inactive_split_MB: 209.9 +rank4 [2026-10-28 19:29:50,732] INFO : Step 11700/50000 rank4 | Train Loss: 1.7739 | Train Loss Unweighted: 0.9706 | Train Loss Components: Auxiliary: 0.3531 | Cardinality: 0.2418 | Classification: 0.1518 | Classification_binary: 0.2821 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2638 | Regression_eta: 0.0028 | Regression_pt: 0.2671 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1034 | Classification_binary: 0.2519 | Regression_cos_phi: 0.1239 | Regression_energy: 0.1276 | Regression_eta: 0.1073 | Regression_pt: 0.1305 | Regression_sin_phi: 0.1184 | LR: 8.84e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.4615s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8789 | max_loss: 0.2057 | max_backward: 2.3846 | max_optimizer_step: 0.1875 | max_tensorboard_logging: 1.0403 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6389.5 | uss_MB: 4921.9 | child_rss_MB: 5640.9 | allocated_MB: 343.8 | reserved_MB: 27624.0 | inactive_split_MB: 186.6 +rank4 [2026-10-28 19:33:54,004] INFO : Step 11800/50000 rank4 | Train Loss: 1.8223 | Train Loss Unweighted: 0.9745 | Train Loss Components: Auxiliary: 0.3609 | Cardinality: 0.2329 | Classification: 0.1589 | Classification_binary: 0.3100 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2473 | Regression_eta: 0.0028 | Regression_pt: 0.2523 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1144 | Classification_binary: 0.3286 | Regression_cos_phi: 0.1451 | Regression_energy: 0.1104 | Regression_eta: 0.1033 | Regression_pt: 0.1174 | Regression_sin_phi: 0.1664 | LR: 8.82e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.3872s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0070 | max_forward: 1.2302 | max_loss: 0.2767 | max_backward: 2.3745 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6390.5 | uss_MB: 5402.7 | child_rss_MB: 5661.5 | allocated_MB: 346.0 | reserved_MB: 27624.0 | inactive_split_MB: 280.5 +rank4 [2026-10-28 19:37:48,966] INFO : Step 11900/50000 rank4 | Train Loss: 1.5797 | Train Loss Unweighted: 0.8638 | Train Loss Components: Auxiliary: 0.3225 | Cardinality: 0.1935 | Classification: 0.1358 | Classification_binary: 0.2351 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2427 | Regression_eta: 0.0025 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.2058 | Regression_cos_phi: 0.0658 | Regression_energy: 0.1047 | Regression_eta: 0.0764 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0556 | LR: 8.80e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.1379s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0787 | max_forward: 0.7011 | max_loss: 0.1484 | max_backward: 1.2002 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.9463 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 6390.3 | uss_MB: 5402.5 | child_rss_MB: 5642.2 | allocated_MB: 340.1 | reserved_MB: 28620.0 | inactive_split_MB: 264.1 +rank4 [2026-10-28 19:41:43,168] INFO : Step 12000/50000 rank4 | Train Loss: 1.6036 | Train Loss Unweighted: 0.8706 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1972 | Classification: 0.1481 | Classification_binary: 0.2459 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2330 | Regression_eta: 0.0025 | Regression_pt: 0.2385 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.2225 | Regression_cos_phi: 0.0822 | Regression_energy: 0.1158 | Regression_eta: 0.0391 | Regression_pt: 0.1216 | Regression_sin_phi: 0.0813 | LR: 8.78e-04 | DataLoad Time: 0.0148s | Model Forward Time: 3.6992s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0072 | max_forward: 0.6194 | max_loss: 0.1546 | max_backward: 1.4628 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5143 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6383.3 | uss_MB: 5409.4 | child_rss_MB: 5646.2 | allocated_MB: 342.9 | reserved_MB: 28620.0 | inactive_split_MB: 215.8 +rank4 [2026-10-28 19:41:44,123] INFO : Running validation on rank4 +rank4 [2026-10-28 19:41:44,123] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 19:41:44,123] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 19:42:35,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:42:35,719] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 19:43:02,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 19:43:04,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 19:43:05,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 19:43:06,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 19:43:16,369] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-28 19:43:17,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:43:17,650] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 19:43:33,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 19:43:34,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 19:43:35,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 19:43:36,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 19:43:46,461] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 19:43:48,085] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:43:48,651] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 19:44:03,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 19:44:04,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 19:44:05,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 19:44:05,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 19:44:15,991] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 19:48:34,083] INFO : Step 12100/50000 rank4 | Train Loss: 1.8149 | Train Loss Unweighted: 0.9793 | Train Loss Components: Auxiliary: 0.3632 | Cardinality: 0.2324 | Classification: 0.1506 | Classification_binary: 0.3149 | HitGrouping: 0.2337 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2518 | Regression_eta: 0.0029 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1036 | Classification_binary: 0.3368 | Regression_cos_phi: 0.1669 | Regression_energy: 0.1648 | Regression_eta: 0.0875 | Regression_pt: 0.1738 | Regression_sin_phi: 0.1387 | LR: 8.76e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.5287s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8305 | max_loss: 0.1678 | max_backward: 1.6498 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8610 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7131.3 | uss_MB: 6150.6 | child_rss_MB: 5646.8 | allocated_MB: 345.8 | reserved_MB: 23512.0 | inactive_split_MB: 299.6 +rank4 [2026-10-28 19:52:54,661] INFO : Step 12200/50000 rank4 | Train Loss: 1.6619 | Train Loss Unweighted: 0.9043 | Train Loss Components: Auxiliary: 0.3290 | Cardinality: 0.1906 | Classification: 0.1445 | Classification_binary: 0.2674 | HitGrouping: 0.2303 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2435 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0817 | Classification_binary: 0.2754 | Regression_cos_phi: 0.1220 | Regression_energy: 0.1190 | Regression_eta: 0.0729 | Regression_pt: 0.1249 | Regression_sin_phi: 0.1348 | LR: 8.74e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8238s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.7862 | max_loss: 0.1438 | max_backward: 1.5991 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.8843 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7133.3 | uss_MB: 5953.1 | child_rss_MB: 5653.1 | allocated_MB: 341.2 | reserved_MB: 23512.0 | inactive_split_MB: 206.8 +rank4 [2026-10-28 19:56:47,338] INFO : Step 12300/50000 rank4 | Train Loss: 1.4699 | Train Loss Unweighted: 0.8181 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1474 | Classification: 0.1404 | Classification_binary: 0.2280 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2222 | Regression_eta: 0.0028 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2241 | Regression_cos_phi: 0.0993 | Regression_energy: 0.0984 | Regression_eta: 0.0666 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0771 | LR: 8.72e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7238s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6177 | max_loss: 0.2232 | max_backward: 1.1179 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 0.7974 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6497.9 | uss_MB: 5527.4 | child_rss_MB: 5654.1 | allocated_MB: 343.2 | reserved_MB: 24286.0 | inactive_split_MB: 290.2 +rank4 [2026-10-28 20:00:36,191] INFO : Step 12400/50000 rank4 | Train Loss: 1.6135 | Train Loss Unweighted: 0.8782 | Train Loss Components: Auxiliary: 0.3265 | Cardinality: 0.2056 | Classification: 0.1546 | Classification_binary: 0.2540 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2308 | Regression_eta: 0.0025 | Regression_pt: 0.2337 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0799 | Classification_binary: 0.2487 | Regression_cos_phi: 0.1331 | Regression_energy: 0.1194 | Regression_eta: 0.0603 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0750 | LR: 8.70e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.9944s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6928 | max_loss: 0.1486 | max_backward: 1.3860 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6497.9 | uss_MB: 5528.1 | child_rss_MB: 5671.7 | allocated_MB: 344.8 | reserved_MB: 24510.0 | inactive_split_MB: 295.5 +rank4 [2026-10-28 20:04:30,060] INFO : Step 12500/50000 rank4 | Train Loss: 1.6527 | Train Loss Unweighted: 0.8797 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1890 | Classification: 0.1391 | Classification_binary: 0.2779 | HitGrouping: 0.2315 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2273 | Regression_eta: 0.0030 | Regression_pt: 0.2295 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.2863 | Regression_cos_phi: 0.1564 | Regression_energy: 0.1101 | Regression_eta: 0.1278 | Regression_pt: 0.1124 | Regression_sin_phi: 0.1546 | LR: 8.68e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.1672s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0535 | max_forward: 0.7747 | max_loss: 0.1534 | max_backward: 1.3339 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6494.1 | uss_MB: 5528.8 | child_rss_MB: 5649.7 | allocated_MB: 347.2 | reserved_MB: 24510.0 | inactive_split_MB: 456.8 +rank4 [2026-10-28 20:08:30,520] INFO : Step 12600/50000 rank4 | Train Loss: 1.4969 | Train Loss Unweighted: 0.8185 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1595 | Classification: 0.1210 | Classification_binary: 0.2357 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2279 | Regression_eta: 0.0024 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.2204 | Regression_cos_phi: 0.1088 | Regression_energy: 0.1035 | Regression_eta: 0.0638 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0932 | LR: 8.66e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7603s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0062 | max_forward: 0.6571 | max_loss: 0.1429 | max_backward: 1.4364 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6482.1 | uss_MB: 5531.3 | child_rss_MB: 5638.2 | allocated_MB: 342.9 | reserved_MB: 25526.0 | inactive_split_MB: 234.6 +rank4 [2026-10-28 20:12:34,583] INFO : Step 12700/50000 rank4 | Train Loss: 1.6234 | Train Loss Unweighted: 0.8661 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.2004 | Classification: 0.1459 | Classification_binary: 0.2387 | HitGrouping: 0.2403 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2360 | Regression_eta: 0.0027 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0949 | Classification_binary: 0.2621 | Regression_cos_phi: 0.1055 | Regression_energy: 0.0964 | Regression_eta: 0.0807 | Regression_pt: 0.1015 | Regression_sin_phi: 0.1295 | LR: 8.64e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.4808s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6221 | max_loss: 0.1846 | max_backward: 1.1554 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6469.4 | uss_MB: 5531.3 | child_rss_MB: 5648.7 | allocated_MB: 345.9 | reserved_MB: 26286.0 | inactive_split_MB: 241.7 +rank4 [2026-10-28 20:16:22,943] INFO : Step 12800/50000 rank4 | Train Loss: 1.7877 | Train Loss Unweighted: 0.9398 | Train Loss Components: Auxiliary: 0.3651 | Cardinality: 0.2327 | Classification: 0.1495 | Classification_binary: 0.3094 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2357 | Regression_eta: 0.0027 | Regression_pt: 0.2394 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1143 | Classification_binary: 0.3365 | Regression_cos_phi: 0.1321 | Regression_energy: 0.1164 | Regression_eta: 0.0813 | Regression_pt: 0.1217 | Regression_sin_phi: 0.1646 | LR: 8.62e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.2583s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.8468 | max_loss: 0.1534 | max_backward: 1.7420 | max_optimizer_step: 0.3013 | max_tensorboard_logging: 0.5725 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6458.4 | uss_MB: 5535.9 | child_rss_MB: 5651.5 | allocated_MB: 345.6 | reserved_MB: 26286.0 | inactive_split_MB: 284.4 +rank4 [2026-10-28 20:20:13,787] INFO : Step 12900/50000 rank4 | Train Loss: 1.5881 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.3191 | Cardinality: 0.2029 | Classification: 0.1304 | Classification_binary: 0.2564 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2291 | Regression_eta: 0.0023 | Regression_pt: 0.2312 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0883 | Classification_binary: 0.2828 | Regression_cos_phi: 0.1187 | Regression_energy: 0.1017 | Regression_eta: 0.0750 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0912 | LR: 8.60e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9629s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7180 | max_loss: 0.2380 | max_backward: 1.4488 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6932 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6444.6 | uss_MB: 5531.5 | child_rss_MB: 5646.1 | allocated_MB: 341.8 | reserved_MB: 26286.0 | inactive_split_MB: 252.2 +rank4 [2026-10-28 20:24:06,550] INFO : Step 13000/50000 rank4 | Train Loss: 1.5217 | Train Loss Unweighted: 0.8547 | Train Loss Components: Auxiliary: 0.3141 | Cardinality: 0.1693 | Classification: 0.1382 | Classification_binary: 0.2135 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2470 | Regression_eta: 0.0024 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0944 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0963 | Regression_energy: 0.1104 | Regression_eta: 0.0759 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0866 | LR: 8.58e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8311s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7101 | max_loss: 0.1763 | max_backward: 1.1845 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 0.8282 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6429.6 | uss_MB: 5536.9 | child_rss_MB: 5653.9 | allocated_MB: 339.8 | reserved_MB: 26286.0 | inactive_split_MB: 213.4 +rank4 [2026-10-28 20:24:07,450] INFO : Running validation on rank4 +rank4 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 20:24:58,457] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:24:59,079] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 20:25:24,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 20:25:25,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 20:25:26,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 20:25:27,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 20:25:37,484] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 20:25:38,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:25:38,650] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 20:25:55,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 20:25:55,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 20:25:56,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 20:25:57,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 20:26:07,768] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 20:26:08,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:26:09,008] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 20:26:24,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 20:26:25,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 20:26:26,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 20:26:26,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 20:26:36,732] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 20:30:47,340] INFO : Step 13100/50000 rank4 | Train Loss: 1.6168 | Train Loss Unweighted: 0.8649 | Train Loss Components: Auxiliary: 0.3213 | Cardinality: 0.1869 | Classification: 0.1467 | Classification_binary: 0.2696 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2210 | Regression_eta: 0.0027 | Regression_pt: 0.2221 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1192 | Classification_binary: 0.3014 | Regression_cos_phi: 0.1135 | Regression_energy: 0.0981 | Regression_eta: 0.0712 | Regression_pt: 0.0990 | Regression_sin_phi: 0.1249 | LR: 8.56e-04 | DataLoad Time: 0.0122s | Model Forward Time: 6.3785s | Phase Times: max_pre_step_barrier: 0.0337 | max_batch_to_device: 0.0136 | max_forward: 1.2842 | max_loss: 0.2583 | max_backward: 3.1281 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.9677 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7302.9 | uss_MB: 6406.9 | child_rss_MB: 5646.9 | allocated_MB: 341.8 | reserved_MB: 22940.0 | inactive_split_MB: 270.2 +rank4 [2026-10-28 20:34:56,293] INFO : Step 13200/50000 rank4 | Train Loss: 1.6996 | Train Loss Unweighted: 0.8855 | Train Loss Components: Auxiliary: 0.3386 | Cardinality: 0.2474 | Classification: 0.1410 | Classification_binary: 0.2769 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2299 | Regression_eta: 0.0027 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0852 | Classification_binary: 0.2854 | Regression_cos_phi: 0.1189 | Regression_energy: 0.1106 | Regression_eta: 0.0702 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0967 | LR: 8.54e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.1537s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0072 | max_forward: 0.7543 | max_loss: 0.1532 | max_backward: 1.3317 | max_optimizer_step: 0.3180 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7300.0 | uss_MB: 6407.2 | child_rss_MB: 5649.5 | allocated_MB: 343.4 | reserved_MB: 23942.0 | inactive_split_MB: 249.6 +rank4 [2026-10-28 20:38:53,370] INFO : Step 13300/50000 rank4 | Train Loss: 1.4876 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2970 | Cardinality: 0.2018 | Classification: 0.1100 | Classification_binary: 0.2192 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2205 | Regression_eta: 0.0026 | Regression_pt: 0.2230 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0954 | Regression_eta: 0.0603 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0905 | LR: 8.52e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.6364s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0259 | max_forward: 0.9208 | max_loss: 0.2245 | max_backward: 1.9523 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6473.0 | uss_MB: 4937.1 | child_rss_MB: 5535.0 | allocated_MB: 341.7 | reserved_MB: 24474.0 | inactive_split_MB: 265.4 +rank4 [2026-10-28 20:42:52,943] INFO : Step 13400/50000 rank4 | Train Loss: 1.4253 | Train Loss Unweighted: 0.7944 | Train Loss Components: Auxiliary: 0.2889 | Cardinality: 0.1585 | Classification: 0.1406 | Classification_binary: 0.2158 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2158 | Regression_eta: 0.0022 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0710 | Regression_energy: 0.1033 | Regression_eta: 0.0606 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0842 | LR: 8.50e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.7379s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6533 | max_loss: 0.2500 | max_backward: 1.4458 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.5454 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6474.0 | uss_MB: 5610.2 | child_rss_MB: 5538.3 | allocated_MB: 341.6 | reserved_MB: 25334.0 | inactive_split_MB: 266.6 +rank4 [2026-10-28 20:47:05,652] INFO : Step 13500/50000 rank4 | Train Loss: 1.6021 | Train Loss Unweighted: 0.8749 | Train Loss Components: Auxiliary: 0.3241 | Cardinality: 0.1832 | Classification: 0.1473 | Classification_binary: 0.2801 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2200 | Regression_eta: 0.0024 | Regression_pt: 0.2225 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.2512 | Regression_cos_phi: 0.1075 | Regression_energy: 0.1102 | Regression_eta: 0.0735 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0947 | LR: 8.48e-04 | DataLoad Time: 0.0147s | Model Forward Time: 6.1065s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.7859 | max_loss: 0.3303 | max_backward: 2.5111 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4974 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6412.7 | uss_MB: 5395.8 | child_rss_MB: 5343.5 | allocated_MB: 344.3 | reserved_MB: 25574.0 | inactive_split_MB: 284.1 +rank4 [2026-10-28 20:51:04,210] INFO : Step 13600/50000 rank4 | Train Loss: 1.5296 | Train Loss Unweighted: 0.8114 | Train Loss Components: Auxiliary: 0.3050 | Cardinality: 0.1872 | Classification: 0.1291 | Classification_binary: 0.2450 | HitGrouping: 0.2218 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2158 | Regression_eta: 0.0025 | Regression_pt: 0.2165 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0946 | Classification_binary: 0.2427 | Regression_cos_phi: 0.1159 | Regression_energy: 0.1119 | Regression_eta: 0.0903 | Regression_pt: 0.1149 | Regression_sin_phi: 0.1264 | LR: 8.45e-04 | DataLoad Time: 0.0222s | Model Forward Time: 3.6632s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.7508 | max_loss: 0.1507 | max_backward: 1.2891 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6404.7 | uss_MB: 5608.6 | child_rss_MB: 5283.3 | allocated_MB: 345.0 | reserved_MB: 26608.0 | inactive_split_MB: 257.3 +rank4 [2026-10-28 20:55:12,337] INFO : Step 13700/50000 rank4 | Train Loss: 1.4515 | Train Loss Unweighted: 0.7895 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1575 | Classification: 0.1261 | Classification_binary: 0.2121 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2219 | Regression_eta: 0.0023 | Regression_pt: 0.2245 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1815 | Regression_cos_phi: 0.1130 | Regression_energy: 0.1020 | Regression_eta: 0.0877 | Regression_pt: 0.1042 | Regression_sin_phi: 0.1056 | LR: 8.43e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.8524s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.6804 | max_loss: 0.2128 | max_backward: 1.1331 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6398.1 | uss_MB: 5609.6 | child_rss_MB: 5245.6 | allocated_MB: 342.5 | reserved_MB: 26608.0 | inactive_split_MB: 241.7 +rank4 [2026-10-28 20:59:19,101] INFO : Step 13800/50000 rank4 | Train Loss: 1.5060 | Train Loss Unweighted: 0.8320 | Train Loss Components: Auxiliary: 0.3084 | Cardinality: 0.1762 | Classification: 0.1368 | Classification_binary: 0.2567 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2156 | Regression_eta: 0.0023 | Regression_pt: 0.2184 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0880 | Classification_binary: 0.2359 | Regression_cos_phi: 0.0781 | Regression_energy: 0.1204 | Regression_eta: 0.0604 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0738 | LR: 8.41e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4119s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7967 | max_loss: 0.1491 | max_backward: 1.5976 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6392.1 | uss_MB: 5613.3 | child_rss_MB: 5216.1 | allocated_MB: 342.1 | reserved_MB: 26608.0 | inactive_split_MB: 241.3 +rank4 [2026-10-28 21:03:21,786] INFO : Step 13900/50000 rank4 | Train Loss: 1.6108 | Train Loss Unweighted: 0.8469 | Train Loss Components: Auxiliary: 0.3204 | Cardinality: 0.1942 | Classification: 0.1303 | Classification_binary: 0.2751 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2168 | Regression_eta: 0.0028 | Regression_pt: 0.2192 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0870 | Classification_binary: 0.2998 | Regression_cos_phi: 0.1223 | Regression_energy: 0.0986 | Regression_eta: 0.0792 | Regression_pt: 0.1036 | Regression_sin_phi: 0.1296 | LR: 8.39e-04 | DataLoad Time: 0.0198s | Model Forward Time: 6.7334s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8010 | max_loss: 0.1552 | max_backward: 2.5526 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 1.2862 | max_tensorboard_flush: 0.0041 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 6389.1 | uss_MB: 5603.5 | child_rss_MB: 5217.8 | allocated_MB: 345.6 | reserved_MB: 26608.0 | inactive_split_MB: 292.8 +rank4 [2026-10-28 21:07:29,378] INFO : Step 14000/50000 rank4 | Train Loss: 1.3659 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1707 | Classification: 0.0974 | Classification_binary: 0.1997 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2149 | Regression_eta: 0.0023 | Regression_pt: 0.2181 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2310 | Regression_cos_phi: 0.1297 | Regression_energy: 0.1250 | Regression_eta: 0.0964 | Regression_pt: 0.1307 | Regression_sin_phi: 0.0676 | LR: 8.37e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.4512s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7574 | max_loss: 0.1908 | max_backward: 1.5870 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6393.2 | uss_MB: 4632.1 | child_rss_MB: 5217.4 | allocated_MB: 342.3 | reserved_MB: 26608.0 | inactive_split_MB: 238.4 +rank4 [2026-10-28 21:07:30,351] INFO : Running validation on rank4 +rank4 [2026-10-28 21:07:30,352] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 21:07:30,352] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 21:08:21,199] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:08:21,764] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 21:08:48,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 21:08:49,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 21:08:50,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 21:08:51,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 21:09:01,864] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 21:09:05,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:09:06,443] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 21:09:22,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 21:09:23,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 21:09:24,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 21:09:25,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 21:09:35,021] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 21:09:36,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:09:36,854] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 21:09:52,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 21:09:52,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 21:09:53,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 21:09:54,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 21:10:04,357] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 21:14:16,742] INFO : Step 14100/50000 rank4 | Train Loss: 1.5416 | Train Loss Unweighted: 0.8176 | Train Loss Components: Auxiliary: 0.3012 | Cardinality: 0.1751 | Classification: 0.1363 | Classification_binary: 0.2297 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2177 | Regression_eta: 0.0028 | Regression_pt: 0.2283 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1025 | Classification_binary: 0.2574 | Regression_cos_phi: 0.1577 | Regression_energy: 0.0911 | Regression_eta: 0.0882 | Regression_pt: 0.0998 | Regression_sin_phi: 0.1703 | LR: 8.35e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.2013s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7718 | max_loss: 0.1710 | max_backward: 1.6376 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 7412.7 | uss_MB: 6170.5 | child_rss_MB: 5197.6 | allocated_MB: 347.1 | reserved_MB: 23970.0 | inactive_split_MB: 244.9 +rank4 [2026-10-28 21:18:17,877] INFO : Step 14200/50000 rank4 | Train Loss: 1.6164 | Train Loss Unweighted: 0.8542 | Train Loss Components: Auxiliary: 0.3190 | Cardinality: 0.1950 | Classification: 0.1478 | Classification_binary: 0.2794 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2094 | Regression_eta: 0.0027 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1000 | Classification_binary: 0.3073 | Regression_cos_phi: 0.1628 | Regression_energy: 0.1060 | Regression_eta: 0.0941 | Regression_pt: 0.1116 | Regression_sin_phi: 0.1451 | LR: 8.32e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.3721s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1153 | max_forward: 0.8788 | max_loss: 0.1615 | max_backward: 1.5286 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7413.4 | uss_MB: 6634.1 | child_rss_MB: 5195.0 | allocated_MB: 344.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.9 +rank4 [2026-10-28 21:22:21,673] INFO : Step 14300/50000 rank4 | Train Loss: 1.4246 | Train Loss Unweighted: 0.7744 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1685 | Classification: 0.1198 | Classification_binary: 0.2357 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2084 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.2453 | Regression_cos_phi: 0.0897 | Regression_energy: 0.0906 | Regression_eta: 0.0589 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0884 | LR: 8.30e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.8791s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7183 | max_loss: 0.1494 | max_backward: 1.3371 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6454.4 | uss_MB: 5677.1 | child_rss_MB: 5184.2 | allocated_MB: 341.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.1 +rank4 [2026-10-28 21:26:25,336] INFO : Step 14400/50000 rank4 | Train Loss: 1.3539 | Train Loss Unweighted: 0.7267 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1402 | Classification: 0.1303 | Classification_binary: 0.2015 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1957 | Regression_eta: 0.0022 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.1783 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0922 | Regression_eta: 0.0615 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0796 | LR: 8.28e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6670s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6350 | max_loss: 0.2071 | max_backward: 1.2597 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6458.7 | uss_MB: 5681.9 | child_rss_MB: 5188.8 | allocated_MB: 346.1 | reserved_MB: 24962.0 | inactive_split_MB: 231.6 +rank4 [2026-10-28 21:30:23,756] INFO : Step 14500/50000 rank4 | Train Loss: 1.5651 | Train Loss Unweighted: 0.7967 | Train Loss Components: Auxiliary: 0.3070 | Cardinality: 0.2312 | Classification: 0.1364 | Classification_binary: 0.2565 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1984 | Regression_eta: 0.0026 | Regression_pt: 0.2005 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2158 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0934 | Regression_eta: 0.0513 | Regression_pt: 0.0983 | Regression_sin_phi: 0.0793 | LR: 8.26e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.4652s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0069 | max_forward: 0.8745 | max_loss: 0.1717 | max_backward: 1.8485 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 1.4922 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6443.5 | uss_MB: 5682.8 | child_rss_MB: 5113.1 | allocated_MB: 344.2 | reserved_MB: 25374.0 | inactive_split_MB: 206.9 +rank4 [2026-10-28 21:34:34,947] INFO : Step 14600/50000 rank4 | Train Loss: 1.6485 | Train Loss Unweighted: 0.8413 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.2375 | Classification: 0.1249 | Classification_binary: 0.2902 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2104 | Regression_eta: 0.0023 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2618 | Regression_cos_phi: 0.0920 | Regression_energy: 0.1075 | Regression_eta: 0.0486 | Regression_pt: 0.1098 | Regression_sin_phi: 0.1281 | LR: 8.24e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3685s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0068 | max_forward: 0.7866 | max_loss: 0.1515 | max_backward: 1.5469 | max_optimizer_step: 0.3128 | max_tensorboard_logging: 0.7882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6427.1 | uss_MB: 5683.8 | child_rss_MB: 5049.1 | allocated_MB: 345.7 | reserved_MB: 25374.0 | inactive_split_MB: 292.3 +rank4 [2026-10-28 21:38:39,704] INFO : Step 14700/50000 rank4 | Train Loss: 1.3645 | Train Loss Unweighted: 0.7326 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1507 | Classification: 0.1080 | Classification_binary: 0.2034 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.1962 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0692 | Regression_eta: 0.0639 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0665 | LR: 8.21e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9099s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6571 | max_loss: 0.1494 | max_backward: 1.2978 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7751 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0922 | Memory: rss_MB: 6423.6 | uss_MB: 5684.9 | child_rss_MB: 5040.9 | allocated_MB: 341.1 | reserved_MB: 25374.0 | inactive_split_MB: 264.9 +rank4 [2026-10-28 21:42:53,742] INFO : Step 14800/50000 rank4 | Train Loss: 1.3987 | Train Loss Unweighted: 0.7335 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1827 | Classification: 0.1136 | Classification_binary: 0.2244 | HitGrouping: 0.2204 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1949 | Regression_eta: 0.0022 | Regression_pt: 0.1963 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2346 | Regression_cos_phi: 0.0768 | Regression_energy: 0.1053 | Regression_eta: 0.0574 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0613 | LR: 8.19e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.2145s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.1679 | max_loss: 0.2171 | max_backward: 2.8051 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 6420.4 | uss_MB: 4434.9 | child_rss_MB: 5026.4 | allocated_MB: 345.2 | reserved_MB: 25374.0 | inactive_split_MB: 222.5 +rank4 [2026-10-28 21:47:06,020] INFO : Step 14900/50000 rank4 | Train Loss: 1.5046 | Train Loss Unweighted: 0.7991 | Train Loss Components: Auxiliary: 0.3055 | Cardinality: 0.1834 | Classification: 0.1311 | Classification_binary: 0.2643 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1982 | Regression_eta: 0.0023 | Regression_pt: 0.2008 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.2608 | Regression_cos_phi: 0.1064 | Regression_energy: 0.0932 | Regression_eta: 0.0683 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0838 | LR: 8.17e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.0338s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0071 | max_forward: 0.7642 | max_loss: 0.1561 | max_backward: 1.5975 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6421.6 | uss_MB: 5685.9 | child_rss_MB: 5030.2 | allocated_MB: 345.4 | reserved_MB: 25374.0 | inactive_split_MB: 289.7 +rank4 [2026-10-28 21:51:16,625] INFO : Step 15000/50000 rank4 | Train Loss: 1.3836 | Train Loss Unweighted: 0.7341 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1614 | Classification: 0.1182 | Classification_binary: 0.2252 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1930 | Regression_eta: 0.0023 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0823 | Classification_binary: 0.2380 | Regression_cos_phi: 0.1080 | Regression_energy: 0.0790 | Regression_eta: 0.0899 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0930 | LR: 8.15e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.7146s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0066 | max_forward: 0.6828 | max_loss: 0.1453 | max_backward: 1.3064 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1064 | Memory: rss_MB: 6423.8 | uss_MB: 5686.3 | child_rss_MB: 5021.1 | allocated_MB: 342.6 | reserved_MB: 25374.0 | inactive_split_MB: 262.7 +rank4 [2026-10-28 21:51:17,569] INFO : Running validation on rank4 +rank4 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 21:52:06,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:52:07,025] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 21:52:33,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 21:52:34,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 21:52:34,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 21:52:35,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 21:52:45,993] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 21:52:46,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:52:47,271] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 21:53:03,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 21:53:04,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 21:53:05,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 21:53:06,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 21:53:16,137] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 21:53:16,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:53:17,338] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 21:53:33,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 21:53:33,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 21:53:34,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 21:53:35,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 21:53:45,192] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 21:57:58,525] INFO : Step 15100/50000 rank4 | Train Loss: 1.3706 | Train Loss Unweighted: 0.7159 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1535 | Classification: 0.1271 | Classification_binary: 0.1954 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1940 | Regression_eta: 0.0022 | Regression_pt: 0.1948 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.1974 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0880 | Regression_eta: 0.0520 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0665 | LR: 8.12e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.0254s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.6658 | max_loss: 0.2401 | max_backward: 1.3105 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8277 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7654.5 | uss_MB: 6910.5 | child_rss_MB: 5047.2 | allocated_MB: 341.1 | reserved_MB: 22096.0 | inactive_split_MB: 215.5 +rank4 [2026-10-28 22:01:59,131] INFO : Step 15200/50000 rank4 | Train Loss: 1.5024 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3072 | Cardinality: 0.2080 | Classification: 0.1273 | Classification_binary: 0.2385 | HitGrouping: 0.2164 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1989 | Regression_eta: 0.0022 | Regression_pt: 0.2007 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2256 | Regression_cos_phi: 0.1018 | Regression_energy: 0.1026 | Regression_eta: 0.0501 | Regression_pt: 0.1065 | Regression_sin_phi: 0.0987 | LR: 8.10e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.5450s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7502 | max_loss: 0.1534 | max_backward: 1.8002 | max_optimizer_step: 0.2270 | max_tensorboard_logging: 0.8682 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 7649.1 | uss_MB: 6910.3 | child_rss_MB: 5045.3 | allocated_MB: 345.6 | reserved_MB: 22420.0 | inactive_split_MB: 257.7 +rank4 [2026-10-28 22:05:42,233] INFO : Step 15300/50000 rank4 | Train Loss: 1.4920 | Train Loss Unweighted: 0.7880 | Train Loss Components: Auxiliary: 0.3065 | Cardinality: 0.1970 | Classification: 0.1161 | Classification_binary: 0.2589 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2026 | Regression_eta: 0.0022 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2615 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0853 | Regression_eta: 0.0533 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0851 | LR: 8.08e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5781s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0192 | max_forward: 0.7642 | max_loss: 0.1530 | max_backward: 1.6292 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 0.9125 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6524.0 | uss_MB: 5797.3 | child_rss_MB: 5044.3 | allocated_MB: 341.2 | reserved_MB: 23096.0 | inactive_split_MB: 271.7 +rank4 [2026-10-28 22:09:28,151] INFO : Step 15400/50000 rank4 | Train Loss: 1.3549 | Train Loss Unweighted: 0.7210 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1572 | Classification: 0.1148 | Classification_binary: 0.2052 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1983 | Regression_eta: 0.0022 | Regression_pt: 0.1984 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0912 | Classification_binary: 0.2284 | Regression_cos_phi: 0.1090 | Regression_energy: 0.1026 | Regression_eta: 0.0598 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0925 | LR: 8.05e-04 | DataLoad Time: 0.0214s | Model Forward Time: 3.7047s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6594 | max_loss: 0.2185 | max_backward: 1.2416 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6511.0 | uss_MB: 5790.6 | child_rss_MB: 5047.2 | allocated_MB: 340.8 | reserved_MB: 23394.0 | inactive_split_MB: 166.6 +rank4 [2026-10-28 22:13:12,490] INFO : Step 15500/50000 rank4 | Train Loss: 1.4313 | Train Loss Unweighted: 0.7527 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1666 | Classification: 0.1321 | Classification_binary: 0.2182 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1979 | Regression_eta: 0.0025 | Regression_pt: 0.1996 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2546 | Regression_cos_phi: 0.1269 | Regression_energy: 0.0829 | Regression_eta: 0.0695 | Regression_pt: 0.0862 | Regression_sin_phi: 0.1016 | LR: 8.03e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2554s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0792 | max_forward: 0.7874 | max_loss: 0.1514 | max_backward: 1.8676 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5380 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6511.0 | uss_MB: 5805.3 | child_rss_MB: 5052.2 | allocated_MB: 345.6 | reserved_MB: 24092.0 | inactive_split_MB: 260.5 +rank4 [2026-10-28 22:17:01,249] INFO : Step 15600/50000 rank4 | Train Loss: 1.5305 | Train Loss Unweighted: 0.8109 | Train Loss Components: Auxiliary: 0.3137 | Cardinality: 0.1790 | Classification: 0.1304 | Classification_binary: 0.2721 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2011 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.3098 | Regression_cos_phi: 0.0948 | Regression_energy: 0.1078 | Regression_eta: 0.0669 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0978 | LR: 8.01e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2113s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7645 | max_loss: 0.2230 | max_backward: 1.7015 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1398 | Memory: rss_MB: 6508.8 | uss_MB: 5803.4 | child_rss_MB: 5052.3 | allocated_MB: 344.1 | reserved_MB: 24402.0 | inactive_split_MB: 287.9 +rank4 [2026-10-28 22:20:46,548] INFO : Step 15700/50000 rank4 | Train Loss: 1.4083 | Train Loss Unweighted: 0.7205 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1775 | Classification: 0.1238 | Classification_binary: 0.2322 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1785 | Regression_eta: 0.0024 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0797 | Classification_binary: 0.2498 | Regression_cos_phi: 0.0885 | Regression_energy: 0.1044 | Regression_eta: 0.0753 | Regression_pt: 0.1092 | Regression_sin_phi: 0.0973 | LR: 7.98e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.0326s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2392 | max_backward: 1.6606 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6506.8 | uss_MB: 5802.2 | child_rss_MB: 5055.5 | allocated_MB: 344.4 | reserved_MB: 25380.0 | inactive_split_MB: 228.2 +rank4 [2026-10-28 22:24:39,044] INFO : Step 15800/50000 rank4 | Train Loss: 1.2988 | Train Loss Unweighted: 0.6968 | Train Loss Components: Auxiliary: 0.2671 | Cardinality: 0.1436 | Classification: 0.1112 | Classification_binary: 0.1916 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1933 | Regression_eta: 0.0020 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0901 | Regression_energy: 0.0764 | Regression_eta: 0.0768 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0642 | LR: 7.96e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7266s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6109 | max_loss: 0.2015 | max_backward: 1.1291 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6506.5 | uss_MB: 5803.6 | child_rss_MB: 5051.3 | allocated_MB: 341.6 | reserved_MB: 25380.0 | inactive_split_MB: 263.9 +rank4 [2026-10-28 22:28:35,066] INFO : Step 15900/50000 rank4 | Train Loss: 1.4020 | Train Loss Unweighted: 0.7322 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1800 | Classification: 0.1295 | Classification_binary: 0.2346 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1816 | Regression_eta: 0.0023 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0995 | Classification_binary: 0.2604 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0767 | Regression_eta: 0.0699 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0907 | LR: 7.94e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2689s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0933 | max_forward: 0.7823 | max_loss: 0.1445 | max_backward: 1.4740 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6508.5 | uss_MB: 5807.0 | child_rss_MB: 5047.2 | allocated_MB: 342.9 | reserved_MB: 25380.0 | inactive_split_MB: 283.5 +rank4 [2026-10-28 22:32:40,675] INFO : Step 16000/50000 rank4 | Train Loss: 1.4978 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3126 | Cardinality: 0.2093 | Classification: 0.1190 | Classification_binary: 0.2580 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1920 | Regression_eta: 0.0021 | Regression_pt: 0.1969 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.2501 | Regression_cos_phi: 0.0936 | Regression_energy: 0.0929 | Regression_eta: 0.0548 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0871 | LR: 7.91e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4898s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.7474 | max_loss: 0.1892 | max_backward: 1.4114 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.7981 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6509.3 | uss_MB: 5675.5 | child_rss_MB: 5048.3 | allocated_MB: 342.7 | reserved_MB: 25380.0 | inactive_split_MB: 292.9 +rank4 [2026-10-28 22:32:41,702] INFO : Running validation on rank4 +rank4 [2026-10-28 22:32:41,702] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 22:32:41,702] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 22:33:34,621] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:33:35,152] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 22:34:00,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 22:34:01,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 22:34:02,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 22:34:03,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 22:34:13,819] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 22:34:14,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:34:14,938] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 22:34:31,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 22:34:32,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 22:34:32,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 22:34:33,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 22:34:44,001] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 22:34:44,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:34:44,851] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 22:35:00,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 22:35:00,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 22:35:01,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 22:35:02,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 22:35:12,135] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 22:39:22,135] INFO : Step 16100/50000 rank4 | Train Loss: 1.3441 | Train Loss Unweighted: 0.7192 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1631 | Classification: 0.1197 | Classification_binary: 0.2058 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1931 | Regression_eta: 0.0020 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0990 | Regression_eta: 0.0433 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0785 | LR: 7.89e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1517s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0584 | max_forward: 0.7257 | max_loss: 0.1421 | max_backward: 1.2590 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.0197 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6577.1 | uss_MB: 5871.7 | child_rss_MB: 5053.6 | allocated_MB: 341.3 | reserved_MB: 23342.0 | inactive_split_MB: 221.6 +rank4 [2026-10-28 22:43:09,260] INFO : Step 16200/50000 rank4 | Train Loss: 1.3737 | Train Loss Unweighted: 0.7229 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1695 | Classification: 0.1213 | Classification_binary: 0.2167 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2025 | Regression_cos_phi: 0.1078 | Regression_energy: 0.1140 | Regression_eta: 0.0436 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0664 | LR: 7.86e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0507s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.1062 | max_forward: 0.7366 | max_loss: 0.1469 | max_backward: 1.4999 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.6015 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6580.7 | uss_MB: 5882.6 | child_rss_MB: 5053.8 | allocated_MB: 342.5 | reserved_MB: 24304.0 | inactive_split_MB: 205.9 +rank4 [2026-10-28 22:46:54,662] INFO : Step 16300/50000 rank4 | Train Loss: 1.4671 | Train Loss Unweighted: 0.7592 | Train Loss Components: Auxiliary: 0.3008 | Cardinality: 0.1775 | Classification: 0.1342 | Classification_binary: 0.2702 | HitGrouping: 0.2148 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0023 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2639 | Regression_cos_phi: 0.1210 | Regression_energy: 0.0843 | Regression_eta: 0.0580 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0809 | LR: 7.84e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4599s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7718 | max_loss: 0.1628 | max_backward: 1.7017 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.8513 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6574.7 | uss_MB: 5880.8 | child_rss_MB: 5053.1 | allocated_MB: 345.8 | reserved_MB: 24304.0 | inactive_split_MB: 281.0 +rank4 [2026-10-28 22:50:39,547] INFO : Step 16400/50000 rank4 | Train Loss: 1.4200 | Train Loss Unweighted: 0.7161 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.2086 | Classification: 0.1206 | Classification_binary: 0.2158 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0024 | Regression_pt: 0.1885 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0991 | Regression_energy: 0.0992 | Regression_eta: 0.0725 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0772 | LR: 7.81e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.8037s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7570 | max_loss: 0.1546 | max_backward: 1.2568 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5199 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1436 | Memory: rss_MB: 6571.7 | uss_MB: 5879.8 | child_rss_MB: 5049.7 | allocated_MB: 342.7 | reserved_MB: 25268.0 | inactive_split_MB: 213.6 +rank4 [2026-10-28 22:54:25,074] INFO : Step 16500/50000 rank4 | Train Loss: 1.2201 | Train Loss Unweighted: 0.6592 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1195 | Classification: 0.1029 | Classification_binary: 0.1835 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1836 | Regression_eta: 0.0021 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1356 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0814 | Regression_eta: 0.0395 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0510 | LR: 7.79e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.3180s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0128 | max_forward: 0.7866 | max_loss: 0.2200 | max_backward: 1.9590 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.4913 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6572.2 | uss_MB: 5885.7 | child_rss_MB: 5046.8 | allocated_MB: 343.6 | reserved_MB: 26238.0 | inactive_split_MB: 248.8 +rank4 [2026-10-28 22:58:25,288] INFO : Step 16600/50000 rank4 | Train Loss: 1.4335 | Train Loss Unweighted: 0.7657 | Train Loss Components: Auxiliary: 0.2949 | Cardinality: 0.1698 | Classification: 0.1386 | Classification_binary: 0.2312 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1941 | Regression_eta: 0.0021 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0789 | Regression_eta: 0.0411 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0724 | LR: 7.77e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.9939s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0139 | max_forward: 1.0329 | max_loss: 0.3279 | max_backward: 2.5489 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.5244 | max_tensorboard_flush: 0.0014 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6575.3 | uss_MB: 4925.2 | child_rss_MB: 5045.0 | allocated_MB: 344.2 | reserved_MB: 27228.0 | inactive_split_MB: 219.6 +rank4 [2026-10-28 23:02:38,763] INFO : Step 16700/50000 rank4 | Train Loss: 1.4062 | Train Loss Unweighted: 0.7378 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1840 | Classification: 0.1203 | Classification_binary: 0.2439 | HitGrouping: 0.2108 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1839 | Regression_eta: 0.0020 | Regression_pt: 0.1855 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2741 | Regression_cos_phi: 0.0920 | Regression_energy: 0.0871 | Regression_eta: 0.0673 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0909 | LR: 7.74e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0477s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7336 | max_loss: 0.2460 | max_backward: 1.6747 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6577.4 | uss_MB: 5888.0 | child_rss_MB: 5049.7 | allocated_MB: 344.6 | reserved_MB: 27228.0 | inactive_split_MB: 279.4 +rank4 [2026-10-28 23:06:47,051] INFO : Step 16800/50000 rank4 | Train Loss: 1.3172 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1533 | Classification: 0.1173 | Classification_binary: 0.1949 | HitGrouping: 0.2175 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1819 | Regression_eta: 0.0027 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1928 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0746 | Regression_eta: 0.0522 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0515 | LR: 7.72e-04 | DataLoad Time: 0.0225s | Model Forward Time: 3.6834s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0962 | max_forward: 0.7772 | max_loss: 0.1423 | max_backward: 1.3121 | max_optimizer_step: 0.2117 | max_tensorboard_logging: 0.4865 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6580.1 | uss_MB: 5886.7 | child_rss_MB: 5043.1 | allocated_MB: 342.1 | reserved_MB: 27250.0 | inactive_split_MB: 200.8 +rank4 [2026-10-28 23:11:05,115] INFO : Step 16900/50000 rank4 | Train Loss: 1.3132 | Train Loss Unweighted: 0.6868 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1474 | Classification: 0.1260 | Classification_binary: 0.1988 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1779 | Regression_eta: 0.0021 | Regression_pt: 0.1798 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.1972 | Regression_cos_phi: 0.0980 | Regression_energy: 0.1045 | Regression_eta: 0.0698 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0704 | LR: 7.69e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.7613s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7203 | max_loss: 0.1660 | max_backward: 1.1458 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6581.3 | uss_MB: 5889.6 | child_rss_MB: 5044.4 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 193.9 +rank4 [2026-10-28 23:15:09,203] INFO : Step 17000/50000 rank4 | Train Loss: 1.3575 | Train Loss Unweighted: 0.7115 | Train Loss Components: Auxiliary: 0.2906 | Cardinality: 0.1781 | Classification: 0.1179 | Classification_binary: 0.2325 | HitGrouping: 0.2089 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0018 | Regression_pt: 0.1792 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0806 | Classification_binary: 0.2215 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0767 | Regression_eta: 0.0376 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0651 | LR: 7.67e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4156s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.9008 | max_loss: 0.1474 | max_backward: 1.9229 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6575.6 | uss_MB: 5894.0 | child_rss_MB: 5025.6 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 278.9 +rank4 [2026-10-28 23:15:10,041] INFO : Running validation on rank4 +rank4 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 23:16:00,503] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:16:01,059] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 23:16:27,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 23:16:27,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 23:16:28,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 23:16:30,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 23:16:40,491] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 23:16:42,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:16:42,521] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 23:16:58,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 23:16:59,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 23:17:00,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 23:17:01,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 23:17:11,333] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 23:17:11,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:17:12,023] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 23:17:27,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 23:17:28,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 23:17:29,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 23:17:29,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 23:17:39,852] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-28 23:21:59,503] INFO : Step 17100/50000 rank4 | Train Loss: 1.3790 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1704 | Classification: 0.1078 | Classification_binary: 0.2095 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1968 | Regression_eta: 0.0020 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.2136 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0814 | Regression_eta: 0.0444 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0572 | LR: 7.64e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.8988s | Phase Times: max_pre_step_barrier: 0.0234 | max_batch_to_device: 0.0065 | max_forward: 0.8292 | max_loss: 0.1455 | max_backward: 1.4833 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6666.8 | uss_MB: 5307.9 | child_rss_MB: 5009.6 | allocated_MB: 343.8 | reserved_MB: 22688.0 | inactive_split_MB: 254.6 +rank4 [2026-10-28 23:26:06,863] INFO : Step 17200/50000 rank4 | Train Loss: 1.2786 | Train Loss Unweighted: 0.6598 | Train Loss Components: Auxiliary: 0.2576 | Cardinality: 0.1378 | Classification: 0.1161 | Classification_binary: 0.1910 | HitGrouping: 0.2191 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0025 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0809 | Regression_eta: 0.0389 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0563 | LR: 7.62e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8634s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6470 | max_loss: 0.2505 | max_backward: 1.4380 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6320 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6664.1 | uss_MB: 5982.7 | child_rss_MB: 5028.0 | allocated_MB: 344.1 | reserved_MB: 22688.0 | inactive_split_MB: 295.9 +rank4 [2026-10-28 23:30:05,516] INFO : Step 17300/50000 rank4 | Train Loss: 1.3571 | Train Loss Unweighted: 0.6976 | Train Loss Components: Auxiliary: 0.2789 | Cardinality: 0.1665 | Classification: 0.1193 | Classification_binary: 0.2287 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1724 | Regression_eta: 0.0021 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2340 | Regression_cos_phi: 0.1594 | Regression_energy: 0.0898 | Regression_eta: 0.0770 | Regression_pt: 0.0936 | Regression_sin_phi: 0.1055 | LR: 7.59e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5423s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7276 | max_loss: 0.2520 | max_backward: 1.8579 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8525 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6666.1 | uss_MB: 5121.5 | child_rss_MB: 5026.9 | allocated_MB: 344.2 | reserved_MB: 23616.0 | inactive_split_MB: 266.5 +rank4 [2026-10-28 23:34:25,421] INFO : Step 17400/50000 rank4 | Train Loss: 1.3668 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2811 | Cardinality: 0.1760 | Classification: 0.1195 | Classification_binary: 0.2351 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1732 | Regression_eta: 0.0019 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2156 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0866 | Regression_eta: 0.0419 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0535 | LR: 7.57e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.4868s | Phase Times: max_pre_step_barrier: 0.1038 | max_batch_to_device: 0.0306 | max_forward: 0.7782 | max_loss: 0.2429 | max_backward: 1.3743 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.8348 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0757 | Memory: rss_MB: 6667.1 | uss_MB: 5984.6 | child_rss_MB: 5026.4 | allocated_MB: 343.3 | reserved_MB: 24604.0 | inactive_split_MB: 262.4 +rank4 [2026-10-28 23:38:51,400] INFO : Step 17500/50000 rank4 | Train Loss: 1.3114 | Train Loss Unweighted: 0.6526 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.1481 | Classification: 0.1145 | Classification_binary: 0.1884 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1714 | Regression_eta: 0.0024 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.1729 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0859 | Regression_eta: 0.0615 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0753 | LR: 7.54e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9149s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.6394 | max_loss: 0.1427 | max_backward: 1.2079 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.8375 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 6667.1 | uss_MB: 5986.2 | child_rss_MB: 5023.0 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 240.0 +rank4 [2026-10-28 23:43:03,145] INFO : Step 17600/50000 rank4 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2688 | Cardinality: 0.1362 | Classification: 0.1273 | Classification_binary: 0.2065 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0019 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.1925 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0742 | Regression_eta: 0.0549 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0779 | LR: 7.52e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.6085s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.1120 | max_forward: 1.2883 | max_loss: 0.2624 | max_backward: 1.5380 | max_optimizer_step: 0.2147 | max_tensorboard_logging: 0.7644 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 6665.9 | uss_MB: 5985.5 | child_rss_MB: 5028.5 | allocated_MB: 344.2 | reserved_MB: 25596.0 | inactive_split_MB: 237.9 +rank4 [2026-10-28 23:47:02,149] INFO : Step 17700/50000 rank4 | Train Loss: 1.3491 | Train Loss Unweighted: 0.6939 | Train Loss Components: Auxiliary: 0.2835 | Cardinality: 0.1622 | Classification: 0.1186 | Classification_binary: 0.2310 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0022 | Regression_pt: 0.1701 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0844 | Classification_binary: 0.2211 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0906 | Regression_eta: 0.0810 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0910 | LR: 7.49e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.6539s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0704 | max_forward: 0.9841 | max_loss: 0.1994 | max_backward: 1.8911 | max_optimizer_step: 0.2227 | max_tensorboard_logging: 0.5849 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6667.9 | uss_MB: 5989.0 | child_rss_MB: 5031.2 | allocated_MB: 345.5 | reserved_MB: 25596.0 | inactive_split_MB: 287.6 +rank4 [2026-10-28 23:51:17,534] INFO : Step 17800/50000 rank4 | Train Loss: 1.3063 | Train Loss Unweighted: 0.6692 | Train Loss Components: Auxiliary: 0.2664 | Cardinality: 0.1579 | Classification: 0.1042 | Classification_binary: 0.2130 | HitGrouping: 0.2150 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1741 | Regression_eta: 0.0019 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.2218 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0788 | Regression_eta: 0.0333 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0879 | LR: 7.47e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8700s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.7351 | max_loss: 0.1686 | max_backward: 1.4616 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5023 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 6666.0 | uss_MB: 5988.8 | child_rss_MB: 5032.9 | allocated_MB: 341.0 | reserved_MB: 25596.0 | inactive_split_MB: 197.8 +rank4 [2026-10-28 23:55:22,790] INFO : Step 17900/50000 rank4 | Train Loss: 1.2213 | Train Loss Unweighted: 0.6433 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1427 | Classification: 0.1147 | Classification_binary: 0.1725 | HitGrouping: 0.2070 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0769 | Regression_eta: 0.0426 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0910 | LR: 7.44e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8178s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7249 | max_loss: 0.1579 | max_backward: 1.1981 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6654.3 | uss_MB: 5990.9 | child_rss_MB: 5033.3 | allocated_MB: 341.6 | reserved_MB: 25596.0 | inactive_split_MB: 297.9 +rank4 [2026-10-28 23:59:20,282] INFO : Step 18000/50000 rank4 | Train Loss: 1.3597 | Train Loss Unweighted: 0.7083 | Train Loss Components: Auxiliary: 0.2781 | Cardinality: 0.1599 | Classification: 0.1247 | Classification_binary: 0.2275 | HitGrouping: 0.2122 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0021 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0837 | Classification_binary: 0.2578 | Regression_cos_phi: 0.1043 | Regression_energy: 0.0802 | Regression_eta: 0.0741 | Regression_pt: 0.0827 | Regression_sin_phi: 0.1252 | LR: 7.42e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6429s | Phase Times: max_pre_step_barrier: 0.0816 | max_batch_to_device: 0.0068 | max_forward: 0.8980 | max_loss: 0.1586 | max_backward: 1.9185 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.7546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6654.3 | uss_MB: 5995.0 | child_rss_MB: 5035.3 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 254.8 +rank4 [2026-10-28 23:59:21,239] INFO : Running validation on rank4 +rank4 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 00:00:12,209] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:00:12,738] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 00:00:39,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 00:00:40,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 00:00:41,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 00:00:42,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 00:00:52,140] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 00:00:52,803] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:00:53,360] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 00:01:09,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 00:01:10,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 00:01:11,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 00:01:12,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 00:01:22,060] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 00:01:22,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:01:23,291] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 00:01:38,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 00:01:39,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 00:01:39,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 00:01:40,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 00:01:50,712] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 00:05:57,254] INFO : Step 18100/50000 rank4 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7216 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1666 | Classification: 0.1199 | Classification_binary: 0.2340 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1807 | Regression_eta: 0.0024 | Regression_pt: 0.1824 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.2539 | Regression_cos_phi: 0.0901 | Regression_energy: 0.0846 | Regression_eta: 0.0487 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0824 | LR: 7.39e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.5286s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.9127 | max_loss: 0.4546 | max_backward: 1.7406 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.7463 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7789.9 | uss_MB: 7130.3 | child_rss_MB: 5031.5 | allocated_MB: 344.6 | reserved_MB: 22256.0 | inactive_split_MB: 291.4 +rank4 [2026-10-29 00:09:47,529] INFO : Step 18200/50000 rank4 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6468 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1359 | Classification: 0.1208 | Classification_binary: 0.1803 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1700 | Regression_eta: 0.0020 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0831 | Regression_eta: 0.0509 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0631 | LR: 7.36e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7595s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7114 | max_loss: 0.2369 | max_backward: 1.1051 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6784.1 | uss_MB: 6119.0 | child_rss_MB: 5031.9 | allocated_MB: 341.2 | reserved_MB: 23772.0 | inactive_split_MB: 170.7 +rank4 [2026-10-29 00:13:32,739] INFO : Step 18300/50000 rank4 | Train Loss: 1.3100 | Train Loss Unweighted: 0.6891 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1522 | Classification: 0.1230 | Classification_binary: 0.1938 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1834 | Regression_eta: 0.0019 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0814 | Classification_binary: 0.2038 | Regression_cos_phi: 0.0937 | Regression_energy: 0.1031 | Regression_eta: 0.0419 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0900 | LR: 7.34e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1460s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7219 | max_loss: 0.1491 | max_backward: 1.5825 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.7390 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 6786.1 | uss_MB: 6124.1 | child_rss_MB: 5031.6 | allocated_MB: 344.4 | reserved_MB: 24778.0 | inactive_split_MB: 234.4 +rank4 [2026-10-29 00:17:16,846] INFO : Step 18400/50000 rank4 | Train Loss: 1.3667 | Train Loss Unweighted: 0.7337 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1448 | Classification: 0.1277 | Classification_binary: 0.2393 | HitGrouping: 0.2095 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1805 | Regression_eta: 0.0022 | Regression_pt: 0.1818 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.2559 | Regression_cos_phi: 0.0906 | Regression_energy: 0.0924 | Regression_eta: 0.0612 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0840 | LR: 7.31e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.4875s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1744 | max_backward: 1.7308 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6782.0 | uss_MB: 6123.4 | child_rss_MB: 5027.3 | allocated_MB: 344.6 | reserved_MB: 24778.0 | inactive_split_MB: 281.4 +rank4 [2026-10-29 00:21:03,776] INFO : Step 18500/50000 rank4 | Train Loss: 1.2528 | Train Loss Unweighted: 0.6816 | Train Loss Components: Auxiliary: 0.2631 | Cardinality: 0.1350 | Classification: 0.1140 | Classification_binary: 0.1973 | HitGrouping: 0.2066 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1821 | Regression_eta: 0.0017 | Regression_pt: 0.1847 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0972 | Regression_eta: 0.0278 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0489 | LR: 7.29e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2656s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0113 | max_forward: 0.8281 | max_loss: 0.2418 | max_backward: 1.4683 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7561 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1431 | Memory: rss_MB: 6783.0 | uss_MB: 6122.8 | child_rss_MB: 5026.7 | allocated_MB: 343.3 | reserved_MB: 24778.0 | inactive_split_MB: 226.5 +rank4 [2026-10-29 00:24:44,690] INFO : Step 18600/50000 rank4 | Train Loss: 1.1881 | Train Loss Unweighted: 0.6335 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1331 | Classification: 0.1084 | Classification_binary: 0.1723 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1746 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1697 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0931 | Regression_eta: 0.0470 | Regression_pt: 0.0958 | Regression_sin_phi: 0.0834 | LR: 7.26e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8522s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7299 | max_loss: 0.2279 | max_backward: 1.0799 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8080 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6782.4 | uss_MB: 6120.8 | child_rss_MB: 5022.4 | allocated_MB: 341.0 | reserved_MB: 24778.0 | inactive_split_MB: 229.3 +rank4 [2026-10-29 00:28:24,928] INFO : Step 18700/50000 rank4 | Train Loss: 1.3122 | Train Loss Unweighted: 0.6534 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1756 | Classification: 0.1225 | Classification_binary: 0.2098 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1571 | Regression_eta: 0.0020 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.1992 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0771 | Regression_eta: 0.0564 | Regression_pt: 0.0793 | Regression_sin_phi: 0.1032 | LR: 7.24e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6252s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9129 | max_loss: 0.1606 | max_backward: 1.9397 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6785.3 | uss_MB: 6128.9 | child_rss_MB: 5021.8 | allocated_MB: 344.8 | reserved_MB: 24778.0 | inactive_split_MB: 239.8 +rank4 [2026-10-29 00:32:05,525] INFO : Step 18800/50000 rank4 | Train Loss: 1.2696 | Train Loss Unweighted: 0.6643 | Train Loss Components: Auxiliary: 0.2674 | Cardinality: 0.1602 | Classification: 0.1070 | Classification_binary: 0.2200 | HitGrouping: 0.2058 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1652 | Regression_eta: 0.0018 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2105 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0854 | Regression_eta: 0.0563 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0833 | LR: 7.21e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6473s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7444 | max_loss: 0.2527 | max_backward: 1.7167 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.9324 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 6785.1 | uss_MB: 6124.1 | child_rss_MB: 5021.8 | allocated_MB: 345.4 | reserved_MB: 25044.0 | inactive_split_MB: 284.4 +rank4 [2026-10-29 00:35:49,546] INFO : Step 18900/50000 rank4 | Train Loss: 1.1718 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1382 | Classification: 0.0996 | Classification_binary: 0.1741 | HitGrouping: 0.2036 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1652 | Regression_eta: 0.0017 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0951 | Regression_eta: 0.0404 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0728 | LR: 7.18e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.0769s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0056 | max_forward: 0.6830 | max_loss: 0.1421 | max_backward: 1.2812 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.8931 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6786.1 | uss_MB: 6126.4 | child_rss_MB: 5006.2 | allocated_MB: 338.7 | reserved_MB: 25044.0 | inactive_split_MB: 232.6 +rank4 [2026-10-29 00:39:31,233] INFO : Step 19000/50000 rank4 | Train Loss: 1.2397 | Train Loss Unweighted: 0.6317 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1277 | Classification: 0.1188 | Classification_binary: 0.1866 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1600 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1861 | Regression_cos_phi: 0.1009 | Regression_energy: 0.0804 | Regression_eta: 0.0754 | Regression_pt: 0.0831 | Regression_sin_phi: 0.1219 | LR: 7.16e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.2161s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6593 | max_loss: 0.2467 | max_backward: 1.6203 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7732 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6787.7 | uss_MB: 6128.3 | child_rss_MB: 5026.8 | allocated_MB: 342.2 | reserved_MB: 25790.0 | inactive_split_MB: 229.6 +rank4 [2026-10-29 00:39:32,174] INFO : Running validation on rank4 +rank4 [2026-10-29 00:39:32,174] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 00:39:32,175] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 00:40:23,806] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:40:24,368] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 00:40:50,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 00:40:50,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 00:40:51,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 00:40:52,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 00:41:02,885] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 00:41:03,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:41:04,088] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 00:41:19,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 00:41:20,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 00:41:21,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 00:41:22,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 00:41:32,414] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 00:41:33,292] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:41:33,790] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 00:41:49,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 00:41:49,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 00:41:50,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 00:41:51,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 00:42:01,324] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 00:46:01,996] INFO : Step 19100/50000 rank4 | Train Loss: 1.3562 | Train Loss Unweighted: 0.6933 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1644 | Classification: 0.1204 | Classification_binary: 0.2426 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1627 | Regression_eta: 0.0023 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.2480 | Regression_cos_phi: 0.0913 | Regression_energy: 0.0784 | Regression_eta: 0.0492 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0585 | LR: 7.13e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.5916s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7796 | max_loss: 0.1634 | max_backward: 1.6942 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.8232 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 8027.6 | uss_MB: 7365.9 | child_rss_MB: 5027.2 | allocated_MB: 345.2 | reserved_MB: 23900.0 | inactive_split_MB: 240.8 +rank4 [2026-10-29 00:49:48,722] INFO : Step 19200/50000 rank4 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5946 | Train Loss Components: Auxiliary: 0.2384 | Cardinality: 0.1391 | Classification: 0.0965 | Classification_binary: 0.1984 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1472 | Regression_eta: 0.0018 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.2116 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0593 | Regression_eta: 0.0554 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0914 | LR: 7.10e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9582s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7138 | max_loss: 0.2391 | max_backward: 1.2379 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.7395 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6876.1 | uss_MB: 6216.1 | child_rss_MB: 5025.0 | allocated_MB: 341.3 | reserved_MB: 23900.0 | inactive_split_MB: 206.8 +rank4 [2026-10-29 00:53:30,059] INFO : Step 19300/50000 rank4 | Train Loss: 1.1768 | Train Loss Unweighted: 0.6083 | Train Loss Components: Auxiliary: 0.2422 | Cardinality: 0.1345 | Classification: 0.1035 | Classification_binary: 0.1709 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0018 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1724 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0746 | Regression_eta: 0.0394 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0599 | LR: 7.08e-04 | DataLoad Time: 0.0220s | Model Forward Time: 3.8955s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0503 | max_forward: 0.7163 | max_loss: 0.1442 | max_backward: 1.3745 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6878.1 | uss_MB: 6217.4 | child_rss_MB: 5020.2 | allocated_MB: 341.0 | reserved_MB: 23900.0 | inactive_split_MB: 218.1 +rank4 [2026-10-29 00:57:11,121] INFO : Step 19400/50000 rank4 | Train Loss: 1.4590 | Train Loss Unweighted: 0.7109 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.2328 | Classification: 0.1237 | Classification_binary: 0.2265 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1770 | Regression_eta: 0.0021 | Regression_pt: 0.1794 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2183 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0925 | Regression_eta: 0.0602 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0616 | LR: 7.05e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.4243s | Phase Times: max_pre_step_barrier: 0.1118 | max_batch_to_device: 0.0070 | max_forward: 0.7578 | max_loss: 0.2459 | max_backward: 1.7975 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6876.3 | uss_MB: 6217.1 | child_rss_MB: 5016.1 | allocated_MB: 347.7 | reserved_MB: 24186.0 | inactive_split_MB: 276.3 +rank4 [2026-10-29 01:00:48,550] INFO : Step 19500/50000 rank4 | Train Loss: 1.3753 | Train Loss Unweighted: 0.6794 | Train Loss Components: Auxiliary: 0.2810 | Cardinality: 0.1865 | Classification: 0.1153 | Classification_binary: 0.2310 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1637 | Regression_eta: 0.0023 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2305 | Regression_cos_phi: 0.1021 | Regression_energy: 0.0850 | Regression_eta: 0.0684 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0961 | LR: 7.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8980s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7197 | max_loss: 0.2164 | max_backward: 1.3937 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.4987 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6876.3 | uss_MB: 6219.5 | child_rss_MB: 5017.7 | allocated_MB: 345.2 | reserved_MB: 24186.0 | inactive_split_MB: 275.2 +rank4 [2026-10-29 01:04:42,200] INFO : Step 19600/50000 rank4 | Train Loss: 1.2086 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1557 | Classification: 0.0942 | Classification_binary: 0.1768 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1735 | Regression_eta: 0.0015 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1717 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0803 | Regression_eta: 0.0382 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0587 | LR: 7.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7340s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6937 | max_loss: 0.1942 | max_backward: 1.2727 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6877.2 | uss_MB: 6219.0 | child_rss_MB: 5004.9 | allocated_MB: 340.6 | reserved_MB: 24186.0 | inactive_split_MB: 277.9 +rank4 [2026-10-29 01:08:29,168] INFO : Step 19700/50000 rank4 | Train Loss: 1.2115 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1478 | Classification: 0.1094 | Classification_binary: 0.1919 | HitGrouping: 0.2027 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.1892 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0663 | Regression_eta: 0.0539 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0801 | LR: 6.97e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8675s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0914 | max_forward: 0.7091 | max_loss: 0.1429 | max_backward: 1.4681 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0652 | Memory: rss_MB: 6879.2 | uss_MB: 6222.3 | child_rss_MB: 5016.1 | allocated_MB: 344.0 | reserved_MB: 24630.0 | inactive_split_MB: 143.8 +rank4 [2026-10-29 01:12:36,832] INFO : Step 19800/50000 rank4 | Train Loss: 1.2919 | Train Loss Unweighted: 0.6584 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1453 | Classification: 0.1132 | Classification_binary: 0.2375 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0020 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.2255 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0687 | Regression_eta: 0.0433 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0873 | LR: 6.94e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.4518s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0074 | max_forward: 0.7828 | max_loss: 0.2508 | max_backward: 1.8604 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6879.1 | uss_MB: 6223.6 | child_rss_MB: 5015.7 | allocated_MB: 348.0 | reserved_MB: 25634.0 | inactive_split_MB: 277.9 +rank4 [2026-10-29 01:16:27,498] INFO : Step 19900/50000 rank4 | Train Loss: 1.2010 | Train Loss Unweighted: 0.6167 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1346 | Classification: 0.0885 | Classification_binary: 0.1961 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1633 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.2064 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0866 | Regression_eta: 0.0473 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0752 | LR: 6.92e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9746s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6671 | max_loss: 0.2548 | max_backward: 1.5862 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6881.1 | uss_MB: 6224.6 | child_rss_MB: 5016.9 | allocated_MB: 340.3 | reserved_MB: 25634.0 | inactive_split_MB: 201.7 +rank4 [2026-10-29 01:20:16,835] INFO : Step 20000/50000 rank4 | Train Loss: 1.2077 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1487 | Classification: 0.1030 | Classification_binary: 0.1729 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1639 | Regression_eta: 0.0019 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0700 | Regression_eta: 0.0462 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0877 | LR: 6.89e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.8160s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.6570 | max_loss: 0.1550 | max_backward: 1.4848 | max_optimizer_step: 0.2205 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6884.1 | uss_MB: 6226.8 | child_rss_MB: 5009.1 | allocated_MB: 341.8 | reserved_MB: 26642.0 | inactive_split_MB: 237.7 +rank4 [2026-10-29 01:20:17,835] INFO : Running validation on rank4 +rank4 [2026-10-29 01:20:17,835] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 01:20:17,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 01:21:07,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:21:08,250] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 01:21:34,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 01:21:35,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 01:21:36,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 01:21:37,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 01:21:47,480] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 01:21:48,205] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:21:48,734] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 01:22:04,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 01:22:05,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 01:22:06,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 01:22:07,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 01:22:17,657] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 01:22:18,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:22:18,871] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 01:22:34,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 01:22:35,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 01:22:35,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 01:22:36,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 01:22:46,570] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 01:26:57,041] INFO : Step 20100/50000 rank4 | Train Loss: 1.3825 | Train Loss Unweighted: 0.6739 | Train Loss Components: Auxiliary: 0.2830 | Cardinality: 0.1854 | Classification: 0.1189 | Classification_binary: 0.2349 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1561 | Regression_eta: 0.0022 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2684 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0838 | Regression_eta: 0.0896 | Regression_pt: 0.0893 | Regression_sin_phi: 0.1010 | LR: 6.86e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2467s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.9496 | max_loss: 0.1965 | max_backward: 1.6425 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.5117 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1294 | Memory: rss_MB: 8120.6 | uss_MB: 7458.3 | child_rss_MB: 5010.7 | allocated_MB: 344.8 | reserved_MB: 21328.0 | inactive_split_MB: 198.7 +rank4 [2026-10-29 01:30:57,230] INFO : Step 20200/50000 rank4 | Train Loss: 1.3220 | Train Loss Unweighted: 0.6624 | Train Loss Components: Auxiliary: 0.2728 | Cardinality: 0.1545 | Classification: 0.1109 | Classification_binary: 0.2232 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1618 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.2178 | Regression_cos_phi: 0.0945 | Regression_energy: 0.0748 | Regression_eta: 0.0564 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0677 | LR: 6.84e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8622s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.9337 | max_loss: 0.2287 | max_backward: 1.4619 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6969.7 | uss_MB: 6305.7 | child_rss_MB: 5014.0 | allocated_MB: 344.4 | reserved_MB: 21328.0 | inactive_split_MB: 287.4 +rank4 [2026-10-29 01:34:46,196] INFO : Step 20300/50000 rank4 | Train Loss: 1.2742 | Train Loss Unweighted: 0.6629 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1615 | Classification: 0.1132 | Classification_binary: 0.1937 | HitGrouping: 0.2129 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1761 | Regression_eta: 0.0018 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0921 | Regression_eta: 0.0474 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0716 | LR: 6.81e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.8209s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6857 | max_loss: 0.2423 | max_backward: 1.2822 | max_optimizer_step: 0.2837 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0419 | Memory: rss_MB: 6967.2 | uss_MB: 6303.7 | child_rss_MB: 5019.2 | allocated_MB: 340.8 | reserved_MB: 21826.0 | inactive_split_MB: 239.4 +rank4 [2026-10-29 01:38:43,604] INFO : Step 20400/50000 rank4 | Train Loss: 1.2743 | Train Loss Unweighted: 0.6521 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1507 | Classification: 0.1279 | Classification_binary: 0.1990 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1604 | Regression_eta: 0.0019 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1922 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0719 | Regression_eta: 0.0410 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0465 | LR: 6.78e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7474s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0283 | max_forward: 1.3648 | max_loss: 0.1816 | max_backward: 1.7875 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 6969.3 | uss_MB: 6306.4 | child_rss_MB: 5010.2 | allocated_MB: 343.0 | reserved_MB: 22154.0 | inactive_split_MB: 243.5 +rank4 [2026-10-29 01:42:48,353] INFO : Step 20500/50000 rank4 | Train Loss: 1.2712 | Train Loss Unweighted: 0.6509 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1409 | Classification: 0.1147 | Classification_binary: 0.2202 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1558 | Regression_eta: 0.0019 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2579 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0796 | Regression_eta: 0.0646 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0868 | LR: 6.76e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2559s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7205 | max_loss: 0.2283 | max_backward: 1.6623 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.6254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0704 | Memory: rss_MB: 6969.0 | uss_MB: 6306.9 | child_rss_MB: 5000.2 | allocated_MB: 345.0 | reserved_MB: 22154.0 | inactive_split_MB: 208.2 +rank4 [2026-10-29 01:46:45,622] INFO : Step 20600/50000 rank4 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6338 | Train Loss Components: Auxiliary: 0.2625 | Cardinality: 0.1623 | Classification: 0.1101 | Classification_binary: 0.1900 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0018 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.1999 | Regression_cos_phi: 0.0833 | Regression_energy: 0.0778 | Regression_eta: 0.0591 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0555 | LR: 6.73e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9352s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 1.1520 | max_loss: 0.2849 | max_backward: 2.1471 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.6141 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6963.6 | uss_MB: 6301.3 | child_rss_MB: 5002.5 | allocated_MB: 341.4 | reserved_MB: 22154.0 | inactive_split_MB: 183.5 +rank4 [2026-10-29 01:50:54,179] INFO : Step 20700/50000 rank4 | Train Loss: 1.1613 | Train Loss Unweighted: 0.6066 | Train Loss Components: Auxiliary: 0.2395 | Cardinality: 0.1327 | Classification: 0.1061 | Classification_binary: 0.1677 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0016 | Regression_pt: 0.1666 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0741 | Regression_eta: 0.0438 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0594 | LR: 6.70e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5596s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6276 | max_loss: 0.2330 | max_backward: 1.4954 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6965.5 | uss_MB: 5936.8 | child_rss_MB: 5004.9 | allocated_MB: 341.0 | reserved_MB: 22154.0 | inactive_split_MB: 285.0 +rank4 [2026-10-29 01:55:01,769] INFO : Step 20800/50000 rank4 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1493 | Classification: 0.1205 | Classification_binary: 0.2020 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.2166 | Regression_cos_phi: 0.0901 | Regression_energy: 0.0761 | Regression_eta: 0.0534 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0952 | LR: 6.67e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.2099s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.7679 | max_loss: 0.1604 | max_backward: 1.7694 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.4914 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6972.0 | uss_MB: 6311.7 | child_rss_MB: 5027.1 | allocated_MB: 344.1 | reserved_MB: 22154.0 | inactive_split_MB: 258.7 +rank4 [2026-10-29 01:59:06,394] INFO : Step 20900/50000 rank4 | Train Loss: 1.2934 | Train Loss Unweighted: 0.6525 | Train Loss Components: Auxiliary: 0.2719 | Cardinality: 0.1620 | Classification: 0.1167 | Classification_binary: 0.2121 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1591 | Regression_eta: 0.0021 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0693 | Regression_eta: 0.0585 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0558 | LR: 6.65e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1352s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7787 | max_loss: 0.1747 | max_backward: 1.6678 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 6974.0 | uss_MB: 5772.6 | child_rss_MB: 5037.2 | allocated_MB: 343.6 | reserved_MB: 23208.0 | inactive_split_MB: 296.4 +rank4 [2026-10-29 02:03:08,876] INFO : Step 21000/50000 rank4 | Train Loss: 1.1402 | Train Loss Unweighted: 0.5874 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1236 | Classification: 0.0967 | Classification_binary: 0.1701 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1583 | Regression_eta: 0.0017 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0870 | Regression_eta: 0.0288 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0554 | LR: 6.62e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.8975s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6629 | max_loss: 0.1833 | max_backward: 1.4689 | max_optimizer_step: 0.2206 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6973.2 | uss_MB: 6313.8 | child_rss_MB: 5014.5 | allocated_MB: 340.4 | reserved_MB: 23208.0 | inactive_split_MB: 264.1 +rank4 [2026-10-29 02:03:09,907] INFO : Running validation on rank4 +rank4 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 02:04:00,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:04:01,073] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 02:04:27,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 02:04:27,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 02:04:28,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 02:04:29,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 02:04:39,904] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 02:04:40,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:04:41,352] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 02:04:57,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 02:04:57,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 02:04:58,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 02:04:59,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 02:05:09,820] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 02:05:10,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:05:11,184] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 02:05:26,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 02:05:27,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 02:05:28,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 02:05:28,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 02:05:38,874] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 02:10:06,180] INFO : Step 21100/50000 rank4 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6632 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1574 | Classification: 0.1277 | Classification_binary: 0.2026 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0024 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0798 | Classification_binary: 0.2038 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0741 | Regression_eta: 0.0592 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0872 | LR: 6.59e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2656s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7159 | max_loss: 0.1411 | max_backward: 1.5805 | max_optimizer_step: 0.3396 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8247.8 | uss_MB: 7584.2 | child_rss_MB: 5046.5 | allocated_MB: 343.8 | reserved_MB: 21372.0 | inactive_split_MB: 255.4 +rank4 [2026-10-29 02:10:38,255] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank4 [2026-10-29 02:10:38,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-29 02:14:15,483] INFO : Step 21200/50000 rank4 | Train Loss: 1.2354 | Train Loss Unweighted: 0.6366 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1422 | Classification: 0.1056 | Classification_binary: 0.1876 | HitGrouping: 0.2030 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0017 | Regression_pt: 0.1709 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1949 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0766 | Regression_eta: 0.0486 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0614 | LR: 6.56e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.1590s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7385 | max_loss: 0.2183 | max_backward: 1.4276 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.7483 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7137.9 | uss_MB: 6469.9 | child_rss_MB: 5019.5 | allocated_MB: 342.5 | reserved_MB: 23286.0 | inactive_split_MB: 273.0 +rank4 [2026-10-29 02:18:17,991] INFO : Step 21300/50000 rank4 | Train Loss: 1.1511 | Train Loss Unweighted: 0.5782 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1240 | Classification: 0.1021 | Classification_binary: 0.1706 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1505 | Regression_eta: 0.0017 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1895 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0756 | Regression_eta: 0.0504 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0978 | LR: 6.54e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.4911s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0290 | max_forward: 1.3238 | max_loss: 0.2105 | max_backward: 1.6672 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.6878 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7140.4 | uss_MB: 6475.6 | child_rss_MB: 5012.2 | allocated_MB: 343.8 | reserved_MB: 23286.0 | inactive_split_MB: 272.8 +rank4 [2026-10-29 02:22:14,589] INFO : Step 21400/50000 rank4 | Train Loss: 1.2993 | Train Loss Unweighted: 0.6458 | Train Loss Components: Auxiliary: 0.2724 | Cardinality: 0.1796 | Classification: 0.1204 | Classification_binary: 0.2158 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1518 | Regression_eta: 0.0018 | Regression_pt: 0.1542 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.2140 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0606 | Regression_eta: 0.0530 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0949 | LR: 6.51e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.5458s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8696 | max_loss: 0.1572 | max_backward: 1.8002 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 0.6992 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7114.7 | uss_MB: 6081.0 | child_rss_MB: 5015.7 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 201.9 +rank4 [2026-10-29 02:26:14,445] INFO : Step 21500/50000 rank4 | Train Loss: 1.2839 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2693 | Cardinality: 0.1478 | Classification: 0.1058 | Classification_binary: 0.2162 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2055 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0782 | Regression_eta: 0.0519 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0848 | LR: 6.48e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7753s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.8684 | max_loss: 0.1577 | max_backward: 1.9355 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.7593 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0648 | Memory: rss_MB: 7121.8 | uss_MB: 6458.0 | child_rss_MB: 5014.4 | allocated_MB: 341.9 | reserved_MB: 25248.0 | inactive_split_MB: 284.1 +rank4 [2026-10-29 02:30:00,713] INFO : Step 21600/50000 rank4 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1290 | Classification: 0.1062 | Classification_binary: 0.1618 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1486 | Regression_eta: 0.0018 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0594 | Regression_eta: 0.0485 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0687 | LR: 6.45e-04 | DataLoad Time: 0.0221s | Model Forward Time: 3.9644s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0061 | max_forward: 0.7663 | max_loss: 0.1883 | max_backward: 1.4257 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.4997 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7120.6 | uss_MB: 6456.9 | child_rss_MB: 5032.2 | allocated_MB: 340.5 | reserved_MB: 25248.0 | inactive_split_MB: 234.8 +rank4 [2026-10-29 02:33:57,426] INFO : Step 21700/50000 rank4 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6283 | Train Loss Components: Auxiliary: 0.2577 | Cardinality: 0.1575 | Classification: 0.1186 | Classification_binary: 0.1853 | HitGrouping: 0.2068 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1594 | Regression_eta: 0.0022 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.1837 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0811 | Regression_eta: 0.0759 | Regression_pt: 0.0866 | Regression_sin_phi: 0.1028 | LR: 6.43e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7315s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.6796 | max_loss: 0.1417 | max_backward: 1.3881 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7122.2 | uss_MB: 6458.0 | child_rss_MB: 5034.2 | allocated_MB: 343.8 | reserved_MB: 25248.0 | inactive_split_MB: 244.5 +rank4 [2026-10-29 02:37:54,439] INFO : Step 21800/50000 rank4 | Train Loss: 1.2260 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2691 | Cardinality: 0.1439 | Classification: 0.1105 | Classification_binary: 0.2043 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1576 | Regression_eta: 0.0016 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2100 | Regression_cos_phi: 0.0892 | Regression_energy: 0.0727 | Regression_eta: 0.0598 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0932 | LR: 6.40e-04 | DataLoad Time: 0.0227s | Model Forward Time: 5.9200s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 0.8043 | max_loss: 0.1895 | max_backward: 2.4027 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.9442 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7124.3 | uss_MB: 5827.3 | child_rss_MB: 5037.0 | allocated_MB: 344.5 | reserved_MB: 25248.0 | inactive_split_MB: 277.8 +rank4 [2026-10-29 02:41:42,513] INFO : Step 21900/50000 rank4 | Train Loss: 1.2119 | Train Loss Unweighted: 0.5952 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1861 | Classification: 0.1033 | Classification_binary: 0.1865 | HitGrouping: 0.1989 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0015 | Regression_pt: 0.1511 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1510 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0661 | Regression_eta: 0.0282 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0534 | LR: 6.37e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5512s | Phase Times: max_pre_step_barrier: 0.0174 | max_batch_to_device: 0.0163 | max_forward: 0.9684 | max_loss: 0.1448 | max_backward: 1.6354 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.8990 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 7124.6 | uss_MB: 6460.4 | child_rss_MB: 5033.8 | allocated_MB: 342.7 | reserved_MB: 25248.0 | inactive_split_MB: 192.3 +rank4 [2026-10-29 02:45:41,732] INFO : Step 22000/50000 rank4 | Train Loss: 1.1094 | Train Loss Unweighted: 0.5651 | Train Loss Components: Auxiliary: 0.2344 | Cardinality: 0.1248 | Classification: 0.1037 | Classification_binary: 0.1579 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1497 | Regression_eta: 0.0017 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.1455 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0684 | Regression_eta: 0.0404 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0539 | LR: 6.34e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.4205s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.6825 | max_loss: 0.1653 | max_backward: 1.4396 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.0256 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 7124.1 | uss_MB: 6463.7 | child_rss_MB: 5009.1 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 247.3 +rank4 [2026-10-29 02:45:42,811] INFO : Running validation on rank4 +rank4 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 02:46:37,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:46:38,046] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 02:46:57,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 02:46:58,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 02:46:59,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 02:47:00,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 02:47:10,454] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 02:47:11,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:47:11,959] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 02:47:28,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 02:47:29,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 02:47:30,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 02:47:31,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 02:47:41,180] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 02:47:41,460] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:47:42,030] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 02:47:57,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 02:47:58,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 02:47:58,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 02:47:59,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 02:48:09,654] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 02:52:24,844] INFO : Step 22100/50000 rank4 | Train Loss: 1.2514 | Train Loss Unweighted: 0.6219 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1530 | Classification: 0.1140 | Classification_binary: 0.2076 | HitGrouping: 0.2031 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1470 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.2177 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0672 | Regression_eta: 0.0500 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0723 | LR: 6.32e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5396s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7250 | max_loss: 0.2514 | max_backward: 1.5956 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.8632 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0587 | Memory: rss_MB: 8032.9 | uss_MB: 7365.3 | child_rss_MB: 5019.4 | allocated_MB: 347.3 | reserved_MB: 23386.0 | inactive_split_MB: 278.6 +rank4 [2026-10-29 02:56:29,084] INFO : Step 22200/50000 rank4 | Train Loss: 1.2727 | Train Loss Unweighted: 0.6485 | Train Loss Components: Auxiliary: 0.2709 | Cardinality: 0.1498 | Classification: 0.1104 | Classification_binary: 0.2184 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1574 | Regression_eta: 0.0018 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2258 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0762 | Regression_eta: 0.0475 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0566 | LR: 6.29e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.9314s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0074 | max_forward: 0.7776 | max_loss: 0.1472 | max_backward: 1.7604 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.3305 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7158.4 | uss_MB: 5092.5 | child_rss_MB: 5016.1 | allocated_MB: 342.3 | reserved_MB: 23578.0 | inactive_split_MB: 277.9 +rank4 [2026-10-29 03:00:35,125] INFO : Step 22300/50000 rank4 | Train Loss: 1.1242 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2347 | Cardinality: 0.1367 | Classification: 0.1013 | Classification_binary: 0.1688 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1730 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0693 | Regression_eta: 0.0630 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0622 | LR: 6.26e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.7793s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7697 | max_loss: 0.1538 | max_backward: 1.3553 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.4896 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 7162.9 | uss_MB: 6498.9 | child_rss_MB: 4987.3 | allocated_MB: 341.2 | reserved_MB: 23578.0 | inactive_split_MB: 233.8 +rank4 [2026-10-29 03:04:32,773] INFO : Step 22400/50000 rank4 | Train Loss: 1.2357 | Train Loss Unweighted: 0.6255 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1493 | Classification: 0.1171 | Classification_binary: 0.1885 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1575 | Regression_eta: 0.0021 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0797 | Classification_binary: 0.1809 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0762 | Regression_eta: 0.0447 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0587 | LR: 6.23e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9150s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0067 | max_forward: 0.6425 | max_loss: 0.1530 | max_backward: 1.4136 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.5876 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 7158.4 | uss_MB: 6496.8 | child_rss_MB: 5021.9 | allocated_MB: 342.5 | reserved_MB: 23578.0 | inactive_split_MB: 251.5 +rank4 [2026-10-29 03:08:26,684] INFO : Step 22500/50000 rank4 | Train Loss: 1.2633 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2638 | Cardinality: 0.1676 | Classification: 0.0954 | Classification_binary: 0.2181 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0020 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2390 | Regression_cos_phi: 0.1089 | Regression_energy: 0.0707 | Regression_eta: 0.0585 | Regression_pt: 0.0729 | Regression_sin_phi: 0.1049 | LR: 6.20e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4303s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1522 | max_backward: 1.5631 | max_optimizer_step: 0.3261 | max_tensorboard_logging: 0.7552 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 7160.2 | uss_MB: 6502.4 | child_rss_MB: 5011.7 | allocated_MB: 342.9 | reserved_MB: 24052.0 | inactive_split_MB: 295.1 +rank4 [2026-10-29 03:12:18,755] INFO : Step 22600/50000 rank4 | Train Loss: 1.1636 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1329 | Classification: 0.0997 | Classification_binary: 0.1889 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1493 | Regression_eta: 0.0020 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0813 | Regression_eta: 0.0919 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0600 | LR: 6.18e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9329s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7375 | max_loss: 0.2403 | max_backward: 1.4584 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7163.5 | uss_MB: 6503.3 | child_rss_MB: 5024.4 | allocated_MB: 344.2 | reserved_MB: 24590.0 | inactive_split_MB: 288.5 +rank4 [2026-10-29 03:16:20,064] INFO : Step 22700/50000 rank4 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1212 | Classification: 0.0976 | Classification_binary: 0.1566 | HitGrouping: 0.2101 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1455 | Regression_eta: 0.0015 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1326 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0599 | Regression_eta: 0.0260 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0513 | LR: 6.15e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.9687s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.6124 | max_loss: 0.1376 | max_backward: 1.1766 | max_optimizer_step: 0.2237 | max_tensorboard_logging: 3.8568 | max_tensorboard_flush: 0.0169 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 7164.2 | uss_MB: 6160.1 | child_rss_MB: 5007.8 | allocated_MB: 341.6 | reserved_MB: 25058.0 | inactive_split_MB: 256.4 +rank4 [2026-10-29 03:20:29,740] INFO : Step 22800/50000 rank4 | Train Loss: 1.1840 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2533 | Cardinality: 0.1443 | Classification: 0.1191 | Classification_binary: 0.1924 | HitGrouping: 0.1959 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1444 | Regression_eta: 0.0016 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0866 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0716 | Regression_eta: 0.0452 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0919 | LR: 6.12e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4526s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7010 | max_loss: 0.2312 | max_backward: 1.6853 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8726 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 7164.4 | uss_MB: 6502.8 | child_rss_MB: 5021.2 | allocated_MB: 341.7 | reserved_MB: 25058.0 | inactive_split_MB: 281.3 +rank4 [2026-10-29 03:24:21,161] INFO : Step 22900/50000 rank4 | Train Loss: 1.2680 | Train Loss Unweighted: 0.6318 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1792 | Classification: 0.1020 | Classification_binary: 0.2074 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1580 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1809 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0708 | Regression_eta: 0.0493 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0510 | LR: 6.09e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.0018s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 1.1637 | max_loss: 0.4031 | max_backward: 1.8442 | max_optimizer_step: 0.3320 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7165.6 | uss_MB: 6506.2 | child_rss_MB: 5023.4 | allocated_MB: 344.8 | reserved_MB: 25058.0 | inactive_split_MB: 300.1 +rank4 [2026-10-29 03:28:05,593] INFO : Step 23000/50000 rank4 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5484 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1129 | Classification: 0.0836 | Classification_binary: 0.1519 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1547 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1345 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0652 | Regression_eta: 0.0436 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0477 | LR: 6.06e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7525s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0063 | max_forward: 0.6963 | max_loss: 0.2382 | max_backward: 1.2870 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 7162.6 | uss_MB: 6506.3 | child_rss_MB: 5008.7 | allocated_MB: 340.3 | reserved_MB: 26066.0 | inactive_split_MB: 218.4 +rank4 [2026-10-29 03:28:06,534] INFO : Running validation on rank4 +rank4 [2026-10-29 03:28:06,534] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 03:28:06,534] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 03:28:56,134] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:28:56,639] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 03:29:22,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 03:29:22,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 03:29:23,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 03:29:25,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 03:29:35,097] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 03:29:35,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:29:36,300] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 03:29:52,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 03:29:53,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 03:29:53,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 03:29:54,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 03:30:04,925] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 03:30:05,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:30:05,703] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 03:30:20,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 03:30:21,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 03:30:22,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 03:30:22,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 03:30:32,853] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 03:34:45,381] INFO : Step 23100/50000 rank4 | Train Loss: 1.2422 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1379 | Classification: 0.1246 | Classification_binary: 0.1795 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1482 | Regression_eta: 0.0023 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.1822 | Regression_cos_phi: 0.1339 | Regression_energy: 0.0792 | Regression_eta: 0.0621 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0951 | LR: 6.04e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8243s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7203 | max_loss: 0.1522 | max_backward: 1.1000 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.6183 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 8419.6 | uss_MB: 7758.7 | child_rss_MB: 5039.2 | allocated_MB: 344.7 | reserved_MB: 22598.0 | inactive_split_MB: 244.5 +rank4 [2026-10-29 03:38:33,524] INFO : Step 23200/50000 rank4 | Train Loss: 1.2507 | Train Loss Unweighted: 0.6359 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1400 | Classification: 0.1121 | Classification_binary: 0.2208 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1488 | Regression_eta: 0.0020 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.2187 | Regression_cos_phi: 0.0948 | Regression_energy: 0.0742 | Regression_eta: 0.0617 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0719 | LR: 6.01e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.5956s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0727 | max_forward: 0.9486 | max_loss: 0.1744 | max_backward: 1.7031 | max_optimizer_step: 0.2405 | max_tensorboard_logging: 0.6717 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1109 | Memory: rss_MB: 7225.3 | uss_MB: 6564.6 | child_rss_MB: 5046.8 | allocated_MB: 343.7 | reserved_MB: 23792.0 | inactive_split_MB: 223.2 +rank4 [2026-10-29 03:42:24,564] INFO : Step 23300/50000 rank4 | Train Loss: 1.1880 | Train Loss Unweighted: 0.5927 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1464 | Classification: 0.0987 | Classification_binary: 0.1851 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1524 | Regression_eta: 0.0019 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1545 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0633 | Regression_eta: 0.0528 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0479 | LR: 5.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8603s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7557 | max_loss: 0.1467 | max_backward: 1.4708 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.4854 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 7231.3 | uss_MB: 6572.4 | child_rss_MB: 5045.1 | allocated_MB: 342.6 | reserved_MB: 24534.0 | inactive_split_MB: 271.6 +rank4 [2026-10-29 03:46:11,297] INFO : Step 23400/50000 rank4 | Train Loss: 1.1136 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2402 | Cardinality: 0.1202 | Classification: 0.1070 | Classification_binary: 0.1497 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1644 | Regression_eta: 0.0017 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1395 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0751 | Regression_eta: 0.0329 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0643 | LR: 5.95e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.6668s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.5743 | max_loss: 0.2297 | max_backward: 1.0127 | max_optimizer_step: 0.3321 | max_tensorboard_logging: 0.6182 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 7232.3 | uss_MB: 6571.7 | child_rss_MB: 5046.9 | allocated_MB: 340.5 | reserved_MB: 24848.0 | inactive_split_MB: 241.3 +rank4 [2026-10-29 03:50:07,300] INFO : Step 23500/50000 rank4 | Train Loss: 1.1601 | Train Loss Unweighted: 0.5799 | Train Loss Components: Auxiliary: 0.2477 | Cardinality: 0.1161 | Classification: 0.1057 | Classification_binary: 0.1852 | HitGrouping: 0.2029 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1416 | Regression_eta: 0.0017 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.1825 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0646 | Regression_eta: 0.0471 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0573 | LR: 5.92e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4409s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8864 | max_loss: 0.1573 | max_backward: 1.9036 | max_optimizer_step: 0.2294 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7232.1 | uss_MB: 6574.4 | child_rss_MB: 5036.9 | allocated_MB: 341.9 | reserved_MB: 24848.0 | inactive_split_MB: 233.9 +rank4 [2026-10-29 03:53:52,162] INFO : Step 23600/50000 rank4 | Train Loss: 1.2402 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1447 | Classification: 0.1136 | Classification_binary: 0.2049 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1534 | Regression_eta: 0.0020 | Regression_pt: 0.1550 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2114 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0729 | Regression_eta: 0.0540 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0749 | LR: 5.90e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9419s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8397 | max_loss: 0.1470 | max_backward: 1.4395 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7234.1 | uss_MB: 6575.7 | child_rss_MB: 5045.6 | allocated_MB: 342.5 | reserved_MB: 24848.0 | inactive_split_MB: 215.5 +rank4 [2026-10-29 03:57:35,779] INFO : Step 23700/50000 rank4 | Train Loss: 1.1009 | Train Loss Unweighted: 0.5682 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1218 | Classification: 0.0954 | Classification_binary: 0.1599 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1528 | Regression_eta: 0.0014 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1853 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0877 | Regression_eta: 0.0482 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0615 | LR: 5.87e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.6819s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6589 | max_loss: 0.1387 | max_backward: 1.2152 | max_optimizer_step: 0.2251 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7230.0 | uss_MB: 6574.3 | child_rss_MB: 5051.4 | allocated_MB: 342.5 | reserved_MB: 25508.0 | inactive_split_MB: 226.3 +rank4 [2026-10-29 04:01:35,999] INFO : Step 23800/50000 rank4 | Train Loss: 1.1390 | Train Loss Unweighted: 0.5882 | Train Loss Components: Auxiliary: 0.2427 | Cardinality: 0.1196 | Classification: 0.1035 | Classification_binary: 0.1735 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1745 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0850 | Regression_eta: 0.0603 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0506 | LR: 5.84e-04 | DataLoad Time: 0.0221s | Model Forward Time: 3.7616s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.7283 | max_loss: 0.1547 | max_backward: 1.2415 | max_optimizer_step: 0.2259 | max_tensorboard_logging: 0.5270 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7231.0 | uss_MB: 6110.2 | child_rss_MB: 5050.1 | allocated_MB: 345.2 | reserved_MB: 25508.0 | inactive_split_MB: 241.8 +rank4 [2026-10-29 04:05:42,354] INFO : Step 23900/50000 rank4 | Train Loss: 1.2349 | Train Loss Unweighted: 0.6078 | Train Loss Components: Auxiliary: 0.2621 | Cardinality: 0.1596 | Classification: 0.1044 | Classification_binary: 0.2077 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1451 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.2057 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0698 | Regression_eta: 0.0486 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0904 | LR: 5.81e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.4882s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.9077 | max_loss: 0.1538 | max_backward: 1.9088 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7234.7 | uss_MB: 6579.6 | child_rss_MB: 5044.4 | allocated_MB: 342.1 | reserved_MB: 27474.0 | inactive_split_MB: 282.4 +rank4 [2026-10-29 04:09:40,154] INFO : Step 24000/50000 rank4 | Train Loss: 1.1074 | Train Loss Unweighted: 0.5714 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1170 | Classification: 0.1007 | Classification_binary: 0.1792 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1431 | Regression_eta: 0.0015 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0676 | Regression_eta: 0.0392 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0470 | LR: 5.78e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.2188s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 1.1428 | max_loss: 0.2613 | max_backward: 2.5767 | max_optimizer_step: 0.2298 | max_tensorboard_logging: 0.5998 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7236.7 | uss_MB: 6577.2 | child_rss_MB: 5038.1 | allocated_MB: 342.1 | reserved_MB: 27476.0 | inactive_split_MB: 218.6 +rank4 [2026-10-29 04:09:41,046] INFO : Running validation on rank4 +rank4 [2026-10-29 04:09:41,046] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 04:09:41,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 04:10:38,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:10:39,334] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 04:11:04,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 04:11:05,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 04:11:06,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 04:11:10,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 04:11:20,175] INFO : Time taken to make predictions on device 4 is: 0.28 min +rank4 [2026-10-29 04:11:20,671] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:11:21,254] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 04:11:36,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 04:11:37,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 04:11:38,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 04:11:39,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 04:11:49,516] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 04:11:50,532] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:11:51,026] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 04:12:06,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 04:12:07,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 04:12:08,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 04:12:08,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 04:12:18,748] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 04:16:38,288] INFO : Step 24100/50000 rank4 | Train Loss: 1.1244 | Train Loss Unweighted: 0.5584 | Train Loss Components: Auxiliary: 0.2373 | Cardinality: 0.1285 | Classification: 0.0978 | Classification_binary: 0.1606 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0599 | Regression_eta: 0.0496 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0499 | LR: 5.75e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1999s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0860 | max_forward: 0.7361 | max_loss: 0.1432 | max_backward: 1.4268 | max_optimizer_step: 0.3451 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 8228.7 | uss_MB: 7570.1 | child_rss_MB: 5001.0 | allocated_MB: 343.2 | reserved_MB: 21608.0 | inactive_split_MB: 269.9 +rank4 [2026-10-29 04:20:36,776] INFO : Step 24200/50000 rank4 | Train Loss: 1.3067 | Train Loss Unweighted: 0.6341 | Train Loss Components: Auxiliary: 0.2739 | Cardinality: 0.1760 | Classification: 0.1145 | Classification_binary: 0.2101 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0018 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1933 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0737 | Regression_eta: 0.0429 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0731 | LR: 5.73e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.0800s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1066 | max_forward: 1.0531 | max_loss: 0.2796 | max_backward: 2.3820 | max_optimizer_step: 0.2441 | max_tensorboard_logging: 0.6187 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 8233.6 | uss_MB: 7574.6 | child_rss_MB: 5010.3 | allocated_MB: 345.0 | reserved_MB: 23864.0 | inactive_split_MB: 203.0 +rank4 [2026-10-29 04:24:33,011] INFO : Step 24300/50000 rank4 | Train Loss: 1.2207 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1582 | Classification: 0.1151 | Classification_binary: 0.2013 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1450 | Regression_eta: 0.0017 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0620 | Regression_eta: 0.0456 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0566 | LR: 5.70e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3064s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7395 | max_loss: 0.2490 | max_backward: 1.4969 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.6555 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7355.3 | uss_MB: 6697.1 | child_rss_MB: 5007.1 | allocated_MB: 344.8 | reserved_MB: 23864.0 | inactive_split_MB: 284.8 +rank4 [2026-10-29 04:28:19,172] INFO : Step 24400/50000 rank4 | Train Loss: 1.1288 | Train Loss Unweighted: 0.5921 | Train Loss Components: Auxiliary: 0.2439 | Cardinality: 0.1229 | Classification: 0.1017 | Classification_binary: 0.1613 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1631 | Regression_eta: 0.0015 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0813 | Classification_binary: 0.1783 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0838 | Regression_eta: 0.0673 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0822 | LR: 5.67e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.9874s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.6685 | max_loss: 0.1564 | max_backward: 1.4736 | max_optimizer_step: 0.2223 | max_tensorboard_logging: 0.5369 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7357.1 | uss_MB: 6698.7 | child_rss_MB: 5016.5 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 267.2 +rank4 [2026-10-29 04:32:03,440] INFO : Step 24500/50000 rank4 | Train Loss: 1.2319 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2586 | Cardinality: 0.1511 | Classification: 0.1107 | Classification_binary: 0.1876 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1540 | Regression_eta: 0.0018 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0810 | Regression_eta: 0.0457 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0579 | LR: 5.64e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1614s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0677 | max_forward: 0.7219 | max_loss: 0.1439 | max_backward: 1.4253 | max_optimizer_step: 0.3416 | max_tensorboard_logging: 0.5859 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 7356.5 | uss_MB: 6698.2 | child_rss_MB: 5010.5 | allocated_MB: 343.1 | reserved_MB: 23864.0 | inactive_split_MB: 275.8 +rank4 [2026-10-29 04:36:01,552] INFO : Step 24600/50000 rank4 | Train Loss: 1.2474 | Train Loss Unweighted: 0.6241 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1630 | Classification: 0.1074 | Classification_binary: 0.2102 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0840 | Regression_eta: 0.0340 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0666 | LR: 5.61e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5133s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7876 | max_loss: 0.1528 | max_backward: 1.7437 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.6204 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7358.2 | uss_MB: 6700.7 | child_rss_MB: 5006.8 | allocated_MB: 344.3 | reserved_MB: 23864.0 | inactive_split_MB: 285.7 +rank4 [2026-10-29 04:39:51,298] INFO : Step 24700/50000 rank4 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1193 | Classification: 0.0990 | Classification_binary: 0.1723 | HitGrouping: 0.2019 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1471 | Regression_eta: 0.0016 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2163 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0625 | Regression_eta: 0.0499 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0926 | LR: 5.58e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9517s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6978 | max_loss: 0.2493 | max_backward: 1.2925 | max_optimizer_step: 0.2314 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7361.3 | uss_MB: 6700.3 | child_rss_MB: 5009.8 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 223.9 +rank4 [2026-10-29 04:43:37,808] INFO : Step 24800/50000 rank4 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5554 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1264 | Classification: 0.0932 | Classification_binary: 0.1583 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0726 | Regression_eta: 0.0520 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0706 | LR: 5.56e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.2773s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6582 | max_loss: 0.2533 | max_backward: 1.5858 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.6923 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1086 | Memory: rss_MB: 7365.2 | uss_MB: 6703.3 | child_rss_MB: 5005.7 | allocated_MB: 342.9 | reserved_MB: 23864.0 | inactive_split_MB: 278.6 +rank4 [2026-10-29 04:47:36,372] INFO : Step 24900/50000 rank4 | Train Loss: 1.1348 | Train Loss Unweighted: 0.5816 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1381 | Classification: 0.1126 | Classification_binary: 0.1845 | HitGrouping: 0.1937 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1403 | Regression_eta: 0.0015 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0741 | Regression_eta: 0.0459 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0599 | LR: 5.53e-04 | DataLoad Time: 0.0154s | Model Forward Time: 8.7035s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0067 | max_forward: 0.7276 | max_loss: 0.2411 | max_backward: 1.3958 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 5.1754 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1038 | Memory: rss_MB: 7366.2 | uss_MB: 6703.5 | child_rss_MB: 5007.2 | allocated_MB: 343.9 | reserved_MB: 23864.0 | inactive_split_MB: 239.7 +rank4 [2026-10-29 04:51:28,372] INFO : Step 25000/50000 rank4 | Train Loss: 1.1995 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1590 | Classification: 0.0891 | Classification_binary: 0.2009 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1421 | Regression_eta: 0.0018 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2123 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0709 | Regression_eta: 0.0496 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0624 | LR: 5.50e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.2043s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7692 | max_loss: 0.2091 | max_backward: 1.3616 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6652 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1392 | Memory: rss_MB: 7366.4 | uss_MB: 6702.5 | child_rss_MB: 5014.2 | allocated_MB: 343.5 | reserved_MB: 23864.0 | inactive_split_MB: 293.8 +rank4 [2026-10-29 04:51:29,310] INFO : Running validation on rank4 +rank4 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 04:52:22,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:52:22,822] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 04:52:48,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 04:52:49,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 04:52:50,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 04:52:52,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 04:53:02,112] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 04:53:02,764] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:53:03,283] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 04:53:19,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 04:53:20,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 04:53:21,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 04:53:22,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 04:53:32,440] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 04:53:32,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:53:32,991] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 04:53:48,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 04:53:49,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 04:53:49,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 04:53:50,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 04:54:00,484] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 04:58:07,811] INFO : Step 25100/50000 rank4 | Train Loss: 1.0833 | Train Loss Unweighted: 0.5514 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1317 | Classification: 0.0920 | Classification_binary: 0.1568 | HitGrouping: 0.2008 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1491 | Regression_eta: 0.0016 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0255 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0986 | Regression_eta: 0.0477 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0549 | LR: 5.47e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7882s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7295 | max_loss: 0.1401 | max_backward: 1.2439 | max_optimizer_step: 0.2490 | max_tensorboard_logging: 0.4940 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8607.6 | uss_MB: 7944.9 | child_rss_MB: 5042.3 | allocated_MB: 342.3 | reserved_MB: 23608.0 | inactive_split_MB: 222.3 +rank4 [2026-10-29 05:01:53,212] INFO : Step 25200/50000 rank4 | Train Loss: 1.1886 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1523 | Classification: 0.1158 | Classification_binary: 0.1800 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1512 | Regression_eta: 0.0016 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0730 | Regression_eta: 0.0380 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0505 | LR: 5.44e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7328s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8134 | max_loss: 0.1416 | max_backward: 1.5095 | max_optimizer_step: 0.3471 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 8606.1 | uss_MB: 7945.9 | child_rss_MB: 5039.5 | allocated_MB: 343.6 | reserved_MB: 24590.0 | inactive_split_MB: 260.9 +rank4 [2026-10-29 05:05:52,267] INFO : Step 25300/50000 rank4 | Train Loss: 1.2031 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2624 | Cardinality: 0.1455 | Classification: 0.1145 | Classification_binary: 0.1966 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1448 | Regression_eta: 0.0016 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.1909 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0714 | Regression_eta: 0.0396 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0582 | LR: 5.42e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5361s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.8219 | max_loss: 0.1491 | max_backward: 1.6406 | max_optimizer_step: 0.2735 | max_tensorboard_logging: 0.7204 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0488 | Memory: rss_MB: 7456.6 | uss_MB: 6796.5 | child_rss_MB: 5036.7 | allocated_MB: 341.8 | reserved_MB: 25572.0 | inactive_split_MB: 253.5 +rank4 [2026-10-29 05:09:44,971] INFO : Step 25400/50000 rank4 | Train Loss: 1.1194 | Train Loss Unweighted: 0.5655 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1346 | Classification: 0.0983 | Classification_binary: 0.1754 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1438 | Regression_eta: 0.0015 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1745 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0807 | Regression_eta: 0.0451 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0559 | LR: 5.39e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1080s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.6986 | max_loss: 0.1530 | max_backward: 1.2606 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.8427 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7454.9 | uss_MB: 6795.9 | child_rss_MB: 5040.6 | allocated_MB: 345.2 | reserved_MB: 25572.0 | inactive_split_MB: 223.8 +rank4 [2026-10-29 05:13:34,054] INFO : Step 25500/50000 rank4 | Train Loss: 1.0289 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1068 | Classification: 0.1034 | Classification_binary: 0.1444 | HitGrouping: 0.1913 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1484 | Regression_eta: 0.0014 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0843 | Regression_eta: 0.0489 | Regression_pt: 0.0875 | Regression_sin_phi: 0.1238 | LR: 5.36e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.1588s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0554 | max_forward: 0.7008 | max_loss: 0.1433 | max_backward: 1.4571 | max_optimizer_step: 0.3349 | max_tensorboard_logging: 0.6083 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 7460.4 | uss_MB: 6802.0 | child_rss_MB: 5038.5 | allocated_MB: 345.4 | reserved_MB: 25572.0 | inactive_split_MB: 240.6 +rank4 [2026-10-29 05:17:32,215] INFO : Step 25600/50000 rank4 | Train Loss: 1.2028 | Train Loss Unweighted: 0.5968 | Train Loss Components: Auxiliary: 0.2590 | Cardinality: 0.1607 | Classification: 0.1135 | Classification_binary: 0.1926 | HitGrouping: 0.1978 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1429 | Regression_eta: 0.0016 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0642 | Regression_eta: 0.0373 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0684 | LR: 5.33e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6833s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8800 | max_loss: 0.1578 | max_backward: 1.8564 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 7459.9 | uss_MB: 6802.4 | child_rss_MB: 5035.4 | allocated_MB: 345.5 | reserved_MB: 26054.0 | inactive_split_MB: 284.5 +rank4 [2026-10-29 05:21:26,002] INFO : Step 25700/50000 rank4 | Train Loss: 1.1800 | Train Loss Unweighted: 0.5964 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1298 | Classification: 0.1086 | Classification_binary: 0.1961 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0019 | Regression_pt: 0.1444 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1864 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0684 | Regression_eta: 0.0302 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0671 | LR: 5.30e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.5215s | Phase Times: max_pre_step_barrier: 0.0086 | max_batch_to_device: 0.0137 | max_forward: 1.0340 | max_loss: 0.2483 | max_backward: 1.6704 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 0.7612 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7459.7 | uss_MB: 6801.4 | child_rss_MB: 5033.4 | allocated_MB: 343.5 | reserved_MB: 26054.0 | inactive_split_MB: 270.8 +rank4 [2026-10-29 05:25:16,515] INFO : Step 25800/50000 rank4 | Train Loss: 1.1337 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2388 | Cardinality: 0.1264 | Classification: 0.0898 | Classification_binary: 0.1617 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0019 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0629 | Regression_eta: 0.0445 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0494 | LR: 5.27e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9358s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0986 | max_forward: 0.8129 | max_loss: 0.1662 | max_backward: 1.2979 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1103 | Memory: rss_MB: 7457.8 | uss_MB: 6799.8 | child_rss_MB: 5030.2 | allocated_MB: 341.8 | reserved_MB: 26054.0 | inactive_split_MB: 273.2 +rank4 [2026-10-29 05:29:08,212] INFO : Step 25900/50000 rank4 | Train Loss: 1.1019 | Train Loss Unweighted: 0.5481 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1165 | Classification: 0.0977 | Classification_binary: 0.1594 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1434 | Regression_eta: 0.0016 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0676 | Regression_eta: 0.0423 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0573 | LR: 5.25e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5207s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7034 | max_loss: 0.2469 | max_backward: 1.7130 | max_optimizer_step: 0.2363 | max_tensorboard_logging: 0.8382 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 7454.8 | uss_MB: 6800.6 | child_rss_MB: 5037.1 | allocated_MB: 343.0 | reserved_MB: 26054.0 | inactive_split_MB: 187.0 +rank4 [2026-10-29 05:33:07,469] INFO : Step 26000/50000 rank4 | Train Loss: 1.1708 | Train Loss Unweighted: 0.5680 | Train Loss Components: Auxiliary: 0.2498 | Cardinality: 0.1352 | Classification: 0.0999 | Classification_binary: 0.2021 | HitGrouping: 0.1994 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1304 | Regression_eta: 0.0021 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1777 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0586 | Regression_eta: 0.0542 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0551 | LR: 5.22e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2255s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.7877 | max_loss: 0.1637 | max_backward: 1.6550 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.4954 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 7454.4 | uss_MB: 6802.1 | child_rss_MB: 5032.6 | allocated_MB: 345.0 | reserved_MB: 27098.0 | inactive_split_MB: 285.6 +rank4 [2026-10-29 05:33:08,394] INFO : Running validation on rank4 +rank4 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 05:33:56,766] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:33:57,245] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 05:34:24,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 05:34:25,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 05:34:26,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 05:34:27,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 05:34:37,354] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 05:34:38,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:34:38,538] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 05:34:54,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 05:34:55,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 05:34:56,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 05:34:57,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 05:35:07,474] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 05:35:08,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:35:08,510] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 05:35:24,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 05:35:24,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 05:35:25,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 05:35:26,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 05:35:36,084] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 05:39:49,627] INFO : Step 26100/50000 rank4 | Train Loss: 1.0974 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2307 | Cardinality: 0.1119 | Classification: 0.0956 | Classification_binary: 0.1689 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1513 | Regression_eta: 0.0014 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1794 | Regression_cos_phi: 0.1023 | Regression_energy: 0.0823 | Regression_eta: 0.0351 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0695 | LR: 5.19e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1602s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0117 | max_forward: 0.7200 | max_loss: 0.1501 | max_backward: 1.2286 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.8284 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 8630.6 | uss_MB: 7973.7 | child_rss_MB: 5008.3 | allocated_MB: 340.1 | reserved_MB: 21762.0 | inactive_split_MB: 215.7 +rank4 [2026-10-29 05:43:56,539] INFO : Step 26200/50000 rank4 | Train Loss: 1.0876 | Train Loss Unweighted: 0.5502 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1065 | Classification: 0.0968 | Classification_binary: 0.1527 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1476 | Regression_eta: 0.0018 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0401 | Regression_energy: 0.0782 | Regression_eta: 0.0342 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0488 | LR: 5.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3254s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6530 | max_loss: 0.1421 | max_backward: 1.3973 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.8695 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8630.6 | uss_MB: 7973.8 | child_rss_MB: 5031.7 | allocated_MB: 342.3 | reserved_MB: 22678.0 | inactive_split_MB: 242.5 +rank4 [2026-10-29 05:47:48,031] INFO : Step 26300/50000 rank4 | Train Loss: 1.2051 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1512 | Classification: 0.1193 | Classification_binary: 0.1877 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1458 | Regression_eta: 0.0017 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1891 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0754 | Regression_eta: 0.0702 | Regression_pt: 0.0806 | Regression_sin_phi: 0.1015 | LR: 5.13e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4691s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0072 | max_forward: 0.7601 | max_loss: 0.2267 | max_backward: 1.8902 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7522.0 | uss_MB: 6871.8 | child_rss_MB: 4996.1 | allocated_MB: 345.4 | reserved_MB: 22678.0 | inactive_split_MB: 270.9 +rank4 [2026-10-29 05:51:47,278] INFO : Step 26400/50000 rank4 | Train Loss: 1.2256 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1648 | Classification: 0.1020 | Classification_binary: 0.1981 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1520 | Regression_eta: 0.0015 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1868 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0827 | Regression_eta: 0.0323 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0710 | LR: 5.10e-04 | DataLoad Time: 0.0130s | Model Forward Time: 4.3878s | Phase Times: max_pre_step_barrier: 0.0351 | max_batch_to_device: 0.0070 | max_forward: 0.8129 | max_loss: 0.1477 | max_backward: 1.4015 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8384 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 7520.2 | uss_MB: 6869.2 | child_rss_MB: 4985.0 | allocated_MB: 342.8 | reserved_MB: 22678.0 | inactive_split_MB: 196.9 +rank4 [2026-10-29 05:55:40,759] INFO : Step 26500/50000 rank4 | Train Loss: 1.1110 | Train Loss Unweighted: 0.5356 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1488 | Classification: 0.0891 | Classification_binary: 0.1546 | HitGrouping: 0.2012 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0659 | Regression_eta: 0.0476 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0698 | LR: 5.08e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0226s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6841 | max_loss: 0.2421 | max_backward: 1.1616 | max_optimizer_step: 0.2383 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7505.4 | uss_MB: 6872.6 | child_rss_MB: 4957.6 | allocated_MB: 340.6 | reserved_MB: 22678.0 | inactive_split_MB: 269.9 +rank4 [2026-10-29 05:59:33,068] INFO : Step 26600/50000 rank4 | Train Loss: 1.0895 | Train Loss Unweighted: 0.5529 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1261 | Classification: 0.1083 | Classification_binary: 0.1603 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1386 | Regression_eta: 0.0015 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0750 | Regression_eta: 0.0349 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0453 | LR: 5.05e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.4528s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0696 | max_forward: 0.6926 | max_loss: 0.1459 | max_backward: 1.4149 | max_optimizer_step: 0.2385 | max_tensorboard_logging: 0.8927 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7477.7 | uss_MB: 6873.1 | child_rss_MB: 4831.1 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 232.0 +rank4 [2026-10-29 06:03:23,764] INFO : Step 26700/50000 rank4 | Train Loss: 1.2592 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1717 | Classification: 0.1100 | Classification_binary: 0.2137 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1420 | Regression_eta: 0.0017 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1895 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0803 | Regression_eta: 0.0353 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0640 | LR: 5.02e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.6758s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7750 | max_loss: 0.1526 | max_backward: 1.6427 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.8866 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7338.5 | uss_MB: 6135.7 | child_rss_MB: 4788.2 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 267.4 +rank4 [2026-10-29 06:07:16,644] INFO : Step 26800/50000 rank4 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5792 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1288 | Classification: 0.0911 | Classification_binary: 0.1633 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1600 | Regression_eta: 0.0015 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0772 | Regression_eta: 0.0534 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0410 | LR: 4.99e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0415s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7108 | max_loss: 0.1467 | max_backward: 1.2855 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7173 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 7253.0 | uss_MB: 6872.2 | child_rss_MB: 4809.0 | allocated_MB: 340.8 | reserved_MB: 22846.0 | inactive_split_MB: 253.2 +rank4 [2026-10-29 06:11:09,652] INFO : Step 26900/50000 rank4 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.0996 | Classification: 0.0867 | Classification_binary: 0.1428 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1483 | Regression_eta: 0.0018 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0494 | Regression_eta: 0.0228 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0328 | LR: 4.96e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8398s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0336 | max_forward: 0.6534 | max_loss: 0.1408 | max_backward: 1.3898 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0912 | Memory: rss_MB: 7214.0 | uss_MB: 6872.7 | child_rss_MB: 4790.0 | allocated_MB: 342.5 | reserved_MB: 22846.0 | inactive_split_MB: 233.5 +rank4 [2026-10-29 06:15:05,368] INFO : Step 27000/50000 rank4 | Train Loss: 1.0945 | Train Loss Unweighted: 0.5396 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1196 | Classification: 0.0957 | Classification_binary: 0.1912 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1247 | Regression_eta: 0.0015 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1927 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0582 | Regression_eta: 0.0394 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0656 | LR: 4.94e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1536s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7531 | max_loss: 0.1689 | max_backward: 1.5554 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6017 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7214.1 | uss_MB: 6877.0 | child_rss_MB: 4792.2 | allocated_MB: 345.3 | reserved_MB: 23694.0 | inactive_split_MB: 251.5 +rank4 [2026-10-29 06:15:06,391] INFO : Running validation on rank4 +rank4 [2026-10-29 06:15:06,391] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 06:15:06,391] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 06:16:00,132] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:16:00,714] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 06:16:29,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 06:16:30,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 06:16:31,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 06:16:32,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 06:16:42,564] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 06:16:43,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:16:43,720] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 06:16:59,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 06:17:00,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 06:17:01,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 06:17:02,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 06:17:12,549] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 06:17:13,249] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:17:13,778] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 06:17:29,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 06:17:30,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 06:17:30,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 06:17:31,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 06:17:41,651] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 06:21:51,977] INFO : Step 27100/50000 rank4 | Train Loss: 1.1577 | Train Loss Unweighted: 0.5595 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1497 | Classification: 0.0884 | Classification_binary: 0.1879 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1399 | Regression_eta: 0.0018 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1712 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0683 | Regression_eta: 0.0433 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0517 | LR: 4.91e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.0528s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8543 | max_loss: 0.1477 | max_backward: 1.8762 | max_optimizer_step: 0.2540 | max_tensorboard_logging: 0.9462 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 8306.7 | uss_MB: 7966.7 | child_rss_MB: 4814.1 | allocated_MB: 342.9 | reserved_MB: 21640.0 | inactive_split_MB: 268.0 +rank4 [2026-10-29 06:25:39,973] INFO : Step 27200/50000 rank4 | Train Loss: 1.0028 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1065 | Classification: 0.0834 | Classification_binary: 0.1354 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1443 | Regression_eta: 0.0015 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0705 | Regression_eta: 0.0277 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0552 | LR: 4.88e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.0910s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6839 | max_loss: 0.1724 | max_backward: 1.0628 | max_optimizer_step: 0.2515 | max_tensorboard_logging: 0.9641 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0689 | Memory: rss_MB: 8305.7 | uss_MB: 7964.4 | child_rss_MB: 4815.8 | allocated_MB: 340.5 | reserved_MB: 22612.0 | inactive_split_MB: 267.2 +rank4 [2026-10-29 06:29:18,757] INFO : Step 27300/50000 rank4 | Train Loss: 1.1913 | Train Loss Unweighted: 0.5857 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1563 | Classification: 0.1169 | Classification_binary: 0.1769 | HitGrouping: 0.1965 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1426 | Regression_eta: 0.0019 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0650 | Regression_eta: 0.0404 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0842 | LR: 4.85e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.2513s | Phase Times: max_pre_step_barrier: 0.0244 | max_batch_to_device: 0.0064 | max_forward: 0.6813 | max_loss: 0.1454 | max_backward: 1.4618 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7699 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 7290.8 | uss_MB: 6952.0 | child_rss_MB: 4811.5 | allocated_MB: 345.6 | reserved_MB: 23586.0 | inactive_split_MB: 267.9 +rank4 [2026-10-29 06:33:03,428] INFO : Step 27400/50000 rank4 | Train Loss: 1.1723 | Train Loss Unweighted: 0.5888 | Train Loss Components: Auxiliary: 0.2572 | Cardinality: 0.1400 | Classification: 0.1088 | Classification_binary: 0.2016 | HitGrouping: 0.1987 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1370 | Regression_eta: 0.0017 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2006 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0550 | Regression_eta: 0.0503 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0654 | LR: 4.82e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6653s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8048 | max_loss: 0.1475 | max_backward: 1.6655 | max_optimizer_step: 0.2279 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7292.0 | uss_MB: 6951.9 | child_rss_MB: 4814.0 | allocated_MB: 344.5 | reserved_MB: 23586.0 | inactive_split_MB: 279.5 +rank4 [2026-10-29 06:37:01,482] INFO : Step 27500/50000 rank4 | Train Loss: 1.0739 | Train Loss Unweighted: 0.5273 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1251 | Classification: 0.0933 | Classification_binary: 0.1655 | HitGrouping: 0.1957 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1633 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0640 | Regression_eta: 0.0566 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0585 | LR: 4.80e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2217s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7686 | max_loss: 0.1542 | max_backward: 1.3750 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.8628 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 7288.8 | uss_MB: 6945.7 | child_rss_MB: 4804.4 | allocated_MB: 343.6 | reserved_MB: 23586.0 | inactive_split_MB: 241.4 +rank4 [2026-10-29 06:40:55,528] INFO : Step 27600/50000 rank4 | Train Loss: 1.0507 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1163 | Classification: 0.1024 | Classification_binary: 0.1467 | HitGrouping: 0.1992 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0582 | Regression_eta: 0.1285 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0529 | LR: 4.77e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4912s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0060 | max_forward: 0.6744 | max_loss: 0.2492 | max_backward: 1.6867 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.5257 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 7290.8 | uss_MB: 6853.7 | child_rss_MB: 4808.0 | allocated_MB: 341.2 | reserved_MB: 24480.0 | inactive_split_MB: 234.8 +rank4 [2026-10-29 06:44:47,871] INFO : Step 27700/50000 rank4 | Train Loss: 1.1881 | Train Loss Unweighted: 0.5834 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1544 | Classification: 0.1001 | Classification_binary: 0.1885 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1444 | Regression_eta: 0.0015 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.2196 | Regression_cos_phi: 0.0964 | Regression_energy: 0.0763 | Regression_eta: 0.0512 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0715 | LR: 4.74e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.2924s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7716 | max_loss: 0.1527 | max_backward: 1.4941 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 0.7272 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7291.1 | uss_MB: 6952.7 | child_rss_MB: 4804.6 | allocated_MB: 344.5 | reserved_MB: 24480.0 | inactive_split_MB: 194.2 +rank4 [2026-10-29 06:48:38,929] INFO : Step 27800/50000 rank4 | Train Loss: 1.1903 | Train Loss Unweighted: 0.5862 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1435 | Classification: 0.1032 | Classification_binary: 0.2006 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1387 | Regression_eta: 0.0018 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.2243 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0785 | Regression_eta: 0.0656 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0919 | LR: 4.71e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1884s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8137 | max_loss: 0.1509 | max_backward: 1.4748 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6041 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7291.1 | uss_MB: 6953.1 | child_rss_MB: 4808.7 | allocated_MB: 348.4 | reserved_MB: 24592.0 | inactive_split_MB: 226.8 +rank4 [2026-10-29 06:52:39,106] INFO : Step 27900/50000 rank4 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5521 | Train Loss Components: Auxiliary: 0.2284 | Cardinality: 0.1139 | Classification: 0.0902 | Classification_binary: 0.1544 | HitGrouping: 0.2057 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1519 | Regression_eta: 0.0017 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0614 | Regression_eta: 0.0369 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0607 | LR: 4.68e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6322s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6397 | max_loss: 0.2456 | max_backward: 1.5234 | max_optimizer_step: 0.2276 | max_tensorboard_logging: 1.1626 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 7291.1 | uss_MB: 6955.2 | child_rss_MB: 4805.2 | allocated_MB: 342.4 | reserved_MB: 25612.0 | inactive_split_MB: 259.0 +rank4 [2026-10-29 06:56:30,764] INFO : Step 28000/50000 rank4 | Train Loss: 1.1111 | Train Loss Unweighted: 0.5401 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1274 | Classification: 0.1035 | Classification_binary: 0.1699 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0574 | Regression_eta: 0.0427 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0705 | LR: 4.66e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.0947s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.7503 | max_loss: 0.1492 | max_backward: 1.7566 | max_optimizer_step: 0.2764 | max_tensorboard_logging: 1.2295 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 7291.2 | uss_MB: 5984.8 | child_rss_MB: 4794.8 | allocated_MB: 344.1 | reserved_MB: 25612.0 | inactive_split_MB: 243.2 +rank4 [2026-10-29 06:56:31,846] INFO : Running validation on rank4 +rank4 [2026-10-29 06:56:31,846] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 06:56:31,846] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 06:57:24,999] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:57:25,561] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 06:57:52,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 06:57:53,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 06:57:54,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 06:57:55,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 06:58:05,106] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 06:58:05,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:58:06,179] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 06:58:22,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 06:58:23,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 06:58:23,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 06:58:24,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 06:58:34,836] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 06:58:35,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:58:35,799] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 06:58:51,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 06:58:51,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 06:58:52,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 06:58:53,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 06:59:03,140] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 07:03:11,252] INFO : Step 28100/50000 rank4 | Train Loss: 1.1727 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1644 | Classification: 0.0934 | Classification_binary: 0.1929 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1365 | Regression_eta: 0.0015 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.2077 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0627 | Regression_eta: 0.0342 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0635 | LR: 4.63e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9503s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8425 | max_loss: 0.2571 | max_backward: 1.8157 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 1.0134 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 8340.2 | uss_MB: 8005.6 | child_rss_MB: 4802.9 | allocated_MB: 344.8 | reserved_MB: 22140.0 | inactive_split_MB: 293.2 +rank4 [2026-10-29 07:07:01,163] INFO : Step 28200/50000 rank4 | Train Loss: 1.0520 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2296 | Cardinality: 0.1203 | Classification: 0.0934 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1466 | Regression_eta: 0.0014 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0780 | Regression_eta: 0.0508 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0838 | LR: 4.60e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3551s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7943 | max_loss: 0.1496 | max_backward: 1.4503 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 0.7872 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 8343.2 | uss_MB: 8006.4 | child_rss_MB: 4796.0 | allocated_MB: 341.0 | reserved_MB: 24034.0 | inactive_split_MB: 210.4 +rank4 [2026-10-29 07:10:52,729] INFO : Step 28300/50000 rank4 | Train Loss: 1.0602 | Train Loss Unweighted: 0.5453 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1043 | Classification: 0.0923 | Classification_binary: 0.1501 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1496 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0715 | Regression_eta: 0.0365 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0547 | LR: 4.57e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2453s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6482 | max_loss: 0.2266 | max_backward: 1.5658 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.6870 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0836 | Memory: rss_MB: 7393.0 | uss_MB: 7059.5 | child_rss_MB: 4787.3 | allocated_MB: 342.7 | reserved_MB: 25016.0 | inactive_split_MB: 243.9 +rank4 [2026-10-29 07:14:40,574] INFO : Step 28400/50000 rank4 | Train Loss: 1.1507 | Train Loss Unweighted: 0.5391 | Train Loss Components: Auxiliary: 0.2436 | Cardinality: 0.1500 | Classification: 0.0965 | Classification_binary: 0.1896 | HitGrouping: 0.1911 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1241 | Regression_eta: 0.0018 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2003 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0652 | Regression_eta: 0.0678 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0642 | LR: 4.55e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6297s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7664 | max_loss: 0.1532 | max_backward: 1.7342 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.8455 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7394.0 | uss_MB: 7059.9 | child_rss_MB: 4785.1 | allocated_MB: 344.2 | reserved_MB: 25016.0 | inactive_split_MB: 278.9 +rank4 [2026-10-29 07:18:29,002] INFO : Step 28500/50000 rank4 | Train Loss: 1.1595 | Train Loss Unweighted: 0.5640 | Train Loss Components: Auxiliary: 0.2492 | Cardinality: 0.1556 | Classification: 0.0927 | Classification_binary: 0.1908 | HitGrouping: 0.1954 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1384 | Regression_eta: 0.0015 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0715 | Regression_eta: 0.0246 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0499 | LR: 4.52e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2720s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.8033 | max_loss: 0.1665 | max_backward: 1.5231 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.6178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0709 | Memory: rss_MB: 7393.5 | uss_MB: 7056.4 | child_rss_MB: 4808.7 | allocated_MB: 347.4 | reserved_MB: 26000.0 | inactive_split_MB: 283.2 +rank4 [2026-10-29 07:22:17,774] INFO : Step 28600/50000 rank4 | Train Loss: 1.0672 | Train Loss Unweighted: 0.5222 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1172 | Classification: 0.0893 | Classification_binary: 0.1544 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0587 | Regression_eta: 0.0480 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0668 | LR: 4.49e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.0225s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6704 | max_loss: 0.1829 | max_backward: 1.2353 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.8074 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7396.5 | uss_MB: 7057.8 | child_rss_MB: 4793.1 | allocated_MB: 342.3 | reserved_MB: 26000.0 | inactive_split_MB: 243.7 +rank4 [2026-10-29 07:26:10,155] INFO : Step 28700/50000 rank4 | Train Loss: 1.1535 | Train Loss Unweighted: 0.5811 | Train Loss Components: Auxiliary: 0.2469 | Cardinality: 0.1254 | Classification: 0.1220 | Classification_binary: 0.1661 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1442 | Regression_eta: 0.0021 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0751 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0767 | Regression_eta: 0.0621 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0778 | LR: 4.46e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5816s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6905 | max_loss: 0.2072 | max_backward: 1.6876 | max_optimizer_step: 0.2353 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 7397.5 | uss_MB: 7064.1 | child_rss_MB: 4802.4 | allocated_MB: 343.5 | reserved_MB: 26000.0 | inactive_split_MB: 234.1 +rank4 [2026-10-29 07:30:12,141] INFO : Step 28800/50000 rank4 | Train Loss: 1.2131 | Train Loss Unweighted: 0.5739 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1788 | Classification: 0.1057 | Classification_binary: 0.2127 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2318 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0542 | Regression_eta: 0.0399 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0778 | LR: 4.44e-04 | DataLoad Time: 0.0119s | Model Forward Time: 6.8479s | Phase Times: max_pre_step_barrier: 0.0263 | max_batch_to_device: 0.0149 | max_forward: 1.2928 | max_loss: 0.2713 | max_backward: 3.0161 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.8895 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 7398.3 | uss_MB: 5722.3 | child_rss_MB: 4796.9 | allocated_MB: 343.8 | reserved_MB: 26000.0 | inactive_split_MB: 273.3 +rank4 [2026-10-29 07:34:11,576] INFO : Step 28900/50000 rank4 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1252 | Classification: 0.0967 | Classification_binary: 0.1637 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1509 | Regression_eta: 0.0015 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1531 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0632 | Regression_eta: 0.0272 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0857 | LR: 4.41e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.1518s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7512 | max_loss: 0.2189 | max_backward: 1.6229 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.9330 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7399.6 | uss_MB: 5634.1 | child_rss_MB: 4800.1 | allocated_MB: 339.6 | reserved_MB: 26000.0 | inactive_split_MB: 223.0 +rank4 [2026-10-29 07:38:06,079] INFO : Step 29000/50000 rank4 | Train Loss: 1.0238 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1070 | Classification: 0.0861 | Classification_binary: 0.1439 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1389 | Regression_eta: 0.0014 | Regression_pt: 0.1414 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0662 | Regression_eta: 0.0320 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0448 | LR: 4.38e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9808s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.6636 | max_loss: 0.2429 | max_backward: 1.5394 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.5216 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7400.5 | uss_MB: 7067.2 | child_rss_MB: 4794.8 | allocated_MB: 345.2 | reserved_MB: 26984.0 | inactive_split_MB: 235.2 +rank4 [2026-10-29 07:38:07,029] INFO : Running validation on rank4 +rank4 [2026-10-29 07:38:07,029] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 07:38:07,030] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 07:38:57,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:38:58,450] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 07:39:24,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 07:39:25,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 07:39:26,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 07:39:27,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 07:39:37,398] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 07:39:38,026] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:39:38,580] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 07:39:54,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 07:39:55,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 07:39:56,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 07:39:57,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 07:40:07,348] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 07:40:07,779] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:40:08,277] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 07:40:23,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 07:40:24,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 07:40:24,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 07:40:25,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 07:40:35,720] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 07:45:01,997] INFO : Step 29100/50000 rank4 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5808 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1279 | Classification: 0.1113 | Classification_binary: 0.1973 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1339 | Regression_eta: 0.0017 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1991 | Regression_cos_phi: 0.0885 | Regression_energy: 0.0567 | Regression_eta: 0.0563 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0637 | LR: 4.35e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.8736s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.8060 | max_loss: 0.1503 | max_backward: 1.9210 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 8379.4 | uss_MB: 8043.2 | child_rss_MB: 4787.2 | allocated_MB: 344.6 | reserved_MB: 24404.0 | inactive_split_MB: 274.9 +rank4 [2026-10-29 07:48:58,054] INFO : Step 29200/50000 rank4 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5633 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1265 | Classification: 0.0969 | Classification_binary: 0.1883 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1368 | Regression_eta: 0.0014 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2012 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0680 | Regression_eta: 0.0420 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0812 | LR: 4.33e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8366s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0742 | max_forward: 1.0303 | max_loss: 0.2183 | max_backward: 1.8745 | max_optimizer_step: 0.2501 | max_tensorboard_logging: 0.5570 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8379.4 | uss_MB: 8042.7 | child_rss_MB: 4798.0 | allocated_MB: 343.1 | reserved_MB: 24404.0 | inactive_split_MB: 228.1 +rank4 [2026-10-29 07:52:51,792] INFO : Step 29300/50000 rank4 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2330 | Cardinality: 0.1231 | Classification: 0.0898 | Classification_binary: 0.1516 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1428 | Regression_eta: 0.0017 | Regression_pt: 0.1453 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1352 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0644 | Regression_eta: 0.0308 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0347 | LR: 4.30e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.7356s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0067 | max_forward: 0.7139 | max_loss: 0.1578 | max_backward: 1.5096 | max_optimizer_step: 0.2586 | max_tensorboard_logging: 1.1324 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7473.7 | uss_MB: 6607.6 | child_rss_MB: 4797.1 | allocated_MB: 341.0 | reserved_MB: 24404.0 | inactive_split_MB: 247.0 +rank4 [2026-10-29 07:56:44,366] INFO : Step 29400/50000 rank4 | Train Loss: 1.1400 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1377 | Classification: 0.1135 | Classification_binary: 0.1661 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1386 | Regression_eta: 0.0019 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.1839 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0745 | Regression_eta: 0.1602 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0632 | LR: 4.27e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.2888s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.6285 | max_loss: 0.1485 | max_backward: 1.3787 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.9179 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7474.7 | uss_MB: 7139.2 | child_rss_MB: 4788.9 | allocated_MB: 343.3 | reserved_MB: 24404.0 | inactive_split_MB: 277.1 +rank4 [2026-10-29 08:00:58,645] INFO : Step 29500/50000 rank4 | Train Loss: 1.1987 | Train Loss Unweighted: 0.6106 | Train Loss Components: Auxiliary: 0.2551 | Cardinality: 0.1277 | Classification: 0.1201 | Classification_binary: 0.2013 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1419 | Regression_eta: 0.0019 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2131 | Regression_cos_phi: 0.1182 | Regression_energy: 0.0774 | Regression_eta: 0.0514 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0798 | LR: 4.24e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.8978s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0749 | max_forward: 0.8153 | max_loss: 0.3465 | max_backward: 2.8565 | max_optimizer_step: 0.3442 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7477.7 | uss_MB: 7142.6 | child_rss_MB: 4792.1 | allocated_MB: 343.9 | reserved_MB: 24404.0 | inactive_split_MB: 271.5 +rank4 [2026-10-29 08:05:08,852] INFO : Step 29600/50000 rank4 | Train Loss: 1.1065 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2391 | Cardinality: 0.1179 | Classification: 0.0927 | Classification_binary: 0.1666 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0018 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0661 | Regression_eta: 0.0311 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0499 | LR: 4.22e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4750s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8025 | max_loss: 0.1485 | max_backward: 1.4674 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.8982 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 7476.5 | uss_MB: 7138.1 | child_rss_MB: 4790.4 | allocated_MB: 340.8 | reserved_MB: 24404.0 | inactive_split_MB: 290.7 +rank4 [2026-10-29 08:09:13,801] INFO : Step 29700/50000 rank4 | Train Loss: 1.0488 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2273 | Cardinality: 0.1414 | Classification: 0.0952 | Classification_binary: 0.1459 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1667 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0668 | Regression_eta: 0.0355 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0654 | LR: 4.19e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5180s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0057 | max_forward: 0.7342 | max_loss: 0.1652 | max_backward: 1.4907 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.9743 | max_tensorboard_flush: 0.0140 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7477.5 | uss_MB: 6998.2 | child_rss_MB: 4799.0 | allocated_MB: 342.3 | reserved_MB: 24404.0 | inactive_split_MB: 236.6 +rank4 [2026-10-29 08:13:32,079] INFO : Step 29800/50000 rank4 | Train Loss: 1.1686 | Train Loss Unweighted: 0.5644 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1551 | Classification: 0.0952 | Classification_binary: 0.1922 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1357 | Regression_eta: 0.0014 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0355 | Regression_energy: 0.0533 | Regression_eta: 0.0254 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0440 | LR: 4.16e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.6788s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7294 | max_loss: 0.1685 | max_backward: 1.6547 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9409 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7478.5 | uss_MB: 7143.5 | child_rss_MB: 4794.4 | allocated_MB: 345.0 | reserved_MB: 24404.0 | inactive_split_MB: 265.1 +rank4 [2026-10-29 08:17:37,376] INFO : Step 29900/50000 rank4 | Train Loss: 1.1305 | Train Loss Unweighted: 0.5666 | Train Loss Components: Auxiliary: 0.2471 | Cardinality: 0.1380 | Classification: 0.0944 | Classification_binary: 0.1750 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1464 | Regression_eta: 0.0015 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0663 | Regression_eta: 0.0380 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0601 | LR: 4.14e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3513s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0611 | max_forward: 0.8060 | max_loss: 0.1494 | max_backward: 1.3251 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 7479.5 | uss_MB: 7143.0 | child_rss_MB: 4800.7 | allocated_MB: 344.7 | reserved_MB: 26422.0 | inactive_split_MB: 298.4 +rank4 [2026-10-29 08:22:04,988] INFO : Step 30000/50000 rank4 | Train Loss: 1.0854 | Train Loss Unweighted: 0.5586 | Train Loss Components: Auxiliary: 0.2416 | Cardinality: 0.1337 | Classification: 0.1000 | Classification_binary: 0.1541 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1499 | Regression_eta: 0.0014 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0333 | Regression_energy: 0.0575 | Regression_eta: 0.0216 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0456 | LR: 4.11e-04 | DataLoad Time: 0.0206s | Model Forward Time: 7.0740s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0172 | max_forward: 0.9685 | max_loss: 0.2166 | max_backward: 4.0112 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.8430 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7461.9 | uss_MB: 7142.5 | child_rss_MB: 4797.1 | allocated_MB: 338.0 | reserved_MB: 26422.0 | inactive_split_MB: 288.0 +rank4 [2026-10-29 08:22:05,952] INFO : Running validation on rank4 +rank4 [2026-10-29 08:22:05,952] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 08:22:05,952] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 08:22:57,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:22:58,360] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 08:23:24,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 08:23:25,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 08:23:26,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 08:23:27,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 08:23:37,821] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 08:23:38,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:23:38,901] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 08:23:55,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 08:23:56,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 08:23:57,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 08:23:57,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 08:24:07,947] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 08:24:08,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:24:08,615] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 08:24:24,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 08:24:25,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 08:24:25,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 08:24:26,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 08:24:36,512] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 08:29:00,406] INFO : Step 30100/50000 rank4 | Train Loss: 1.0762 | Train Loss Unweighted: 0.5358 | Train Loss Components: Auxiliary: 0.2378 | Cardinality: 0.1106 | Classification: 0.1097 | Classification_binary: 0.1679 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0838 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0760 | Regression_eta: 0.0634 | Regression_pt: 0.0790 | Regression_sin_phi: 0.1061 | LR: 4.08e-04 | DataLoad Time: 0.0168s | Model Forward Time: 8.2919s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7352 | max_loss: 0.1443 | max_backward: 1.7375 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 1.7731 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.3497 | Memory: rss_MB: 7528.6 | uss_MB: 6331.6 | child_rss_MB: 4801.2 | allocated_MB: 344.1 | reserved_MB: 22922.0 | inactive_split_MB: 248.8 +rank4 [2026-10-29 08:33:16,462] INFO : Step 30200/50000 rank4 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5304 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1421 | Classification: 0.0934 | Classification_binary: 0.1839 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0016 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.2013 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0564 | Regression_eta: 0.0762 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0813 | LR: 4.06e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.6016s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8008 | max_loss: 0.1560 | max_backward: 1.6706 | max_optimizer_step: 0.2663 | max_tensorboard_logging: 0.7725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0555 | Memory: rss_MB: 7535.6 | uss_MB: 7212.9 | child_rss_MB: 4775.8 | allocated_MB: 344.1 | reserved_MB: 23882.0 | inactive_split_MB: 274.8 +rank4 [2026-10-29 08:37:30,483] INFO : Step 30300/50000 rank4 | Train Loss: 1.0429 | Train Loss Unweighted: 0.5252 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1223 | Classification: 0.0922 | Classification_binary: 0.1566 | HitGrouping: 0.1929 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0616 | Regression_eta: 0.0298 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0365 | LR: 4.03e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.6255s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0060 | max_forward: 1.1719 | max_loss: 0.2884 | max_backward: 2.4994 | max_optimizer_step: 0.2255 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7536.6 | uss_MB: 7213.9 | child_rss_MB: 4771.6 | allocated_MB: 341.8 | reserved_MB: 23882.0 | inactive_split_MB: 270.7 +rank4 [2026-10-29 08:41:40,106] INFO : Step 30400/50000 rank4 | Train Loss: 1.0543 | Train Loss Unweighted: 0.5182 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1318 | Classification: 0.0889 | Classification_binary: 0.1487 | HitGrouping: 0.1973 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1382 | Regression_eta: 0.0014 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0334 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0590 | Regression_eta: 0.0319 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0512 | LR: 4.00e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.0682s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0150 | max_forward: 0.8845 | max_loss: 0.2482 | max_backward: 2.1745 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.5112 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7452.2 | uss_MB: 7214.6 | child_rss_MB: 4475.9 | allocated_MB: 344.7 | reserved_MB: 24842.0 | inactive_split_MB: 245.3 +rank4 [2026-10-29 08:45:49,061] INFO : Step 30500/50000 rank4 | Train Loss: 1.1589 | Train Loss Unweighted: 0.5596 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1407 | Classification: 0.1085 | Classification_binary: 0.1952 | HitGrouping: 0.1960 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0581 | Regression_eta: 0.0553 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0859 | LR: 3.98e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.7059s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7791 | max_loss: 0.2528 | max_backward: 1.9433 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.5210 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0672 | Memory: rss_MB: 7432.7 | uss_MB: 7008.9 | child_rss_MB: 4374.7 | allocated_MB: 346.2 | reserved_MB: 26806.0 | inactive_split_MB: 227.8 +rank4 [2026-10-29 08:49:58,496] INFO : Step 30600/50000 rank4 | Train Loss: 1.1269 | Train Loss Unweighted: 0.5561 | Train Loss Components: Auxiliary: 0.2414 | Cardinality: 0.1456 | Classification: 0.0973 | Classification_binary: 0.1804 | HitGrouping: 0.1953 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1373 | Regression_eta: 0.0014 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0657 | Regression_eta: 0.0343 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0552 | LR: 3.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4781s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1011 | max_forward: 0.8824 | max_loss: 0.1534 | max_backward: 1.4258 | max_optimizer_step: 0.3067 | max_tensorboard_logging: 0.7925 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 7378.2 | uss_MB: 7215.1 | child_rss_MB: 4177.7 | allocated_MB: 345.8 | reserved_MB: 27800.0 | inactive_split_MB: 283.5 +rank4 [2026-10-29 08:54:17,063] INFO : Step 30700/50000 rank4 | Train Loss: 1.0989 | Train Loss Unweighted: 0.5515 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1377 | Classification: 0.0890 | Classification_binary: 0.1568 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0013 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1497 | Regression_cos_phi: 0.0472 | Regression_energy: 0.1114 | Regression_eta: 0.0295 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0400 | LR: 3.92e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.2933s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0056 | max_forward: 0.7546 | max_loss: 0.2396 | max_backward: 1.3389 | max_optimizer_step: 0.2508 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0572 | Memory: rss_MB: 7380.2 | uss_MB: 7216.5 | child_rss_MB: 4178.9 | allocated_MB: 340.3 | reserved_MB: 27800.0 | inactive_split_MB: 223.2 +rank4 [2026-10-29 08:58:31,924] INFO : Step 30800/50000 rank4 | Train Loss: 1.1241 | Train Loss Unweighted: 0.5488 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1377 | Classification: 0.1081 | Classification_binary: 0.1717 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1317 | Regression_eta: 0.0015 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0677 | Regression_eta: 0.0565 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0617 | LR: 3.90e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.8027s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7775 | max_loss: 0.2735 | max_backward: 1.9298 | max_optimizer_step: 0.3489 | max_tensorboard_logging: 0.8756 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0726 | Memory: rss_MB: 7357.4 | uss_MB: 5945.6 | child_rss_MB: 4116.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 275.1 +rank4 [2026-10-29 09:02:40,911] INFO : Step 30900/50000 rank4 | Train Loss: 1.1501 | Train Loss Unweighted: 0.5762 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1300 | Classification: 0.0976 | Classification_binary: 0.1996 | HitGrouping: 0.2000 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1374 | Regression_eta: 0.0017 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0717 | Regression_eta: 0.0400 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0479 | LR: 3.87e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.5952s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.3511 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 1.0306 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 7360.4 | uss_MB: 7217.8 | child_rss_MB: 4118.5 | allocated_MB: 342.2 | reserved_MB: 27800.0 | inactive_split_MB: 198.3 +rank4 [2026-10-29 09:06:46,925] INFO : Step 31000/50000 rank4 | Train Loss: 1.1054 | Train Loss Unweighted: 0.5218 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1446 | Classification: 0.0888 | Classification_binary: 0.1693 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0016 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0443 | Regression_energy: 0.0727 | Regression_eta: 0.0344 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0792 | LR: 3.84e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.7160s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.8505 | max_loss: 0.2491 | max_backward: 1.6460 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9731 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7337.9 | uss_MB: 6674.8 | child_rss_MB: 4046.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 190.7 +rank4 [2026-10-29 09:06:47,902] INFO : Running validation on rank4 +rank4 [2026-10-29 09:06:47,902] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 09:06:47,903] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 09:07:39,008] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:07:39,555] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 09:08:08,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 09:08:09,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 09:08:10,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 09:08:11,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 09:08:21,354] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 09:08:22,017] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:08:22,479] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 09:08:38,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 09:08:40,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 09:08:41,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 09:08:42,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 09:08:52,555] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 09:08:52,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:08:53,433] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 09:09:08,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 09:09:09,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 09:09:09,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 09:09:10,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 09:09:20,575] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 09:13:45,022] INFO : Step 31100/50000 rank4 | Train Loss: 1.0306 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2182 | Cardinality: 0.1238 | Classification: 0.0932 | Classification_binary: 0.1415 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0663 | Regression_eta: 0.0259 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0682 | LR: 3.82e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.5111s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1068 | max_forward: 0.8067 | max_loss: 0.1436 | max_backward: 1.1427 | max_optimizer_step: 0.2576 | max_tensorboard_logging: 1.1475 | max_tensorboard_flush: 0.0054 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 8196.3 | uss_MB: 5921.9 | child_rss_MB: 4032.2 | allocated_MB: 343.8 | reserved_MB: 24492.0 | inactive_split_MB: 293.3 +rank4 [2026-10-29 09:17:44,912] INFO : Step 31200/50000 rank4 | Train Loss: 1.1102 | Train Loss Unweighted: 0.5369 | Train Loss Components: Auxiliary: 0.2472 | Cardinality: 0.1495 | Classification: 0.0959 | Classification_binary: 0.1708 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1328 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.1819 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0521 | Regression_eta: 0.0379 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0667 | LR: 3.79e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1833s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7127 | max_loss: 0.1476 | max_backward: 1.5707 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7428.7 | uss_MB: 7313.8 | child_rss_MB: 4003.5 | allocated_MB: 344.5 | reserved_MB: 24648.0 | inactive_split_MB: 235.5 +rank4 [2026-10-29 09:21:47,087] INFO : Step 31300/50000 rank4 | Train Loss: 1.1865 | Train Loss Unweighted: 0.5472 | Train Loss Components: Auxiliary: 0.2511 | Cardinality: 0.1748 | Classification: 0.0979 | Classification_binary: 0.1851 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0019 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1904 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0601 | Regression_eta: 0.0411 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0763 | LR: 3.76e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2308s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0064 | max_forward: 0.7815 | max_loss: 0.1495 | max_backward: 1.6330 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.5234 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7366.1 | uss_MB: 7313.9 | child_rss_MB: 3706.8 | allocated_MB: 344.4 | reserved_MB: 25640.0 | inactive_split_MB: 233.6 +rank4 [2026-10-29 09:25:42,888] INFO : Step 31400/50000 rank4 | Train Loss: 1.0272 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1191 | Classification: 0.0784 | Classification_binary: 0.1451 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0600 | Regression_eta: 0.0294 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0492 | LR: 3.74e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5263s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0063 | max_forward: 0.7252 | max_loss: 0.1478 | max_backward: 1.6173 | max_optimizer_step: 0.2404 | max_tensorboard_logging: 0.9541 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7335.5 | uss_MB: 7312.2 | child_rss_MB: 3625.5 | allocated_MB: 342.1 | reserved_MB: 25640.0 | inactive_split_MB: 296.4 +rank4 [2026-10-29 09:29:44,710] INFO : Step 31500/50000 rank4 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5247 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1206 | Classification: 0.1037 | Classification_binary: 0.1583 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1297 | Regression_eta: 0.0014 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1698 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0707 | Regression_eta: 0.0361 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0546 | LR: 3.71e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.3657s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6643 | max_loss: 0.2487 | max_backward: 1.5648 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.8248 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7336.0 | uss_MB: 7313.0 | child_rss_MB: 3638.4 | allocated_MB: 344.4 | reserved_MB: 26092.0 | inactive_split_MB: 238.2 +rank4 [2026-10-29 09:33:42,078] INFO : Step 31600/50000 rank4 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5471 | Train Loss Components: Auxiliary: 0.2424 | Cardinality: 0.1284 | Classification: 0.1033 | Classification_binary: 0.1828 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1282 | Regression_eta: 0.0017 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1801 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0560 | Regression_eta: 0.0430 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0674 | LR: 3.69e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.0251s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7603 | max_loss: 0.1471 | max_backward: 2.0475 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.9076 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7328.4 | uss_MB: 7314.4 | child_rss_MB: 3585.6 | allocated_MB: 341.5 | reserved_MB: 26640.0 | inactive_split_MB: 287.0 +rank4 [2026-10-29 09:37:47,825] INFO : Step 31700/50000 rank4 | Train Loss: 1.0533 | Train Loss Unweighted: 0.5009 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1416 | Classification: 0.0871 | Classification_binary: 0.1632 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0596 | Regression_eta: 0.0536 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0651 | LR: 3.66e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.1113s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7635 | max_loss: 0.1543 | max_backward: 1.2075 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7330.4 | uss_MB: 7312.9 | child_rss_MB: 3557.0 | allocated_MB: 341.9 | reserved_MB: 26640.0 | inactive_split_MB: 274.1 +rank4 [2026-10-29 09:45:40,561] INFO : Step 31800/50000 rank4 | Train Loss: 1.0385 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1148 | Classification: 0.0877 | Classification_binary: 0.1481 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1418 | Regression_eta: 0.0012 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1644 | Regression_cos_phi: 0.1087 | Regression_energy: 0.0746 | Regression_eta: 0.0411 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0601 | LR: 3.64e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0347s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0072 | max_forward: 0.6606 | max_loss: 0.1610 | max_backward: 1.1159 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.8258 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1417 | Memory: rss_MB: 7328.9 | uss_MB: 7312.1 | child_rss_MB: 3561.3 | allocated_MB: 345.5 | reserved_MB: 27080.0 | inactive_split_MB: 260.5 +rank4 [2026-10-29 09:49:44,090] INFO : Step 31900/50000 rank4 | Train Loss: 1.1464 | Train Loss Unweighted: 0.5585 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1372 | Classification: 0.1078 | Classification_binary: 0.1815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0017 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0630 | Regression_eta: 0.0382 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0631 | LR: 3.61e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.5157s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7718 | max_loss: 0.1553 | max_backward: 1.7857 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.6721 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7333.9 | uss_MB: 7320.8 | child_rss_MB: 3569.3 | allocated_MB: 343.8 | reserved_MB: 27080.0 | inactive_split_MB: 280.2 +rank4 [2026-10-29 09:53:38,294] INFO : Step 32000/50000 rank4 | Train Loss: 1.1115 | Train Loss Unweighted: 0.5351 | Train Loss Components: Auxiliary: 0.2393 | Cardinality: 0.1406 | Classification: 0.0954 | Classification_binary: 0.1748 | HitGrouping: 0.1912 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1302 | Regression_eta: 0.0018 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.2002 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0622 | Regression_eta: 0.1531 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0917 | LR: 3.58e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.2445s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.9767 | max_loss: 0.6919 | max_backward: 2.1955 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.9277 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 7329.3 | uss_MB: 7313.9 | child_rss_MB: 3566.4 | allocated_MB: 344.2 | reserved_MB: 27080.0 | inactive_split_MB: 265.7 +rank4 [2026-10-29 09:53:39,335] INFO : Running validation on rank4 +rank4 [2026-10-29 09:53:39,335] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 09:53:39,335] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 09:54:28,722] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:54:29,292] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 09:54:57,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 09:54:58,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 09:54:59,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 09:55:00,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 09:55:10,284] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 09:55:10,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:55:11,416] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 09:55:27,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 09:55:28,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 09:55:29,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 09:55:30,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 09:55:40,252] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 09:55:41,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:55:41,776] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 09:55:57,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 09:55:57,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 09:55:58,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 09:55:59,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 09:56:09,187] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 10:00:19,239] INFO : Step 32100/50000 rank4 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1218 | Classification: 0.0914 | Classification_binary: 0.1509 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1333 | Regression_eta: 0.0016 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0701 | Regression_eta: 0.0331 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0746 | LR: 3.56e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.3626s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7098 | max_loss: 0.1454 | max_backward: 1.1394 | max_optimizer_step: 0.2650 | max_tensorboard_logging: 0.9832 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 8404.9 | uss_MB: 8386.6 | child_rss_MB: 3573.9 | allocated_MB: 344.0 | reserved_MB: 22148.0 | inactive_split_MB: 276.8 +rank4 [2026-10-29 10:04:33,375] INFO : Step 32200/50000 rank4 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1241 | Classification: 0.1035 | Classification_binary: 0.1630 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1279 | Regression_eta: 0.0014 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1603 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0747 | Regression_eta: 0.0293 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0432 | LR: 3.53e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6699s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0891 | max_forward: 0.7517 | max_loss: 0.1441 | max_backward: 1.7835 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.7808 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7418.2 | uss_MB: 7403.1 | child_rss_MB: 3576.2 | allocated_MB: 345.5 | reserved_MB: 23122.0 | inactive_split_MB: 233.4 +rank4 [2026-10-29 10:08:31,627] INFO : Step 32300/50000 rank4 | Train Loss: 1.1438 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2535 | Cardinality: 0.1335 | Classification: 0.1042 | Classification_binary: 0.1958 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1303 | Regression_eta: 0.0015 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2184 | Regression_cos_phi: 0.1360 | Regression_energy: 0.0664 | Regression_eta: 0.0551 | Regression_pt: 0.0704 | Regression_sin_phi: 0.1011 | LR: 3.51e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7839s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7857 | max_loss: 0.1780 | max_backward: 1.7580 | max_optimizer_step: 0.2428 | max_tensorboard_logging: 0.8954 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 7421.3 | uss_MB: 7404.5 | child_rss_MB: 3576.8 | allocated_MB: 343.9 | reserved_MB: 24108.0 | inactive_split_MB: 280.1 +rank4 [2026-10-29 10:12:37,147] INFO : Step 32400/50000 rank4 | Train Loss: 1.0306 | Train Loss Unweighted: 0.5063 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1168 | Classification: 0.0867 | Classification_binary: 0.1599 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0013 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0531 | Regression_eta: 0.0398 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0601 | LR: 3.48e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.3210s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.1456 | max_backward: 1.3401 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.8752 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7420.4 | uss_MB: 7401.9 | child_rss_MB: 3574.4 | allocated_MB: 344.8 | reserved_MB: 24108.0 | inactive_split_MB: 271.0 +rank4 [2026-10-29 10:16:48,486] INFO : Step 32500/50000 rank4 | Train Loss: 1.0033 | Train Loss Unweighted: 0.5126 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.0995 | Classification: 0.1015 | Classification_binary: 0.1402 | HitGrouping: 0.1898 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1332 | Regression_eta: 0.0015 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1367 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0759 | Regression_eta: 0.0329 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0507 | LR: 3.46e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9953s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6088 | max_loss: 0.1480 | max_backward: 1.0729 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.8045 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7420.4 | uss_MB: 7401.8 | child_rss_MB: 3575.0 | allocated_MB: 341.9 | reserved_MB: 25096.0 | inactive_split_MB: 268.1 +rank4 [2026-10-29 10:21:04,067] INFO : Step 32600/50000 rank4 | Train Loss: 1.0981 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1271 | Classification: 0.1053 | Classification_binary: 0.1773 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1306 | Regression_eta: 0.0014 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0685 | Regression_eta: 0.0453 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0635 | LR: 3.43e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.2981s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8529 | max_loss: 0.3369 | max_backward: 2.9831 | max_optimizer_step: 0.2386 | max_tensorboard_logging: 0.5557 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7420.7 | uss_MB: 5748.3 | child_rss_MB: 3585.6 | allocated_MB: 344.1 | reserved_MB: 25096.0 | inactive_split_MB: 287.9 +rank4 [2026-10-29 10:25:23,204] INFO : Step 32700/50000 rank4 | Train Loss: 1.0956 | Train Loss Unweighted: 0.5307 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.1282 | Classification: 0.1011 | Classification_binary: 0.1731 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1730 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0558 | Regression_eta: 0.0312 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0471 | LR: 3.41e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.1109s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0062 | max_forward: 0.8233 | max_loss: 0.1478 | max_backward: 1.5120 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0564 | Memory: rss_MB: 7420.7 | uss_MB: 7403.4 | child_rss_MB: 3588.8 | allocated_MB: 342.9 | reserved_MB: 25096.0 | inactive_split_MB: 272.6 +rank4 [2026-10-29 10:29:23,500] INFO : Step 32800/50000 rank4 | Train Loss: 1.0857 | Train Loss Unweighted: 0.5241 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1299 | Classification: 0.1011 | Classification_binary: 0.1551 | HitGrouping: 0.1982 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1327 | Regression_eta: 0.0016 | Regression_pt: 0.1319 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.1424 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0622 | Regression_eta: 0.0592 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0640 | LR: 3.38e-04 | DataLoad Time: 0.0243s | Model Forward Time: 4.2376s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0179 | max_forward: 0.9787 | max_loss: 0.1782 | max_backward: 1.5075 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 0.5808 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.1577 | Memory: rss_MB: 7419.5 | uss_MB: 7401.5 | child_rss_MB: 3585.9 | allocated_MB: 345.1 | reserved_MB: 25096.0 | inactive_split_MB: 263.9 +rank4 [2026-10-29 10:33:26,398] INFO : Step 32900/50000 rank4 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1116 | Classification: 0.1066 | Classification_binary: 0.1631 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1300 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1848 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0670 | Regression_eta: 0.0400 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0682 | LR: 3.36e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.7024s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 1.3284 | max_loss: 0.5826 | max_backward: 1.9391 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7423.7 | uss_MB: 7407.1 | child_rss_MB: 3589.0 | allocated_MB: 344.3 | reserved_MB: 26086.0 | inactive_split_MB: 212.1 +rank4 [2026-10-29 10:37:13,221] INFO : Step 33000/50000 rank4 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5373 | Train Loss Components: Auxiliary: 0.2310 | Cardinality: 0.1017 | Classification: 0.1099 | Classification_binary: 0.1844 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0802 | Classification_binary: 0.1854 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0514 | Regression_eta: 0.0366 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0869 | LR: 3.33e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.3121s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 0.7919 | max_loss: 0.2231 | max_backward: 1.5724 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.5025 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1340 | Memory: rss_MB: 7423.5 | uss_MB: 7406.2 | child_rss_MB: 3581.4 | allocated_MB: 344.2 | reserved_MB: 26086.0 | inactive_split_MB: 269.8 +rank4 [2026-10-29 10:37:14,208] INFO : Running validation on rank4 +rank4 [2026-10-29 10:37:14,208] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 10:37:14,209] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 10:38:03,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:38:04,389] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 10:38:31,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 10:38:32,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 10:38:33,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 10:38:34,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 10:38:44,756] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 10:38:45,833] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:38:46,345] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 10:39:02,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 10:39:03,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 10:39:03,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 10:39:04,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 10:39:14,862] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 10:39:15,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:39:16,281] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 10:39:31,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 10:39:32,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 10:39:33,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 10:39:33,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 10:39:43,872] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 10:43:56,311] INFO : Step 33100/50000 rank4 | Train Loss: 1.1040 | Train Loss Unweighted: 0.5268 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1427 | Classification: 0.0921 | Classification_binary: 0.1655 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1319 | Regression_eta: 0.0015 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0605 | Regression_eta: 0.0684 | Regression_pt: 0.0637 | Regression_sin_phi: 0.1032 | LR: 3.31e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.3497s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0325 | max_forward: 0.7755 | max_loss: 0.1478 | max_backward: 1.3916 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.6610 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0760 | Memory: rss_MB: 8784.9 | uss_MB: 8766.5 | child_rss_MB: 3557.4 | allocated_MB: 343.8 | reserved_MB: 23356.0 | inactive_split_MB: 267.3 +rank4 [2026-10-29 10:47:56,312] INFO : Step 33200/50000 rank4 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4955 | Train Loss Components: Auxiliary: 0.2164 | Cardinality: 0.1253 | Classification: 0.1001 | Classification_binary: 0.1407 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0019 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1292 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0716 | Regression_eta: 0.0346 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0684 | LR: 3.28e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.0965s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0401 | max_forward: 0.9954 | max_loss: 0.3116 | max_backward: 2.2139 | max_optimizer_step: 0.2657 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7500.4 | uss_MB: 7483.0 | child_rss_MB: 3560.3 | allocated_MB: 344.1 | reserved_MB: 23356.0 | inactive_split_MB: 241.6 +rank4 [2026-10-29 10:51:55,085] INFO : Step 33300/50000 rank4 | Train Loss: 1.0288 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2302 | Cardinality: 0.1166 | Classification: 0.0975 | Classification_binary: 0.1653 | HitGrouping: 0.1885 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1220 | Regression_eta: 0.0012 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1562 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0573 | Regression_eta: 0.0288 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0564 | LR: 3.26e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5490s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7054 | max_loss: 0.2213 | max_backward: 1.6633 | max_optimizer_step: 0.2474 | max_tensorboard_logging: 0.8237 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0708 | Memory: rss_MB: 7500.4 | uss_MB: 7483.1 | child_rss_MB: 3556.5 | allocated_MB: 343.5 | reserved_MB: 23356.0 | inactive_split_MB: 261.9 +rank4 [2026-10-29 10:55:54,911] INFO : Step 33400/50000 rank4 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5337 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1249 | Classification: 0.0970 | Classification_binary: 0.1720 | HitGrouping: 0.1947 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0013 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0547 | Regression_eta: 0.0361 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0897 | LR: 3.23e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.4580s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0073 | max_forward: 1.2444 | max_loss: 0.2025 | max_backward: 2.5109 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7502.5 | uss_MB: 7485.1 | child_rss_MB: 3549.5 | allocated_MB: 347.6 | reserved_MB: 23358.0 | inactive_split_MB: 303.0 +rank4 [2026-10-29 10:59:53,595] INFO : Step 33500/50000 rank4 | Train Loss: 1.0164 | Train Loss Unweighted: 0.4956 | Train Loss Components: Auxiliary: 0.2212 | Cardinality: 0.1274 | Classification: 0.0875 | Classification_binary: 0.1421 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1309 | Regression_eta: 0.0013 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0812 | Regression_eta: 0.0294 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0507 | LR: 3.21e-04 | DataLoad Time: 0.0112s | Model Forward Time: 4.6267s | Phase Times: max_pre_step_barrier: 0.0271 | max_batch_to_device: 0.0558 | max_forward: 0.9437 | max_loss: 0.1437 | max_backward: 1.5429 | max_optimizer_step: 0.2447 | max_tensorboard_logging: 0.8833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7505.9 | uss_MB: 7486.7 | child_rss_MB: 3558.2 | allocated_MB: 341.2 | reserved_MB: 23358.0 | inactive_split_MB: 237.5 +rank4 [2026-10-29 11:05:22,802] INFO : Step 33600/50000 rank4 | Train Loss: 1.0074 | Train Loss Unweighted: 0.4988 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1109 | Classification: 0.0969 | Classification_binary: 0.1509 | HitGrouping: 0.1901 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0012 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0885 | Regression_energy: 0.0792 | Regression_eta: 0.0363 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0588 | LR: 3.19e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1890s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0970 | max_forward: 0.7667 | max_loss: 0.1478 | max_backward: 1.4192 | max_optimizer_step: 0.2426 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7500.7 | uss_MB: 7485.1 | child_rss_MB: 3550.7 | allocated_MB: 344.5 | reserved_MB: 24528.0 | inactive_split_MB: 238.9 +rank4 [2026-10-29 11:09:17,748] INFO : Step 33700/50000 rank4 | Train Loss: 1.1891 | Train Loss Unweighted: 0.5658 | Train Loss Components: Auxiliary: 0.2602 | Cardinality: 0.1587 | Classification: 0.1116 | Classification_binary: 0.1953 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1270 | Regression_eta: 0.0015 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1876 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0559 | Regression_eta: 0.0425 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0580 | LR: 3.16e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8104s | Phase Times: max_pre_step_barrier: 0.0253 | max_batch_to_device: 0.0066 | max_forward: 1.3051 | max_loss: 0.1764 | max_backward: 1.7331 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.5706 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0895 | Memory: rss_MB: 7502.0 | uss_MB: 7487.9 | child_rss_MB: 3548.7 | allocated_MB: 347.7 | reserved_MB: 24528.0 | inactive_split_MB: 288.7 +rank4 [2026-10-29 11:13:14,784] INFO : Step 33800/50000 rank4 | Train Loss: 1.0442 | Train Loss Unweighted: 0.4970 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1421 | Classification: 0.0764 | Classification_binary: 0.1525 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1318 | Regression_eta: 0.0014 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1726 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0683 | Regression_eta: 0.0423 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0468 | LR: 3.14e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2985s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7370 | max_loss: 0.2458 | max_backward: 1.5902 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.6292 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1331 | Memory: rss_MB: 7496.1 | uss_MB: 7482.2 | child_rss_MB: 3539.4 | allocated_MB: 344.2 | reserved_MB: 24528.0 | inactive_split_MB: 271.5 +rank4 [2026-10-29 11:17:10,800] INFO : Step 33900/50000 rank4 | Train Loss: 1.0341 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1352 | Classification: 0.0865 | Classification_binary: 0.1441 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1254 | Regression_eta: 0.0016 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1429 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0720 | Regression_eta: 0.0833 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0608 | LR: 3.11e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3852s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.6305 | max_loss: 0.1507 | max_backward: 1.4015 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9471 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0605 | Memory: rss_MB: 7498.1 | uss_MB: 7488.3 | child_rss_MB: 3543.5 | allocated_MB: 344.1 | reserved_MB: 24528.0 | inactive_split_MB: 268.7 +rank4 [2026-10-29 11:21:05,086] INFO : Step 34000/50000 rank4 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1309 | Classification: 0.0985 | Classification_binary: 0.1707 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1170 | Regression_eta: 0.0014 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0560 | Regression_eta: 0.0421 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0515 | LR: 3.09e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.6373s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7414 | max_loss: 0.1458 | max_backward: 1.7218 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 7499.1 | uss_MB: 7491.4 | child_rss_MB: 3548.4 | allocated_MB: 342.9 | reserved_MB: 24528.0 | inactive_split_MB: 277.5 +rank4 [2026-10-29 11:21:06,162] INFO : Running validation on rank4 +rank4 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 11:21:56,884] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:21:57,379] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 11:22:16,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 11:22:18,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 11:22:19,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 11:22:21,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 11:22:31,440] INFO : Time taken to make predictions on device 4 is: 0.28 min +rank4 [2026-10-29 11:22:32,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:22:32,507] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 11:22:48,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 11:22:49,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 11:22:50,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 11:22:51,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 11:23:01,033] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 11:23:02,011] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:23:02,514] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 11:23:18,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 11:23:19,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 11:23:19,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 11:23:20,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 11:23:30,405] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 11:27:45,686] INFO : Step 34100/50000 rank4 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5582 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1263 | Classification: 0.1031 | Classification_binary: 0.1801 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1358 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.1580 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0587 | Regression_eta: 0.0411 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0548 | LR: 3.06e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8613s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9025 | max_loss: 0.1870 | max_backward: 1.7732 | max_optimizer_step: 0.2826 | max_tensorboard_logging: 0.8215 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0738 | Memory: rss_MB: 9109.5 | uss_MB: 8140.4 | child_rss_MB: 3555.5 | allocated_MB: 342.7 | reserved_MB: 22124.0 | inactive_split_MB: 211.6 +rank4 [2026-10-29 11:31:40,882] INFO : Step 34200/50000 rank4 | Train Loss: 0.9931 | Train Loss Unweighted: 0.4794 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1232 | Classification: 0.0771 | Classification_binary: 0.1377 | HitGrouping: 0.1996 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1301 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1270 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0697 | Regression_eta: 0.0271 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0580 | LR: 3.04e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.7181s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.8809 | max_loss: 0.3036 | max_backward: 1.8297 | max_optimizer_step: 0.2897 | max_tensorboard_logging: 0.6835 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7608.6 | uss_MB: 6291.1 | child_rss_MB: 3559.3 | allocated_MB: 339.0 | reserved_MB: 22508.0 | inactive_split_MB: 220.4 +rank4 [2026-10-29 11:35:37,831] INFO : Step 34300/50000 rank4 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4920 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1192 | Classification: 0.0965 | Classification_binary: 0.1484 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0662 | Regression_eta: 0.0451 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0763 | LR: 3.02e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.3206s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7255 | max_loss: 0.1427 | max_backward: 1.3652 | max_optimizer_step: 0.2496 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7610.6 | uss_MB: 7596.5 | child_rss_MB: 3557.3 | allocated_MB: 345.0 | reserved_MB: 22508.0 | inactive_split_MB: 216.4 +rank4 [2026-10-29 11:39:28,036] INFO : Step 34400/50000 rank4 | Train Loss: 1.0662 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2383 | Cardinality: 0.1289 | Classification: 0.0974 | Classification_binary: 0.1749 | HitGrouping: 0.1864 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1281 | Regression_eta: 0.0014 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0858 | Classification_binary: 0.2060 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0721 | Regression_eta: 0.0415 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0705 | LR: 2.99e-04 | DataLoad Time: 0.0184s | Model Forward Time: 6.1262s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9438 | max_loss: 0.1560 | max_backward: 2.9359 | max_optimizer_step: 0.2526 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7613.7 | uss_MB: 6304.5 | child_rss_MB: 3550.6 | allocated_MB: 345.3 | reserved_MB: 23132.0 | inactive_split_MB: 282.7 +rank4 [2026-10-29 11:43:17,762] INFO : Step 34500/50000 rank4 | Train Loss: 1.0328 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1103 | Classification: 0.0867 | Classification_binary: 0.1518 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0014 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1693 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0563 | Regression_eta: 0.0455 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0649 | LR: 2.97e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.6267s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.8153 | max_loss: 0.1429 | max_backward: 1.8299 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.5315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7615.7 | uss_MB: 6643.2 | child_rss_MB: 3549.5 | allocated_MB: 341.4 | reserved_MB: 23132.0 | inactive_split_MB: 238.6 +rank4 [2026-10-29 11:47:05,943] INFO : Step 34600/50000 rank4 | Train Loss: 1.0317 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1201 | Classification: 0.0877 | Classification_binary: 0.1434 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1359 | Regression_eta: 0.0014 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1396 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0633 | Regression_eta: 0.0567 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0623 | LR: 2.95e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2929s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.6882 | max_loss: 0.1360 | max_backward: 1.3984 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5787 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7607.9 | uss_MB: 6553.3 | child_rss_MB: 3582.8 | allocated_MB: 343.8 | reserved_MB: 23132.0 | inactive_split_MB: 233.1 +rank4 [2026-10-29 11:51:12,923] INFO : Step 34700/50000 rank4 | Train Loss: 1.1152 | Train Loss Unweighted: 0.5285 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1612 | Classification: 0.0937 | Classification_binary: 0.1734 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1283 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1836 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0763 | Regression_eta: 0.0293 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0582 | LR: 2.92e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5253s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7859 | max_loss: 0.2571 | max_backward: 1.9119 | max_optimizer_step: 0.2516 | max_tensorboard_logging: 0.5032 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7610.6 | uss_MB: 7599.4 | child_rss_MB: 3559.6 | allocated_MB: 343.9 | reserved_MB: 23132.0 | inactive_split_MB: 446.8 +rank4 [2026-10-29 11:54:56,884] INFO : Step 34800/50000 rank4 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5142 | Train Loss Components: Auxiliary: 0.2371 | Cardinality: 0.1583 | Classification: 0.0857 | Classification_binary: 0.1750 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1255 | Regression_eta: 0.0013 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0483 | Regression_eta: 0.0309 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0348 | LR: 2.90e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2204s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0423 | max_forward: 0.8214 | max_loss: 0.1472 | max_backward: 1.4019 | max_optimizer_step: 0.3234 | max_tensorboard_logging: 0.5950 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7613.6 | uss_MB: 7602.4 | child_rss_MB: 3541.0 | allocated_MB: 344.5 | reserved_MB: 23132.0 | inactive_split_MB: 201.5 +rank4 [2026-10-29 11:58:48,855] INFO : Step 34900/50000 rank4 | Train Loss: 0.9512 | Train Loss Unweighted: 0.4678 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1120 | Classification: 0.0788 | Classification_binary: 0.1318 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1262 | Regression_eta: 0.0014 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1121 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0474 | Regression_eta: 0.0315 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0635 | LR: 2.88e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2630s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0986 | max_forward: 0.7397 | max_loss: 0.1421 | max_backward: 1.3918 | max_optimizer_step: 0.3560 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7614.6 | uss_MB: 7602.0 | child_rss_MB: 3543.9 | allocated_MB: 341.5 | reserved_MB: 23500.0 | inactive_split_MB: 262.9 +rank4 [2026-10-29 12:02:43,664] INFO : Step 35000/50000 rank4 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1401 | Classification: 0.0852 | Classification_binary: 0.1555 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0012 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0490 | Regression_eta: 0.0319 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0458 | LR: 2.85e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.7033s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0159 | max_forward: 0.7961 | max_loss: 0.2615 | max_backward: 2.0104 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6382 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7618.3 | uss_MB: 7606.3 | child_rss_MB: 3544.7 | allocated_MB: 345.2 | reserved_MB: 23500.0 | inactive_split_MB: 284.5 +rank4 [2026-10-29 12:02:44,719] INFO : Running validation on rank4 +rank4 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 12:03:33,662] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 12:03:34,210] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 12:04:01,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 12:04:02,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 12:04:03,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 12:04:05,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 12:04:15,253] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 12:04:18,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 12:04:18,514] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 12:04:34,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 12:04:35,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 12:04:36,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 12:04:37,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 12:04:47,337] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 12:04:48,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 12:04:48,826] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 12:05:04,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 12:05:05,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 12:05:05,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 12:05:06,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 12:05:16,385] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 12:09:34,457] INFO : Step 35100/50000 rank4 | Train Loss: 1.1060 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1387 | Classification: 0.0926 | Classification_binary: 0.1848 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0014 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1636 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0564 | Regression_eta: 0.0235 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0408 | LR: 2.83e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.6642s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.9070 | max_loss: 0.1513 | max_backward: 2.1746 | max_optimizer_step: 0.2850 | max_tensorboard_logging: 1.1439 | max_tensorboard_flush: 0.0024 | max_post_step_barrier: 0.0768 | Memory: rss_MB: 9217.3 | uss_MB: 7564.1 | child_rss_MB: 3570.6 | allocated_MB: 344.3 | reserved_MB: 22622.0 | inactive_split_MB: 852.9 +rank4 [2026-10-29 12:13:24,279] INFO : Step 35200/50000 rank4 | Train Loss: 1.0386 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1293 | Classification: 0.0826 | Classification_binary: 0.1559 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1247 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0664 | Regression_eta: 0.0731 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0755 | LR: 2.81e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.5308s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.8193 | max_loss: 0.1608 | max_backward: 1.3752 | max_optimizer_step: 0.2535 | max_tensorboard_logging: 0.9185 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0700 | Memory: rss_MB: 7738.0 | uss_MB: 7717.0 | child_rss_MB: 3546.7 | allocated_MB: 341.8 | reserved_MB: 25614.0 | inactive_split_MB: 192.0 +rank4 [2026-10-29 12:17:18,033] INFO : Step 35300/50000 rank4 | Train Loss: 0.9736 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1059 | Classification: 0.0970 | Classification_binary: 0.1371 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0016 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0496 | Regression_eta: 0.0381 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0649 | LR: 2.79e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6480s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6434 | max_loss: 0.2341 | max_backward: 1.6834 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0588 | Memory: rss_MB: 7740.0 | uss_MB: 7297.3 | child_rss_MB: 3543.2 | allocated_MB: 343.2 | reserved_MB: 25614.0 | inactive_split_MB: 226.8 +rank4 [2026-10-29 12:21:06,032] INFO : Step 35400/50000 rank4 | Train Loss: 1.1186 | Train Loss Unweighted: 0.5363 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1393 | Classification: 0.1069 | Classification_binary: 0.1767 | HitGrouping: 0.1943 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0016 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0549 | Regression_eta: 0.0508 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0684 | LR: 2.76e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8138s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0060 | max_forward: 0.7591 | max_loss: 0.1562 | max_backward: 1.6040 | max_optimizer_step: 0.2543 | max_tensorboard_logging: 0.9624 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.0563 | Memory: rss_MB: 7735.5 | uss_MB: 7248.0 | child_rss_MB: 3540.1 | allocated_MB: 345.3 | reserved_MB: 25762.0 | inactive_split_MB: 277.5 +rank4 [2026-10-29 12:24:54,117] INFO : Step 35500/50000 rank4 | Train Loss: 1.1216 | Train Loss Unweighted: 0.5466 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1527 | Classification: 0.0970 | Classification_binary: 0.1767 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1342 | Regression_eta: 0.0014 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1617 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0610 | Regression_eta: 0.0414 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0611 | LR: 2.74e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4067s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0648 | max_forward: 0.8748 | max_loss: 0.1544 | max_backward: 1.6387 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 0.5130 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0632 | Memory: rss_MB: 7738.5 | uss_MB: 7721.7 | child_rss_MB: 3543.4 | allocated_MB: 343.4 | reserved_MB: 25762.0 | inactive_split_MB: 284.4 +rank4 [2026-10-29 12:28:50,969] INFO : Step 35600/50000 rank4 | Train Loss: 1.0380 | Train Loss Unweighted: 0.5039 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1124 | Classification: 0.0922 | Classification_binary: 0.1405 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1493 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0541 | Regression_eta: 0.0420 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0568 | LR: 2.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8991s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0559 | max_forward: 0.7376 | max_loss: 0.1415 | max_backward: 1.1754 | max_optimizer_step: 0.2524 | max_tensorboard_logging: 0.5225 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1603 | Memory: rss_MB: 7739.4 | uss_MB: 7724.9 | child_rss_MB: 3557.9 | allocated_MB: 343.8 | reserved_MB: 25762.0 | inactive_split_MB: 233.6 +rank4 [2026-10-29 12:32:54,485] INFO : Step 35700/50000 rank4 | Train Loss: 1.0352 | Train Loss Unweighted: 0.5000 | Train Loss Components: Auxiliary: 0.2288 | Cardinality: 0.1199 | Classification: 0.1043 | Classification_binary: 0.1492 | HitGrouping: 0.1850 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1212 | Regression_eta: 0.0015 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1456 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0541 | Regression_eta: 0.0347 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0735 | LR: 2.70e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3738s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0360 | max_forward: 0.7358 | max_loss: 0.1419 | max_backward: 1.6761 | max_optimizer_step: 0.2531 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7741.1 | uss_MB: 7726.8 | child_rss_MB: 3543.5 | allocated_MB: 344.2 | reserved_MB: 26034.0 | inactive_split_MB: 271.8 +rank4 [2026-10-29 12:36:36,039] INFO : Step 35800/50000 rank4 | Train Loss: 1.0929 | Train Loss Unweighted: 0.5323 | Train Loss Components: Auxiliary: 0.2446 | Cardinality: 0.1241 | Classification: 0.0986 | Classification_binary: 0.1801 | HitGrouping: 0.1908 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0014 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.2112 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0597 | Regression_eta: 0.0379 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0852 | LR: 2.68e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1526s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7732 | max_loss: 0.1506 | max_backward: 1.4048 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0581 | Memory: rss_MB: 7740.3 | uss_MB: 7206.4 | child_rss_MB: 3548.3 | allocated_MB: 344.1 | reserved_MB: 26034.0 | inactive_split_MB: 276.1 +rank4 [2026-10-29 14:48:22,181] INFO : Initializing model with seed=12345; process seed=12349 +rank4 [2026-10-29 14:48:22,186] INFO : configured dtype=torch.bfloat16 for autocast +rank4 [2026-10-29 14:48:45,046] INFO : Instantiating model +rank4 [2026-10-29 14:48:45,046] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank4 [2026-10-29 14:48:45,047] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank4 [2026-10-29 14:48:45,065] INFO : Input encoding initialization took 0.02s +rank4 [2026-10-29 14:48:45,065] INFO : nn0 parameters: 171776 +rank4 [2026-10-29 14:48:45,065] INFO : Initializing shared backbone layers, num_convs=6 +rank4 [2026-10-29 14:48:45,065] INFO : Initializing backbone layer 0 +rank4 [2026-10-29 14:48:45,065] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:48:45,071] INFO : Initializing backbone layer 1 +rank4 [2026-10-29 14:48:45,071] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:48:45,076] INFO : Initializing backbone layer 2 +rank4 [2026-10-29 14:48:45,076] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:48:45,082] INFO : Initializing backbone layer 3 +rank4 [2026-10-29 14:48:45,082] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:48:45,087] INFO : Initializing backbone layer 4 +rank4 [2026-10-29 14:48:45,087] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:48:45,092] INFO : Initializing backbone layer 5 +rank4 [2026-10-29 14:48:45,092] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank4 [2026-10-29 14:48:45,097] INFO : Backbone initialization took 0.03s +rank4 [2026-10-29 14:48:45,098] INFO : backbone parameters: 2374656 +rank4 [2026-10-29 14:48:45,098] INFO : Initializing output DNNs +rank4 [2026-10-29 14:48:45,136] INFO : Output DNNs initialization took 0.04s +rank4 [2026-10-29 14:48:45,136] INFO : backbone_mode=BackboneMode.SHARED +rank4 [2026-10-29 14:48:45,136] INFO : jagged_attention enabled=True +rank4 [2026-10-29 14:48:45,136] INFO : flash_attn_varlen enabled=True +rank4 [2026-10-29 14:48:45,136] INFO : task_queries enabled=False +rank4 [2026-10-29 14:48:45,136] INFO : classification_norm parameters: 0 +rank4 [2026-10-29 14:48:45,136] INFO : regression_norm parameters: 0 +rank4 [2026-10-29 14:48:45,136] INFO : classification_query parameters: 0 +rank4 [2026-10-29 14:48:45,136] INFO : regression_query parameters: 0 +rank4 [2026-10-29 14:48:45,136] INFO : classification_readout parameters: 0 +rank4 [2026-10-29 14:48:45,136] INFO : regression_readout parameters: 0 +rank4 [2026-10-29 14:48:45,137] INFO : output_mode=set +rank4 [2026-10-29 14:48:45,137] INFO : set_decoder parameters: 4713812 +rank4 [2026-10-29 14:48:45,137] INFO : nn_binary_particle parameters: 0 +rank4 [2026-10-29 14:48:45,137] INFO : nn_pid parameters: 0 +rank4 [2026-10-29 14:48:45,137] INFO : nn_pt parameters: 0 +rank4 [2026-10-29 14:48:45,137] INFO : nn_eta parameters: 0 +rank4 [2026-10-29 14:48:45,137] INFO : nn_sin_phi parameters: 0 +rank4 [2026-10-29 14:48:45,137] INFO : nn_cos_phi parameters: 0 +rank4 [2026-10-29 14:48:45,137] INFO : nn_energy parameters: 0 +rank4 [2026-10-29 14:48:45,138] INFO : Total MLPF parameters: 7260244 +rank4 [2026-10-29 14:48:45,138] INFO : MLPF __init__ done +rank4 [2026-10-29 14:48:45,138] INFO : Instantiated model +rank4 [2026-10-29 14:48:45,138] INFO : Moving model to device rank=4 +rank4 [2026-10-29 14:48:45,170] INFO : Moved model to device rank=4 +rank4 [2026-10-29 14:48:45,172] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank4 [2026-10-29 14:48:45,515] INFO : Restoring training from step 35001 +rank4 [2026-10-29 14:48:45,521] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth +rank4 [2026-10-29 14:48:45,521] INFO : Configured model for SyncBatchNorm rank=4 +rank4 [2026-10-29 14:48:45,761] INFO : Configured model for DistributedDataParallel rank=4 +rank4 [2026-10-29 14:48:45,874] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank4 [2026-10-29 14:48:45,874] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank4 [2026-10-29 14:48:45,876] INFO : Trainable parameters: 7260244 +rank4 [2026-10-29 14:48:45,876] INFO : Non-trainable parameters: 0 +rank4 [2026-10-29 14:48:45,876] INFO : Total parameters: 7260244 +rank4 [2026-10-29 14:48:45,878] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank4 [2026-10-29 14:48:45,878] INFO : Getting dataloaders +rank4 [2026-10-29 14:48:45,879] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:48:51,692] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:48:52,731] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:48:53,925] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-29 14:48:53,925] INFO : Creating WeightedMixIterator with 1 data loaders. +rank4 [2026-10-29 14:48:53,925] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:48:54,191] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:48:54,456] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:48:54,715] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-29 14:48:54,716] INFO : Creating InterleavedIterator with 1 data loaders. +rank4 [2026-10-29 14:49:08,535] INFO : Got dataloaders +rank4 [2026-10-29 14:49:08,535] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13888] of epochs [1] +rank4 [2026-10-29 14:49:08,535] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank4 [2026-10-29 14:49:08,535] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank4 [2026-10-29 14:49:08,535] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank4 [2026-10-29 14:49:10,015] INFO : loader split=train rank=4 len=21112 +rank4 [2026-10-29 14:49:10,015] INFO : loader split=valid rank=4 len=12 +rank4 [2026-10-29 14:49:10,015] INFO : Starting training from step 35001 to 50000 on rank 4 of 8 +rank4 [2026-10-29 14:49:10,016] INFO : Creating train_iterator +rank4 [2026-10-29 14:49:10,016] INFO : ResumableSampler physical:train loading state: start_index=222208 +rank4 [2026-10-29 14:49:24,185] INFO : Created train_iterator +rank4 [2026-10-29 14:53:12,251] INFO : Step 35100/50000 rank4 | Train Loss: 1.0826 | Train Loss Unweighted: 0.5185 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1393 | Classification: 0.0961 | Classification_binary: 0.1822 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1574 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0577 | Regression_eta: 0.0210 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0320 | LR: 2.83e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.4924s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.7340 | max_loss: 0.1651 | max_backward: 1.4152 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4761 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4625.8 | uss_MB: 3388.6 | child_rss_MB: 5578.9 | allocated_MB: 372.6 | reserved_MB: 22674.0 | inactive_split_MB: 823.1 +rank4 [2026-10-29 14:56:45,512] INFO : Step 35200/50000 rank4 | Train Loss: 1.0562 | Train Loss Unweighted: 0.5125 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1301 | Classification: 0.0878 | Classification_binary: 0.1540 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1331 | Regression_eta: 0.0017 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1908 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0681 | Regression_eta: 0.0675 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0917 | LR: 2.81e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.1446s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6042 | max_loss: 0.1411 | max_backward: 1.1580 | max_optimizer_step: 0.1447 | max_tensorboard_logging: 0.5108 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4625.0 | uss_MB: 3391.6 | child_rss_MB: 5591.1 | allocated_MB: 368.1 | reserved_MB: 25668.0 | inactive_split_MB: 192.7 +rank4 [2026-10-29 15:00:19,321] INFO : Step 35300/50000 rank4 | Train Loss: 0.9962 | Train Loss Unweighted: 0.4725 | Train Loss Components: Auxiliary: 0.2160 | Cardinality: 0.1172 | Classification: 0.0927 | Classification_binary: 0.1341 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1251 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0628 | Regression_eta: 0.0410 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0525 | LR: 2.79e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.0283s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.5940 | max_loss: 0.1358 | max_backward: 1.1911 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0027 | Memory: rss_MB: 4636.4 | uss_MB: 3402.6 | child_rss_MB: 5598.9 | allocated_MB: 371.5 | reserved_MB: 25668.0 | inactive_split_MB: 207.6 +rank4 [2026-10-29 15:03:53,851] INFO : Step 35400/50000 rank4 | Train Loss: 1.1270 | Train Loss Unweighted: 0.5445 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1356 | Classification: 0.0990 | Classification_binary: 0.1805 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1297 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1996 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0539 | Regression_eta: 0.0501 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0651 | LR: 2.76e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.6489s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1507 | max_backward: 1.5023 | max_optimizer_step: 0.2459 | max_tensorboard_logging: 0.5679 | max_tensorboard_flush: 0.0064 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4646.3 | uss_MB: 3410.8 | child_rss_MB: 5599.4 | allocated_MB: 371.4 | reserved_MB: 25816.0 | inactive_split_MB: 263.6 +rank4 [2026-10-29 15:07:33,751] INFO : Step 35500/50000 rank4 | Train Loss: 1.0502 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1302 | Classification: 0.0904 | Classification_binary: 0.1703 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1250 | Regression_eta: 0.0013 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0649 | Regression_eta: 0.0246 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0466 | LR: 2.74e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.6104s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0737 | max_forward: 0.7546 | max_loss: 0.1581 | max_backward: 1.4977 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4644.7 | uss_MB: 3412.9 | child_rss_MB: 5568.4 | allocated_MB: 371.1 | reserved_MB: 25816.0 | inactive_split_MB: 263.4 +rank4 [2026-10-29 15:11:19,389] INFO : Step 35600/50000 rank4 | Train Loss: 1.0258 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2226 | Cardinality: 0.1119 | Classification: 0.0922 | Classification_binary: 0.1418 | HitGrouping: 0.1950 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1286 | Regression_eta: 0.0015 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0444 | Regression_eta: 0.0425 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0494 | LR: 2.72e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.2539s | Phase Times: max_pre_step_barrier: 0.0929 | max_batch_to_device: 0.0064 | max_forward: 0.5696 | max_loss: 0.1430 | max_backward: 1.1627 | max_optimizer_step: 0.1475 | max_tensorboard_logging: 0.5805 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4646.9 | uss_MB: 3412.7 | child_rss_MB: 5602.7 | allocated_MB: 370.5 | reserved_MB: 25816.0 | inactive_split_MB: 212.3 +rank4 [2026-10-29 15:15:08,570] INFO : Step 35700/50000 rank4 | Train Loss: 1.0162 | Train Loss Unweighted: 0.5013 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1185 | Classification: 0.0946 | Classification_binary: 0.1537 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1248 | Regression_eta: 0.0015 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1541 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0626 | Regression_eta: 0.0480 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0729 | LR: 2.70e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.5921s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7397 | max_loss: 0.1413 | max_backward: 1.6688 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4646.6 | uss_MB: 3411.7 | child_rss_MB: 5604.2 | allocated_MB: 372.2 | reserved_MB: 26084.0 | inactive_split_MB: 249.8 +rank4 [2026-10-29 15:18:49,408] INFO : Step 35800/50000 rank4 | Train Loss: 1.0884 | Train Loss Unweighted: 0.5265 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1296 | Classification: 0.0948 | Classification_binary: 0.1803 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2123 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0640 | Regression_eta: 0.0408 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0733 | LR: 2.68e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2023s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0199 | max_forward: 0.9402 | max_loss: 0.2389 | max_backward: 1.7889 | max_optimizer_step: 0.2466 | max_tensorboard_logging: 0.7071 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4643.4 | uss_MB: 3414.5 | child_rss_MB: 5602.6 | allocated_MB: 371.6 | reserved_MB: 26084.0 | inactive_split_MB: 257.5 +rank4 [2026-10-29 15:22:30,828] INFO : Step 35900/50000 rank4 | Train Loss: 1.0760 | Train Loss Unweighted: 0.5299 | Train Loss Components: Auxiliary: 0.2328 | Cardinality: 0.1277 | Classification: 0.0983 | Classification_binary: 0.1568 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1349 | Regression_eta: 0.0016 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0571 | Regression_eta: 0.0396 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0646 | LR: 2.65e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.2894s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6268 | max_loss: 0.2373 | max_backward: 1.2570 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.4934 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0016 | Memory: rss_MB: 4639.4 | uss_MB: 3412.6 | child_rss_MB: 5605.5 | allocated_MB: 369.7 | reserved_MB: 26800.0 | inactive_split_MB: 294.3 +rank4 [2026-10-29 15:26:13,159] INFO : Step 36000/50000 rank4 | Train Loss: 1.0831 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1276 | Classification: 0.1031 | Classification_binary: 0.1434 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1452 | Regression_eta: 0.0017 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0583 | Regression_eta: 0.0474 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0599 | LR: 2.63e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.2181s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6266 | max_loss: 0.1437 | max_backward: 1.3209 | max_optimizer_step: 0.1464 | max_tensorboard_logging: 0.4977 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4620.1 | uss_MB: 3411.5 | child_rss_MB: 5582.3 | allocated_MB: 368.7 | reserved_MB: 26800.0 | inactive_split_MB: 270.9 +rank4 [2026-10-29 15:26:14,074] INFO : Running validation on rank4 +rank4 [2026-10-29 15:26:14,074] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 15:26:14,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 15:27:03,341] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:27:03,871] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 15:27:29,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 15:27:30,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 15:27:31,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 15:27:32,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 15:27:42,275] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 15:27:42,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:27:43,314] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 15:27:59,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 15:28:00,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 15:28:01,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 15:28:01,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 15:28:11,955] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 15:28:12,651] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:28:13,126] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 15:28:28,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 15:28:29,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 15:28:29,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 15:28:30,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 15:28:40,594] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 15:32:50,524] INFO : Step 36100/50000 rank4 | Train Loss: 1.0137 | Train Loss Unweighted: 0.5026 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1079 | Classification: 0.1014 | Classification_binary: 0.1522 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1216 | Regression_eta: 0.0013 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0630 | Regression_eta: 0.0309 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0787 | LR: 2.61e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0460s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0065 | max_forward: 0.6968 | max_loss: 0.2782 | max_backward: 1.4278 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.9130 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6101.3 | uss_MB: 4907.2 | child_rss_MB: 5609.1 | allocated_MB: 372.5 | reserved_MB: 23174.0 | inactive_split_MB: 246.6 +rank4 [2026-10-29 15:36:33,882] INFO : Step 36200/50000 rank4 | Train Loss: 1.0439 | Train Loss Unweighted: 0.5147 | Train Loss Components: Auxiliary: 0.2338 | Cardinality: 0.1304 | Classification: 0.0909 | Classification_binary: 0.1697 | HitGrouping: 0.1917 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1255 | Regression_eta: 0.0011 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0401 | Regression_energy: 0.0652 | Regression_eta: 0.0254 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0502 | LR: 2.59e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5404s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7440 | max_loss: 0.1544 | max_backward: 1.4037 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 6101.4 | uss_MB: 4909.6 | child_rss_MB: 5587.9 | allocated_MB: 367.8 | reserved_MB: 23174.0 | inactive_split_MB: 282.8 +rank4 [2026-10-29 15:40:20,924] INFO : Step 36300/50000 rank4 | Train Loss: 1.0379 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1216 | Classification: 0.0886 | Classification_binary: 0.1447 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1266 | Regression_eta: 0.0016 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0714 | Regression_eta: 0.0348 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0677 | LR: 2.57e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.3302s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6106 | max_loss: 0.2412 | max_backward: 1.3407 | max_optimizer_step: 0.1535 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1063 | Memory: rss_MB: 5450.1 | uss_MB: 4269.0 | child_rss_MB: 5611.1 | allocated_MB: 370.5 | reserved_MB: 23174.0 | inactive_split_MB: 274.2 +rank4 [2026-10-29 15:44:06,295] INFO : Step 36400/50000 rank4 | Train Loss: 1.0182 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.0976 | Classification: 0.0939 | Classification_binary: 0.1520 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1167 | Regression_eta: 0.0016 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0547 | Regression_eta: 0.0695 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0949 | LR: 2.55e-04 | DataLoad Time: 0.0145s | Model Forward Time: 3.5234s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.6772 | max_loss: 0.1711 | max_backward: 1.4058 | max_optimizer_step: 0.1518 | max_tensorboard_logging: 0.5460 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0089 | Memory: rss_MB: 5451.9 | uss_MB: 4272.1 | child_rss_MB: 5615.2 | allocated_MB: 371.8 | reserved_MB: 23382.0 | inactive_split_MB: 230.6 +rank4 [2026-10-29 15:47:54,065] INFO : Step 36500/50000 rank4 | Train Loss: 1.0953 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1370 | Classification: 0.0895 | Classification_binary: 0.1838 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0533 | Regression_eta: 0.0270 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0517 | LR: 2.52e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9819s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7475 | max_loss: 0.2537 | max_backward: 1.7352 | max_optimizer_step: 0.1520 | max_tensorboard_logging: 0.6267 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5440.5 | uss_MB: 4262.4 | child_rss_MB: 5614.9 | allocated_MB: 372.3 | reserved_MB: 24160.0 | inactive_split_MB: 275.1 +rank4 [2026-10-29 15:51:40,079] INFO : Step 36600/50000 rank4 | Train Loss: 0.9967 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1165 | Classification: 0.0804 | Classification_binary: 0.1438 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1366 | Regression_eta: 0.0013 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0647 | Regression_eta: 0.0439 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0661 | LR: 2.50e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6286s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6592 | max_loss: 0.2501 | max_backward: 1.4213 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5953 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5450.1 | uss_MB: 4272.8 | child_rss_MB: 5617.4 | allocated_MB: 367.8 | reserved_MB: 27180.0 | inactive_split_MB: 225.4 +rank4 [2026-10-29 15:55:26,196] INFO : Step 36700/50000 rank4 | Train Loss: 0.9770 | Train Loss Unweighted: 0.4817 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1070 | Classification: 0.0901 | Classification_binary: 0.1347 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0578 | Regression_eta: 0.0758 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0483 | LR: 2.48e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.2736s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0062 | max_forward: 0.5676 | max_loss: 0.2381 | max_backward: 1.1343 | max_optimizer_step: 0.2417 | max_tensorboard_logging: 0.4693 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5448.5 | uss_MB: 4274.1 | child_rss_MB: 5620.1 | allocated_MB: 370.8 | reserved_MB: 27180.0 | inactive_split_MB: 235.2 +rank4 [2026-10-29 15:59:20,982] INFO : Step 36800/50000 rank4 | Train Loss: 1.0234 | Train Loss Unweighted: 0.4967 | Train Loss Components: Auxiliary: 0.2267 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1662 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1197 | Regression_eta: 0.0013 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0612 | Regression_eta: 0.0374 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0708 | LR: 2.46e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5118s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 1.1394 | max_loss: 0.1518 | max_backward: 2.3088 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5450.3 | uss_MB: 4278.9 | child_rss_MB: 5629.5 | allocated_MB: 371.9 | reserved_MB: 27180.0 | inactive_split_MB: 230.7 +rank4 [2026-10-29 16:03:11,411] INFO : Step 36900/50000 rank4 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2268 | Cardinality: 0.1211 | Classification: 0.0855 | Classification_binary: 0.1637 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1219 | Regression_eta: 0.0015 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.1783 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0677 | Regression_eta: 0.0667 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0378 | LR: 2.44e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.4991s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0071 | max_forward: 0.7005 | max_loss: 0.1465 | max_backward: 1.2640 | max_optimizer_step: 0.1579 | max_tensorboard_logging: 0.4728 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5451.1 | uss_MB: 4280.1 | child_rss_MB: 5636.6 | allocated_MB: 372.3 | reserved_MB: 27180.0 | inactive_split_MB: 201.7 +rank4 [2026-10-29 16:07:15,145] INFO : Step 37000/50000 rank4 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2194 | Cardinality: 0.1268 | Classification: 0.0926 | Classification_binary: 0.1402 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0014 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.1361 | Regression_cos_phi: 0.1212 | Regression_energy: 0.0653 | Regression_eta: 0.0960 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0592 | LR: 2.42e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5224s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6075 | max_loss: 0.2475 | max_backward: 1.2956 | max_optimizer_step: 0.1585 | max_tensorboard_logging: 0.7613 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5445.1 | uss_MB: 4277.2 | child_rss_MB: 5637.5 | allocated_MB: 368.1 | reserved_MB: 27180.0 | inactive_split_MB: 226.3 +rank4 [2026-10-29 16:07:16,060] INFO : Running validation on rank4 +rank4 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 16:08:14,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:08:14,584] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 16:08:39,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 16:08:40,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 16:08:41,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 16:08:42,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 16:08:52,398] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 16:08:53,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:08:53,527] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 16:09:10,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 16:09:10,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 16:09:11,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 16:09:12,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 16:09:22,666] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 16:09:23,588] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:09:24,059] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 16:09:39,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 16:09:40,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 16:09:40,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 16:09:41,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 16:09:51,688] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 16:13:56,322] INFO : Step 37100/50000 rank4 | Train Loss: 1.1150 | Train Loss Unweighted: 0.5431 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1226 | Classification: 0.1105 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1340 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0636 | Regression_eta: 0.0331 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0420 | LR: 2.40e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8583s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7539 | max_loss: 0.3988 | max_backward: 1.7903 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4747 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6593.7 | uss_MB: 5426.2 | child_rss_MB: 5642.5 | allocated_MB: 372.5 | reserved_MB: 23160.0 | inactive_split_MB: 265.5 +rank4 [2026-10-29 16:17:46,448] INFO : Step 37200/50000 rank4 | Train Loss: 1.0752 | Train Loss Unweighted: 0.5181 | Train Loss Components: Auxiliary: 0.2406 | Cardinality: 0.1258 | Classification: 0.0924 | Classification_binary: 0.1812 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1200 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0588 | Regression_eta: 0.0547 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0826 | LR: 2.38e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1702s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0549 | max_forward: 0.8058 | max_loss: 0.1510 | max_backward: 1.8476 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5280 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6591.2 | uss_MB: 5426.9 | child_rss_MB: 5644.6 | allocated_MB: 371.5 | reserved_MB: 24140.0 | inactive_split_MB: 271.0 +rank4 [2026-10-29 16:21:30,617] INFO : Step 37300/50000 rank4 | Train Loss: 1.0230 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1563 | HitGrouping: 0.1981 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1213 | Regression_eta: 0.0016 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1419 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0486 | Regression_eta: 0.0380 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0440 | LR: 2.36e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0290s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.6853 | max_loss: 0.1502 | max_backward: 1.3245 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 1.0657 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5580.1 | uss_MB: 4415.2 | child_rss_MB: 5668.9 | allocated_MB: 370.9 | reserved_MB: 24140.0 | inactive_split_MB: 261.7 +rank4 [2026-10-29 16:25:14,029] INFO : Step 37400/50000 rank4 | Train Loss: 1.0130 | Train Loss Unweighted: 0.4929 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1180 | Classification: 0.0961 | Classification_binary: 0.1390 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1273 | Regression_eta: 0.0015 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1409 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0797 | Regression_eta: 0.0360 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0654 | LR: 2.34e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5034s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.5424 | max_loss: 0.1464 | max_backward: 1.0792 | max_optimizer_step: 0.1675 | max_tensorboard_logging: 0.8668 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5585.6 | uss_MB: 4420.8 | child_rss_MB: 5670.0 | allocated_MB: 368.8 | reserved_MB: 25122.0 | inactive_split_MB: 232.7 +rank4 [2026-10-29 16:30:41,705] INFO : Step 37500/50000 rank4 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1315 | Classification: 0.0992 | Classification_binary: 0.1679 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1176 | Regression_eta: 0.0013 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0623 | Regression_eta: 0.0399 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0758 | LR: 2.32e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1480s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.7240 | max_loss: 0.1472 | max_backward: 1.5381 | max_optimizer_step: 0.2707 | max_tensorboard_logging: 0.8554 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5585.8 | uss_MB: 4422.3 | child_rss_MB: 5642.8 | allocated_MB: 370.2 | reserved_MB: 26134.0 | inactive_split_MB: 179.7 +rank4 [2026-10-29 16:34:38,361] INFO : Step 37600/50000 rank4 | Train Loss: 1.0243 | Train Loss Unweighted: 0.4867 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1182 | Classification: 0.0808 | Classification_binary: 0.1619 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0680 | Regression_eta: 0.0269 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0552 | LR: 2.30e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1654s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.7069 | max_loss: 0.1543 | max_backward: 1.5139 | max_optimizer_step: 0.1704 | max_tensorboard_logging: 1.0159 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5587.9 | uss_MB: 4422.7 | child_rss_MB: 5672.0 | allocated_MB: 370.8 | reserved_MB: 26846.0 | inactive_split_MB: 273.3 +rank4 [2026-10-29 16:38:29,293] INFO : Step 37700/50000 rank4 | Train Loss: 0.9610 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2102 | Cardinality: 0.1049 | Classification: 0.0974 | Classification_binary: 0.1317 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0012 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1304 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0567 | Regression_eta: 0.0268 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0482 | LR: 2.28e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5153s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6233 | max_loss: 0.2433 | max_backward: 1.3306 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5644 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5591.0 | uss_MB: 4425.5 | child_rss_MB: 5643.2 | allocated_MB: 369.4 | reserved_MB: 28822.0 | inactive_split_MB: 231.5 +rank4 [2026-10-29 16:42:35,307] INFO : Step 37800/50000 rank4 | Train Loss: 0.9994 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1105 | Classification: 0.0952 | Classification_binary: 0.1458 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1161 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0489 | Regression_eta: 0.0232 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0441 | LR: 2.26e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8886s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.6659 | max_loss: 0.1473 | max_backward: 1.3840 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.9558 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5593.5 | uss_MB: 4427.0 | child_rss_MB: 5671.1 | allocated_MB: 371.5 | reserved_MB: 28822.0 | inactive_split_MB: 170.1 +rank4 [2026-10-29 16:46:40,061] INFO : Step 37900/50000 rank4 | Train Loss: 1.0837 | Train Loss Unweighted: 0.5094 | Train Loss Components: Auxiliary: 0.2372 | Cardinality: 0.1419 | Classification: 0.0937 | Classification_binary: 0.1795 | HitGrouping: 0.1848 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1160 | Regression_eta: 0.0015 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1845 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0538 | Regression_eta: 0.0418 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0802 | LR: 2.24e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2201s | Phase Times: max_pre_step_barrier: 0.1076 | max_batch_to_device: 0.0071 | max_forward: 0.7572 | max_loss: 0.1547 | max_backward: 1.4632 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 0.9205 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 5593.0 | uss_MB: 4429.3 | child_rss_MB: 5670.1 | allocated_MB: 370.2 | reserved_MB: 28822.0 | inactive_split_MB: 265.3 +rank4 [2026-10-29 16:50:31,519] INFO : Step 38000/50000 rank4 | Train Loss: 0.9811 | Train Loss Unweighted: 0.4782 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1213 | Classification: 0.0781 | Classification_binary: 0.1413 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1277 | Regression_eta: 0.0012 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1175 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0445 | Regression_eta: 0.0278 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0380 | LR: 2.22e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.5078s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.6120 | max_loss: 0.2171 | max_backward: 1.2526 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5836 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5590.4 | uss_MB: 4430.0 | child_rss_MB: 5667.2 | allocated_MB: 368.7 | reserved_MB: 28822.0 | inactive_split_MB: 223.4 +rank4 [2026-10-29 16:50:32,415] INFO : Running validation on rank4 +rank4 [2026-10-29 16:50:32,415] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 16:50:32,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 16:51:23,718] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:51:24,264] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 16:51:50,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 16:51:51,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 16:51:52,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 16:51:53,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 16:52:03,639] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 16:52:04,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:52:05,008] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 16:52:21,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 16:52:21,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 16:52:22,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 16:52:23,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 16:52:33,687] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 16:52:34,974] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:52:35,419] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 16:52:51,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 16:52:51,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 16:52:52,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 16:52:53,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 16:53:03,292] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 16:57:14,096] INFO : Step 38100/50000 rank4 | Train Loss: 1.0158 | Train Loss Unweighted: 0.4912 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1220 | Classification: 0.0982 | Classification_binary: 0.1455 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1224 | Regression_eta: 0.0013 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0619 | Regression_eta: 0.0342 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0574 | LR: 2.20e-04 | DataLoad Time: 0.0247s | Model Forward Time: 4.9416s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8406 | max_loss: 0.1820 | max_backward: 1.8536 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.3354 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0179 | Memory: rss_MB: 5714.6 | uss_MB: 4482.4 | child_rss_MB: 5669.8 | allocated_MB: 370.5 | reserved_MB: 24292.0 | inactive_split_MB: 257.8 +rank4 [2026-10-29 17:01:01,944] INFO : Step 38200/50000 rank4 | Train Loss: 1.0273 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2292 | Cardinality: 0.1060 | Classification: 0.0997 | Classification_binary: 0.1650 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1452 | Regression_cos_phi: 0.0352 | Regression_energy: 0.0548 | Regression_eta: 0.0254 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0312 | LR: 2.18e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.8896s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7758 | max_loss: 0.1794 | max_backward: 1.7969 | max_optimizer_step: 0.1713 | max_tensorboard_logging: 0.4698 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5717.4 | uss_MB: 4558.1 | child_rss_MB: 5664.1 | allocated_MB: 372.9 | reserved_MB: 26212.0 | inactive_split_MB: 221.1 +rank4 [2026-10-29 17:05:07,130] INFO : Step 38300/50000 rank4 | Train Loss: 1.0498 | Train Loss Unweighted: 0.5122 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1249 | Classification: 0.0913 | Classification_binary: 0.1583 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1294 | Regression_eta: 0.0013 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.1820 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0777 | Regression_eta: 0.0546 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0521 | LR: 2.16e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7354s | Phase Times: max_pre_step_barrier: 0.1103 | max_batch_to_device: 0.0086 | max_forward: 0.6730 | max_loss: 0.1509 | max_backward: 1.3787 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.4782 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5715.4 | uss_MB: 4556.5 | child_rss_MB: 5665.1 | allocated_MB: 369.2 | reserved_MB: 26212.0 | inactive_split_MB: 171.8 +rank4 [2026-10-29 17:09:08,354] INFO : Step 38400/50000 rank4 | Train Loss: 0.9354 | Train Loss Unweighted: 0.4735 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.0968 | Classification: 0.0760 | Classification_binary: 0.1246 | HitGrouping: 0.1889 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1340 | Regression_eta: 0.0013 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1177 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0681 | Regression_eta: 0.0325 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0467 | LR: 2.14e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.6857s | Phase Times: max_pre_step_barrier: 0.1107 | max_batch_to_device: 0.0062 | max_forward: 0.6635 | max_loss: 0.1419 | max_backward: 1.4771 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 0.4619 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5721.1 | uss_MB: 4559.4 | child_rss_MB: 5662.2 | allocated_MB: 370.2 | reserved_MB: 26212.0 | inactive_split_MB: 267.8 +rank4 [2026-10-29 17:13:06,090] INFO : Step 38500/50000 rank4 | Train Loss: 1.0482 | Train Loss Unweighted: 0.5077 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1228 | Classification: 0.1147 | Classification_binary: 0.1473 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1208 | Regression_eta: 0.0016 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0498 | Regression_eta: 0.0276 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0454 | LR: 2.12e-04 | DataLoad Time: 0.0198s | Model Forward Time: 3.4595s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.6383 | max_loss: 0.1393 | max_backward: 1.3136 | max_optimizer_step: 0.1695 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 5717.0 | uss_MB: 4562.8 | child_rss_MB: 5658.1 | allocated_MB: 370.2 | reserved_MB: 27188.0 | inactive_split_MB: 210.0 +rank4 [2026-10-29 17:17:04,838] INFO : Step 38600/50000 rank4 | Train Loss: 1.0321 | Train Loss Unweighted: 0.4950 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1157 | Classification: 0.0910 | Classification_binary: 0.1735 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1130 | Regression_eta: 0.0014 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1771 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0566 | Regression_eta: 0.0363 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0638 | LR: 2.11e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.0842s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0127 | max_forward: 0.7516 | max_loss: 0.1551 | max_backward: 1.6059 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.7250 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5714.3 | uss_MB: 4563.8 | child_rss_MB: 5662.6 | allocated_MB: 372.6 | reserved_MB: 28172.0 | inactive_split_MB: 283.9 +rank4 [2026-10-29 17:20:59,239] INFO : Step 38700/50000 rank4 | Train Loss: 0.9420 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1147 | Classification: 0.0828 | Classification_binary: 0.1348 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1155 | Regression_eta: 0.0013 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0536 | Regression_eta: 0.0442 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0447 | LR: 2.09e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.6830s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.8052 | max_loss: 0.2133 | max_backward: 1.3215 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5712.0 | uss_MB: 4410.4 | child_rss_MB: 5659.5 | allocated_MB: 367.4 | reserved_MB: 28172.0 | inactive_split_MB: 206.7 +rank4 [2026-10-29 17:25:08,218] INFO : Step 38800/50000 rank4 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4825 | Train Loss Components: Auxiliary: 0.2201 | Cardinality: 0.1186 | Classification: 0.0965 | Classification_binary: 0.1429 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1198 | Regression_eta: 0.0012 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0586 | Regression_eta: 0.0377 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0521 | LR: 2.07e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.6539s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.9011 | max_loss: 0.1880 | max_backward: 1.7487 | max_optimizer_step: 0.2166 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5710.9 | uss_MB: 4431.1 | child_rss_MB: 5666.7 | allocated_MB: 371.5 | reserved_MB: 28172.0 | inactive_split_MB: 220.5 +rank4 [2026-10-29 17:29:07,476] INFO : Step 38900/50000 rank4 | Train Loss: 0.9986 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2211 | Cardinality: 0.1078 | Classification: 0.0988 | Classification_binary: 0.1578 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0667 | Regression_eta: 0.0332 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0577 | LR: 2.05e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.4272s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0073 | max_forward: 0.9800 | max_loss: 0.2159 | max_backward: 2.2158 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0079 | Memory: rss_MB: 5714.2 | uss_MB: 3754.5 | child_rss_MB: 5665.7 | allocated_MB: 373.3 | reserved_MB: 29164.0 | inactive_split_MB: 189.2 +rank4 [2026-10-29 17:33:02,382] INFO : Step 39000/50000 rank4 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4758 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1227 | Classification: 0.0847 | Classification_binary: 0.1518 | HitGrouping: 0.1853 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1179 | Regression_eta: 0.0013 | Regression_pt: 0.1189 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1353 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0618 | Regression_eta: 0.0227 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0299 | LR: 2.03e-04 | DataLoad Time: 0.0188s | Model Forward Time: 3.7824s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0515 | max_forward: 0.7410 | max_loss: 0.1790 | max_backward: 1.4655 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5713.4 | uss_MB: 4569.9 | child_rss_MB: 5663.9 | allocated_MB: 370.5 | reserved_MB: 29164.0 | inactive_split_MB: 192.2 +rank4 [2026-10-29 17:33:03,294] INFO : Running validation on rank4 +rank4 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 17:33:56,304] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:33:56,830] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 17:34:24,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 17:34:25,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 17:34:26,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 17:34:27,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 17:34:37,411] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 17:34:38,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:34:38,630] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 17:34:54,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 17:34:55,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 17:34:56,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 17:34:57,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 17:35:07,064] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 17:35:07,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:35:08,263] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 17:35:24,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 17:35:24,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 17:35:25,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 17:35:26,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 17:35:36,083] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 17:39:50,704] INFO : Step 39100/50000 rank4 | Train Loss: 1.0043 | Train Loss Unweighted: 0.4886 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1149 | Classification: 0.0878 | Classification_binary: 0.1320 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1329 | Regression_eta: 0.0012 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1362 | Regression_cos_phi: 0.1039 | Regression_energy: 0.0650 | Regression_eta: 0.0274 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0616 | LR: 2.01e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6386s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0736 | max_forward: 0.7590 | max_loss: 0.4062 | max_backward: 1.4330 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.4941 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6810.5 | uss_MB: 5664.9 | child_rss_MB: 5641.2 | allocated_MB: 371.1 | reserved_MB: 23656.0 | inactive_split_MB: 194.8 +rank4 [2026-10-29 17:43:44,582] INFO : Step 39200/50000 rank4 | Train Loss: 1.0325 | Train Loss Unweighted: 0.5174 | Train Loss Components: Auxiliary: 0.2315 | Cardinality: 0.1117 | Classification: 0.1086 | Classification_binary: 0.1481 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0633 | Regression_eta: 0.0479 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0513 | LR: 2.00e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.6950s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0102 | max_forward: 0.7546 | max_loss: 0.4268 | max_backward: 1.4370 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5801.6 | uss_MB: 4660.4 | child_rss_MB: 5666.2 | allocated_MB: 371.2 | reserved_MB: 23934.0 | inactive_split_MB: 196.6 +rank4 [2026-10-29 17:47:41,749] INFO : Step 39300/50000 rank4 | Train Loss: 1.0373 | Train Loss Unweighted: 0.5050 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1241 | Classification: 0.0956 | Classification_binary: 0.1745 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1150 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0636 | Regression_eta: 0.0382 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0595 | LR: 1.98e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.2675s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0844 | max_forward: 0.8662 | max_loss: 0.2199 | max_backward: 1.7416 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5805.0 | uss_MB: 4500.0 | child_rss_MB: 5642.3 | allocated_MB: 373.1 | reserved_MB: 23934.0 | inactive_split_MB: 256.8 +rank4 [2026-10-29 17:51:43,103] INFO : Step 39400/50000 rank4 | Train Loss: 0.9741 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2189 | Cardinality: 0.1078 | Classification: 0.0808 | Classification_binary: 0.1461 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1223 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0490 | Regression_eta: 0.0274 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0309 | LR: 1.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5596s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.6778 | max_loss: 0.1515 | max_backward: 1.4156 | max_optimizer_step: 0.1740 | max_tensorboard_logging: 0.4653 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5807.5 | uss_MB: 4663.4 | child_rss_MB: 5665.4 | allocated_MB: 368.0 | reserved_MB: 23934.0 | inactive_split_MB: 199.1 +rank4 [2026-10-29 17:55:40,086] INFO : Step 39500/50000 rank4 | Train Loss: 0.9809 | Train Loss Unweighted: 0.4664 | Train Loss Components: Auxiliary: 0.2175 | Cardinality: 0.1105 | Classification: 0.0963 | Classification_binary: 0.1347 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1158 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0537 | Regression_eta: 0.0403 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0595 | LR: 1.94e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5610s | Phase Times: max_pre_step_barrier: 0.0259 | max_batch_to_device: 0.1050 | max_forward: 0.6813 | max_loss: 0.1541 | max_backward: 1.3449 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 0.5375 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0111 | Memory: rss_MB: 5806.7 | uss_MB: 4606.4 | child_rss_MB: 5664.0 | allocated_MB: 367.2 | reserved_MB: 24970.0 | inactive_split_MB: 230.9 +rank4 [2026-10-29 17:59:42,115] INFO : Step 39600/50000 rank4 | Train Loss: 1.0814 | Train Loss Unweighted: 0.5099 | Train Loss Components: Auxiliary: 0.2411 | Cardinality: 0.1344 | Classification: 0.0978 | Classification_binary: 0.1731 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0507 | Regression_eta: 0.0334 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0431 | LR: 1.93e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1494s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8839 | max_loss: 0.1464 | max_backward: 1.8057 | max_optimizer_step: 0.2568 | max_tensorboard_logging: 0.5748 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5805.3 | uss_MB: 4666.8 | child_rss_MB: 5646.1 | allocated_MB: 370.6 | reserved_MB: 24970.0 | inactive_split_MB: 243.4 +rank4 [2026-10-29 18:03:55,339] INFO : Step 39700/50000 rank4 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4877 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1185 | Classification: 0.0909 | Classification_binary: 0.1674 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1134 | Regression_eta: 0.0013 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0491 | Regression_eta: 0.0236 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0331 | LR: 1.91e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8046s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.6954 | max_loss: 0.2515 | max_backward: 1.5651 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.5599 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5806.9 | uss_MB: 4667.8 | child_rss_MB: 5665.2 | allocated_MB: 371.1 | reserved_MB: 24970.0 | inactive_split_MB: 244.9 +rank4 [2026-10-29 18:07:56,647] INFO : Step 39800/50000 rank4 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1022 | Classification: 0.0711 | Classification_binary: 0.1322 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0327 | Regression_energy: 0.0554 | Regression_eta: 0.0250 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0352 | LR: 1.89e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.9375s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0063 | max_forward: 1.0012 | max_loss: 0.2859 | max_backward: 2.1063 | max_optimizer_step: 0.1800 | max_tensorboard_logging: 0.4881 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5810.9 | uss_MB: 3839.1 | child_rss_MB: 5666.9 | allocated_MB: 368.3 | reserved_MB: 24970.0 | inactive_split_MB: 241.7 +rank4 [2026-10-29 18:11:54,171] INFO : Step 39900/50000 rank4 | Train Loss: 0.9667 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.2188 | Cardinality: 0.1055 | Classification: 0.0897 | Classification_binary: 0.1526 | HitGrouping: 0.1883 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1101 | Regression_eta: 0.0012 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0837 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0621 | Regression_eta: 0.0454 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0942 | LR: 1.88e-04 | DataLoad Time: 0.0198s | Model Forward Time: 6.9944s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.3186 | max_loss: 0.4377 | max_backward: 3.6189 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.6123 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 5815.3 | uss_MB: 4217.6 | child_rss_MB: 5667.9 | allocated_MB: 370.5 | reserved_MB: 24970.0 | inactive_split_MB: 229.4 +rank4 [2026-10-29 18:15:59,744] INFO : Step 40000/50000 rank4 | Train Loss: 1.0232 | Train Loss Unweighted: 0.5131 | Train Loss Components: Auxiliary: 0.2348 | Cardinality: 0.1100 | Classification: 0.0962 | Classification_binary: 0.1712 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1206 | Regression_eta: 0.0013 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1865 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0592 | Regression_eta: 0.0384 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0518 | LR: 1.86e-04 | DataLoad Time: 0.0183s | Model Forward Time: 7.4572s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0077 | max_forward: 0.8168 | max_loss: 0.2161 | max_backward: 1.5709 | max_optimizer_step: 0.1733 | max_tensorboard_logging: 1.1681 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.1427 | Memory: rss_MB: 5816.6 | uss_MB: 3786.6 | child_rss_MB: 5665.4 | allocated_MB: 375.8 | reserved_MB: 24970.0 | inactive_split_MB: 259.2 +rank4 [2026-10-29 18:16:02,308] INFO : Running validation on rank4 +rank4 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 18:16:53,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:16:54,180] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 18:17:19,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 18:17:20,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 18:17:21,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 18:17:22,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 18:17:32,743] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 18:17:33,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:17:33,667] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 18:17:49,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 18:17:50,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 18:17:51,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 18:17:52,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 18:18:02,343] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 18:18:03,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:18:03,746] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 18:18:19,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 18:18:20,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 18:18:21,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 18:18:21,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 18:18:31,783] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 18:22:46,459] INFO : Step 40100/50000 rank4 | Train Loss: 0.9349 | Train Loss Unweighted: 0.4412 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1220 | Classification: 0.0762 | Classification_binary: 0.1328 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1148 | Regression_eta: 0.0012 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0543 | Regression_eta: 0.0546 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0646 | LR: 1.84e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.6905s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.7093 | max_loss: 0.3974 | max_backward: 1.3310 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.5555 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 7103.1 | uss_MB: 5953.7 | child_rss_MB: 5672.5 | allocated_MB: 367.6 | reserved_MB: 22178.0 | inactive_split_MB: 265.9 +rank4 [2026-10-29 18:26:52,186] INFO : Step 40200/50000 rank4 | Train Loss: 0.9570 | Train Loss Unweighted: 0.4556 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1119 | Classification: 0.0905 | Classification_binary: 0.1293 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0013 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1353 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0619 | Regression_eta: 0.0297 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0457 | LR: 1.83e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5886s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6287 | max_loss: 0.4129 | max_backward: 1.3901 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 0.5609 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5918.3 | uss_MB: 4768.5 | child_rss_MB: 5665.4 | allocated_MB: 370.8 | reserved_MB: 22294.0 | inactive_split_MB: 233.2 +rank4 [2026-10-29 18:30:59,535] INFO : Step 40300/50000 rank4 | Train Loss: 1.0209 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1232 | Classification: 0.0991 | Classification_binary: 0.1688 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1068 | Regression_eta: 0.0015 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0925 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0554 | Regression_eta: 0.0487 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0823 | LR: 1.81e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9657s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0749 | max_forward: 0.7563 | max_loss: 0.1523 | max_backward: 1.5452 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5632 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5917.0 | uss_MB: 4772.2 | child_rss_MB: 5652.3 | allocated_MB: 371.8 | reserved_MB: 23296.0 | inactive_split_MB: 234.2 +rank4 [2026-10-29 18:35:20,932] INFO : Step 40400/50000 rank4 | Train Loss: 1.0486 | Train Loss Unweighted: 0.5003 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1357 | Classification: 0.0888 | Classification_binary: 0.1690 | HitGrouping: 0.1879 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1196 | Regression_eta: 0.0015 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1538 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0650 | Regression_eta: 0.0295 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0362 | LR: 1.79e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.7352s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0072 | max_forward: 0.7368 | max_loss: 0.2443 | max_backward: 3.1596 | max_optimizer_step: 0.1851 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5913.1 | uss_MB: 3802.2 | child_rss_MB: 5663.8 | allocated_MB: 371.2 | reserved_MB: 23386.0 | inactive_split_MB: 254.8 +rank4 [2026-10-29 18:39:23,871] INFO : Step 40500/50000 rank4 | Train Loss: 0.9309 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.0955 | Classification: 0.0829 | Classification_binary: 0.1355 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0014 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0556 | Regression_eta: 0.0841 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0671 | LR: 1.78e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7236s | Phase Times: max_pre_step_barrier: 0.0186 | max_batch_to_device: 0.0076 | max_forward: 0.6863 | max_loss: 0.2393 | max_backward: 1.3065 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.7004 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5916.1 | uss_MB: 4771.6 | child_rss_MB: 5669.9 | allocated_MB: 369.2 | reserved_MB: 24370.0 | inactive_split_MB: 237.3 +rank4 [2026-10-29 18:43:23,922] INFO : Step 40600/50000 rank4 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1087 | Classification: 0.0887 | Classification_binary: 0.1521 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.1822 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0513 | Regression_eta: 0.0479 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0971 | LR: 1.76e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3432s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.1102 | max_forward: 0.7412 | max_loss: 0.1464 | max_backward: 1.4782 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 1.0719 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5917.7 | uss_MB: 4772.7 | child_rss_MB: 5655.3 | allocated_MB: 372.3 | reserved_MB: 24370.0 | inactive_split_MB: 234.4 +rank4 [2026-10-29 18:47:14,843] INFO : Step 40700/50000 rank4 | Train Loss: 1.0611 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1374 | Classification: 0.0904 | Classification_binary: 0.1773 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1190 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1847 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0537 | Regression_eta: 0.0352 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0601 | LR: 1.75e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2489s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7422 | max_loss: 0.1589 | max_backward: 1.5298 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5918.6 | uss_MB: 4774.9 | child_rss_MB: 5665.5 | allocated_MB: 372.7 | reserved_MB: 24370.0 | inactive_split_MB: 255.3 +rank4 [2026-10-29 18:51:03,370] INFO : Step 40800/50000 rank4 | Train Loss: 0.9701 | Train Loss Unweighted: 0.4785 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.0983 | Classification: 0.0901 | Classification_binary: 0.1373 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0538 | Regression_eta: 0.0364 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0656 | LR: 1.73e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7277s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5843 | max_loss: 0.1422 | max_backward: 1.0661 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.0389 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5915.8 | uss_MB: 4772.0 | child_rss_MB: 5661.2 | allocated_MB: 367.5 | reserved_MB: 24658.0 | inactive_split_MB: 205.4 +rank4 [2026-10-29 18:54:43,248] INFO : Step 40900/50000 rank4 | Train Loss: 0.9731 | Train Loss Unweighted: 0.4636 | Train Loss Components: Auxiliary: 0.2169 | Cardinality: 0.1073 | Classification: 0.0883 | Classification_binary: 0.1426 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1140 | Regression_eta: 0.0013 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1158 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0590 | Regression_eta: 0.0392 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0628 | LR: 1.72e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8711s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.1047 | max_forward: 0.7107 | max_loss: 0.1679 | max_backward: 1.3343 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.8664 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5919.0 | uss_MB: 4778.3 | child_rss_MB: 5681.4 | allocated_MB: 372.0 | reserved_MB: 25640.0 | inactive_split_MB: 260.9 +rank4 [2026-10-29 18:58:25,754] INFO : Step 41000/50000 rank4 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1031 | Classification: 0.0930 | Classification_binary: 0.1618 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1164 | Regression_eta: 0.0014 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0631 | Regression_eta: 0.0350 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0599 | LR: 1.70e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4151s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.7604 | max_loss: 0.2558 | max_backward: 1.7603 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.8399 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 5926.0 | uss_MB: 4782.0 | child_rss_MB: 5654.3 | allocated_MB: 371.9 | reserved_MB: 25640.0 | inactive_split_MB: 195.8 +rank4 [2026-10-29 18:58:26,765] INFO : Running validation on rank4 +rank4 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 18:59:14,711] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:59:15,274] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 18:59:40,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 18:59:41,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 18:59:42,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 18:59:43,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 18:59:53,087] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 18:59:53,544] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:59:54,025] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 19:00:09,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 19:00:10,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 19:00:11,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 19:00:12,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 19:00:22,516] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 19:00:23,208] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:00:23,684] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 19:00:38,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 19:00:39,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 19:00:40,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 19:00:40,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 19:00:50,898] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 19:04:58,255] INFO : Step 41100/50000 rank4 | Train Loss: 1.0684 | Train Loss Unweighted: 0.5053 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1438 | Classification: 0.0856 | Classification_binary: 0.1616 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1271 | Regression_eta: 0.0014 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0412 | Regression_energy: 0.0718 | Regression_eta: 0.0346 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0672 | LR: 1.69e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.3313s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0066 | max_forward: 0.8006 | max_loss: 0.1726 | max_backward: 1.6699 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.6214 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0566 | Memory: rss_MB: 7184.9 | uss_MB: 5901.7 | child_rss_MB: 5647.6 | allocated_MB: 371.5 | reserved_MB: 22898.0 | inactive_split_MB: 258.4 +rank4 [2026-10-29 19:08:41,623] INFO : Step 41200/50000 rank4 | Train Loss: 0.9455 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1027 | Classification: 0.0887 | Classification_binary: 0.1242 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1223 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1088 | Regression_cos_phi: 0.0412 | Regression_energy: 0.0508 | Regression_eta: 0.0211 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0474 | LR: 1.67e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.4120s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0589 | max_forward: 0.6517 | max_loss: 0.1401 | max_backward: 1.1761 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5685 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5984.6 | uss_MB: 4847.1 | child_rss_MB: 5628.1 | allocated_MB: 370.2 | reserved_MB: 24878.0 | inactive_split_MB: 199.0 +rank4 [2026-10-29 19:12:28,248] INFO : Step 41300/50000 rank4 | Train Loss: 0.9373 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1011 | Classification: 0.0853 | Classification_binary: 0.1448 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0013 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0555 | Regression_eta: 0.0379 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0484 | LR: 1.66e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8980s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.8726 | max_loss: 0.1617 | max_backward: 1.6974 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5417 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5987.6 | uss_MB: 4852.4 | child_rss_MB: 5623.6 | allocated_MB: 370.7 | reserved_MB: 24878.0 | inactive_split_MB: 233.6 +rank4 [2026-10-29 19:16:15,464] INFO : Step 41400/50000 rank4 | Train Loss: 1.0355 | Train Loss Unweighted: 0.4965 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1363 | Classification: 0.0957 | Classification_binary: 0.1654 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1162 | Regression_eta: 0.0012 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0595 | Regression_eta: 0.0326 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0659 | LR: 1.64e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9194s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7129 | max_loss: 0.2460 | max_backward: 1.6340 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5967.8 | uss_MB: 4849.9 | child_rss_MB: 5563.5 | allocated_MB: 372.0 | reserved_MB: 24878.0 | inactive_split_MB: 305.4 +rank4 [2026-10-29 19:19:52,655] INFO : Step 41500/50000 rank4 | Train Loss: 0.9689 | Train Loss Unweighted: 0.4689 | Train Loss Components: Auxiliary: 0.2118 | Cardinality: 0.1040 | Classification: 0.0862 | Classification_binary: 0.1340 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1227 | Regression_eta: 0.0012 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1240 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0546 | Regression_eta: 0.0434 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0626 | LR: 1.63e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.4468s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.6415 | max_loss: 0.1502 | max_backward: 1.1845 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5935.8 | uss_MB: 4853.4 | child_rss_MB: 5403.5 | allocated_MB: 368.5 | reserved_MB: 24878.0 | inactive_split_MB: 173.5 +rank4 [2026-10-29 19:23:40,393] INFO : Step 41600/50000 rank4 | Train Loss: 0.9559 | Train Loss Unweighted: 0.4625 | Train Loss Components: Auxiliary: 0.2141 | Cardinality: 0.1097 | Classification: 0.0872 | Classification_binary: 0.1298 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1211 | Regression_eta: 0.0012 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1161 | Regression_cos_phi: 0.0323 | Regression_energy: 0.0528 | Regression_eta: 0.0244 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0372 | LR: 1.61e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.5876s | Phase Times: max_pre_step_barrier: 0.0755 | max_batch_to_device: 0.0062 | max_forward: 0.6849 | max_loss: 0.2047 | max_backward: 1.1646 | max_optimizer_step: 0.2235 | max_tensorboard_logging: 0.5806 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5909.3 | uss_MB: 4827.6 | child_rss_MB: 5308.7 | allocated_MB: 370.4 | reserved_MB: 24878.0 | inactive_split_MB: 287.4 +rank4 [2026-10-29 19:27:23,743] INFO : Step 41700/50000 rank4 | Train Loss: 1.0204 | Train Loss Unweighted: 0.4905 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1165 | Classification: 0.0860 | Classification_binary: 0.1552 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0497 | Regression_eta: 0.0395 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0851 | LR: 1.60e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.0308s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0773 | max_forward: 0.7754 | max_loss: 0.1486 | max_backward: 1.5759 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 0.5773 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5907.3 | uss_MB: 4857.3 | child_rss_MB: 5302.2 | allocated_MB: 370.8 | reserved_MB: 24878.0 | inactive_split_MB: 238.3 +rank4 [2026-10-29 19:31:24,301] INFO : Step 41800/50000 rank4 | Train Loss: 1.0101 | Train Loss Unweighted: 0.4703 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1282 | Classification: 0.0818 | Classification_binary: 0.1618 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1616 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0463 | Regression_eta: 0.0304 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0491 | LR: 1.58e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9778s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8147 | max_loss: 0.4259 | max_backward: 1.4886 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1200 | Memory: rss_MB: 5899.0 | uss_MB: 4855.0 | child_rss_MB: 5276.1 | allocated_MB: 371.9 | reserved_MB: 25802.0 | inactive_split_MB: 266.5 +rank4 [2026-10-29 19:35:32,345] INFO : Step 41900/50000 rank4 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4672 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1185 | Classification: 0.0878 | Classification_binary: 0.1282 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1241 | Regression_eta: 0.0011 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1130 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0552 | Regression_eta: 0.0216 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0416 | LR: 1.57e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.4633s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0473 | max_forward: 0.6783 | max_loss: 0.1431 | max_backward: 1.2347 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.5634 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5868.0 | uss_MB: 4858.2 | child_rss_MB: 5156.0 | allocated_MB: 370.8 | reserved_MB: 25802.0 | inactive_split_MB: 166.5 +rank4 [2026-10-29 19:39:32,016] INFO : Step 42000/50000 rank4 | Train Loss: 0.9837 | Train Loss Unweighted: 0.4719 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.1093 | Classification: 0.1015 | Classification_binary: 0.1413 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1128 | Regression_eta: 0.0014 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0517 | Regression_eta: 0.0273 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0381 | LR: 1.56e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.6381s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6848 | max_loss: 0.2509 | max_backward: 1.3746 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5866.0 | uss_MB: 4860.4 | child_rss_MB: 5161.1 | allocated_MB: 370.2 | reserved_MB: 25802.0 | inactive_split_MB: 219.9 +rank4 [2026-10-29 19:39:32,941] INFO : Running validation on rank4 +rank4 [2026-10-29 19:39:32,941] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 19:39:32,941] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 19:40:23,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:40:23,889] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 19:40:52,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 19:40:53,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 19:40:54,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 19:40:55,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 19:41:05,451] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 19:41:06,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:41:06,796] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 19:41:22,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 19:41:24,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 19:41:26,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 19:41:27,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 19:41:37,069] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 19:41:38,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:41:38,914] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 19:41:54,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 19:41:54,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 19:41:55,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 19:41:56,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 19:42:06,356] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 19:46:29,430] INFO : Step 42100/50000 rank4 | Train Loss: 1.0639 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1434 | Classification: 0.0999 | Classification_binary: 0.1737 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1097 | Regression_eta: 0.0012 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1617 | Regression_cos_phi: 0.0319 | Regression_energy: 0.0488 | Regression_eta: 0.0303 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0451 | LR: 1.54e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0362s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.7424 | max_loss: 0.1586 | max_backward: 1.4434 | max_optimizer_step: 0.3014 | max_tensorboard_logging: 0.6771 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 7018.7 | uss_MB: 6010.1 | child_rss_MB: 5164.1 | allocated_MB: 370.8 | reserved_MB: 23298.0 | inactive_split_MB: 265.3 +rank4 [2026-10-29 19:50:21,662] INFO : Step 42200/50000 rank4 | Train Loss: 1.0152 | Train Loss Unweighted: 0.4831 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1169 | Classification: 0.0913 | Classification_binary: 0.1515 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1190 | Regression_eta: 0.0015 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1597 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0561 | Regression_eta: 0.0643 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0602 | LR: 1.53e-04 | DataLoad Time: 0.0186s | Model Forward Time: 3.5448s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.6567 | max_loss: 0.2000 | max_backward: 1.2379 | max_optimizer_step: 0.1803 | max_tensorboard_logging: 0.5782 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5943.7 | uss_MB: 4948.2 | child_rss_MB: 5171.4 | allocated_MB: 370.9 | reserved_MB: 24984.0 | inactive_split_MB: 272.4 +rank4 [2026-10-29 19:51:24,069] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank4 [2026-10-29 19:51:24,069] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-29 19:54:15,428] INFO : Step 42300/50000 rank4 | Train Loss: 1.0224 | Train Loss Unweighted: 0.4869 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.1261 | Classification: 0.0943 | Classification_binary: 0.1704 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1897 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0647 | Regression_eta: 0.0319 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0676 | LR: 1.52e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7694s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7065 | max_loss: 0.2237 | max_backward: 1.4902 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.5353 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6010.9 | uss_MB: 5019.0 | child_rss_MB: 5170.2 | allocated_MB: 371.9 | reserved_MB: 24984.0 | inactive_split_MB: 183.6 +rank4 [2026-10-29 19:58:13,284] INFO : Step 42400/50000 rank4 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4895 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1253 | Classification: 0.0872 | Classification_binary: 0.1616 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1184 | Regression_eta: 0.0013 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1816 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0641 | Regression_eta: 0.0368 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0726 | LR: 1.50e-04 | DataLoad Time: 0.0147s | Model Forward Time: 3.7987s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7436 | max_loss: 0.2477 | max_backward: 1.5279 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5169 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5933.8 | uss_MB: 4947.5 | child_rss_MB: 5164.9 | allocated_MB: 373.0 | reserved_MB: 24984.0 | inactive_split_MB: 275.6 +rank4 [2026-10-29 20:02:13,260] INFO : Step 42500/50000 rank4 | Train Loss: 0.9211 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1056 | Classification: 0.0789 | Classification_binary: 0.1296 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1162 | Regression_eta: 0.0011 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.0985 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0402 | Regression_eta: 0.0283 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0416 | LR: 1.49e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.5528s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.6196 | max_loss: 0.2440 | max_backward: 1.3403 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 2.3828 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5926.8 | uss_MB: 4949.7 | child_rss_MB: 5158.5 | allocated_MB: 371.3 | reserved_MB: 25980.0 | inactive_split_MB: 274.9 +rank4 [2026-10-29 20:06:17,859] INFO : Step 42600/50000 rank4 | Train Loss: 0.9688 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1101 | Classification: 0.0874 | Classification_binary: 0.1472 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0012 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0485 | Regression_eta: 0.0331 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0451 | LR: 1.48e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9498s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6353 | max_loss: 0.2547 | max_backward: 1.3961 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 5918.3 | uss_MB: 4947.3 | child_rss_MB: 5167.3 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 272.5 +rank4 [2026-10-29 20:10:16,915] INFO : Step 42700/50000 rank4 | Train Loss: 1.0476 | Train Loss Unweighted: 0.4808 | Train Loss Components: Auxiliary: 0.2377 | Cardinality: 0.1284 | Classification: 0.0883 | Classification_binary: 0.1739 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1071 | Regression_eta: 0.0014 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0440 | Regression_eta: 0.0291 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0670 | LR: 1.47e-04 | DataLoad Time: 0.0103s | Model Forward Time: 5.7788s | Phase Times: max_pre_step_barrier: 0.0320 | max_batch_to_device: 0.0114 | max_forward: 2.1726 | max_loss: 0.5101 | max_backward: 2.7494 | max_optimizer_step: 0.2679 | max_tensorboard_logging: 0.6520 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5921.8 | uss_MB: 4952.4 | child_rss_MB: 5151.8 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 482.9 +rank4 [2026-10-29 20:14:17,049] INFO : Step 42800/50000 rank4 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4792 | Train Loss Components: Auxiliary: 0.2262 | Cardinality: 0.1277 | Classification: 0.0841 | Classification_binary: 0.1487 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1209 | Regression_eta: 0.0012 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0319 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0627 | Regression_eta: 0.0377 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0630 | LR: 1.45e-04 | DataLoad Time: 0.0189s | Model Forward Time: 6.2817s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0175 | max_forward: 1.3870 | max_loss: 0.5106 | max_backward: 1.9221 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 2.4017 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5924.3 | uss_MB: 4952.5 | child_rss_MB: 5165.0 | allocated_MB: 372.6 | reserved_MB: 25980.0 | inactive_split_MB: 234.4 +rank4 [2026-10-29 20:18:09,439] INFO : Step 42900/50000 rank4 | Train Loss: 0.8847 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.0993 | Classification: 0.0847 | Classification_binary: 0.1207 | HitGrouping: 0.1828 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1069 | Regression_eta: 0.0011 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0328 | Classification_binary: 0.1064 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0532 | Regression_eta: 0.0295 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0339 | LR: 1.44e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.7390s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.6133 | max_loss: 0.1478 | max_backward: 1.3445 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.7551 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0762 | Memory: rss_MB: 5924.9 | uss_MB: 4957.1 | child_rss_MB: 5179.7 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 276.9 +rank4 [2026-10-29 20:22:02,171] INFO : Step 43000/50000 rank4 | Train Loss: 0.9800 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1096 | Classification: 0.0987 | Classification_binary: 0.1606 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0012 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.1750 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0537 | Regression_eta: 0.0322 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0665 | LR: 1.43e-04 | DataLoad Time: 0.0198s | Model Forward Time: 3.9045s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0071 | max_forward: 0.7477 | max_loss: 0.1477 | max_backward: 1.5663 | max_optimizer_step: 0.1600 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 5929.7 | uss_MB: 4961.8 | child_rss_MB: 5171.4 | allocated_MB: 369.1 | reserved_MB: 25980.0 | inactive_split_MB: 209.4 +rank4 [2026-10-29 20:22:03,144] INFO : Running validation on rank4 +rank4 [2026-10-29 20:22:03,145] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 20:22:03,145] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 20:22:51,801] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:22:52,330] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 20:23:18,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 20:23:19,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 20:23:20,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 20:23:21,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 20:23:31,400] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 20:23:31,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:23:32,369] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 20:23:48,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 20:23:49,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 20:23:50,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 20:23:51,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 20:24:01,164] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 20:24:01,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:24:02,214] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 20:24:17,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 20:24:18,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 20:24:18,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 20:24:19,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 20:24:29,584] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 20:28:29,103] INFO : Step 43100/50000 rank4 | Train Loss: 0.9692 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1151 | Classification: 0.0766 | Classification_binary: 0.1586 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1062 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0692 | Regression_eta: 0.0404 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0750 | LR: 1.42e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7248s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.8051 | max_loss: 0.1438 | max_backward: 1.3224 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 7068.8 | uss_MB: 6101.3 | child_rss_MB: 5191.0 | allocated_MB: 370.5 | reserved_MB: 21096.0 | inactive_split_MB: 247.5 +rank4 [2026-10-29 20:32:30,812] INFO : Step 43200/50000 rank4 | Train Loss: 0.9431 | Train Loss Unweighted: 0.4458 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1175 | Classification: 0.0811 | Classification_binary: 0.1294 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0012 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0563 | Regression_eta: 0.0298 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0853 | LR: 1.40e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5213s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0120 | max_forward: 0.6342 | max_loss: 0.1417 | max_backward: 1.2752 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6068.4 | uss_MB: 5101.7 | child_rss_MB: 5194.2 | allocated_MB: 370.4 | reserved_MB: 22140.0 | inactive_split_MB: 198.0 +rank4 [2026-10-29 20:36:16,428] INFO : Step 43300/50000 rank4 | Train Loss: 1.0382 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1221 | Classification: 0.1031 | Classification_binary: 0.1555 | HitGrouping: 0.1894 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1173 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0510 | Regression_eta: 0.0339 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0620 | LR: 1.39e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2392s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0266 | max_forward: 0.7527 | max_loss: 0.1476 | max_backward: 1.6633 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.7701 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6071.5 | uss_MB: 5107.2 | child_rss_MB: 5175.9 | allocated_MB: 371.1 | reserved_MB: 23082.0 | inactive_split_MB: 236.4 +rank4 [2026-10-29 20:40:05,495] INFO : Step 43400/50000 rank4 | Train Loss: 1.0532 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1292 | Classification: 0.0981 | Classification_binary: 0.1785 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0015 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1873 | Regression_cos_phi: 0.0406 | Regression_energy: 0.0664 | Regression_eta: 0.0271 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0740 | LR: 1.38e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2961s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0691 | max_forward: 0.8057 | max_loss: 0.1503 | max_backward: 1.8319 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.6703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6067.0 | uss_MB: 5102.4 | child_rss_MB: 5192.5 | allocated_MB: 371.5 | reserved_MB: 23082.0 | inactive_split_MB: 776.0 +rank4 [2026-10-29 20:43:57,009] INFO : Step 43500/50000 rank4 | Train Loss: 0.9308 | Train Loss Unweighted: 0.4531 | Train Loss Components: Auxiliary: 0.2084 | Cardinality: 0.0966 | Classification: 0.0846 | Classification_binary: 0.1384 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1135 | Regression_eta: 0.0013 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1497 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0478 | Regression_eta: 0.0328 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0421 | LR: 1.37e-04 | DataLoad Time: 0.0146s | Model Forward Time: 3.6278s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0171 | max_forward: 0.6718 | max_loss: 0.1455 | max_backward: 1.4141 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 6070.4 | uss_MB: 5108.5 | child_rss_MB: 5191.0 | allocated_MB: 372.0 | reserved_MB: 23082.0 | inactive_split_MB: 228.0 +rank4 [2026-10-29 20:47:52,744] INFO : Step 43600/50000 rank4 | Train Loss: 0.9195 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.0926 | Classification: 0.0847 | Classification_binary: 0.1298 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1175 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1284 | Regression_cos_phi: 0.0244 | Regression_energy: 0.0619 | Regression_eta: 0.0272 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0414 | LR: 1.36e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5152s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.6453 | max_loss: 0.1399 | max_backward: 1.3514 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0942 | Memory: rss_MB: 6067.9 | uss_MB: 5106.0 | child_rss_MB: 5190.1 | allocated_MB: 367.6 | reserved_MB: 23082.0 | inactive_split_MB: 270.4 +rank4 [2026-10-29 20:51:54,645] INFO : Step 43700/50000 rank4 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1181 | Classification: 0.0916 | Classification_binary: 0.1553 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1009 | Regression_eta: 0.0014 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0441 | Regression_eta: 0.0353 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0362 | LR: 1.35e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.0846s | Phase Times: max_pre_step_barrier: 0.0199 | max_batch_to_device: 0.0121 | max_forward: 0.8005 | max_loss: 0.2526 | max_backward: 1.9159 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 6075.4 | uss_MB: 5110.5 | child_rss_MB: 5168.5 | allocated_MB: 371.6 | reserved_MB: 23674.0 | inactive_split_MB: 256.4 +rank4 [2026-10-29 20:55:51,187] INFO : Step 43800/50000 rank4 | Train Loss: 1.0325 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1341 | Classification: 0.0892 | Classification_binary: 0.1542 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0012 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1710 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0452 | Regression_eta: 0.0643 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0760 | LR: 1.34e-04 | DataLoad Time: 0.0186s | Model Forward Time: 3.6834s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7618 | max_loss: 0.1715 | max_backward: 1.3818 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6073.5 | uss_MB: 5110.6 | child_rss_MB: 5186.9 | allocated_MB: 371.7 | reserved_MB: 23674.0 | inactive_split_MB: 282.3 +rank4 [2026-10-29 20:59:46,231] INFO : Step 43900/50000 rank4 | Train Loss: 0.9104 | Train Loss Unweighted: 0.4325 | Train Loss Components: Auxiliary: 0.2009 | Cardinality: 0.1078 | Classification: 0.0725 | Classification_binary: 0.1272 | HitGrouping: 0.1855 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1146 | Regression_eta: 0.0011 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1233 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0570 | Regression_eta: 0.0393 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0507 | LR: 1.33e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.3855s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.6145 | max_loss: 0.1412 | max_backward: 1.1415 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5764 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6071.8 | uss_MB: 5111.9 | child_rss_MB: 5178.4 | allocated_MB: 368.3 | reserved_MB: 23674.0 | inactive_split_MB: 193.1 +rank4 [2026-10-29 21:03:43,512] INFO : Step 44000/50000 rank4 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4573 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1078 | Classification: 0.0956 | Classification_binary: 0.1421 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1083 | Regression_eta: 0.0011 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0527 | Regression_eta: 0.0418 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0478 | LR: 1.32e-04 | DataLoad Time: 0.0188s | Model Forward Time: 3.9490s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0061 | max_forward: 1.1697 | max_loss: 0.2455 | max_backward: 1.7366 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6074.9 | uss_MB: 5118.1 | child_rss_MB: 5179.2 | allocated_MB: 370.7 | reserved_MB: 23674.0 | inactive_split_MB: 207.3 +rank4 [2026-10-29 21:03:44,527] INFO : Running validation on rank4 +rank4 [2026-10-29 21:03:44,528] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 21:03:44,528] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 21:04:32,715] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:04:33,227] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 21:04:58,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 21:04:59,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 21:05:00,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 21:05:01,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 21:05:11,865] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 21:05:12,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:05:13,000] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 21:05:28,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 21:05:29,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 21:05:30,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 21:05:31,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 21:05:41,431] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 21:05:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:05:42,732] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 21:05:58,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 21:05:58,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 21:05:59,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 21:06:00,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 21:06:10,113] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 21:10:20,117] INFO : Step 44100/50000 rank4 | Train Loss: 1.0009 | Train Loss Unweighted: 0.4605 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1400 | Classification: 0.0841 | Classification_binary: 0.1644 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1044 | Regression_eta: 0.0011 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1804 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0517 | Regression_eta: 0.0278 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0484 | LR: 1.31e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.5834s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 1.2763 | max_loss: 0.5349 | max_backward: 2.2676 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 1.2755 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 7076.5 | uss_MB: 6129.8 | child_rss_MB: 5164.2 | allocated_MB: 372.3 | reserved_MB: 24224.0 | inactive_split_MB: 272.2 +rank4 [2026-10-29 21:14:15,311] INFO : Step 44200/50000 rank4 | Train Loss: 0.9801 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1146 | Classification: 0.0882 | Classification_binary: 0.1482 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1167 | Regression_eta: 0.0014 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.1659 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0504 | Regression_eta: 0.0597 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0611 | LR: 1.30e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.0041s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0132 | max_forward: 0.6993 | max_loss: 0.1574 | max_backward: 1.3862 | max_optimizer_step: 0.2915 | max_tensorboard_logging: 0.7724 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6137.2 | uss_MB: 5191.5 | child_rss_MB: 5174.4 | allocated_MB: 370.2 | reserved_MB: 26182.0 | inactive_split_MB: 176.5 +rank4 [2026-10-29 21:17:58,584] INFO : Step 44300/50000 rank4 | Train Loss: 0.8787 | Train Loss Unweighted: 0.4332 | Train Loss Components: Auxiliary: 0.1952 | Cardinality: 0.0950 | Classification: 0.0869 | Classification_binary: 0.1153 | HitGrouping: 0.1807 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0011 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1143 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0610 | Regression_eta: 0.0343 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0422 | LR: 1.29e-04 | DataLoad Time: 0.0198s | Model Forward Time: 3.7427s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0142 | max_forward: 0.7699 | max_loss: 0.2088 | max_backward: 1.4049 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 0.4839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 6137.2 | uss_MB: 4195.8 | child_rss_MB: 5178.8 | allocated_MB: 367.2 | reserved_MB: 27204.0 | inactive_split_MB: 205.8 +rank4 [2026-10-29 21:21:59,794] INFO : Step 44400/50000 rank4 | Train Loss: 0.9558 | Train Loss Unweighted: 0.4680 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.0905 | Classification: 0.0911 | Classification_binary: 0.1587 | HitGrouping: 0.1858 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1296 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0526 | Regression_eta: 0.0284 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0411 | LR: 1.28e-04 | DataLoad Time: 0.0195s | Model Forward Time: 6.3287s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0370 | max_forward: 0.7829 | max_loss: 0.1727 | max_backward: 1.7862 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 2.7361 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6138.4 | uss_MB: 5206.0 | child_rss_MB: 5161.4 | allocated_MB: 373.1 | reserved_MB: 27586.0 | inactive_split_MB: 176.3 +rank4 [2026-10-29 21:26:11,867] INFO : Step 44500/50000 rank4 | Train Loss: 0.9739 | Train Loss Unweighted: 0.4610 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1139 | Classification: 0.0849 | Classification_binary: 0.1578 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1077 | Regression_eta: 0.0012 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0490 | Regression_eta: 0.0327 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0463 | LR: 1.27e-04 | DataLoad Time: 0.0121s | Model Forward Time: 3.9230s | Phase Times: max_pre_step_barrier: 0.0250 | max_batch_to_device: 0.0065 | max_forward: 0.7930 | max_loss: 0.1668 | max_backward: 1.3363 | max_optimizer_step: 0.2871 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6136.3 | uss_MB: 5200.8 | child_rss_MB: 5177.3 | allocated_MB: 371.5 | reserved_MB: 27586.0 | inactive_split_MB: 195.8 +rank4 [2026-10-29 21:30:04,560] INFO : Step 44600/50000 rank4 | Train Loss: 0.8904 | Train Loss Unweighted: 0.4364 | Train Loss Components: Auxiliary: 0.2007 | Cardinality: 0.1041 | Classification: 0.0775 | Classification_binary: 0.1238 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1161 | Regression_eta: 0.0010 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0536 | Regression_eta: 0.0293 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0497 | LR: 1.26e-04 | DataLoad Time: 0.0181s | Model Forward Time: 3.5122s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.6364 | max_loss: 0.1431 | max_backward: 1.3712 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.4900 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6139.4 | uss_MB: 5205.0 | child_rss_MB: 5167.5 | allocated_MB: 368.4 | reserved_MB: 27586.0 | inactive_split_MB: 170.2 +rank4 [2026-10-29 21:33:51,598] INFO : Step 44700/50000 rank4 | Train Loss: 1.0003 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1252 | Classification: 0.0899 | Classification_binary: 0.1520 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1117 | Regression_eta: 0.0013 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0321 | Regression_energy: 0.0697 | Regression_eta: 0.0261 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0456 | LR: 1.25e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6392s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6952 | max_loss: 0.1455 | max_backward: 1.5015 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6145.1 | uss_MB: 5206.5 | child_rss_MB: 5162.4 | allocated_MB: 371.9 | reserved_MB: 28574.0 | inactive_split_MB: 214.1 +rank4 [2026-10-29 21:37:55,986] INFO : Step 44800/50000 rank4 | Train Loss: 0.9665 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1115 | Classification: 0.0889 | Classification_binary: 0.1605 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1042 | Regression_eta: 0.0011 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1355 | Regression_cos_phi: 0.0349 | Regression_energy: 0.0423 | Regression_eta: 0.0287 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0281 | LR: 1.24e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.8729s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7593 | max_loss: 0.1924 | max_backward: 2.8086 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.6087 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 6148.4 | uss_MB: 5210.8 | child_rss_MB: 5160.0 | allocated_MB: 370.8 | reserved_MB: 28574.0 | inactive_split_MB: 257.7 +rank4 [2026-10-29 21:41:36,914] INFO : Step 44900/50000 rank4 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.0886 | Classification_binary: 0.1466 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1121 | Regression_eta: 0.0013 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0912 | Regression_energy: 0.0473 | Regression_eta: 0.0348 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0610 | LR: 1.23e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.5194s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0101 | max_forward: 0.6531 | max_loss: 0.1431 | max_backward: 1.3230 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6145.9 | uss_MB: 5207.3 | child_rss_MB: 5182.8 | allocated_MB: 368.1 | reserved_MB: 28574.0 | inactive_split_MB: 178.8 +rank4 [2026-10-29 21:45:50,735] INFO : Step 45000/50000 rank4 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4443 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1053 | Classification: 0.0832 | Classification_binary: 0.1311 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1274 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0522 | Regression_eta: 0.0387 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0553 | LR: 1.22e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.1253s | Phase Times: max_pre_step_barrier: 0.0074 | max_batch_to_device: 0.0159 | max_forward: 1.1897 | max_loss: 0.2457 | max_backward: 1.2951 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.8342 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0919 | Memory: rss_MB: 6152.1 | uss_MB: 5213.7 | child_rss_MB: 5170.7 | allocated_MB: 368.8 | reserved_MB: 28576.0 | inactive_split_MB: 266.4 +rank4 [2026-10-29 21:45:51,739] INFO : Running validation on rank4 +rank4 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 21:46:40,561] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:46:41,117] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 21:47:08,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 21:47:09,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 21:47:10,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 21:47:12,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 21:47:22,035] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 21:47:23,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:47:23,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 21:47:40,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 21:47:41,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 21:47:42,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 21:47:42,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 21:47:52,967] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 21:47:53,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:47:53,969] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 21:48:10,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 21:48:10,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 21:48:11,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 21:48:12,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 21:48:22,315] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 21:52:56,452] INFO : Step 45100/50000 rank4 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1151 | Classification: 0.1010 | Classification_binary: 0.1562 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1118 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0550 | Regression_eta: 0.0307 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0452 | LR: 1.21e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.3051s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7307 | max_loss: 0.4409 | max_backward: 1.5823 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 7066.7 | uss_MB: 6122.4 | child_rss_MB: 5160.9 | allocated_MB: 373.1 | reserved_MB: 23280.0 | inactive_split_MB: 227.2 +rank4 [2026-10-29 21:56:57,436] INFO : Step 45200/50000 rank4 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1108 | Classification: 0.0815 | Classification_binary: 0.1593 | HitGrouping: 0.1884 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0494 | Regression_eta: 0.0471 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0507 | LR: 1.20e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.2197s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.9482 | max_loss: 0.4476 | max_backward: 1.7165 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.4656 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7064.1 | uss_MB: 6120.3 | child_rss_MB: 5157.7 | allocated_MB: 373.0 | reserved_MB: 23338.0 | inactive_split_MB: 277.4 +rank4 [2026-10-29 22:01:28,910] INFO : Step 45300/50000 rank4 | Train Loss: 0.9312 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1141 | Classification: 0.0882 | Classification_binary: 0.1299 | HitGrouping: 0.1891 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0010 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0622 | Regression_eta: 0.0270 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0350 | LR: 1.19e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.4276s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6106 | max_loss: 0.2394 | max_backward: 1.1660 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 6271.7 | uss_MB: 5329.4 | child_rss_MB: 5157.3 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 223.2 +rank4 [2026-10-29 22:05:37,388] INFO : Step 45400/50000 rank4 | Train Loss: 0.9251 | Train Loss Unweighted: 0.4554 | Train Loss Components: Auxiliary: 0.2112 | Cardinality: 0.1034 | Classification: 0.0967 | Classification_binary: 0.1405 | HitGrouping: 0.1801 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1072 | Regression_eta: 0.0012 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0332 | Regression_energy: 0.0548 | Regression_eta: 0.0257 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0431 | LR: 1.19e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9541s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7158 | max_loss: 0.2520 | max_backward: 1.7727 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6276.9 | uss_MB: 5335.5 | child_rss_MB: 5162.7 | allocated_MB: 372.6 | reserved_MB: 23338.0 | inactive_split_MB: 212.6 +rank4 [2026-10-29 22:09:49,791] INFO : Step 45500/50000 rank4 | Train Loss: 1.0136 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1238 | Classification: 0.0902 | Classification_binary: 0.1661 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1060 | Regression_eta: 0.0013 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1707 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0530 | Regression_eta: 0.0301 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0421 | LR: 1.18e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.1586s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0147 | max_forward: 1.9023 | max_loss: 0.2979 | max_backward: 3.5478 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5661 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 6272.0 | uss_MB: 4281.7 | child_rss_MB: 5159.4 | allocated_MB: 372.4 | reserved_MB: 23338.0 | inactive_split_MB: 256.1 +rank4 [2026-10-29 22:14:16,023] INFO : Step 45600/50000 rank4 | Train Loss: 0.9675 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2172 | Cardinality: 0.1193 | Classification: 0.0849 | Classification_binary: 0.1400 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1218 | Regression_eta: 0.0010 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0306 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0499 | Regression_eta: 0.0183 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0395 | LR: 1.17e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.7194s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6781 | max_loss: 0.1576 | max_backward: 1.5089 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 0.5447 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6272.0 | uss_MB: 5337.5 | child_rss_MB: 5161.4 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 216.2 +rank4 [2026-10-29 22:18:15,631] INFO : Step 45700/50000 rank4 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1961 | Cardinality: 0.0908 | Classification: 0.0837 | Classification_binary: 0.1223 | HitGrouping: 0.1794 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1108 | Regression_eta: 0.0011 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1310 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0652 | Regression_eta: 0.0182 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0402 | LR: 1.16e-04 | DataLoad Time: 0.0220s | Model Forward Time: 3.4717s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6216 | max_loss: 0.1658 | max_backward: 1.1797 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6234.4 | uss_MB: 5336.4 | child_rss_MB: 5162.7 | allocated_MB: 370.8 | reserved_MB: 23338.0 | inactive_split_MB: 226.6 +rank4 [2026-10-29 22:22:19,535] INFO : Step 45800/50000 rank4 | Train Loss: 0.9372 | Train Loss Unweighted: 0.4491 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1073 | Classification: 0.0880 | Classification_binary: 0.1481 | HitGrouping: 0.1781 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0561 | Regression_eta: 0.0340 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0379 | LR: 1.16e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.2368s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8017 | max_loss: 0.2428 | max_backward: 1.9917 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0978 | Memory: rss_MB: 6240.7 | uss_MB: 5347.0 | child_rss_MB: 5162.1 | allocated_MB: 375.2 | reserved_MB: 23338.0 | inactive_split_MB: 268.8 +rank4 [2026-10-29 22:26:26,930] INFO : Step 45900/50000 rank4 | Train Loss: 0.9600 | Train Loss Unweighted: 0.4592 | Train Loss Components: Auxiliary: 0.2179 | Cardinality: 0.1027 | Classification: 0.0816 | Classification_binary: 0.1590 | HitGrouping: 0.1875 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0421 | Regression_eta: 0.0273 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0532 | LR: 1.15e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.8431s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 1.3210 | max_loss: 0.4294 | max_backward: 3.2770 | max_optimizer_step: 0.2740 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 6235.6 | uss_MB: 5343.7 | child_rss_MB: 5161.3 | allocated_MB: 370.5 | reserved_MB: 23338.0 | inactive_split_MB: 259.6 +rank4 [2026-10-29 22:30:40,886] INFO : Step 46000/50000 rank4 | Train Loss: 0.8490 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.0938 | Classification: 0.0817 | Classification_binary: 0.1110 | HitGrouping: 0.1819 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0010 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1065 | Regression_cos_phi: 0.0290 | Regression_energy: 0.0495 | Regression_eta: 0.0242 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0597 | LR: 1.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5922s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.5955 | max_loss: 0.1430 | max_backward: 1.2483 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 0.6657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 6228.9 | uss_MB: 5344.9 | child_rss_MB: 5164.3 | allocated_MB: 365.7 | reserved_MB: 23338.0 | inactive_split_MB: 222.2 +rank4 [2026-10-29 22:30:41,828] INFO : Running validation on rank4 +rank4 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 22:31:32,964] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:31:33,519] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 22:31:59,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 22:32:01,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 22:32:02,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 22:32:04,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 22:32:14,411] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 22:32:15,390] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:32:16,222] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 22:32:32,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 22:32:33,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 22:32:33,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 22:32:34,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 22:32:44,907] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 22:32:45,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:32:46,016] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 22:33:01,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 22:33:01,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 22:33:02,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 22:33:03,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 22:33:13,243] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 22:37:44,431] INFO : Step 46100/50000 rank4 | Train Loss: 0.9299 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1082 | Classification: 0.0899 | Classification_binary: 0.1357 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1081 | Regression_eta: 0.0010 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1219 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0521 | Regression_eta: 0.0271 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0526 | LR: 1.13e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.9685s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 2.5146 | max_loss: 0.4579 | max_backward: 2.4958 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1014 | Memory: rss_MB: 6980.2 | uss_MB: 6096.7 | child_rss_MB: 5172.0 | allocated_MB: 371.5 | reserved_MB: 22784.0 | inactive_split_MB: 245.0 +rank4 [2026-10-29 22:41:43,511] INFO : Step 46200/50000 rank4 | Train Loss: 0.9944 | Train Loss Unweighted: 0.4707 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1286 | Classification: 0.0940 | Classification_binary: 0.1623 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1052 | Regression_eta: 0.0011 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0551 | Regression_eta: 0.0356 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0573 | LR: 1.13e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2024s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7685 | max_loss: 0.2359 | max_backward: 1.6855 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1253 | Memory: rss_MB: 6976.0 | uss_MB: 6094.5 | child_rss_MB: 5141.0 | allocated_MB: 373.1 | reserved_MB: 22784.0 | inactive_split_MB: 264.3 +rank4 [2026-10-29 22:45:39,133] INFO : Step 46300/50000 rank4 | Train Loss: 0.9564 | Train Loss Unweighted: 0.4515 | Train Loss Components: Auxiliary: 0.2114 | Cardinality: 0.1124 | Classification: 0.0869 | Classification_binary: 0.1435 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0408 | Regression_eta: 0.0394 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0426 | LR: 1.12e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8625s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6709 | max_loss: 0.2410 | max_backward: 1.2189 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.8303 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6276.5 | uss_MB: 5413.4 | child_rss_MB: 5109.4 | allocated_MB: 372.3 | reserved_MB: 23784.0 | inactive_split_MB: 228.2 +rank4 [2026-10-29 22:49:34,649] INFO : Step 46400/50000 rank4 | Train Loss: 0.9514 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1147 | Classification: 0.0822 | Classification_binary: 0.1248 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0010 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0507 | Regression_eta: 0.0173 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0428 | LR: 1.11e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.9556s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.6748 | max_loss: 0.4371 | max_backward: 1.4077 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.8211 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6270.0 | uss_MB: 5413.5 | child_rss_MB: 5075.5 | allocated_MB: 374.2 | reserved_MB: 23988.0 | inactive_split_MB: 241.8 +rank4 [2026-10-29 22:53:41,742] INFO : Step 46500/50000 rank4 | Train Loss: 0.9517 | Train Loss Unweighted: 0.4566 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.0937 | Classification: 0.0928 | Classification_binary: 0.1533 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1031 | Regression_eta: 0.0013 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0411 | Regression_eta: 0.0365 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0551 | LR: 1.11e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9711s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0087 | max_forward: 0.7136 | max_loss: 0.1507 | max_backward: 1.5229 | max_optimizer_step: 0.2767 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6272.9 | uss_MB: 5416.3 | child_rss_MB: 5079.6 | allocated_MB: 372.0 | reserved_MB: 23988.0 | inactive_split_MB: 248.1 +rank4 [2026-10-29 22:57:41,614] INFO : Step 46600/50000 rank4 | Train Loss: 0.9541 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.2166 | Cardinality: 0.1078 | Classification: 0.0915 | Classification_binary: 0.1560 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0372 | Regression_energy: 0.0530 | Regression_eta: 0.0282 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0596 | LR: 1.10e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.5733s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7127 | max_loss: 0.1604 | max_backward: 1.3079 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4722 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 6261.6 | uss_MB: 5413.8 | child_rss_MB: 5045.9 | allocated_MB: 370.5 | reserved_MB: 23988.0 | inactive_split_MB: 179.8 +rank4 [2026-10-29 23:02:40,690] INFO : Step 46700/50000 rank4 | Train Loss: 0.9378 | Train Loss Unweighted: 0.4497 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.0977 | Classification: 0.0857 | Classification_binary: 0.1264 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0015 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0556 | Regression_eta: 0.0346 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0445 | LR: 1.10e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.7494s | Phase Times: max_pre_step_barrier: 0.0197 | max_batch_to_device: 0.0481 | max_forward: 0.6699 | max_loss: 0.2593 | max_backward: 1.4356 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5914 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6261.6 | uss_MB: 5415.0 | child_rss_MB: 5040.6 | allocated_MB: 368.9 | reserved_MB: 24784.0 | inactive_split_MB: 229.2 +rank4 [2026-10-29 23:06:36,577] INFO : Step 46800/50000 rank4 | Train Loss: 0.9152 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.2022 | Cardinality: 0.1011 | Classification: 0.0910 | Classification_binary: 0.1376 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0988 | Regression_eta: 0.0014 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1087 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0457 | Regression_eta: 0.0216 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0531 | LR: 1.09e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.6072s | Phase Times: max_pre_step_barrier: 0.1216 | max_batch_to_device: 0.0144 | max_forward: 1.0361 | max_loss: 0.2737 | max_backward: 1.7951 | max_optimizer_step: 0.2978 | max_tensorboard_logging: 0.5843 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6263.1 | uss_MB: 5417.5 | child_rss_MB: 5044.7 | allocated_MB: 368.9 | reserved_MB: 24994.0 | inactive_split_MB: 204.7 +rank4 [2026-10-29 23:10:45,824] INFO : Step 46900/50000 rank4 | Train Loss: 0.9278 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2149 | Cardinality: 0.1073 | Classification: 0.0827 | Classification_binary: 0.1607 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1027 | Regression_eta: 0.0010 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1541 | Regression_cos_phi: 0.0352 | Regression_energy: 0.0487 | Regression_eta: 0.0297 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0354 | LR: 1.09e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.0951s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0446 | max_forward: 0.7815 | max_loss: 0.1553 | max_backward: 1.4732 | max_optimizer_step: 0.2929 | max_tensorboard_logging: 0.6668 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6256.7 | uss_MB: 5416.7 | child_rss_MB: 5041.4 | allocated_MB: 369.3 | reserved_MB: 24994.0 | inactive_split_MB: 274.7 +rank4 [2026-10-29 23:14:40,387] INFO : Step 47000/50000 rank4 | Train Loss: 0.9023 | Train Loss Unweighted: 0.4500 | Train Loss Components: Auxiliary: 0.2093 | Cardinality: 0.0953 | Classification: 0.0802 | Classification_binary: 0.1325 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1175 | Regression_eta: 0.0010 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0584 | Regression_eta: 0.0267 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0402 | LR: 1.08e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.5097s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6624 | max_loss: 0.1440 | max_backward: 1.2028 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 6259.7 | uss_MB: 5421.0 | child_rss_MB: 5043.0 | allocated_MB: 369.4 | reserved_MB: 25784.0 | inactive_split_MB: 224.5 +rank4 [2026-10-29 23:14:41,333] INFO : Running validation on rank4 +rank4 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 23:15:32,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:15:33,164] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 23:16:02,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 23:16:03,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 23:16:04,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 23:16:05,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 23:16:15,674] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 23:16:16,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:16:16,906] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 23:16:33,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 23:16:34,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 23:16:35,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 23:16:36,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 23:16:46,738] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 23:16:47,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:16:47,863] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 23:17:03,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 23:17:04,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 23:17:04,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 23:17:05,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 23:17:15,479] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-29 23:21:36,066] INFO : Step 47100/50000 rank4 | Train Loss: 0.9138 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.2058 | Cardinality: 0.0938 | Classification: 0.0946 | Classification_binary: 0.1243 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1103 | Regression_eta: 0.0011 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1216 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0663 | Regression_eta: 0.0286 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0423 | LR: 1.07e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1760s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0063 | max_forward: 0.8048 | max_loss: 0.2433 | max_backward: 1.3159 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 1.0353 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 6952.3 | uss_MB: 6113.6 | child_rss_MB: 5037.2 | allocated_MB: 368.1 | reserved_MB: 22106.0 | inactive_split_MB: 269.0 +rank4 [2026-10-29 23:25:45,623] INFO : Step 47200/50000 rank4 | Train Loss: 0.9875 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2263 | Cardinality: 0.1259 | Classification: 0.0903 | Classification_binary: 0.1586 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1110 | Regression_eta: 0.0012 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0377 | Regression_energy: 0.0516 | Regression_eta: 0.0346 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0463 | LR: 1.07e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2834s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0486 | max_forward: 0.7581 | max_loss: 0.1506 | max_backward: 1.5914 | max_optimizer_step: 0.2300 | max_tensorboard_logging: 0.7953 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 6951.3 | uss_MB: 6117.8 | child_rss_MB: 5032.9 | allocated_MB: 370.5 | reserved_MB: 23104.0 | inactive_split_MB: 195.8 +rank4 [2026-10-29 23:29:53,763] INFO : Step 47300/50000 rank4 | Train Loss: 0.9350 | Train Loss Unweighted: 0.4467 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.0958 | Classification: 0.0857 | Classification_binary: 0.1512 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1558 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0575 | Regression_eta: 0.0301 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0436 | LR: 1.06e-04 | DataLoad Time: 0.0211s | Model Forward Time: 6.4010s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0184 | max_forward: 1.9890 | max_loss: 0.4771 | max_backward: 3.1656 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6309.1 | uss_MB: 5475.6 | child_rss_MB: 5028.4 | allocated_MB: 370.8 | reserved_MB: 23104.0 | inactive_split_MB: 271.1 +rank4 [2026-10-29 23:33:58,356] INFO : Step 47400/50000 rank4 | Train Loss: 0.8593 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.0807 | Classification: 0.0812 | Classification_binary: 0.1182 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1114 | Regression_eta: 0.0011 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1309 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0629 | Regression_eta: 0.0276 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0385 | LR: 1.06e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.0135s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0125 | max_forward: 0.7544 | max_loss: 0.1826 | max_backward: 1.4299 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8027 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6311.1 | uss_MB: 5475.9 | child_rss_MB: 5031.1 | allocated_MB: 369.9 | reserved_MB: 23104.0 | inactive_split_MB: 253.6 +rank4 [2026-10-29 23:37:53,325] INFO : Step 47500/50000 rank4 | Train Loss: 0.9089 | Train Loss Unweighted: 0.4235 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1000 | Classification: 0.0799 | Classification_binary: 0.1391 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1008 | Regression_eta: 0.0012 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0525 | Regression_eta: 0.0350 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0551 | LR: 1.06e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.2723s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0068 | max_forward: 0.7329 | max_loss: 0.1827 | max_backward: 1.5814 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9161 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6306.2 | uss_MB: 5478.3 | child_rss_MB: 5026.1 | allocated_MB: 371.7 | reserved_MB: 23104.0 | inactive_split_MB: 243.6 +rank4 [2026-10-29 23:41:48,171] INFO : Step 47600/50000 rank4 | Train Loss: 0.9781 | Train Loss Unweighted: 0.4635 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1226 | Classification: 0.0910 | Classification_binary: 0.1607 | HitGrouping: 0.1784 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0567 | Regression_eta: 0.0386 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0392 | LR: 1.05e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.4258s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8435 | max_loss: 0.1581 | max_backward: 1.8989 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5341 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6310.2 | uss_MB: 5483.0 | child_rss_MB: 5033.7 | allocated_MB: 370.6 | reserved_MB: 24122.0 | inactive_split_MB: 257.4 +rank4 [2026-10-29 23:46:00,211] INFO : Step 47700/50000 rank4 | Train Loss: 0.9311 | Train Loss Unweighted: 0.4523 | Train Loss Components: Auxiliary: 0.2105 | Cardinality: 0.0992 | Classification: 0.0889 | Classification_binary: 0.1321 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1138 | Regression_eta: 0.0012 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0472 | Regression_eta: 0.0335 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0530 | LR: 1.05e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.5064s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0068 | max_forward: 0.6708 | max_loss: 0.1613 | max_backward: 1.2326 | max_optimizer_step: 0.1946 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6306.7 | uss_MB: 5481.0 | child_rss_MB: 5031.8 | allocated_MB: 368.1 | reserved_MB: 24122.0 | inactive_split_MB: 276.0 +rank4 [2026-10-29 23:50:02,910] INFO : Step 47800/50000 rank4 | Train Loss: 0.9063 | Train Loss Unweighted: 0.4379 | Train Loss Components: Auxiliary: 0.2020 | Cardinality: 0.0932 | Classification: 0.0920 | Classification_binary: 0.1284 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0011 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1067 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0489 | Regression_eta: 0.0283 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0409 | LR: 1.04e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8432s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0610 | max_forward: 0.6626 | max_loss: 0.1349 | max_backward: 1.5091 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6302.7 | uss_MB: 5485.7 | child_rss_MB: 5029.8 | allocated_MB: 369.5 | reserved_MB: 24122.0 | inactive_split_MB: 236.5 +rank4 [2026-10-29 23:54:07,643] INFO : Step 47900/50000 rank4 | Train Loss: 0.9591 | Train Loss Unweighted: 0.4478 | Train Loss Components: Auxiliary: 0.2152 | Cardinality: 0.1022 | Classification: 0.0882 | Classification_binary: 0.1545 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1003 | Regression_eta: 0.0014 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0413 | Regression_eta: 0.0407 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0473 | LR: 1.04e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.1699s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8744 | max_loss: 0.2241 | max_backward: 1.6370 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 6302.2 | uss_MB: 5483.9 | child_rss_MB: 5030.3 | allocated_MB: 370.5 | reserved_MB: 24122.0 | inactive_split_MB: 218.9 +rank4 [2026-10-30 00:00:28,988] INFO : Step 48000/50000 rank4 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1221 | Classification: 0.0823 | Classification_binary: 0.1525 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1079 | Regression_eta: 0.0017 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1593 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0569 | Regression_eta: 0.0257 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0465 | LR: 1.04e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1275s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0057 | max_forward: 0.7069 | max_loss: 0.2447 | max_backward: 1.4090 | max_optimizer_step: 0.1933 | max_tensorboard_logging: 0.8132 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1250 | Memory: rss_MB: 6301.2 | uss_MB: 5485.0 | child_rss_MB: 5024.5 | allocated_MB: 368.8 | reserved_MB: 24122.0 | inactive_split_MB: 269.6 +rank4 [2026-10-30 00:00:29,931] INFO : Running validation on rank4 +rank4 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 00:01:19,556] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:01:20,066] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 00:01:46,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 00:01:48,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 00:01:49,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 00:01:50,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 00:02:00,504] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-30 00:02:01,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:02:01,859] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 00:02:17,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 00:02:18,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 00:02:19,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 00:02:20,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 00:02:30,303] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-30 00:02:32,510] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:02:32,993] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 00:02:48,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 00:02:49,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 00:02:49,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 00:02:50,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 00:03:00,407] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-30 00:07:25,204] INFO : Step 48100/50000 rank4 | Train Loss: 0.9084 | Train Loss Unweighted: 0.4360 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1162 | Classification: 0.0769 | Classification_binary: 0.1204 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1183 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1135 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0563 | Regression_eta: 0.0987 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0454 | LR: 1.03e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9239s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1372 | max_backward: 1.2024 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.9158 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7374.2 | uss_MB: 6557.5 | child_rss_MB: 5038.7 | allocated_MB: 367.7 | reserved_MB: 24830.0 | inactive_split_MB: 222.3 +rank4 [2026-10-30 00:11:33,472] INFO : Step 48200/50000 rank4 | Train Loss: 0.9585 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2151 | Cardinality: 0.1199 | Classification: 0.0839 | Classification_binary: 0.1503 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1456 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0443 | Regression_eta: 0.0450 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0747 | LR: 1.03e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4377s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7766 | max_loss: 0.1498 | max_backward: 1.7414 | max_optimizer_step: 0.3242 | max_tensorboard_logging: 0.7504 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 7374.2 | uss_MB: 6560.6 | child_rss_MB: 5031.5 | allocated_MB: 370.4 | reserved_MB: 24830.0 | inactive_split_MB: 179.3 +rank4 [2026-10-30 00:15:41,698] INFO : Step 48300/50000 rank4 | Train Loss: 1.0575 | Train Loss Unweighted: 0.4963 | Train Loss Components: Auxiliary: 0.2408 | Cardinality: 0.1341 | Classification: 0.0958 | Classification_binary: 0.1687 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1138 | Regression_eta: 0.0013 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0455 | Regression_eta: 0.0389 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0600 | LR: 1.03e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2031s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7497 | max_loss: 0.2201 | max_backward: 1.7233 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6378.2 | uss_MB: 5566.5 | child_rss_MB: 5032.3 | allocated_MB: 372.3 | reserved_MB: 24830.0 | inactive_split_MB: 269.6 +rank4 [2026-10-30 00:19:56,563] INFO : Step 48400/50000 rank4 | Train Loss: 0.9108 | Train Loss Unweighted: 0.4349 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1093 | Classification: 0.0691 | Classification_binary: 0.1311 | HitGrouping: 0.1830 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1153 | Regression_eta: 0.0013 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1595 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0565 | Regression_eta: 0.0402 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0663 | LR: 1.02e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8539s | Phase Times: max_pre_step_barrier: 0.1141 | max_batch_to_device: 0.0064 | max_forward: 0.6489 | max_loss: 0.1390 | max_backward: 1.1801 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6705 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6380.2 | uss_MB: 5566.3 | child_rss_MB: 5030.1 | allocated_MB: 370.1 | reserved_MB: 24830.0 | inactive_split_MB: 261.5 +rank4 [2026-10-30 00:23:53,558] INFO : Step 48500/50000 rank4 | Train Loss: 0.9694 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1004 | Classification: 0.0844 | Classification_binary: 0.1367 | HitGrouping: 0.1928 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1194 | Regression_eta: 0.0012 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1343 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0661 | Regression_eta: 0.0316 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0775 | LR: 1.02e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.7553s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 1.7512 | max_loss: 0.2678 | max_backward: 1.9538 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.4804 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 6374.4 | uss_MB: 5565.1 | child_rss_MB: 5023.6 | allocated_MB: 371.5 | reserved_MB: 26842.0 | inactive_split_MB: 176.5 +rank4 [2026-10-30 00:27:51,108] INFO : Step 48600/50000 rank4 | Train Loss: 0.9724 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1148 | Classification: 0.0854 | Classification_binary: 0.1551 | HitGrouping: 0.1844 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1616 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0463 | Regression_eta: 0.0251 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0498 | LR: 1.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0070s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0086 | max_forward: 0.6984 | max_loss: 0.2525 | max_backward: 1.6611 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6375.4 | uss_MB: 5569.8 | child_rss_MB: 5026.2 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 234.8 +rank4 [2026-10-30 00:31:52,432] INFO : Step 48700/50000 rank4 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4863 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1212 | Classification: 0.0968 | Classification_binary: 0.1569 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1143 | Regression_eta: 0.0012 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1779 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0540 | Regression_eta: 0.0322 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0667 | LR: 1.02e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9631s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7410 | max_loss: 0.2472 | max_backward: 1.4647 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.6585 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6316.1 | uss_MB: 5565.7 | child_rss_MB: 4823.4 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 275.8 +rank4 [2026-10-30 00:35:40,394] INFO : Step 48800/50000 rank4 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1981 | Cardinality: 0.0898 | Classification: 0.0814 | Classification_binary: 0.1196 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1094 | Regression_eta: 0.0012 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.1162 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0417 | Regression_eta: 0.0351 | Regression_pt: 0.0421 | Regression_sin_phi: 0.0459 | LR: 1.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7708s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.6122 | max_loss: 0.1775 | max_backward: 1.1932 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6316.1 | uss_MB: 5568.8 | child_rss_MB: 4813.9 | allocated_MB: 368.0 | reserved_MB: 26842.0 | inactive_split_MB: 263.7 +rank4 [2026-10-30 00:39:23,571] INFO : Step 48900/50000 rank4 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1005 | Classification: 0.0873 | Classification_binary: 0.1401 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0432 | Regression_eta: 0.0242 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0415 | LR: 1.01e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.9261s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0809 | max_forward: 0.7397 | max_loss: 0.1455 | max_backward: 1.5027 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6303.5 | uss_MB: 5491.8 | child_rss_MB: 4768.8 | allocated_MB: 373.0 | reserved_MB: 26842.0 | inactive_split_MB: 191.3 +rank4 [2026-10-30 00:43:08,805] INFO : Step 49000/50000 rank4 | Train Loss: 1.0019 | Train Loss Unweighted: 0.4657 | Train Loss Components: Auxiliary: 0.2334 | Cardinality: 0.1354 | Classification: 0.0893 | Classification_binary: 0.1587 | HitGrouping: 0.1838 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0010 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0501 | Regression_eta: 0.0267 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0350 | LR: 1.01e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4532s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1085 | max_forward: 0.8072 | max_loss: 0.1558 | max_backward: 1.7809 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.6082 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6305.0 | uss_MB: 5577.9 | child_rss_MB: 4749.4 | allocated_MB: 370.9 | reserved_MB: 26842.0 | inactive_split_MB: 271.3 +rank4 [2026-10-30 00:43:09,868] INFO : Running validation on rank4 +rank4 [2026-10-30 00:43:09,869] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 00:43:09,869] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 00:43:59,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:43:59,787] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 00:44:27,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 00:44:28,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 00:44:29,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 00:44:29,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 00:44:40,005] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-30 00:44:40,646] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:44:41,121] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 00:44:56,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 00:44:57,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 00:44:58,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 00:44:59,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 00:45:09,476] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-30 00:45:10,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:45:10,743] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 00:45:26,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 00:45:27,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 00:45:27,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 00:45:28,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 00:45:38,591] INFO : Time taken to make predictions on device 4 is: 0.22 min +rank4 [2026-10-30 00:49:42,470] INFO : Step 49100/50000 rank4 | Train Loss: 0.9797 | Train Loss Unweighted: 0.4561 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1125 | Classification: 0.0877 | Classification_binary: 0.1433 | HitGrouping: 0.1905 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0013 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.1497 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0596 | Regression_eta: 0.0671 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0887 | LR: 1.01e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.7316s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6869 | max_loss: 0.1436 | max_backward: 1.2639 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.4741 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0455 | Memory: rss_MB: 7470.2 | uss_MB: 6734.9 | child_rss_MB: 4778.3 | allocated_MB: 370.4 | reserved_MB: 22116.0 | inactive_split_MB: 274.9 +rank4 [2026-10-30 00:53:47,122] INFO : Step 49200/50000 rank4 | Train Loss: 0.8929 | Train Loss Unweighted: 0.4393 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1020 | Classification: 0.0903 | Classification_binary: 0.1250 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0009 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0498 | Regression_eta: 0.0212 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0380 | LR: 1.01e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7539s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6890 | max_loss: 0.2430 | max_backward: 1.3427 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.6347 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7470.5 | uss_MB: 6738.6 | child_rss_MB: 4771.6 | allocated_MB: 368.3 | reserved_MB: 24092.0 | inactive_split_MB: 187.3 +rank4 [2026-10-30 00:57:33,941] INFO : Step 49300/50000 rank4 | Train Loss: 1.0122 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1210 | Classification: 0.0951 | Classification_binary: 0.1621 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1066 | Regression_eta: 0.0015 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1628 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0516 | Regression_eta: 0.0392 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0511 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8970s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1495 | max_backward: 1.5906 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6415.4 | uss_MB: 5684.6 | child_rss_MB: 4746.9 | allocated_MB: 370.9 | reserved_MB: 24092.0 | inactive_split_MB: 226.9 +rank4 [2026-10-30 01:01:33,227] INFO : Step 49400/50000 rank4 | Train Loss: 0.9790 | Train Loss Unweighted: 0.4695 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1137 | Classification: 0.0883 | Classification_binary: 0.1513 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1131 | Regression_eta: 0.0013 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0541 | Regression_eta: 0.0456 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0440 | LR: 1.00e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.0353s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2433 | max_backward: 1.8823 | max_optimizer_step: 0.3081 | max_tensorboard_logging: 0.5951 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6413.4 | uss_MB: 5608.6 | child_rss_MB: 4765.4 | allocated_MB: 372.2 | reserved_MB: 24092.0 | inactive_split_MB: 278.9 +rank4 [2026-10-30 01:05:37,233] INFO : Step 49500/50000 rank4 | Train Loss: 0.9706 | Train Loss Unweighted: 0.4581 | Train Loss Components: Auxiliary: 0.2120 | Cardinality: 0.1158 | Classification: 0.0879 | Classification_binary: 0.1279 | HitGrouping: 0.1920 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1356 | Regression_cos_phi: 0.0979 | Regression_energy: 0.0584 | Regression_eta: 0.0813 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0434 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9629s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6307 | max_loss: 0.1505 | max_backward: 1.3337 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.8584 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6409.4 | uss_MB: 5683.7 | child_rss_MB: 4759.9 | allocated_MB: 370.4 | reserved_MB: 24614.0 | inactive_split_MB: 171.1 +rank4 [2026-10-30 01:09:49,675] INFO : Step 49600/50000 rank4 | Train Loss: 0.9965 | Train Loss Unweighted: 0.4752 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1098 | Classification: 0.1016 | Classification_binary: 0.1437 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1133 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0610 | Regression_eta: 0.0386 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0606 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1640s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.6933 | max_loss: 0.1496 | max_backward: 1.5639 | max_optimizer_step: 0.3077 | max_tensorboard_logging: 0.7702 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6412.2 | uss_MB: 5687.4 | child_rss_MB: 4755.4 | allocated_MB: 371.3 | reserved_MB: 24614.0 | inactive_split_MB: 250.0 +rank4 [2026-10-30 01:13:44,423] INFO : Step 49700/50000 rank4 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4775 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1108 | Classification: 0.1000 | Classification_binary: 0.1702 | HitGrouping: 0.1796 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1019 | Regression_eta: 0.0014 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0555 | Regression_eta: 0.0457 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0696 | LR: 1.00e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.2321s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7591 | max_loss: 0.1933 | max_backward: 1.7410 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 6407.4 | uss_MB: 5675.2 | child_rss_MB: 4703.7 | allocated_MB: 372.4 | reserved_MB: 24614.0 | inactive_split_MB: 259.5 +rank4 [2026-10-30 01:17:46,084] INFO : Step 49800/50000 rank4 | Train Loss: 0.9472 | Train Loss Unweighted: 0.4401 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1188 | Classification: 0.0821 | Classification_binary: 0.1427 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1307 | Regression_cos_phi: 0.0322 | Regression_energy: 0.0493 | Regression_eta: 0.0294 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0488 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7722s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7430 | max_loss: 0.2153 | max_backward: 1.0846 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.7647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6388.3 | uss_MB: 5552.5 | child_rss_MB: 4630.2 | allocated_MB: 368.0 | reserved_MB: 24614.0 | inactive_split_MB: 273.7 +rank4 [2026-10-30 01:21:39,660] INFO : Step 49900/50000 rank4 | Train Loss: 0.9411 | Train Loss Unweighted: 0.4586 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.0996 | Classification: 0.0961 | Classification_binary: 0.1284 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1145 | Regression_eta: 0.0013 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1352 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0467 | Regression_eta: 0.0393 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0814 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0422s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.6992 | max_loss: 0.1527 | max_backward: 1.3836 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8576 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6363.3 | uss_MB: 5692.1 | child_rss_MB: 4528.3 | allocated_MB: 368.7 | reserved_MB: 24614.0 | inactive_split_MB: 241.3 +rank4 [2026-10-30 01:25:29,378] INFO : Step 50000/50000 rank4 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2297 | Cardinality: 0.1196 | Classification: 0.0902 | Classification_binary: 0.1583 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1843 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0499 | Regression_eta: 0.0378 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0759 | LR: 1.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2532s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.7146 | max_loss: 0.1481 | max_backward: 1.6207 | max_optimizer_step: 0.2721 | max_tensorboard_logging: 0.8195 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6364.5 | uss_MB: 5690.8 | child_rss_MB: 4519.4 | allocated_MB: 371.5 | reserved_MB: 25088.0 | inactive_split_MB: 268.9 +rank4 [2026-10-30 01:25:30,314] INFO : Running validation on rank4 +rank4 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 01:26:21,849] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:26:22,339] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 01:26:49,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 01:26:51,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 01:26:52,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 01:26:53,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 01:27:03,463] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-30 01:27:03,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:27:04,489] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 01:27:20,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 01:27:21,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 01:27:22,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 01:27:23,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 01:27:33,537] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-30 01:27:34,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:27:34,756] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 01:27:50,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 01:27:51,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 01:27:52,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 01:27:52,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 01:28:02,871] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 01:28:03,350] INFO : Training completed. Total time on device 4: 638.889min diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.5 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.5 new file mode 100644 index 0000000000000000000000000000000000000000..1a74a1723f2387a1e1c38e07a1265d34e6569edf --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.5 @@ -0,0 +1,2484 @@ +rank5 [2026-10-28 11:12:33,093] INFO : Initializing model with seed=12345; process seed=12350 +rank5 [2026-10-28 11:12:33,094] INFO : configured dtype=torch.bfloat16 for autocast +rank5 [2026-10-28 11:12:54,339] INFO : Instantiating model +rank5 [2026-10-28 11:12:54,339] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank5 [2026-10-28 11:12:54,339] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank5 [2026-10-28 11:12:54,358] INFO : Input encoding initialization took 0.02s +rank5 [2026-10-28 11:12:54,358] INFO : nn0 parameters: 171776 +rank5 [2026-10-28 11:12:54,358] INFO : Initializing shared backbone layers, num_convs=6 +rank5 [2026-10-28 11:12:54,358] INFO : Initializing backbone layer 0 +rank5 [2026-10-28 11:12:54,358] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:54,362] INFO : Initializing backbone layer 1 +rank5 [2026-10-28 11:12:54,362] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:54,365] INFO : Initializing backbone layer 2 +rank5 [2026-10-28 11:12:54,365] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:54,368] INFO : Initializing backbone layer 3 +rank5 [2026-10-28 11:12:54,368] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:54,371] INFO : Initializing backbone layer 4 +rank5 [2026-10-28 11:12:54,371] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:54,374] INFO : Initializing backbone layer 5 +rank5 [2026-10-28 11:12:54,374] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-28 11:12:54,378] INFO : Backbone initialization took 0.02s +rank5 [2026-10-28 11:12:54,378] INFO : backbone parameters: 2374656 +rank5 [2026-10-28 11:12:54,378] INFO : Initializing output DNNs +rank5 [2026-10-28 11:12:54,427] INFO : Output DNNs initialization took 0.05s +rank5 [2026-10-28 11:12:54,427] INFO : backbone_mode=BackboneMode.SHARED +rank5 [2026-10-28 11:12:54,427] INFO : jagged_attention enabled=True +rank5 [2026-10-28 11:12:54,427] INFO : flash_attn_varlen enabled=True +rank5 [2026-10-28 11:12:54,427] INFO : task_queries enabled=False +rank5 [2026-10-28 11:12:54,427] INFO : classification_norm parameters: 0 +rank5 [2026-10-28 11:12:54,427] INFO : regression_norm parameters: 0 +rank5 [2026-10-28 11:12:54,427] INFO : classification_query parameters: 0 +rank5 [2026-10-28 11:12:54,427] INFO : regression_query parameters: 0 +rank5 [2026-10-28 11:12:54,427] INFO : classification_readout parameters: 0 +rank5 [2026-10-28 11:12:54,427] INFO : regression_readout parameters: 0 +rank5 [2026-10-28 11:12:54,427] INFO : output_mode=set +rank5 [2026-10-28 11:12:54,428] INFO : set_decoder parameters: 4713812 +rank5 [2026-10-28 11:12:54,428] INFO : nn_binary_particle parameters: 0 +rank5 [2026-10-28 11:12:54,428] INFO : nn_pid parameters: 0 +rank5 [2026-10-28 11:12:54,428] INFO : nn_pt parameters: 0 +rank5 [2026-10-28 11:12:54,428] INFO : nn_eta parameters: 0 +rank5 [2026-10-28 11:12:54,428] INFO : nn_sin_phi parameters: 0 +rank5 [2026-10-28 11:12:54,428] INFO : nn_cos_phi parameters: 0 +rank5 [2026-10-28 11:12:54,428] INFO : nn_energy parameters: 0 +rank5 [2026-10-28 11:12:54,429] INFO : Total MLPF parameters: 7260244 +rank5 [2026-10-28 11:12:54,429] INFO : MLPF __init__ done +rank5 [2026-10-28 11:12:54,430] INFO : Instantiated model +rank5 [2026-10-28 11:12:54,430] INFO : Moving model to device rank=5 +rank5 [2026-10-28 11:12:54,447] INFO : Moved model to device rank=5 +rank5 [2026-10-28 11:12:54,449] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank5 [2026-10-28 11:12:54,449] INFO : Configured model for SyncBatchNorm rank=5 +rank5 [2026-10-28 11:12:54,644] INFO : Configured model for DistributedDataParallel rank=5 +rank5 [2026-10-28 11:12:54,763] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank5 [2026-10-28 11:12:54,763] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank5 [2026-10-28 11:12:54,764] INFO : Trainable parameters: 7260244 +rank5 [2026-10-28 11:12:54,764] INFO : Non-trainable parameters: 0 +rank5 [2026-10-28 11:12:54,764] INFO : Total parameters: 7260244 +rank5 [2026-10-28 11:12:54,766] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank5 [2026-10-28 11:12:54,767] INFO : Getting dataloaders +rank5 [2026-10-28 11:12:54,767] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:03,760] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:04,813] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,149] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-28 11:13:06,149] INFO : Creating WeightedMixIterator with 1 data loaders. +rank5 [2026-10-28 11:13:06,149] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,421] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,680] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,938] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-28 11:13:06,939] INFO : Creating InterleavedIterator with 1 data loaders. +rank5 [2026-10-28 11:13:21,271] INFO : Got dataloaders +rank5 [2026-10-28 11:13:21,271] INFO : loader split=train rank=5 len=21112 +rank5 [2026-10-28 11:13:21,271] INFO : loader split=valid rank=5 len=12 +rank5 [2026-10-28 11:13:21,271] INFO : Starting training from step 1 to 50000 on rank 5 of 8 +rank5 [2026-10-28 11:13:21,272] INFO : Creating train_iterator +rank5 [2026-10-28 11:13:21,272] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-28 11:13:35,616] INFO : Created train_iterator +rank5 [2026-10-28 11:17:15,587] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5434697668626904, 'Classification': 0.9147019968181849, 'Regression_pt': 1.0888657218962907, 'Regression_eta': 0.019410275079426356, 'Regression_sin_phi': 0.00660347732307855, 'Regression_cos_phi': 0.006484371119877323, 'Regression_energy': 1.1263945941627025}, weights={'Classification_binary': 1.0, 'Classification': 0.5941495299339294, 'Regression_pt': 0.4991154968738556, 'Regression_eta': 27.99907684326172, 'Regression_sin_phi': 82.30054473876953, 'Regression_cos_phi': 83.812255859375, 'Regression_energy': 0.48248612880706787} +rank5 [2026-10-28 11:17:17,665] INFO : Step 100/50000 rank5 | Train Loss: 5.4941 | Train Loss Unweighted: 3.3573 | Train Loss Components: Auxiliary: 1.0155 | Cardinality: 0.6417 | Classification: 0.8373 | Classification_binary: 0.4757 | HitGrouping: 0.4797 | Regression_cos_phi: 0.0063 | Regression_energy: 1.0524 | Regression_eta: 0.0130 | Regression_pt: 0.9668 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8594 | Classification_binary: 0.5009 | Regression_cos_phi: 0.0069 | Regression_energy: 1.1807 | Regression_eta: 0.0144 | Regression_pt: 1.0768 | Regression_sin_phi: 0.0058 | LR: 1.00e-03 | DataLoad Time: 0.0146s | Model Forward Time: 3.0165s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.5597 | max_loss: 0.2615 | max_backward: 1.0157 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.4654 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4647.0 | uss_MB: 3401.7 | child_rss_MB: 5567.8 | allocated_MB: 341.0 | reserved_MB: 22914.0 | inactive_split_MB: 237.2 +rank5 [2026-10-28 11:20:50,234] INFO : Step 200/50000 rank5 | Train Loss: 4.4651 | Train Loss Unweighted: 2.7232 | Train Loss Components: Auxiliary: 0.8610 | Cardinality: 0.5153 | Classification: 0.6425 | Classification_binary: 0.5162 | HitGrouping: 0.4582 | Regression_cos_phi: 0.0043 | Regression_energy: 0.7694 | Regression_eta: 0.0086 | Regression_pt: 0.7776 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3879 | Classification_binary: 0.4775 | Regression_cos_phi: 0.2870 | Regression_energy: 0.3982 | Regression_eta: 0.1885 | Regression_pt: 0.4186 | Regression_sin_phi: 0.3268 | LR: 1.00e-03 | DataLoad Time: 0.0154s | Model Forward Time: 3.7240s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7368 | max_loss: 0.2463 | max_backward: 1.7032 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5468 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4651.2 | uss_MB: 3409.7 | child_rss_MB: 5571.3 | allocated_MB: 343.8 | reserved_MB: 23264.0 | inactive_split_MB: 284.1 +rank5 [2026-10-28 11:24:24,162] INFO : Step 300/50000 rank5 | Train Loss: 3.6383 | Train Loss Unweighted: 2.1819 | Train Loss Components: Auxiliary: 0.6931 | Cardinality: 0.3447 | Classification: 0.5089 | Classification_binary: 0.4824 | HitGrouping: 0.4305 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5908 | Regression_eta: 0.0074 | Regression_pt: 0.5852 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3112 | Classification_binary: 0.4707 | Regression_cos_phi: 0.2114 | Regression_energy: 0.3069 | Regression_eta: 0.1589 | Regression_pt: 0.3134 | Regression_sin_phi: 0.2402 | LR: 1.00e-03 | DataLoad Time: 0.0145s | Model Forward Time: 3.8723s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0315 | max_forward: 0.6855 | max_loss: 0.1568 | max_backward: 1.3632 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 1.0182 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4652.7 | uss_MB: 3415.2 | child_rss_MB: 5569.6 | allocated_MB: 341.4 | reserved_MB: 23916.0 | inactive_split_MB: 289.5 +rank5 [2026-10-28 11:27:58,397] INFO : Step 400/50000 rank5 | Train Loss: 3.5064 | Train Loss Unweighted: 1.9938 | Train Loss Components: Auxiliary: 0.6582 | Cardinality: 0.3738 | Classification: 0.4591 | Classification_binary: 0.4306 | HitGrouping: 0.4041 | Regression_cos_phi: 0.0039 | Regression_energy: 0.5433 | Regression_eta: 0.0080 | Regression_pt: 0.5456 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2404 | Classification_binary: 0.4433 | Regression_cos_phi: 0.2704 | Regression_energy: 0.2597 | Regression_eta: 0.1825 | Regression_pt: 0.2677 | Regression_sin_phi: 0.2826 | LR: 1.00e-03 | DataLoad Time: 0.0149s | Model Forward Time: 3.7434s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.6100 | max_loss: 0.1486 | max_backward: 1.2396 | max_optimizer_step: 0.1474 | max_tensorboard_logging: 1.0157 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4650.3 | uss_MB: 3416.8 | child_rss_MB: 5584.9 | allocated_MB: 342.9 | reserved_MB: 23982.0 | inactive_split_MB: 297.5 +rank5 [2026-10-28 11:31:35,335] INFO : Step 500/50000 rank5 | Train Loss: 3.8899 | Train Loss Unweighted: 2.1856 | Train Loss Components: Auxiliary: 0.6982 | Cardinality: 0.3682 | Classification: 0.4707 | Classification_binary: 0.4836 | HitGrouping: 0.4071 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6048 | Regression_eta: 0.0103 | Regression_pt: 0.6070 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2625 | Classification_binary: 0.4215 | Regression_cos_phi: 0.2825 | Regression_energy: 0.3013 | Regression_eta: 0.2314 | Regression_pt: 0.3146 | Regression_sin_phi: 0.2627 | LR: 1.00e-03 | DataLoad Time: 0.0161s | Model Forward Time: 4.1821s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.7460 | max_loss: 0.2622 | max_backward: 1.7372 | max_optimizer_step: 0.1478 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 4651.3 | uss_MB: 3418.2 | child_rss_MB: 5567.0 | allocated_MB: 344.2 | reserved_MB: 24988.0 | inactive_split_MB: 288.4 +rank5 [2026-10-28 11:35:12,860] INFO : Step 600/50000 rank5 | Train Loss: 3.5648 | Train Loss Unweighted: 2.0654 | Train Loss Components: Auxiliary: 0.6756 | Cardinality: 0.3628 | Classification: 0.4159 | Classification_binary: 0.5246 | HitGrouping: 0.3632 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5539 | Regression_eta: 0.0077 | Regression_pt: 0.5557 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2560 | Classification_binary: 0.5281 | Regression_cos_phi: 0.3566 | Regression_energy: 0.2732 | Regression_eta: 0.2537 | Regression_pt: 0.2807 | Regression_sin_phi: 0.2865 | LR: 1.00e-03 | DataLoad Time: 0.0156s | Model Forward Time: 4.1929s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0071 | max_forward: 0.7483 | max_loss: 0.2574 | max_backward: 1.6305 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.9709 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 4647.1 | uss_MB: 3418.2 | child_rss_MB: 5567.3 | allocated_MB: 344.6 | reserved_MB: 24988.0 | inactive_split_MB: 289.5 +rank5 [2026-10-28 11:38:47,057] INFO : Step 700/50000 rank5 | Train Loss: 3.0543 | Train Loss Unweighted: 1.8237 | Train Loss Components: Auxiliary: 0.6036 | Cardinality: 0.3094 | Classification: 0.3407 | Classification_binary: 0.4142 | HitGrouping: 0.3367 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5254 | Regression_eta: 0.0060 | Regression_pt: 0.5314 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1982 | Classification_binary: 0.4041 | Regression_cos_phi: 0.2787 | Regression_energy: 0.2732 | Regression_eta: 0.1807 | Regression_pt: 0.2843 | Regression_sin_phi: 0.3405 | LR: 1.00e-03 | DataLoad Time: 0.0167s | Model Forward Time: 3.4869s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.6445 | max_loss: 0.2136 | max_backward: 1.2179 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.7893 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4647.0 | uss_MB: 3416.3 | child_rss_MB: 5577.3 | allocated_MB: 341.4 | reserved_MB: 24988.0 | inactive_split_MB: 289.7 +rank5 [2026-10-28 11:42:25,056] INFO : Step 800/50000 rank5 | Train Loss: 3.1854 | Train Loss Unweighted: 1.9126 | Train Loss Components: Auxiliary: 0.6154 | Cardinality: 0.2934 | Classification: 0.4077 | Classification_binary: 0.4086 | HitGrouping: 0.3317 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5404 | Regression_eta: 0.0067 | Regression_pt: 0.5423 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2489 | Classification_binary: 0.3987 | Regression_cos_phi: 0.1996 | Regression_energy: 0.2782 | Regression_eta: 0.1748 | Regression_pt: 0.2890 | Regression_sin_phi: 0.2219 | LR: 9.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.3178s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.6526 | max_loss: 0.2001 | max_backward: 1.2539 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5713 | max_tensorboard_flush: 0.0138 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 4642.6 | uss_MB: 3416.3 | child_rss_MB: 5579.0 | allocated_MB: 346.1 | reserved_MB: 24988.0 | inactive_split_MB: 222.7 +rank5 [2026-10-28 11:46:07,653] INFO : Step 900/50000 rank5 | Train Loss: 3.4950 | Train Loss Unweighted: 2.0678 | Train Loss Components: Auxiliary: 0.6566 | Cardinality: 0.3469 | Classification: 0.4160 | Classification_binary: 0.4953 | HitGrouping: 0.3295 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5663 | Regression_eta: 0.0083 | Regression_pt: 0.5742 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2603 | Classification_binary: 0.4961 | Regression_cos_phi: 0.3642 | Regression_energy: 0.2841 | Regression_eta: 0.2368 | Regression_pt: 0.2965 | Regression_sin_phi: 0.4337 | LR: 9.99e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.8994s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8131 | max_loss: 0.2548 | max_backward: 1.8396 | max_optimizer_step: 0.1492 | max_tensorboard_logging: 0.4938 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 4643.4 | uss_MB: 3417.5 | child_rss_MB: 5588.2 | allocated_MB: 344.2 | reserved_MB: 26006.0 | inactive_split_MB: 273.7 +rank5 [2026-10-28 11:49:58,411] INFO : Step 1000/50000 rank5 | Train Loss: 3.0581 | Train Loss Unweighted: 1.8801 | Train Loss Components: Auxiliary: 0.5942 | Cardinality: 0.2760 | Classification: 0.3603 | Classification_binary: 0.4232 | HitGrouping: 0.3177 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5411 | Regression_eta: 0.0066 | Regression_pt: 0.5426 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2117 | Classification_binary: 0.4371 | Regression_cos_phi: 0.3009 | Regression_energy: 0.2961 | Regression_eta: 0.3107 | Regression_pt: 0.3076 | Regression_sin_phi: 0.2989 | LR: 9.99e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2164s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1006 | max_forward: 0.7513 | max_loss: 0.1459 | max_backward: 1.3658 | max_optimizer_step: 0.1511 | max_tensorboard_logging: 1.2506 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4639.1 | uss_MB: 3415.3 | child_rss_MB: 5584.2 | allocated_MB: 344.4 | reserved_MB: 26006.0 | inactive_split_MB: 271.1 +rank5 [2026-10-28 11:49:59,499] INFO : Running validation on rank5 +rank5 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 11:50:52,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:50:53,226] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 11:51:19,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 11:51:20,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 11:51:21,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 11:51:22,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 11:51:32,353] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 11:51:32,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:51:33,321] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 11:51:49,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 11:51:50,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 11:51:51,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 11:51:52,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 11:52:02,128] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 11:52:02,913] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:52:03,476] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 11:52:19,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 11:52:19,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 11:52:20,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 11:52:21,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 11:52:31,205] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 11:56:35,454] INFO : Step 1100/50000 rank5 | Train Loss: 2.9181 | Train Loss Unweighted: 1.8556 | Train Loss Components: Auxiliary: 0.5873 | Cardinality: 0.2540 | Classification: 0.3370 | Classification_binary: 0.3789 | HitGrouping: 0.3247 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5584 | Regression_eta: 0.0064 | Regression_pt: 0.5695 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1895 | Classification_binary: 0.3288 | Regression_cos_phi: 0.0980 | Regression_energy: 0.3220 | Regression_eta: 0.1317 | Regression_pt: 0.3366 | Regression_sin_phi: 0.1528 | LR: 9.99e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9462s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8528 | max_loss: 0.1390 | max_backward: 1.4220 | max_optimizer_step: 0.2689 | max_tensorboard_logging: 0.9135 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 6058.3 | uss_MB: 4835.8 | child_rss_MB: 5619.2 | allocated_MB: 344.1 | reserved_MB: 21080.0 | inactive_split_MB: 227.9 +rank5 [2026-10-28 12:00:25,220] INFO : Step 1200/50000 rank5 | Train Loss: 3.1885 | Train Loss Unweighted: 1.9202 | Train Loss Components: Auxiliary: 0.6265 | Cardinality: 0.3516 | Classification: 0.3718 | Classification_binary: 0.4294 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5507 | Regression_eta: 0.0065 | Regression_pt: 0.5556 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2174 | Classification_binary: 0.4323 | Regression_cos_phi: 0.2720 | Regression_energy: 0.2972 | Regression_eta: 0.2104 | Regression_pt: 0.3128 | Regression_sin_phi: 0.2188 | LR: 9.99e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.4088s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0076 | max_forward: 0.7370 | max_loss: 0.3921 | max_backward: 1.7945 | max_optimizer_step: 0.1639 | max_tensorboard_logging: 1.0479 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 6061.1 | uss_MB: 4838.4 | child_rss_MB: 5619.8 | allocated_MB: 344.9 | reserved_MB: 23662.0 | inactive_split_MB: 241.9 +rank5 [2026-10-28 12:04:16,198] INFO : Step 1300/50000 rank5 | Train Loss: 3.2694 | Train Loss Unweighted: 1.9280 | Train Loss Components: Auxiliary: 0.6193 | Cardinality: 0.3461 | Classification: 0.3601 | Classification_binary: 0.4731 | HitGrouping: 0.3156 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5359 | Regression_eta: 0.0069 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2423 | Classification_binary: 0.5165 | Regression_cos_phi: 0.2804 | Regression_energy: 0.2532 | Regression_eta: 0.1913 | Regression_pt: 0.2720 | Regression_sin_phi: 0.5265 | LR: 9.98e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.5229s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0540 | max_forward: 0.9909 | max_loss: 0.3234 | max_backward: 2.2964 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 1.1252 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5498.9 | uss_MB: 4193.7 | child_rss_MB: 5645.6 | allocated_MB: 344.9 | reserved_MB: 24052.0 | inactive_split_MB: 299.1 +rank5 [2026-10-28 12:07:55,472] INFO : Step 1400/50000 rank5 | Train Loss: 2.8491 | Train Loss Unweighted: 1.7889 | Train Loss Components: Auxiliary: 0.5645 | Cardinality: 0.2711 | Classification: 0.3226 | Classification_binary: 0.3768 | HitGrouping: 0.3109 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5376 | Regression_eta: 0.0061 | Regression_pt: 0.5407 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1923 | Classification_binary: 0.3943 | Regression_cos_phi: 0.1687 | Regression_energy: 0.2515 | Regression_eta: 0.1432 | Regression_pt: 0.2623 | Regression_sin_phi: 0.1553 | LR: 9.98e-04 | DataLoad Time: 0.0149s | Model Forward Time: 3.7587s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0094 | max_forward: 0.6378 | max_loss: 0.4388 | max_backward: 1.2886 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 0.9631 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 5501.2 | uss_MB: 4277.0 | child_rss_MB: 5652.8 | allocated_MB: 350.4 | reserved_MB: 24916.0 | inactive_split_MB: 216.1 +rank5 [2026-10-28 12:11:42,090] INFO : Step 1500/50000 rank5 | Train Loss: 2.8928 | Train Loss Unweighted: 1.7749 | Train Loss Components: Auxiliary: 0.5690 | Cardinality: 0.2892 | Classification: 0.3566 | Classification_binary: 0.3704 | HitGrouping: 0.3078 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5147 | Regression_eta: 0.0058 | Regression_pt: 0.5216 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1867 | Classification_binary: 0.3695 | Regression_cos_phi: 0.2282 | Regression_energy: 0.2664 | Regression_eta: 0.1546 | Regression_pt: 0.2733 | Regression_sin_phi: 0.2018 | LR: 9.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5608s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.6094 | max_loss: 0.4244 | max_backward: 1.4479 | max_optimizer_step: 0.1602 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5488.8 | uss_MB: 4270.8 | child_rss_MB: 5621.5 | allocated_MB: 342.2 | reserved_MB: 24916.0 | inactive_split_MB: 182.9 +rank5 [2026-10-28 12:15:27,599] INFO : Step 1600/50000 rank5 | Train Loss: 2.9805 | Train Loss Unweighted: 1.8512 | Train Loss Components: Auxiliary: 0.5725 | Cardinality: 0.2336 | Classification: 0.3605 | Classification_binary: 0.4404 | HitGrouping: 0.3159 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5142 | Regression_eta: 0.0065 | Regression_pt: 0.5235 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2352 | Classification_binary: 0.4446 | Regression_cos_phi: 0.2301 | Regression_energy: 0.2526 | Regression_eta: 0.1731 | Regression_pt: 0.2700 | Regression_sin_phi: 0.2406 | LR: 9.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7173s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7382 | max_loss: 0.1451 | max_backward: 1.5359 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.4679 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5499.3 | uss_MB: 4281.1 | child_rss_MB: 5637.2 | allocated_MB: 343.2 | reserved_MB: 25046.0 | inactive_split_MB: 260.8 +rank5 [2026-10-28 12:19:19,666] INFO : Step 1700/50000 rank5 | Train Loss: 2.9423 | Train Loss Unweighted: 1.7603 | Train Loss Components: Auxiliary: 0.5618 | Cardinality: 0.2834 | Classification: 0.3350 | Classification_binary: 0.4141 | HitGrouping: 0.3155 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4976 | Regression_eta: 0.0064 | Regression_pt: 0.5011 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2104 | Classification_binary: 0.4389 | Regression_cos_phi: 0.2975 | Regression_energy: 0.2296 | Regression_eta: 0.2191 | Regression_pt: 0.2406 | Regression_sin_phi: 0.3111 | LR: 9.97e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2301s | Phase Times: max_pre_step_barrier: 0.0070 | max_batch_to_device: 0.0065 | max_forward: 0.8351 | max_loss: 0.2466 | max_backward: 1.7292 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0857 | Memory: rss_MB: 5499.7 | uss_MB: 4254.7 | child_rss_MB: 5622.9 | allocated_MB: 342.0 | reserved_MB: 25046.0 | inactive_split_MB: 198.9 +rank5 [2026-10-28 12:23:11,816] INFO : Step 1800/50000 rank5 | Train Loss: 2.7182 | Train Loss Unweighted: 1.6765 | Train Loss Components: Auxiliary: 0.5305 | Cardinality: 0.2556 | Classification: 0.3337 | Classification_binary: 0.3613 | HitGrouping: 0.3103 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4855 | Regression_eta: 0.0054 | Regression_pt: 0.4854 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1774 | Classification_binary: 0.3566 | Regression_cos_phi: 0.1855 | Regression_energy: 0.2774 | Regression_eta: 0.1451 | Regression_pt: 0.2872 | Regression_sin_phi: 0.1633 | LR: 9.97e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0939s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6573 | max_loss: 0.2402 | max_backward: 1.4111 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 1.0385 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0959 | Memory: rss_MB: 5501.3 | uss_MB: 4283.5 | child_rss_MB: 5636.1 | allocated_MB: 340.6 | reserved_MB: 25046.0 | inactive_split_MB: 181.6 +rank5 [2026-10-28 12:26:53,741] INFO : Step 1900/50000 rank5 | Train Loss: 2.9893 | Train Loss Unweighted: 1.8606 | Train Loss Components: Auxiliary: 0.5839 | Cardinality: 0.2900 | Classification: 0.3568 | Classification_binary: 0.4041 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5399 | Regression_eta: 0.0063 | Regression_pt: 0.5477 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2044 | Classification_binary: 0.4157 | Regression_cos_phi: 0.2033 | Regression_energy: 0.2536 | Regression_eta: 0.1462 | Regression_pt: 0.2670 | Regression_sin_phi: 0.2076 | LR: 9.97e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8711s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.6296 | max_loss: 0.1482 | max_backward: 1.2931 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 1.0537 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 5503.8 | uss_MB: 4286.6 | child_rss_MB: 5619.9 | allocated_MB: 344.8 | reserved_MB: 25048.0 | inactive_split_MB: 225.2 +rank5 [2026-10-28 12:30:53,515] INFO : Step 2000/50000 rank5 | Train Loss: 3.0007 | Train Loss Unweighted: 1.8040 | Train Loss Components: Auxiliary: 0.5673 | Cardinality: 0.3091 | Classification: 0.3519 | Classification_binary: 0.4482 | HitGrouping: 0.3137 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4922 | Regression_eta: 0.0062 | Regression_pt: 0.4996 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1979 | Classification_binary: 0.4413 | Regression_cos_phi: 0.2028 | Regression_energy: 0.2203 | Regression_eta: 0.1696 | Regression_pt: 0.2304 | Regression_sin_phi: 0.1964 | LR: 9.96e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.1686s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6813 | max_loss: 0.2416 | max_backward: 1.5165 | max_optimizer_step: 0.1635 | max_tensorboard_logging: 0.9842 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5502.9 | uss_MB: 4287.2 | child_rss_MB: 5669.7 | allocated_MB: 346.7 | reserved_MB: 25696.0 | inactive_split_MB: 287.8 +rank5 [2026-10-28 12:30:54,413] INFO : Running validation on rank5 +rank5 [2026-10-28 12:30:54,414] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 12:30:54,414] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 12:31:46,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:31:46,631] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 12:32:13,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 12:32:14,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 12:32:15,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 12:32:16,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 12:32:26,607] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 12:32:27,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:32:27,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 12:32:43,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 12:32:44,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 12:32:45,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 12:32:46,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 12:32:56,128] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 12:32:57,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:32:57,629] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 12:33:13,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 12:33:14,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 12:33:14,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 12:33:15,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 12:33:25,710] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 12:37:37,594] INFO : Step 2100/50000 rank5 | Train Loss: 2.6299 | Train Loss Unweighted: 1.6429 | Train Loss Components: Auxiliary: 0.5172 | Cardinality: 0.2489 | Classification: 0.3119 | Classification_binary: 0.3591 | HitGrouping: 0.3047 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4785 | Regression_eta: 0.0053 | Regression_pt: 0.4834 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1949 | Classification_binary: 0.3756 | Regression_cos_phi: 0.2098 | Regression_energy: 0.2539 | Regression_eta: 0.1615 | Regression_pt: 0.2653 | Regression_sin_phi: 0.1947 | LR: 9.96e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.0187s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0685 | max_forward: 0.7343 | max_loss: 0.1422 | max_backward: 1.3363 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.0015 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6447.9 | uss_MB: 5231.1 | child_rss_MB: 5629.3 | allocated_MB: 341.1 | reserved_MB: 22546.0 | inactive_split_MB: 269.0 +rank5 [2026-10-28 12:41:24,842] INFO : Step 2200/50000 rank5 | Train Loss: 2.8483 | Train Loss Unweighted: 1.7068 | Train Loss Components: Auxiliary: 0.5372 | Cardinality: 0.2547 | Classification: 0.3411 | Classification_binary: 0.3679 | HitGrouping: 0.3030 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4876 | Regression_eta: 0.0061 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1531 | Classification_binary: 0.3427 | Regression_cos_phi: 0.1992 | Regression_energy: 0.2465 | Regression_eta: 0.1541 | Regression_pt: 0.2556 | Regression_sin_phi: 0.2534 | LR: 9.96e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.4911s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6352 | max_loss: 0.1414 | max_backward: 1.3119 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.4703 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6448.9 | uss_MB: 5233.9 | child_rss_MB: 5636.2 | allocated_MB: 341.4 | reserved_MB: 22598.0 | inactive_split_MB: 313.4 +rank5 [2026-10-28 12:45:27,860] INFO : Step 2300/50000 rank5 | Train Loss: 3.1410 | Train Loss Unweighted: 1.8301 | Train Loss Components: Auxiliary: 0.5918 | Cardinality: 0.3583 | Classification: 0.3517 | Classification_binary: 0.4348 | HitGrouping: 0.3083 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5121 | Regression_eta: 0.0065 | Regression_pt: 0.5184 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2040 | Classification_binary: 0.3980 | Regression_cos_phi: 0.2453 | Regression_energy: 0.2701 | Regression_eta: 0.1391 | Regression_pt: 0.2832 | Regression_sin_phi: 0.2393 | LR: 9.95e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8601s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.7258 | max_loss: 0.1520 | max_backward: 1.6288 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.6027 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0166 | Memory: rss_MB: 5633.9 | uss_MB: 4422.4 | child_rss_MB: 5644.1 | allocated_MB: 346.5 | reserved_MB: 23596.0 | inactive_split_MB: 273.5 +rank5 [2026-10-28 12:49:22,411] INFO : Step 2400/50000 rank5 | Train Loss: 2.8063 | Train Loss Unweighted: 1.7109 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.2683 | Classification: 0.3276 | Classification_binary: 0.3929 | HitGrouping: 0.3025 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4873 | Regression_eta: 0.0054 | Regression_pt: 0.4919 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1399 | Classification_binary: 0.3805 | Regression_cos_phi: 0.1710 | Regression_energy: 0.2202 | Regression_eta: 0.1324 | Regression_pt: 0.2303 | Regression_sin_phi: 0.1581 | LR: 9.95e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.5890s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0314 | max_forward: 0.7331 | max_loss: 0.1502 | max_backward: 1.4174 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.5071 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0157 | Memory: rss_MB: 5635.8 | uss_MB: 4426.4 | child_rss_MB: 5648.9 | allocated_MB: 344.4 | reserved_MB: 23596.0 | inactive_split_MB: 276.8 +rank5 [2026-10-28 12:53:10,912] INFO : Step 2500/50000 rank5 | Train Loss: 2.5300 | Train Loss Unweighted: 1.6034 | Train Loss Components: Auxiliary: 0.5018 | Cardinality: 0.2387 | Classification: 0.2872 | Classification_binary: 0.3313 | HitGrouping: 0.3000 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4857 | Regression_eta: 0.0042 | Regression_pt: 0.4904 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1730 | Classification_binary: 0.3256 | Regression_cos_phi: 0.1527 | Regression_energy: 0.2046 | Regression_eta: 0.1178 | Regression_pt: 0.2127 | Regression_sin_phi: 0.1944 | LR: 9.94e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.0242s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0177 | max_forward: 0.6104 | max_loss: 0.1394 | max_backward: 1.2397 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 1.1029 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5639.0 | uss_MB: 4422.1 | child_rss_MB: 5639.4 | allocated_MB: 342.5 | reserved_MB: 23596.0 | inactive_split_MB: 302.8 +rank5 [2026-10-28 12:56:55,919] INFO : Step 2600/50000 rank5 | Train Loss: 2.8992 | Train Loss Unweighted: 1.7075 | Train Loss Components: Auxiliary: 0.5471 | Cardinality: 0.3219 | Classification: 0.3420 | Classification_binary: 0.4005 | HitGrouping: 0.3027 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4737 | Regression_eta: 0.0057 | Regression_pt: 0.4796 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1893 | Classification_binary: 0.4136 | Regression_cos_phi: 0.2468 | Regression_energy: 0.2285 | Regression_eta: 0.1320 | Regression_pt: 0.2374 | Regression_sin_phi: 0.2436 | LR: 9.94e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.7068s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8849 | max_loss: 0.1495 | max_backward: 1.9136 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1254 | Memory: rss_MB: 5638.3 | uss_MB: 4434.4 | child_rss_MB: 5644.3 | allocated_MB: 343.4 | reserved_MB: 23596.0 | inactive_split_MB: 290.6 +rank5 [2026-10-28 13:00:46,527] INFO : Step 2700/50000 rank5 | Train Loss: 2.8481 | Train Loss Unweighted: 1.7200 | Train Loss Components: Auxiliary: 0.5468 | Cardinality: 0.2918 | Classification: 0.3199 | Classification_binary: 0.4344 | HitGrouping: 0.3029 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4756 | Regression_eta: 0.0057 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1920 | Classification_binary: 0.4288 | Regression_cos_phi: 0.2485 | Regression_energy: 0.2256 | Regression_eta: 0.1725 | Regression_pt: 0.2323 | Regression_sin_phi: 0.2109 | LR: 9.94e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.3238s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0064 | max_forward: 0.7554 | max_loss: 0.2027 | max_backward: 1.6333 | max_optimizer_step: 0.1633 | max_tensorboard_logging: 0.9978 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5637.9 | uss_MB: 4434.7 | child_rss_MB: 5643.6 | allocated_MB: 343.8 | reserved_MB: 23598.0 | inactive_split_MB: 284.2 +rank5 [2026-10-28 13:04:40,295] INFO : Step 2800/50000 rank5 | Train Loss: 2.6381 | Train Loss Unweighted: 1.5955 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2387 | Classification: 0.2848 | Classification_binary: 0.3420 | HitGrouping: 0.2974 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4746 | Regression_eta: 0.0056 | Regression_pt: 0.4829 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1751 | Classification_binary: 0.3626 | Regression_cos_phi: 0.2972 | Regression_energy: 0.2296 | Regression_eta: 0.1844 | Regression_pt: 0.2451 | Regression_sin_phi: 0.2998 | LR: 9.93e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7375s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6805 | max_loss: 0.2368 | max_backward: 1.1707 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5635.7 | uss_MB: 4432.9 | child_rss_MB: 5638.4 | allocated_MB: 341.2 | reserved_MB: 24554.0 | inactive_split_MB: 308.0 +rank5 [2026-10-28 13:08:36,458] INFO : Step 2900/50000 rank5 | Train Loss: 2.7831 | Train Loss Unweighted: 1.6528 | Train Loss Components: Auxiliary: 0.5323 | Cardinality: 0.3031 | Classification: 0.3069 | Classification_binary: 0.3681 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4798 | Regression_eta: 0.0056 | Regression_pt: 0.4869 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1870 | Classification_binary: 0.3362 | Regression_cos_phi: 0.2879 | Regression_energy: 0.2610 | Regression_eta: 0.1597 | Regression_pt: 0.2716 | Regression_sin_phi: 0.2673 | LR: 9.93e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7850s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6242 | max_loss: 0.1475 | max_backward: 1.3573 | max_optimizer_step: 0.2831 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5637.0 | uss_MB: 4433.8 | child_rss_MB: 5645.5 | allocated_MB: 341.4 | reserved_MB: 25570.0 | inactive_split_MB: 292.0 +rank5 [2026-10-28 13:12:15,920] INFO : Step 3000/50000 rank5 | Train Loss: 2.9336 | Train Loss Unweighted: 1.7260 | Train Loss Components: Auxiliary: 0.5466 | Cardinality: 0.2829 | Classification: 0.3524 | Classification_binary: 0.4205 | HitGrouping: 0.3085 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4673 | Regression_eta: 0.0061 | Regression_pt: 0.4733 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2449 | Classification_binary: 0.4410 | Regression_cos_phi: 0.3899 | Regression_energy: 0.2307 | Regression_eta: 0.1896 | Regression_pt: 0.2434 | Regression_sin_phi: 0.3210 | LR: 9.92e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1513s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0386 | max_forward: 0.8057 | max_loss: 0.1488 | max_backward: 1.9376 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5635.9 | uss_MB: 4440.1 | child_rss_MB: 5640.1 | allocated_MB: 346.5 | reserved_MB: 25570.0 | inactive_split_MB: 281.4 +rank5 [2026-10-28 13:12:16,814] INFO : Running validation on rank5 +rank5 [2026-10-28 13:12:16,814] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 13:12:16,814] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 13:13:09,907] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:13:10,420] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 13:13:30,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 13:13:31,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 13:13:32,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 13:13:32,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 13:13:42,980] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 13:13:43,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:13:44,015] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 13:13:59,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 13:14:00,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 13:14:01,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 13:14:03,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 13:14:13,055] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 13:14:13,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:14:14,219] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 13:14:29,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 13:14:30,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 13:14:30,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 13:14:31,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 13:14:41,484] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 13:18:52,944] INFO : Step 3100/50000 rank5 | Train Loss: 2.7478 | Train Loss Unweighted: 1.6734 | Train Loss Components: Auxiliary: 0.5216 | Cardinality: 0.2531 | Classification: 0.2981 | Classification_binary: 0.3781 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4881 | Regression_eta: 0.0057 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1646 | Classification_binary: 0.3782 | Regression_cos_phi: 0.1836 | Regression_energy: 0.2315 | Regression_eta: 0.1191 | Regression_pt: 0.2442 | Regression_sin_phi: 0.1428 | LR: 9.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5532s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.6511 | max_loss: 0.3988 | max_backward: 1.4306 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 1.2631 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5749.3 | uss_MB: 4552.2 | child_rss_MB: 5658.3 | allocated_MB: 342.5 | reserved_MB: 23120.0 | inactive_split_MB: 194.6 +rank5 [2026-10-28 13:22:38,494] INFO : Step 3200/50000 rank5 | Train Loss: 2.5254 | Train Loss Unweighted: 1.5710 | Train Loss Components: Auxiliary: 0.4888 | Cardinality: 0.2129 | Classification: 0.2845 | Classification_binary: 0.3271 | HitGrouping: 0.2964 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4770 | Regression_eta: 0.0050 | Regression_pt: 0.4723 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1352 | Classification_binary: 0.3015 | Regression_cos_phi: 0.1592 | Regression_energy: 0.2414 | Regression_eta: 0.1571 | Regression_pt: 0.2461 | Regression_sin_phi: 0.2334 | LR: 9.91e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.5508s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6134 | max_loss: 0.2441 | max_backward: 1.2833 | max_optimizer_step: 0.2753 | max_tensorboard_logging: 0.5629 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5733.9 | uss_MB: 4558.8 | child_rss_MB: 5628.8 | allocated_MB: 343.6 | reserved_MB: 23340.0 | inactive_split_MB: 191.9 +rank5 [2026-10-28 13:26:30,566] INFO : Step 3300/50000 rank5 | Train Loss: 2.8037 | Train Loss Unweighted: 1.6635 | Train Loss Components: Auxiliary: 0.5345 | Cardinality: 0.2841 | Classification: 0.3310 | Classification_binary: 0.3857 | HitGrouping: 0.3002 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4633 | Regression_eta: 0.0058 | Regression_pt: 0.4718 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2019 | Classification_binary: 0.3686 | Regression_cos_phi: 0.2544 | Regression_energy: 0.2182 | Regression_eta: 0.1678 | Regression_pt: 0.2281 | Regression_sin_phi: 0.2516 | LR: 9.90e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2958s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9147 | max_loss: 0.1535 | max_backward: 1.9100 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.6631 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5735.1 | uss_MB: 4562.9 | child_rss_MB: 5661.7 | allocated_MB: 344.0 | reserved_MB: 23618.0 | inactive_split_MB: 250.0 +rank5 [2026-10-28 13:30:24,443] INFO : Step 3400/50000 rank5 | Train Loss: 2.8855 | Train Loss Unweighted: 1.7067 | Train Loss Components: Auxiliary: 0.5454 | Cardinality: 0.2859 | Classification: 0.3252 | Classification_binary: 0.4155 | HitGrouping: 0.2965 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4731 | Regression_eta: 0.0067 | Regression_pt: 0.4802 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1825 | Classification_binary: 0.4423 | Regression_cos_phi: 0.2493 | Regression_energy: 0.2031 | Regression_eta: 0.1599 | Regression_pt: 0.2137 | Regression_sin_phi: 0.2063 | LR: 9.90e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5642s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0438 | max_forward: 0.7271 | max_loss: 0.1506 | max_backward: 1.3643 | max_optimizer_step: 0.1716 | max_tensorboard_logging: 0.4827 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5729.2 | uss_MB: 4562.1 | child_rss_MB: 5658.8 | allocated_MB: 346.0 | reserved_MB: 23618.0 | inactive_split_MB: 235.6 +rank5 [2026-10-28 13:34:14,209] INFO : Step 3500/50000 rank5 | Train Loss: 2.4422 | Train Loss Unweighted: 1.4809 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.2120 | Classification: 0.2615 | Classification_binary: 0.3296 | HitGrouping: 0.2898 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4369 | Regression_eta: 0.0054 | Regression_pt: 0.4427 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1527 | Classification_binary: 0.3094 | Regression_cos_phi: 0.1876 | Regression_energy: 0.2339 | Regression_eta: 0.1325 | Regression_pt: 0.2424 | Regression_sin_phi: 0.1781 | LR: 9.89e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7300s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0058 | max_forward: 0.7056 | max_loss: 0.1971 | max_backward: 1.3359 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.6307 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5720.3 | uss_MB: 4563.4 | child_rss_MB: 5630.0 | allocated_MB: 339.0 | reserved_MB: 24610.0 | inactive_split_MB: 280.8 +rank5 [2026-10-28 13:38:02,721] INFO : Step 3600/50000 rank5 | Train Loss: 2.5780 | Train Loss Unweighted: 1.5250 | Train Loss Components: Auxiliary: 0.4892 | Cardinality: 0.2463 | Classification: 0.3079 | Classification_binary: 0.3446 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4295 | Regression_eta: 0.0052 | Regression_pt: 0.4324 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1887 | Classification_binary: 0.3372 | Regression_cos_phi: 0.2839 | Regression_energy: 0.1864 | Regression_eta: 0.1488 | Regression_pt: 0.1951 | Regression_sin_phi: 0.1986 | LR: 9.89e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.4636s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1453 | max_backward: 1.4106 | max_optimizer_step: 0.1681 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0126 | Memory: rss_MB: 5726.7 | uss_MB: 4567.6 | child_rss_MB: 5631.8 | allocated_MB: 343.9 | reserved_MB: 24610.0 | inactive_split_MB: 244.5 +rank5 [2026-10-28 13:41:50,384] INFO : Step 3700/50000 rank5 | Train Loss: 2.9213 | Train Loss Unweighted: 1.6952 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.3153 | Classification: 0.3418 | Classification_binary: 0.4227 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4570 | Regression_eta: 0.0063 | Regression_pt: 0.4612 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2261 | Classification_binary: 0.4699 | Regression_cos_phi: 0.3783 | Regression_energy: 0.2221 | Regression_eta: 0.1915 | Regression_pt: 0.2349 | Regression_sin_phi: 0.2860 | LR: 9.88e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2204s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7710 | max_loss: 0.1507 | max_backward: 1.6430 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.8888 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0145 | Memory: rss_MB: 5726.2 | uss_MB: 4569.4 | child_rss_MB: 5661.5 | allocated_MB: 345.4 | reserved_MB: 24610.0 | inactive_split_MB: 268.9 +rank5 [2026-10-28 13:45:46,299] INFO : Step 3800/50000 rank5 | Train Loss: 2.6063 | Train Loss Unweighted: 1.5289 | Train Loss Components: Auxiliary: 0.4933 | Cardinality: 0.2661 | Classification: 0.2839 | Classification_binary: 0.3652 | HitGrouping: 0.2879 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4319 | Regression_eta: 0.0058 | Regression_pt: 0.4369 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1725 | Classification_binary: 0.3553 | Regression_cos_phi: 0.2801 | Regression_energy: 0.2056 | Regression_eta: 0.1721 | Regression_pt: 0.2165 | Regression_sin_phi: 0.2119 | LR: 9.87e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1725s | Phase Times: max_pre_step_barrier: 0.1134 | max_batch_to_device: 0.0063 | max_forward: 0.6784 | max_loss: 0.1451 | max_backward: 1.4269 | max_optimizer_step: 0.2795 | max_tensorboard_logging: 0.9060 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5726.3 | uss_MB: 4569.5 | child_rss_MB: 5664.0 | allocated_MB: 343.8 | reserved_MB: 24610.0 | inactive_split_MB: 202.2 +rank5 [2026-10-28 13:49:32,932] INFO : Step 3900/50000 rank5 | Train Loss: 2.2761 | Train Loss Unweighted: 1.4165 | Train Loss Components: Auxiliary: 0.4517 | Cardinality: 0.1938 | Classification: 0.2553 | Classification_binary: 0.2930 | HitGrouping: 0.2882 | Regression_cos_phi: 0.0020 | Regression_energy: 0.4264 | Regression_eta: 0.0045 | Regression_pt: 0.4331 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1368 | Classification_binary: 0.2871 | Regression_cos_phi: 0.1554 | Regression_energy: 0.1968 | Regression_eta: 0.1056 | Regression_pt: 0.2103 | Regression_sin_phi: 0.1267 | LR: 9.87e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8911s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6197 | max_loss: 0.1501 | max_backward: 1.2810 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.0075 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 5727.5 | uss_MB: 3764.5 | child_rss_MB: 5661.9 | allocated_MB: 343.5 | reserved_MB: 24610.0 | inactive_split_MB: 286.6 +rank5 [2026-10-28 13:53:23,686] INFO : Step 4000/50000 rank5 | Train Loss: 2.6760 | Train Loss Unweighted: 1.5903 | Train Loss Components: Auxiliary: 0.5112 | Cardinality: 0.2445 | Classification: 0.3070 | Classification_binary: 0.3812 | HitGrouping: 0.2906 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4413 | Regression_eta: 0.0062 | Regression_pt: 0.4492 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1599 | Classification_binary: 0.3712 | Regression_cos_phi: 0.2101 | Regression_energy: 0.2032 | Regression_eta: 0.1721 | Regression_pt: 0.2186 | Regression_sin_phi: 0.1959 | LR: 9.86e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.0148s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0492 | max_forward: 0.7834 | max_loss: 0.1586 | max_backward: 1.6903 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.5589 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5731.5 | uss_MB: 4577.2 | child_rss_MB: 5661.6 | allocated_MB: 345.5 | reserved_MB: 24610.0 | inactive_split_MB: 252.6 +rank5 [2026-10-28 13:53:24,557] INFO : Running validation on rank5 +rank5 [2026-10-28 13:53:24,557] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 13:53:24,557] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 13:54:13,891] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:54:14,418] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 13:54:39,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 13:54:41,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 13:54:42,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 13:54:43,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 13:54:53,092] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 13:54:53,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:54:53,946] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 13:55:10,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 13:55:10,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 13:55:11,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 13:55:12,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 13:55:22,818] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 13:55:23,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:55:23,752] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 13:55:38,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 13:55:39,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 13:55:40,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 13:55:40,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 13:55:50,793] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 13:59:55,049] INFO : Step 4100/50000 rank5 | Train Loss: 2.6923 | Train Loss Unweighted: 1.5640 | Train Loss Components: Auxiliary: 0.5058 | Cardinality: 0.2671 | Classification: 0.2981 | Classification_binary: 0.4073 | HitGrouping: 0.2911 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4210 | Regression_eta: 0.0055 | Regression_pt: 0.4265 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1723 | Classification_binary: 0.4196 | Regression_cos_phi: 0.2345 | Regression_energy: 0.2149 | Regression_eta: 0.1376 | Regression_pt: 0.2251 | Regression_sin_phi: 0.2894 | LR: 9.85e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1488s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0141 | max_forward: 0.7422 | max_loss: 0.3919 | max_backward: 1.3629 | max_optimizer_step: 0.1890 | max_tensorboard_logging: 0.9903 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6680.3 | uss_MB: 5525.0 | child_rss_MB: 5670.9 | allocated_MB: 345.2 | reserved_MB: 23430.0 | inactive_split_MB: 294.4 +rank5 [2026-10-28 14:03:47,899] INFO : Step 4200/50000 rank5 | Train Loss: 2.3415 | Train Loss Unweighted: 1.4424 | Train Loss Components: Auxiliary: 0.4542 | Cardinality: 0.2058 | Classification: 0.2540 | Classification_binary: 0.3143 | HitGrouping: 0.2846 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4310 | Regression_eta: 0.0050 | Regression_pt: 0.4337 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1565 | Classification_binary: 0.3169 | Regression_cos_phi: 0.2307 | Regression_energy: 0.2180 | Regression_eta: 0.1554 | Regression_pt: 0.2266 | Regression_sin_phi: 0.2085 | LR: 9.84e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.7394s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0168 | max_forward: 0.8335 | max_loss: 0.2425 | max_backward: 1.5586 | max_optimizer_step: 0.1744 | max_tensorboard_logging: 0.4781 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5904.9 | uss_MB: 4751.1 | child_rss_MB: 5672.4 | allocated_MB: 341.2 | reserved_MB: 24406.0 | inactive_split_MB: 273.9 +rank5 [2026-10-28 14:07:35,176] INFO : Step 4300/50000 rank5 | Train Loss: 2.5102 | Train Loss Unweighted: 1.4857 | Train Loss Components: Auxiliary: 0.4757 | Cardinality: 0.2309 | Classification: 0.2786 | Classification_binary: 0.3543 | HitGrouping: 0.2938 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4177 | Regression_eta: 0.0054 | Regression_pt: 0.4246 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1903 | Classification_binary: 0.3381 | Regression_cos_phi: 0.2770 | Regression_energy: 0.2117 | Regression_eta: 0.1806 | Regression_pt: 0.2282 | Regression_sin_phi: 0.2892 | LR: 9.84e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5992s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6520 | max_loss: 0.2453 | max_backward: 1.4084 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.5540 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5910.3 | uss_MB: 4757.7 | child_rss_MB: 5668.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 239.6 +rank5 [2026-10-28 14:11:23,917] INFO : Step 4400/50000 rank5 | Train Loss: 2.8258 | Train Loss Unweighted: 1.5691 | Train Loss Components: Auxiliary: 0.5223 | Cardinality: 0.3313 | Classification: 0.2877 | Classification_binary: 0.4120 | HitGrouping: 0.2902 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4255 | Regression_eta: 0.0063 | Regression_pt: 0.4316 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1614 | Classification_binary: 0.3863 | Regression_cos_phi: 0.2765 | Regression_energy: 0.1843 | Regression_eta: 0.2019 | Regression_pt: 0.1951 | Regression_sin_phi: 0.3075 | LR: 9.83e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.0436s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.7635 | max_loss: 0.2580 | max_backward: 1.7456 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.5221 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 5909.1 | uss_MB: 4757.0 | child_rss_MB: 5674.0 | allocated_MB: 344.6 | reserved_MB: 24406.0 | inactive_split_MB: 261.0 +rank5 [2026-10-28 14:15:17,053] INFO : Step 4500/50000 rank5 | Train Loss: 2.5622 | Train Loss Unweighted: 1.4664 | Train Loss Components: Auxiliary: 0.4942 | Cardinality: 0.3132 | Classification: 0.2685 | Classification_binary: 0.3695 | HitGrouping: 0.2876 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4070 | Regression_eta: 0.0048 | Regression_pt: 0.4117 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1663 | Classification_binary: 0.4160 | Regression_cos_phi: 0.2149 | Regression_energy: 0.2121 | Regression_eta: 0.1366 | Regression_pt: 0.2203 | Regression_sin_phi: 0.1867 | LR: 9.82e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.2146s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0462 | max_forward: 1.1330 | max_loss: 0.3526 | max_backward: 2.1744 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5571 | max_tensorboard_flush: 0.0016 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 5912.8 | uss_MB: 4736.1 | child_rss_MB: 5674.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 288.8 +rank5 [2026-10-28 14:19:09,439] INFO : Step 4600/50000 rank5 | Train Loss: 2.2516 | Train Loss Unweighted: 1.3548 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2151 | Classification: 0.2428 | Classification_binary: 0.2886 | HitGrouping: 0.2811 | Regression_cos_phi: 0.0019 | Regression_energy: 0.4059 | Regression_eta: 0.0042 | Regression_pt: 0.4088 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1365 | Classification_binary: 0.2719 | Regression_cos_phi: 0.1607 | Regression_energy: 0.2264 | Regression_eta: 0.1270 | Regression_pt: 0.2343 | Regression_sin_phi: 0.1790 | LR: 9.81e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7001s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0445 | max_forward: 0.8897 | max_loss: 0.2214 | max_backward: 1.8748 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5913.2 | uss_MB: 4741.7 | child_rss_MB: 5653.3 | allocated_MB: 341.6 | reserved_MB: 25392.0 | inactive_split_MB: 298.4 +rank5 [2026-10-28 14:23:11,223] INFO : Step 4700/50000 rank5 | Train Loss: 2.5991 | Train Loss Unweighted: 1.5142 | Train Loss Components: Auxiliary: 0.4914 | Cardinality: 0.2679 | Classification: 0.2997 | Classification_binary: 0.3514 | HitGrouping: 0.2867 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4228 | Regression_eta: 0.0061 | Regression_pt: 0.4290 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1795 | Classification_binary: 0.3405 | Regression_cos_phi: 0.2275 | Regression_energy: 0.1821 | Regression_eta: 0.1668 | Regression_pt: 0.1921 | Regression_sin_phi: 0.2397 | LR: 9.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8892s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7362 | max_loss: 0.2547 | max_backward: 1.8331 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5918.9 | uss_MB: 4764.8 | child_rss_MB: 5675.1 | allocated_MB: 343.1 | reserved_MB: 26384.0 | inactive_split_MB: 292.4 +rank5 [2026-10-28 14:27:13,056] INFO : Step 4800/50000 rank5 | Train Loss: 2.7077 | Train Loss Unweighted: 1.5200 | Train Loss Components: Auxiliary: 0.5118 | Cardinality: 0.3360 | Classification: 0.2606 | Classification_binary: 0.3918 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4264 | Regression_eta: 0.0053 | Regression_pt: 0.4305 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1768 | Classification_binary: 0.4289 | Regression_cos_phi: 0.2870 | Regression_energy: 0.2028 | Regression_eta: 0.1480 | Regression_pt: 0.2135 | Regression_sin_phi: 0.2545 | LR: 9.80e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.2493s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7154 | max_loss: 0.1552 | max_backward: 1.4398 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.8979 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0854 | Memory: rss_MB: 5917.3 | uss_MB: 4226.4 | child_rss_MB: 5678.8 | allocated_MB: 346.1 | reserved_MB: 26384.0 | inactive_split_MB: 284.0 +rank5 [2026-10-28 14:31:09,413] INFO : Step 4900/50000 rank5 | Train Loss: 2.3682 | Train Loss Unweighted: 1.4242 | Train Loss Components: Auxiliary: 0.4616 | Cardinality: 0.2176 | Classification: 0.2570 | Classification_binary: 0.3149 | HitGrouping: 0.2900 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4191 | Regression_eta: 0.0047 | Regression_pt: 0.4239 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1661 | Classification_binary: 0.3068 | Regression_cos_phi: 0.2077 | Regression_energy: 0.1833 | Regression_eta: 0.1672 | Regression_pt: 0.1869 | Regression_sin_phi: 0.1992 | LR: 9.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.4624s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0516 | max_forward: 0.6608 | max_loss: 0.1430 | max_backward: 1.2657 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5916.2 | uss_MB: 4765.4 | child_rss_MB: 5677.6 | allocated_MB: 343.8 | reserved_MB: 26384.0 | inactive_split_MB: 201.9 +rank5 [2026-10-28 14:35:19,045] INFO : Step 5000/50000 rank5 | Train Loss: 2.4060 | Train Loss Unweighted: 1.4130 | Train Loss Components: Auxiliary: 0.4756 | Cardinality: 0.2573 | Classification: 0.2748 | Classification_binary: 0.3198 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4020 | Regression_eta: 0.0043 | Regression_pt: 0.4074 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1360 | Classification_binary: 0.3215 | Regression_cos_phi: 0.2450 | Regression_energy: 0.1797 | Regression_eta: 0.1242 | Regression_pt: 0.1890 | Regression_sin_phi: 0.1950 | LR: 9.78e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8090s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6153 | max_loss: 0.1441 | max_backward: 1.2119 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.9741 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5914.8 | uss_MB: 4767.1 | child_rss_MB: 5669.5 | allocated_MB: 342.5 | reserved_MB: 27388.0 | inactive_split_MB: 218.4 +rank5 [2026-10-28 14:35:19,962] INFO : Running validation on rank5 +rank5 [2026-10-28 14:35:19,963] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 14:35:19,963] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 14:36:10,799] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:36:11,318] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 14:36:36,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 14:36:37,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 14:36:38,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 14:36:39,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 14:36:49,616] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 14:36:50,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:36:50,780] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 14:37:06,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 14:37:07,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 14:37:08,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 14:37:09,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 14:37:19,561] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 14:37:20,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:37:20,582] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 14:37:36,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 14:37:36,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 14:37:37,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 14:37:38,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 14:37:48,181] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 14:42:02,659] INFO : Step 5100/50000 rank5 | Train Loss: 2.7059 | Train Loss Unweighted: 1.5631 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2507 | Classification: 0.3068 | Classification_binary: 0.4110 | HitGrouping: 0.2884 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4148 | Regression_eta: 0.0060 | Regression_pt: 0.4186 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1743 | Classification_binary: 0.4153 | Regression_cos_phi: 0.1966 | Regression_energy: 0.1935 | Regression_eta: 0.1687 | Regression_pt: 0.2052 | Regression_sin_phi: 0.2063 | LR: 9.77e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4420s | Phase Times: max_pre_step_barrier: 0.1071 | max_batch_to_device: 0.0070 | max_forward: 0.7192 | max_loss: 0.2509 | max_backward: 1.6066 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.8859 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7032.2 | uss_MB: 5882.9 | child_rss_MB: 5662.0 | allocated_MB: 343.2 | reserved_MB: 23542.0 | inactive_split_MB: 236.2 +rank5 [2026-10-28 14:45:56,261] INFO : Step 5200/50000 rank5 | Train Loss: 2.4117 | Train Loss Unweighted: 1.4393 | Train Loss Components: Auxiliary: 0.4667 | Cardinality: 0.2215 | Classification: 0.2426 | Classification_binary: 0.3524 | HitGrouping: 0.2783 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4149 | Regression_eta: 0.0050 | Regression_pt: 0.4196 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1747 | Classification_binary: 0.3790 | Regression_cos_phi: 0.1651 | Regression_energy: 0.1926 | Regression_eta: 0.1490 | Regression_pt: 0.2015 | Regression_sin_phi: 0.2397 | LR: 9.76e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9025s | Phase Times: max_pre_step_barrier: 0.1151 | max_batch_to_device: 0.0079 | max_forward: 0.8909 | max_loss: 0.1450 | max_backward: 1.3654 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.5791 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 6020.7 | uss_MB: 4875.4 | child_rss_MB: 5666.4 | allocated_MB: 346.1 | reserved_MB: 23690.0 | inactive_split_MB: 197.9 +rank5 [2026-10-28 14:50:03,850] INFO : Step 5300/50000 rank5 | Train Loss: 2.2417 | Train Loss Unweighted: 1.3238 | Train Loss Components: Auxiliary: 0.4353 | Cardinality: 0.2226 | Classification: 0.2387 | Classification_binary: 0.3027 | HitGrouping: 0.2808 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3844 | Regression_eta: 0.0041 | Regression_pt: 0.3895 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1350 | Classification_binary: 0.3292 | Regression_cos_phi: 0.2469 | Regression_energy: 0.1932 | Regression_eta: 0.1090 | Regression_pt: 0.2036 | Regression_sin_phi: 0.2136 | LR: 9.75e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.1978s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0155 | max_forward: 1.3549 | max_loss: 0.4272 | max_backward: 1.7517 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 6024.3 | uss_MB: 4878.1 | child_rss_MB: 5664.8 | allocated_MB: 341.7 | reserved_MB: 25496.0 | inactive_split_MB: 195.4 +rank5 [2026-10-28 14:54:26,616] INFO : Step 5400/50000 rank5 | Train Loss: 2.5857 | Train Loss Unweighted: 1.4820 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2715 | Classification: 0.2898 | Classification_binary: 0.3613 | HitGrouping: 0.2829 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4076 | Regression_eta: 0.0053 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1871 | Classification_binary: 0.3421 | Regression_cos_phi: 0.2267 | Regression_energy: 0.2115 | Regression_eta: 0.1612 | Regression_pt: 0.2235 | Regression_sin_phi: 0.2424 | LR: 9.74e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.8617s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7472 | max_loss: 0.1664 | max_backward: 1.6216 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4956 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6028.8 | uss_MB: 4880.2 | child_rss_MB: 5668.3 | allocated_MB: 345.8 | reserved_MB: 25496.0 | inactive_split_MB: 226.8 +rank5 [2026-10-28 14:58:29,775] INFO : Step 5500/50000 rank5 | Train Loss: 2.4383 | Train Loss Unweighted: 1.4361 | Train Loss Components: Auxiliary: 0.4636 | Cardinality: 0.2531 | Classification: 0.2547 | Classification_binary: 0.3846 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3900 | Regression_eta: 0.0045 | Regression_pt: 0.3974 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1411 | Classification_binary: 0.3785 | Regression_cos_phi: 0.2137 | Regression_energy: 0.1997 | Regression_eta: 0.1190 | Regression_pt: 0.2123 | Regression_sin_phi: 0.2102 | LR: 9.73e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3517s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7547 | max_loss: 0.2915 | max_backward: 1.8143 | max_optimizer_step: 0.2819 | max_tensorboard_logging: 0.5958 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6029.1 | uss_MB: 4882.8 | child_rss_MB: 5667.9 | allocated_MB: 342.5 | reserved_MB: 26496.0 | inactive_split_MB: 471.5 +rank5 [2026-10-28 15:02:37,794] INFO : Step 5600/50000 rank5 | Train Loss: 2.2622 | Train Loss Unweighted: 1.3177 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.1990 | Classification: 0.2257 | Classification_binary: 0.3268 | HitGrouping: 0.2800 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3758 | Regression_eta: 0.0051 | Regression_pt: 0.3799 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1330 | Classification_binary: 0.2981 | Regression_cos_phi: 0.1478 | Regression_energy: 0.2185 | Regression_eta: 0.1162 | Regression_pt: 0.2313 | Regression_sin_phi: 0.1298 | LR: 9.72e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.1697s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0347 | max_forward: 1.1910 | max_loss: 0.2555 | max_backward: 1.5431 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 6024.4 | uss_MB: 4885.9 | child_rss_MB: 5669.8 | allocated_MB: 340.8 | reserved_MB: 26496.0 | inactive_split_MB: 191.2 +rank5 [2026-10-28 15:06:24,089] INFO : Step 5700/50000 rank5 | Train Loss: 2.4588 | Train Loss Unweighted: 1.4026 | Train Loss Components: Auxiliary: 0.4666 | Cardinality: 0.2348 | Classification: 0.2768 | Classification_binary: 0.3365 | HitGrouping: 0.2916 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3864 | Regression_eta: 0.0055 | Regression_pt: 0.3923 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1815 | Classification_binary: 0.3699 | Regression_cos_phi: 0.2572 | Regression_energy: 0.1700 | Regression_eta: 0.1476 | Regression_pt: 0.1791 | Regression_sin_phi: 0.2195 | LR: 9.71e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.5060s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6547 | max_loss: 0.1874 | max_backward: 1.2748 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6027.1 | uss_MB: 4889.2 | child_rss_MB: 5674.0 | allocated_MB: 344.4 | reserved_MB: 26496.0 | inactive_split_MB: 224.5 +rank5 [2026-10-28 15:10:14,857] INFO : Step 5800/50000 rank5 | Train Loss: 2.4947 | Train Loss Unweighted: 1.4616 | Train Loss Components: Auxiliary: 0.4752 | Cardinality: 0.2715 | Classification: 0.2623 | Classification_binary: 0.3729 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4032 | Regression_eta: 0.0047 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1476 | Classification_binary: 0.3564 | Regression_cos_phi: 0.1680 | Regression_energy: 0.1864 | Regression_eta: 0.1441 | Regression_pt: 0.1986 | Regression_sin_phi: 0.2409 | LR: 9.70e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7379s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.9768 | max_loss: 0.2512 | max_backward: 2.0573 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.6978 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6026.0 | uss_MB: 4048.8 | child_rss_MB: 5650.8 | allocated_MB: 343.4 | reserved_MB: 27494.0 | inactive_split_MB: 198.3 +rank5 [2026-10-28 15:14:12,312] INFO : Step 5900/50000 rank5 | Train Loss: 2.3800 | Train Loss Unweighted: 1.3838 | Train Loss Components: Auxiliary: 0.4549 | Cardinality: 0.2203 | Classification: 0.2438 | Classification_binary: 0.3435 | HitGrouping: 0.2757 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3900 | Regression_eta: 0.0050 | Regression_pt: 0.3965 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1453 | Classification_binary: 0.3292 | Regression_cos_phi: 0.1831 | Regression_energy: 0.1826 | Regression_eta: 0.1110 | Regression_pt: 0.1930 | Regression_sin_phi: 0.1753 | LR: 9.69e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.7155s | Phase Times: max_pre_step_barrier: 0.0158 | max_batch_to_device: 0.0069 | max_forward: 1.0355 | max_loss: 0.2311 | max_backward: 2.2749 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5131 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 6029.8 | uss_MB: 3868.6 | child_rss_MB: 5658.5 | allocated_MB: 342.1 | reserved_MB: 27494.0 | inactive_split_MB: 281.2 +rank5 [2026-10-28 15:18:13,894] INFO : Step 6000/50000 rank5 | Train Loss: 2.1372 | Train Loss Unweighted: 1.3015 | Train Loss Components: Auxiliary: 0.4179 | Cardinality: 0.1916 | Classification: 0.2091 | Classification_binary: 0.2848 | HitGrouping: 0.2746 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3966 | Regression_eta: 0.0041 | Regression_pt: 0.4029 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1095 | Classification_binary: 0.2640 | Regression_cos_phi: 0.1852 | Regression_energy: 0.2121 | Regression_eta: 0.0901 | Regression_pt: 0.2203 | Regression_sin_phi: 0.1150 | LR: 9.68e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6786s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6457 | max_loss: 0.2157 | max_backward: 1.4984 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.5590 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6033.8 | uss_MB: 4891.5 | child_rss_MB: 5658.2 | allocated_MB: 343.1 | reserved_MB: 27494.0 | inactive_split_MB: 260.7 +rank5 [2026-10-28 15:18:14,997] INFO : Running validation on rank5 +rank5 [2026-10-28 15:18:14,997] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 15:18:14,998] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 15:19:17,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:19:17,851] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 15:19:43,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 15:19:44,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 15:19:45,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 15:19:46,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 15:19:56,881] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 15:19:57,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:19:58,136] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 15:20:13,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 15:20:14,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 15:20:15,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 15:20:16,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 15:20:26,679] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 15:20:27,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:20:28,077] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 15:20:43,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 15:20:44,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 15:20:44,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 15:20:45,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 15:20:55,597] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 15:25:29,848] INFO : Step 6100/50000 rank5 | Train Loss: 2.4040 | Train Loss Unweighted: 1.3907 | Train Loss Components: Auxiliary: 0.4661 | Cardinality: 0.2610 | Classification: 0.2427 | Classification_binary: 0.3401 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3968 | Regression_eta: 0.0047 | Regression_pt: 0.4016 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1465 | Classification_binary: 0.3449 | Regression_cos_phi: 0.2493 | Regression_energy: 0.1938 | Regression_eta: 0.1360 | Regression_pt: 0.2044 | Regression_sin_phi: 0.2362 | LR: 9.67e-04 | DataLoad Time: 0.0207s | Model Forward Time: 6.4051s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0073 | max_forward: 1.0567 | max_loss: 0.2452 | max_backward: 2.4106 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.9836 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 7214.6 | uss_MB: 5292.3 | child_rss_MB: 5654.8 | allocated_MB: 344.4 | reserved_MB: 23648.0 | inactive_split_MB: 194.8 +rank5 [2026-10-28 15:29:38,612] INFO : Step 6200/50000 rank5 | Train Loss: 2.4942 | Train Loss Unweighted: 1.4287 | Train Loss Components: Auxiliary: 0.4744 | Cardinality: 0.2537 | Classification: 0.2580 | Classification_binary: 0.3817 | HitGrouping: 0.2748 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3867 | Regression_eta: 0.0050 | Regression_pt: 0.3920 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1444 | Classification_binary: 0.3639 | Regression_cos_phi: 0.2048 | Regression_energy: 0.1684 | Regression_eta: 0.1382 | Regression_pt: 0.1751 | Regression_sin_phi: 0.2199 | LR: 9.66e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0643s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7838 | max_loss: 0.1532 | max_backward: 1.6314 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6093.8 | uss_MB: 4537.3 | child_rss_MB: 5656.7 | allocated_MB: 343.8 | reserved_MB: 23830.0 | inactive_split_MB: 215.7 +rank5 [2026-10-28 15:34:14,251] INFO : Step 6300/50000 rank5 | Train Loss: 2.2204 | Train Loss Unweighted: 1.2763 | Train Loss Components: Auxiliary: 0.4285 | Cardinality: 0.2506 | Classification: 0.2258 | Classification_binary: 0.2983 | HitGrouping: 0.2710 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3704 | Regression_eta: 0.0045 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1343 | Classification_binary: 0.2881 | Regression_cos_phi: 0.2050 | Regression_energy: 0.1730 | Regression_eta: 0.1278 | Regression_pt: 0.1829 | Regression_sin_phi: 0.1644 | LR: 9.65e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.6719s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.5854 | max_loss: 0.1923 | max_backward: 1.0848 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.8586 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6096.2 | uss_MB: 4957.2 | child_rss_MB: 5651.4 | allocated_MB: 339.7 | reserved_MB: 24826.0 | inactive_split_MB: 251.6 +rank5 [2026-10-28 15:38:39,607] INFO : Step 6400/50000 rank5 | Train Loss: 2.2882 | Train Loss Unweighted: 1.3014 | Train Loss Components: Auxiliary: 0.4348 | Cardinality: 0.2271 | Classification: 0.2431 | Classification_binary: 0.3058 | HitGrouping: 0.2702 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3679 | Regression_eta: 0.0046 | Regression_pt: 0.3751 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1466 | Classification_binary: 0.3151 | Regression_cos_phi: 0.1929 | Regression_energy: 0.1775 | Regression_eta: 0.1384 | Regression_pt: 0.1842 | Regression_sin_phi: 0.2295 | LR: 9.64e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.8565s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7489 | max_loss: 0.3608 | max_backward: 2.5327 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6095.6 | uss_MB: 4594.4 | child_rss_MB: 5649.8 | allocated_MB: 345.8 | reserved_MB: 24826.0 | inactive_split_MB: 253.0 +rank5 [2026-10-28 15:42:45,601] INFO : Step 6500/50000 rank5 | Train Loss: 2.4626 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.3022 | Classification: 0.2393 | Classification_binary: 0.3805 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3783 | Regression_eta: 0.0049 | Regression_pt: 0.3831 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1610 | Classification_binary: 0.3906 | Regression_cos_phi: 0.1908 | Regression_energy: 0.1875 | Regression_eta: 0.1412 | Regression_pt: 0.1937 | Regression_sin_phi: 0.1950 | LR: 9.63e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.1910s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7600 | max_loss: 0.1555 | max_backward: 1.3910 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6094.6 | uss_MB: 4960.0 | child_rss_MB: 5656.1 | allocated_MB: 344.4 | reserved_MB: 24826.0 | inactive_split_MB: 269.0 +rank5 [2026-10-28 15:46:47,733] INFO : Step 6600/50000 rank5 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3617 | Train Loss Components: Auxiliary: 0.4493 | Cardinality: 0.2405 | Classification: 0.2289 | Classification_binary: 0.3388 | HitGrouping: 0.2650 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3914 | Regression_eta: 0.0045 | Regression_pt: 0.3935 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1327 | Classification_binary: 0.3277 | Regression_cos_phi: 0.1483 | Regression_energy: 0.1816 | Regression_eta: 0.1057 | Regression_pt: 0.1872 | Regression_sin_phi: 0.1692 | LR: 9.62e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7769s | Phase Times: max_pre_step_barrier: 0.1106 | max_batch_to_device: 0.0061 | max_forward: 0.7587 | max_loss: 0.2869 | max_backward: 2.2641 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1321 | Memory: rss_MB: 6092.2 | uss_MB: 4956.2 | child_rss_MB: 5651.6 | allocated_MB: 343.8 | reserved_MB: 25846.0 | inactive_split_MB: 267.9 +rank5 [2026-10-28 15:51:24,435] INFO : Step 6700/50000 rank5 | Train Loss: 2.1225 | Train Loss Unweighted: 1.2385 | Train Loss Components: Auxiliary: 0.4092 | Cardinality: 0.2100 | Classification: 0.2079 | Classification_binary: 0.2916 | HitGrouping: 0.2751 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3623 | Regression_eta: 0.0041 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1620 | Classification_binary: 0.3205 | Regression_cos_phi: 0.1890 | Regression_energy: 0.1864 | Regression_eta: 0.1385 | Regression_pt: 0.1966 | Regression_sin_phi: 0.2432 | LR: 9.61e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.9906s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.6452 | max_loss: 0.1453 | max_backward: 1.2984 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.0269 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6098.1 | uss_MB: 4965.1 | child_rss_MB: 5657.8 | allocated_MB: 345.0 | reserved_MB: 25846.0 | inactive_split_MB: 258.8 +rank5 [2026-10-28 15:55:32,973] INFO : Step 6800/50000 rank5 | Train Loss: 2.4601 | Train Loss Unweighted: 1.3772 | Train Loss Components: Auxiliary: 0.4589 | Cardinality: 0.2427 | Classification: 0.2631 | Classification_binary: 0.3490 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3747 | Regression_eta: 0.0054 | Regression_pt: 0.3793 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1422 | Classification_binary: 0.2825 | Regression_cos_phi: 0.1754 | Regression_energy: 0.1602 | Regression_eta: 0.1326 | Regression_pt: 0.1642 | Regression_sin_phi: 0.2303 | LR: 9.60e-04 | DataLoad Time: 0.0245s | Model Forward Time: 5.6144s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0662 | max_forward: 1.6558 | max_loss: 0.3855 | max_backward: 2.1309 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 1.0709 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6092.6 | uss_MB: 4961.3 | child_rss_MB: 5647.9 | allocated_MB: 345.8 | reserved_MB: 26622.0 | inactive_split_MB: 194.2 +rank5 [2026-10-28 15:59:35,210] INFO : Step 6900/50000 rank5 | Train Loss: 2.3793 | Train Loss Unweighted: 1.3337 | Train Loss Components: Auxiliary: 0.4495 | Cardinality: 0.2757 | Classification: 0.2300 | Classification_binary: 0.3711 | HitGrouping: 0.2647 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3605 | Regression_eta: 0.0045 | Regression_pt: 0.3628 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1447 | Classification_binary: 0.3802 | Regression_cos_phi: 0.1976 | Regression_energy: 0.1866 | Regression_eta: 0.1136 | Regression_pt: 0.1919 | Regression_sin_phi: 0.1788 | LR: 9.58e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2964s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1120 | max_forward: 0.8190 | max_loss: 0.1496 | max_backward: 1.4356 | max_optimizer_step: 0.2847 | max_tensorboard_logging: 0.8440 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6096.9 | uss_MB: 4962.9 | child_rss_MB: 5656.8 | allocated_MB: 345.4 | reserved_MB: 26622.0 | inactive_split_MB: 300.5 +rank5 [2026-10-28 16:03:56,953] INFO : Step 7000/50000 rank5 | Train Loss: 2.2410 | Train Loss Unweighted: 1.2707 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2746 | Classification: 0.1970 | Classification_binary: 0.3002 | HitGrouping: 0.2655 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3795 | Regression_eta: 0.0045 | Regression_pt: 0.3852 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1217 | Classification_binary: 0.3162 | Regression_cos_phi: 0.1905 | Regression_energy: 0.1841 | Regression_eta: 0.1217 | Regression_pt: 0.1953 | Regression_sin_phi: 0.1841 | LR: 9.57e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.3862s | Phase Times: max_pre_step_barrier: 0.0229 | max_batch_to_device: 0.0154 | max_forward: 1.0393 | max_loss: 0.2251 | max_backward: 2.0451 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.9425 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6097.4 | uss_MB: 4965.6 | child_rss_MB: 5656.9 | allocated_MB: 345.5 | reserved_MB: 26622.0 | inactive_split_MB: 296.5 +rank5 [2026-10-28 16:03:58,353] INFO : Running validation on rank5 +rank5 [2026-10-28 16:03:58,353] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 16:03:58,353] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 16:04:51,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:04:52,383] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 16:05:18,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 16:05:19,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 16:05:20,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 16:05:21,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 16:05:31,564] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 16:05:32,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:05:32,550] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 16:05:48,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 16:05:49,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 16:05:50,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 16:05:51,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 16:06:01,148] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 16:06:02,144] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:06:02,711] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 16:06:18,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 16:06:19,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 16:06:19,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 16:06:20,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 16:06:30,456] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 16:10:49,043] INFO : Step 7100/50000 rank5 | Train Loss: 2.1086 | Train Loss Unweighted: 1.2525 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.1821 | Classification: 0.2186 | Classification_binary: 0.2903 | HitGrouping: 0.2613 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3655 | Regression_eta: 0.0041 | Regression_pt: 0.3696 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1429 | Classification_binary: 0.2661 | Regression_cos_phi: 0.1746 | Regression_energy: 0.1581 | Regression_eta: 0.1420 | Regression_pt: 0.1689 | Regression_sin_phi: 0.1943 | LR: 9.56e-04 | DataLoad Time: 0.0193s | Model Forward Time: 3.9392s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6376 | max_loss: 0.1424 | max_backward: 1.3436 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8454 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 7191.1 | uss_MB: 6057.4 | child_rss_MB: 5686.3 | allocated_MB: 345.6 | reserved_MB: 22618.0 | inactive_split_MB: 206.0 +rank5 [2026-10-28 16:15:05,704] INFO : Step 7200/50000 rank5 | Train Loss: 2.4589 | Train Loss Unweighted: 1.3873 | Train Loss Components: Auxiliary: 0.4658 | Cardinality: 0.2721 | Classification: 0.2648 | Classification_binary: 0.3806 | HitGrouping: 0.2633 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3642 | Regression_eta: 0.0051 | Regression_pt: 0.3675 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1604 | Classification_binary: 0.3689 | Regression_cos_phi: 0.1891 | Regression_energy: 0.1954 | Regression_eta: 0.1779 | Regression_pt: 0.2053 | Regression_sin_phi: 0.1980 | LR: 9.55e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3051s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 1.0191 | max_loss: 0.1674 | max_backward: 1.7813 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6183.3 | uss_MB: 5050.5 | child_rss_MB: 5662.0 | allocated_MB: 343.5 | reserved_MB: 22618.0 | inactive_split_MB: 315.3 +rank5 [2026-10-28 16:19:17,627] INFO : Step 7300/50000 rank5 | Train Loss: 2.1617 | Train Loss Unweighted: 1.2404 | Train Loss Components: Auxiliary: 0.4141 | Cardinality: 0.2258 | Classification: 0.1979 | Classification_binary: 0.3107 | HitGrouping: 0.2671 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3613 | Regression_eta: 0.0041 | Regression_pt: 0.3621 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1308 | Classification_binary: 0.3455 | Regression_cos_phi: 0.1831 | Regression_energy: 0.1809 | Regression_eta: 0.1329 | Regression_pt: 0.1897 | Regression_sin_phi: 0.1901 | LR: 9.53e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.7538s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0084 | max_forward: 0.7227 | max_loss: 0.1491 | max_backward: 1.4232 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.5902 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6180.9 | uss_MB: 5051.2 | child_rss_MB: 5696.9 | allocated_MB: 345.0 | reserved_MB: 23608.0 | inactive_split_MB: 295.0 +rank5 [2026-10-28 16:23:25,991] INFO : Step 7400/50000 rank5 | Train Loss: 2.0741 | Train Loss Unweighted: 1.2014 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2287 | Classification: 0.2035 | Classification_binary: 0.2781 | HitGrouping: 0.2646 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3549 | Regression_eta: 0.0040 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1041 | Classification_binary: 0.2755 | Regression_cos_phi: 0.1353 | Regression_energy: 0.1927 | Regression_eta: 0.0893 | Regression_pt: 0.1996 | Regression_sin_phi: 0.1455 | LR: 9.52e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6551s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6129 | max_loss: 0.2127 | max_backward: 1.2944 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6450 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6180.9 | uss_MB: 5049.5 | child_rss_MB: 5695.6 | allocated_MB: 343.8 | reserved_MB: 23608.0 | inactive_split_MB: 229.5 +rank5 [2026-10-28 16:27:18,559] INFO : Step 7500/50000 rank5 | Train Loss: 2.4640 | Train Loss Unweighted: 1.3686 | Train Loss Components: Auxiliary: 0.4637 | Cardinality: 0.2640 | Classification: 0.2570 | Classification_binary: 0.3594 | HitGrouping: 0.2687 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3684 | Regression_eta: 0.0054 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1781 | Classification_binary: 0.3930 | Regression_cos_phi: 0.2499 | Regression_energy: 0.1797 | Regression_eta: 0.1698 | Regression_pt: 0.1916 | Regression_sin_phi: 0.2920 | LR: 9.51e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1731s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0887 | max_forward: 0.7717 | max_loss: 0.1508 | max_backward: 1.3590 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.9634 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6179.4 | uss_MB: 5053.3 | child_rss_MB: 5684.8 | allocated_MB: 348.6 | reserved_MB: 23608.0 | inactive_split_MB: 283.7 +rank5 [2026-10-28 16:31:22,304] INFO : Step 7600/50000 rank5 | Train Loss: 2.3745 | Train Loss Unweighted: 1.3447 | Train Loss Components: Auxiliary: 0.4509 | Cardinality: 0.2616 | Classification: 0.2519 | Classification_binary: 0.3743 | HitGrouping: 0.2627 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3532 | Regression_eta: 0.0050 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1198 | Classification_binary: 0.3532 | Regression_cos_phi: 0.1475 | Regression_energy: 0.1755 | Regression_eta: 0.1265 | Regression_pt: 0.1832 | Regression_sin_phi: 0.2064 | LR: 9.50e-04 | DataLoad Time: 0.0163s | Model Forward Time: 16.0028s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.7345 | max_loss: 0.1617 | max_backward: 1.4897 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 12.6867 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6179.5 | uss_MB: 5051.2 | child_rss_MB: 5695.8 | allocated_MB: 345.3 | reserved_MB: 23608.0 | inactive_split_MB: 284.8 +rank5 [2026-10-28 16:35:23,036] INFO : Step 7700/50000 rank5 | Train Loss: 2.0023 | Train Loss Unweighted: 1.1475 | Train Loss Components: Auxiliary: 0.3866 | Cardinality: 0.2005 | Classification: 0.1968 | Classification_binary: 0.2784 | HitGrouping: 0.2562 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3315 | Regression_eta: 0.0040 | Regression_pt: 0.3329 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1094 | Classification_binary: 0.2883 | Regression_cos_phi: 0.1421 | Regression_energy: 0.1436 | Regression_eta: 0.1239 | Regression_pt: 0.1506 | Regression_sin_phi: 0.1726 | LR: 9.48e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.0215s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7891 | max_loss: 0.1927 | max_backward: 1.4291 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6182.9 | uss_MB: 5055.0 | child_rss_MB: 5702.0 | allocated_MB: 343.9 | reserved_MB: 23608.0 | inactive_split_MB: 279.7 +rank5 [2026-10-28 16:39:23,043] INFO : Step 7800/50000 rank5 | Train Loss: 2.0473 | Train Loss Unweighted: 1.2037 | Train Loss Components: Auxiliary: 0.4013 | Cardinality: 0.1891 | Classification: 0.2102 | Classification_binary: 0.2760 | HitGrouping: 0.2604 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3526 | Regression_eta: 0.0038 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1131 | Classification_binary: 0.2560 | Regression_cos_phi: 0.1756 | Regression_energy: 0.1587 | Regression_eta: 0.0915 | Regression_pt: 0.1683 | Regression_sin_phi: 0.1235 | LR: 9.47e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.4327s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.7326 | max_loss: 0.2464 | max_backward: 1.8124 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9395 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6182.4 | uss_MB: 5059.4 | child_rss_MB: 5675.6 | allocated_MB: 343.5 | reserved_MB: 24612.0 | inactive_split_MB: 224.8 +rank5 [2026-10-28 16:43:21,636] INFO : Step 7900/50000 rank5 | Train Loss: 2.3872 | Train Loss Unweighted: 1.3279 | Train Loss Components: Auxiliary: 0.4525 | Cardinality: 0.2722 | Classification: 0.2401 | Classification_binary: 0.3580 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3573 | Regression_eta: 0.0051 | Regression_pt: 0.3627 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1694 | Classification_binary: 0.3885 | Regression_cos_phi: 0.2365 | Regression_energy: 0.1811 | Regression_eta: 0.1798 | Regression_pt: 0.1901 | Regression_sin_phi: 0.2270 | LR: 9.46e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7827s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7032 | max_loss: 0.2052 | max_backward: 1.5183 | max_optimizer_step: 0.1922 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6185.2 | uss_MB: 5060.7 | child_rss_MB: 5695.7 | allocated_MB: 347.1 | reserved_MB: 24612.0 | inactive_split_MB: 279.0 +rank5 [2026-10-28 16:47:21,752] INFO : Step 8000/50000 rank5 | Train Loss: 2.1303 | Train Loss Unweighted: 1.2019 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2451 | Classification: 0.2031 | Classification_binary: 0.3087 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3382 | Regression_eta: 0.0041 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1353 | Classification_binary: 0.3492 | Regression_cos_phi: 0.2004 | Regression_energy: 0.1746 | Regression_eta: 0.1046 | Regression_pt: 0.1826 | Regression_sin_phi: 0.1759 | LR: 9.44e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.7973s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0554 | max_forward: 0.7608 | max_loss: 0.1617 | max_backward: 1.3810 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 6186.5 | uss_MB: 4730.8 | child_rss_MB: 5692.5 | allocated_MB: 344.2 | reserved_MB: 24612.0 | inactive_split_MB: 224.1 +rank5 [2026-10-28 16:47:22,744] INFO : Running validation on rank5 +rank5 [2026-10-28 16:47:22,744] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 16:47:22,745] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 16:48:13,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:48:14,392] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 16:48:40,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 16:48:41,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 16:48:42,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 16:48:43,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 16:48:53,453] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 16:48:53,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:48:54,468] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 16:49:10,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 16:49:11,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 16:49:12,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 16:49:13,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 16:49:23,553] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 16:49:24,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:49:24,963] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 16:49:40,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 16:49:41,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 16:49:41,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 16:49:42,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 16:49:52,447] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 16:54:09,126] INFO : Step 8100/50000 rank5 | Train Loss: 2.0320 | Train Loss Unweighted: 1.1744 | Train Loss Components: Auxiliary: 0.3893 | Cardinality: 0.1986 | Classification: 0.1938 | Classification_binary: 0.2691 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3510 | Regression_eta: 0.0039 | Regression_pt: 0.3525 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1374 | Classification_binary: 0.2799 | Regression_cos_phi: 0.2463 | Regression_energy: 0.1883 | Regression_eta: 0.1100 | Regression_pt: 0.1945 | Regression_sin_phi: 0.1752 | LR: 9.43e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0439s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6251 | max_loss: 0.2465 | max_backward: 1.3564 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8507 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7244.9 | uss_MB: 6118.4 | child_rss_MB: 5660.8 | allocated_MB: 342.9 | reserved_MB: 23510.0 | inactive_split_MB: 258.1 +rank5 [2026-10-28 16:58:00,709] INFO : Step 8200/50000 rank5 | Train Loss: 2.1917 | Train Loss Unweighted: 1.2329 | Train Loss Components: Auxiliary: 0.4200 | Cardinality: 0.2405 | Classification: 0.2359 | Classification_binary: 0.3328 | HitGrouping: 0.2582 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3251 | Regression_eta: 0.0043 | Regression_pt: 0.3305 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1493 | Classification_binary: 0.3325 | Regression_cos_phi: 0.1603 | Regression_energy: 0.1382 | Regression_eta: 0.1186 | Regression_pt: 0.1449 | Regression_sin_phi: 0.1754 | LR: 9.42e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2776s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0064 | max_forward: 0.7679 | max_loss: 0.2484 | max_backward: 1.6620 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.8049 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1289 | Memory: rss_MB: 6232.6 | uss_MB: 5106.1 | child_rss_MB: 5673.2 | allocated_MB: 343.2 | reserved_MB: 23510.0 | inactive_split_MB: 292.4 +rank5 [2026-10-28 17:02:03,651] INFO : Step 8300/50000 rank5 | Train Loss: 2.1924 | Train Loss Unweighted: 1.2399 | Train Loss Components: Auxiliary: 0.4122 | Cardinality: 0.2039 | Classification: 0.2083 | Classification_binary: 0.3661 | HitGrouping: 0.2599 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3243 | Regression_eta: 0.0043 | Regression_pt: 0.3323 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1331 | Classification_binary: 0.3697 | Regression_cos_phi: 0.2098 | Regression_energy: 0.1595 | Regression_eta: 0.1248 | Regression_pt: 0.1703 | Regression_sin_phi: 0.1994 | LR: 9.40e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.8401s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0061 | max_forward: 0.9407 | max_loss: 0.2296 | max_backward: 1.9622 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6228.1 | uss_MB: 5055.3 | child_rss_MB: 5669.2 | allocated_MB: 344.6 | reserved_MB: 24580.0 | inactive_split_MB: 267.5 +rank5 [2026-10-28 17:06:07,748] INFO : Step 8400/50000 rank5 | Train Loss: 1.9542 | Train Loss Unweighted: 1.1189 | Train Loss Components: Auxiliary: 0.3756 | Cardinality: 0.1979 | Classification: 0.1926 | Classification_binary: 0.2783 | HitGrouping: 0.2585 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3198 | Regression_eta: 0.0035 | Regression_pt: 0.3209 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0975 | Classification_binary: 0.2906 | Regression_cos_phi: 0.1013 | Regression_energy: 0.1465 | Regression_eta: 0.0860 | Regression_pt: 0.1536 | Regression_sin_phi: 0.1570 | LR: 9.39e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2368s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6939 | max_loss: 0.1474 | max_backward: 1.5357 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 0.9892 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 6216.9 | uss_MB: 5103.3 | child_rss_MB: 5675.6 | allocated_MB: 341.1 | reserved_MB: 24580.0 | inactive_split_MB: 178.9 +rank5 [2026-10-28 17:10:17,599] INFO : Step 8500/50000 rank5 | Train Loss: 2.0642 | Train Loss Unweighted: 1.1724 | Train Loss Components: Auxiliary: 0.3964 | Cardinality: 0.2023 | Classification: 0.2152 | Classification_binary: 0.2897 | HitGrouping: 0.2607 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3289 | Regression_eta: 0.0041 | Regression_pt: 0.3302 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1784 | Classification_binary: 0.3363 | Regression_cos_phi: 0.2258 | Regression_energy: 0.1746 | Regression_eta: 0.1490 | Regression_pt: 0.1834 | Regression_sin_phi: 0.1882 | LR: 9.37e-04 | DataLoad Time: 0.0147s | Model Forward Time: 3.7410s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0069 | max_forward: 0.6424 | max_loss: 0.1972 | max_backward: 1.2711 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.7327 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6218.9 | uss_MB: 5108.3 | child_rss_MB: 5667.6 | allocated_MB: 344.4 | reserved_MB: 25586.0 | inactive_split_MB: 317.0 +rank5 [2026-10-28 17:14:24,635] INFO : Step 8600/50000 rank5 | Train Loss: 2.2820 | Train Loss Unweighted: 1.2960 | Train Loss Components: Auxiliary: 0.4378 | Cardinality: 0.2546 | Classification: 0.2294 | Classification_binary: 0.3478 | HitGrouping: 0.2584 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3521 | Regression_eta: 0.0046 | Regression_pt: 0.3576 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1192 | Classification_binary: 0.3505 | Regression_cos_phi: 0.1750 | Regression_energy: 0.1683 | Regression_eta: 0.1410 | Regression_pt: 0.1763 | Regression_sin_phi: 0.1756 | LR: 9.36e-04 | DataLoad Time: 0.0198s | Model Forward Time: 3.9621s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0069 | max_forward: 0.7396 | max_loss: 0.1527 | max_backward: 1.6088 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5007 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6218.9 | uss_MB: 5112.5 | child_rss_MB: 5664.7 | allocated_MB: 344.8 | reserved_MB: 25586.0 | inactive_split_MB: 283.2 +rank5 [2026-10-28 17:18:21,031] INFO : Step 8700/50000 rank5 | Train Loss: 2.0581 | Train Loss Unweighted: 1.1705 | Train Loss Components: Auxiliary: 0.4038 | Cardinality: 0.2463 | Classification: 0.2066 | Classification_binary: 0.3017 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3267 | Regression_eta: 0.0035 | Regression_pt: 0.3282 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1184 | Classification_binary: 0.3082 | Regression_cos_phi: 0.1601 | Regression_energy: 0.1531 | Regression_eta: 0.0890 | Regression_pt: 0.1567 | Regression_sin_phi: 0.1155 | LR: 9.34e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.7155s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6873 | max_loss: 0.2444 | max_backward: 1.2985 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6214.6 | uss_MB: 5113.2 | child_rss_MB: 5650.1 | allocated_MB: 343.7 | reserved_MB: 25586.0 | inactive_split_MB: 260.4 +rank5 [2026-10-28 17:22:21,713] INFO : Step 8800/50000 rank5 | Train Loss: 1.8415 | Train Loss Unweighted: 1.0591 | Train Loss Components: Auxiliary: 0.3597 | Cardinality: 0.1811 | Classification: 0.1734 | Classification_binary: 0.2490 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3144 | Regression_eta: 0.0034 | Regression_pt: 0.3153 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0842 | Classification_binary: 0.2349 | Regression_cos_phi: 0.1023 | Regression_energy: 0.1520 | Regression_eta: 0.0804 | Regression_pt: 0.1572 | Regression_sin_phi: 0.1394 | LR: 9.33e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.5733s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6737 | max_loss: 0.2385 | max_backward: 1.3073 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.5819 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6215.6 | uss_MB: 5114.6 | child_rss_MB: 5649.0 | allocated_MB: 344.1 | reserved_MB: 25586.0 | inactive_split_MB: 205.9 +rank5 [2026-10-28 17:26:43,764] INFO : Step 8900/50000 rank5 | Train Loss: 2.1706 | Train Loss Unweighted: 1.1768 | Train Loss Components: Auxiliary: 0.4167 | Cardinality: 0.3000 | Classification: 0.2064 | Classification_binary: 0.3037 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3284 | Regression_eta: 0.0039 | Regression_pt: 0.3304 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0909 | Classification_binary: 0.2720 | Regression_cos_phi: 0.1551 | Regression_energy: 0.1577 | Regression_eta: 0.0688 | Regression_pt: 0.1651 | Regression_sin_phi: 0.1270 | LR: 9.31e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.8183s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0073 | max_forward: 0.9973 | max_loss: 0.4040 | max_backward: 3.4126 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6194.6 | uss_MB: 5119.9 | child_rss_MB: 5654.4 | allocated_MB: 347.1 | reserved_MB: 25586.0 | inactive_split_MB: 275.1 +rank5 [2026-10-28 17:30:46,803] INFO : Step 9000/50000 rank5 | Train Loss: 2.1357 | Train Loss Unweighted: 1.1883 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.2351 | Classification: 0.2003 | Classification_binary: 0.3467 | HitGrouping: 0.2532 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3143 | Regression_eta: 0.0041 | Regression_pt: 0.3186 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1348 | Classification_binary: 0.4096 | Regression_cos_phi: 0.1979 | Regression_energy: 0.1522 | Regression_eta: 0.1198 | Regression_pt: 0.1582 | Regression_sin_phi: 0.2238 | LR: 9.30e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.7201s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.7873 | max_loss: 0.2178 | max_backward: 1.5407 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.3125 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 6192.6 | uss_MB: 5121.2 | child_rss_MB: 5660.4 | allocated_MB: 344.6 | reserved_MB: 25586.0 | inactive_split_MB: 289.9 +rank5 [2026-10-28 17:30:47,949] INFO : Running validation on rank5 +rank5 [2026-10-28 17:30:47,949] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 17:30:47,950] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 17:31:40,962] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:31:41,475] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 17:32:08,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 17:32:10,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 17:32:11,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 17:32:12,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 17:32:22,043] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-28 17:32:22,657] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:32:23,200] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 17:32:39,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 17:32:40,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 17:32:41,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 17:32:42,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 17:32:52,009] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 17:32:52,552] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:32:53,101] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 17:33:08,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 17:33:09,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 17:33:09,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 17:33:10,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 17:33:20,697] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 17:37:41,205] INFO : Step 9100/50000 rank5 | Train Loss: 1.8448 | Train Loss Unweighted: 1.0695 | Train Loss Components: Auxiliary: 0.3538 | Cardinality: 0.1722 | Classification: 0.1878 | Classification_binary: 0.2661 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3012 | Regression_eta: 0.0033 | Regression_pt: 0.3075 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1173 | Classification_binary: 0.2589 | Regression_cos_phi: 0.1330 | Regression_energy: 0.1453 | Regression_eta: 0.0990 | Regression_pt: 0.1564 | Regression_sin_phi: 0.1833 | LR: 9.28e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4373s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0153 | max_forward: 1.2776 | max_loss: 0.3881 | max_backward: 1.3802 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9203 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7188.5 | uss_MB: 6128.3 | child_rss_MB: 5636.7 | allocated_MB: 345.3 | reserved_MB: 22582.0 | inactive_split_MB: 275.2 +rank5 [2026-10-28 17:41:50,228] INFO : Step 9200/50000 rank5 | Train Loss: 1.9883 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3829 | Cardinality: 0.2086 | Classification: 0.1812 | Classification_binary: 0.2802 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3207 | Regression_eta: 0.0035 | Regression_pt: 0.3257 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1046 | Classification_binary: 0.2748 | Regression_cos_phi: 0.1421 | Regression_energy: 0.1755 | Regression_eta: 0.1221 | Regression_pt: 0.1870 | Regression_sin_phi: 0.1168 | LR: 9.27e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5141s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0073 | max_forward: 0.6839 | max_loss: 0.2414 | max_backward: 1.6336 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 1.0636 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6214.5 | uss_MB: 5161.4 | child_rss_MB: 5636.3 | allocated_MB: 345.6 | reserved_MB: 23224.0 | inactive_split_MB: 190.1 +rank5 [2026-10-28 17:45:52,428] INFO : Step 9300/50000 rank5 | Train Loss: 2.0749 | Train Loss Unweighted: 1.1591 | Train Loss Components: Auxiliary: 0.4049 | Cardinality: 0.2573 | Classification: 0.2065 | Classification_binary: 0.3180 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3111 | Regression_eta: 0.0036 | Regression_pt: 0.3160 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1169 | Classification_binary: 0.3023 | Regression_cos_phi: 0.1219 | Regression_energy: 0.1701 | Regression_eta: 0.0890 | Regression_pt: 0.1802 | Regression_sin_phi: 0.1229 | LR: 9.25e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2165s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8329 | max_loss: 0.1499 | max_backward: 1.4059 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.8766 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6214.5 | uss_MB: 5164.3 | child_rss_MB: 5654.0 | allocated_MB: 344.8 | reserved_MB: 24586.0 | inactive_split_MB: 287.2 +rank5 [2026-10-28 17:50:16,942] INFO : Step 9400/50000 rank5 | Train Loss: 1.8467 | Train Loss Unweighted: 1.0417 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1520 | Classification: 0.1837 | Classification_binary: 0.2977 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2742 | Regression_eta: 0.0040 | Regression_pt: 0.2784 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0946 | Classification_binary: 0.2871 | Regression_cos_phi: 0.1498 | Regression_energy: 0.1269 | Regression_eta: 0.1120 | Regression_pt: 0.1336 | Regression_sin_phi: 0.1626 | LR: 9.24e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.5888s | Phase Times: max_pre_step_barrier: 0.0180 | max_batch_to_device: 0.0062 | max_forward: 1.5076 | max_loss: 0.3748 | max_backward: 2.6247 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.5247 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6206.4 | uss_MB: 4649.5 | child_rss_MB: 5650.0 | allocated_MB: 343.9 | reserved_MB: 24586.0 | inactive_split_MB: 267.3 +rank5 [2026-10-28 17:54:18,159] INFO : Step 9500/50000 rank5 | Train Loss: 1.8199 | Train Loss Unweighted: 0.9995 | Train Loss Components: Auxiliary: 0.3464 | Cardinality: 0.1875 | Classification: 0.1651 | Classification_binary: 0.2558 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2851 | Regression_eta: 0.0035 | Regression_pt: 0.2864 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0986 | Classification_binary: 0.2565 | Regression_cos_phi: 0.1908 | Regression_energy: 0.1297 | Regression_eta: 0.1541 | Regression_pt: 0.1342 | Regression_sin_phi: 0.1423 | LR: 9.22e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.8050s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6532 | max_loss: 0.2385 | max_backward: 1.0043 | max_optimizer_step: 0.3001 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6205.2 | uss_MB: 5165.5 | child_rss_MB: 5654.5 | allocated_MB: 341.2 | reserved_MB: 24586.0 | inactive_split_MB: 180.8 +rank5 [2026-10-28 17:58:23,634] INFO : Step 9600/50000 rank5 | Train Loss: 2.0900 | Train Loss Unweighted: 1.1491 | Train Loss Components: Auxiliary: 0.4030 | Cardinality: 0.2464 | Classification: 0.2089 | Classification_binary: 0.3189 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3034 | Regression_eta: 0.0042 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1298 | Classification_binary: 0.3043 | Regression_cos_phi: 0.1662 | Regression_energy: 0.1349 | Regression_eta: 0.1160 | Regression_pt: 0.1427 | Regression_sin_phi: 0.1507 | LR: 9.21e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8843s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1083 | max_forward: 0.7381 | max_loss: 0.1457 | max_backward: 1.4890 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4970 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6206.2 | uss_MB: 5167.2 | child_rss_MB: 5637.2 | allocated_MB: 345.2 | reserved_MB: 25594.0 | inactive_split_MB: 190.6 +rank5 [2026-10-28 18:02:35,091] INFO : Step 9700/50000 rank5 | Train Loss: 2.1397 | Train Loss Unweighted: 1.1700 | Train Loss Components: Auxiliary: 0.4082 | Cardinality: 0.2377 | Classification: 0.1942 | Classification_binary: 0.3460 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3095 | Regression_eta: 0.0043 | Regression_pt: 0.3118 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1368 | Classification_binary: 0.3787 | Regression_cos_phi: 0.2042 | Regression_energy: 0.1338 | Regression_eta: 0.1564 | Regression_pt: 0.1413 | Regression_sin_phi: 0.1707 | LR: 9.19e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1034s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7744 | max_loss: 0.1497 | max_backward: 1.6846 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4710 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 6209.3 | uss_MB: 5170.1 | child_rss_MB: 5650.1 | allocated_MB: 345.9 | reserved_MB: 25594.0 | inactive_split_MB: 288.1 +rank5 [2026-10-28 18:06:42,228] INFO : Step 9800/50000 rank5 | Train Loss: 1.8589 | Train Loss Unweighted: 1.0245 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2213 | Classification: 0.1590 | Classification_binary: 0.2751 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2908 | Regression_eta: 0.0031 | Regression_pt: 0.2931 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0990 | Classification_binary: 0.2769 | Regression_cos_phi: 0.1545 | Regression_energy: 0.1258 | Regression_eta: 0.0878 | Regression_pt: 0.1319 | Regression_sin_phi: 0.1483 | LR: 9.17e-04 | DataLoad Time: 0.0231s | Model Forward Time: 4.1212s | Phase Times: max_pre_step_barrier: 0.1099 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.1713 | max_backward: 1.7200 | max_optimizer_step: 0.1898 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6188.3 | uss_MB: 5169.7 | child_rss_MB: 5642.5 | allocated_MB: 342.5 | reserved_MB: 26610.0 | inactive_split_MB: 264.5 +rank5 [2026-10-28 18:10:58,313] INFO : Step 9900/50000 rank5 | Train Loss: 1.8366 | Train Loss Unweighted: 1.0077 | Train Loss Components: Auxiliary: 0.3557 | Cardinality: 0.2091 | Classification: 0.1810 | Classification_binary: 0.2609 | HitGrouping: 0.2462 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2775 | Regression_eta: 0.0035 | Regression_pt: 0.2814 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0970 | Classification_binary: 0.2914 | Regression_cos_phi: 0.1769 | Regression_energy: 0.1202 | Regression_eta: 0.0878 | Regression_pt: 0.1266 | Regression_sin_phi: 0.1472 | LR: 9.16e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5794s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6346 | max_loss: 0.1692 | max_backward: 1.3542 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6188.4 | uss_MB: 5170.9 | child_rss_MB: 5645.5 | allocated_MB: 343.7 | reserved_MB: 26610.0 | inactive_split_MB: 205.7 +rank5 [2026-10-28 18:15:06,636] INFO : Step 10000/50000 rank5 | Train Loss: 2.0706 | Train Loss Unweighted: 1.1295 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2648 | Classification: 0.1819 | Classification_binary: 0.3225 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3077 | Regression_eta: 0.0037 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1229 | Classification_binary: 0.2969 | Regression_cos_phi: 0.1712 | Regression_energy: 0.1456 | Regression_eta: 0.0924 | Regression_pt: 0.1473 | Regression_sin_phi: 0.1386 | LR: 9.14e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.7020s | Phase Times: max_pre_step_barrier: 0.0136 | max_batch_to_device: 0.0708 | max_forward: 1.8842 | max_loss: 0.5484 | max_backward: 2.6620 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.5314 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6190.4 | uss_MB: 5174.2 | child_rss_MB: 5642.5 | allocated_MB: 345.2 | reserved_MB: 26610.0 | inactive_split_MB: 288.8 +rank5 [2026-10-28 18:15:07,627] INFO : Running validation on rank5 +rank5 [2026-10-28 18:15:07,627] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 18:15:07,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 18:15:55,923] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:15:56,538] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 18:16:22,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 18:16:23,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 18:16:24,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 18:16:25,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 18:16:35,711] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 18:16:36,561] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:16:37,076] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 18:16:53,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 18:16:54,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 18:16:55,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 18:16:57,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 18:17:07,091] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 18:17:09,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:17:10,079] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 18:17:25,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 18:17:26,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 18:17:26,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 18:17:27,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 18:17:37,558] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 18:22:01,597] INFO : Step 10100/50000 rank5 | Train Loss: 1.8115 | Train Loss Unweighted: 1.0221 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.1999 | Classification: 0.1589 | Classification_binary: 0.2815 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2868 | Regression_eta: 0.0032 | Regression_pt: 0.2886 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0936 | Classification_binary: 0.2908 | Regression_cos_phi: 0.1244 | Regression_energy: 0.1303 | Regression_eta: 0.0996 | Regression_pt: 0.1332 | Regression_sin_phi: 0.1538 | LR: 9.12e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9766s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7090 | max_loss: 0.1507 | max_backward: 1.2179 | max_optimizer_step: 0.3146 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 7166.8 | uss_MB: 6150.2 | child_rss_MB: 5645.8 | allocated_MB: 343.1 | reserved_MB: 21794.0 | inactive_split_MB: 243.8 +rank5 [2026-10-28 18:26:06,876] INFO : Step 10200/50000 rank5 | Train Loss: 1.6749 | Train Loss Unweighted: 0.9660 | Train Loss Components: Auxiliary: 0.3310 | Cardinality: 0.1773 | Classification: 0.1544 | Classification_binary: 0.2377 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2827 | Regression_eta: 0.0027 | Regression_pt: 0.2857 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0878 | Classification_binary: 0.2186 | Regression_cos_phi: 0.1239 | Regression_energy: 0.1301 | Regression_eta: 0.0691 | Regression_pt: 0.1346 | Regression_sin_phi: 0.1312 | LR: 9.11e-04 | DataLoad Time: 0.0120s | Model Forward Time: 3.9990s | Phase Times: max_pre_step_barrier: 0.0352 | max_batch_to_device: 0.0241 | max_forward: 0.6625 | max_loss: 0.1442 | max_backward: 1.3835 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7009 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1256 | Memory: rss_MB: 7164.8 | uss_MB: 6150.1 | child_rss_MB: 5645.0 | allocated_MB: 345.9 | reserved_MB: 21794.0 | inactive_split_MB: 292.4 +rank5 [2026-10-28 18:30:14,208] INFO : Step 10300/50000 rank5 | Train Loss: 1.9113 | Train Loss Unweighted: 1.0360 | Train Loss Components: Auxiliary: 0.3660 | Cardinality: 0.2108 | Classification: 0.1830 | Classification_binary: 0.2931 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0033 | Regression_pt: 0.2786 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1395 | Classification_binary: 0.3206 | Regression_cos_phi: 0.2780 | Regression_energy: 0.1306 | Regression_eta: 0.1583 | Regression_pt: 0.1388 | Regression_sin_phi: 0.2235 | LR: 9.09e-04 | DataLoad Time: 0.0143s | Model Forward Time: 3.9120s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7205 | max_loss: 0.1496 | max_backward: 1.6083 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 6287.2 | uss_MB: 5273.5 | child_rss_MB: 5650.9 | allocated_MB: 343.8 | reserved_MB: 22764.0 | inactive_split_MB: 257.9 +rank5 [2026-10-28 18:34:19,878] INFO : Step 10400/50000 rank5 | Train Loss: 1.9303 | Train Loss Unweighted: 1.0569 | Train Loss Components: Auxiliary: 0.3739 | Cardinality: 0.2318 | Classification: 0.1704 | Classification_binary: 0.3342 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2718 | Regression_eta: 0.0033 | Regression_pt: 0.2738 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1114 | Classification_binary: 0.3470 | Regression_cos_phi: 0.1653 | Regression_energy: 0.1166 | Regression_eta: 0.0932 | Regression_pt: 0.1201 | Regression_sin_phi: 0.1690 | LR: 9.07e-04 | DataLoad Time: 0.0148s | Model Forward Time: 3.7369s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7465 | max_loss: 0.1499 | max_backward: 1.4212 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.4712 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6287.2 | uss_MB: 5272.0 | child_rss_MB: 5660.4 | allocated_MB: 345.6 | reserved_MB: 22792.0 | inactive_split_MB: 259.8 +rank5 [2026-10-28 18:38:24,242] INFO : Step 10500/50000 rank5 | Train Loss: 1.6601 | Train Loss Unweighted: 0.9268 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1892 | Classification: 0.1410 | Classification_binary: 0.2342 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2693 | Regression_eta: 0.0029 | Regression_pt: 0.2766 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2707 | Regression_cos_phi: 0.1144 | Regression_energy: 0.1322 | Regression_eta: 0.1010 | Regression_pt: 0.1384 | Regression_sin_phi: 0.1032 | LR: 9.06e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.5602s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 0.6364 | max_loss: 0.2260 | max_backward: 1.1629 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6452 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6284.6 | uss_MB: 5269.3 | child_rss_MB: 5635.7 | allocated_MB: 340.4 | reserved_MB: 22792.0 | inactive_split_MB: 230.3 +rank5 [2026-10-28 18:42:22,546] INFO : Step 10600/50000 rank5 | Train Loss: 1.9388 | Train Loss Unweighted: 1.0719 | Train Loss Components: Auxiliary: 0.3737 | Cardinality: 0.2008 | Classification: 0.1863 | Classification_binary: 0.2709 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3047 | Regression_eta: 0.0042 | Regression_pt: 0.3019 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0826 | Classification_binary: 0.2838 | Regression_cos_phi: 0.1272 | Regression_energy: 0.1489 | Regression_eta: 0.0972 | Regression_pt: 0.1536 | Regression_sin_phi: 0.1311 | LR: 9.04e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9680s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6880 | max_loss: 0.2458 | max_backward: 1.4354 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7481 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 6285.6 | uss_MB: 5273.7 | child_rss_MB: 5653.3 | allocated_MB: 344.6 | reserved_MB: 22798.0 | inactive_split_MB: 286.4 +rank5 [2026-10-28 18:46:28,871] INFO : Step 10700/50000 rank5 | Train Loss: 1.8955 | Train Loss Unweighted: 1.0230 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.2139 | Classification: 0.1752 | Classification_binary: 0.3186 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2598 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0842 | Classification_binary: 0.2896 | Regression_cos_phi: 0.1295 | Regression_energy: 0.1333 | Regression_eta: 0.0876 | Regression_pt: 0.1364 | Regression_sin_phi: 0.1471 | LR: 9.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1535s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0072 | max_forward: 1.0363 | max_loss: 0.1757 | max_backward: 2.3255 | max_optimizer_step: 0.1834 | max_tensorboard_logging: 0.7232 | max_tensorboard_flush: 0.0051 | max_post_step_barrier: 0.1576 | Memory: rss_MB: 6287.7 | uss_MB: 5274.9 | child_rss_MB: 5649.7 | allocated_MB: 342.1 | reserved_MB: 22798.0 | inactive_split_MB: 282.9 +rank5 [2026-10-28 18:50:28,325] INFO : Step 10800/50000 rank5 | Train Loss: 1.7679 | Train Loss Unweighted: 0.9563 | Train Loss Components: Auxiliary: 0.3446 | Cardinality: 0.2074 | Classification: 0.1518 | Classification_binary: 0.2766 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2590 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1041 | Classification_binary: 0.3012 | Regression_cos_phi: 0.1493 | Regression_energy: 0.1211 | Regression_eta: 0.1112 | Regression_pt: 0.1246 | Regression_sin_phi: 0.1764 | LR: 9.00e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7796s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0995 | max_forward: 0.7884 | max_loss: 0.2225 | max_backward: 2.0144 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.5230 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6288.7 | uss_MB: 5271.1 | child_rss_MB: 5654.7 | allocated_MB: 343.6 | reserved_MB: 23784.0 | inactive_split_MB: 286.4 +rank5 [2026-10-28 18:54:31,737] INFO : Step 10900/50000 rank5 | Train Loss: 1.6025 | Train Loss Unweighted: 0.8934 | Train Loss Components: Auxiliary: 0.3208 | Cardinality: 0.1729 | Classification: 0.1446 | Classification_binary: 0.2284 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2574 | Regression_eta: 0.0027 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0855 | Classification_binary: 0.2300 | Regression_cos_phi: 0.0841 | Regression_energy: 0.1184 | Regression_eta: 0.0697 | Regression_pt: 0.1233 | Regression_sin_phi: 0.1255 | LR: 8.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.4462s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.5765 | max_loss: 0.1490 | max_backward: 1.1882 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6288.0 | uss_MB: 5275.5 | child_rss_MB: 5639.7 | allocated_MB: 342.1 | reserved_MB: 23796.0 | inactive_split_MB: 272.7 +rank5 [2026-10-28 18:58:45,418] INFO : Step 11000/50000 rank5 | Train Loss: 1.9315 | Train Loss Unweighted: 1.0466 | Train Loss Components: Auxiliary: 0.3721 | Cardinality: 0.2078 | Classification: 0.1806 | Classification_binary: 0.3087 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0037 | Regression_pt: 0.2758 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1123 | Classification_binary: 0.3129 | Regression_cos_phi: 0.1654 | Regression_energy: 0.1342 | Regression_eta: 0.1189 | Regression_pt: 0.1384 | Regression_sin_phi: 0.1614 | LR: 8.97e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.1701s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7424 | max_loss: 0.1704 | max_backward: 1.5866 | max_optimizer_step: 0.2906 | max_tensorboard_logging: 0.6287 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6277.0 | uss_MB: 5281.8 | child_rss_MB: 5663.8 | allocated_MB: 347.8 | reserved_MB: 23796.0 | inactive_split_MB: 262.8 +rank5 [2026-10-28 18:58:46,360] INFO : Running validation on rank5 +rank5 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 18:59:40,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:59:40,904] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 19:00:07,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 19:00:08,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 19:00:10,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 19:00:11,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 19:00:21,149] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-28 19:00:21,678] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:00:22,257] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 19:00:38,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 19:00:39,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 19:00:40,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 19:00:41,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 19:00:51,564] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 19:00:52,166] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:00:52,895] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 19:01:08,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 19:01:08,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 19:01:09,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 19:01:10,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 19:01:20,168] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 19:05:41,394] INFO : Step 11100/50000 rank5 | Train Loss: 1.9118 | Train Loss Unweighted: 1.0064 | Train Loss Components: Auxiliary: 0.3723 | Cardinality: 0.2793 | Classification: 0.1563 | Classification_binary: 0.3082 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2660 | Regression_eta: 0.0034 | Regression_pt: 0.2693 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1103 | Classification_binary: 0.3397 | Regression_cos_phi: 0.1512 | Regression_energy: 0.1314 | Regression_eta: 0.1011 | Regression_pt: 0.1390 | Regression_sin_phi: 0.1458 | LR: 8.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0434s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.9217 | max_loss: 0.2304 | max_backward: 1.5570 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 7169.9 | uss_MB: 6179.3 | child_rss_MB: 5664.2 | allocated_MB: 343.2 | reserved_MB: 22146.0 | inactive_split_MB: 281.7 +rank5 [2026-10-28 19:09:39,138] INFO : Step 11200/50000 rank5 | Train Loss: 1.7366 | Train Loss Unweighted: 0.9397 | Train Loss Components: Auxiliary: 0.3529 | Cardinality: 0.2466 | Classification: 0.1384 | Classification_binary: 0.2548 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2689 | Regression_eta: 0.0027 | Regression_pt: 0.2722 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0871 | Classification_binary: 0.2692 | Regression_cos_phi: 0.0862 | Regression_energy: 0.1337 | Regression_eta: 0.0799 | Regression_pt: 0.1404 | Regression_sin_phi: 0.1536 | LR: 8.93e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7184s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7793 | max_loss: 0.1825 | max_backward: 1.3743 | max_optimizer_step: 0.2128 | max_tensorboard_logging: 0.4676 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7167.9 | uss_MB: 6179.2 | child_rss_MB: 5641.7 | allocated_MB: 344.6 | reserved_MB: 22972.0 | inactive_split_MB: 273.4 +rank5 [2026-10-28 19:13:41,797] INFO : Step 11300/50000 rank5 | Train Loss: 1.6876 | Train Loss Unweighted: 0.9423 | Train Loss Components: Auxiliary: 0.3334 | Cardinality: 0.1862 | Classification: 0.1581 | Classification_binary: 0.2713 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2524 | Regression_eta: 0.0027 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0965 | Classification_binary: 0.2727 | Regression_cos_phi: 0.1331 | Regression_energy: 0.1400 | Regression_eta: 0.0854 | Regression_pt: 0.1477 | Regression_sin_phi: 0.1385 | LR: 8.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6827s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0223 | max_forward: 0.9315 | max_loss: 0.2308 | max_backward: 1.9859 | max_optimizer_step: 0.1928 | max_tensorboard_logging: 0.5681 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6388.7 | uss_MB: 5273.7 | child_rss_MB: 5658.4 | allocated_MB: 345.5 | reserved_MB: 24368.0 | inactive_split_MB: 269.6 +rank5 [2026-10-28 19:17:45,333] INFO : Step 11400/50000 rank5 | Train Loss: 1.8563 | Train Loss Unweighted: 0.9980 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.2376 | Classification: 0.1627 | Classification_binary: 0.3164 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2551 | Regression_eta: 0.0029 | Regression_pt: 0.2577 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0838 | Classification_binary: 0.2743 | Regression_cos_phi: 0.1083 | Regression_energy: 0.1293 | Regression_eta: 0.0620 | Regression_pt: 0.1348 | Regression_sin_phi: 0.1164 | LR: 8.89e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.0875s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.7684 | max_loss: 0.1511 | max_backward: 1.6966 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5034 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 6391.7 | uss_MB: 5402.6 | child_rss_MB: 5664.5 | allocated_MB: 343.2 | reserved_MB: 25642.0 | inactive_split_MB: 251.6 +rank5 [2026-10-28 19:21:55,044] INFO : Step 11500/50000 rank5 | Train Loss: 1.6323 | Train Loss Unweighted: 0.8976 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1755 | Classification: 0.1375 | Classification_binary: 0.2717 | HitGrouping: 0.2346 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2408 | Regression_eta: 0.0028 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.2900 | Regression_cos_phi: 0.1085 | Regression_energy: 0.1212 | Regression_eta: 0.0678 | Regression_pt: 0.1264 | Regression_sin_phi: 0.1015 | LR: 8.88e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.6950s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7232 | max_loss: 0.1557 | max_backward: 1.4770 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6390.5 | uss_MB: 5401.3 | child_rss_MB: 5644.3 | allocated_MB: 345.3 | reserved_MB: 25642.0 | inactive_split_MB: 274.7 +rank5 [2026-10-28 19:25:54,594] INFO : Step 11600/50000 rank5 | Train Loss: 1.6133 | Train Loss Unweighted: 0.9062 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1718 | Classification: 0.1502 | Classification_binary: 0.2356 | HitGrouping: 0.2377 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2569 | Regression_eta: 0.0025 | Regression_pt: 0.2581 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1050 | Classification_binary: 0.2403 | Regression_cos_phi: 0.1260 | Regression_energy: 0.1482 | Regression_eta: 0.0716 | Regression_pt: 0.1528 | Regression_sin_phi: 0.1469 | LR: 8.86e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6492s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6646 | max_loss: 0.1710 | max_backward: 1.3208 | max_optimizer_step: 0.1963 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6391.5 | uss_MB: 5404.2 | child_rss_MB: 5665.3 | allocated_MB: 344.1 | reserved_MB: 26630.0 | inactive_split_MB: 209.9 +rank5 [2026-10-28 19:29:50,732] INFO : Step 11700/50000 rank5 | Train Loss: 1.7739 | Train Loss Unweighted: 0.9706 | Train Loss Components: Auxiliary: 0.3531 | Cardinality: 0.2418 | Classification: 0.1518 | Classification_binary: 0.2821 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2638 | Regression_eta: 0.0028 | Regression_pt: 0.2671 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0817 | Classification_binary: 0.2931 | Regression_cos_phi: 0.1292 | Regression_energy: 0.1327 | Regression_eta: 0.0850 | Regression_pt: 0.1426 | Regression_sin_phi: 0.1170 | LR: 8.84e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.4606s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8789 | max_loss: 0.2057 | max_backward: 2.3846 | max_optimizer_step: 0.1875 | max_tensorboard_logging: 1.0403 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6389.5 | uss_MB: 4921.9 | child_rss_MB: 5640.9 | allocated_MB: 343.8 | reserved_MB: 27624.0 | inactive_split_MB: 186.6 +rank5 [2026-10-28 19:33:54,004] INFO : Step 11800/50000 rank5 | Train Loss: 1.8223 | Train Loss Unweighted: 0.9745 | Train Loss Components: Auxiliary: 0.3609 | Cardinality: 0.2329 | Classification: 0.1589 | Classification_binary: 0.3100 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2473 | Regression_eta: 0.0028 | Regression_pt: 0.2523 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0831 | Classification_binary: 0.2789 | Regression_cos_phi: 0.1125 | Regression_energy: 0.1292 | Regression_eta: 0.0716 | Regression_pt: 0.1376 | Regression_sin_phi: 0.1226 | LR: 8.82e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.3860s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0070 | max_forward: 1.2302 | max_loss: 0.2767 | max_backward: 2.3745 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6390.5 | uss_MB: 5402.7 | child_rss_MB: 5661.5 | allocated_MB: 346.0 | reserved_MB: 27624.0 | inactive_split_MB: 280.5 +rank5 [2026-10-28 19:37:48,967] INFO : Step 11900/50000 rank5 | Train Loss: 1.5797 | Train Loss Unweighted: 0.8638 | Train Loss Components: Auxiliary: 0.3225 | Cardinality: 0.1935 | Classification: 0.1358 | Classification_binary: 0.2351 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2427 | Regression_eta: 0.0025 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.2862 | Regression_cos_phi: 0.1216 | Regression_energy: 0.1357 | Regression_eta: 0.0608 | Regression_pt: 0.1424 | Regression_sin_phi: 0.1001 | LR: 8.80e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.1382s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0787 | max_forward: 0.7011 | max_loss: 0.1484 | max_backward: 1.2002 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.9463 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 6390.3 | uss_MB: 5402.5 | child_rss_MB: 5642.2 | allocated_MB: 340.1 | reserved_MB: 28620.0 | inactive_split_MB: 264.1 +rank5 [2026-10-28 19:41:43,167] INFO : Step 12000/50000 rank5 | Train Loss: 1.6036 | Train Loss Unweighted: 0.8706 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1972 | Classification: 0.1481 | Classification_binary: 0.2459 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2330 | Regression_eta: 0.0025 | Regression_pt: 0.2385 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0899 | Classification_binary: 0.2584 | Regression_cos_phi: 0.1444 | Regression_energy: 0.1151 | Regression_eta: 0.0753 | Regression_pt: 0.1237 | Regression_sin_phi: 0.1388 | LR: 8.78e-04 | DataLoad Time: 0.0143s | Model Forward Time: 3.6988s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0072 | max_forward: 0.6194 | max_loss: 0.1546 | max_backward: 1.4628 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5143 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6383.3 | uss_MB: 5409.4 | child_rss_MB: 5646.2 | allocated_MB: 342.9 | reserved_MB: 28620.0 | inactive_split_MB: 215.8 +rank5 [2026-10-28 19:41:44,123] INFO : Running validation on rank5 +rank5 [2026-10-28 19:41:44,123] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 19:41:44,123] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 19:42:35,080] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:42:35,719] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 19:43:02,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 19:43:04,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 19:43:05,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 19:43:06,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 19:43:16,694] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-28 19:43:17,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:43:17,650] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 19:43:33,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 19:43:34,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 19:43:35,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 19:43:36,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 19:43:46,704] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 19:43:48,086] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:43:48,651] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 19:44:04,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 19:44:04,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 19:44:05,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 19:44:06,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 19:44:16,294] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 19:48:34,082] INFO : Step 12100/50000 rank5 | Train Loss: 1.8149 | Train Loss Unweighted: 0.9793 | Train Loss Components: Auxiliary: 0.3632 | Cardinality: 0.2324 | Classification: 0.1506 | Classification_binary: 0.3149 | HitGrouping: 0.2337 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2518 | Regression_eta: 0.0029 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1002 | Classification_binary: 0.3295 | Regression_cos_phi: 0.1238 | Regression_energy: 0.1176 | Regression_eta: 0.0916 | Regression_pt: 0.1234 | Regression_sin_phi: 0.1176 | LR: 8.76e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5263s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8305 | max_loss: 0.1678 | max_backward: 1.6498 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8610 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7131.3 | uss_MB: 6150.6 | child_rss_MB: 5646.8 | allocated_MB: 345.8 | reserved_MB: 23512.0 | inactive_split_MB: 299.6 +rank5 [2026-10-28 19:52:54,661] INFO : Step 12200/50000 rank5 | Train Loss: 1.6619 | Train Loss Unweighted: 0.9043 | Train Loss Components: Auxiliary: 0.3290 | Cardinality: 0.1906 | Classification: 0.1445 | Classification_binary: 0.2674 | HitGrouping: 0.2303 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2435 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0751 | Classification_binary: 0.2536 | Regression_cos_phi: 0.0966 | Regression_energy: 0.1153 | Regression_eta: 0.0674 | Regression_pt: 0.1180 | Regression_sin_phi: 0.1131 | LR: 8.74e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8240s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.7862 | max_loss: 0.1438 | max_backward: 1.5991 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.8843 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7133.3 | uss_MB: 5953.1 | child_rss_MB: 5653.1 | allocated_MB: 341.2 | reserved_MB: 23512.0 | inactive_split_MB: 206.8 +rank5 [2026-10-28 19:56:47,337] INFO : Step 12300/50000 rank5 | Train Loss: 1.4699 | Train Loss Unweighted: 0.8181 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1474 | Classification: 0.1404 | Classification_binary: 0.2280 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2222 | Regression_eta: 0.0028 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1170 | Classification_binary: 0.2203 | Regression_cos_phi: 0.0857 | Regression_energy: 0.1073 | Regression_eta: 0.0783 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0955 | LR: 8.72e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.7228s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6177 | max_loss: 0.2232 | max_backward: 1.1179 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 0.7974 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6497.9 | uss_MB: 5527.4 | child_rss_MB: 5654.1 | allocated_MB: 343.2 | reserved_MB: 24286.0 | inactive_split_MB: 290.2 +rank5 [2026-10-28 20:00:36,191] INFO : Step 12400/50000 rank5 | Train Loss: 1.6135 | Train Loss Unweighted: 0.8782 | Train Loss Components: Auxiliary: 0.3265 | Cardinality: 0.2056 | Classification: 0.1546 | Classification_binary: 0.2540 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2308 | Regression_eta: 0.0025 | Regression_pt: 0.2337 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0995 | Classification_binary: 0.2112 | Regression_cos_phi: 0.0943 | Regression_energy: 0.1108 | Regression_eta: 0.0604 | Regression_pt: 0.1138 | Regression_sin_phi: 0.1075 | LR: 8.70e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.9928s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6928 | max_loss: 0.1486 | max_backward: 1.3860 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6497.9 | uss_MB: 5528.1 | child_rss_MB: 5671.7 | allocated_MB: 344.8 | reserved_MB: 24510.0 | inactive_split_MB: 295.5 +rank5 [2026-10-28 20:04:30,060] INFO : Step 12500/50000 rank5 | Train Loss: 1.6527 | Train Loss Unweighted: 0.8797 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1890 | Classification: 0.1391 | Classification_binary: 0.2779 | HitGrouping: 0.2315 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2273 | Regression_eta: 0.0030 | Regression_pt: 0.2295 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2876 | Regression_cos_phi: 0.0896 | Regression_energy: 0.1151 | Regression_eta: 0.0776 | Regression_pt: 0.1207 | Regression_sin_phi: 0.1195 | LR: 8.68e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.1679s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0535 | max_forward: 0.7747 | max_loss: 0.1534 | max_backward: 1.3339 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6494.1 | uss_MB: 5528.8 | child_rss_MB: 5649.7 | allocated_MB: 347.2 | reserved_MB: 24510.0 | inactive_split_MB: 456.8 +rank5 [2026-10-28 20:08:30,520] INFO : Step 12600/50000 rank5 | Train Loss: 1.4969 | Train Loss Unweighted: 0.8185 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1595 | Classification: 0.1210 | Classification_binary: 0.2357 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2279 | Regression_eta: 0.0024 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2339 | Regression_cos_phi: 0.1045 | Regression_energy: 0.1090 | Regression_eta: 0.0672 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0781 | LR: 8.66e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7599s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0062 | max_forward: 0.6571 | max_loss: 0.1429 | max_backward: 1.4364 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6482.1 | uss_MB: 5531.3 | child_rss_MB: 5638.2 | allocated_MB: 342.9 | reserved_MB: 25526.0 | inactive_split_MB: 234.6 +rank5 [2026-10-28 20:12:34,582] INFO : Step 12700/50000 rank5 | Train Loss: 1.6234 | Train Loss Unweighted: 0.8661 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.2004 | Classification: 0.1459 | Classification_binary: 0.2387 | HitGrouping: 0.2403 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2360 | Regression_eta: 0.0027 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0838 | Classification_binary: 0.2711 | Regression_cos_phi: 0.1380 | Regression_energy: 0.1114 | Regression_eta: 0.0792 | Regression_pt: 0.1179 | Regression_sin_phi: 0.1111 | LR: 8.64e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.4795s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6221 | max_loss: 0.1846 | max_backward: 1.1554 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6469.4 | uss_MB: 5531.3 | child_rss_MB: 5648.7 | allocated_MB: 345.9 | reserved_MB: 26286.0 | inactive_split_MB: 241.7 +rank5 [2026-10-28 20:16:22,943] INFO : Step 12800/50000 rank5 | Train Loss: 1.7877 | Train Loss Unweighted: 0.9398 | Train Loss Components: Auxiliary: 0.3651 | Cardinality: 0.2327 | Classification: 0.1495 | Classification_binary: 0.3094 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2357 | Regression_eta: 0.0027 | Regression_pt: 0.2394 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1013 | Classification_binary: 0.3059 | Regression_cos_phi: 0.1280 | Regression_energy: 0.1189 | Regression_eta: 0.0764 | Regression_pt: 0.1228 | Regression_sin_phi: 0.1588 | LR: 8.62e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2593s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.8468 | max_loss: 0.1534 | max_backward: 1.7420 | max_optimizer_step: 0.3013 | max_tensorboard_logging: 0.5725 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6458.4 | uss_MB: 5535.9 | child_rss_MB: 5651.5 | allocated_MB: 345.6 | reserved_MB: 26286.0 | inactive_split_MB: 284.4 +rank5 [2026-10-28 20:20:13,787] INFO : Step 12900/50000 rank5 | Train Loss: 1.5881 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.3191 | Cardinality: 0.2029 | Classification: 0.1304 | Classification_binary: 0.2564 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2291 | Regression_eta: 0.0023 | Regression_pt: 0.2312 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0795 | Classification_binary: 0.2898 | Regression_cos_phi: 0.0839 | Regression_energy: 0.1156 | Regression_eta: 0.0710 | Regression_pt: 0.1203 | Regression_sin_phi: 0.1087 | LR: 8.60e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9622s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7180 | max_loss: 0.2380 | max_backward: 1.4488 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6932 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6444.6 | uss_MB: 5531.5 | child_rss_MB: 5646.1 | allocated_MB: 341.8 | reserved_MB: 26286.0 | inactive_split_MB: 252.2 +rank5 [2026-10-28 20:24:06,548] INFO : Step 13000/50000 rank5 | Train Loss: 1.5217 | Train Loss Unweighted: 0.8547 | Train Loss Components: Auxiliary: 0.3141 | Cardinality: 0.1693 | Classification: 0.1382 | Classification_binary: 0.2135 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2470 | Regression_eta: 0.0024 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2007 | Regression_cos_phi: 0.1015 | Regression_energy: 0.1241 | Regression_eta: 0.0502 | Regression_pt: 0.1303 | Regression_sin_phi: 0.1220 | LR: 8.58e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8286s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7101 | max_loss: 0.1763 | max_backward: 1.1845 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 0.8282 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6429.6 | uss_MB: 5536.9 | child_rss_MB: 5653.9 | allocated_MB: 339.8 | reserved_MB: 26286.0 | inactive_split_MB: 213.4 +rank5 [2026-10-28 20:24:07,450] INFO : Running validation on rank5 +rank5 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 20:24:58,457] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:24:59,078] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 20:25:24,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 20:25:25,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 20:25:26,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 20:25:27,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 20:25:37,612] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 20:25:38,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:25:38,650] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 20:25:54,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 20:25:55,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 20:25:56,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 20:25:57,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 20:26:07,624] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 20:26:08,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:26:09,008] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 20:26:24,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 20:26:24,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 20:26:25,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 20:26:26,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 20:26:36,128] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 20:30:47,340] INFO : Step 13100/50000 rank5 | Train Loss: 1.6168 | Train Loss Unweighted: 0.8649 | Train Loss Components: Auxiliary: 0.3213 | Cardinality: 0.1869 | Classification: 0.1467 | Classification_binary: 0.2696 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2210 | Regression_eta: 0.0027 | Regression_pt: 0.2221 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0839 | Classification_binary: 0.2833 | Regression_cos_phi: 0.1320 | Regression_energy: 0.0991 | Regression_eta: 0.0630 | Regression_pt: 0.1008 | Regression_sin_phi: 0.1185 | LR: 8.56e-04 | DataLoad Time: 0.0130s | Model Forward Time: 6.3736s | Phase Times: max_pre_step_barrier: 0.0337 | max_batch_to_device: 0.0136 | max_forward: 1.2842 | max_loss: 0.2583 | max_backward: 3.1281 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.9677 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7302.9 | uss_MB: 6406.9 | child_rss_MB: 5646.9 | allocated_MB: 341.8 | reserved_MB: 22940.0 | inactive_split_MB: 270.2 +rank5 [2026-10-28 20:34:56,292] INFO : Step 13200/50000 rank5 | Train Loss: 1.6996 | Train Loss Unweighted: 0.8855 | Train Loss Components: Auxiliary: 0.3386 | Cardinality: 0.2474 | Classification: 0.1410 | Classification_binary: 0.2769 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2299 | Regression_eta: 0.0027 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.2765 | Regression_cos_phi: 0.0992 | Regression_energy: 0.1184 | Regression_eta: 0.0681 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0770 | LR: 8.54e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.1534s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0072 | max_forward: 0.7543 | max_loss: 0.1532 | max_backward: 1.3317 | max_optimizer_step: 0.3180 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7300.0 | uss_MB: 6407.2 | child_rss_MB: 5649.5 | allocated_MB: 343.4 | reserved_MB: 23942.0 | inactive_split_MB: 249.6 +rank5 [2026-10-28 20:38:53,363] INFO : Step 13300/50000 rank5 | Train Loss: 1.4876 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2970 | Cardinality: 0.2018 | Classification: 0.1100 | Classification_binary: 0.2192 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2205 | Regression_eta: 0.0026 | Regression_pt: 0.2230 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0815 | Classification_binary: 0.2593 | Regression_cos_phi: 0.0962 | Regression_energy: 0.1061 | Regression_eta: 0.0806 | Regression_pt: 0.1107 | Regression_sin_phi: 0.1110 | LR: 8.52e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.6362s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0259 | max_forward: 0.9208 | max_loss: 0.2245 | max_backward: 1.9523 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6473.0 | uss_MB: 4937.1 | child_rss_MB: 5535.0 | allocated_MB: 341.7 | reserved_MB: 24474.0 | inactive_split_MB: 265.4 +rank5 [2026-10-28 20:42:52,944] INFO : Step 13400/50000 rank5 | Train Loss: 1.4253 | Train Loss Unweighted: 0.7944 | Train Loss Components: Auxiliary: 0.2889 | Cardinality: 0.1585 | Classification: 0.1406 | Classification_binary: 0.2158 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2158 | Regression_eta: 0.0022 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.2003 | Regression_cos_phi: 0.1027 | Regression_energy: 0.1011 | Regression_eta: 0.0680 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0767 | LR: 8.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7381s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6533 | max_loss: 0.2500 | max_backward: 1.4458 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.5454 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6474.0 | uss_MB: 5610.2 | child_rss_MB: 5538.3 | allocated_MB: 341.6 | reserved_MB: 25334.0 | inactive_split_MB: 266.6 +rank5 [2026-10-28 20:47:05,652] INFO : Step 13500/50000 rank5 | Train Loss: 1.6021 | Train Loss Unweighted: 0.8749 | Train Loss Components: Auxiliary: 0.3241 | Cardinality: 0.1832 | Classification: 0.1473 | Classification_binary: 0.2801 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2200 | Regression_eta: 0.0024 | Regression_pt: 0.2225 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0946 | Classification_binary: 0.3020 | Regression_cos_phi: 0.1460 | Regression_energy: 0.1015 | Regression_eta: 0.0805 | Regression_pt: 0.1061 | Regression_sin_phi: 0.1435 | LR: 8.48e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.1058s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.7859 | max_loss: 0.3303 | max_backward: 2.5111 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4974 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6412.7 | uss_MB: 5395.8 | child_rss_MB: 5343.5 | allocated_MB: 344.3 | reserved_MB: 25574.0 | inactive_split_MB: 284.1 +rank5 [2026-10-28 20:51:04,210] INFO : Step 13600/50000 rank5 | Train Loss: 1.5296 | Train Loss Unweighted: 0.8114 | Train Loss Components: Auxiliary: 0.3050 | Cardinality: 0.1872 | Classification: 0.1291 | Classification_binary: 0.2450 | HitGrouping: 0.2218 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2158 | Regression_eta: 0.0025 | Regression_pt: 0.2165 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2225 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0897 | Regression_eta: 0.0545 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0914 | LR: 8.45e-04 | DataLoad Time: 0.0224s | Model Forward Time: 3.6631s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.7508 | max_loss: 0.1507 | max_backward: 1.2891 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6404.7 | uss_MB: 5608.6 | child_rss_MB: 5283.3 | allocated_MB: 345.0 | reserved_MB: 26608.0 | inactive_split_MB: 257.3 +rank5 [2026-10-28 20:55:12,337] INFO : Step 13700/50000 rank5 | Train Loss: 1.4515 | Train Loss Unweighted: 0.7895 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1575 | Classification: 0.1261 | Classification_binary: 0.2121 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2219 | Regression_eta: 0.0023 | Regression_pt: 0.2245 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0840 | Classification_binary: 0.2161 | Regression_cos_phi: 0.0844 | Regression_energy: 0.1358 | Regression_eta: 0.0549 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0880 | LR: 8.43e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.8532s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.6804 | max_loss: 0.2128 | max_backward: 1.1331 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6398.1 | uss_MB: 5609.6 | child_rss_MB: 5245.6 | allocated_MB: 342.5 | reserved_MB: 26608.0 | inactive_split_MB: 241.7 +rank5 [2026-10-28 20:59:19,101] INFO : Step 13800/50000 rank5 | Train Loss: 1.5060 | Train Loss Unweighted: 0.8320 | Train Loss Components: Auxiliary: 0.3084 | Cardinality: 0.1762 | Classification: 0.1368 | Classification_binary: 0.2567 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2156 | Regression_eta: 0.0023 | Regression_pt: 0.2184 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2363 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0987 | Regression_eta: 0.0456 | Regression_pt: 0.1037 | Regression_sin_phi: 0.1017 | LR: 8.41e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4130s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7967 | max_loss: 0.1491 | max_backward: 1.5976 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6392.1 | uss_MB: 5613.3 | child_rss_MB: 5216.1 | allocated_MB: 342.1 | reserved_MB: 26608.0 | inactive_split_MB: 241.3 +rank5 [2026-10-28 21:03:21,775] INFO : Step 13900/50000 rank5 | Train Loss: 1.6108 | Train Loss Unweighted: 0.8469 | Train Loss Components: Auxiliary: 0.3204 | Cardinality: 0.1942 | Classification: 0.1303 | Classification_binary: 0.2751 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2168 | Regression_eta: 0.0028 | Regression_pt: 0.2192 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.2968 | Regression_cos_phi: 0.1546 | Regression_energy: 0.1000 | Regression_eta: 0.1357 | Regression_pt: 0.1005 | Regression_sin_phi: 0.1842 | LR: 8.39e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.7325s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8010 | max_loss: 0.1552 | max_backward: 2.5526 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 1.2862 | max_tensorboard_flush: 0.0041 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 6389.1 | uss_MB: 5603.5 | child_rss_MB: 5217.8 | allocated_MB: 345.6 | reserved_MB: 26608.0 | inactive_split_MB: 292.8 +rank5 [2026-10-28 21:07:29,377] INFO : Step 14000/50000 rank5 | Train Loss: 1.3659 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1707 | Classification: 0.0974 | Classification_binary: 0.1997 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2149 | Regression_eta: 0.0023 | Regression_pt: 0.2181 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1817 | Regression_cos_phi: 0.0431 | Regression_energy: 0.1087 | Regression_eta: 0.0375 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0914 | LR: 8.37e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.4508s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7574 | max_loss: 0.1908 | max_backward: 1.5870 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6393.2 | uss_MB: 4632.1 | child_rss_MB: 5217.4 | allocated_MB: 342.3 | reserved_MB: 26608.0 | inactive_split_MB: 238.4 +rank5 [2026-10-28 21:07:30,351] INFO : Running validation on rank5 +rank5 [2026-10-28 21:07:30,352] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 21:07:30,352] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 21:08:21,199] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:08:21,764] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 21:08:49,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 21:08:50,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 21:08:51,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 21:08:52,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 21:09:02,170] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 21:09:05,934] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:09:06,443] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 21:09:22,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 21:09:23,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 21:09:24,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 21:09:25,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 21:09:35,268] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 21:09:36,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:09:36,854] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 21:09:52,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 21:09:52,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 21:09:53,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 21:09:54,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 21:10:04,210] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 21:14:16,743] INFO : Step 14100/50000 rank5 | Train Loss: 1.5416 | Train Loss Unweighted: 0.8176 | Train Loss Components: Auxiliary: 0.3012 | Cardinality: 0.1751 | Classification: 0.1363 | Classification_binary: 0.2297 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2177 | Regression_eta: 0.0028 | Regression_pt: 0.2283 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0937 | Classification_binary: 0.2410 | Regression_cos_phi: 0.1045 | Regression_energy: 0.0915 | Regression_eta: 0.0620 | Regression_pt: 0.0987 | Regression_sin_phi: 0.1090 | LR: 8.35e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.2018s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7718 | max_loss: 0.1710 | max_backward: 1.6376 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 7412.7 | uss_MB: 6170.5 | child_rss_MB: 5197.6 | allocated_MB: 347.1 | reserved_MB: 23970.0 | inactive_split_MB: 244.9 +rank5 [2026-10-28 21:18:17,875] INFO : Step 14200/50000 rank5 | Train Loss: 1.6164 | Train Loss Unweighted: 0.8542 | Train Loss Components: Auxiliary: 0.3190 | Cardinality: 0.1950 | Classification: 0.1478 | Classification_binary: 0.2794 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2094 | Regression_eta: 0.0027 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1043 | Classification_binary: 0.3012 | Regression_cos_phi: 0.1319 | Regression_energy: 0.1096 | Regression_eta: 0.0916 | Regression_pt: 0.1112 | Regression_sin_phi: 0.1232 | LR: 8.32e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.3693s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1153 | max_forward: 0.8788 | max_loss: 0.1615 | max_backward: 1.5286 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7413.4 | uss_MB: 6634.1 | child_rss_MB: 5195.0 | allocated_MB: 344.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.9 +rank5 [2026-10-28 21:22:21,673] INFO : Step 14300/50000 rank5 | Train Loss: 1.4246 | Train Loss Unweighted: 0.7744 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1685 | Classification: 0.1198 | Classification_binary: 0.2357 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2084 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0899 | Classification_binary: 0.2408 | Regression_cos_phi: 0.1190 | Regression_energy: 0.1034 | Regression_eta: 0.0658 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0976 | LR: 8.30e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.8794s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7183 | max_loss: 0.1494 | max_backward: 1.3371 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6454.4 | uss_MB: 5677.1 | child_rss_MB: 5184.2 | allocated_MB: 341.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.1 +rank5 [2026-10-28 21:26:25,335] INFO : Step 14400/50000 rank5 | Train Loss: 1.3539 | Train Loss Unweighted: 0.7267 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1402 | Classification: 0.1303 | Classification_binary: 0.2015 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1957 | Regression_eta: 0.0022 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0873 | Classification_binary: 0.2000 | Regression_cos_phi: 0.0958 | Regression_energy: 0.1126 | Regression_eta: 0.0629 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0930 | LR: 8.28e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.6670s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6350 | max_loss: 0.2071 | max_backward: 1.2597 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6458.7 | uss_MB: 5681.9 | child_rss_MB: 5188.8 | allocated_MB: 346.1 | reserved_MB: 24962.0 | inactive_split_MB: 231.6 +rank5 [2026-10-28 21:30:23,755] INFO : Step 14500/50000 rank5 | Train Loss: 1.5651 | Train Loss Unweighted: 0.7967 | Train Loss Components: Auxiliary: 0.3070 | Cardinality: 0.2312 | Classification: 0.1364 | Classification_binary: 0.2565 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1984 | Regression_eta: 0.0026 | Regression_pt: 0.2005 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2239 | Regression_cos_phi: 0.0673 | Regression_energy: 0.1057 | Regression_eta: 0.0532 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0764 | LR: 8.26e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.4640s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0069 | max_forward: 0.8745 | max_loss: 0.1717 | max_backward: 1.8485 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 1.4922 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6443.5 | uss_MB: 5682.8 | child_rss_MB: 5113.1 | allocated_MB: 344.2 | reserved_MB: 25374.0 | inactive_split_MB: 206.9 +rank5 [2026-10-28 21:34:34,947] INFO : Step 14600/50000 rank5 | Train Loss: 1.6485 | Train Loss Unweighted: 0.8413 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.2375 | Classification: 0.1249 | Classification_binary: 0.2902 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2104 | Regression_eta: 0.0023 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2904 | Regression_cos_phi: 0.1053 | Regression_energy: 0.1077 | Regression_eta: 0.0616 | Regression_pt: 0.1117 | Regression_sin_phi: 0.1115 | LR: 8.24e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.3665s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0068 | max_forward: 0.7866 | max_loss: 0.1515 | max_backward: 1.5469 | max_optimizer_step: 0.3128 | max_tensorboard_logging: 0.7882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6427.1 | uss_MB: 5683.8 | child_rss_MB: 5049.1 | allocated_MB: 345.7 | reserved_MB: 25374.0 | inactive_split_MB: 292.3 +rank5 [2026-10-28 21:38:39,704] INFO : Step 14700/50000 rank5 | Train Loss: 1.3645 | Train Loss Unweighted: 0.7326 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1507 | Classification: 0.1080 | Classification_binary: 0.2034 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.2233 | Regression_cos_phi: 0.1015 | Regression_energy: 0.1089 | Regression_eta: 0.1283 | Regression_pt: 0.1106 | Regression_sin_phi: 0.1260 | LR: 8.21e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9104s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6571 | max_loss: 0.1494 | max_backward: 1.2978 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7751 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0922 | Memory: rss_MB: 6423.6 | uss_MB: 5684.9 | child_rss_MB: 5040.9 | allocated_MB: 341.1 | reserved_MB: 25374.0 | inactive_split_MB: 264.9 +rank5 [2026-10-28 21:42:53,741] INFO : Step 14800/50000 rank5 | Train Loss: 1.3987 | Train Loss Unweighted: 0.7335 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1827 | Classification: 0.1136 | Classification_binary: 0.2244 | HitGrouping: 0.2204 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1949 | Regression_eta: 0.0022 | Regression_pt: 0.1963 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.2428 | Regression_cos_phi: 0.1077 | Regression_energy: 0.0977 | Regression_eta: 0.0765 | Regression_pt: 0.1012 | Regression_sin_phi: 0.0952 | LR: 8.19e-04 | DataLoad Time: 0.0228s | Model Forward Time: 5.2133s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.1679 | max_loss: 0.2171 | max_backward: 2.8051 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 6420.4 | uss_MB: 4434.9 | child_rss_MB: 5026.4 | allocated_MB: 345.2 | reserved_MB: 25374.0 | inactive_split_MB: 222.5 +rank5 [2026-10-28 21:47:06,021] INFO : Step 14900/50000 rank5 | Train Loss: 1.5046 | Train Loss Unweighted: 0.7991 | Train Loss Components: Auxiliary: 0.3055 | Cardinality: 0.1834 | Classification: 0.1311 | Classification_binary: 0.2643 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1982 | Regression_eta: 0.0023 | Regression_pt: 0.2008 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0855 | Classification_binary: 0.2963 | Regression_cos_phi: 0.1143 | Regression_energy: 0.0945 | Regression_eta: 0.0674 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0949 | LR: 8.17e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.0349s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0071 | max_forward: 0.7642 | max_loss: 0.1561 | max_backward: 1.5975 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6421.6 | uss_MB: 5685.9 | child_rss_MB: 5030.2 | allocated_MB: 345.4 | reserved_MB: 25374.0 | inactive_split_MB: 289.7 +rank5 [2026-10-28 21:51:16,625] INFO : Step 15000/50000 rank5 | Train Loss: 1.3836 | Train Loss Unweighted: 0.7341 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1614 | Classification: 0.1182 | Classification_binary: 0.2252 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1930 | Regression_eta: 0.0023 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2332 | Regression_cos_phi: 0.0877 | Regression_energy: 0.0997 | Regression_eta: 0.0690 | Regression_pt: 0.1017 | Regression_sin_phi: 0.1181 | LR: 8.15e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.7144s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0066 | max_forward: 0.6828 | max_loss: 0.1453 | max_backward: 1.3064 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1064 | Memory: rss_MB: 6423.8 | uss_MB: 5686.3 | child_rss_MB: 5021.1 | allocated_MB: 342.6 | reserved_MB: 25374.0 | inactive_split_MB: 262.7 +rank5 [2026-10-28 21:51:17,569] INFO : Running validation on rank5 +rank5 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 21:52:06,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:52:07,025] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 21:52:33,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 21:52:34,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 21:52:35,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 21:52:36,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 21:52:46,332] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 21:52:46,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:52:47,272] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 21:53:03,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 21:53:04,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 21:53:05,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 21:53:06,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 21:53:16,155] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 21:53:16,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:53:17,344] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 21:53:32,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 21:53:33,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 21:53:33,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 21:53:34,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 21:53:44,560] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 21:57:58,525] INFO : Step 15100/50000 rank5 | Train Loss: 1.3706 | Train Loss Unweighted: 0.7159 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1535 | Classification: 0.1271 | Classification_binary: 0.1954 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1940 | Regression_eta: 0.0022 | Regression_pt: 0.1948 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2207 | Regression_cos_phi: 0.1259 | Regression_energy: 0.0983 | Regression_eta: 0.0635 | Regression_pt: 0.1042 | Regression_sin_phi: 0.0901 | LR: 8.12e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.0253s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.6658 | max_loss: 0.2401 | max_backward: 1.3105 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8277 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7654.5 | uss_MB: 6910.5 | child_rss_MB: 5047.2 | allocated_MB: 341.1 | reserved_MB: 22096.0 | inactive_split_MB: 215.5 +rank5 [2026-10-28 22:01:59,131] INFO : Step 15200/50000 rank5 | Train Loss: 1.5024 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3072 | Cardinality: 0.2080 | Classification: 0.1273 | Classification_binary: 0.2385 | HitGrouping: 0.2164 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1989 | Regression_eta: 0.0022 | Regression_pt: 0.2007 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2390 | Regression_cos_phi: 0.0897 | Regression_energy: 0.0939 | Regression_eta: 0.0558 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0693 | LR: 8.10e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5458s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7502 | max_loss: 0.1534 | max_backward: 1.8002 | max_optimizer_step: 0.2270 | max_tensorboard_logging: 0.8682 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 7649.1 | uss_MB: 6910.3 | child_rss_MB: 5045.3 | allocated_MB: 345.6 | reserved_MB: 22420.0 | inactive_split_MB: 257.7 +rank5 [2026-10-28 22:05:42,233] INFO : Step 15300/50000 rank5 | Train Loss: 1.4920 | Train Loss Unweighted: 0.7880 | Train Loss Components: Auxiliary: 0.3065 | Cardinality: 0.1970 | Classification: 0.1161 | Classification_binary: 0.2589 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2026 | Regression_eta: 0.0022 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0835 | Classification_binary: 0.2369 | Regression_cos_phi: 0.1028 | Regression_energy: 0.1183 | Regression_eta: 0.0585 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0904 | LR: 8.08e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5780s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0192 | max_forward: 0.7642 | max_loss: 0.1530 | max_backward: 1.6292 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 0.9125 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6524.0 | uss_MB: 5797.3 | child_rss_MB: 5044.3 | allocated_MB: 341.2 | reserved_MB: 23096.0 | inactive_split_MB: 271.7 +rank5 [2026-10-28 22:09:28,150] INFO : Step 15400/50000 rank5 | Train Loss: 1.3549 | Train Loss Unweighted: 0.7210 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1572 | Classification: 0.1148 | Classification_binary: 0.2052 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1983 | Regression_eta: 0.0022 | Regression_pt: 0.1984 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1886 | Regression_cos_phi: 0.0614 | Regression_energy: 0.1035 | Regression_eta: 0.0534 | Regression_pt: 0.1092 | Regression_sin_phi: 0.0647 | LR: 8.05e-04 | DataLoad Time: 0.0214s | Model Forward Time: 3.7034s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6594 | max_loss: 0.2185 | max_backward: 1.2416 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6511.0 | uss_MB: 5790.6 | child_rss_MB: 5047.2 | allocated_MB: 340.8 | reserved_MB: 23394.0 | inactive_split_MB: 166.6 +rank5 [2026-10-28 22:13:12,490] INFO : Step 15500/50000 rank5 | Train Loss: 1.4313 | Train Loss Unweighted: 0.7527 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1666 | Classification: 0.1321 | Classification_binary: 0.2182 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1979 | Regression_eta: 0.0025 | Regression_pt: 0.1996 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0927 | Classification_binary: 0.1916 | Regression_cos_phi: 0.1037 | Regression_energy: 0.0934 | Regression_eta: 0.0621 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0823 | LR: 8.03e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2552s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0792 | max_forward: 0.7874 | max_loss: 0.1514 | max_backward: 1.8676 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5380 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6511.0 | uss_MB: 5805.3 | child_rss_MB: 5052.2 | allocated_MB: 345.6 | reserved_MB: 24092.0 | inactive_split_MB: 260.5 +rank5 [2026-10-28 22:17:01,250] INFO : Step 15600/50000 rank5 | Train Loss: 1.5305 | Train Loss Unweighted: 0.8109 | Train Loss Components: Auxiliary: 0.3137 | Cardinality: 0.1790 | Classification: 0.1304 | Classification_binary: 0.2721 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2011 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0795 | Classification_binary: 0.3008 | Regression_cos_phi: 0.1132 | Regression_energy: 0.0988 | Regression_eta: 0.0760 | Regression_pt: 0.1044 | Regression_sin_phi: 0.1193 | LR: 8.01e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2116s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7645 | max_loss: 0.2230 | max_backward: 1.7015 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1398 | Memory: rss_MB: 6508.8 | uss_MB: 5803.4 | child_rss_MB: 5052.3 | allocated_MB: 344.1 | reserved_MB: 24402.0 | inactive_split_MB: 287.9 +rank5 [2026-10-28 22:20:46,546] INFO : Step 15700/50000 rank5 | Train Loss: 1.4083 | Train Loss Unweighted: 0.7205 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1775 | Classification: 0.1238 | Classification_binary: 0.2322 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1785 | Regression_eta: 0.0024 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2442 | Regression_cos_phi: 0.1237 | Regression_energy: 0.1010 | Regression_eta: 0.0726 | Regression_pt: 0.1055 | Regression_sin_phi: 0.1143 | LR: 7.98e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.0317s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2392 | max_backward: 1.6606 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6506.8 | uss_MB: 5802.2 | child_rss_MB: 5055.5 | allocated_MB: 344.4 | reserved_MB: 25380.0 | inactive_split_MB: 228.2 +rank5 [2026-10-28 22:24:39,043] INFO : Step 15800/50000 rank5 | Train Loss: 1.2988 | Train Loss Unweighted: 0.6968 | Train Loss Components: Auxiliary: 0.2671 | Cardinality: 0.1436 | Classification: 0.1112 | Classification_binary: 0.1916 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1933 | Regression_eta: 0.0020 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.2122 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0960 | Regression_eta: 0.0555 | Regression_pt: 0.1002 | Regression_sin_phi: 0.0780 | LR: 7.96e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7259s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6109 | max_loss: 0.2015 | max_backward: 1.1291 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6506.5 | uss_MB: 5803.6 | child_rss_MB: 5051.3 | allocated_MB: 341.6 | reserved_MB: 25380.0 | inactive_split_MB: 263.9 +rank5 [2026-10-28 22:28:35,067] INFO : Step 15900/50000 rank5 | Train Loss: 1.4020 | Train Loss Unweighted: 0.7322 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1800 | Classification: 0.1295 | Classification_binary: 0.2346 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1816 | Regression_eta: 0.0023 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.2378 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0912 | Regression_eta: 0.0557 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0894 | LR: 7.94e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2690s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0933 | max_forward: 0.7823 | max_loss: 0.1445 | max_backward: 1.4740 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6508.5 | uss_MB: 5807.0 | child_rss_MB: 5047.2 | allocated_MB: 342.9 | reserved_MB: 25380.0 | inactive_split_MB: 283.5 +rank5 [2026-10-28 22:32:40,685] INFO : Step 16000/50000 rank5 | Train Loss: 1.4978 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3126 | Cardinality: 0.2093 | Classification: 0.1190 | Classification_binary: 0.2580 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1920 | Regression_eta: 0.0021 | Regression_pt: 0.1969 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.2448 | Regression_cos_phi: 0.0686 | Regression_energy: 0.1146 | Regression_eta: 0.0446 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0751 | LR: 7.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4905s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.7474 | max_loss: 0.1892 | max_backward: 1.4114 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.7981 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6509.3 | uss_MB: 5675.5 | child_rss_MB: 5048.3 | allocated_MB: 342.7 | reserved_MB: 25380.0 | inactive_split_MB: 292.9 +rank5 [2026-10-28 22:32:41,713] INFO : Running validation on rank5 +rank5 [2026-10-28 22:32:41,713] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 22:32:41,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 22:33:34,622] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:33:35,150] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 22:34:01,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 22:34:02,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 22:34:03,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 22:34:04,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 22:34:14,104] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 22:34:14,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:34:14,938] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 22:34:30,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 22:34:31,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 22:34:32,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 22:34:33,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 22:34:43,682] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 22:34:44,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:34:44,855] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 22:34:59,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 22:35:00,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 22:35:01,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 22:35:01,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 22:35:11,936] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 22:39:22,135] INFO : Step 16100/50000 rank5 | Train Loss: 1.3441 | Train Loss Unweighted: 0.7192 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1631 | Classification: 0.1197 | Classification_binary: 0.2058 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1931 | Regression_eta: 0.0020 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2250 | Regression_cos_phi: 0.0945 | Regression_energy: 0.0942 | Regression_eta: 0.0649 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0823 | LR: 7.89e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1519s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0584 | max_forward: 0.7257 | max_loss: 0.1421 | max_backward: 1.2590 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.0197 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6577.1 | uss_MB: 5871.7 | child_rss_MB: 5053.6 | allocated_MB: 341.3 | reserved_MB: 23342.0 | inactive_split_MB: 221.6 +rank5 [2026-10-28 22:43:09,260] INFO : Step 16200/50000 rank5 | Train Loss: 1.3737 | Train Loss Unweighted: 0.7229 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1695 | Classification: 0.1213 | Classification_binary: 0.2167 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.2128 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0911 | Regression_eta: 0.0580 | Regression_pt: 0.0958 | Regression_sin_phi: 0.0831 | LR: 7.86e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0507s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.1062 | max_forward: 0.7366 | max_loss: 0.1469 | max_backward: 1.4999 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.6015 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6580.7 | uss_MB: 5882.6 | child_rss_MB: 5053.8 | allocated_MB: 342.5 | reserved_MB: 24304.0 | inactive_split_MB: 205.9 +rank5 [2026-10-28 22:46:54,662] INFO : Step 16300/50000 rank5 | Train Loss: 1.4671 | Train Loss Unweighted: 0.7592 | Train Loss Components: Auxiliary: 0.3008 | Cardinality: 0.1775 | Classification: 0.1342 | Classification_binary: 0.2702 | HitGrouping: 0.2148 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0023 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0829 | Classification_binary: 0.2666 | Regression_cos_phi: 0.1090 | Regression_energy: 0.0806 | Regression_eta: 0.0875 | Regression_pt: 0.0829 | Regression_sin_phi: 0.1243 | LR: 7.84e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.4605s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7718 | max_loss: 0.1628 | max_backward: 1.7017 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.8513 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6574.7 | uss_MB: 5880.8 | child_rss_MB: 5053.1 | allocated_MB: 345.8 | reserved_MB: 24304.0 | inactive_split_MB: 281.0 +rank5 [2026-10-28 22:50:39,547] INFO : Step 16400/50000 rank5 | Train Loss: 1.4200 | Train Loss Unweighted: 0.7161 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.2086 | Classification: 0.1206 | Classification_binary: 0.2158 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0024 | Regression_pt: 0.1885 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0921 | Classification_binary: 0.2678 | Regression_cos_phi: 0.1459 | Regression_energy: 0.0945 | Regression_eta: 0.1045 | Regression_pt: 0.0974 | Regression_sin_phi: 0.1195 | LR: 7.81e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.8028s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7570 | max_loss: 0.1546 | max_backward: 1.2568 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5199 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1436 | Memory: rss_MB: 6571.7 | uss_MB: 5879.8 | child_rss_MB: 5049.7 | allocated_MB: 342.7 | reserved_MB: 25268.0 | inactive_split_MB: 213.6 +rank5 [2026-10-28 22:54:25,075] INFO : Step 16500/50000 rank5 | Train Loss: 1.2201 | Train Loss Unweighted: 0.6592 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1195 | Classification: 0.1029 | Classification_binary: 0.1835 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1836 | Regression_eta: 0.0021 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1779 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0901 | Regression_eta: 0.0466 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0462 | LR: 7.79e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.3182s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0128 | max_forward: 0.7866 | max_loss: 0.2200 | max_backward: 1.9590 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.4913 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6572.2 | uss_MB: 5885.7 | child_rss_MB: 5046.8 | allocated_MB: 343.6 | reserved_MB: 26238.0 | inactive_split_MB: 248.8 +rank5 [2026-10-28 22:58:25,287] INFO : Step 16600/50000 rank5 | Train Loss: 1.4335 | Train Loss Unweighted: 0.7657 | Train Loss Components: Auxiliary: 0.2949 | Cardinality: 0.1698 | Classification: 0.1386 | Classification_binary: 0.2312 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1941 | Regression_eta: 0.0021 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.2340 | Regression_cos_phi: 0.0776 | Regression_energy: 0.1075 | Regression_eta: 0.0579 | Regression_pt: 0.1114 | Regression_sin_phi: 0.1427 | LR: 7.77e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.9926s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0139 | max_forward: 1.0329 | max_loss: 0.3279 | max_backward: 2.5489 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.5244 | max_tensorboard_flush: 0.0014 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6575.3 | uss_MB: 4925.2 | child_rss_MB: 5045.0 | allocated_MB: 344.2 | reserved_MB: 27228.0 | inactive_split_MB: 219.6 +rank5 [2026-10-28 23:02:38,763] INFO : Step 16700/50000 rank5 | Train Loss: 1.4062 | Train Loss Unweighted: 0.7378 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1840 | Classification: 0.1203 | Classification_binary: 0.2439 | HitGrouping: 0.2108 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1839 | Regression_eta: 0.0020 | Regression_pt: 0.1855 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2320 | Regression_cos_phi: 0.1080 | Regression_energy: 0.0884 | Regression_eta: 0.0477 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0571 | LR: 7.74e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0467s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7336 | max_loss: 0.2460 | max_backward: 1.6747 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6577.4 | uss_MB: 5888.0 | child_rss_MB: 5049.7 | allocated_MB: 344.6 | reserved_MB: 27228.0 | inactive_split_MB: 279.4 +rank5 [2026-10-28 23:06:47,051] INFO : Step 16800/50000 rank5 | Train Loss: 1.3172 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1533 | Classification: 0.1173 | Classification_binary: 0.1949 | HitGrouping: 0.2175 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1819 | Regression_eta: 0.0027 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0829 | Regression_energy: 0.1047 | Regression_eta: 0.0552 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0701 | LR: 7.72e-04 | DataLoad Time: 0.0221s | Model Forward Time: 3.6834s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0962 | max_forward: 0.7772 | max_loss: 0.1423 | max_backward: 1.3121 | max_optimizer_step: 0.2117 | max_tensorboard_logging: 0.4865 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6580.1 | uss_MB: 5886.7 | child_rss_MB: 5043.1 | allocated_MB: 342.1 | reserved_MB: 27250.0 | inactive_split_MB: 200.8 +rank5 [2026-10-28 23:11:05,116] INFO : Step 16900/50000 rank5 | Train Loss: 1.3132 | Train Loss Unweighted: 0.6868 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1474 | Classification: 0.1260 | Classification_binary: 0.1988 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1779 | Regression_eta: 0.0021 | Regression_pt: 0.1798 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0713 | Classification_binary: 0.2093 | Regression_cos_phi: 0.1163 | Regression_energy: 0.0884 | Regression_eta: 0.0820 | Regression_pt: 0.0907 | Regression_sin_phi: 0.1135 | LR: 7.69e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.7611s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7203 | max_loss: 0.1660 | max_backward: 1.1458 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6581.3 | uss_MB: 5889.6 | child_rss_MB: 5044.4 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 193.9 +rank5 [2026-10-28 23:15:09,202] INFO : Step 17000/50000 rank5 | Train Loss: 1.3575 | Train Loss Unweighted: 0.7115 | Train Loss Components: Auxiliary: 0.2906 | Cardinality: 0.1781 | Classification: 0.1179 | Classification_binary: 0.2325 | HitGrouping: 0.2089 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0018 | Regression_pt: 0.1792 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2607 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0808 | Regression_eta: 0.0635 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0748 | LR: 7.67e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4155s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.9008 | max_loss: 0.1474 | max_backward: 1.9229 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6575.6 | uss_MB: 5894.0 | child_rss_MB: 5025.6 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 278.9 +rank5 [2026-10-28 23:15:10,041] INFO : Running validation on rank5 +rank5 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 23:16:00,503] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:16:01,061] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 23:16:27,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 23:16:28,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 23:16:29,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 23:16:31,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 23:16:41,072] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 23:16:42,017] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:16:42,520] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 23:16:58,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 23:16:59,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 23:16:59,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 23:17:00,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 23:17:10,922] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 23:17:11,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:17:12,022] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 23:17:27,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 23:17:28,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 23:17:28,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 23:17:29,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 23:17:39,543] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-28 23:21:59,506] INFO : Step 17100/50000 rank5 | Train Loss: 1.3790 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1704 | Classification: 0.1078 | Classification_binary: 0.2095 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1968 | Regression_eta: 0.0020 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.2286 | Regression_cos_phi: 0.1061 | Regression_energy: 0.0848 | Regression_eta: 0.0498 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0856 | LR: 7.64e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.9030s | Phase Times: max_pre_step_barrier: 0.0234 | max_batch_to_device: 0.0065 | max_forward: 0.8292 | max_loss: 0.1455 | max_backward: 1.4833 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6666.8 | uss_MB: 5307.9 | child_rss_MB: 5009.6 | allocated_MB: 343.8 | reserved_MB: 22688.0 | inactive_split_MB: 254.6 +rank5 [2026-10-28 23:26:06,862] INFO : Step 17200/50000 rank5 | Train Loss: 1.2786 | Train Loss Unweighted: 0.6598 | Train Loss Components: Auxiliary: 0.2576 | Cardinality: 0.1378 | Classification: 0.1161 | Classification_binary: 0.1910 | HitGrouping: 0.2191 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0025 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2118 | Regression_cos_phi: 0.1068 | Regression_energy: 0.0814 | Regression_eta: 0.0576 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0693 | LR: 7.62e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8625s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6470 | max_loss: 0.2505 | max_backward: 1.4380 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6320 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6664.1 | uss_MB: 5982.7 | child_rss_MB: 5028.0 | allocated_MB: 344.1 | reserved_MB: 22688.0 | inactive_split_MB: 295.9 +rank5 [2026-10-28 23:30:05,514] INFO : Step 17300/50000 rank5 | Train Loss: 1.3571 | Train Loss Unweighted: 0.6976 | Train Loss Components: Auxiliary: 0.2789 | Cardinality: 0.1665 | Classification: 0.1193 | Classification_binary: 0.2287 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1724 | Regression_eta: 0.0021 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2217 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0910 | Regression_eta: 0.0821 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0877 | LR: 7.59e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5413s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7276 | max_loss: 0.2520 | max_backward: 1.8579 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8525 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6666.1 | uss_MB: 5121.5 | child_rss_MB: 5026.9 | allocated_MB: 344.2 | reserved_MB: 23616.0 | inactive_split_MB: 266.5 +rank5 [2026-10-28 23:34:25,421] INFO : Step 17400/50000 rank5 | Train Loss: 1.3668 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2811 | Cardinality: 0.1760 | Classification: 0.1195 | Classification_binary: 0.2351 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1732 | Regression_eta: 0.0019 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.2313 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0739 | Regression_eta: 0.0482 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0729 | LR: 7.57e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.4901s | Phase Times: max_pre_step_barrier: 0.1038 | max_batch_to_device: 0.0306 | max_forward: 0.7782 | max_loss: 0.2429 | max_backward: 1.3743 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.8348 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0757 | Memory: rss_MB: 6667.1 | uss_MB: 5984.6 | child_rss_MB: 5026.4 | allocated_MB: 343.3 | reserved_MB: 24604.0 | inactive_split_MB: 262.4 +rank5 [2026-10-28 23:38:51,400] INFO : Step 17500/50000 rank5 | Train Loss: 1.3114 | Train Loss Unweighted: 0.6526 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.1481 | Classification: 0.1145 | Classification_binary: 0.1884 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1714 | Regression_eta: 0.0024 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1948 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0799 | Regression_eta: 0.0512 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0779 | LR: 7.54e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9139s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.6394 | max_loss: 0.1427 | max_backward: 1.2079 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.8375 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 6667.1 | uss_MB: 5986.2 | child_rss_MB: 5023.0 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 240.0 +rank5 [2026-10-28 23:43:03,145] INFO : Step 17600/50000 rank5 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2688 | Cardinality: 0.1362 | Classification: 0.1273 | Classification_binary: 0.2065 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0019 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.1846 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0835 | Regression_eta: 0.0519 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0805 | LR: 7.52e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.6077s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.1120 | max_forward: 1.2883 | max_loss: 0.2624 | max_backward: 1.5380 | max_optimizer_step: 0.2147 | max_tensorboard_logging: 0.7644 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 6665.9 | uss_MB: 5985.5 | child_rss_MB: 5028.5 | allocated_MB: 344.2 | reserved_MB: 25596.0 | inactive_split_MB: 237.9 +rank5 [2026-10-28 23:47:02,148] INFO : Step 17700/50000 rank5 | Train Loss: 1.3491 | Train Loss Unweighted: 0.6939 | Train Loss Components: Auxiliary: 0.2835 | Cardinality: 0.1622 | Classification: 0.1186 | Classification_binary: 0.2310 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0022 | Regression_pt: 0.1701 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.2128 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0837 | Regression_eta: 0.0469 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0678 | LR: 7.49e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.6520s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0704 | max_forward: 0.9841 | max_loss: 0.1994 | max_backward: 1.8911 | max_optimizer_step: 0.2227 | max_tensorboard_logging: 0.5849 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6667.9 | uss_MB: 5989.0 | child_rss_MB: 5031.2 | allocated_MB: 345.5 | reserved_MB: 25596.0 | inactive_split_MB: 287.6 +rank5 [2026-10-28 23:51:17,533] INFO : Step 17800/50000 rank5 | Train Loss: 1.3063 | Train Loss Unweighted: 0.6692 | Train Loss Components: Auxiliary: 0.2664 | Cardinality: 0.1579 | Classification: 0.1042 | Classification_binary: 0.2130 | HitGrouping: 0.2150 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1741 | Regression_eta: 0.0019 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2161 | Regression_cos_phi: 0.0982 | Regression_energy: 0.0796 | Regression_eta: 0.0466 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0590 | LR: 7.47e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8686s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.7351 | max_loss: 0.1686 | max_backward: 1.4616 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5023 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 6666.0 | uss_MB: 5988.8 | child_rss_MB: 5032.9 | allocated_MB: 341.0 | reserved_MB: 25596.0 | inactive_split_MB: 197.8 +rank5 [2026-10-28 23:55:22,791] INFO : Step 17900/50000 rank5 | Train Loss: 1.2213 | Train Loss Unweighted: 0.6433 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1427 | Classification: 0.1147 | Classification_binary: 0.1725 | HitGrouping: 0.2070 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0873 | Regression_energy: 0.0775 | Regression_eta: 0.0429 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0530 | LR: 7.44e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8178s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7249 | max_loss: 0.1579 | max_backward: 1.1981 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6654.3 | uss_MB: 5990.9 | child_rss_MB: 5033.3 | allocated_MB: 341.6 | reserved_MB: 25596.0 | inactive_split_MB: 297.9 +rank5 [2026-10-28 23:59:20,281] INFO : Step 18000/50000 rank5 | Train Loss: 1.3597 | Train Loss Unweighted: 0.7083 | Train Loss Components: Auxiliary: 0.2781 | Cardinality: 0.1599 | Classification: 0.1247 | Classification_binary: 0.2275 | HitGrouping: 0.2122 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0021 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2039 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0863 | Regression_eta: 0.0451 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0714 | LR: 7.42e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6429s | Phase Times: max_pre_step_barrier: 0.0816 | max_batch_to_device: 0.0068 | max_forward: 0.8980 | max_loss: 0.1586 | max_backward: 1.9185 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.7546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6654.3 | uss_MB: 5995.0 | child_rss_MB: 5035.3 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 254.8 +rank5 [2026-10-28 23:59:21,239] INFO : Running validation on rank5 +rank5 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 00:00:12,210] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:00:12,738] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 00:00:39,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 00:00:40,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 00:00:41,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 00:00:42,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 00:00:52,384] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 00:00:52,803] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:00:53,360] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 00:01:09,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 00:01:10,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 00:01:11,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 00:01:12,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 00:01:22,296] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 00:01:22,746] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:01:23,292] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 00:01:38,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 00:01:39,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 00:01:40,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 00:01:40,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 00:01:50,912] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 00:05:57,252] INFO : Step 18100/50000 rank5 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7216 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1666 | Classification: 0.1199 | Classification_binary: 0.2340 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1807 | Regression_eta: 0.0024 | Regression_pt: 0.1824 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.2321 | Regression_cos_phi: 0.0965 | Regression_energy: 0.0819 | Regression_eta: 0.0720 | Regression_pt: 0.0865 | Regression_sin_phi: 0.1202 | LR: 7.39e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.5286s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.9127 | max_loss: 0.4546 | max_backward: 1.7406 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.7463 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7789.9 | uss_MB: 7130.3 | child_rss_MB: 5031.5 | allocated_MB: 344.6 | reserved_MB: 22256.0 | inactive_split_MB: 291.4 +rank5 [2026-10-29 00:09:47,528] INFO : Step 18200/50000 rank5 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6468 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1359 | Classification: 0.1208 | Classification_binary: 0.1803 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1700 | Regression_eta: 0.0020 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.1847 | Regression_cos_phi: 0.1126 | Regression_energy: 0.0725 | Regression_eta: 0.0699 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0996 | LR: 7.36e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7599s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7114 | max_loss: 0.2369 | max_backward: 1.1051 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6784.1 | uss_MB: 6119.0 | child_rss_MB: 5031.9 | allocated_MB: 341.2 | reserved_MB: 23772.0 | inactive_split_MB: 170.7 +rank5 [2026-10-29 00:13:32,739] INFO : Step 18300/50000 rank5 | Train Loss: 1.3100 | Train Loss Unweighted: 0.6891 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1522 | Classification: 0.1230 | Classification_binary: 0.1938 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1834 | Regression_eta: 0.0019 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.1783 | Regression_cos_phi: 0.0911 | Regression_energy: 0.0861 | Regression_eta: 0.0615 | Regression_pt: 0.0892 | Regression_sin_phi: 0.1029 | LR: 7.34e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1464s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7219 | max_loss: 0.1491 | max_backward: 1.5825 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.7390 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 6786.1 | uss_MB: 6124.1 | child_rss_MB: 5031.6 | allocated_MB: 344.4 | reserved_MB: 24778.0 | inactive_split_MB: 234.4 +rank5 [2026-10-29 00:17:16,846] INFO : Step 18400/50000 rank5 | Train Loss: 1.3667 | Train Loss Unweighted: 0.7337 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1448 | Classification: 0.1277 | Classification_binary: 0.2393 | HitGrouping: 0.2095 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1805 | Regression_eta: 0.0022 | Regression_pt: 0.1818 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0878 | Classification_binary: 0.2321 | Regression_cos_phi: 0.1110 | Regression_energy: 0.0965 | Regression_eta: 0.0599 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0947 | LR: 7.31e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4870s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1744 | max_backward: 1.7308 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6782.0 | uss_MB: 6123.4 | child_rss_MB: 5027.3 | allocated_MB: 344.6 | reserved_MB: 24778.0 | inactive_split_MB: 281.4 +rank5 [2026-10-29 00:21:03,776] INFO : Step 18500/50000 rank5 | Train Loss: 1.2528 | Train Loss Unweighted: 0.6816 | Train Loss Components: Auxiliary: 0.2631 | Cardinality: 0.1350 | Classification: 0.1140 | Classification_binary: 0.1973 | HitGrouping: 0.2066 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1821 | Regression_eta: 0.0017 | Regression_pt: 0.1847 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.1980 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0747 | Regression_eta: 0.0409 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0639 | LR: 7.29e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2644s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0113 | max_forward: 0.8281 | max_loss: 0.2418 | max_backward: 1.4683 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7561 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1431 | Memory: rss_MB: 6783.0 | uss_MB: 6122.8 | child_rss_MB: 5026.7 | allocated_MB: 343.3 | reserved_MB: 24778.0 | inactive_split_MB: 226.5 +rank5 [2026-10-29 00:24:44,690] INFO : Step 18600/50000 rank5 | Train Loss: 1.1881 | Train Loss Unweighted: 0.6335 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1331 | Classification: 0.1084 | Classification_binary: 0.1723 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1746 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0857 | Regression_eta: 0.0351 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0702 | LR: 7.26e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8517s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7299 | max_loss: 0.2279 | max_backward: 1.0799 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8080 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6782.4 | uss_MB: 6120.8 | child_rss_MB: 5022.4 | allocated_MB: 341.0 | reserved_MB: 24778.0 | inactive_split_MB: 229.3 +rank5 [2026-10-29 00:28:24,928] INFO : Step 18700/50000 rank5 | Train Loss: 1.3122 | Train Loss Unweighted: 0.6534 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1756 | Classification: 0.1225 | Classification_binary: 0.2098 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1571 | Regression_eta: 0.0020 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0682 | Regression_eta: 0.0552 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0822 | LR: 7.24e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.6247s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9129 | max_loss: 0.1606 | max_backward: 1.9397 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6785.3 | uss_MB: 6128.9 | child_rss_MB: 5021.8 | allocated_MB: 344.8 | reserved_MB: 24778.0 | inactive_split_MB: 239.8 +rank5 [2026-10-29 00:32:05,525] INFO : Step 18800/50000 rank5 | Train Loss: 1.2696 | Train Loss Unweighted: 0.6643 | Train Loss Components: Auxiliary: 0.2674 | Cardinality: 0.1602 | Classification: 0.1070 | Classification_binary: 0.2200 | HitGrouping: 0.2058 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1652 | Regression_eta: 0.0018 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2286 | Regression_cos_phi: 0.1263 | Regression_energy: 0.0862 | Regression_eta: 0.0632 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0959 | LR: 7.21e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6469s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7444 | max_loss: 0.2527 | max_backward: 1.7167 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.9324 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 6785.1 | uss_MB: 6124.1 | child_rss_MB: 5021.8 | allocated_MB: 345.4 | reserved_MB: 25044.0 | inactive_split_MB: 284.4 +rank5 [2026-10-29 00:35:49,544] INFO : Step 18900/50000 rank5 | Train Loss: 1.1718 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1382 | Classification: 0.0996 | Classification_binary: 0.1741 | HitGrouping: 0.2036 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1652 | Regression_eta: 0.0017 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1938 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0858 | Regression_eta: 0.0420 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0924 | LR: 7.18e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.0753s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0056 | max_forward: 0.6830 | max_loss: 0.1421 | max_backward: 1.2812 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.8931 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6786.1 | uss_MB: 6126.4 | child_rss_MB: 5006.2 | allocated_MB: 338.7 | reserved_MB: 25044.0 | inactive_split_MB: 232.6 +rank5 [2026-10-29 00:39:31,233] INFO : Step 19000/50000 rank5 | Train Loss: 1.2397 | Train Loss Unweighted: 0.6317 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1277 | Classification: 0.1188 | Classification_binary: 0.1866 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1600 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.1907 | Regression_cos_phi: 0.0858 | Regression_energy: 0.0729 | Regression_eta: 0.0620 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0811 | LR: 7.16e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.2161s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6593 | max_loss: 0.2467 | max_backward: 1.6203 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7732 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6787.7 | uss_MB: 6128.3 | child_rss_MB: 5026.8 | allocated_MB: 342.2 | reserved_MB: 25790.0 | inactive_split_MB: 229.6 +rank5 [2026-10-29 00:39:32,174] INFO : Running validation on rank5 +rank5 [2026-10-29 00:39:32,174] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 00:39:32,175] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 00:40:23,807] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:40:24,368] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 00:40:50,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 00:40:51,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 00:40:52,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 00:40:52,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 00:41:03,009] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 00:41:03,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:41:04,089] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 00:41:20,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 00:41:20,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 00:41:21,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 00:41:22,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 00:41:32,892] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 00:41:33,292] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:41:33,790] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 00:41:49,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 00:41:50,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 00:41:50,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 00:41:51,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 00:42:01,545] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 00:46:01,994] INFO : Step 19100/50000 rank5 | Train Loss: 1.3562 | Train Loss Unweighted: 0.6933 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1644 | Classification: 0.1204 | Classification_binary: 0.2426 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1627 | Regression_eta: 0.0023 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2352 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0727 | Regression_eta: 0.0536 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0939 | LR: 7.13e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.5891s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7796 | max_loss: 0.1634 | max_backward: 1.6942 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.8232 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 8027.6 | uss_MB: 7365.9 | child_rss_MB: 5027.2 | allocated_MB: 345.2 | reserved_MB: 23900.0 | inactive_split_MB: 240.8 +rank5 [2026-10-29 00:49:48,722] INFO : Step 19200/50000 rank5 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5946 | Train Loss Components: Auxiliary: 0.2384 | Cardinality: 0.1391 | Classification: 0.0965 | Classification_binary: 0.1984 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1472 | Regression_eta: 0.0018 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2160 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0721 | Regression_eta: 0.0467 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0540 | LR: 7.10e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9573s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7138 | max_loss: 0.2391 | max_backward: 1.2379 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.7395 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6876.1 | uss_MB: 6216.1 | child_rss_MB: 5025.0 | allocated_MB: 341.3 | reserved_MB: 23900.0 | inactive_split_MB: 206.8 +rank5 [2026-10-29 00:53:30,059] INFO : Step 19300/50000 rank5 | Train Loss: 1.1768 | Train Loss Unweighted: 0.6083 | Train Loss Components: Auxiliary: 0.2422 | Cardinality: 0.1345 | Classification: 0.1035 | Classification_binary: 0.1709 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0018 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0731 | Regression_eta: 0.0384 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0568 | LR: 7.08e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.8973s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0503 | max_forward: 0.7163 | max_loss: 0.1442 | max_backward: 1.3745 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6878.1 | uss_MB: 6217.4 | child_rss_MB: 5020.2 | allocated_MB: 341.0 | reserved_MB: 23900.0 | inactive_split_MB: 218.1 +rank5 [2026-10-29 00:57:11,120] INFO : Step 19400/50000 rank5 | Train Loss: 1.4590 | Train Loss Unweighted: 0.7109 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.2328 | Classification: 0.1237 | Classification_binary: 0.2265 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1770 | Regression_eta: 0.0021 | Regression_pt: 0.1794 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2312 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0833 | Regression_eta: 0.0583 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0667 | LR: 7.05e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.4237s | Phase Times: max_pre_step_barrier: 0.1118 | max_batch_to_device: 0.0070 | max_forward: 0.7578 | max_loss: 0.2459 | max_backward: 1.7975 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6876.3 | uss_MB: 6217.1 | child_rss_MB: 5016.1 | allocated_MB: 347.7 | reserved_MB: 24186.0 | inactive_split_MB: 276.3 +rank5 [2026-10-29 01:00:48,549] INFO : Step 19500/50000 rank5 | Train Loss: 1.3753 | Train Loss Unweighted: 0.6794 | Train Loss Components: Auxiliary: 0.2810 | Cardinality: 0.1865 | Classification: 0.1153 | Classification_binary: 0.2310 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1637 | Regression_eta: 0.0023 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2229 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0779 | Regression_eta: 0.0551 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0587 | LR: 7.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8972s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7197 | max_loss: 0.2164 | max_backward: 1.3937 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.4987 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6876.3 | uss_MB: 6219.5 | child_rss_MB: 5017.7 | allocated_MB: 345.2 | reserved_MB: 24186.0 | inactive_split_MB: 275.2 +rank5 [2026-10-29 01:04:42,199] INFO : Step 19600/50000 rank5 | Train Loss: 1.2086 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1557 | Classification: 0.0942 | Classification_binary: 0.1768 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1735 | Regression_eta: 0.0015 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1828 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0923 | Regression_eta: 0.0387 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0411 | LR: 7.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7343s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6937 | max_loss: 0.1942 | max_backward: 1.2727 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6877.2 | uss_MB: 6219.0 | child_rss_MB: 5004.9 | allocated_MB: 340.6 | reserved_MB: 24186.0 | inactive_split_MB: 277.9 +rank5 [2026-10-29 01:08:29,169] INFO : Step 19700/50000 rank5 | Train Loss: 1.2115 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1478 | Classification: 0.1094 | Classification_binary: 0.1919 | HitGrouping: 0.2027 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.1916 | Regression_cos_phi: 0.0990 | Regression_energy: 0.0672 | Regression_eta: 0.0540 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0951 | LR: 6.97e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8685s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0914 | max_forward: 0.7091 | max_loss: 0.1429 | max_backward: 1.4681 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0652 | Memory: rss_MB: 6879.2 | uss_MB: 6222.3 | child_rss_MB: 5016.1 | allocated_MB: 344.0 | reserved_MB: 24630.0 | inactive_split_MB: 143.8 +rank5 [2026-10-29 01:12:36,831] INFO : Step 19800/50000 rank5 | Train Loss: 1.2919 | Train Loss Unweighted: 0.6584 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1453 | Classification: 0.1132 | Classification_binary: 0.2375 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0020 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.2338 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0734 | Regression_eta: 0.0568 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0782 | LR: 6.94e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.4525s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0074 | max_forward: 0.7828 | max_loss: 0.2508 | max_backward: 1.8604 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6879.1 | uss_MB: 6223.6 | child_rss_MB: 5015.7 | allocated_MB: 348.0 | reserved_MB: 25634.0 | inactive_split_MB: 277.9 +rank5 [2026-10-29 01:16:27,497] INFO : Step 19900/50000 rank5 | Train Loss: 1.2010 | Train Loss Unweighted: 0.6167 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1346 | Classification: 0.0885 | Classification_binary: 0.1961 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1633 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.2231 | Regression_cos_phi: 0.0993 | Regression_energy: 0.0765 | Regression_eta: 0.0659 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0892 | LR: 6.92e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9734s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6671 | max_loss: 0.2548 | max_backward: 1.5862 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6881.1 | uss_MB: 6224.6 | child_rss_MB: 5016.9 | allocated_MB: 340.3 | reserved_MB: 25634.0 | inactive_split_MB: 201.7 +rank5 [2026-10-29 01:20:16,836] INFO : Step 20000/50000 rank5 | Train Loss: 1.2077 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1487 | Classification: 0.1030 | Classification_binary: 0.1729 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1639 | Regression_eta: 0.0019 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1935 | Regression_cos_phi: 0.0998 | Regression_energy: 0.0726 | Regression_eta: 0.0491 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0797 | LR: 6.89e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8158s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.6570 | max_loss: 0.1550 | max_backward: 1.4848 | max_optimizer_step: 0.2205 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6884.1 | uss_MB: 6226.8 | child_rss_MB: 5009.1 | allocated_MB: 341.8 | reserved_MB: 26642.0 | inactive_split_MB: 237.7 +rank5 [2026-10-29 01:20:17,835] INFO : Running validation on rank5 +rank5 [2026-10-29 01:20:17,835] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 01:20:17,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 01:21:07,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:21:08,246] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 01:21:35,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 01:21:36,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 01:21:37,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 01:21:37,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 01:21:47,929] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 01:21:48,206] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:21:48,734] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 01:22:04,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 01:22:05,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 01:22:06,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 01:22:07,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 01:22:17,276] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 01:22:18,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:22:18,871] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 01:22:34,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 01:22:34,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 01:22:35,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 01:22:36,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 01:22:46,152] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 01:26:57,040] INFO : Step 20100/50000 rank5 | Train Loss: 1.3825 | Train Loss Unweighted: 0.6739 | Train Loss Components: Auxiliary: 0.2830 | Cardinality: 0.1854 | Classification: 0.1189 | Classification_binary: 0.2349 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1561 | Regression_eta: 0.0022 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2403 | Regression_cos_phi: 0.1021 | Regression_energy: 0.0786 | Regression_eta: 0.0602 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0988 | LR: 6.86e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2463s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.9496 | max_loss: 0.1965 | max_backward: 1.6425 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.5117 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1294 | Memory: rss_MB: 8120.6 | uss_MB: 7458.3 | child_rss_MB: 5010.7 | allocated_MB: 344.8 | reserved_MB: 21328.0 | inactive_split_MB: 198.7 +rank5 [2026-10-29 01:30:57,229] INFO : Step 20200/50000 rank5 | Train Loss: 1.3220 | Train Loss Unweighted: 0.6624 | Train Loss Components: Auxiliary: 0.2728 | Cardinality: 0.1545 | Classification: 0.1109 | Classification_binary: 0.2232 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1618 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2354 | Regression_cos_phi: 0.0883 | Regression_energy: 0.0866 | Regression_eta: 0.0637 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0939 | LR: 6.84e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.8629s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.9337 | max_loss: 0.2287 | max_backward: 1.4619 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6969.7 | uss_MB: 6305.7 | child_rss_MB: 5014.0 | allocated_MB: 344.4 | reserved_MB: 21328.0 | inactive_split_MB: 287.4 +rank5 [2026-10-29 01:34:46,196] INFO : Step 20300/50000 rank5 | Train Loss: 1.2742 | Train Loss Unweighted: 0.6629 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1615 | Classification: 0.1132 | Classification_binary: 0.1937 | HitGrouping: 0.2129 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1761 | Regression_eta: 0.0018 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.1938 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0844 | Regression_eta: 0.0519 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0656 | LR: 6.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8207s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6857 | max_loss: 0.2423 | max_backward: 1.2822 | max_optimizer_step: 0.2837 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0419 | Memory: rss_MB: 6967.2 | uss_MB: 6303.7 | child_rss_MB: 5019.2 | allocated_MB: 340.8 | reserved_MB: 21826.0 | inactive_split_MB: 239.4 +rank5 [2026-10-29 01:38:43,604] INFO : Step 20400/50000 rank5 | Train Loss: 1.2743 | Train Loss Unweighted: 0.6521 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1507 | Classification: 0.1279 | Classification_binary: 0.1990 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1604 | Regression_eta: 0.0019 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1865 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0960 | Regression_eta: 0.0486 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0906 | LR: 6.78e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.7414s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0283 | max_forward: 1.3648 | max_loss: 0.1816 | max_backward: 1.7875 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 6969.3 | uss_MB: 6306.4 | child_rss_MB: 5010.2 | allocated_MB: 343.0 | reserved_MB: 22154.0 | inactive_split_MB: 243.5 +rank5 [2026-10-29 01:42:48,353] INFO : Step 20500/50000 rank5 | Train Loss: 1.2712 | Train Loss Unweighted: 0.6509 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1409 | Classification: 0.1147 | Classification_binary: 0.2202 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1558 | Regression_eta: 0.0019 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2021 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0739 | Regression_eta: 0.0448 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0708 | LR: 6.76e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2556s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7205 | max_loss: 0.2283 | max_backward: 1.6623 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.6254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0704 | Memory: rss_MB: 6969.0 | uss_MB: 6306.9 | child_rss_MB: 5000.2 | allocated_MB: 345.0 | reserved_MB: 22154.0 | inactive_split_MB: 208.2 +rank5 [2026-10-29 01:46:45,622] INFO : Step 20600/50000 rank5 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6338 | Train Loss Components: Auxiliary: 0.2625 | Cardinality: 0.1623 | Classification: 0.1101 | Classification_binary: 0.1900 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0018 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2049 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0740 | Regression_eta: 0.0566 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0531 | LR: 6.73e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.9346s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 1.1520 | max_loss: 0.2849 | max_backward: 2.1471 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.6141 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6963.6 | uss_MB: 6301.3 | child_rss_MB: 5002.5 | allocated_MB: 341.4 | reserved_MB: 22154.0 | inactive_split_MB: 183.5 +rank5 [2026-10-29 01:50:54,179] INFO : Step 20700/50000 rank5 | Train Loss: 1.1613 | Train Loss Unweighted: 0.6066 | Train Loss Components: Auxiliary: 0.2395 | Cardinality: 0.1327 | Classification: 0.1061 | Classification_binary: 0.1677 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0016 | Regression_pt: 0.1666 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1854 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0883 | Regression_eta: 0.0456 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0718 | LR: 6.70e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5607s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6276 | max_loss: 0.2330 | max_backward: 1.4954 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6965.5 | uss_MB: 5936.8 | child_rss_MB: 5004.9 | allocated_MB: 341.0 | reserved_MB: 22154.0 | inactive_split_MB: 285.0 +rank5 [2026-10-29 01:55:01,768] INFO : Step 20800/50000 rank5 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1493 | Classification: 0.1205 | Classification_binary: 0.2020 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1859 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0751 | Regression_eta: 0.0534 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0834 | LR: 6.67e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.2092s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.7679 | max_loss: 0.1604 | max_backward: 1.7694 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.4914 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6972.0 | uss_MB: 6311.7 | child_rss_MB: 5027.1 | allocated_MB: 344.1 | reserved_MB: 22154.0 | inactive_split_MB: 258.7 +rank5 [2026-10-29 01:59:06,393] INFO : Step 20900/50000 rank5 | Train Loss: 1.2934 | Train Loss Unweighted: 0.6525 | Train Loss Components: Auxiliary: 0.2719 | Cardinality: 0.1620 | Classification: 0.1167 | Classification_binary: 0.2121 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1591 | Regression_eta: 0.0021 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1938 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0756 | Regression_eta: 0.0513 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0596 | LR: 6.65e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1333s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7787 | max_loss: 0.1747 | max_backward: 1.6678 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 6974.0 | uss_MB: 5772.6 | child_rss_MB: 5037.2 | allocated_MB: 343.6 | reserved_MB: 23208.0 | inactive_split_MB: 296.4 +rank5 [2026-10-29 02:03:08,876] INFO : Step 21000/50000 rank5 | Train Loss: 1.1402 | Train Loss Unweighted: 0.5874 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1236 | Classification: 0.0967 | Classification_binary: 0.1701 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1583 | Regression_eta: 0.0017 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0816 | Regression_eta: 0.0719 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0726 | LR: 6.62e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.8971s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6629 | max_loss: 0.1833 | max_backward: 1.4689 | max_optimizer_step: 0.2206 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6973.2 | uss_MB: 6313.8 | child_rss_MB: 5014.5 | allocated_MB: 340.4 | reserved_MB: 23208.0 | inactive_split_MB: 264.1 +rank5 [2026-10-29 02:03:09,907] INFO : Running validation on rank5 +rank5 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 02:04:00,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:04:01,076] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 02:04:27,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 02:04:28,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 02:04:29,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 02:04:30,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 02:04:40,159] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 02:04:40,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:04:41,351] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 02:04:57,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 02:04:58,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 02:04:59,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 02:05:00,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 02:05:10,508] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 02:05:10,624] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:05:11,216] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 02:05:27,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 02:05:27,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 02:05:28,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 02:05:32,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 02:05:42,504] INFO : Time taken to make predictions on device 5 is: 0.28 min +rank5 [2026-10-29 02:10:06,178] INFO : Step 21100/50000 rank5 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6632 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1574 | Classification: 0.1277 | Classification_binary: 0.2026 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0024 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0850 | Classification_binary: 0.2253 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0788 | Regression_eta: 0.0706 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0992 | LR: 6.59e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2632s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7159 | max_loss: 0.1411 | max_backward: 1.5805 | max_optimizer_step: 0.3396 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8247.8 | uss_MB: 7584.2 | child_rss_MB: 5046.5 | allocated_MB: 343.8 | reserved_MB: 21372.0 | inactive_split_MB: 255.4 +rank5 [2026-10-29 02:10:38,255] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank5 [2026-10-29 02:10:38,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-29 02:14:15,482] INFO : Step 21200/50000 rank5 | Train Loss: 1.2354 | Train Loss Unweighted: 0.6366 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1422 | Classification: 0.1056 | Classification_binary: 0.1876 | HitGrouping: 0.2030 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0017 | Regression_pt: 0.1709 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.2020 | Regression_cos_phi: 0.1039 | Regression_energy: 0.0815 | Regression_eta: 0.0586 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0818 | LR: 6.56e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.1581s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7385 | max_loss: 0.2183 | max_backward: 1.4276 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.7483 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7137.9 | uss_MB: 6469.9 | child_rss_MB: 5019.5 | allocated_MB: 342.5 | reserved_MB: 23286.0 | inactive_split_MB: 273.0 +rank5 [2026-10-29 02:18:17,990] INFO : Step 21300/50000 rank5 | Train Loss: 1.1511 | Train Loss Unweighted: 0.5782 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1240 | Classification: 0.1021 | Classification_binary: 0.1706 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1505 | Regression_eta: 0.0017 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0878 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0694 | Regression_eta: 0.0486 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0852 | LR: 6.54e-04 | DataLoad Time: 0.0250s | Model Forward Time: 4.4882s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0290 | max_forward: 1.3238 | max_loss: 0.2105 | max_backward: 1.6672 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.6878 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7140.4 | uss_MB: 6475.6 | child_rss_MB: 5012.2 | allocated_MB: 343.8 | reserved_MB: 23286.0 | inactive_split_MB: 272.8 +rank5 [2026-10-29 02:22:14,589] INFO : Step 21400/50000 rank5 | Train Loss: 1.2993 | Train Loss Unweighted: 0.6458 | Train Loss Components: Auxiliary: 0.2724 | Cardinality: 0.1796 | Classification: 0.1204 | Classification_binary: 0.2158 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1518 | Regression_eta: 0.0018 | Regression_pt: 0.1542 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1003 | Classification_binary: 0.2121 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0770 | Regression_eta: 0.0660 | Regression_pt: 0.0825 | Regression_sin_phi: 0.1040 | LR: 6.51e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5447s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8696 | max_loss: 0.1572 | max_backward: 1.8002 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 0.6992 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7114.7 | uss_MB: 6081.0 | child_rss_MB: 5015.7 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 201.9 +rank5 [2026-10-29 02:26:14,444] INFO : Step 21500/50000 rank5 | Train Loss: 1.2839 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2693 | Cardinality: 0.1478 | Classification: 0.1058 | Classification_binary: 0.2162 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1843 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0841 | Regression_eta: 0.0343 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0466 | LR: 6.48e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.7752s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.8684 | max_loss: 0.1577 | max_backward: 1.9355 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.7593 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0648 | Memory: rss_MB: 7121.8 | uss_MB: 6458.0 | child_rss_MB: 5014.4 | allocated_MB: 341.9 | reserved_MB: 25248.0 | inactive_split_MB: 284.1 +rank5 [2026-10-29 02:30:00,712] INFO : Step 21600/50000 rank5 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1290 | Classification: 0.1062 | Classification_binary: 0.1618 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1486 | Regression_eta: 0.0018 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1568 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0628 | Regression_eta: 0.0517 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0628 | LR: 6.45e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9644s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0061 | max_forward: 0.7663 | max_loss: 0.1883 | max_backward: 1.4257 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.4997 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7120.6 | uss_MB: 6456.9 | child_rss_MB: 5032.2 | allocated_MB: 340.5 | reserved_MB: 25248.0 | inactive_split_MB: 234.8 +rank5 [2026-10-29 02:33:57,426] INFO : Step 21700/50000 rank5 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6283 | Train Loss Components: Auxiliary: 0.2577 | Cardinality: 0.1575 | Classification: 0.1186 | Classification_binary: 0.1853 | HitGrouping: 0.2068 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1594 | Regression_eta: 0.0022 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0735 | Regression_eta: 0.0514 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0710 | LR: 6.43e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7308s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.6796 | max_loss: 0.1417 | max_backward: 1.3881 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7122.2 | uss_MB: 6458.0 | child_rss_MB: 5034.2 | allocated_MB: 343.8 | reserved_MB: 25248.0 | inactive_split_MB: 244.5 +rank5 [2026-10-29 02:37:54,438] INFO : Step 21800/50000 rank5 | Train Loss: 1.2260 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2691 | Cardinality: 0.1439 | Classification: 0.1105 | Classification_binary: 0.2043 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1576 | Regression_eta: 0.0016 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2299 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0753 | Regression_eta: 0.0310 | Regression_pt: 0.0794 | Regression_sin_phi: 0.0545 | LR: 6.40e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.9216s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 0.8043 | max_loss: 0.1895 | max_backward: 2.4027 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.9442 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7124.3 | uss_MB: 5827.3 | child_rss_MB: 5037.0 | allocated_MB: 344.5 | reserved_MB: 25248.0 | inactive_split_MB: 277.8 +rank5 [2026-10-29 02:41:42,511] INFO : Step 21900/50000 rank5 | Train Loss: 1.2119 | Train Loss Unweighted: 0.5952 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1861 | Classification: 0.1033 | Classification_binary: 0.1865 | HitGrouping: 0.1989 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0015 | Regression_pt: 0.1511 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.1965 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0731 | Regression_eta: 0.0344 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0641 | LR: 6.37e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.5480s | Phase Times: max_pre_step_barrier: 0.0174 | max_batch_to_device: 0.0163 | max_forward: 0.9684 | max_loss: 0.1448 | max_backward: 1.6354 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.8990 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 7124.6 | uss_MB: 6460.4 | child_rss_MB: 5033.8 | allocated_MB: 342.7 | reserved_MB: 25248.0 | inactive_split_MB: 192.3 +rank5 [2026-10-29 02:45:41,732] INFO : Step 22000/50000 rank5 | Train Loss: 1.1094 | Train Loss Unweighted: 0.5651 | Train Loss Components: Auxiliary: 0.2344 | Cardinality: 0.1248 | Classification: 0.1037 | Classification_binary: 0.1579 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1497 | Regression_eta: 0.0017 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1351 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0784 | Regression_eta: 0.0375 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0683 | LR: 6.34e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.4221s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.6825 | max_loss: 0.1653 | max_backward: 1.4396 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.0256 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 7124.1 | uss_MB: 6463.7 | child_rss_MB: 5009.1 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 247.3 +rank5 [2026-10-29 02:45:42,811] INFO : Running validation on rank5 +rank5 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 02:46:37,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:46:38,049] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 02:46:57,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 02:46:58,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 02:46:59,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 02:47:00,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 02:47:10,821] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 02:47:11,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:47:11,957] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 02:47:28,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 02:47:29,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 02:47:30,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 02:47:31,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 02:47:41,136] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 02:47:41,460] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:47:42,035] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 02:47:57,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 02:47:58,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 02:47:59,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 02:47:59,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 02:48:09,841] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 02:52:24,843] INFO : Step 22100/50000 rank5 | Train Loss: 1.2514 | Train Loss Unweighted: 0.6219 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1530 | Classification: 0.1140 | Classification_binary: 0.2076 | HitGrouping: 0.2031 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1470 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1884 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0680 | Regression_eta: 0.0320 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0554 | LR: 6.32e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5396s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7250 | max_loss: 0.2514 | max_backward: 1.5956 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.8632 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0587 | Memory: rss_MB: 8032.9 | uss_MB: 7365.3 | child_rss_MB: 5019.4 | allocated_MB: 347.3 | reserved_MB: 23386.0 | inactive_split_MB: 278.6 +rank5 [2026-10-29 02:56:29,085] INFO : Step 22200/50000 rank5 | Train Loss: 1.2727 | Train Loss Unweighted: 0.6485 | Train Loss Components: Auxiliary: 0.2709 | Cardinality: 0.1498 | Classification: 0.1104 | Classification_binary: 0.2184 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1574 | Regression_eta: 0.0018 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2332 | Regression_cos_phi: 0.1087 | Regression_energy: 0.0670 | Regression_eta: 0.0471 | Regression_pt: 0.0689 | Regression_sin_phi: 0.1014 | LR: 6.29e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.9315s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0074 | max_forward: 0.7776 | max_loss: 0.1472 | max_backward: 1.7604 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.3305 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7158.4 | uss_MB: 5092.5 | child_rss_MB: 5016.1 | allocated_MB: 342.3 | reserved_MB: 23578.0 | inactive_split_MB: 277.9 +rank5 [2026-10-29 03:00:35,125] INFO : Step 22300/50000 rank5 | Train Loss: 1.1242 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2347 | Cardinality: 0.1367 | Classification: 0.1013 | Classification_binary: 0.1688 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0807 | Regression_eta: 0.0427 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0407 | LR: 6.26e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.7792s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7697 | max_loss: 0.1538 | max_backward: 1.3553 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.4896 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 7162.9 | uss_MB: 6498.9 | child_rss_MB: 4987.3 | allocated_MB: 341.2 | reserved_MB: 23578.0 | inactive_split_MB: 233.8 +rank5 [2026-10-29 03:04:32,773] INFO : Step 22400/50000 rank5 | Train Loss: 1.2357 | Train Loss Unweighted: 0.6255 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1493 | Classification: 0.1171 | Classification_binary: 0.1885 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1575 | Regression_eta: 0.0021 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1999 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0883 | Regression_eta: 0.0491 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0713 | LR: 6.23e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9146s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0067 | max_forward: 0.6425 | max_loss: 0.1530 | max_backward: 1.4136 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.5876 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 7158.4 | uss_MB: 6496.8 | child_rss_MB: 5021.9 | allocated_MB: 342.5 | reserved_MB: 23578.0 | inactive_split_MB: 251.5 +rank5 [2026-10-29 03:08:26,683] INFO : Step 22500/50000 rank5 | Train Loss: 1.2633 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2638 | Cardinality: 0.1676 | Classification: 0.0954 | Classification_binary: 0.2181 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0020 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1901 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0628 | Regression_eta: 0.0415 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0604 | LR: 6.20e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.4283s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1522 | max_backward: 1.5631 | max_optimizer_step: 0.3261 | max_tensorboard_logging: 0.7552 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 7160.2 | uss_MB: 6502.4 | child_rss_MB: 5011.7 | allocated_MB: 342.9 | reserved_MB: 24052.0 | inactive_split_MB: 295.1 +rank5 [2026-10-29 03:12:18,754] INFO : Step 22600/50000 rank5 | Train Loss: 1.1636 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1329 | Classification: 0.0997 | Classification_binary: 0.1889 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1493 | Regression_eta: 0.0020 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1874 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0749 | Regression_eta: 0.0365 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0469 | LR: 6.18e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9322s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7375 | max_loss: 0.2403 | max_backward: 1.4584 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7163.5 | uss_MB: 6503.3 | child_rss_MB: 5024.4 | allocated_MB: 344.2 | reserved_MB: 24590.0 | inactive_split_MB: 288.5 +rank5 [2026-10-29 03:16:20,063] INFO : Step 22700/50000 rank5 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1212 | Classification: 0.0976 | Classification_binary: 0.1566 | HitGrouping: 0.2101 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1455 | Regression_eta: 0.0015 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1529 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0833 | Regression_eta: 0.0413 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0707 | LR: 6.15e-04 | DataLoad Time: 0.0166s | Model Forward Time: 6.9680s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.6124 | max_loss: 0.1376 | max_backward: 1.1766 | max_optimizer_step: 0.2237 | max_tensorboard_logging: 3.8568 | max_tensorboard_flush: 0.0169 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 7164.2 | uss_MB: 6160.1 | child_rss_MB: 5007.8 | allocated_MB: 341.6 | reserved_MB: 25058.0 | inactive_split_MB: 256.4 +rank5 [2026-10-29 03:20:29,740] INFO : Step 22800/50000 rank5 | Train Loss: 1.1840 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2533 | Cardinality: 0.1443 | Classification: 0.1191 | Classification_binary: 0.1924 | HitGrouping: 0.1959 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1444 | Regression_eta: 0.0016 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.2102 | Regression_cos_phi: 0.1049 | Regression_energy: 0.0605 | Regression_eta: 0.0584 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0730 | LR: 6.12e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4521s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7010 | max_loss: 0.2312 | max_backward: 1.6853 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8726 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 7164.4 | uss_MB: 6502.8 | child_rss_MB: 5021.2 | allocated_MB: 341.7 | reserved_MB: 25058.0 | inactive_split_MB: 281.3 +rank5 [2026-10-29 03:24:21,161] INFO : Step 22900/50000 rank5 | Train Loss: 1.2680 | Train Loss Unweighted: 0.6318 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1792 | Classification: 0.1020 | Classification_binary: 0.2074 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1580 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0740 | Regression_eta: 0.0359 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0535 | LR: 6.09e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.0000s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 1.1637 | max_loss: 0.4031 | max_backward: 1.8442 | max_optimizer_step: 0.3320 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7165.6 | uss_MB: 6506.2 | child_rss_MB: 5023.4 | allocated_MB: 344.8 | reserved_MB: 25058.0 | inactive_split_MB: 300.1 +rank5 [2026-10-29 03:28:05,593] INFO : Step 23000/50000 rank5 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5484 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1129 | Classification: 0.0836 | Classification_binary: 0.1519 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1547 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1900 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0697 | Regression_eta: 0.0371 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0613 | LR: 6.06e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7528s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0063 | max_forward: 0.6963 | max_loss: 0.2382 | max_backward: 1.2870 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 7162.6 | uss_MB: 6506.3 | child_rss_MB: 5008.7 | allocated_MB: 340.3 | reserved_MB: 26066.0 | inactive_split_MB: 218.4 +rank5 [2026-10-29 03:28:06,534] INFO : Running validation on rank5 +rank5 [2026-10-29 03:28:06,534] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 03:28:06,534] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 03:28:56,134] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:28:56,639] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 03:29:22,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 03:29:23,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 03:29:24,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 03:29:25,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 03:29:35,353] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 03:29:35,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:29:36,304] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 03:29:52,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 03:29:53,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 03:29:54,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 03:29:55,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 03:30:05,111] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 03:30:05,223] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:30:05,704] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 03:30:20,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 03:30:21,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 03:30:22,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 03:30:22,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 03:30:32,905] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 03:34:45,381] INFO : Step 23100/50000 rank5 | Train Loss: 1.2422 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1379 | Classification: 0.1246 | Classification_binary: 0.1795 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1482 | Regression_eta: 0.0023 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.2021 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0725 | Regression_eta: 0.0929 | Regression_pt: 0.0749 | Regression_sin_phi: 0.1309 | LR: 6.04e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8247s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7203 | max_loss: 0.1522 | max_backward: 1.1000 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.6183 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 8419.6 | uss_MB: 7758.7 | child_rss_MB: 5039.2 | allocated_MB: 344.7 | reserved_MB: 22598.0 | inactive_split_MB: 244.5 +rank5 [2026-10-29 03:38:33,524] INFO : Step 23200/50000 rank5 | Train Loss: 1.2507 | Train Loss Unweighted: 0.6359 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1400 | Classification: 0.1121 | Classification_binary: 0.2208 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1488 | Regression_eta: 0.0020 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2179 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0645 | Regression_eta: 0.0482 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0946 | LR: 6.01e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.5952s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0727 | max_forward: 0.9486 | max_loss: 0.1744 | max_backward: 1.7031 | max_optimizer_step: 0.2405 | max_tensorboard_logging: 0.6717 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1109 | Memory: rss_MB: 7225.3 | uss_MB: 6564.6 | child_rss_MB: 5046.8 | allocated_MB: 343.7 | reserved_MB: 23792.0 | inactive_split_MB: 223.2 +rank5 [2026-10-29 03:42:24,565] INFO : Step 23300/50000 rank5 | Train Loss: 1.1880 | Train Loss Unweighted: 0.5927 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1464 | Classification: 0.0987 | Classification_binary: 0.1851 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1524 | Regression_eta: 0.0019 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.2233 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0781 | Regression_eta: 0.0414 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0627 | LR: 5.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8600s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7557 | max_loss: 0.1467 | max_backward: 1.4708 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.4854 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 7231.3 | uss_MB: 6572.4 | child_rss_MB: 5045.1 | allocated_MB: 342.6 | reserved_MB: 24534.0 | inactive_split_MB: 271.6 +rank5 [2026-10-29 03:46:11,296] INFO : Step 23400/50000 rank5 | Train Loss: 1.1136 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2402 | Cardinality: 0.1202 | Classification: 0.1070 | Classification_binary: 0.1497 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1644 | Regression_eta: 0.0017 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.1256 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0993 | Regression_eta: 0.0524 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0393 | LR: 5.95e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.6656s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.5743 | max_loss: 0.2297 | max_backward: 1.0127 | max_optimizer_step: 0.3321 | max_tensorboard_logging: 0.6182 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 7232.3 | uss_MB: 6571.7 | child_rss_MB: 5046.9 | allocated_MB: 340.5 | reserved_MB: 24848.0 | inactive_split_MB: 241.3 +rank5 [2026-10-29 03:50:07,301] INFO : Step 23500/50000 rank5 | Train Loss: 1.1601 | Train Loss Unweighted: 0.5799 | Train Loss Components: Auxiliary: 0.2477 | Cardinality: 0.1161 | Classification: 0.1057 | Classification_binary: 0.1852 | HitGrouping: 0.2029 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1416 | Regression_eta: 0.0017 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1819 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0649 | Regression_eta: 0.0336 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0926 | LR: 5.92e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4422s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8864 | max_loss: 0.1573 | max_backward: 1.9036 | max_optimizer_step: 0.2294 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7232.1 | uss_MB: 6574.4 | child_rss_MB: 5036.9 | allocated_MB: 341.9 | reserved_MB: 24848.0 | inactive_split_MB: 233.9 +rank5 [2026-10-29 03:53:52,162] INFO : Step 23600/50000 rank5 | Train Loss: 1.2402 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1447 | Classification: 0.1136 | Classification_binary: 0.2049 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1534 | Regression_eta: 0.0020 | Regression_pt: 0.1550 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.1876 | Regression_cos_phi: 0.0877 | Regression_energy: 0.0685 | Regression_eta: 0.0494 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0943 | LR: 5.90e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.9438s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8397 | max_loss: 0.1470 | max_backward: 1.4395 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7234.1 | uss_MB: 6575.7 | child_rss_MB: 5045.6 | allocated_MB: 342.5 | reserved_MB: 24848.0 | inactive_split_MB: 215.5 +rank5 [2026-10-29 03:57:35,779] INFO : Step 23700/50000 rank5 | Train Loss: 1.1009 | Train Loss Unweighted: 0.5682 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1218 | Classification: 0.0954 | Classification_binary: 0.1599 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1528 | Regression_eta: 0.0014 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0754 | Regression_eta: 0.0293 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0329 | LR: 5.87e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6823s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6589 | max_loss: 0.1387 | max_backward: 1.2152 | max_optimizer_step: 0.2251 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7230.0 | uss_MB: 6574.3 | child_rss_MB: 5051.4 | allocated_MB: 342.5 | reserved_MB: 25508.0 | inactive_split_MB: 226.3 +rank5 [2026-10-29 04:01:35,999] INFO : Step 23800/50000 rank5 | Train Loss: 1.1390 | Train Loss Unweighted: 0.5882 | Train Loss Components: Auxiliary: 0.2427 | Cardinality: 0.1196 | Classification: 0.1035 | Classification_binary: 0.1735 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0628 | Regression_eta: 0.0449 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0816 | LR: 5.84e-04 | DataLoad Time: 0.0224s | Model Forward Time: 3.7615s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.7283 | max_loss: 0.1547 | max_backward: 1.2415 | max_optimizer_step: 0.2259 | max_tensorboard_logging: 0.5270 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7231.0 | uss_MB: 6110.2 | child_rss_MB: 5050.1 | allocated_MB: 345.2 | reserved_MB: 25508.0 | inactive_split_MB: 241.8 +rank5 [2026-10-29 04:05:42,354] INFO : Step 23900/50000 rank5 | Train Loss: 1.2349 | Train Loss Unweighted: 0.6078 | Train Loss Components: Auxiliary: 0.2621 | Cardinality: 0.1596 | Classification: 0.1044 | Classification_binary: 0.2077 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1451 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.2074 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0646 | Regression_eta: 0.0954 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0509 | LR: 5.81e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4880s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.9077 | max_loss: 0.1538 | max_backward: 1.9088 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7234.7 | uss_MB: 6579.6 | child_rss_MB: 5044.4 | allocated_MB: 342.1 | reserved_MB: 27474.0 | inactive_split_MB: 282.4 +rank5 [2026-10-29 04:09:40,154] INFO : Step 24000/50000 rank5 | Train Loss: 1.1074 | Train Loss Unweighted: 0.5714 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1170 | Classification: 0.1007 | Classification_binary: 0.1792 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1431 | Regression_eta: 0.0015 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0803 | Regression_eta: 0.0372 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0573 | LR: 5.78e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.2198s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 1.1428 | max_loss: 0.2613 | max_backward: 2.5767 | max_optimizer_step: 0.2298 | max_tensorboard_logging: 0.5998 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7236.7 | uss_MB: 6577.2 | child_rss_MB: 5038.1 | allocated_MB: 342.1 | reserved_MB: 27476.0 | inactive_split_MB: 218.6 +rank5 [2026-10-29 04:09:41,046] INFO : Running validation on rank5 +rank5 [2026-10-29 04:09:41,046] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 04:09:41,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 04:10:38,743] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:10:39,335] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 04:11:05,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 04:11:06,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 04:11:07,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 04:11:10,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 04:11:20,232] INFO : Time taken to make predictions on device 5 is: 0.28 min +rank5 [2026-10-29 04:11:20,671] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:11:21,254] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 04:11:37,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 04:11:38,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 04:11:39,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 04:11:40,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 04:11:50,227] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 04:11:50,532] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:11:51,026] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 04:12:06,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 04:12:07,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 04:12:07,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 04:12:08,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 04:12:18,416] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 04:16:38,286] INFO : Step 24100/50000 rank5 | Train Loss: 1.1244 | Train Loss Unweighted: 0.5584 | Train Loss Components: Auxiliary: 0.2373 | Cardinality: 0.1285 | Classification: 0.0978 | Classification_binary: 0.1606 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0764 | Regression_eta: 0.0392 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0716 | LR: 5.75e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1979s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0860 | max_forward: 0.7361 | max_loss: 0.1432 | max_backward: 1.4268 | max_optimizer_step: 0.3451 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 8228.7 | uss_MB: 7570.1 | child_rss_MB: 5001.0 | allocated_MB: 343.2 | reserved_MB: 21608.0 | inactive_split_MB: 269.9 +rank5 [2026-10-29 04:20:36,776] INFO : Step 24200/50000 rank5 | Train Loss: 1.3067 | Train Loss Unweighted: 0.6341 | Train Loss Components: Auxiliary: 0.2739 | Cardinality: 0.1760 | Classification: 0.1145 | Classification_binary: 0.2101 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0018 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.1990 | Regression_cos_phi: 0.0954 | Regression_energy: 0.0740 | Regression_eta: 0.0886 | Regression_pt: 0.0780 | Regression_sin_phi: 0.1075 | LR: 5.73e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.0788s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1066 | max_forward: 1.0531 | max_loss: 0.2796 | max_backward: 2.3820 | max_optimizer_step: 0.2441 | max_tensorboard_logging: 0.6187 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 8233.6 | uss_MB: 7574.6 | child_rss_MB: 5010.3 | allocated_MB: 345.0 | reserved_MB: 23864.0 | inactive_split_MB: 203.0 +rank5 [2026-10-29 04:24:33,010] INFO : Step 24300/50000 rank5 | Train Loss: 1.2207 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1582 | Classification: 0.1151 | Classification_binary: 0.2013 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1450 | Regression_eta: 0.0017 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.2269 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0742 | Regression_eta: 0.0463 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0788 | LR: 5.70e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3066s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7395 | max_loss: 0.2490 | max_backward: 1.4969 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.6555 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7355.3 | uss_MB: 6697.1 | child_rss_MB: 5007.1 | allocated_MB: 344.8 | reserved_MB: 23864.0 | inactive_split_MB: 284.8 +rank5 [2026-10-29 04:28:19,173] INFO : Step 24400/50000 rank5 | Train Loss: 1.1288 | Train Loss Unweighted: 0.5921 | Train Loss Components: Auxiliary: 0.2439 | Cardinality: 0.1229 | Classification: 0.1017 | Classification_binary: 0.1613 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1631 | Regression_eta: 0.0015 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0664 | Regression_eta: 0.0591 | Regression_pt: 0.0689 | Regression_sin_phi: 0.1155 | LR: 5.67e-04 | DataLoad Time: 0.0187s | Model Forward Time: 3.9878s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.6685 | max_loss: 0.1564 | max_backward: 1.4736 | max_optimizer_step: 0.2223 | max_tensorboard_logging: 0.5369 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7357.1 | uss_MB: 6698.7 | child_rss_MB: 5016.5 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 267.2 +rank5 [2026-10-29 04:32:03,440] INFO : Step 24500/50000 rank5 | Train Loss: 1.2319 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2586 | Cardinality: 0.1511 | Classification: 0.1107 | Classification_binary: 0.1876 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1540 | Regression_eta: 0.0018 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.1874 | Regression_cos_phi: 0.1113 | Regression_energy: 0.0811 | Regression_eta: 0.0470 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0762 | LR: 5.64e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1620s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0677 | max_forward: 0.7219 | max_loss: 0.1439 | max_backward: 1.4253 | max_optimizer_step: 0.3416 | max_tensorboard_logging: 0.5859 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 7356.5 | uss_MB: 6698.2 | child_rss_MB: 5010.5 | allocated_MB: 343.1 | reserved_MB: 23864.0 | inactive_split_MB: 275.8 +rank5 [2026-10-29 04:36:01,552] INFO : Step 24600/50000 rank5 | Train Loss: 1.2474 | Train Loss Unweighted: 0.6241 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1630 | Classification: 0.1074 | Classification_binary: 0.2102 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.2139 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0798 | Regression_eta: 0.0424 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0612 | LR: 5.61e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5145s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7876 | max_loss: 0.1528 | max_backward: 1.7437 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.6204 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7358.2 | uss_MB: 6700.7 | child_rss_MB: 5006.8 | allocated_MB: 344.3 | reserved_MB: 23864.0 | inactive_split_MB: 285.7 +rank5 [2026-10-29 04:39:51,298] INFO : Step 24700/50000 rank5 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1193 | Classification: 0.0990 | Classification_binary: 0.1723 | HitGrouping: 0.2019 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1471 | Regression_eta: 0.0016 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1715 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0846 | Regression_eta: 0.0324 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0593 | LR: 5.58e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9523s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6978 | max_loss: 0.2493 | max_backward: 1.2925 | max_optimizer_step: 0.2314 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7361.3 | uss_MB: 6700.3 | child_rss_MB: 5009.8 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 223.9 +rank5 [2026-10-29 04:43:37,808] INFO : Step 24800/50000 rank5 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5554 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1264 | Classification: 0.0932 | Classification_binary: 0.1583 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0675 | Regression_eta: 0.0354 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0629 | LR: 5.56e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2772s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6582 | max_loss: 0.2533 | max_backward: 1.5858 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.6923 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1086 | Memory: rss_MB: 7365.2 | uss_MB: 6703.3 | child_rss_MB: 5005.7 | allocated_MB: 342.9 | reserved_MB: 23864.0 | inactive_split_MB: 278.6 +rank5 [2026-10-29 04:47:36,371] INFO : Step 24900/50000 rank5 | Train Loss: 1.1348 | Train Loss Unweighted: 0.5816 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1381 | Classification: 0.1126 | Classification_binary: 0.1845 | HitGrouping: 0.1937 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1403 | Regression_eta: 0.0015 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.1907 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0637 | Regression_eta: 0.0403 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0517 | LR: 5.53e-04 | DataLoad Time: 0.0158s | Model Forward Time: 8.7024s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0067 | max_forward: 0.7276 | max_loss: 0.2411 | max_backward: 1.3958 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 5.1754 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1038 | Memory: rss_MB: 7366.2 | uss_MB: 6703.5 | child_rss_MB: 5007.2 | allocated_MB: 343.9 | reserved_MB: 23864.0 | inactive_split_MB: 239.7 +rank5 [2026-10-29 04:51:28,372] INFO : Step 25000/50000 rank5 | Train Loss: 1.1995 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1590 | Classification: 0.0891 | Classification_binary: 0.2009 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1421 | Regression_eta: 0.0018 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.2139 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0834 | Regression_eta: 0.0484 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0655 | LR: 5.50e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2030s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7692 | max_loss: 0.2091 | max_backward: 1.3616 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6652 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1392 | Memory: rss_MB: 7366.4 | uss_MB: 6702.5 | child_rss_MB: 5014.2 | allocated_MB: 343.5 | reserved_MB: 23864.0 | inactive_split_MB: 293.8 +rank5 [2026-10-29 04:51:29,310] INFO : Running validation on rank5 +rank5 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 04:52:22,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:52:22,822] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 04:52:49,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 04:52:50,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 04:52:51,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 04:52:52,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 04:53:02,601] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 04:53:02,764] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:53:03,282] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 04:53:19,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 04:53:20,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 04:53:21,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 04:53:22,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 04:53:32,446] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 04:53:32,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:53:32,991] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 04:53:48,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 04:53:49,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 04:53:49,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 04:53:50,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 04:54:00,435] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 04:58:07,811] INFO : Step 25100/50000 rank5 | Train Loss: 1.0833 | Train Loss Unweighted: 0.5514 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1317 | Classification: 0.0920 | Classification_binary: 0.1568 | HitGrouping: 0.2008 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1491 | Regression_eta: 0.0016 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1597 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0752 | Regression_eta: 0.0312 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0569 | LR: 5.47e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7889s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7295 | max_loss: 0.1401 | max_backward: 1.2439 | max_optimizer_step: 0.2490 | max_tensorboard_logging: 0.4940 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8607.6 | uss_MB: 7944.9 | child_rss_MB: 5042.3 | allocated_MB: 342.3 | reserved_MB: 23608.0 | inactive_split_MB: 222.3 +rank5 [2026-10-29 05:01:53,204] INFO : Step 25200/50000 rank5 | Train Loss: 1.1886 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1523 | Classification: 0.1158 | Classification_binary: 0.1800 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1512 | Regression_eta: 0.0016 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.1862 | Regression_cos_phi: 0.1063 | Regression_energy: 0.0620 | Regression_eta: 0.0630 | Regression_pt: 0.0652 | Regression_sin_phi: 0.1282 | LR: 5.44e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.7326s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8134 | max_loss: 0.1416 | max_backward: 1.5095 | max_optimizer_step: 0.3471 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 8606.1 | uss_MB: 7945.9 | child_rss_MB: 5039.5 | allocated_MB: 343.6 | reserved_MB: 24590.0 | inactive_split_MB: 260.9 +rank5 [2026-10-29 05:05:52,267] INFO : Step 25300/50000 rank5 | Train Loss: 1.2031 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2624 | Cardinality: 0.1455 | Classification: 0.1145 | Classification_binary: 0.1966 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1448 | Regression_eta: 0.0016 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1541 | Regression_cos_phi: 0.0566 | Regression_energy: 0.0589 | Regression_eta: 0.0358 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0521 | LR: 5.42e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.5368s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.8219 | max_loss: 0.1491 | max_backward: 1.6406 | max_optimizer_step: 0.2735 | max_tensorboard_logging: 0.7204 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0488 | Memory: rss_MB: 7456.6 | uss_MB: 6796.5 | child_rss_MB: 5036.7 | allocated_MB: 341.8 | reserved_MB: 25572.0 | inactive_split_MB: 253.5 +rank5 [2026-10-29 05:09:44,971] INFO : Step 25400/50000 rank5 | Train Loss: 1.1194 | Train Loss Unweighted: 0.5655 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1346 | Classification: 0.0983 | Classification_binary: 0.1754 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1438 | Regression_eta: 0.0015 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.2106 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0821 | Regression_eta: 0.0496 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0559 | LR: 5.39e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1076s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.6986 | max_loss: 0.1530 | max_backward: 1.2606 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.8427 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7454.9 | uss_MB: 6795.9 | child_rss_MB: 5040.6 | allocated_MB: 345.2 | reserved_MB: 25572.0 | inactive_split_MB: 223.8 +rank5 [2026-10-29 05:13:34,054] INFO : Step 25500/50000 rank5 | Train Loss: 1.0289 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1068 | Classification: 0.1034 | Classification_binary: 0.1444 | HitGrouping: 0.1913 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1484 | Regression_eta: 0.0014 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1625 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0733 | Regression_eta: 0.0373 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0528 | LR: 5.36e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.1581s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0554 | max_forward: 0.7008 | max_loss: 0.1433 | max_backward: 1.4571 | max_optimizer_step: 0.3349 | max_tensorboard_logging: 0.6083 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 7460.4 | uss_MB: 6802.0 | child_rss_MB: 5038.5 | allocated_MB: 345.4 | reserved_MB: 25572.0 | inactive_split_MB: 240.6 +rank5 [2026-10-29 05:17:32,215] INFO : Step 25600/50000 rank5 | Train Loss: 1.2028 | Train Loss Unweighted: 0.5968 | Train Loss Components: Auxiliary: 0.2590 | Cardinality: 0.1607 | Classification: 0.1135 | Classification_binary: 0.1926 | HitGrouping: 0.1978 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1429 | Regression_eta: 0.0016 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0790 | Classification_binary: 0.1838 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0767 | Regression_eta: 0.0396 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0782 | LR: 5.33e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.6832s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8800 | max_loss: 0.1578 | max_backward: 1.8564 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 7459.9 | uss_MB: 6802.4 | child_rss_MB: 5035.4 | allocated_MB: 345.5 | reserved_MB: 26054.0 | inactive_split_MB: 284.5 +rank5 [2026-10-29 05:21:26,002] INFO : Step 25700/50000 rank5 | Train Loss: 1.1800 | Train Loss Unweighted: 0.5964 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1298 | Classification: 0.1086 | Classification_binary: 0.1961 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0019 | Regression_pt: 0.1444 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0752 | Regression_eta: 0.0509 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0767 | LR: 5.30e-04 | DataLoad Time: 0.0136s | Model Forward Time: 4.5154s | Phase Times: max_pre_step_barrier: 0.0086 | max_batch_to_device: 0.0137 | max_forward: 1.0340 | max_loss: 0.2483 | max_backward: 1.6704 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 0.7612 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7459.7 | uss_MB: 6801.4 | child_rss_MB: 5033.4 | allocated_MB: 343.5 | reserved_MB: 26054.0 | inactive_split_MB: 270.8 +rank5 [2026-10-29 05:25:16,515] INFO : Step 25800/50000 rank5 | Train Loss: 1.1337 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2388 | Cardinality: 0.1264 | Classification: 0.0898 | Classification_binary: 0.1617 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0019 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0972 | Regression_energy: 0.0619 | Regression_eta: 0.0497 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0604 | LR: 5.27e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.9356s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0986 | max_forward: 0.8129 | max_loss: 0.1662 | max_backward: 1.2979 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1103 | Memory: rss_MB: 7457.8 | uss_MB: 6799.8 | child_rss_MB: 5030.2 | allocated_MB: 341.8 | reserved_MB: 26054.0 | inactive_split_MB: 273.2 +rank5 [2026-10-29 05:29:08,211] INFO : Step 25900/50000 rank5 | Train Loss: 1.1019 | Train Loss Unweighted: 0.5481 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1165 | Classification: 0.0977 | Classification_binary: 0.1594 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1434 | Regression_eta: 0.0016 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0656 | Regression_eta: 0.0503 | Regression_pt: 0.0687 | Regression_sin_phi: 0.1203 | LR: 5.25e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5194s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7034 | max_loss: 0.2469 | max_backward: 1.7130 | max_optimizer_step: 0.2363 | max_tensorboard_logging: 0.8382 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 7454.8 | uss_MB: 6800.6 | child_rss_MB: 5037.1 | allocated_MB: 343.0 | reserved_MB: 26054.0 | inactive_split_MB: 187.0 +rank5 [2026-10-29 05:33:07,468] INFO : Step 26000/50000 rank5 | Train Loss: 1.1708 | Train Loss Unweighted: 0.5680 | Train Loss Components: Auxiliary: 0.2498 | Cardinality: 0.1352 | Classification: 0.0999 | Classification_binary: 0.2021 | HitGrouping: 0.1994 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1304 | Regression_eta: 0.0021 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0598 | Regression_eta: 0.0520 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0633 | LR: 5.22e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2240s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.7877 | max_loss: 0.1637 | max_backward: 1.6550 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.4954 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 7454.4 | uss_MB: 6802.1 | child_rss_MB: 5032.6 | allocated_MB: 345.0 | reserved_MB: 27098.0 | inactive_split_MB: 285.6 +rank5 [2026-10-29 05:33:08,394] INFO : Running validation on rank5 +rank5 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 05:33:56,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:33:57,250] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 05:34:24,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 05:34:25,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 05:34:26,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 05:34:27,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 05:34:37,468] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 05:34:38,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:34:38,538] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 05:34:54,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 05:34:55,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 05:34:56,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 05:34:57,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 05:35:07,552] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 05:35:08,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:35:08,514] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 05:35:24,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 05:35:24,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 05:35:25,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 05:35:26,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 05:35:36,084] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 05:39:49,627] INFO : Step 26100/50000 rank5 | Train Loss: 1.0974 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2307 | Cardinality: 0.1119 | Classification: 0.0956 | Classification_binary: 0.1689 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1513 | Regression_eta: 0.0014 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1673 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0664 | Regression_eta: 0.0355 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0673 | LR: 5.19e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.1615s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0117 | max_forward: 0.7200 | max_loss: 0.1501 | max_backward: 1.2286 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.8284 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 8630.6 | uss_MB: 7973.7 | child_rss_MB: 5008.3 | allocated_MB: 340.1 | reserved_MB: 21762.0 | inactive_split_MB: 215.7 +rank5 [2026-10-29 05:43:56,538] INFO : Step 26200/50000 rank5 | Train Loss: 1.0876 | Train Loss Unweighted: 0.5502 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1065 | Classification: 0.0968 | Classification_binary: 0.1527 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1476 | Regression_eta: 0.0018 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0832 | Regression_eta: 0.0485 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0509 | LR: 5.16e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3259s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6530 | max_loss: 0.1421 | max_backward: 1.3973 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.8695 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8630.6 | uss_MB: 7973.8 | child_rss_MB: 5031.7 | allocated_MB: 342.3 | reserved_MB: 22678.0 | inactive_split_MB: 242.5 +rank5 [2026-10-29 05:47:48,030] INFO : Step 26300/50000 rank5 | Train Loss: 1.2051 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1512 | Classification: 0.1193 | Classification_binary: 0.1877 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1458 | Regression_eta: 0.0017 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1854 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0661 | Regression_eta: 0.0482 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0603 | LR: 5.13e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.4689s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0072 | max_forward: 0.7601 | max_loss: 0.2267 | max_backward: 1.8902 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7522.0 | uss_MB: 6871.8 | child_rss_MB: 4996.1 | allocated_MB: 345.4 | reserved_MB: 22678.0 | inactive_split_MB: 270.9 +rank5 [2026-10-29 05:51:47,278] INFO : Step 26400/50000 rank5 | Train Loss: 1.2256 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1648 | Classification: 0.1020 | Classification_binary: 0.1981 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1520 | Regression_eta: 0.0015 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1904 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0798 | Regression_eta: 0.0422 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0538 | LR: 5.10e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.3854s | Phase Times: max_pre_step_barrier: 0.0351 | max_batch_to_device: 0.0070 | max_forward: 0.8129 | max_loss: 0.1477 | max_backward: 1.4015 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8384 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 7520.2 | uss_MB: 6869.2 | child_rss_MB: 4985.0 | allocated_MB: 342.8 | reserved_MB: 22678.0 | inactive_split_MB: 196.9 +rank5 [2026-10-29 05:55:40,758] INFO : Step 26500/50000 rank5 | Train Loss: 1.1110 | Train Loss Unweighted: 0.5356 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1488 | Classification: 0.0891 | Classification_binary: 0.1546 | HitGrouping: 0.2012 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1593 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0656 | Regression_eta: 0.0364 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0659 | LR: 5.08e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0218s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6841 | max_loss: 0.2421 | max_backward: 1.1616 | max_optimizer_step: 0.2383 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7505.4 | uss_MB: 6872.6 | child_rss_MB: 4957.6 | allocated_MB: 340.6 | reserved_MB: 22678.0 | inactive_split_MB: 269.9 +rank5 [2026-10-29 05:59:33,068] INFO : Step 26600/50000 rank5 | Train Loss: 1.0895 | Train Loss Unweighted: 0.5529 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1261 | Classification: 0.1083 | Classification_binary: 0.1603 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1386 | Regression_eta: 0.0015 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0695 | Regression_eta: 0.0439 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0582 | LR: 5.05e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4533s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0696 | max_forward: 0.6926 | max_loss: 0.1459 | max_backward: 1.4149 | max_optimizer_step: 0.2385 | max_tensorboard_logging: 0.8927 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7477.7 | uss_MB: 6873.1 | child_rss_MB: 4831.1 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 232.0 +rank5 [2026-10-29 06:03:23,755] INFO : Step 26700/50000 rank5 | Train Loss: 1.2592 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1717 | Classification: 0.1100 | Classification_binary: 0.2137 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1420 | Regression_eta: 0.0017 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0796 | Classification_binary: 0.2658 | Regression_cos_phi: 0.0994 | Regression_energy: 0.0742 | Regression_eta: 0.0530 | Regression_pt: 0.0765 | Regression_sin_phi: 0.1008 | LR: 5.02e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.6743s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7750 | max_loss: 0.1526 | max_backward: 1.6427 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.8866 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7338.5 | uss_MB: 6135.7 | child_rss_MB: 4788.2 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 267.4 +rank5 [2026-10-29 06:07:16,644] INFO : Step 26800/50000 rank5 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5792 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1288 | Classification: 0.0911 | Classification_binary: 0.1633 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1600 | Regression_eta: 0.0015 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1916 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0710 | Regression_eta: 0.0349 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0572 | LR: 4.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0418s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7108 | max_loss: 0.1467 | max_backward: 1.2855 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7173 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 7253.0 | uss_MB: 6872.2 | child_rss_MB: 4809.0 | allocated_MB: 340.8 | reserved_MB: 22846.0 | inactive_split_MB: 253.2 +rank5 [2026-10-29 06:11:09,652] INFO : Step 26900/50000 rank5 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.0996 | Classification: 0.0867 | Classification_binary: 0.1428 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1483 | Regression_eta: 0.0018 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1213 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0625 | Regression_eta: 0.0406 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0598 | LR: 4.96e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8391s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0336 | max_forward: 0.6534 | max_loss: 0.1408 | max_backward: 1.3898 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0912 | Memory: rss_MB: 7214.0 | uss_MB: 6872.7 | child_rss_MB: 4790.0 | allocated_MB: 342.5 | reserved_MB: 22846.0 | inactive_split_MB: 233.5 +rank5 [2026-10-29 06:15:05,369] INFO : Step 27000/50000 rank5 | Train Loss: 1.0945 | Train Loss Unweighted: 0.5396 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1196 | Classification: 0.0957 | Classification_binary: 0.1912 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1247 | Regression_eta: 0.0015 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0583 | Regression_eta: 0.0344 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0690 | LR: 4.94e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.1547s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7531 | max_loss: 0.1689 | max_backward: 1.5554 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6017 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7214.1 | uss_MB: 6877.0 | child_rss_MB: 4792.2 | allocated_MB: 345.3 | reserved_MB: 23694.0 | inactive_split_MB: 251.5 +rank5 [2026-10-29 06:15:06,391] INFO : Running validation on rank5 +rank5 [2026-10-29 06:15:06,391] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 06:15:06,391] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 06:16:00,133] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:16:00,715] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 06:16:29,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 06:16:30,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 06:16:31,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 06:16:32,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 06:16:42,699] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 06:16:43,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:16:43,719] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 06:16:59,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 06:17:00,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 06:17:01,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 06:17:02,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 06:17:12,498] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 06:17:13,249] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:17:13,779] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 06:17:29,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 06:17:30,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 06:17:30,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 06:17:31,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 06:17:41,598] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 06:21:51,976] INFO : Step 27100/50000 rank5 | Train Loss: 1.1577 | Train Loss Unweighted: 0.5595 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1497 | Classification: 0.0884 | Classification_binary: 0.1879 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1399 | Regression_eta: 0.0018 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2219 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0714 | Regression_eta: 0.0517 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0918 | LR: 4.91e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0510s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8543 | max_loss: 0.1477 | max_backward: 1.8762 | max_optimizer_step: 0.2540 | max_tensorboard_logging: 0.9462 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 8306.7 | uss_MB: 7966.7 | child_rss_MB: 4814.1 | allocated_MB: 342.9 | reserved_MB: 21640.0 | inactive_split_MB: 268.0 +rank5 [2026-10-29 06:25:39,973] INFO : Step 27200/50000 rank5 | Train Loss: 1.0028 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1065 | Classification: 0.0834 | Classification_binary: 0.1354 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1443 | Regression_eta: 0.0015 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1580 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0694 | Regression_eta: 0.0402 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0559 | LR: 4.88e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.0906s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6839 | max_loss: 0.1724 | max_backward: 1.0628 | max_optimizer_step: 0.2515 | max_tensorboard_logging: 0.9641 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0689 | Memory: rss_MB: 8305.7 | uss_MB: 7964.4 | child_rss_MB: 4815.8 | allocated_MB: 340.5 | reserved_MB: 22612.0 | inactive_split_MB: 267.2 +rank5 [2026-10-29 06:29:18,756] INFO : Step 27300/50000 rank5 | Train Loss: 1.1913 | Train Loss Unweighted: 0.5857 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1563 | Classification: 0.1169 | Classification_binary: 0.1769 | HitGrouping: 0.1965 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1426 | Regression_eta: 0.0019 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0599 | Regression_eta: 0.0374 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0657 | LR: 4.85e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2496s | Phase Times: max_pre_step_barrier: 0.0244 | max_batch_to_device: 0.0064 | max_forward: 0.6813 | max_loss: 0.1454 | max_backward: 1.4618 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7699 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 7290.8 | uss_MB: 6952.0 | child_rss_MB: 4811.5 | allocated_MB: 345.6 | reserved_MB: 23586.0 | inactive_split_MB: 267.9 +rank5 [2026-10-29 06:33:03,428] INFO : Step 27400/50000 rank5 | Train Loss: 1.1723 | Train Loss Unweighted: 0.5888 | Train Loss Components: Auxiliary: 0.2572 | Cardinality: 0.1400 | Classification: 0.1088 | Classification_binary: 0.2016 | HitGrouping: 0.1987 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1370 | Regression_eta: 0.0017 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2070 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0667 | Regression_eta: 0.0445 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0477 | LR: 4.82e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6667s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8048 | max_loss: 0.1475 | max_backward: 1.6655 | max_optimizer_step: 0.2279 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7292.0 | uss_MB: 6951.9 | child_rss_MB: 4814.0 | allocated_MB: 344.5 | reserved_MB: 23586.0 | inactive_split_MB: 279.5 +rank5 [2026-10-29 06:37:01,482] INFO : Step 27500/50000 rank5 | Train Loss: 1.0739 | Train Loss Unweighted: 0.5273 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1251 | Classification: 0.0933 | Classification_binary: 0.1655 | HitGrouping: 0.1957 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0648 | Regression_eta: 0.0288 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0473 | LR: 4.80e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2219s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7686 | max_loss: 0.1542 | max_backward: 1.3750 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.8628 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 7288.8 | uss_MB: 6945.7 | child_rss_MB: 4804.4 | allocated_MB: 343.6 | reserved_MB: 23586.0 | inactive_split_MB: 241.4 +rank5 [2026-10-29 06:40:55,530] INFO : Step 27600/50000 rank5 | Train Loss: 1.0507 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1163 | Classification: 0.1024 | Classification_binary: 0.1467 | HitGrouping: 0.1992 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0663 | Regression_eta: 0.0421 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0489 | LR: 4.77e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4907s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0060 | max_forward: 0.6744 | max_loss: 0.2492 | max_backward: 1.6867 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.5257 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 7290.8 | uss_MB: 6853.7 | child_rss_MB: 4808.0 | allocated_MB: 341.2 | reserved_MB: 24480.0 | inactive_split_MB: 234.8 +rank5 [2026-10-29 06:44:47,870] INFO : Step 27700/50000 rank5 | Train Loss: 1.1881 | Train Loss Unweighted: 0.5834 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1544 | Classification: 0.1001 | Classification_binary: 0.1885 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1444 | Regression_eta: 0.0015 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1791 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0649 | Regression_eta: 0.0412 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0782 | LR: 4.74e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.2915s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7716 | max_loss: 0.1527 | max_backward: 1.4941 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 0.7272 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7291.1 | uss_MB: 6952.7 | child_rss_MB: 4804.6 | allocated_MB: 344.5 | reserved_MB: 24480.0 | inactive_split_MB: 194.2 +rank5 [2026-10-29 06:48:38,928] INFO : Step 27800/50000 rank5 | Train Loss: 1.1903 | Train Loss Unweighted: 0.5862 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1435 | Classification: 0.1032 | Classification_binary: 0.2006 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1387 | Regression_eta: 0.0018 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.2014 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0710 | Regression_eta: 0.0572 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0602 | LR: 4.71e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1876s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8137 | max_loss: 0.1509 | max_backward: 1.4748 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6041 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7291.1 | uss_MB: 6953.1 | child_rss_MB: 4808.7 | allocated_MB: 348.4 | reserved_MB: 24592.0 | inactive_split_MB: 226.8 +rank5 [2026-10-29 06:52:39,106] INFO : Step 27900/50000 rank5 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5521 | Train Loss Components: Auxiliary: 0.2284 | Cardinality: 0.1139 | Classification: 0.0902 | Classification_binary: 0.1544 | HitGrouping: 0.2057 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1519 | Regression_eta: 0.0017 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0812 | Regression_eta: 0.0363 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0685 | LR: 4.68e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6319s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6397 | max_loss: 0.2456 | max_backward: 1.5234 | max_optimizer_step: 0.2276 | max_tensorboard_logging: 1.1626 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 7291.1 | uss_MB: 6955.2 | child_rss_MB: 4805.2 | allocated_MB: 342.4 | reserved_MB: 25612.0 | inactive_split_MB: 259.0 +rank5 [2026-10-29 06:56:30,759] INFO : Step 28000/50000 rank5 | Train Loss: 1.1111 | Train Loss Unweighted: 0.5401 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1274 | Classification: 0.1035 | Classification_binary: 0.1699 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1594 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0685 | Regression_eta: 0.0469 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0824 | LR: 4.66e-04 | DataLoad Time: 0.0225s | Model Forward Time: 5.0929s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.7503 | max_loss: 0.1492 | max_backward: 1.7566 | max_optimizer_step: 0.2764 | max_tensorboard_logging: 1.2295 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 7291.2 | uss_MB: 5984.8 | child_rss_MB: 4794.8 | allocated_MB: 344.1 | reserved_MB: 25612.0 | inactive_split_MB: 243.2 +rank5 [2026-10-29 06:56:31,846] INFO : Running validation on rank5 +rank5 [2026-10-29 06:56:31,846] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 06:56:31,846] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 06:57:24,999] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:57:25,560] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 06:57:52,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 06:57:53,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 06:57:54,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 06:57:55,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 06:58:05,143] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 06:58:05,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:58:06,179] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 06:58:21,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 06:58:22,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 06:58:23,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 06:58:24,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 06:58:34,798] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 06:58:35,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:58:35,800] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 06:58:51,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 06:58:52,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 06:58:52,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 06:58:53,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 06:59:03,411] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 07:03:11,252] INFO : Step 28100/50000 rank5 | Train Loss: 1.1727 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1644 | Classification: 0.0934 | Classification_binary: 0.1929 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1365 | Regression_eta: 0.0015 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.1827 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0654 | Regression_eta: 0.0362 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0642 | LR: 4.63e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.9515s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8425 | max_loss: 0.2571 | max_backward: 1.8157 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 1.0134 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 8340.2 | uss_MB: 8005.6 | child_rss_MB: 4802.9 | allocated_MB: 344.8 | reserved_MB: 22140.0 | inactive_split_MB: 293.2 +rank5 [2026-10-29 07:07:01,163] INFO : Step 28200/50000 rank5 | Train Loss: 1.0520 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2296 | Cardinality: 0.1203 | Classification: 0.0934 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1466 | Regression_eta: 0.0014 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0380 | Regression_energy: 0.0883 | Regression_eta: 0.0386 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0791 | LR: 4.60e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3562s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7943 | max_loss: 0.1496 | max_backward: 1.4503 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 0.7872 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 8343.2 | uss_MB: 8006.4 | child_rss_MB: 4796.0 | allocated_MB: 341.0 | reserved_MB: 24034.0 | inactive_split_MB: 210.4 +rank5 [2026-10-29 07:10:52,728] INFO : Step 28300/50000 rank5 | Train Loss: 1.0602 | Train Loss Unweighted: 0.5453 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1043 | Classification: 0.0923 | Classification_binary: 0.1501 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1496 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0543 | Regression_eta: 0.0278 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0689 | LR: 4.57e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2446s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6482 | max_loss: 0.2266 | max_backward: 1.5658 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.6870 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0836 | Memory: rss_MB: 7393.0 | uss_MB: 7059.5 | child_rss_MB: 4787.3 | allocated_MB: 342.7 | reserved_MB: 25016.0 | inactive_split_MB: 243.9 +rank5 [2026-10-29 07:14:40,573] INFO : Step 28400/50000 rank5 | Train Loss: 1.1507 | Train Loss Unweighted: 0.5391 | Train Loss Components: Auxiliary: 0.2436 | Cardinality: 0.1500 | Classification: 0.0965 | Classification_binary: 0.1896 | HitGrouping: 0.1911 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1241 | Regression_eta: 0.0018 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0581 | Regression_eta: 0.0492 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0984 | LR: 4.55e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6286s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7664 | max_loss: 0.1532 | max_backward: 1.7342 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.8455 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7394.0 | uss_MB: 7059.9 | child_rss_MB: 4785.1 | allocated_MB: 344.2 | reserved_MB: 25016.0 | inactive_split_MB: 278.9 +rank5 [2026-10-29 07:18:29,002] INFO : Step 28500/50000 rank5 | Train Loss: 1.1595 | Train Loss Unweighted: 0.5640 | Train Loss Components: Auxiliary: 0.2492 | Cardinality: 0.1556 | Classification: 0.0927 | Classification_binary: 0.1908 | HitGrouping: 0.1954 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1384 | Regression_eta: 0.0015 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0877 | Regression_energy: 0.0641 | Regression_eta: 0.0429 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0607 | LR: 4.52e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2718s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.8033 | max_loss: 0.1665 | max_backward: 1.5231 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.6178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0709 | Memory: rss_MB: 7393.5 | uss_MB: 7056.4 | child_rss_MB: 4808.7 | allocated_MB: 347.4 | reserved_MB: 26000.0 | inactive_split_MB: 283.2 +rank5 [2026-10-29 07:22:17,774] INFO : Step 28600/50000 rank5 | Train Loss: 1.0672 | Train Loss Unweighted: 0.5222 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1172 | Classification: 0.0893 | Classification_binary: 0.1544 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1516 | Regression_cos_phi: 0.0351 | Regression_energy: 0.0767 | Regression_eta: 0.0410 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0394 | LR: 4.49e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.0218s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6704 | max_loss: 0.1829 | max_backward: 1.2353 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.8074 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7396.5 | uss_MB: 7057.8 | child_rss_MB: 4793.1 | allocated_MB: 342.3 | reserved_MB: 26000.0 | inactive_split_MB: 243.7 +rank5 [2026-10-29 07:26:10,154] INFO : Step 28700/50000 rank5 | Train Loss: 1.1535 | Train Loss Unweighted: 0.5811 | Train Loss Components: Auxiliary: 0.2469 | Cardinality: 0.1254 | Classification: 0.1220 | Classification_binary: 0.1661 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1442 | Regression_eta: 0.0021 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0909 | Regression_energy: 0.1007 | Regression_eta: 0.0666 | Regression_pt: 0.1026 | Regression_sin_phi: 0.1148 | LR: 4.46e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5816s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6905 | max_loss: 0.2072 | max_backward: 1.6876 | max_optimizer_step: 0.2353 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 7397.5 | uss_MB: 7064.1 | child_rss_MB: 4802.4 | allocated_MB: 343.5 | reserved_MB: 26000.0 | inactive_split_MB: 234.1 +rank5 [2026-10-29 07:30:12,152] INFO : Step 28800/50000 rank5 | Train Loss: 1.2131 | Train Loss Unweighted: 0.5739 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1788 | Classification: 0.1057 | Classification_binary: 0.2127 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.2309 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0584 | Regression_eta: 0.0408 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0672 | LR: 4.44e-04 | DataLoad Time: 0.0102s | Model Forward Time: 6.8422s | Phase Times: max_pre_step_barrier: 0.0263 | max_batch_to_device: 0.0149 | max_forward: 1.2928 | max_loss: 0.2713 | max_backward: 3.0161 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.8895 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 7398.3 | uss_MB: 5722.3 | child_rss_MB: 4796.9 | allocated_MB: 343.8 | reserved_MB: 26000.0 | inactive_split_MB: 273.3 +rank5 [2026-10-29 07:34:11,570] INFO : Step 28900/50000 rank5 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1252 | Classification: 0.0967 | Classification_binary: 0.1637 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1509 | Regression_eta: 0.0015 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0639 | Regression_eta: 0.0444 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0892 | LR: 4.41e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.1514s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7512 | max_loss: 0.2189 | max_backward: 1.6229 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.9330 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7399.6 | uss_MB: 5634.1 | child_rss_MB: 4800.1 | allocated_MB: 339.6 | reserved_MB: 26000.0 | inactive_split_MB: 223.0 +rank5 [2026-10-29 07:38:06,079] INFO : Step 29000/50000 rank5 | Train Loss: 1.0238 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1070 | Classification: 0.0861 | Classification_binary: 0.1439 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1389 | Regression_eta: 0.0014 | Regression_pt: 0.1414 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1214 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0655 | Regression_eta: 0.0352 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0553 | LR: 4.38e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.9822s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.6636 | max_loss: 0.2429 | max_backward: 1.5394 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.5216 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7400.5 | uss_MB: 7067.2 | child_rss_MB: 4794.8 | allocated_MB: 345.2 | reserved_MB: 26984.0 | inactive_split_MB: 235.2 +rank5 [2026-10-29 07:38:07,029] INFO : Running validation on rank5 +rank5 [2026-10-29 07:38:07,029] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 07:38:07,030] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 07:38:57,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:38:58,450] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 07:39:24,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 07:39:25,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 07:39:26,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 07:39:27,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 07:39:37,640] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 07:39:38,026] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:39:38,576] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 07:39:54,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 07:39:55,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 07:39:56,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 07:39:57,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 07:40:07,509] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 07:40:07,780] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:40:08,277] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 07:40:23,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 07:40:24,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 07:40:25,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 07:40:25,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 07:40:35,960] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 07:45:01,997] INFO : Step 29100/50000 rank5 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5808 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1279 | Classification: 0.1113 | Classification_binary: 0.1973 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1339 | Regression_eta: 0.0017 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.2086 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0597 | Regression_eta: 0.0380 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0649 | LR: 4.35e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8721s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.8060 | max_loss: 0.1503 | max_backward: 1.9210 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 8379.4 | uss_MB: 8043.2 | child_rss_MB: 4787.2 | allocated_MB: 344.6 | reserved_MB: 24404.0 | inactive_split_MB: 274.9 +rank5 [2026-10-29 07:48:58,055] INFO : Step 29200/50000 rank5 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5633 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1265 | Classification: 0.0969 | Classification_binary: 0.1883 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1368 | Regression_eta: 0.0014 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2089 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0734 | Regression_eta: 0.0457 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0722 | LR: 4.33e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.8375s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0742 | max_forward: 1.0303 | max_loss: 0.2183 | max_backward: 1.8745 | max_optimizer_step: 0.2501 | max_tensorboard_logging: 0.5570 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8379.4 | uss_MB: 8042.7 | child_rss_MB: 4798.0 | allocated_MB: 343.1 | reserved_MB: 24404.0 | inactive_split_MB: 228.1 +rank5 [2026-10-29 07:52:51,792] INFO : Step 29300/50000 rank5 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2330 | Cardinality: 0.1231 | Classification: 0.0898 | Classification_binary: 0.1516 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1428 | Regression_eta: 0.0017 | Regression_pt: 0.1453 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0692 | Regression_eta: 0.0671 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0859 | LR: 4.30e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.7360s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0067 | max_forward: 0.7139 | max_loss: 0.1578 | max_backward: 1.5096 | max_optimizer_step: 0.2586 | max_tensorboard_logging: 1.1324 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7473.7 | uss_MB: 6607.6 | child_rss_MB: 4797.1 | allocated_MB: 341.0 | reserved_MB: 24404.0 | inactive_split_MB: 247.0 +rank5 [2026-10-29 07:56:44,367] INFO : Step 29400/50000 rank5 | Train Loss: 1.1400 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1377 | Classification: 0.1135 | Classification_binary: 0.1661 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1386 | Regression_eta: 0.0019 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0847 | Regression_energy: 0.0602 | Regression_eta: 0.0475 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0756 | LR: 4.27e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.2880s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.6285 | max_loss: 0.1485 | max_backward: 1.3787 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.9179 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7474.7 | uss_MB: 7139.2 | child_rss_MB: 4788.9 | allocated_MB: 343.3 | reserved_MB: 24404.0 | inactive_split_MB: 277.1 +rank5 [2026-10-29 08:00:58,645] INFO : Step 29500/50000 rank5 | Train Loss: 1.1987 | Train Loss Unweighted: 0.6106 | Train Loss Components: Auxiliary: 0.2551 | Cardinality: 0.1277 | Classification: 0.1201 | Classification_binary: 0.2013 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1419 | Regression_eta: 0.0019 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0925 | Regression_energy: 0.0609 | Regression_eta: 0.0717 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0817 | LR: 4.24e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.8985s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0749 | max_forward: 0.8153 | max_loss: 0.3465 | max_backward: 2.8565 | max_optimizer_step: 0.3442 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7477.7 | uss_MB: 7142.6 | child_rss_MB: 4792.1 | allocated_MB: 343.9 | reserved_MB: 24404.0 | inactive_split_MB: 271.5 +rank5 [2026-10-29 08:05:08,852] INFO : Step 29600/50000 rank5 | Train Loss: 1.1065 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2391 | Cardinality: 0.1179 | Classification: 0.0927 | Classification_binary: 0.1666 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0018 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1752 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0754 | Regression_eta: 0.0471 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0639 | LR: 4.22e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4748s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8025 | max_loss: 0.1485 | max_backward: 1.4674 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.8982 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 7476.5 | uss_MB: 7138.1 | child_rss_MB: 4790.4 | allocated_MB: 340.8 | reserved_MB: 24404.0 | inactive_split_MB: 290.7 +rank5 [2026-10-29 08:09:13,801] INFO : Step 29700/50000 rank5 | Train Loss: 1.0488 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2273 | Cardinality: 0.1414 | Classification: 0.0952 | Classification_binary: 0.1459 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1387 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0641 | Regression_eta: 0.0407 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0455 | LR: 4.19e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.5184s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0057 | max_forward: 0.7342 | max_loss: 0.1652 | max_backward: 1.4907 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.9743 | max_tensorboard_flush: 0.0140 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7477.5 | uss_MB: 6998.2 | child_rss_MB: 4799.0 | allocated_MB: 342.3 | reserved_MB: 24404.0 | inactive_split_MB: 236.6 +rank5 [2026-10-29 08:13:32,079] INFO : Step 29800/50000 rank5 | Train Loss: 1.1686 | Train Loss Unweighted: 0.5644 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1551 | Classification: 0.0952 | Classification_binary: 0.1922 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1357 | Regression_eta: 0.0014 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1807 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0648 | Regression_eta: 0.0353 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0609 | LR: 4.16e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6781s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7294 | max_loss: 0.1685 | max_backward: 1.6547 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9409 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7478.5 | uss_MB: 7143.5 | child_rss_MB: 4794.4 | allocated_MB: 345.0 | reserved_MB: 24404.0 | inactive_split_MB: 265.1 +rank5 [2026-10-29 08:17:37,375] INFO : Step 29900/50000 rank5 | Train Loss: 1.1305 | Train Loss Unweighted: 0.5666 | Train Loss Components: Auxiliary: 0.2471 | Cardinality: 0.1380 | Classification: 0.0944 | Classification_binary: 0.1750 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1464 | Regression_eta: 0.0015 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1767 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0646 | Regression_eta: 0.0355 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0699 | LR: 4.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3505s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0611 | max_forward: 0.8060 | max_loss: 0.1494 | max_backward: 1.3251 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 7479.5 | uss_MB: 7143.0 | child_rss_MB: 4800.7 | allocated_MB: 344.7 | reserved_MB: 26422.0 | inactive_split_MB: 298.4 +rank5 [2026-10-29 08:22:04,988] INFO : Step 30000/50000 rank5 | Train Loss: 1.0854 | Train Loss Unweighted: 0.5586 | Train Loss Components: Auxiliary: 0.2416 | Cardinality: 0.1337 | Classification: 0.1000 | Classification_binary: 0.1541 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1499 | Regression_eta: 0.0014 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0334 | Regression_energy: 0.0738 | Regression_eta: 0.0421 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0608 | LR: 4.11e-04 | DataLoad Time: 0.0205s | Model Forward Time: 7.0733s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0172 | max_forward: 0.9685 | max_loss: 0.2166 | max_backward: 4.0112 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.8430 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7461.9 | uss_MB: 7142.5 | child_rss_MB: 4797.1 | allocated_MB: 338.0 | reserved_MB: 26422.0 | inactive_split_MB: 288.0 +rank5 [2026-10-29 08:22:05,952] INFO : Running validation on rank5 +rank5 [2026-10-29 08:22:05,952] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 08:22:05,952] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 08:22:57,875] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:22:58,360] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 08:23:24,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 08:23:25,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 08:23:26,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 08:23:28,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 08:23:38,317] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 08:23:38,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:23:38,903] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 08:23:55,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 08:23:55,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 08:23:56,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 08:23:57,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 08:24:07,828] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 08:24:08,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:24:08,615] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 08:24:23,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 08:24:24,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 08:24:25,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 08:24:26,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 08:24:36,277] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 08:29:00,406] INFO : Step 30100/50000 rank5 | Train Loss: 1.0762 | Train Loss Unweighted: 0.5358 | Train Loss Components: Auxiliary: 0.2378 | Cardinality: 0.1106 | Classification: 0.1097 | Classification_binary: 0.1679 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0404 | Regression_energy: 0.0507 | Regression_eta: 0.0291 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0342 | LR: 4.08e-04 | DataLoad Time: 0.0167s | Model Forward Time: 8.2919s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7352 | max_loss: 0.1443 | max_backward: 1.7375 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 1.7731 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.3497 | Memory: rss_MB: 7528.6 | uss_MB: 6331.6 | child_rss_MB: 4801.2 | allocated_MB: 344.1 | reserved_MB: 22922.0 | inactive_split_MB: 248.8 +rank5 [2026-10-29 08:33:16,461] INFO : Step 30200/50000 rank5 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5304 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1421 | Classification: 0.0934 | Classification_binary: 0.1839 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0016 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.2006 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0663 | Regression_eta: 0.0401 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0743 | LR: 4.06e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6007s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8008 | max_loss: 0.1560 | max_backward: 1.6706 | max_optimizer_step: 0.2663 | max_tensorboard_logging: 0.7725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0555 | Memory: rss_MB: 7535.6 | uss_MB: 7212.9 | child_rss_MB: 4775.8 | allocated_MB: 344.1 | reserved_MB: 23882.0 | inactive_split_MB: 274.8 +rank5 [2026-10-29 08:37:30,482] INFO : Step 30300/50000 rank5 | Train Loss: 1.0429 | Train Loss Unweighted: 0.5252 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1223 | Classification: 0.0922 | Classification_binary: 0.1566 | HitGrouping: 0.1929 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0893 | Regression_eta: 0.0254 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0616 | LR: 4.03e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.6262s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0060 | max_forward: 1.1719 | max_loss: 0.2884 | max_backward: 2.4994 | max_optimizer_step: 0.2255 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7536.6 | uss_MB: 7213.9 | child_rss_MB: 4771.6 | allocated_MB: 341.8 | reserved_MB: 23882.0 | inactive_split_MB: 270.7 +rank5 [2026-10-29 08:41:40,106] INFO : Step 30400/50000 rank5 | Train Loss: 1.0543 | Train Loss Unweighted: 0.5182 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1318 | Classification: 0.0889 | Classification_binary: 0.1487 | HitGrouping: 0.1973 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1382 | Regression_eta: 0.0014 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1205 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0658 | Regression_eta: 0.0242 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0492 | LR: 4.00e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.0685s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0150 | max_forward: 0.8845 | max_loss: 0.2482 | max_backward: 2.1745 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.5112 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7452.2 | uss_MB: 7214.6 | child_rss_MB: 4475.9 | allocated_MB: 344.7 | reserved_MB: 24842.0 | inactive_split_MB: 245.3 +rank5 [2026-10-29 08:45:49,048] INFO : Step 30500/50000 rank5 | Train Loss: 1.1589 | Train Loss Unweighted: 0.5596 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1407 | Classification: 0.1085 | Classification_binary: 0.1952 | HitGrouping: 0.1960 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0907 | Classification_binary: 0.1912 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0625 | Regression_eta: 0.0483 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0597 | LR: 3.98e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.7054s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7791 | max_loss: 0.2528 | max_backward: 1.9433 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.5210 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0672 | Memory: rss_MB: 7432.7 | uss_MB: 7008.9 | child_rss_MB: 4374.7 | allocated_MB: 346.2 | reserved_MB: 26806.0 | inactive_split_MB: 227.8 +rank5 [2026-10-29 08:49:58,495] INFO : Step 30600/50000 rank5 | Train Loss: 1.1269 | Train Loss Unweighted: 0.5561 | Train Loss Components: Auxiliary: 0.2414 | Cardinality: 0.1456 | Classification: 0.0973 | Classification_binary: 0.1804 | HitGrouping: 0.1953 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1373 | Regression_eta: 0.0014 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1853 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0712 | Regression_eta: 0.0408 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0640 | LR: 3.95e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4788s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1011 | max_forward: 0.8824 | max_loss: 0.1534 | max_backward: 1.4258 | max_optimizer_step: 0.3067 | max_tensorboard_logging: 0.7925 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 7378.2 | uss_MB: 7215.1 | child_rss_MB: 4177.7 | allocated_MB: 345.8 | reserved_MB: 27800.0 | inactive_split_MB: 283.5 +rank5 [2026-10-29 08:54:17,062] INFO : Step 30700/50000 rank5 | Train Loss: 1.0989 | Train Loss Unweighted: 0.5515 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1377 | Classification: 0.0890 | Classification_binary: 0.1568 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0013 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1584 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0632 | Regression_eta: 0.0408 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0796 | LR: 3.92e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2933s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0056 | max_forward: 0.7546 | max_loss: 0.2396 | max_backward: 1.3389 | max_optimizer_step: 0.2508 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0572 | Memory: rss_MB: 7380.2 | uss_MB: 7216.5 | child_rss_MB: 4178.9 | allocated_MB: 340.3 | reserved_MB: 27800.0 | inactive_split_MB: 223.2 +rank5 [2026-10-29 08:58:31,928] INFO : Step 30800/50000 rank5 | Train Loss: 1.1241 | Train Loss Unweighted: 0.5488 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1377 | Classification: 0.1081 | Classification_binary: 0.1717 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1317 | Regression_eta: 0.0015 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0649 | Regression_eta: 0.0389 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0661 | LR: 3.90e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.8018s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7775 | max_loss: 0.2735 | max_backward: 1.9298 | max_optimizer_step: 0.3489 | max_tensorboard_logging: 0.8756 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0726 | Memory: rss_MB: 7357.4 | uss_MB: 5945.6 | child_rss_MB: 4116.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 275.1 +rank5 [2026-10-29 09:02:40,910] INFO : Step 30900/50000 rank5 | Train Loss: 1.1501 | Train Loss Unweighted: 0.5762 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1300 | Classification: 0.0976 | Classification_binary: 0.1996 | HitGrouping: 0.2000 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1374 | Regression_eta: 0.0017 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1903 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0655 | Regression_eta: 0.0425 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0478 | LR: 3.87e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.5938s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.3511 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 1.0306 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 7360.4 | uss_MB: 7217.8 | child_rss_MB: 4118.5 | allocated_MB: 342.2 | reserved_MB: 27800.0 | inactive_split_MB: 198.3 +rank5 [2026-10-29 09:06:46,925] INFO : Step 31000/50000 rank5 | Train Loss: 1.1054 | Train Loss Unweighted: 0.5218 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1446 | Classification: 0.0888 | Classification_binary: 0.1693 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0016 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1866 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0577 | Regression_eta: 0.0341 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0576 | LR: 3.84e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.7155s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.8505 | max_loss: 0.2491 | max_backward: 1.6460 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9731 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7337.9 | uss_MB: 6674.8 | child_rss_MB: 4046.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 190.7 +rank5 [2026-10-29 09:06:47,902] INFO : Running validation on rank5 +rank5 [2026-10-29 09:06:47,903] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 09:06:47,903] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 09:07:39,008] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:07:39,556] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 09:08:08,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 09:08:09,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 09:08:10,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 09:08:11,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 09:08:21,720] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 09:08:22,017] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:08:22,479] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 09:08:39,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 09:08:40,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 09:08:41,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 09:08:42,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 09:08:52,414] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 09:08:52,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:08:53,433] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 09:09:09,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 09:09:09,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 09:09:10,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 09:09:11,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 09:09:21,024] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 09:13:45,042] INFO : Step 31100/50000 rank5 | Train Loss: 1.0306 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2182 | Cardinality: 0.1238 | Classification: 0.0932 | Classification_binary: 0.1415 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1315 | Regression_cos_phi: 0.0339 | Regression_energy: 0.0582 | Regression_eta: 0.0277 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0351 | LR: 3.82e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.5109s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1068 | max_forward: 0.8067 | max_loss: 0.1436 | max_backward: 1.1427 | max_optimizer_step: 0.2576 | max_tensorboard_logging: 1.1475 | max_tensorboard_flush: 0.0054 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 8196.3 | uss_MB: 5921.9 | child_rss_MB: 4032.2 | allocated_MB: 343.8 | reserved_MB: 24492.0 | inactive_split_MB: 293.3 +rank5 [2026-10-29 09:17:44,912] INFO : Step 31200/50000 rank5 | Train Loss: 1.1102 | Train Loss Unweighted: 0.5369 | Train Loss Components: Auxiliary: 0.2472 | Cardinality: 0.1495 | Classification: 0.0959 | Classification_binary: 0.1708 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1328 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1903 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0679 | Regression_eta: 0.0379 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0571 | LR: 3.79e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.1839s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7127 | max_loss: 0.1476 | max_backward: 1.5707 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7428.7 | uss_MB: 7313.8 | child_rss_MB: 4003.5 | allocated_MB: 344.5 | reserved_MB: 24648.0 | inactive_split_MB: 235.5 +rank5 [2026-10-29 09:21:47,087] INFO : Step 31300/50000 rank5 | Train Loss: 1.1865 | Train Loss Unweighted: 0.5472 | Train Loss Components: Auxiliary: 0.2511 | Cardinality: 0.1748 | Classification: 0.0979 | Classification_binary: 0.1851 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0019 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1663 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0586 | Regression_eta: 0.0507 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0566 | LR: 3.76e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.2331s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0064 | max_forward: 0.7815 | max_loss: 0.1495 | max_backward: 1.6330 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.5234 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7366.1 | uss_MB: 7313.9 | child_rss_MB: 3706.8 | allocated_MB: 344.4 | reserved_MB: 25640.0 | inactive_split_MB: 233.6 +rank5 [2026-10-29 09:25:42,888] INFO : Step 31400/50000 rank5 | Train Loss: 1.0272 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1191 | Classification: 0.0784 | Classification_binary: 0.1451 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0659 | Regression_eta: 0.0336 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0812 | LR: 3.74e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.5260s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0063 | max_forward: 0.7252 | max_loss: 0.1478 | max_backward: 1.6173 | max_optimizer_step: 0.2404 | max_tensorboard_logging: 0.9541 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7335.5 | uss_MB: 7312.2 | child_rss_MB: 3625.5 | allocated_MB: 342.1 | reserved_MB: 25640.0 | inactive_split_MB: 296.4 +rank5 [2026-10-29 09:29:44,710] INFO : Step 31500/50000 rank5 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5247 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1206 | Classification: 0.1037 | Classification_binary: 0.1583 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1297 | Regression_eta: 0.0014 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0375 | Regression_energy: 0.0631 | Regression_eta: 0.0366 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0497 | LR: 3.71e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.3651s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6643 | max_loss: 0.2487 | max_backward: 1.5648 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.8248 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7336.0 | uss_MB: 7313.0 | child_rss_MB: 3638.4 | allocated_MB: 344.4 | reserved_MB: 26092.0 | inactive_split_MB: 238.2 +rank5 [2026-10-29 09:33:42,077] INFO : Step 31600/50000 rank5 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5471 | Train Loss Components: Auxiliary: 0.2424 | Cardinality: 0.1284 | Classification: 0.1033 | Classification_binary: 0.1828 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1282 | Regression_eta: 0.0017 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0579 | Regression_eta: 0.0347 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0456 | LR: 3.69e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0240s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7603 | max_loss: 0.1471 | max_backward: 2.0475 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.9076 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7328.4 | uss_MB: 7314.4 | child_rss_MB: 3585.6 | allocated_MB: 341.5 | reserved_MB: 26640.0 | inactive_split_MB: 287.0 +rank5 [2026-10-29 09:37:47,825] INFO : Step 31700/50000 rank5 | Train Loss: 1.0533 | Train Loss Unweighted: 0.5009 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1416 | Classification: 0.0871 | Classification_binary: 0.1632 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1854 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0669 | Regression_eta: 0.0427 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0443 | LR: 3.66e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.1118s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7635 | max_loss: 0.1543 | max_backward: 1.2075 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7330.4 | uss_MB: 7312.9 | child_rss_MB: 3557.0 | allocated_MB: 341.9 | reserved_MB: 26640.0 | inactive_split_MB: 274.1 +rank5 [2026-10-29 09:45:40,561] INFO : Step 31800/50000 rank5 | Train Loss: 1.0385 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1148 | Classification: 0.0877 | Classification_binary: 0.1481 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1418 | Regression_eta: 0.0012 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0540 | Regression_eta: 0.0401 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0576 | LR: 3.64e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0346s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0072 | max_forward: 0.6606 | max_loss: 0.1610 | max_backward: 1.1159 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.8258 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1417 | Memory: rss_MB: 7328.9 | uss_MB: 7312.1 | child_rss_MB: 3561.3 | allocated_MB: 345.5 | reserved_MB: 27080.0 | inactive_split_MB: 260.5 +rank5 [2026-10-29 09:49:44,090] INFO : Step 31900/50000 rank5 | Train Loss: 1.1464 | Train Loss Unweighted: 0.5585 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1372 | Classification: 0.1078 | Classification_binary: 0.1815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0017 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.1830 | Regression_cos_phi: 0.1022 | Regression_energy: 0.0602 | Regression_eta: 0.0721 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0957 | LR: 3.61e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5149s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7718 | max_loss: 0.1553 | max_backward: 1.7857 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.6721 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7333.9 | uss_MB: 7320.8 | child_rss_MB: 3569.3 | allocated_MB: 343.8 | reserved_MB: 27080.0 | inactive_split_MB: 280.2 +rank5 [2026-10-29 09:53:38,293] INFO : Step 32000/50000 rank5 | Train Loss: 1.1115 | Train Loss Unweighted: 0.5351 | Train Loss Components: Auxiliary: 0.2393 | Cardinality: 0.1406 | Classification: 0.0954 | Classification_binary: 0.1748 | HitGrouping: 0.1912 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1302 | Regression_eta: 0.0018 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1823 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0590 | Regression_eta: 0.0433 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0658 | LR: 3.58e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.2441s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.9767 | max_loss: 0.6919 | max_backward: 2.1955 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.9277 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 7329.3 | uss_MB: 7313.9 | child_rss_MB: 3566.4 | allocated_MB: 344.2 | reserved_MB: 27080.0 | inactive_split_MB: 265.7 +rank5 [2026-10-29 09:53:39,352] INFO : Running validation on rank5 +rank5 [2026-10-29 09:53:39,352] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 09:53:39,352] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 09:54:28,723] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:54:29,292] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 09:54:57,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 09:54:58,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 09:54:59,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 09:55:00,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 09:55:10,612] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 09:55:10,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:55:11,414] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 09:55:27,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 09:55:28,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 09:55:29,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 09:55:30,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 09:55:40,453] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 09:55:41,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:55:41,777] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 09:55:56,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 09:55:57,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 09:55:58,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 09:55:58,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 09:56:08,897] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 10:00:19,239] INFO : Step 32100/50000 rank5 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1218 | Classification: 0.0914 | Classification_binary: 0.1509 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1333 | Regression_eta: 0.0016 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0492 | Regression_eta: 0.0444 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0610 | LR: 3.56e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.3628s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7098 | max_loss: 0.1454 | max_backward: 1.1394 | max_optimizer_step: 0.2650 | max_tensorboard_logging: 0.9832 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 8404.9 | uss_MB: 8386.6 | child_rss_MB: 3573.9 | allocated_MB: 344.0 | reserved_MB: 22148.0 | inactive_split_MB: 276.8 +rank5 [2026-10-29 10:04:33,375] INFO : Step 32200/50000 rank5 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1241 | Classification: 0.1035 | Classification_binary: 0.1630 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1279 | Regression_eta: 0.0014 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0611 | Regression_eta: 0.0345 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0585 | LR: 3.53e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6699s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0891 | max_forward: 0.7517 | max_loss: 0.1441 | max_backward: 1.7835 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.7808 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7418.2 | uss_MB: 7403.1 | child_rss_MB: 3576.2 | allocated_MB: 345.5 | reserved_MB: 23122.0 | inactive_split_MB: 233.4 +rank5 [2026-10-29 10:08:31,626] INFO : Step 32300/50000 rank5 | Train Loss: 1.1438 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2535 | Cardinality: 0.1335 | Classification: 0.1042 | Classification_binary: 0.1958 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1303 | Regression_eta: 0.0015 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1810 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0543 | Regression_eta: 0.0443 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0741 | LR: 3.51e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7837s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7857 | max_loss: 0.1780 | max_backward: 1.7580 | max_optimizer_step: 0.2428 | max_tensorboard_logging: 0.8954 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 7421.3 | uss_MB: 7404.5 | child_rss_MB: 3576.8 | allocated_MB: 343.9 | reserved_MB: 24108.0 | inactive_split_MB: 280.1 +rank5 [2026-10-29 10:12:37,147] INFO : Step 32400/50000 rank5 | Train Loss: 1.0306 | Train Loss Unweighted: 0.5063 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1168 | Classification: 0.0867 | Classification_binary: 0.1599 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0013 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1624 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0647 | Regression_eta: 0.0350 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0376 | LR: 3.48e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.3208s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.1456 | max_backward: 1.3401 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.8752 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7420.4 | uss_MB: 7401.9 | child_rss_MB: 3574.4 | allocated_MB: 344.8 | reserved_MB: 24108.0 | inactive_split_MB: 271.0 +rank5 [2026-10-29 10:16:48,486] INFO : Step 32500/50000 rank5 | Train Loss: 1.0033 | Train Loss Unweighted: 0.5126 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.0995 | Classification: 0.1015 | Classification_binary: 0.1402 | HitGrouping: 0.1898 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1332 | Regression_eta: 0.0015 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0847 | Regression_energy: 0.0789 | Regression_eta: 0.0716 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0733 | LR: 3.46e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9962s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6088 | max_loss: 0.1480 | max_backward: 1.0729 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.8045 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7420.4 | uss_MB: 7401.8 | child_rss_MB: 3575.0 | allocated_MB: 341.9 | reserved_MB: 25096.0 | inactive_split_MB: 268.1 +rank5 [2026-10-29 10:21:04,066] INFO : Step 32600/50000 rank5 | Train Loss: 1.0981 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1271 | Classification: 0.1053 | Classification_binary: 0.1773 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1306 | Regression_eta: 0.0014 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.1783 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0600 | Regression_eta: 0.0392 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0864 | LR: 3.43e-04 | DataLoad Time: 0.0142s | Model Forward Time: 6.2981s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8529 | max_loss: 0.3369 | max_backward: 2.9831 | max_optimizer_step: 0.2386 | max_tensorboard_logging: 0.5557 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7420.7 | uss_MB: 5748.3 | child_rss_MB: 3585.6 | allocated_MB: 344.1 | reserved_MB: 25096.0 | inactive_split_MB: 287.9 +rank5 [2026-10-29 10:25:23,205] INFO : Step 32700/50000 rank5 | Train Loss: 1.0956 | Train Loss Unweighted: 0.5307 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.1282 | Classification: 0.1011 | Classification_binary: 0.1731 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1877 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0592 | Regression_eta: 0.0649 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0788 | LR: 3.41e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.1099s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0062 | max_forward: 0.8233 | max_loss: 0.1478 | max_backward: 1.5120 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0564 | Memory: rss_MB: 7420.7 | uss_MB: 7403.4 | child_rss_MB: 3588.8 | allocated_MB: 342.9 | reserved_MB: 25096.0 | inactive_split_MB: 272.6 +rank5 [2026-10-29 10:29:23,499] INFO : Step 32800/50000 rank5 | Train Loss: 1.0857 | Train Loss Unweighted: 0.5241 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1299 | Classification: 0.1011 | Classification_binary: 0.1551 | HitGrouping: 0.1982 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1327 | Regression_eta: 0.0016 | Regression_pt: 0.1319 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0409 | Regression_eta: 0.0430 | Regression_pt: 0.0433 | Regression_sin_phi: 0.0730 | LR: 3.38e-04 | DataLoad Time: 0.0244s | Model Forward Time: 4.2372s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0179 | max_forward: 0.9787 | max_loss: 0.1782 | max_backward: 1.5075 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 0.5808 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.1577 | Memory: rss_MB: 7419.5 | uss_MB: 7401.5 | child_rss_MB: 3585.9 | allocated_MB: 345.1 | reserved_MB: 25096.0 | inactive_split_MB: 263.9 +rank5 [2026-10-29 10:33:26,397] INFO : Step 32900/50000 rank5 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1116 | Classification: 0.1066 | Classification_binary: 0.1631 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1300 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0552 | Regression_eta: 0.0439 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0919 | LR: 3.36e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.6991s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 1.3284 | max_loss: 0.5826 | max_backward: 1.9391 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7423.7 | uss_MB: 7407.1 | child_rss_MB: 3589.0 | allocated_MB: 344.3 | reserved_MB: 26086.0 | inactive_split_MB: 212.1 +rank5 [2026-10-29 10:37:13,221] INFO : Step 33000/50000 rank5 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5373 | Train Loss Components: Auxiliary: 0.2310 | Cardinality: 0.1017 | Classification: 0.1099 | Classification_binary: 0.1844 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0668 | Regression_eta: 0.0506 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0543 | LR: 3.33e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.3114s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 0.7919 | max_loss: 0.2231 | max_backward: 1.5724 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.5025 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1340 | Memory: rss_MB: 7423.5 | uss_MB: 7406.2 | child_rss_MB: 3581.4 | allocated_MB: 344.2 | reserved_MB: 26086.0 | inactive_split_MB: 269.8 +rank5 [2026-10-29 10:37:14,208] INFO : Running validation on rank5 +rank5 [2026-10-29 10:37:14,208] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 10:37:14,208] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 10:38:03,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:38:04,389] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 10:38:31,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 10:38:32,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 10:38:34,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 10:38:35,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 10:38:45,164] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 10:38:45,833] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:38:46,345] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 10:39:02,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 10:39:03,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 10:39:04,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 10:39:05,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 10:39:15,017] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 10:39:15,792] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:39:16,280] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 10:39:31,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 10:39:32,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 10:39:32,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 10:39:33,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 10:39:43,700] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 10:43:56,310] INFO : Step 33100/50000 rank5 | Train Loss: 1.1040 | Train Loss Unweighted: 0.5268 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1427 | Classification: 0.0921 | Classification_binary: 0.1655 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1319 | Regression_eta: 0.0015 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0803 | Regression_eta: 0.0460 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0441 | LR: 3.31e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3481s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0325 | max_forward: 0.7755 | max_loss: 0.1478 | max_backward: 1.3916 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.6610 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0760 | Memory: rss_MB: 8784.9 | uss_MB: 8766.5 | child_rss_MB: 3557.4 | allocated_MB: 343.8 | reserved_MB: 23356.0 | inactive_split_MB: 267.3 +rank5 [2026-10-29 10:47:56,312] INFO : Step 33200/50000 rank5 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4955 | Train Loss Components: Auxiliary: 0.2164 | Cardinality: 0.1253 | Classification: 0.1001 | Classification_binary: 0.1407 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0019 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0573 | Regression_eta: 0.0311 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0544 | LR: 3.28e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.0980s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0401 | max_forward: 0.9954 | max_loss: 0.3116 | max_backward: 2.2139 | max_optimizer_step: 0.2657 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7500.4 | uss_MB: 7483.0 | child_rss_MB: 3560.3 | allocated_MB: 344.1 | reserved_MB: 23356.0 | inactive_split_MB: 241.6 +rank5 [2026-10-29 10:51:55,085] INFO : Step 33300/50000 rank5 | Train Loss: 1.0288 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2302 | Cardinality: 0.1166 | Classification: 0.0975 | Classification_binary: 0.1653 | HitGrouping: 0.1885 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1220 | Regression_eta: 0.0012 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0525 | Regression_eta: 0.0435 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0588 | LR: 3.26e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5488s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7054 | max_loss: 0.2213 | max_backward: 1.6633 | max_optimizer_step: 0.2474 | max_tensorboard_logging: 0.8237 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0708 | Memory: rss_MB: 7500.4 | uss_MB: 7483.1 | child_rss_MB: 3556.5 | allocated_MB: 343.5 | reserved_MB: 23356.0 | inactive_split_MB: 261.9 +rank5 [2026-10-29 10:55:54,910] INFO : Step 33400/50000 rank5 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5337 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1249 | Classification: 0.0970 | Classification_binary: 0.1720 | HitGrouping: 0.1947 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0013 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0708 | Regression_eta: 0.0342 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0578 | LR: 3.23e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.4584s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0073 | max_forward: 1.2444 | max_loss: 0.2025 | max_backward: 2.5109 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7502.5 | uss_MB: 7485.1 | child_rss_MB: 3549.5 | allocated_MB: 347.6 | reserved_MB: 23358.0 | inactive_split_MB: 303.0 +rank5 [2026-10-29 10:59:53,594] INFO : Step 33500/50000 rank5 | Train Loss: 1.0164 | Train Loss Unweighted: 0.4956 | Train Loss Components: Auxiliary: 0.2212 | Cardinality: 0.1274 | Classification: 0.0875 | Classification_binary: 0.1421 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1309 | Regression_eta: 0.0013 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1269 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0495 | Regression_eta: 0.0373 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0463 | LR: 3.21e-04 | DataLoad Time: 0.0120s | Model Forward Time: 4.6230s | Phase Times: max_pre_step_barrier: 0.0271 | max_batch_to_device: 0.0558 | max_forward: 0.9437 | max_loss: 0.1437 | max_backward: 1.5429 | max_optimizer_step: 0.2447 | max_tensorboard_logging: 0.8833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7505.9 | uss_MB: 7486.7 | child_rss_MB: 3558.2 | allocated_MB: 341.2 | reserved_MB: 23358.0 | inactive_split_MB: 237.5 +rank5 [2026-10-29 11:05:22,801] INFO : Step 33600/50000 rank5 | Train Loss: 1.0074 | Train Loss Unweighted: 0.4988 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1109 | Classification: 0.0969 | Classification_binary: 0.1509 | HitGrouping: 0.1901 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0012 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0596 | Regression_eta: 0.0394 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0520 | LR: 3.19e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1870s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0970 | max_forward: 0.7667 | max_loss: 0.1478 | max_backward: 1.4192 | max_optimizer_step: 0.2426 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7500.7 | uss_MB: 7485.1 | child_rss_MB: 3550.7 | allocated_MB: 344.5 | reserved_MB: 24528.0 | inactive_split_MB: 238.9 +rank5 [2026-10-29 11:09:17,748] INFO : Step 33700/50000 rank5 | Train Loss: 1.1891 | Train Loss Unweighted: 0.5658 | Train Loss Components: Auxiliary: 0.2602 | Cardinality: 0.1587 | Classification: 0.1116 | Classification_binary: 0.1953 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1270 | Regression_eta: 0.0015 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.1991 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0637 | Regression_eta: 0.0317 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0574 | LR: 3.16e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8107s | Phase Times: max_pre_step_barrier: 0.0253 | max_batch_to_device: 0.0066 | max_forward: 1.3051 | max_loss: 0.1764 | max_backward: 1.7331 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.5706 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0895 | Memory: rss_MB: 7502.0 | uss_MB: 7487.9 | child_rss_MB: 3548.7 | allocated_MB: 347.7 | reserved_MB: 24528.0 | inactive_split_MB: 288.7 +rank5 [2026-10-29 11:13:14,783] INFO : Step 33800/50000 rank5 | Train Loss: 1.0442 | Train Loss Unweighted: 0.4970 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1421 | Classification: 0.0764 | Classification_binary: 0.1525 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1318 | Regression_eta: 0.0014 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1584 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0530 | Regression_eta: 0.0749 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0728 | LR: 3.14e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2982s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7370 | max_loss: 0.2458 | max_backward: 1.5902 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.6292 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1331 | Memory: rss_MB: 7496.1 | uss_MB: 7482.2 | child_rss_MB: 3539.4 | allocated_MB: 344.2 | reserved_MB: 24528.0 | inactive_split_MB: 271.5 +rank5 [2026-10-29 11:17:10,799] INFO : Step 33900/50000 rank5 | Train Loss: 1.0341 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1352 | Classification: 0.0865 | Classification_binary: 0.1441 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1254 | Regression_eta: 0.0016 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0521 | Regression_eta: 0.0293 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0445 | LR: 3.11e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3833s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.6305 | max_loss: 0.1507 | max_backward: 1.4015 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9471 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0605 | Memory: rss_MB: 7498.1 | uss_MB: 7488.3 | child_rss_MB: 3543.5 | allocated_MB: 344.1 | reserved_MB: 24528.0 | inactive_split_MB: 268.7 +rank5 [2026-10-29 11:21:05,085] INFO : Step 34000/50000 rank5 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1309 | Classification: 0.0985 | Classification_binary: 0.1707 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1170 | Regression_eta: 0.0014 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0512 | Regression_eta: 0.0646 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0557 | LR: 3.09e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.6366s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7414 | max_loss: 0.1458 | max_backward: 1.7218 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 7499.1 | uss_MB: 7491.4 | child_rss_MB: 3548.4 | allocated_MB: 342.9 | reserved_MB: 24528.0 | inactive_split_MB: 277.5 +rank5 [2026-10-29 11:21:06,163] INFO : Running validation on rank5 +rank5 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 11:21:56,884] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:21:57,379] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 11:22:17,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 11:22:18,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 11:22:20,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 11:22:21,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 11:22:31,897] INFO : Time taken to make predictions on device 5 is: 0.28 min +rank5 [2026-10-29 11:22:32,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:22:32,506] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 11:22:48,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 11:22:49,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 11:22:50,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 11:22:51,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 11:23:01,053] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 11:23:02,011] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:23:02,513] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 11:23:18,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 11:23:18,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 11:23:19,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 11:23:20,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 11:23:30,204] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 11:27:45,672] INFO : Step 34100/50000 rank5 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5582 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1263 | Classification: 0.1031 | Classification_binary: 0.1801 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1358 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1659 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0510 | Regression_eta: 0.0284 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0447 | LR: 3.06e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8608s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9025 | max_loss: 0.1870 | max_backward: 1.7732 | max_optimizer_step: 0.2826 | max_tensorboard_logging: 0.8215 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0738 | Memory: rss_MB: 9109.5 | uss_MB: 8140.4 | child_rss_MB: 3555.5 | allocated_MB: 342.7 | reserved_MB: 22124.0 | inactive_split_MB: 211.6 +rank5 [2026-10-29 11:31:40,882] INFO : Step 34200/50000 rank5 | Train Loss: 0.9931 | Train Loss Unweighted: 0.4794 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1232 | Classification: 0.0771 | Classification_binary: 0.1377 | HitGrouping: 0.1996 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1301 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1252 | Regression_cos_phi: 0.0315 | Regression_energy: 0.0569 | Regression_eta: 0.0235 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0280 | LR: 3.04e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.7178s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.8809 | max_loss: 0.3036 | max_backward: 1.8297 | max_optimizer_step: 0.2897 | max_tensorboard_logging: 0.6835 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7608.6 | uss_MB: 6291.1 | child_rss_MB: 3559.3 | allocated_MB: 339.0 | reserved_MB: 22508.0 | inactive_split_MB: 220.4 +rank5 [2026-10-29 11:35:37,831] INFO : Step 34300/50000 rank5 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4920 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1192 | Classification: 0.0965 | Classification_binary: 0.1484 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.1517 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0594 | Regression_eta: 0.0537 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0748 | LR: 3.02e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3206s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7255 | max_loss: 0.1427 | max_backward: 1.3652 | max_optimizer_step: 0.2496 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7610.6 | uss_MB: 7596.5 | child_rss_MB: 3557.3 | allocated_MB: 345.0 | reserved_MB: 22508.0 | inactive_split_MB: 216.4 +rank5 [2026-10-29 11:39:28,026] INFO : Step 34400/50000 rank5 | Train Loss: 1.0662 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2383 | Cardinality: 0.1289 | Classification: 0.0974 | Classification_binary: 0.1749 | HitGrouping: 0.1864 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1281 | Regression_eta: 0.0014 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0510 | Regression_eta: 0.0290 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0561 | LR: 2.99e-04 | DataLoad Time: 0.0189s | Model Forward Time: 6.1251s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9438 | max_loss: 0.1560 | max_backward: 2.9359 | max_optimizer_step: 0.2526 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7613.7 | uss_MB: 6304.5 | child_rss_MB: 3550.6 | allocated_MB: 345.3 | reserved_MB: 23132.0 | inactive_split_MB: 282.7 +rank5 [2026-10-29 11:43:17,761] INFO : Step 34500/50000 rank5 | Train Loss: 1.0328 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1103 | Classification: 0.0867 | Classification_binary: 0.1518 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0014 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.1372 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0600 | Regression_eta: 0.0332 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0585 | LR: 2.97e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6261s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.8153 | max_loss: 0.1429 | max_backward: 1.8299 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.5315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7615.7 | uss_MB: 6643.2 | child_rss_MB: 3549.5 | allocated_MB: 341.4 | reserved_MB: 23132.0 | inactive_split_MB: 238.6 +rank5 [2026-10-29 11:47:05,943] INFO : Step 34600/50000 rank5 | Train Loss: 1.0317 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1201 | Classification: 0.0877 | Classification_binary: 0.1434 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1359 | Regression_eta: 0.0014 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0795 | Regression_eta: 0.0322 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0617 | LR: 2.95e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2922s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.6882 | max_loss: 0.1360 | max_backward: 1.3984 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5787 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7607.9 | uss_MB: 6553.3 | child_rss_MB: 3582.8 | allocated_MB: 343.8 | reserved_MB: 23132.0 | inactive_split_MB: 233.1 +rank5 [2026-10-29 11:51:12,922] INFO : Step 34700/50000 rank5 | Train Loss: 1.1152 | Train Loss Unweighted: 0.5285 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1612 | Classification: 0.0937 | Classification_binary: 0.1734 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1283 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1436 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0491 | Regression_eta: 0.0342 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0455 | LR: 2.92e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5242s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7859 | max_loss: 0.2571 | max_backward: 1.9119 | max_optimizer_step: 0.2516 | max_tensorboard_logging: 0.5032 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7610.6 | uss_MB: 7599.4 | child_rss_MB: 3559.6 | allocated_MB: 343.9 | reserved_MB: 23132.0 | inactive_split_MB: 446.8 +rank5 [2026-10-29 11:54:56,885] INFO : Step 34800/50000 rank5 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5142 | Train Loss Components: Auxiliary: 0.2371 | Cardinality: 0.1583 | Classification: 0.0857 | Classification_binary: 0.1750 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1255 | Regression_eta: 0.0013 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1983 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0639 | Regression_eta: 0.0463 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0781 | LR: 2.90e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2210s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0423 | max_forward: 0.8214 | max_loss: 0.1472 | max_backward: 1.4019 | max_optimizer_step: 0.3234 | max_tensorboard_logging: 0.5950 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7613.6 | uss_MB: 7602.4 | child_rss_MB: 3541.0 | allocated_MB: 344.5 | reserved_MB: 23132.0 | inactive_split_MB: 201.5 +rank5 [2026-10-29 11:58:48,856] INFO : Step 34900/50000 rank5 | Train Loss: 0.9512 | Train Loss Unweighted: 0.4678 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1120 | Classification: 0.0788 | Classification_binary: 0.1318 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1262 | Regression_eta: 0.0014 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0737 | Regression_eta: 0.0361 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0428 | LR: 2.88e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2621s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0986 | max_forward: 0.7397 | max_loss: 0.1421 | max_backward: 1.3918 | max_optimizer_step: 0.3560 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7614.6 | uss_MB: 7602.0 | child_rss_MB: 3543.9 | allocated_MB: 341.5 | reserved_MB: 23500.0 | inactive_split_MB: 262.9 +rank5 [2026-10-29 12:02:43,663] INFO : Step 35000/50000 rank5 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1401 | Classification: 0.0852 | Classification_binary: 0.1555 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0012 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0510 | Regression_eta: 0.0315 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0676 | LR: 2.85e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.7030s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0159 | max_forward: 0.7961 | max_loss: 0.2615 | max_backward: 2.0104 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6382 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7618.3 | uss_MB: 7606.3 | child_rss_MB: 3544.7 | allocated_MB: 345.2 | reserved_MB: 23500.0 | inactive_split_MB: 284.5 +rank5 [2026-10-29 12:02:44,719] INFO : Running validation on rank5 +rank5 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 12:03:33,663] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 12:03:34,207] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 12:04:01,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 12:04:02,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 12:04:04,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 12:04:05,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 12:04:15,212] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 12:04:18,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 12:04:18,509] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 12:04:34,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 12:04:35,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 12:04:36,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 12:04:37,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 12:04:47,276] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 12:04:48,304] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 12:04:48,825] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 12:05:04,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 12:05:04,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 12:05:05,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 12:05:06,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 12:05:16,073] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 12:09:34,457] INFO : Step 35100/50000 rank5 | Train Loss: 1.1060 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1387 | Classification: 0.0926 | Classification_binary: 0.1848 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0014 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1799 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0577 | Regression_eta: 0.0327 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0422 | LR: 2.83e-04 | DataLoad Time: 0.0156s | Model Forward Time: 6.6643s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.9070 | max_loss: 0.1513 | max_backward: 2.1746 | max_optimizer_step: 0.2850 | max_tensorboard_logging: 1.1439 | max_tensorboard_flush: 0.0024 | max_post_step_barrier: 0.0768 | Memory: rss_MB: 9217.3 | uss_MB: 7564.1 | child_rss_MB: 3570.6 | allocated_MB: 344.3 | reserved_MB: 22622.0 | inactive_split_MB: 852.9 +rank5 [2026-10-29 12:13:24,279] INFO : Step 35200/50000 rank5 | Train Loss: 1.0386 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1293 | Classification: 0.0826 | Classification_binary: 0.1559 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1247 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1772 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0602 | Regression_eta: 0.0581 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0883 | LR: 2.81e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5316s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.8193 | max_loss: 0.1608 | max_backward: 1.3752 | max_optimizer_step: 0.2535 | max_tensorboard_logging: 0.9185 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0700 | Memory: rss_MB: 7738.0 | uss_MB: 7717.0 | child_rss_MB: 3546.7 | allocated_MB: 341.8 | reserved_MB: 25614.0 | inactive_split_MB: 192.0 +rank5 [2026-10-29 12:17:18,033] INFO : Step 35300/50000 rank5 | Train Loss: 0.9736 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1059 | Classification: 0.0970 | Classification_binary: 0.1371 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0016 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1180 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0452 | Regression_eta: 0.0805 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0483 | LR: 2.79e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6487s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6434 | max_loss: 0.2341 | max_backward: 1.6834 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0588 | Memory: rss_MB: 7740.0 | uss_MB: 7297.3 | child_rss_MB: 3543.2 | allocated_MB: 343.2 | reserved_MB: 25614.0 | inactive_split_MB: 226.8 +rank5 [2026-10-29 12:21:06,024] INFO : Step 35400/50000 rank5 | Train Loss: 1.1186 | Train Loss Unweighted: 0.5363 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1393 | Classification: 0.1069 | Classification_binary: 0.1767 | HitGrouping: 0.1943 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0016 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.1554 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0600 | Regression_eta: 0.0387 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0484 | LR: 2.76e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8140s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0060 | max_forward: 0.7591 | max_loss: 0.1562 | max_backward: 1.6040 | max_optimizer_step: 0.2543 | max_tensorboard_logging: 0.9624 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.0563 | Memory: rss_MB: 7735.5 | uss_MB: 7248.0 | child_rss_MB: 3540.1 | allocated_MB: 345.3 | reserved_MB: 25762.0 | inactive_split_MB: 277.5 +rank5 [2026-10-29 12:24:54,118] INFO : Step 35500/50000 rank5 | Train Loss: 1.1216 | Train Loss Unweighted: 0.5466 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1527 | Classification: 0.0970 | Classification_binary: 0.1767 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1342 | Regression_eta: 0.0014 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0563 | Regression_eta: 0.0424 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0706 | LR: 2.74e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4075s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0648 | max_forward: 0.8748 | max_loss: 0.1544 | max_backward: 1.6387 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 0.5130 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0632 | Memory: rss_MB: 7738.5 | uss_MB: 7721.7 | child_rss_MB: 3543.4 | allocated_MB: 343.4 | reserved_MB: 25762.0 | inactive_split_MB: 284.4 +rank5 [2026-10-29 12:28:50,969] INFO : Step 35600/50000 rank5 | Train Loss: 1.0380 | Train Loss Unweighted: 0.5039 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1124 | Classification: 0.0922 | Classification_binary: 0.1405 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1222 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0653 | Regression_eta: 0.0283 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0703 | LR: 2.72e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.8999s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0559 | max_forward: 0.7376 | max_loss: 0.1415 | max_backward: 1.1754 | max_optimizer_step: 0.2524 | max_tensorboard_logging: 0.5225 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1603 | Memory: rss_MB: 7739.4 | uss_MB: 7724.9 | child_rss_MB: 3557.9 | allocated_MB: 343.8 | reserved_MB: 25762.0 | inactive_split_MB: 233.6 +rank5 [2026-10-29 12:32:54,485] INFO : Step 35700/50000 rank5 | Train Loss: 1.0352 | Train Loss Unweighted: 0.5000 | Train Loss Components: Auxiliary: 0.2288 | Cardinality: 0.1199 | Classification: 0.1043 | Classification_binary: 0.1492 | HitGrouping: 0.1850 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1212 | Regression_eta: 0.0015 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.1399 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0746 | Regression_eta: 0.0498 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0604 | LR: 2.70e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3739s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0360 | max_forward: 0.7358 | max_loss: 0.1419 | max_backward: 1.6761 | max_optimizer_step: 0.2531 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7741.1 | uss_MB: 7726.8 | child_rss_MB: 3543.5 | allocated_MB: 344.2 | reserved_MB: 26034.0 | inactive_split_MB: 271.8 +rank5 [2026-10-29 12:36:36,039] INFO : Step 35800/50000 rank5 | Train Loss: 1.0929 | Train Loss Unweighted: 0.5323 | Train Loss Components: Auxiliary: 0.2446 | Cardinality: 0.1241 | Classification: 0.0986 | Classification_binary: 0.1801 | HitGrouping: 0.1908 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0014 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0510 | Regression_eta: 0.0282 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0727 | LR: 2.68e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1524s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7732 | max_loss: 0.1506 | max_backward: 1.4048 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0581 | Memory: rss_MB: 7740.3 | uss_MB: 7206.4 | child_rss_MB: 3548.3 | allocated_MB: 344.1 | reserved_MB: 26034.0 | inactive_split_MB: 276.1 +rank5 [2026-10-29 14:48:21,996] INFO : Initializing model with seed=12345; process seed=12350 +rank5 [2026-10-29 14:48:22,004] INFO : configured dtype=torch.bfloat16 for autocast +rank5 [2026-10-29 14:48:45,046] INFO : Instantiating model +rank5 [2026-10-29 14:48:45,046] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank5 [2026-10-29 14:48:45,046] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank5 [2026-10-29 14:48:45,065] INFO : Input encoding initialization took 0.02s +rank5 [2026-10-29 14:48:45,065] INFO : nn0 parameters: 171776 +rank5 [2026-10-29 14:48:45,065] INFO : Initializing shared backbone layers, num_convs=6 +rank5 [2026-10-29 14:48:45,065] INFO : Initializing backbone layer 0 +rank5 [2026-10-29 14:48:45,065] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:48:45,069] INFO : Initializing backbone layer 1 +rank5 [2026-10-29 14:48:45,069] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:48:45,072] INFO : Initializing backbone layer 2 +rank5 [2026-10-29 14:48:45,072] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:48:45,075] INFO : Initializing backbone layer 3 +rank5 [2026-10-29 14:48:45,075] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:48:45,078] INFO : Initializing backbone layer 4 +rank5 [2026-10-29 14:48:45,078] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:48:45,081] INFO : Initializing backbone layer 5 +rank5 [2026-10-29 14:48:45,081] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank5 [2026-10-29 14:48:45,085] INFO : Backbone initialization took 0.02s +rank5 [2026-10-29 14:48:45,085] INFO : backbone parameters: 2374656 +rank5 [2026-10-29 14:48:45,085] INFO : Initializing output DNNs +rank5 [2026-10-29 14:48:45,133] INFO : Output DNNs initialization took 0.05s +rank5 [2026-10-29 14:48:45,133] INFO : backbone_mode=BackboneMode.SHARED +rank5 [2026-10-29 14:48:45,133] INFO : jagged_attention enabled=True +rank5 [2026-10-29 14:48:45,133] INFO : flash_attn_varlen enabled=True +rank5 [2026-10-29 14:48:45,134] INFO : task_queries enabled=False +rank5 [2026-10-29 14:48:45,134] INFO : classification_norm parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : regression_norm parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : classification_query parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : regression_query parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : classification_readout parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : regression_readout parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : output_mode=set +rank5 [2026-10-29 14:48:45,134] INFO : set_decoder parameters: 4713812 +rank5 [2026-10-29 14:48:45,134] INFO : nn_binary_particle parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : nn_pid parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : nn_pt parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : nn_eta parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : nn_sin_phi parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : nn_cos_phi parameters: 0 +rank5 [2026-10-29 14:48:45,134] INFO : nn_energy parameters: 0 +rank5 [2026-10-29 14:48:45,135] INFO : Total MLPF parameters: 7260244 +rank5 [2026-10-29 14:48:45,135] INFO : MLPF __init__ done +rank5 [2026-10-29 14:48:45,136] INFO : Instantiated model +rank5 [2026-10-29 14:48:45,136] INFO : Moving model to device rank=5 +rank5 [2026-10-29 14:48:45,168] INFO : Moved model to device rank=5 +rank5 [2026-10-29 14:48:45,170] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank5 [2026-10-29 14:48:45,513] INFO : Restoring training from step 35001 +rank5 [2026-10-29 14:48:45,519] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth +rank5 [2026-10-29 14:48:45,520] INFO : Configured model for SyncBatchNorm rank=5 +rank5 [2026-10-29 14:48:45,764] INFO : Configured model for DistributedDataParallel rank=5 +rank5 [2026-10-29 14:48:45,876] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank5 [2026-10-29 14:48:45,876] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank5 [2026-10-29 14:48:45,877] INFO : Trainable parameters: 7260244 +rank5 [2026-10-29 14:48:45,877] INFO : Non-trainable parameters: 0 +rank5 [2026-10-29 14:48:45,877] INFO : Total parameters: 7260244 +rank5 [2026-10-29 14:48:45,880] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank5 [2026-10-29 14:48:45,880] INFO : Getting dataloaders +rank5 [2026-10-29 14:48:45,880] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:48:51,703] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:48:52,753] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:48:53,926] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-29 14:48:53,926] INFO : Creating WeightedMixIterator with 1 data loaders. +rank5 [2026-10-29 14:48:53,926] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:48:54,196] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:48:54,455] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:48:54,705] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-29 14:48:54,706] INFO : Creating InterleavedIterator with 1 data loaders. +rank5 [2026-10-29 14:49:08,292] INFO : Got dataloaders +rank5 [2026-10-29 14:49:08,292] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13888] of epochs [1] +rank5 [2026-10-29 14:49:08,292] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank5 [2026-10-29 14:49:08,292] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank5 [2026-10-29 14:49:08,292] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank5 [2026-10-29 14:49:09,967] INFO : loader split=train rank=5 len=21112 +rank5 [2026-10-29 14:49:09,967] INFO : loader split=valid rank=5 len=12 +rank5 [2026-10-29 14:49:09,967] INFO : Starting training from step 35001 to 50000 on rank 5 of 8 +rank5 [2026-10-29 14:49:09,967] INFO : Creating train_iterator +rank5 [2026-10-29 14:49:09,968] INFO : ResumableSampler physical:train loading state: start_index=222208 +rank5 [2026-10-29 14:49:24,140] INFO : Created train_iterator +rank5 [2026-10-29 14:53:12,251] INFO : Step 35100/50000 rank5 | Train Loss: 1.0826 | Train Loss Unweighted: 0.5185 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1393 | Classification: 0.0961 | Classification_binary: 0.1822 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1816 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0740 | Regression_eta: 0.0290 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0450 | LR: 2.83e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.4927s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.7340 | max_loss: 0.1651 | max_backward: 1.4152 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4761 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4625.8 | uss_MB: 3388.6 | child_rss_MB: 5578.9 | allocated_MB: 372.6 | reserved_MB: 22674.0 | inactive_split_MB: 823.1 +rank5 [2026-10-29 14:56:45,511] INFO : Step 35200/50000 rank5 | Train Loss: 1.0562 | Train Loss Unweighted: 0.5125 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1301 | Classification: 0.0878 | Classification_binary: 0.1540 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1331 | Regression_eta: 0.0017 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1715 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0586 | Regression_eta: 0.0632 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0711 | LR: 2.81e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.1440s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6042 | max_loss: 0.1411 | max_backward: 1.1580 | max_optimizer_step: 0.1447 | max_tensorboard_logging: 0.5108 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4625.0 | uss_MB: 3391.6 | child_rss_MB: 5591.1 | allocated_MB: 368.1 | reserved_MB: 25668.0 | inactive_split_MB: 192.7 +rank5 [2026-10-29 15:00:19,321] INFO : Step 35300/50000 rank5 | Train Loss: 0.9962 | Train Loss Unweighted: 0.4725 | Train Loss Components: Auxiliary: 0.2160 | Cardinality: 0.1172 | Classification: 0.0927 | Classification_binary: 0.1341 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1118 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0470 | Regression_eta: 0.0731 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0445 | LR: 2.79e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.0284s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.5940 | max_loss: 0.1358 | max_backward: 1.1911 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0027 | Memory: rss_MB: 4636.4 | uss_MB: 3402.6 | child_rss_MB: 5598.9 | allocated_MB: 371.5 | reserved_MB: 25668.0 | inactive_split_MB: 207.6 +rank5 [2026-10-29 15:03:53,851] INFO : Step 35400/50000 rank5 | Train Loss: 1.1270 | Train Loss Unweighted: 0.5445 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1356 | Classification: 0.0990 | Classification_binary: 0.1805 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1297 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0667 | Regression_eta: 0.0270 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0460 | LR: 2.76e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.6490s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1507 | max_backward: 1.5023 | max_optimizer_step: 0.2459 | max_tensorboard_logging: 0.5679 | max_tensorboard_flush: 0.0064 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4646.3 | uss_MB: 3410.8 | child_rss_MB: 5599.4 | allocated_MB: 371.4 | reserved_MB: 25816.0 | inactive_split_MB: 263.6 +rank5 [2026-10-29 15:07:33,752] INFO : Step 35500/50000 rank5 | Train Loss: 1.0502 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1302 | Classification: 0.0904 | Classification_binary: 0.1703 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1250 | Regression_eta: 0.0013 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1899 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0508 | Regression_eta: 0.0445 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0524 | LR: 2.74e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6112s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0737 | max_forward: 0.7546 | max_loss: 0.1581 | max_backward: 1.4977 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4644.7 | uss_MB: 3412.9 | child_rss_MB: 5568.4 | allocated_MB: 371.1 | reserved_MB: 25816.0 | inactive_split_MB: 263.4 +rank5 [2026-10-29 15:11:19,389] INFO : Step 35600/50000 rank5 | Train Loss: 1.0258 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2226 | Cardinality: 0.1119 | Classification: 0.0922 | Classification_binary: 0.1418 | HitGrouping: 0.1950 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1286 | Regression_eta: 0.0015 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0659 | Regression_eta: 0.0264 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0532 | LR: 2.72e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.2555s | Phase Times: max_pre_step_barrier: 0.0929 | max_batch_to_device: 0.0064 | max_forward: 0.5696 | max_loss: 0.1430 | max_backward: 1.1627 | max_optimizer_step: 0.1475 | max_tensorboard_logging: 0.5805 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4646.9 | uss_MB: 3412.7 | child_rss_MB: 5602.7 | allocated_MB: 370.5 | reserved_MB: 25816.0 | inactive_split_MB: 212.3 +rank5 [2026-10-29 15:15:08,570] INFO : Step 35700/50000 rank5 | Train Loss: 1.0162 | Train Loss Unweighted: 0.5013 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1185 | Classification: 0.0946 | Classification_binary: 0.1537 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1248 | Regression_eta: 0.0015 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0969 | Regression_energy: 0.0742 | Regression_eta: 0.0457 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0777 | LR: 2.70e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.5926s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7397 | max_loss: 0.1413 | max_backward: 1.6688 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4646.6 | uss_MB: 3411.7 | child_rss_MB: 5604.2 | allocated_MB: 372.2 | reserved_MB: 26084.0 | inactive_split_MB: 249.8 +rank5 [2026-10-29 15:18:49,408] INFO : Step 35800/50000 rank5 | Train Loss: 1.0884 | Train Loss Unweighted: 0.5265 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1296 | Classification: 0.0948 | Classification_binary: 0.1803 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1625 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0536 | Regression_eta: 0.0295 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0458 | LR: 2.68e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2026s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0199 | max_forward: 0.9402 | max_loss: 0.2389 | max_backward: 1.7889 | max_optimizer_step: 0.2466 | max_tensorboard_logging: 0.7071 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4643.4 | uss_MB: 3414.5 | child_rss_MB: 5602.6 | allocated_MB: 371.6 | reserved_MB: 26084.0 | inactive_split_MB: 257.5 +rank5 [2026-10-29 15:22:30,828] INFO : Step 35900/50000 rank5 | Train Loss: 1.0760 | Train Loss Unweighted: 0.5299 | Train Loss Components: Auxiliary: 0.2328 | Cardinality: 0.1277 | Classification: 0.0983 | Classification_binary: 0.1568 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1349 | Regression_eta: 0.0016 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1517 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0557 | Regression_eta: 0.0550 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0535 | LR: 2.65e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.2890s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6268 | max_loss: 0.2373 | max_backward: 1.2570 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.4934 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0016 | Memory: rss_MB: 4639.4 | uss_MB: 3412.6 | child_rss_MB: 5605.5 | allocated_MB: 369.7 | reserved_MB: 26800.0 | inactive_split_MB: 294.3 +rank5 [2026-10-29 15:26:13,159] INFO : Step 36000/50000 rank5 | Train Loss: 1.0831 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1276 | Classification: 0.1031 | Classification_binary: 0.1434 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1452 | Regression_eta: 0.0017 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0759 | Regression_eta: 0.0575 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0693 | LR: 2.63e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.2193s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6266 | max_loss: 0.1437 | max_backward: 1.3209 | max_optimizer_step: 0.1464 | max_tensorboard_logging: 0.4977 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4620.1 | uss_MB: 3411.5 | child_rss_MB: 5582.3 | allocated_MB: 368.7 | reserved_MB: 26800.0 | inactive_split_MB: 270.9 +rank5 [2026-10-29 15:26:14,074] INFO : Running validation on rank5 +rank5 [2026-10-29 15:26:14,074] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 15:26:14,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 15:27:03,342] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:27:03,870] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 15:27:29,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 15:27:30,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 15:27:31,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 15:27:32,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 15:27:42,607] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 15:27:42,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:27:43,315] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 15:27:59,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 15:27:59,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 15:28:00,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 15:28:01,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 15:28:11,867] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 15:28:12,651] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:28:13,126] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 15:28:28,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 15:28:29,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 15:28:29,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 15:28:30,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 15:28:40,548] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 15:32:50,524] INFO : Step 36100/50000 rank5 | Train Loss: 1.0137 | Train Loss Unweighted: 0.5026 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1079 | Classification: 0.1014 | Classification_binary: 0.1522 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1216 | Regression_eta: 0.0013 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0373 | Regression_energy: 0.0482 | Regression_eta: 0.0321 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0434 | LR: 2.61e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0461s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0065 | max_forward: 0.6968 | max_loss: 0.2782 | max_backward: 1.4278 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.9130 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6101.3 | uss_MB: 4907.2 | child_rss_MB: 5609.1 | allocated_MB: 372.5 | reserved_MB: 23174.0 | inactive_split_MB: 246.6 +rank5 [2026-10-29 15:36:33,882] INFO : Step 36200/50000 rank5 | Train Loss: 1.0439 | Train Loss Unweighted: 0.5147 | Train Loss Components: Auxiliary: 0.2338 | Cardinality: 0.1304 | Classification: 0.0909 | Classification_binary: 0.1697 | HitGrouping: 0.1917 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1255 | Regression_eta: 0.0011 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0567 | Regression_eta: 0.0293 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0700 | LR: 2.59e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5410s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7440 | max_loss: 0.1544 | max_backward: 1.4037 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 6101.4 | uss_MB: 4909.6 | child_rss_MB: 5587.9 | allocated_MB: 367.8 | reserved_MB: 23174.0 | inactive_split_MB: 282.8 +rank5 [2026-10-29 15:40:20,925] INFO : Step 36300/50000 rank5 | Train Loss: 1.0379 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1216 | Classification: 0.0886 | Classification_binary: 0.1447 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1266 | Regression_eta: 0.0016 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0499 | Regression_eta: 0.0296 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0342 | LR: 2.57e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.3307s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6106 | max_loss: 0.2412 | max_backward: 1.3407 | max_optimizer_step: 0.1535 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1063 | Memory: rss_MB: 5450.1 | uss_MB: 4269.0 | child_rss_MB: 5611.1 | allocated_MB: 370.5 | reserved_MB: 23174.0 | inactive_split_MB: 274.2 +rank5 [2026-10-29 15:44:06,294] INFO : Step 36400/50000 rank5 | Train Loss: 1.0182 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.0976 | Classification: 0.0939 | Classification_binary: 0.1520 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1167 | Regression_eta: 0.0016 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.1692 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0539 | Regression_eta: 0.0349 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0624 | LR: 2.55e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5206s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.6772 | max_loss: 0.1711 | max_backward: 1.4058 | max_optimizer_step: 0.1518 | max_tensorboard_logging: 0.5460 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0089 | Memory: rss_MB: 5451.9 | uss_MB: 4272.1 | child_rss_MB: 5615.2 | allocated_MB: 371.8 | reserved_MB: 23382.0 | inactive_split_MB: 230.6 +rank5 [2026-10-29 15:47:54,064] INFO : Step 36500/50000 rank5 | Train Loss: 1.0953 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1370 | Classification: 0.0895 | Classification_binary: 0.1838 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2038 | Regression_cos_phi: 0.0970 | Regression_energy: 0.0509 | Regression_eta: 0.0503 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0844 | LR: 2.52e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.9828s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7475 | max_loss: 0.2537 | max_backward: 1.7352 | max_optimizer_step: 0.1520 | max_tensorboard_logging: 0.6267 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5440.5 | uss_MB: 4262.4 | child_rss_MB: 5614.9 | allocated_MB: 372.3 | reserved_MB: 24160.0 | inactive_split_MB: 275.1 +rank5 [2026-10-29 15:51:40,079] INFO : Step 36600/50000 rank5 | Train Loss: 0.9967 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1165 | Classification: 0.0804 | Classification_binary: 0.1438 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1366 | Regression_eta: 0.0013 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0725 | Regression_eta: 0.0274 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0432 | LR: 2.50e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.6293s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6592 | max_loss: 0.2501 | max_backward: 1.4213 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5953 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5450.1 | uss_MB: 4272.8 | child_rss_MB: 5617.4 | allocated_MB: 367.8 | reserved_MB: 27180.0 | inactive_split_MB: 225.4 +rank5 [2026-10-29 15:55:26,196] INFO : Step 36700/50000 rank5 | Train Loss: 0.9770 | Train Loss Unweighted: 0.4817 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1070 | Classification: 0.0901 | Classification_binary: 0.1347 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.1085 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0586 | Regression_eta: 0.0546 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0667 | LR: 2.48e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.2740s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0062 | max_forward: 0.5676 | max_loss: 0.2381 | max_backward: 1.1343 | max_optimizer_step: 0.2417 | max_tensorboard_logging: 0.4693 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5448.5 | uss_MB: 4274.1 | child_rss_MB: 5620.1 | allocated_MB: 370.8 | reserved_MB: 27180.0 | inactive_split_MB: 235.2 +rank5 [2026-10-29 15:59:20,983] INFO : Step 36800/50000 rank5 | Train Loss: 1.0234 | Train Loss Unweighted: 0.4967 | Train Loss Components: Auxiliary: 0.2267 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1662 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1197 | Regression_eta: 0.0013 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1672 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0596 | Regression_eta: 0.0402 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0647 | LR: 2.46e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.5125s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 1.1394 | max_loss: 0.1518 | max_backward: 2.3088 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5450.3 | uss_MB: 4278.9 | child_rss_MB: 5629.5 | allocated_MB: 371.9 | reserved_MB: 27180.0 | inactive_split_MB: 230.7 +rank5 [2026-10-29 16:03:11,411] INFO : Step 36900/50000 rank5 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2268 | Cardinality: 0.1211 | Classification: 0.0855 | Classification_binary: 0.1637 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1219 | Regression_eta: 0.0015 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0580 | Regression_eta: 0.0339 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0521 | LR: 2.44e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.5007s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0071 | max_forward: 0.7005 | max_loss: 0.1465 | max_backward: 1.2640 | max_optimizer_step: 0.1579 | max_tensorboard_logging: 0.4728 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5451.1 | uss_MB: 4280.1 | child_rss_MB: 5636.6 | allocated_MB: 372.3 | reserved_MB: 27180.0 | inactive_split_MB: 201.7 +rank5 [2026-10-29 16:07:15,145] INFO : Step 37000/50000 rank5 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2194 | Cardinality: 0.1268 | Classification: 0.0926 | Classification_binary: 0.1402 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0014 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1197 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0520 | Regression_eta: 0.0244 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0481 | LR: 2.42e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.5222s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6075 | max_loss: 0.2475 | max_backward: 1.2956 | max_optimizer_step: 0.1585 | max_tensorboard_logging: 0.7613 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5445.1 | uss_MB: 4277.2 | child_rss_MB: 5637.5 | allocated_MB: 368.1 | reserved_MB: 27180.0 | inactive_split_MB: 226.3 +rank5 [2026-10-29 16:07:16,060] INFO : Running validation on rank5 +rank5 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 16:08:14,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:08:14,583] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 16:08:39,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 16:08:40,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 16:08:41,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 16:08:42,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 16:08:52,683] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 16:08:53,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:08:53,528] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 16:09:09,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 16:09:10,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 16:09:11,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 16:09:12,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 16:09:22,744] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 16:09:23,588] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:09:24,058] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 16:09:39,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 16:09:40,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 16:09:40,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 16:09:41,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 16:09:51,564] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 16:13:56,322] INFO : Step 37100/50000 rank5 | Train Loss: 1.1150 | Train Loss Unweighted: 0.5431 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1226 | Classification: 0.1105 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1340 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.1513 | Regression_cos_phi: 0.1002 | Regression_energy: 0.0558 | Regression_eta: 0.0560 | Regression_pt: 0.0584 | Regression_sin_phi: 0.1041 | LR: 2.40e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8593s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7539 | max_loss: 0.3988 | max_backward: 1.7903 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4747 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6593.7 | uss_MB: 5426.2 | child_rss_MB: 5642.5 | allocated_MB: 372.5 | reserved_MB: 23160.0 | inactive_split_MB: 265.5 +rank5 [2026-10-29 16:17:46,448] INFO : Step 37200/50000 rank5 | Train Loss: 1.0752 | Train Loss Unweighted: 0.5181 | Train Loss Components: Auxiliary: 0.2406 | Cardinality: 0.1258 | Classification: 0.0924 | Classification_binary: 0.1812 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1200 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0406 | Regression_energy: 0.0443 | Regression_eta: 0.0284 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0324 | LR: 2.38e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1702s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0549 | max_forward: 0.8058 | max_loss: 0.1510 | max_backward: 1.8476 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5280 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6591.2 | uss_MB: 5426.9 | child_rss_MB: 5644.6 | allocated_MB: 371.5 | reserved_MB: 24140.0 | inactive_split_MB: 271.0 +rank5 [2026-10-29 16:21:30,618] INFO : Step 37300/50000 rank5 | Train Loss: 1.0230 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1563 | HitGrouping: 0.1981 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1213 | Regression_eta: 0.0016 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1697 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0507 | Regression_eta: 0.0332 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0534 | LR: 2.36e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0283s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.6853 | max_loss: 0.1502 | max_backward: 1.3245 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 1.0657 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5580.1 | uss_MB: 4415.2 | child_rss_MB: 5668.9 | allocated_MB: 370.9 | reserved_MB: 24140.0 | inactive_split_MB: 261.7 +rank5 [2026-10-29 16:25:14,030] INFO : Step 37400/50000 rank5 | Train Loss: 1.0130 | Train Loss Unweighted: 0.4929 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1180 | Classification: 0.0961 | Classification_binary: 0.1390 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1273 | Regression_eta: 0.0015 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0494 | Regression_eta: 0.0271 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0568 | LR: 2.34e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5045s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.5424 | max_loss: 0.1464 | max_backward: 1.0792 | max_optimizer_step: 0.1675 | max_tensorboard_logging: 0.8668 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5585.6 | uss_MB: 4420.8 | child_rss_MB: 5670.0 | allocated_MB: 368.8 | reserved_MB: 25122.0 | inactive_split_MB: 232.7 +rank5 [2026-10-29 16:30:41,706] INFO : Step 37500/50000 rank5 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1315 | Classification: 0.0992 | Classification_binary: 0.1679 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1176 | Regression_eta: 0.0013 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0540 | Regression_eta: 0.0259 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0491 | LR: 2.32e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.1489s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.7240 | max_loss: 0.1472 | max_backward: 1.5381 | max_optimizer_step: 0.2707 | max_tensorboard_logging: 0.8554 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5585.8 | uss_MB: 4422.3 | child_rss_MB: 5642.8 | allocated_MB: 370.2 | reserved_MB: 26134.0 | inactive_split_MB: 179.7 +rank5 [2026-10-29 16:34:38,362] INFO : Step 37600/50000 rank5 | Train Loss: 1.0243 | Train Loss Unweighted: 0.4867 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1182 | Classification: 0.0808 | Classification_binary: 0.1619 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0588 | Regression_eta: 0.0397 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0736 | LR: 2.30e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1670s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.7069 | max_loss: 0.1543 | max_backward: 1.5139 | max_optimizer_step: 0.1704 | max_tensorboard_logging: 1.0159 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5587.9 | uss_MB: 4422.7 | child_rss_MB: 5672.0 | allocated_MB: 370.8 | reserved_MB: 26846.0 | inactive_split_MB: 273.3 +rank5 [2026-10-29 16:38:29,294] INFO : Step 37700/50000 rank5 | Train Loss: 0.9610 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2102 | Cardinality: 0.1049 | Classification: 0.0974 | Classification_binary: 0.1317 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0012 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1199 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0496 | Regression_eta: 0.0268 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0495 | LR: 2.28e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.5156s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6233 | max_loss: 0.2433 | max_backward: 1.3306 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5644 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5591.0 | uss_MB: 4425.5 | child_rss_MB: 5643.2 | allocated_MB: 369.4 | reserved_MB: 28822.0 | inactive_split_MB: 231.5 +rank5 [2026-10-29 16:42:35,307] INFO : Step 37800/50000 rank5 | Train Loss: 0.9994 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1105 | Classification: 0.0952 | Classification_binary: 0.1458 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1161 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0585 | Regression_eta: 0.0491 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0918 | LR: 2.26e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8898s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.6659 | max_loss: 0.1473 | max_backward: 1.3840 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.9558 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5593.5 | uss_MB: 4427.0 | child_rss_MB: 5671.1 | allocated_MB: 371.5 | reserved_MB: 28822.0 | inactive_split_MB: 170.1 +rank5 [2026-10-29 16:46:40,060] INFO : Step 37900/50000 rank5 | Train Loss: 1.0837 | Train Loss Unweighted: 0.5094 | Train Loss Components: Auxiliary: 0.2372 | Cardinality: 0.1419 | Classification: 0.0937 | Classification_binary: 0.1795 | HitGrouping: 0.1848 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1160 | Regression_eta: 0.0015 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1562 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0613 | Regression_eta: 0.0438 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0557 | LR: 2.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2192s | Phase Times: max_pre_step_barrier: 0.1076 | max_batch_to_device: 0.0071 | max_forward: 0.7572 | max_loss: 0.1547 | max_backward: 1.4632 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 0.9205 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 5593.0 | uss_MB: 4429.3 | child_rss_MB: 5670.1 | allocated_MB: 370.2 | reserved_MB: 28822.0 | inactive_split_MB: 265.3 +rank5 [2026-10-29 16:50:31,519] INFO : Step 38000/50000 rank5 | Train Loss: 0.9811 | Train Loss Unweighted: 0.4782 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1213 | Classification: 0.0781 | Classification_binary: 0.1413 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1277 | Regression_eta: 0.0012 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0708 | Regression_eta: 0.0289 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0472 | LR: 2.22e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.5077s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.6120 | max_loss: 0.2171 | max_backward: 1.2526 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5836 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5590.4 | uss_MB: 4430.0 | child_rss_MB: 5667.2 | allocated_MB: 368.7 | reserved_MB: 28822.0 | inactive_split_MB: 223.4 +rank5 [2026-10-29 16:50:32,415] INFO : Running validation on rank5 +rank5 [2026-10-29 16:50:32,415] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 16:50:32,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 16:51:23,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:51:24,264] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 16:51:51,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 16:51:52,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 16:51:53,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 16:51:53,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 16:52:03,995] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 16:52:04,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:52:05,012] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 16:52:20,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 16:52:21,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 16:52:22,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 16:52:23,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 16:52:33,227] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 16:52:34,975] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:52:35,420] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 16:52:50,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 16:52:51,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 16:52:52,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 16:52:52,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 16:53:02,784] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 16:57:14,095] INFO : Step 38100/50000 rank5 | Train Loss: 1.0158 | Train Loss Unweighted: 0.4912 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1220 | Classification: 0.0982 | Classification_binary: 0.1455 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1224 | Regression_eta: 0.0013 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0609 | Regression_eta: 0.0460 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0718 | LR: 2.20e-04 | DataLoad Time: 0.0245s | Model Forward Time: 4.9409s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8406 | max_loss: 0.1820 | max_backward: 1.8536 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.3354 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0179 | Memory: rss_MB: 5714.6 | uss_MB: 4482.4 | child_rss_MB: 5669.8 | allocated_MB: 370.5 | reserved_MB: 24292.0 | inactive_split_MB: 257.8 +rank5 [2026-10-29 17:01:01,945] INFO : Step 38200/50000 rank5 | Train Loss: 1.0273 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2292 | Cardinality: 0.1060 | Classification: 0.0997 | Classification_binary: 0.1650 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0538 | Regression_eta: 0.0297 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0379 | LR: 2.18e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.8907s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7758 | max_loss: 0.1794 | max_backward: 1.7969 | max_optimizer_step: 0.1713 | max_tensorboard_logging: 0.4698 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5717.4 | uss_MB: 4558.1 | child_rss_MB: 5664.1 | allocated_MB: 372.9 | reserved_MB: 26212.0 | inactive_split_MB: 221.1 +rank5 [2026-10-29 17:05:07,129] INFO : Step 38300/50000 rank5 | Train Loss: 1.0498 | Train Loss Unweighted: 0.5122 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1249 | Classification: 0.0913 | Classification_binary: 0.1583 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1294 | Regression_eta: 0.0013 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1612 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0550 | Regression_eta: 0.0327 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0696 | LR: 2.16e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7368s | Phase Times: max_pre_step_barrier: 0.1103 | max_batch_to_device: 0.0086 | max_forward: 0.6730 | max_loss: 0.1509 | max_backward: 1.3787 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.4782 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5715.4 | uss_MB: 4556.5 | child_rss_MB: 5665.1 | allocated_MB: 369.2 | reserved_MB: 26212.0 | inactive_split_MB: 171.8 +rank5 [2026-10-29 17:09:08,355] INFO : Step 38400/50000 rank5 | Train Loss: 0.9354 | Train Loss Unweighted: 0.4735 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.0968 | Classification: 0.0760 | Classification_binary: 0.1246 | HitGrouping: 0.1889 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1340 | Regression_eta: 0.0013 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1368 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0781 | Regression_eta: 0.0338 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0669 | LR: 2.14e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.6837s | Phase Times: max_pre_step_barrier: 0.1107 | max_batch_to_device: 0.0062 | max_forward: 0.6635 | max_loss: 0.1419 | max_backward: 1.4771 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 0.4619 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5721.1 | uss_MB: 4559.4 | child_rss_MB: 5662.2 | allocated_MB: 370.2 | reserved_MB: 26212.0 | inactive_split_MB: 267.8 +rank5 [2026-10-29 17:13:06,091] INFO : Step 38500/50000 rank5 | Train Loss: 1.0482 | Train Loss Unweighted: 0.5077 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1228 | Classification: 0.1147 | Classification_binary: 0.1473 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1208 | Regression_eta: 0.0016 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1526 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0625 | Regression_eta: 0.0505 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0444 | LR: 2.12e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.4591s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.6383 | max_loss: 0.1393 | max_backward: 1.3136 | max_optimizer_step: 0.1695 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 5717.0 | uss_MB: 4562.8 | child_rss_MB: 5658.1 | allocated_MB: 370.2 | reserved_MB: 27188.0 | inactive_split_MB: 210.0 +rank5 [2026-10-29 17:17:04,838] INFO : Step 38600/50000 rank5 | Train Loss: 1.0321 | Train Loss Unweighted: 0.4950 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1157 | Classification: 0.0910 | Classification_binary: 0.1735 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1130 | Regression_eta: 0.0014 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1678 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0561 | Regression_eta: 0.0415 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0522 | LR: 2.11e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0838s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0127 | max_forward: 0.7516 | max_loss: 0.1551 | max_backward: 1.6059 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.7250 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5714.3 | uss_MB: 4563.8 | child_rss_MB: 5662.6 | allocated_MB: 372.6 | reserved_MB: 28172.0 | inactive_split_MB: 283.9 +rank5 [2026-10-29 17:20:59,240] INFO : Step 38700/50000 rank5 | Train Loss: 0.9420 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1147 | Classification: 0.0828 | Classification_binary: 0.1348 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1155 | Regression_eta: 0.0013 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0946 | Regression_energy: 0.0547 | Regression_eta: 0.0519 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0668 | LR: 2.09e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.6827s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.8052 | max_loss: 0.2133 | max_backward: 1.3215 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5712.0 | uss_MB: 4410.4 | child_rss_MB: 5659.5 | allocated_MB: 367.4 | reserved_MB: 28172.0 | inactive_split_MB: 206.7 +rank5 [2026-10-29 17:25:08,218] INFO : Step 38800/50000 rank5 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4825 | Train Loss Components: Auxiliary: 0.2201 | Cardinality: 0.1186 | Classification: 0.0965 | Classification_binary: 0.1429 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1198 | Regression_eta: 0.0012 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1444 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0595 | Regression_eta: 0.0304 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0474 | LR: 2.07e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.6538s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.9011 | max_loss: 0.1880 | max_backward: 1.7487 | max_optimizer_step: 0.2166 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5710.9 | uss_MB: 4431.1 | child_rss_MB: 5666.7 | allocated_MB: 371.5 | reserved_MB: 28172.0 | inactive_split_MB: 220.5 +rank5 [2026-10-29 17:29:07,496] INFO : Step 38900/50000 rank5 | Train Loss: 0.9986 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2211 | Cardinality: 0.1078 | Classification: 0.0988 | Classification_binary: 0.1578 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0579 | Regression_eta: 0.0428 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0712 | LR: 2.05e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.4384s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0073 | max_forward: 0.9800 | max_loss: 0.2159 | max_backward: 2.2158 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0079 | Memory: rss_MB: 5714.2 | uss_MB: 3754.5 | child_rss_MB: 5665.7 | allocated_MB: 373.3 | reserved_MB: 29164.0 | inactive_split_MB: 189.2 +rank5 [2026-10-29 17:33:02,382] INFO : Step 39000/50000 rank5 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4758 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1227 | Classification: 0.0847 | Classification_binary: 0.1518 | HitGrouping: 0.1853 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1179 | Regression_eta: 0.0013 | Regression_pt: 0.1189 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1161 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0413 | Regression_eta: 0.0291 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0402 | LR: 2.03e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.7808s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0515 | max_forward: 0.7410 | max_loss: 0.1790 | max_backward: 1.4655 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5713.4 | uss_MB: 4569.9 | child_rss_MB: 5663.9 | allocated_MB: 370.5 | reserved_MB: 29164.0 | inactive_split_MB: 192.2 +rank5 [2026-10-29 17:33:03,294] INFO : Running validation on rank5 +rank5 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 17:33:56,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:33:56,830] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 17:34:24,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 17:34:25,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 17:34:26,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 17:34:27,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 17:34:37,765] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 17:34:38,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:34:38,630] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 17:34:54,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 17:34:55,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 17:34:56,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 17:34:57,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 17:35:07,439] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 17:35:07,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:35:08,264] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 17:35:24,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 17:35:24,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 17:35:25,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 17:35:26,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 17:35:36,188] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 17:39:50,703] INFO : Step 39100/50000 rank5 | Train Loss: 1.0043 | Train Loss Unweighted: 0.4886 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1149 | Classification: 0.0878 | Classification_binary: 0.1320 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1329 | Regression_eta: 0.0012 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0629 | Regression_eta: 0.0721 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0654 | LR: 2.01e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6386s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0736 | max_forward: 0.7590 | max_loss: 0.4062 | max_backward: 1.4330 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.4941 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6810.5 | uss_MB: 5664.9 | child_rss_MB: 5641.2 | allocated_MB: 371.1 | reserved_MB: 23656.0 | inactive_split_MB: 194.8 +rank5 [2026-10-29 17:43:44,583] INFO : Step 39200/50000 rank5 | Train Loss: 1.0325 | Train Loss Unweighted: 0.5174 | Train Loss Components: Auxiliary: 0.2315 | Cardinality: 0.1117 | Classification: 0.1086 | Classification_binary: 0.1481 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0540 | Regression_eta: 0.0273 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0496 | LR: 2.00e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.6968s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0102 | max_forward: 0.7546 | max_loss: 0.4268 | max_backward: 1.4370 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5801.6 | uss_MB: 4660.4 | child_rss_MB: 5666.2 | allocated_MB: 371.2 | reserved_MB: 23934.0 | inactive_split_MB: 196.6 +rank5 [2026-10-29 17:47:41,748] INFO : Step 39300/50000 rank5 | Train Loss: 1.0373 | Train Loss Unweighted: 0.5050 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1241 | Classification: 0.0956 | Classification_binary: 0.1745 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1150 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1828 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0571 | Regression_eta: 0.0374 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0561 | LR: 1.98e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2676s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0844 | max_forward: 0.8662 | max_loss: 0.2199 | max_backward: 1.7416 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5805.0 | uss_MB: 4500.0 | child_rss_MB: 5642.3 | allocated_MB: 373.1 | reserved_MB: 23934.0 | inactive_split_MB: 256.8 +rank5 [2026-10-29 17:51:43,102] INFO : Step 39400/50000 rank5 | Train Loss: 0.9741 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2189 | Cardinality: 0.1078 | Classification: 0.0808 | Classification_binary: 0.1461 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0649 | Regression_eta: 0.0461 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0606 | LR: 1.96e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.5586s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.6778 | max_loss: 0.1515 | max_backward: 1.4156 | max_optimizer_step: 0.1740 | max_tensorboard_logging: 0.4653 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5807.5 | uss_MB: 4663.4 | child_rss_MB: 5665.4 | allocated_MB: 368.0 | reserved_MB: 23934.0 | inactive_split_MB: 199.1 +rank5 [2026-10-29 17:55:40,087] INFO : Step 39500/50000 rank5 | Train Loss: 0.9809 | Train Loss Unweighted: 0.4664 | Train Loss Components: Auxiliary: 0.2175 | Cardinality: 0.1105 | Classification: 0.0963 | Classification_binary: 0.1347 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1158 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0614 | Regression_eta: 0.0263 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0521 | LR: 1.94e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.5549s | Phase Times: max_pre_step_barrier: 0.0259 | max_batch_to_device: 0.1050 | max_forward: 0.6813 | max_loss: 0.1541 | max_backward: 1.3449 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 0.5375 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0111 | Memory: rss_MB: 5806.7 | uss_MB: 4606.4 | child_rss_MB: 5664.0 | allocated_MB: 367.2 | reserved_MB: 24970.0 | inactive_split_MB: 230.9 +rank5 [2026-10-29 17:59:42,116] INFO : Step 39600/50000 rank5 | Train Loss: 1.0814 | Train Loss Unweighted: 0.5099 | Train Loss Components: Auxiliary: 0.2411 | Cardinality: 0.1344 | Classification: 0.0978 | Classification_binary: 0.1731 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0543 | Regression_eta: 0.0269 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0714 | LR: 1.93e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1497s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8839 | max_loss: 0.1464 | max_backward: 1.8057 | max_optimizer_step: 0.2568 | max_tensorboard_logging: 0.5748 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5805.3 | uss_MB: 4666.8 | child_rss_MB: 5646.1 | allocated_MB: 370.6 | reserved_MB: 24970.0 | inactive_split_MB: 243.4 +rank5 [2026-10-29 18:03:55,339] INFO : Step 39700/50000 rank5 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4877 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1185 | Classification: 0.0909 | Classification_binary: 0.1674 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1134 | Regression_eta: 0.0013 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0534 | Regression_eta: 0.0421 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0732 | LR: 1.91e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8052s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.6954 | max_loss: 0.2515 | max_backward: 1.5651 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.5599 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5806.9 | uss_MB: 4667.8 | child_rss_MB: 5665.2 | allocated_MB: 371.1 | reserved_MB: 24970.0 | inactive_split_MB: 244.9 +rank5 [2026-10-29 18:07:56,653] INFO : Step 39800/50000 rank5 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1022 | Classification: 0.0711 | Classification_binary: 0.1322 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1444 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0722 | Regression_eta: 0.0700 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0888 | LR: 1.89e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.9374s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0063 | max_forward: 1.0012 | max_loss: 0.2859 | max_backward: 2.1063 | max_optimizer_step: 0.1800 | max_tensorboard_logging: 0.4881 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5810.9 | uss_MB: 3839.1 | child_rss_MB: 5666.9 | allocated_MB: 368.3 | reserved_MB: 24970.0 | inactive_split_MB: 241.7 +rank5 [2026-10-29 18:11:54,153] INFO : Step 39900/50000 rank5 | Train Loss: 0.9667 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.2188 | Cardinality: 0.1055 | Classification: 0.0897 | Classification_binary: 0.1526 | HitGrouping: 0.1883 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1101 | Regression_eta: 0.0012 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1591 | Regression_cos_phi: 0.0780 | Regression_energy: 0.0462 | Regression_eta: 0.0401 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0744 | LR: 1.88e-04 | DataLoad Time: 0.0202s | Model Forward Time: 6.9941s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.3186 | max_loss: 0.4377 | max_backward: 3.6189 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.6123 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 5815.3 | uss_MB: 4217.6 | child_rss_MB: 5667.9 | allocated_MB: 370.5 | reserved_MB: 24970.0 | inactive_split_MB: 229.4 +rank5 [2026-10-29 18:15:59,748] INFO : Step 40000/50000 rank5 | Train Loss: 1.0232 | Train Loss Unweighted: 0.5131 | Train Loss Components: Auxiliary: 0.2348 | Cardinality: 0.1100 | Classification: 0.0962 | Classification_binary: 0.1712 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1206 | Regression_eta: 0.0013 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0663 | Regression_eta: 0.0283 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0531 | LR: 1.86e-04 | DataLoad Time: 0.0190s | Model Forward Time: 7.4607s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0077 | max_forward: 0.8168 | max_loss: 0.2161 | max_backward: 1.5709 | max_optimizer_step: 0.1733 | max_tensorboard_logging: 1.1681 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.1427 | Memory: rss_MB: 5816.6 | uss_MB: 3786.6 | child_rss_MB: 5665.4 | allocated_MB: 375.8 | reserved_MB: 24970.0 | inactive_split_MB: 259.2 +rank5 [2026-10-29 18:16:02,308] INFO : Running validation on rank5 +rank5 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 18:16:53,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:16:54,184] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 18:17:19,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 18:17:20,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 18:17:21,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 18:17:22,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 18:17:32,883] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 18:17:33,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:17:33,664] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 18:17:49,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 18:17:50,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 18:17:51,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 18:17:52,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 18:18:02,559] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 18:18:03,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:18:03,744] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 18:18:19,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 18:18:19,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 18:18:20,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 18:18:21,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 18:18:31,303] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 18:22:46,459] INFO : Step 40100/50000 rank5 | Train Loss: 0.9349 | Train Loss Unweighted: 0.4412 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1220 | Classification: 0.0762 | Classification_binary: 0.1328 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1148 | Regression_eta: 0.0012 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1352 | Regression_cos_phi: 0.0269 | Regression_energy: 0.0671 | Regression_eta: 0.0259 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0341 | LR: 1.84e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.6885s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.7093 | max_loss: 0.3974 | max_backward: 1.3310 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.5555 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 7103.1 | uss_MB: 5953.7 | child_rss_MB: 5672.5 | allocated_MB: 367.6 | reserved_MB: 22178.0 | inactive_split_MB: 265.9 +rank5 [2026-10-29 18:26:52,186] INFO : Step 40200/50000 rank5 | Train Loss: 0.9570 | Train Loss Unweighted: 0.4556 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1119 | Classification: 0.0905 | Classification_binary: 0.1293 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0013 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0562 | Regression_eta: 0.0351 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0436 | LR: 1.83e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.5883s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6287 | max_loss: 0.4129 | max_backward: 1.3901 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 0.5609 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5918.3 | uss_MB: 4768.5 | child_rss_MB: 5665.4 | allocated_MB: 370.8 | reserved_MB: 22294.0 | inactive_split_MB: 233.2 +rank5 [2026-10-29 18:30:59,535] INFO : Step 40300/50000 rank5 | Train Loss: 1.0209 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1232 | Classification: 0.0991 | Classification_binary: 0.1688 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1068 | Regression_eta: 0.0015 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0526 | Regression_eta: 0.0401 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0474 | LR: 1.81e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9649s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0749 | max_forward: 0.7563 | max_loss: 0.1523 | max_backward: 1.5452 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5632 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5917.0 | uss_MB: 4772.2 | child_rss_MB: 5652.3 | allocated_MB: 371.8 | reserved_MB: 23296.0 | inactive_split_MB: 234.2 +rank5 [2026-10-29 18:35:20,932] INFO : Step 40400/50000 rank5 | Train Loss: 1.0486 | Train Loss Unweighted: 0.5003 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1357 | Classification: 0.0888 | Classification_binary: 0.1690 | HitGrouping: 0.1879 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1196 | Regression_eta: 0.0015 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0299 | Regression_energy: 0.0459 | Regression_eta: 0.0286 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0454 | LR: 1.79e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.7349s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0072 | max_forward: 0.7368 | max_loss: 0.2443 | max_backward: 3.1596 | max_optimizer_step: 0.1851 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5913.1 | uss_MB: 3802.2 | child_rss_MB: 5663.8 | allocated_MB: 371.2 | reserved_MB: 23386.0 | inactive_split_MB: 254.8 +rank5 [2026-10-29 18:39:23,871] INFO : Step 40500/50000 rank5 | Train Loss: 0.9309 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.0955 | Classification: 0.0829 | Classification_binary: 0.1355 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0014 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0622 | Regression_eta: 0.0341 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0390 | LR: 1.78e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7230s | Phase Times: max_pre_step_barrier: 0.0186 | max_batch_to_device: 0.0076 | max_forward: 0.6863 | max_loss: 0.2393 | max_backward: 1.3065 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.7004 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5916.1 | uss_MB: 4771.6 | child_rss_MB: 5669.9 | allocated_MB: 369.2 | reserved_MB: 24370.0 | inactive_split_MB: 237.3 +rank5 [2026-10-29 18:43:23,923] INFO : Step 40600/50000 rank5 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1087 | Classification: 0.0887 | Classification_binary: 0.1521 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0603 | Regression_eta: 0.0338 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0337 | LR: 1.76e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3450s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.1102 | max_forward: 0.7412 | max_loss: 0.1464 | max_backward: 1.4782 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 1.0719 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5917.7 | uss_MB: 4772.7 | child_rss_MB: 5655.3 | allocated_MB: 372.3 | reserved_MB: 24370.0 | inactive_split_MB: 234.4 +rank5 [2026-10-29 18:47:14,843] INFO : Step 40700/50000 rank5 | Train Loss: 1.0611 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1374 | Classification: 0.0904 | Classification_binary: 0.1773 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1190 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1788 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0636 | Regression_eta: 0.0497 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0638 | LR: 1.75e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2478s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7422 | max_loss: 0.1589 | max_backward: 1.5298 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5918.6 | uss_MB: 4774.9 | child_rss_MB: 5665.5 | allocated_MB: 372.7 | reserved_MB: 24370.0 | inactive_split_MB: 255.3 +rank5 [2026-10-29 18:51:03,370] INFO : Step 40800/50000 rank5 | Train Loss: 0.9701 | Train Loss Unweighted: 0.4785 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.0983 | Classification: 0.0901 | Classification_binary: 0.1373 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0544 | Regression_eta: 0.0315 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0454 | LR: 1.73e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7292s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5843 | max_loss: 0.1422 | max_backward: 1.0661 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.0389 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5915.8 | uss_MB: 4772.0 | child_rss_MB: 5661.2 | allocated_MB: 367.5 | reserved_MB: 24658.0 | inactive_split_MB: 205.4 +rank5 [2026-10-29 18:54:43,248] INFO : Step 40900/50000 rank5 | Train Loss: 0.9731 | Train Loss Unweighted: 0.4636 | Train Loss Components: Auxiliary: 0.2169 | Cardinality: 0.1073 | Classification: 0.0883 | Classification_binary: 0.1426 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1140 | Regression_eta: 0.0013 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1395 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0619 | Regression_eta: 0.0422 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0462 | LR: 1.72e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8714s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.1047 | max_forward: 0.7107 | max_loss: 0.1679 | max_backward: 1.3343 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.8664 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5919.0 | uss_MB: 4778.3 | child_rss_MB: 5681.4 | allocated_MB: 372.0 | reserved_MB: 25640.0 | inactive_split_MB: 260.9 +rank5 [2026-10-29 18:58:25,754] INFO : Step 41000/50000 rank5 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1031 | Classification: 0.0930 | Classification_binary: 0.1618 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1164 | Regression_eta: 0.0014 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0645 | Regression_eta: 0.0554 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0656 | LR: 1.70e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4161s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.7604 | max_loss: 0.2558 | max_backward: 1.7603 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.8399 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 5926.0 | uss_MB: 4782.0 | child_rss_MB: 5654.3 | allocated_MB: 371.9 | reserved_MB: 25640.0 | inactive_split_MB: 195.8 +rank5 [2026-10-29 18:58:26,765] INFO : Running validation on rank5 +rank5 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 18:59:14,711] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:59:15,274] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 18:59:40,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 18:59:41,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 18:59:42,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 18:59:43,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 18:59:53,205] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:59:53,543] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:59:54,025] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 19:00:09,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 19:00:10,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 19:00:11,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 19:00:12,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 19:00:22,506] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 19:00:23,208] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:00:23,685] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 19:00:38,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 19:00:39,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 19:00:39,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 19:00:40,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 19:00:50,666] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 19:04:58,255] INFO : Step 41100/50000 rank5 | Train Loss: 1.0684 | Train Loss Unweighted: 0.5053 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1438 | Classification: 0.0856 | Classification_binary: 0.1616 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1271 | Regression_eta: 0.0014 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1673 | Regression_cos_phi: 0.0957 | Regression_energy: 0.0628 | Regression_eta: 0.0459 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0687 | LR: 1.69e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.3308s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0066 | max_forward: 0.8006 | max_loss: 0.1726 | max_backward: 1.6699 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.6214 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0566 | Memory: rss_MB: 7184.9 | uss_MB: 5901.7 | child_rss_MB: 5647.6 | allocated_MB: 371.5 | reserved_MB: 22898.0 | inactive_split_MB: 258.4 +rank5 [2026-10-29 19:08:41,622] INFO : Step 41200/50000 rank5 | Train Loss: 0.9455 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1027 | Classification: 0.0887 | Classification_binary: 0.1242 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1223 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1414 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0647 | Regression_eta: 0.0415 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0737 | LR: 1.67e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.4116s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0589 | max_forward: 0.6517 | max_loss: 0.1401 | max_backward: 1.1761 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5685 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5984.6 | uss_MB: 4847.1 | child_rss_MB: 5628.1 | allocated_MB: 370.2 | reserved_MB: 24878.0 | inactive_split_MB: 199.0 +rank5 [2026-10-29 19:12:28,247] INFO : Step 41300/50000 rank5 | Train Loss: 0.9373 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1011 | Classification: 0.0853 | Classification_binary: 0.1448 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0013 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0541 | Regression_eta: 0.0434 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0358 | LR: 1.66e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.8986s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.8726 | max_loss: 0.1617 | max_backward: 1.6974 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5417 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5987.6 | uss_MB: 4852.4 | child_rss_MB: 5623.6 | allocated_MB: 370.7 | reserved_MB: 24878.0 | inactive_split_MB: 233.6 +rank5 [2026-10-29 19:16:15,463] INFO : Step 41400/50000 rank5 | Train Loss: 1.0355 | Train Loss Unweighted: 0.4965 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1363 | Classification: 0.0957 | Classification_binary: 0.1654 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1162 | Regression_eta: 0.0012 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1512 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0593 | Regression_eta: 0.0262 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0578 | LR: 1.64e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9188s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7129 | max_loss: 0.2460 | max_backward: 1.6340 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5967.8 | uss_MB: 4849.9 | child_rss_MB: 5563.5 | allocated_MB: 372.0 | reserved_MB: 24878.0 | inactive_split_MB: 305.4 +rank5 [2026-10-29 19:19:52,653] INFO : Step 41500/50000 rank5 | Train Loss: 0.9689 | Train Loss Unweighted: 0.4689 | Train Loss Components: Auxiliary: 0.2118 | Cardinality: 0.1040 | Classification: 0.0862 | Classification_binary: 0.1340 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1227 | Regression_eta: 0.0012 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1386 | Regression_cos_phi: 0.0972 | Regression_energy: 0.0760 | Regression_eta: 0.0392 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0614 | LR: 1.63e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.4447s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.6415 | max_loss: 0.1502 | max_backward: 1.1845 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5935.8 | uss_MB: 4853.4 | child_rss_MB: 5403.5 | allocated_MB: 368.5 | reserved_MB: 24878.0 | inactive_split_MB: 173.5 +rank5 [2026-10-29 19:23:40,392] INFO : Step 41600/50000 rank5 | Train Loss: 0.9559 | Train Loss Unweighted: 0.4625 | Train Loss Components: Auxiliary: 0.2141 | Cardinality: 0.1097 | Classification: 0.0872 | Classification_binary: 0.1298 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1211 | Regression_eta: 0.0012 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1323 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0567 | Regression_eta: 0.0319 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0693 | LR: 1.61e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.5871s | Phase Times: max_pre_step_barrier: 0.0755 | max_batch_to_device: 0.0062 | max_forward: 0.6849 | max_loss: 0.2047 | max_backward: 1.1646 | max_optimizer_step: 0.2235 | max_tensorboard_logging: 0.5806 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5909.3 | uss_MB: 4827.6 | child_rss_MB: 5308.7 | allocated_MB: 370.4 | reserved_MB: 24878.0 | inactive_split_MB: 287.4 +rank5 [2026-10-29 19:27:23,743] INFO : Step 41700/50000 rank5 | Train Loss: 1.0204 | Train Loss Unweighted: 0.4905 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1165 | Classification: 0.0860 | Classification_binary: 0.1552 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1517 | Regression_cos_phi: 0.0765 | Regression_energy: 0.0876 | Regression_eta: 0.0623 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0698 | LR: 1.60e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.0328s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0773 | max_forward: 0.7754 | max_loss: 0.1486 | max_backward: 1.5759 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 0.5773 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5907.3 | uss_MB: 4857.3 | child_rss_MB: 5302.2 | allocated_MB: 370.8 | reserved_MB: 24878.0 | inactive_split_MB: 238.3 +rank5 [2026-10-29 19:31:24,300] INFO : Step 41800/50000 rank5 | Train Loss: 1.0101 | Train Loss Unweighted: 0.4703 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1282 | Classification: 0.0818 | Classification_binary: 0.1618 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0401 | Regression_energy: 0.0536 | Regression_eta: 0.0290 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0529 | LR: 1.58e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9771s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8147 | max_loss: 0.4259 | max_backward: 1.4886 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1200 | Memory: rss_MB: 5899.0 | uss_MB: 4855.0 | child_rss_MB: 5276.1 | allocated_MB: 371.9 | reserved_MB: 25802.0 | inactive_split_MB: 266.5 +rank5 [2026-10-29 19:35:32,345] INFO : Step 41900/50000 rank5 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4672 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1185 | Classification: 0.0878 | Classification_binary: 0.1282 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1241 | Regression_eta: 0.0011 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1117 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0478 | Regression_eta: 0.0262 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0605 | LR: 1.57e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.4617s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0473 | max_forward: 0.6783 | max_loss: 0.1431 | max_backward: 1.2347 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.5634 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5868.0 | uss_MB: 4858.2 | child_rss_MB: 5156.0 | allocated_MB: 370.8 | reserved_MB: 25802.0 | inactive_split_MB: 166.5 +rank5 [2026-10-29 19:39:32,017] INFO : Step 42000/50000 rank5 | Train Loss: 0.9837 | Train Loss Unweighted: 0.4719 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.1093 | Classification: 0.1015 | Classification_binary: 0.1413 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1128 | Regression_eta: 0.0014 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0513 | Regression_eta: 0.0488 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0929 | LR: 1.56e-04 | DataLoad Time: 0.0187s | Model Forward Time: 3.6414s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6848 | max_loss: 0.2509 | max_backward: 1.3746 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5866.0 | uss_MB: 4860.4 | child_rss_MB: 5161.1 | allocated_MB: 370.2 | reserved_MB: 25802.0 | inactive_split_MB: 219.9 +rank5 [2026-10-29 19:39:32,941] INFO : Running validation on rank5 +rank5 [2026-10-29 19:39:32,941] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 19:39:32,941] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 19:40:23,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:40:24,170] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 19:40:53,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 19:40:54,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 19:40:55,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 19:40:55,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 19:41:05,882] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 19:41:06,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:41:06,795] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 19:41:23,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 19:41:25,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 19:41:26,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 19:41:27,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 19:41:37,091] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 19:41:38,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:41:38,675] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 19:41:54,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 19:41:54,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 19:41:55,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 19:41:56,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 19:42:06,188] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 19:46:29,430] INFO : Step 42100/50000 rank5 | Train Loss: 1.0639 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1434 | Classification: 0.0999 | Classification_binary: 0.1737 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1097 | Regression_eta: 0.0012 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1781 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0526 | Regression_eta: 0.0335 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0458 | LR: 1.54e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.0374s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.7424 | max_loss: 0.1586 | max_backward: 1.4434 | max_optimizer_step: 0.3014 | max_tensorboard_logging: 0.6771 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 7018.7 | uss_MB: 6010.1 | child_rss_MB: 5164.1 | allocated_MB: 370.8 | reserved_MB: 23298.0 | inactive_split_MB: 265.3 +rank5 [2026-10-29 19:50:21,662] INFO : Step 42200/50000 rank5 | Train Loss: 1.0152 | Train Loss Unweighted: 0.4831 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1169 | Classification: 0.0913 | Classification_binary: 0.1515 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1190 | Regression_eta: 0.0015 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1570 | Regression_cos_phi: 0.0436 | Regression_energy: 0.0427 | Regression_eta: 0.0380 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0392 | LR: 1.53e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.5437s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.6567 | max_loss: 0.2000 | max_backward: 1.2379 | max_optimizer_step: 0.1803 | max_tensorboard_logging: 0.5782 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5943.7 | uss_MB: 4948.2 | child_rss_MB: 5171.4 | allocated_MB: 370.9 | reserved_MB: 24984.0 | inactive_split_MB: 272.4 +rank5 [2026-10-29 19:51:24,069] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank5 [2026-10-29 19:51:24,069] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-29 19:54:15,429] INFO : Step 42300/50000 rank5 | Train Loss: 1.0224 | Train Loss Unweighted: 0.4869 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.1261 | Classification: 0.0943 | Classification_binary: 0.1704 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0371 | Regression_energy: 0.0487 | Regression_eta: 0.0272 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0325 | LR: 1.52e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7704s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7065 | max_loss: 0.2237 | max_backward: 1.4902 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.5353 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6010.9 | uss_MB: 5019.0 | child_rss_MB: 5170.2 | allocated_MB: 371.9 | reserved_MB: 24984.0 | inactive_split_MB: 183.6 +rank5 [2026-10-29 19:58:13,284] INFO : Step 42400/50000 rank5 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4895 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1253 | Classification: 0.0872 | Classification_binary: 0.1616 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1184 | Regression_eta: 0.0013 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0553 | Regression_eta: 0.0467 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0428 | LR: 1.50e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7969s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7436 | max_loss: 0.2477 | max_backward: 1.5279 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5169 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5933.8 | uss_MB: 4947.5 | child_rss_MB: 5164.9 | allocated_MB: 373.0 | reserved_MB: 24984.0 | inactive_split_MB: 275.6 +rank5 [2026-10-29 20:02:13,260] INFO : Step 42500/50000 rank5 | Train Loss: 0.9211 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1056 | Classification: 0.0789 | Classification_binary: 0.1296 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1162 | Regression_eta: 0.0011 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1072 | Regression_cos_phi: 0.0325 | Regression_energy: 0.0561 | Regression_eta: 0.0294 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0545 | LR: 1.49e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.5537s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.6196 | max_loss: 0.2440 | max_backward: 1.3403 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 2.3828 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5926.8 | uss_MB: 4949.7 | child_rss_MB: 5158.5 | allocated_MB: 371.3 | reserved_MB: 25980.0 | inactive_split_MB: 274.9 +rank5 [2026-10-29 20:06:17,859] INFO : Step 42600/50000 rank5 | Train Loss: 0.9688 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1101 | Classification: 0.0874 | Classification_binary: 0.1472 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0012 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0297 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0487 | Regression_eta: 0.0243 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0470 | LR: 1.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9494s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6353 | max_loss: 0.2547 | max_backward: 1.3961 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 5918.3 | uss_MB: 4947.3 | child_rss_MB: 5167.3 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 272.5 +rank5 [2026-10-29 20:10:16,914] INFO : Step 42700/50000 rank5 | Train Loss: 1.0476 | Train Loss Unweighted: 0.4808 | Train Loss Components: Auxiliary: 0.2377 | Cardinality: 0.1284 | Classification: 0.0883 | Classification_binary: 0.1739 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1071 | Regression_eta: 0.0014 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0444 | Regression_eta: 0.0354 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0774 | LR: 1.47e-04 | DataLoad Time: 0.0085s | Model Forward Time: 5.7698s | Phase Times: max_pre_step_barrier: 0.0320 | max_batch_to_device: 0.0114 | max_forward: 2.1726 | max_loss: 0.5101 | max_backward: 2.7494 | max_optimizer_step: 0.2679 | max_tensorboard_logging: 0.6520 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5921.8 | uss_MB: 4952.4 | child_rss_MB: 5151.8 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 482.9 +rank5 [2026-10-29 20:14:17,049] INFO : Step 42800/50000 rank5 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4792 | Train Loss Components: Auxiliary: 0.2262 | Cardinality: 0.1277 | Classification: 0.0841 | Classification_binary: 0.1487 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1209 | Regression_eta: 0.0012 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0374 | Regression_energy: 0.0590 | Regression_eta: 0.0333 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0478 | LR: 1.45e-04 | DataLoad Time: 0.0206s | Model Forward Time: 6.2797s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0175 | max_forward: 1.3870 | max_loss: 0.5106 | max_backward: 1.9221 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 2.4017 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5924.3 | uss_MB: 4952.5 | child_rss_MB: 5165.0 | allocated_MB: 372.6 | reserved_MB: 25980.0 | inactive_split_MB: 234.4 +rank5 [2026-10-29 20:18:09,439] INFO : Step 42900/50000 rank5 | Train Loss: 0.8847 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.0993 | Classification: 0.0847 | Classification_binary: 0.1207 | HitGrouping: 0.1828 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1069 | Regression_eta: 0.0011 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1121 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0539 | Regression_eta: 0.0245 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0489 | LR: 1.44e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7396s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.6133 | max_loss: 0.1478 | max_backward: 1.3445 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.7551 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0762 | Memory: rss_MB: 5924.9 | uss_MB: 4957.1 | child_rss_MB: 5179.7 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 276.9 +rank5 [2026-10-29 20:22:02,171] INFO : Step 43000/50000 rank5 | Train Loss: 0.9800 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1096 | Classification: 0.0987 | Classification_binary: 0.1606 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0012 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1569 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0609 | Regression_eta: 0.0339 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0540 | LR: 1.43e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.9030s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0071 | max_forward: 0.7477 | max_loss: 0.1477 | max_backward: 1.5663 | max_optimizer_step: 0.1600 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 5929.7 | uss_MB: 4961.8 | child_rss_MB: 5171.4 | allocated_MB: 369.1 | reserved_MB: 25980.0 | inactive_split_MB: 209.4 +rank5 [2026-10-29 20:22:03,144] INFO : Running validation on rank5 +rank5 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 20:22:51,801] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:22:52,330] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 20:23:18,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 20:23:19,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 20:23:20,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 20:23:21,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 20:23:31,484] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 20:23:31,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:23:32,370] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 20:23:48,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 20:23:49,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 20:23:50,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 20:23:50,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 20:24:00,999] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 20:24:01,725] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:24:02,215] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 20:24:17,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 20:24:18,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 20:24:18,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 20:24:19,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 20:24:29,628] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 20:28:29,103] INFO : Step 43100/50000 rank5 | Train Loss: 0.9692 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1151 | Classification: 0.0766 | Classification_binary: 0.1586 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1062 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1693 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0509 | Regression_eta: 0.0315 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0502 | LR: 1.42e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7247s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.8051 | max_loss: 0.1438 | max_backward: 1.3224 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 7068.8 | uss_MB: 6101.3 | child_rss_MB: 5191.0 | allocated_MB: 370.5 | reserved_MB: 21096.0 | inactive_split_MB: 247.5 +rank5 [2026-10-29 20:32:30,812] INFO : Step 43200/50000 rank5 | Train Loss: 0.9431 | Train Loss Unweighted: 0.4458 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1175 | Classification: 0.0811 | Classification_binary: 0.1294 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0012 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0488 | Regression_eta: 0.0280 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0470 | LR: 1.40e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.5217s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0120 | max_forward: 0.6342 | max_loss: 0.1417 | max_backward: 1.2752 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6068.4 | uss_MB: 5101.7 | child_rss_MB: 5194.2 | allocated_MB: 370.4 | reserved_MB: 22140.0 | inactive_split_MB: 198.0 +rank5 [2026-10-29 20:36:16,429] INFO : Step 43300/50000 rank5 | Train Loss: 1.0382 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1221 | Classification: 0.1031 | Classification_binary: 0.1555 | HitGrouping: 0.1894 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1173 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0547 | Regression_eta: 0.0215 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0457 | LR: 1.39e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2400s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0266 | max_forward: 0.7527 | max_loss: 0.1476 | max_backward: 1.6633 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.7701 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6071.5 | uss_MB: 5107.2 | child_rss_MB: 5175.9 | allocated_MB: 371.1 | reserved_MB: 23082.0 | inactive_split_MB: 236.4 +rank5 [2026-10-29 20:40:05,495] INFO : Step 43400/50000 rank5 | Train Loss: 1.0532 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1292 | Classification: 0.0981 | Classification_binary: 0.1785 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0015 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0464 | Regression_eta: 0.0389 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0626 | LR: 1.38e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2968s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0691 | max_forward: 0.8057 | max_loss: 0.1503 | max_backward: 1.8319 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.6703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6067.0 | uss_MB: 5102.4 | child_rss_MB: 5192.5 | allocated_MB: 371.5 | reserved_MB: 23082.0 | inactive_split_MB: 776.0 +rank5 [2026-10-29 20:43:57,009] INFO : Step 43500/50000 rank5 | Train Loss: 0.9308 | Train Loss Unweighted: 0.4531 | Train Loss Components: Auxiliary: 0.2084 | Cardinality: 0.0966 | Classification: 0.0846 | Classification_binary: 0.1384 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1135 | Regression_eta: 0.0013 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0443 | Regression_eta: 0.0303 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0280 | LR: 1.37e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6269s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0171 | max_forward: 0.6718 | max_loss: 0.1455 | max_backward: 1.4141 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 6070.4 | uss_MB: 5108.5 | child_rss_MB: 5191.0 | allocated_MB: 372.0 | reserved_MB: 23082.0 | inactive_split_MB: 228.0 +rank5 [2026-10-29 20:47:52,743] INFO : Step 43600/50000 rank5 | Train Loss: 0.9195 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.0926 | Classification: 0.0847 | Classification_binary: 0.1298 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1175 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0477 | Regression_eta: 0.0294 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0713 | LR: 1.36e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5154s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.6453 | max_loss: 0.1399 | max_backward: 1.3514 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0942 | Memory: rss_MB: 6067.9 | uss_MB: 5106.0 | child_rss_MB: 5190.1 | allocated_MB: 367.6 | reserved_MB: 23082.0 | inactive_split_MB: 270.4 +rank5 [2026-10-29 20:51:54,645] INFO : Step 43700/50000 rank5 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1181 | Classification: 0.0916 | Classification_binary: 0.1553 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1009 | Regression_eta: 0.0014 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0542 | Regression_eta: 0.0331 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0535 | LR: 1.35e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.0844s | Phase Times: max_pre_step_barrier: 0.0199 | max_batch_to_device: 0.0121 | max_forward: 0.8005 | max_loss: 0.2526 | max_backward: 1.9159 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 6075.4 | uss_MB: 5110.5 | child_rss_MB: 5168.5 | allocated_MB: 371.6 | reserved_MB: 23674.0 | inactive_split_MB: 256.4 +rank5 [2026-10-29 20:55:51,187] INFO : Step 43800/50000 rank5 | Train Loss: 1.0325 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1341 | Classification: 0.0892 | Classification_binary: 0.1542 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0012 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0572 | Regression_eta: 0.0351 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0710 | LR: 1.34e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.6839s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7618 | max_loss: 0.1715 | max_backward: 1.3818 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6073.5 | uss_MB: 5110.6 | child_rss_MB: 5186.9 | allocated_MB: 371.7 | reserved_MB: 23674.0 | inactive_split_MB: 282.3 +rank5 [2026-10-29 20:59:46,232] INFO : Step 43900/50000 rank5 | Train Loss: 0.9104 | Train Loss Unweighted: 0.4325 | Train Loss Components: Auxiliary: 0.2009 | Cardinality: 0.1078 | Classification: 0.0725 | Classification_binary: 0.1272 | HitGrouping: 0.1855 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1146 | Regression_eta: 0.0011 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1347 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0667 | Regression_eta: 0.0296 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0621 | LR: 1.33e-04 | DataLoad Time: 0.0211s | Model Forward Time: 3.3849s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.6145 | max_loss: 0.1412 | max_backward: 1.1415 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5764 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6071.8 | uss_MB: 5111.9 | child_rss_MB: 5178.4 | allocated_MB: 368.3 | reserved_MB: 23674.0 | inactive_split_MB: 193.1 +rank5 [2026-10-29 21:03:43,512] INFO : Step 44000/50000 rank5 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4573 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1078 | Classification: 0.0956 | Classification_binary: 0.1421 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1083 | Regression_eta: 0.0011 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1194 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0437 | Regression_eta: 0.0183 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0457 | LR: 1.32e-04 | DataLoad Time: 0.0185s | Model Forward Time: 3.9493s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0061 | max_forward: 1.1697 | max_loss: 0.2455 | max_backward: 1.7366 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6074.9 | uss_MB: 5118.1 | child_rss_MB: 5179.2 | allocated_MB: 370.7 | reserved_MB: 23674.0 | inactive_split_MB: 207.3 +rank5 [2026-10-29 21:03:44,527] INFO : Running validation on rank5 +rank5 [2026-10-29 21:03:44,528] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 21:03:44,528] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 21:04:32,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:04:33,227] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 21:04:58,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 21:04:59,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 21:05:01,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 21:05:02,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 21:05:12,035] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 21:05:12,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:05:13,000] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 21:05:29,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 21:05:29,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 21:05:30,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 21:05:32,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 21:05:42,205] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 21:05:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:05:42,731] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 21:05:57,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 21:05:58,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 21:05:59,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 21:05:59,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 21:06:09,857] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 21:10:20,117] INFO : Step 44100/50000 rank5 | Train Loss: 1.0009 | Train Loss Unweighted: 0.4605 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1400 | Classification: 0.0841 | Classification_binary: 0.1644 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1044 | Regression_eta: 0.0011 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1729 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0452 | Regression_eta: 0.0361 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0689 | LR: 1.31e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.5837s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 1.2763 | max_loss: 0.5349 | max_backward: 2.2676 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 1.2755 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 7076.5 | uss_MB: 6129.8 | child_rss_MB: 5164.2 | allocated_MB: 372.3 | reserved_MB: 24224.0 | inactive_split_MB: 272.2 +rank5 [2026-10-29 21:14:15,311] INFO : Step 44200/50000 rank5 | Train Loss: 0.9801 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1146 | Classification: 0.0882 | Classification_binary: 0.1482 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1167 | Regression_eta: 0.0014 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1636 | Regression_cos_phi: 0.0396 | Regression_energy: 0.0500 | Regression_eta: 0.0328 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0726 | LR: 1.30e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.0041s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0132 | max_forward: 0.6993 | max_loss: 0.1574 | max_backward: 1.3862 | max_optimizer_step: 0.2915 | max_tensorboard_logging: 0.7724 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6137.2 | uss_MB: 5191.5 | child_rss_MB: 5174.4 | allocated_MB: 370.2 | reserved_MB: 26182.0 | inactive_split_MB: 176.5 +rank5 [2026-10-29 21:17:58,578] INFO : Step 44300/50000 rank5 | Train Loss: 0.8787 | Train Loss Unweighted: 0.4332 | Train Loss Components: Auxiliary: 0.1952 | Cardinality: 0.0950 | Classification: 0.0869 | Classification_binary: 0.1153 | HitGrouping: 0.1807 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0011 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1113 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0578 | Regression_eta: 0.0259 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0355 | LR: 1.29e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.7430s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0142 | max_forward: 0.7699 | max_loss: 0.2088 | max_backward: 1.4049 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 0.4839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 6137.2 | uss_MB: 4195.8 | child_rss_MB: 5178.8 | allocated_MB: 367.2 | reserved_MB: 27204.0 | inactive_split_MB: 205.8 +rank5 [2026-10-29 21:21:59,800] INFO : Step 44400/50000 rank5 | Train Loss: 0.9558 | Train Loss Unweighted: 0.4680 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.0905 | Classification: 0.0911 | Classification_binary: 0.1587 | HitGrouping: 0.1858 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.1756 | Regression_cos_phi: 0.0837 | Regression_energy: 0.0554 | Regression_eta: 0.0459 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0482 | LR: 1.28e-04 | DataLoad Time: 0.0217s | Model Forward Time: 6.3266s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0370 | max_forward: 0.7829 | max_loss: 0.1727 | max_backward: 1.7862 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 2.7361 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6138.4 | uss_MB: 5206.0 | child_rss_MB: 5161.4 | allocated_MB: 373.1 | reserved_MB: 27586.0 | inactive_split_MB: 176.3 +rank5 [2026-10-29 21:26:11,866] INFO : Step 44500/50000 rank5 | Train Loss: 0.9739 | Train Loss Unweighted: 0.4610 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1139 | Classification: 0.0849 | Classification_binary: 0.1578 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1077 | Regression_eta: 0.0012 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0517 | Regression_eta: 0.0270 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0608 | LR: 1.27e-04 | DataLoad Time: 0.0129s | Model Forward Time: 3.9217s | Phase Times: max_pre_step_barrier: 0.0250 | max_batch_to_device: 0.0065 | max_forward: 0.7930 | max_loss: 0.1668 | max_backward: 1.3363 | max_optimizer_step: 0.2871 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6136.3 | uss_MB: 5200.8 | child_rss_MB: 5177.3 | allocated_MB: 371.5 | reserved_MB: 27586.0 | inactive_split_MB: 195.8 +rank5 [2026-10-29 21:30:04,561] INFO : Step 44600/50000 rank5 | Train Loss: 0.8904 | Train Loss Unweighted: 0.4364 | Train Loss Components: Auxiliary: 0.2007 | Cardinality: 0.1041 | Classification: 0.0775 | Classification_binary: 0.1238 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1161 | Regression_eta: 0.0010 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1005 | Regression_cos_phi: 0.0348 | Regression_energy: 0.0468 | Regression_eta: 0.0192 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0409 | LR: 1.26e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5137s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.6364 | max_loss: 0.1431 | max_backward: 1.3712 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.4900 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6139.4 | uss_MB: 5205.0 | child_rss_MB: 5167.5 | allocated_MB: 368.4 | reserved_MB: 27586.0 | inactive_split_MB: 170.2 +rank5 [2026-10-29 21:33:51,597] INFO : Step 44700/50000 rank5 | Train Loss: 1.0003 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1252 | Classification: 0.0899 | Classification_binary: 0.1520 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1117 | Regression_eta: 0.0013 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1318 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0451 | Regression_eta: 0.0358 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0323 | LR: 1.25e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.6385s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6952 | max_loss: 0.1455 | max_backward: 1.5015 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6145.1 | uss_MB: 5206.5 | child_rss_MB: 5162.4 | allocated_MB: 371.9 | reserved_MB: 28574.0 | inactive_split_MB: 214.1 +rank5 [2026-10-29 21:37:55,987] INFO : Step 44800/50000 rank5 | Train Loss: 0.9665 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1115 | Classification: 0.0889 | Classification_binary: 0.1605 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1042 | Regression_eta: 0.0011 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0483 | Regression_eta: 0.0295 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0506 | LR: 1.24e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.8735s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7593 | max_loss: 0.1924 | max_backward: 2.8086 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.6087 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 6148.4 | uss_MB: 5210.8 | child_rss_MB: 5160.0 | allocated_MB: 370.8 | reserved_MB: 28574.0 | inactive_split_MB: 257.7 +rank5 [2026-10-29 21:41:36,914] INFO : Step 44900/50000 rank5 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.0886 | Classification_binary: 0.1466 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1121 | Regression_eta: 0.0013 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1295 | Regression_cos_phi: 0.0384 | Regression_energy: 0.0559 | Regression_eta: 0.0354 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0664 | LR: 1.23e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.5191s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0101 | max_forward: 0.6531 | max_loss: 0.1431 | max_backward: 1.3230 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6145.9 | uss_MB: 5207.3 | child_rss_MB: 5182.8 | allocated_MB: 368.1 | reserved_MB: 28574.0 | inactive_split_MB: 178.8 +rank5 [2026-10-29 21:45:50,735] INFO : Step 45000/50000 rank5 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4443 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1053 | Classification: 0.0832 | Classification_binary: 0.1311 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0327 | Regression_energy: 0.0476 | Regression_eta: 0.0310 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0476 | LR: 1.22e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.1238s | Phase Times: max_pre_step_barrier: 0.0074 | max_batch_to_device: 0.0159 | max_forward: 1.1897 | max_loss: 0.2457 | max_backward: 1.2951 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.8342 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0919 | Memory: rss_MB: 6152.1 | uss_MB: 5213.7 | child_rss_MB: 5170.7 | allocated_MB: 368.8 | reserved_MB: 28576.0 | inactive_split_MB: 266.4 +rank5 [2026-10-29 21:45:51,739] INFO : Running validation on rank5 +rank5 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 21:46:40,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:46:41,116] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 21:47:08,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 21:47:09,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 21:47:11,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 21:47:12,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 21:47:22,356] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 21:47:23,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:47:23,689] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 21:47:39,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 21:47:40,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 21:47:41,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 21:47:42,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 21:47:52,043] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 21:47:53,405] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:47:53,966] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 21:48:09,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 21:48:10,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 21:48:11,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 21:48:12,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 21:48:22,063] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 21:52:56,452] INFO : Step 45100/50000 rank5 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1151 | Classification: 0.1010 | Classification_binary: 0.1562 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1118 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1379 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0453 | Regression_eta: 0.0439 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0613 | LR: 1.21e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3076s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7307 | max_loss: 0.4409 | max_backward: 1.5823 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 7066.7 | uss_MB: 6122.4 | child_rss_MB: 5160.9 | allocated_MB: 373.1 | reserved_MB: 23280.0 | inactive_split_MB: 227.2 +rank5 [2026-10-29 21:56:57,435] INFO : Step 45200/50000 rank5 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1108 | Classification: 0.0815 | Classification_binary: 0.1593 | HitGrouping: 0.1884 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1784 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0519 | Regression_eta: 0.0603 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0864 | LR: 1.20e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.2210s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.9482 | max_loss: 0.4476 | max_backward: 1.7165 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.4656 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7064.1 | uss_MB: 6120.3 | child_rss_MB: 5157.7 | allocated_MB: 373.0 | reserved_MB: 23338.0 | inactive_split_MB: 277.4 +rank5 [2026-10-29 22:01:28,910] INFO : Step 45300/50000 rank5 | Train Loss: 0.9312 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1141 | Classification: 0.0882 | Classification_binary: 0.1299 | HitGrouping: 0.1891 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0010 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0497 | Regression_eta: 0.0324 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0557 | LR: 1.19e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.4286s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6106 | max_loss: 0.2394 | max_backward: 1.1660 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 6271.7 | uss_MB: 5329.4 | child_rss_MB: 5157.3 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 223.2 +rank5 [2026-10-29 22:05:37,387] INFO : Step 45400/50000 rank5 | Train Loss: 0.9251 | Train Loss Unweighted: 0.4554 | Train Loss Components: Auxiliary: 0.2112 | Cardinality: 0.1034 | Classification: 0.0967 | Classification_binary: 0.1405 | HitGrouping: 0.1801 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1072 | Regression_eta: 0.0012 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0448 | Regression_eta: 0.0280 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0554 | LR: 1.19e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9535s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7158 | max_loss: 0.2520 | max_backward: 1.7727 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6276.9 | uss_MB: 5335.5 | child_rss_MB: 5162.7 | allocated_MB: 372.6 | reserved_MB: 23338.0 | inactive_split_MB: 212.6 +rank5 [2026-10-29 22:09:49,791] INFO : Step 45500/50000 rank5 | Train Loss: 1.0136 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1238 | Classification: 0.0902 | Classification_binary: 0.1661 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1060 | Regression_eta: 0.0013 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0508 | Regression_eta: 0.0383 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0649 | LR: 1.18e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.1580s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0147 | max_forward: 1.9023 | max_loss: 0.2979 | max_backward: 3.5478 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5661 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 6272.0 | uss_MB: 4281.7 | child_rss_MB: 5159.4 | allocated_MB: 372.4 | reserved_MB: 23338.0 | inactive_split_MB: 256.1 +rank5 [2026-10-29 22:14:16,023] INFO : Step 45600/50000 rank5 | Train Loss: 0.9675 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2172 | Cardinality: 0.1193 | Classification: 0.0849 | Classification_binary: 0.1400 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1218 | Regression_eta: 0.0010 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1510 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0628 | Regression_eta: 0.0364 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0447 | LR: 1.17e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.7198s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6781 | max_loss: 0.1576 | max_backward: 1.5089 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 0.5447 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6272.0 | uss_MB: 5337.5 | child_rss_MB: 5161.4 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 216.2 +rank5 [2026-10-29 22:18:15,631] INFO : Step 45700/50000 rank5 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1961 | Cardinality: 0.0908 | Classification: 0.0837 | Classification_binary: 0.1223 | HitGrouping: 0.1794 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1108 | Regression_eta: 0.0011 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1170 | Regression_cos_phi: 0.0381 | Regression_energy: 0.0533 | Regression_eta: 0.0210 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0389 | LR: 1.16e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.4735s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6216 | max_loss: 0.1658 | max_backward: 1.1797 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6234.4 | uss_MB: 5336.4 | child_rss_MB: 5162.7 | allocated_MB: 370.8 | reserved_MB: 23338.0 | inactive_split_MB: 226.6 +rank5 [2026-10-29 22:22:19,535] INFO : Step 45800/50000 rank5 | Train Loss: 0.9372 | Train Loss Unweighted: 0.4491 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1073 | Classification: 0.0880 | Classification_binary: 0.1481 | HitGrouping: 0.1781 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1067 | Regression_cos_phi: 0.0401 | Regression_energy: 0.0504 | Regression_eta: 0.0171 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0459 | LR: 1.16e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2356s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8017 | max_loss: 0.2428 | max_backward: 1.9917 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0978 | Memory: rss_MB: 6240.7 | uss_MB: 5347.0 | child_rss_MB: 5162.1 | allocated_MB: 375.2 | reserved_MB: 23338.0 | inactive_split_MB: 268.8 +rank5 [2026-10-29 22:26:26,931] INFO : Step 45900/50000 rank5 | Train Loss: 0.9600 | Train Loss Unweighted: 0.4592 | Train Loss Components: Auxiliary: 0.2179 | Cardinality: 0.1027 | Classification: 0.0816 | Classification_binary: 0.1590 | HitGrouping: 0.1875 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1691 | Regression_cos_phi: 0.0384 | Regression_energy: 0.0471 | Regression_eta: 0.0391 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0559 | LR: 1.15e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.8433s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 1.3210 | max_loss: 0.4294 | max_backward: 3.2770 | max_optimizer_step: 0.2740 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 6235.6 | uss_MB: 5343.7 | child_rss_MB: 5161.3 | allocated_MB: 370.5 | reserved_MB: 23338.0 | inactive_split_MB: 259.6 +rank5 [2026-10-29 22:30:40,886] INFO : Step 46000/50000 rank5 | Train Loss: 0.8490 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.0938 | Classification: 0.0817 | Classification_binary: 0.1110 | HitGrouping: 0.1819 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0010 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1047 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0460 | Regression_eta: 0.0260 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0269 | LR: 1.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5923s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.5955 | max_loss: 0.1430 | max_backward: 1.2483 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 0.6657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 6228.9 | uss_MB: 5344.9 | child_rss_MB: 5164.3 | allocated_MB: 365.7 | reserved_MB: 23338.0 | inactive_split_MB: 222.2 +rank5 [2026-10-29 22:30:41,828] INFO : Running validation on rank5 +rank5 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 22:31:32,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:31:33,519] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 22:32:00,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 22:32:01,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 22:32:03,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 22:32:05,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 22:32:15,144] INFO : Time taken to make predictions on device 5 is: 0.28 min +rank5 [2026-10-29 22:32:15,390] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:32:16,223] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 22:32:32,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 22:32:33,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 22:32:34,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 22:32:35,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 22:32:45,323] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 22:32:45,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:32:46,020] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 22:33:01,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 22:33:02,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 22:33:02,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 22:33:03,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 22:33:13,465] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 22:37:44,431] INFO : Step 46100/50000 rank5 | Train Loss: 0.9299 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1082 | Classification: 0.0899 | Classification_binary: 0.1357 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1081 | Regression_eta: 0.0010 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1438 | Regression_cos_phi: 0.0377 | Regression_energy: 0.0645 | Regression_eta: 0.0246 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0471 | LR: 1.13e-04 | DataLoad Time: 0.0195s | Model Forward Time: 6.9689s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 2.5146 | max_loss: 0.4579 | max_backward: 2.4958 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1014 | Memory: rss_MB: 6980.2 | uss_MB: 6096.7 | child_rss_MB: 5172.0 | allocated_MB: 371.5 | reserved_MB: 22784.0 | inactive_split_MB: 245.0 +rank5 [2026-10-29 22:41:43,512] INFO : Step 46200/50000 rank5 | Train Loss: 0.9944 | Train Loss Unweighted: 0.4707 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1286 | Classification: 0.0940 | Classification_binary: 0.1623 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1052 | Regression_eta: 0.0011 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1496 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0526 | Regression_eta: 0.0302 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0470 | LR: 1.13e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2042s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7685 | max_loss: 0.2359 | max_backward: 1.6855 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1253 | Memory: rss_MB: 6976.0 | uss_MB: 6094.5 | child_rss_MB: 5141.0 | allocated_MB: 373.1 | reserved_MB: 22784.0 | inactive_split_MB: 264.3 +rank5 [2026-10-29 22:45:39,133] INFO : Step 46300/50000 rank5 | Train Loss: 0.9564 | Train Loss Unweighted: 0.4515 | Train Loss Components: Auxiliary: 0.2114 | Cardinality: 0.1124 | Classification: 0.0869 | Classification_binary: 0.1435 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0528 | Regression_eta: 0.0393 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0542 | LR: 1.12e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.8636s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6709 | max_loss: 0.2410 | max_backward: 1.2189 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.8303 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6276.5 | uss_MB: 5413.4 | child_rss_MB: 5109.4 | allocated_MB: 372.3 | reserved_MB: 23784.0 | inactive_split_MB: 228.2 +rank5 [2026-10-29 22:49:34,649] INFO : Step 46400/50000 rank5 | Train Loss: 0.9514 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1147 | Classification: 0.0822 | Classification_binary: 0.1248 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0010 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0691 | Regression_eta: 0.0430 | Regression_pt: 0.0740 | Regression_sin_phi: 0.1565 | LR: 1.11e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.9538s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.6748 | max_loss: 0.4371 | max_backward: 1.4077 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.8211 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6270.0 | uss_MB: 5413.5 | child_rss_MB: 5075.5 | allocated_MB: 374.2 | reserved_MB: 23988.0 | inactive_split_MB: 241.8 +rank5 [2026-10-29 22:53:41,742] INFO : Step 46500/50000 rank5 | Train Loss: 0.9517 | Train Loss Unweighted: 0.4566 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.0937 | Classification: 0.0928 | Classification_binary: 0.1533 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1031 | Regression_eta: 0.0013 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0506 | Regression_eta: 0.0307 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0369 | LR: 1.11e-04 | DataLoad Time: 0.0143s | Model Forward Time: 3.9678s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0087 | max_forward: 0.7136 | max_loss: 0.1507 | max_backward: 1.5229 | max_optimizer_step: 0.2767 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6272.9 | uss_MB: 5416.3 | child_rss_MB: 5079.6 | allocated_MB: 372.0 | reserved_MB: 23988.0 | inactive_split_MB: 248.1 +rank5 [2026-10-29 22:57:41,614] INFO : Step 46600/50000 rank5 | Train Loss: 0.9541 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.2166 | Cardinality: 0.1078 | Classification: 0.0915 | Classification_binary: 0.1560 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0559 | Regression_eta: 0.0226 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0388 | LR: 1.10e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5726s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7127 | max_loss: 0.1604 | max_backward: 1.3079 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4722 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 6261.6 | uss_MB: 5413.8 | child_rss_MB: 5045.9 | allocated_MB: 370.5 | reserved_MB: 23988.0 | inactive_split_MB: 179.8 +rank5 [2026-10-29 23:02:40,690] INFO : Step 46700/50000 rank5 | Train Loss: 0.9378 | Train Loss Unweighted: 0.4497 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.0977 | Classification: 0.0857 | Classification_binary: 0.1264 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0015 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0803 | Regression_eta: 0.0541 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0738 | LR: 1.10e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.7512s | Phase Times: max_pre_step_barrier: 0.0197 | max_batch_to_device: 0.0481 | max_forward: 0.6699 | max_loss: 0.2593 | max_backward: 1.4356 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5914 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6261.6 | uss_MB: 5415.0 | child_rss_MB: 5040.6 | allocated_MB: 368.9 | reserved_MB: 24784.0 | inactive_split_MB: 229.2 +rank5 [2026-10-29 23:06:36,578] INFO : Step 46800/50000 rank5 | Train Loss: 0.9152 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.2022 | Cardinality: 0.1011 | Classification: 0.0910 | Classification_binary: 0.1376 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0988 | Regression_eta: 0.0014 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0583 | Regression_eta: 0.0735 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0795 | LR: 1.09e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6093s | Phase Times: max_pre_step_barrier: 0.1216 | max_batch_to_device: 0.0144 | max_forward: 1.0361 | max_loss: 0.2737 | max_backward: 1.7951 | max_optimizer_step: 0.2978 | max_tensorboard_logging: 0.5843 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6263.1 | uss_MB: 5417.5 | child_rss_MB: 5044.7 | allocated_MB: 368.9 | reserved_MB: 24994.0 | inactive_split_MB: 204.7 +rank5 [2026-10-29 23:10:45,824] INFO : Step 46900/50000 rank5 | Train Loss: 0.9278 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2149 | Cardinality: 0.1073 | Classification: 0.0827 | Classification_binary: 0.1607 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1027 | Regression_eta: 0.0010 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0282 | Regression_energy: 0.0495 | Regression_eta: 0.0174 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0425 | LR: 1.09e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0943s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0446 | max_forward: 0.7815 | max_loss: 0.1553 | max_backward: 1.4732 | max_optimizer_step: 0.2929 | max_tensorboard_logging: 0.6668 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6256.7 | uss_MB: 5416.7 | child_rss_MB: 5041.4 | allocated_MB: 369.3 | reserved_MB: 24994.0 | inactive_split_MB: 274.7 +rank5 [2026-10-29 23:14:40,386] INFO : Step 47000/50000 rank5 | Train Loss: 0.9023 | Train Loss Unweighted: 0.4500 | Train Loss Components: Auxiliary: 0.2093 | Cardinality: 0.0953 | Classification: 0.0802 | Classification_binary: 0.1325 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1175 | Regression_eta: 0.0010 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0617 | Regression_eta: 0.0432 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0454 | LR: 1.08e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.5079s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6624 | max_loss: 0.1440 | max_backward: 1.2028 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 6259.7 | uss_MB: 5421.0 | child_rss_MB: 5043.0 | allocated_MB: 369.4 | reserved_MB: 25784.0 | inactive_split_MB: 224.5 +rank5 [2026-10-29 23:14:41,333] INFO : Running validation on rank5 +rank5 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 23:15:32,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:15:33,157] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 23:16:02,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 23:16:03,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 23:16:05,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 23:16:05,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 23:16:15,886] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 23:16:16,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:16:16,906] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 23:16:33,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 23:16:35,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 23:16:35,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 23:16:36,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 23:16:46,927] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 23:16:47,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:16:47,863] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 23:17:03,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 23:17:04,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 23:17:04,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 23:17:05,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 23:17:15,560] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-29 23:21:36,066] INFO : Step 47100/50000 rank5 | Train Loss: 0.9138 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.2058 | Cardinality: 0.0938 | Classification: 0.0946 | Classification_binary: 0.1243 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1103 | Regression_eta: 0.0011 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1220 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0510 | Regression_eta: 0.0362 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0515 | LR: 1.07e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1754s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0063 | max_forward: 0.8048 | max_loss: 0.2433 | max_backward: 1.3159 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 1.0353 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 6952.3 | uss_MB: 6113.6 | child_rss_MB: 5037.2 | allocated_MB: 368.1 | reserved_MB: 22106.0 | inactive_split_MB: 269.0 +rank5 [2026-10-29 23:25:45,623] INFO : Step 47200/50000 rank5 | Train Loss: 0.9875 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2263 | Cardinality: 0.1259 | Classification: 0.0903 | Classification_binary: 0.1586 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1110 | Regression_eta: 0.0012 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0508 | Regression_eta: 0.0322 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0596 | LR: 1.07e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2835s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0486 | max_forward: 0.7581 | max_loss: 0.1506 | max_backward: 1.5914 | max_optimizer_step: 0.2300 | max_tensorboard_logging: 0.7953 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 6951.3 | uss_MB: 6117.8 | child_rss_MB: 5032.9 | allocated_MB: 370.5 | reserved_MB: 23104.0 | inactive_split_MB: 195.8 +rank5 [2026-10-29 23:29:53,763] INFO : Step 47300/50000 rank5 | Train Loss: 0.9350 | Train Loss Unweighted: 0.4467 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.0958 | Classification: 0.0857 | Classification_binary: 0.1512 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1284 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0526 | Regression_eta: 0.0363 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0504 | LR: 1.06e-04 | DataLoad Time: 0.0225s | Model Forward Time: 6.3995s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0184 | max_forward: 1.9890 | max_loss: 0.4771 | max_backward: 3.1656 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6309.1 | uss_MB: 5475.6 | child_rss_MB: 5028.4 | allocated_MB: 370.8 | reserved_MB: 23104.0 | inactive_split_MB: 271.1 +rank5 [2026-10-29 23:33:58,355] INFO : Step 47400/50000 rank5 | Train Loss: 0.8593 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.0807 | Classification: 0.0812 | Classification_binary: 0.1182 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1114 | Regression_eta: 0.0011 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1022 | Regression_cos_phi: 0.0331 | Regression_energy: 0.0581 | Regression_eta: 0.0230 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0414 | LR: 1.06e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.0119s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0125 | max_forward: 0.7544 | max_loss: 0.1826 | max_backward: 1.4299 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8027 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6311.1 | uss_MB: 5475.9 | child_rss_MB: 5031.1 | allocated_MB: 369.9 | reserved_MB: 23104.0 | inactive_split_MB: 253.6 +rank5 [2026-10-29 23:37:53,326] INFO : Step 47500/50000 rank5 | Train Loss: 0.9089 | Train Loss Unweighted: 0.4235 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1000 | Classification: 0.0799 | Classification_binary: 0.1391 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1008 | Regression_eta: 0.0012 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1213 | Regression_cos_phi: 0.0250 | Regression_energy: 0.0468 | Regression_eta: 0.0188 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0250 | LR: 1.06e-04 | DataLoad Time: 0.0232s | Model Forward Time: 4.2702s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0068 | max_forward: 0.7329 | max_loss: 0.1827 | max_backward: 1.5814 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9161 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6306.2 | uss_MB: 5478.3 | child_rss_MB: 5026.1 | allocated_MB: 371.7 | reserved_MB: 23104.0 | inactive_split_MB: 243.6 +rank5 [2026-10-29 23:41:48,171] INFO : Step 47600/50000 rank5 | Train Loss: 0.9781 | Train Loss Unweighted: 0.4635 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1226 | Classification: 0.0910 | Classification_binary: 0.1607 | HitGrouping: 0.1784 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0463 | Regression_eta: 0.0385 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0382 | LR: 1.05e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.4262s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8435 | max_loss: 0.1581 | max_backward: 1.8989 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5341 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6310.2 | uss_MB: 5483.0 | child_rss_MB: 5033.7 | allocated_MB: 370.6 | reserved_MB: 24122.0 | inactive_split_MB: 257.4 +rank5 [2026-10-29 23:46:00,212] INFO : Step 47700/50000 rank5 | Train Loss: 0.9311 | Train Loss Unweighted: 0.4523 | Train Loss Components: Auxiliary: 0.2105 | Cardinality: 0.0992 | Classification: 0.0889 | Classification_binary: 0.1321 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1138 | Regression_eta: 0.0012 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1166 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0424 | Regression_eta: 0.0204 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0226 | LR: 1.05e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.5060s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0068 | max_forward: 0.6708 | max_loss: 0.1613 | max_backward: 1.2326 | max_optimizer_step: 0.1946 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6306.7 | uss_MB: 5481.0 | child_rss_MB: 5031.8 | allocated_MB: 368.1 | reserved_MB: 24122.0 | inactive_split_MB: 276.0 +rank5 [2026-10-29 23:50:02,911] INFO : Step 47800/50000 rank5 | Train Loss: 0.9063 | Train Loss Unweighted: 0.4379 | Train Loss Components: Auxiliary: 0.2020 | Cardinality: 0.0932 | Classification: 0.0920 | Classification_binary: 0.1284 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0011 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1203 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0455 | Regression_eta: 0.0377 | Regression_pt: 0.0478 | Regression_sin_phi: 0.0610 | LR: 1.04e-04 | DataLoad Time: 0.0147s | Model Forward Time: 3.8454s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0610 | max_forward: 0.6626 | max_loss: 0.1349 | max_backward: 1.5091 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6302.7 | uss_MB: 5485.7 | child_rss_MB: 5029.8 | allocated_MB: 369.5 | reserved_MB: 24122.0 | inactive_split_MB: 236.5 +rank5 [2026-10-29 23:54:07,643] INFO : Step 47900/50000 rank5 | Train Loss: 0.9591 | Train Loss Unweighted: 0.4478 | Train Loss Components: Auxiliary: 0.2152 | Cardinality: 0.1022 | Classification: 0.0882 | Classification_binary: 0.1545 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1003 | Regression_eta: 0.0014 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0504 | Regression_eta: 0.0261 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0538 | LR: 1.04e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.1703s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8744 | max_loss: 0.2241 | max_backward: 1.6370 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 6302.2 | uss_MB: 5483.9 | child_rss_MB: 5030.3 | allocated_MB: 370.5 | reserved_MB: 24122.0 | inactive_split_MB: 218.9 +rank5 [2026-10-30 00:00:28,988] INFO : Step 48000/50000 rank5 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1221 | Classification: 0.0823 | Classification_binary: 0.1525 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1079 | Regression_eta: 0.0017 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1737 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0537 | Regression_eta: 0.0320 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0566 | LR: 1.04e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1277s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0057 | max_forward: 0.7069 | max_loss: 0.2447 | max_backward: 1.4090 | max_optimizer_step: 0.1933 | max_tensorboard_logging: 0.8132 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1250 | Memory: rss_MB: 6301.2 | uss_MB: 5485.0 | child_rss_MB: 5024.5 | allocated_MB: 368.8 | reserved_MB: 24122.0 | inactive_split_MB: 269.6 +rank5 [2026-10-30 00:00:29,931] INFO : Running validation on rank5 +rank5 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 00:01:19,557] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:01:20,066] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 00:01:46,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 00:01:48,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 00:01:49,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 00:01:50,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 00:02:00,487] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-30 00:02:01,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:02:01,859] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 00:02:17,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 00:02:18,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 00:02:19,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 00:02:20,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 00:02:30,683] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-30 00:02:32,510] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:02:32,991] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 00:02:48,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 00:02:48,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 00:02:49,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 00:02:50,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 00:03:00,088] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-30 00:07:25,204] INFO : Step 48100/50000 rank5 | Train Loss: 0.9084 | Train Loss Unweighted: 0.4360 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1162 | Classification: 0.0769 | Classification_binary: 0.1204 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1183 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1294 | Regression_cos_phi: 0.0303 | Regression_energy: 0.0557 | Regression_eta: 0.0217 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0353 | LR: 1.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9247s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1372 | max_backward: 1.2024 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.9158 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7374.2 | uss_MB: 6557.5 | child_rss_MB: 5038.7 | allocated_MB: 367.7 | reserved_MB: 24830.0 | inactive_split_MB: 222.3 +rank5 [2026-10-30 00:11:33,471] INFO : Step 48200/50000 rank5 | Train Loss: 0.9585 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2151 | Cardinality: 0.1199 | Classification: 0.0839 | Classification_binary: 0.1503 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0524 | Regression_eta: 0.0254 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0416 | LR: 1.03e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4360s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7766 | max_loss: 0.1498 | max_backward: 1.7414 | max_optimizer_step: 0.3242 | max_tensorboard_logging: 0.7504 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 7374.2 | uss_MB: 6560.6 | child_rss_MB: 5031.5 | allocated_MB: 370.4 | reserved_MB: 24830.0 | inactive_split_MB: 179.3 +rank5 [2026-10-30 00:15:41,697] INFO : Step 48300/50000 rank5 | Train Loss: 1.0575 | Train Loss Unweighted: 0.4963 | Train Loss Components: Auxiliary: 0.2408 | Cardinality: 0.1341 | Classification: 0.0958 | Classification_binary: 0.1687 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1138 | Regression_eta: 0.0013 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0962 | Regression_energy: 0.0619 | Regression_eta: 0.0411 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0447 | LR: 1.03e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2021s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7497 | max_loss: 0.2201 | max_backward: 1.7233 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6378.2 | uss_MB: 5566.5 | child_rss_MB: 5032.3 | allocated_MB: 372.3 | reserved_MB: 24830.0 | inactive_split_MB: 269.6 +rank5 [2026-10-30 00:19:56,563] INFO : Step 48400/50000 rank5 | Train Loss: 0.9108 | Train Loss Unweighted: 0.4349 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1093 | Classification: 0.0691 | Classification_binary: 0.1311 | HitGrouping: 0.1830 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1153 | Regression_eta: 0.0013 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0673 | Regression_eta: 0.0230 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0532 | LR: 1.02e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.8544s | Phase Times: max_pre_step_barrier: 0.1141 | max_batch_to_device: 0.0064 | max_forward: 0.6489 | max_loss: 0.1390 | max_backward: 1.1801 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6705 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6380.2 | uss_MB: 5566.3 | child_rss_MB: 5030.1 | allocated_MB: 370.1 | reserved_MB: 24830.0 | inactive_split_MB: 261.5 +rank5 [2026-10-30 00:23:53,558] INFO : Step 48500/50000 rank5 | Train Loss: 0.9694 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1004 | Classification: 0.0844 | Classification_binary: 0.1367 | HitGrouping: 0.1928 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1194 | Regression_eta: 0.0012 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1510 | Regression_cos_phi: 0.0961 | Regression_energy: 0.0694 | Regression_eta: 0.0478 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0793 | LR: 1.02e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7563s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 1.7512 | max_loss: 0.2678 | max_backward: 1.9538 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.4804 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 6374.4 | uss_MB: 5565.1 | child_rss_MB: 5023.6 | allocated_MB: 371.5 | reserved_MB: 26842.0 | inactive_split_MB: 176.5 +rank5 [2026-10-30 00:27:51,107] INFO : Step 48600/50000 rank5 | Train Loss: 0.9724 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1148 | Classification: 0.0854 | Classification_binary: 0.1551 | HitGrouping: 0.1844 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0576 | Regression_eta: 0.0468 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0691 | LR: 1.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0055s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0086 | max_forward: 0.6984 | max_loss: 0.2525 | max_backward: 1.6611 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6375.4 | uss_MB: 5569.8 | child_rss_MB: 5026.2 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 234.8 +rank5 [2026-10-30 00:31:52,433] INFO : Step 48700/50000 rank5 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4863 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1212 | Classification: 0.0968 | Classification_binary: 0.1569 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1143 | Regression_eta: 0.0012 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0924 | Regression_energy: 0.0572 | Regression_eta: 0.0592 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0849 | LR: 1.02e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9645s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7410 | max_loss: 0.2472 | max_backward: 1.4647 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.6585 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6316.1 | uss_MB: 5565.7 | child_rss_MB: 4823.4 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 275.8 +rank5 [2026-10-30 00:35:40,394] INFO : Step 48800/50000 rank5 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1981 | Cardinality: 0.0898 | Classification: 0.0814 | Classification_binary: 0.1196 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1094 | Regression_eta: 0.0012 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1389 | Regression_cos_phi: 0.0362 | Regression_energy: 0.0719 | Regression_eta: 0.0245 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0405 | LR: 1.01e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.7708s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.6122 | max_loss: 0.1775 | max_backward: 1.1932 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6316.1 | uss_MB: 5568.8 | child_rss_MB: 4813.9 | allocated_MB: 368.0 | reserved_MB: 26842.0 | inactive_split_MB: 263.7 +rank5 [2026-10-30 00:39:23,571] INFO : Step 48900/50000 rank5 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1005 | Classification: 0.0873 | Classification_binary: 0.1401 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1432 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0571 | Regression_eta: 0.0326 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0835 | LR: 1.01e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.9261s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0809 | max_forward: 0.7397 | max_loss: 0.1455 | max_backward: 1.5027 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6303.5 | uss_MB: 5491.8 | child_rss_MB: 4768.8 | allocated_MB: 373.0 | reserved_MB: 26842.0 | inactive_split_MB: 191.3 +rank5 [2026-10-30 00:43:08,795] INFO : Step 49000/50000 rank5 | Train Loss: 1.0019 | Train Loss Unweighted: 0.4657 | Train Loss Components: Auxiliary: 0.2334 | Cardinality: 0.1354 | Classification: 0.0893 | Classification_binary: 0.1587 | HitGrouping: 0.1838 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0010 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0435 | Regression_eta: 0.0348 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0926 | LR: 1.01e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.4553s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1085 | max_forward: 0.8072 | max_loss: 0.1558 | max_backward: 1.7809 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.6082 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6305.0 | uss_MB: 5577.9 | child_rss_MB: 4749.4 | allocated_MB: 370.9 | reserved_MB: 26842.0 | inactive_split_MB: 271.3 +rank5 [2026-10-30 00:43:09,868] INFO : Running validation on rank5 +rank5 [2026-10-30 00:43:09,869] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 00:43:09,869] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 00:43:59,277] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:43:59,788] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 00:44:27,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 00:44:28,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 00:44:29,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 00:44:30,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 00:44:40,256] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 00:44:40,646] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:44:41,122] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 00:44:56,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 00:44:57,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 00:44:58,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 00:44:59,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 00:45:09,778] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-30 00:45:10,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:45:10,743] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 00:45:25,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 00:45:26,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 00:45:27,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 00:45:27,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 00:45:37,839] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-30 00:49:42,470] INFO : Step 49100/50000 rank5 | Train Loss: 0.9797 | Train Loss Unweighted: 0.4561 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1125 | Classification: 0.0877 | Classification_binary: 0.1433 | HitGrouping: 0.1905 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0013 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1574 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0576 | Regression_eta: 0.0449 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0446 | LR: 1.01e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7317s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6869 | max_loss: 0.1436 | max_backward: 1.2639 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.4741 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0455 | Memory: rss_MB: 7470.2 | uss_MB: 6734.9 | child_rss_MB: 4778.3 | allocated_MB: 370.4 | reserved_MB: 22116.0 | inactive_split_MB: 274.9 +rank5 [2026-10-30 00:53:47,121] INFO : Step 49200/50000 rank5 | Train Loss: 0.8929 | Train Loss Unweighted: 0.4393 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1020 | Classification: 0.0903 | Classification_binary: 0.1250 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0009 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1293 | Regression_cos_phi: 0.0386 | Regression_energy: 0.0503 | Regression_eta: 0.0280 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0455 | LR: 1.01e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7533s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6890 | max_loss: 0.2430 | max_backward: 1.3427 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.6347 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7470.5 | uss_MB: 6738.6 | child_rss_MB: 4771.6 | allocated_MB: 368.3 | reserved_MB: 24092.0 | inactive_split_MB: 187.3 +rank5 [2026-10-30 00:57:33,941] INFO : Step 49300/50000 rank5 | Train Loss: 1.0122 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1210 | Classification: 0.0951 | Classification_binary: 0.1621 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1066 | Regression_eta: 0.0015 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0471 | Regression_eta: 0.0613 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0553 | LR: 1.00e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.8976s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1495 | max_backward: 1.5906 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6415.4 | uss_MB: 5684.6 | child_rss_MB: 4746.9 | allocated_MB: 370.9 | reserved_MB: 24092.0 | inactive_split_MB: 226.9 +rank5 [2026-10-30 01:01:33,235] INFO : Step 49400/50000 rank5 | Train Loss: 0.9790 | Train Loss Unweighted: 0.4695 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1137 | Classification: 0.0883 | Classification_binary: 0.1513 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1131 | Regression_eta: 0.0013 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0490 | Regression_eta: 0.0331 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0544 | LR: 1.00e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.0364s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2433 | max_backward: 1.8823 | max_optimizer_step: 0.3081 | max_tensorboard_logging: 0.5951 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6413.4 | uss_MB: 5608.6 | child_rss_MB: 4765.4 | allocated_MB: 372.2 | reserved_MB: 24092.0 | inactive_split_MB: 278.9 +rank5 [2026-10-30 01:05:37,233] INFO : Step 49500/50000 rank5 | Train Loss: 0.9706 | Train Loss Unweighted: 0.4581 | Train Loss Components: Auxiliary: 0.2120 | Cardinality: 0.1158 | Classification: 0.0879 | Classification_binary: 0.1279 | HitGrouping: 0.1920 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1136 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0536 | Regression_eta: 0.0365 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0564 | LR: 1.00e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9619s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6307 | max_loss: 0.1505 | max_backward: 1.3337 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.8584 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6409.4 | uss_MB: 5683.7 | child_rss_MB: 4759.9 | allocated_MB: 370.4 | reserved_MB: 24614.0 | inactive_split_MB: 171.1 +rank5 [2026-10-30 01:09:49,675] INFO : Step 49600/50000 rank5 | Train Loss: 0.9965 | Train Loss Unweighted: 0.4752 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1098 | Classification: 0.1016 | Classification_binary: 0.1437 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1133 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1373 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0524 | Regression_eta: 0.0295 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0651 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1632s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.6933 | max_loss: 0.1496 | max_backward: 1.5639 | max_optimizer_step: 0.3077 | max_tensorboard_logging: 0.7702 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6412.2 | uss_MB: 5687.4 | child_rss_MB: 4755.4 | allocated_MB: 371.3 | reserved_MB: 24614.0 | inactive_split_MB: 250.0 +rank5 [2026-10-30 01:13:44,418] INFO : Step 49700/50000 rank5 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4775 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1108 | Classification: 0.1000 | Classification_binary: 0.1702 | HitGrouping: 0.1796 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1019 | Regression_eta: 0.0014 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1675 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0574 | Regression_eta: 0.0395 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0613 | LR: 1.00e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.2329s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7591 | max_loss: 0.1933 | max_backward: 1.7410 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 6407.4 | uss_MB: 5675.2 | child_rss_MB: 4703.7 | allocated_MB: 372.4 | reserved_MB: 24614.0 | inactive_split_MB: 259.5 +rank5 [2026-10-30 01:17:46,084] INFO : Step 49800/50000 rank5 | Train Loss: 0.9472 | Train Loss Unweighted: 0.4401 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1188 | Classification: 0.0821 | Classification_binary: 0.1427 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1555 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0468 | Regression_eta: 0.0292 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0448 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7721s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7430 | max_loss: 0.2153 | max_backward: 1.0846 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.7647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6388.3 | uss_MB: 5552.5 | child_rss_MB: 4630.2 | allocated_MB: 368.0 | reserved_MB: 24614.0 | inactive_split_MB: 273.7 +rank5 [2026-10-30 01:21:39,660] INFO : Step 49900/50000 rank5 | Train Loss: 0.9411 | Train Loss Unweighted: 0.4586 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.0996 | Classification: 0.0961 | Classification_binary: 0.1284 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1145 | Regression_eta: 0.0013 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0603 | Regression_eta: 0.0295 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0656 | LR: 1.00e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0424s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.6992 | max_loss: 0.1527 | max_backward: 1.3836 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8576 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6363.3 | uss_MB: 5692.1 | child_rss_MB: 4528.3 | allocated_MB: 368.7 | reserved_MB: 24614.0 | inactive_split_MB: 241.3 +rank5 [2026-10-30 01:25:29,378] INFO : Step 50000/50000 rank5 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2297 | Cardinality: 0.1196 | Classification: 0.0902 | Classification_binary: 0.1583 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0536 | Regression_eta: 0.0321 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0496 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2536s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.7146 | max_loss: 0.1481 | max_backward: 1.6207 | max_optimizer_step: 0.2721 | max_tensorboard_logging: 0.8195 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6364.5 | uss_MB: 5690.8 | child_rss_MB: 4519.4 | allocated_MB: 371.5 | reserved_MB: 25088.0 | inactive_split_MB: 268.9 +rank5 [2026-10-30 01:25:30,314] INFO : Running validation on rank5 +rank5 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 01:26:21,850] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:26:22,339] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 01:26:50,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 01:26:51,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 01:26:52,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 01:26:53,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 01:27:03,719] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-30 01:27:03,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:27:04,489] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 01:27:20,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 01:27:21,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 01:27:22,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 01:27:23,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 01:27:33,449] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-30 01:27:34,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:27:34,757] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 01:27:50,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 01:27:51,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 01:27:52,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 01:27:52,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 01:28:02,868] INFO : Time taken to make predictions on device 5 is: 0.22 min +rank5 [2026-10-30 01:28:03,382] INFO : Training completed. Total time on device 5: 638.890min diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.6 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.6 new file mode 100644 index 0000000000000000000000000000000000000000..6d8964cac69bcb1b3b906fa37cdaaf7b115c4082 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.6 @@ -0,0 +1,2484 @@ +rank6 [2026-10-28 11:12:32,349] INFO : Initializing model with seed=12345; process seed=12351 +rank6 [2026-10-28 11:12:32,350] INFO : configured dtype=torch.bfloat16 for autocast +rank6 [2026-10-28 11:12:54,339] INFO : Instantiating model +rank6 [2026-10-28 11:12:54,339] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank6 [2026-10-28 11:12:54,339] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank6 [2026-10-28 11:12:54,358] INFO : Input encoding initialization took 0.02s +rank6 [2026-10-28 11:12:54,358] INFO : nn0 parameters: 171776 +rank6 [2026-10-28 11:12:54,358] INFO : Initializing shared backbone layers, num_convs=6 +rank6 [2026-10-28 11:12:54,358] INFO : Initializing backbone layer 0 +rank6 [2026-10-28 11:12:54,358] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:54,362] INFO : Initializing backbone layer 1 +rank6 [2026-10-28 11:12:54,362] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:54,365] INFO : Initializing backbone layer 2 +rank6 [2026-10-28 11:12:54,365] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:54,368] INFO : Initializing backbone layer 3 +rank6 [2026-10-28 11:12:54,368] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:54,371] INFO : Initializing backbone layer 4 +rank6 [2026-10-28 11:12:54,371] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:54,374] INFO : Initializing backbone layer 5 +rank6 [2026-10-28 11:12:54,374] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-28 11:12:54,377] INFO : Backbone initialization took 0.02s +rank6 [2026-10-28 11:12:54,377] INFO : backbone parameters: 2374656 +rank6 [2026-10-28 11:12:54,377] INFO : Initializing output DNNs +rank6 [2026-10-28 11:12:54,426] INFO : Output DNNs initialization took 0.05s +rank6 [2026-10-28 11:12:54,426] INFO : backbone_mode=BackboneMode.SHARED +rank6 [2026-10-28 11:12:54,427] INFO : jagged_attention enabled=True +rank6 [2026-10-28 11:12:54,427] INFO : flash_attn_varlen enabled=True +rank6 [2026-10-28 11:12:54,427] INFO : task_queries enabled=False +rank6 [2026-10-28 11:12:54,427] INFO : classification_norm parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : regression_norm parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : classification_query parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : regression_query parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : classification_readout parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : regression_readout parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : output_mode=set +rank6 [2026-10-28 11:12:54,427] INFO : set_decoder parameters: 4713812 +rank6 [2026-10-28 11:12:54,427] INFO : nn_binary_particle parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : nn_pid parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : nn_pt parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : nn_eta parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : nn_sin_phi parameters: 0 +rank6 [2026-10-28 11:12:54,427] INFO : nn_cos_phi parameters: 0 +rank6 [2026-10-28 11:12:54,428] INFO : nn_energy parameters: 0 +rank6 [2026-10-28 11:12:54,428] INFO : Total MLPF parameters: 7260244 +rank6 [2026-10-28 11:12:54,428] INFO : MLPF __init__ done +rank6 [2026-10-28 11:12:54,430] INFO : Instantiated model +rank6 [2026-10-28 11:12:54,430] INFO : Moving model to device rank=6 +rank6 [2026-10-28 11:12:54,448] INFO : Moved model to device rank=6 +rank6 [2026-10-28 11:12:54,449] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank6 [2026-10-28 11:12:54,450] INFO : Configured model for SyncBatchNorm rank=6 +rank6 [2026-10-28 11:12:54,646] INFO : Configured model for DistributedDataParallel rank=6 +rank6 [2026-10-28 11:12:54,764] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank6 [2026-10-28 11:12:54,764] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank6 [2026-10-28 11:12:54,765] INFO : Trainable parameters: 7260244 +rank6 [2026-10-28 11:12:54,765] INFO : Non-trainable parameters: 0 +rank6 [2026-10-28 11:12:54,765] INFO : Total parameters: 7260244 +rank6 [2026-10-28 11:12:54,768] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank6 [2026-10-28 11:12:54,768] INFO : Getting dataloaders +rank6 [2026-10-28 11:12:54,768] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:03,735] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:05,120] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,145] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-28 11:13:06,146] INFO : Creating WeightedMixIterator with 1 data loaders. +rank6 [2026-10-28 11:13:06,146] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,415] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,673] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,937] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-28 11:13:06,937] INFO : Creating InterleavedIterator with 1 data loaders. +rank6 [2026-10-28 11:13:20,976] INFO : Got dataloaders +rank6 [2026-10-28 11:13:20,977] INFO : loader split=train rank=6 len=21112 +rank6 [2026-10-28 11:13:20,977] INFO : loader split=valid rank=6 len=12 +rank6 [2026-10-28 11:13:20,977] INFO : Starting training from step 1 to 50000 on rank 6 of 8 +rank6 [2026-10-28 11:13:20,977] INFO : Creating train_iterator +rank6 [2026-10-28 11:13:20,977] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-28 11:13:35,604] INFO : Created train_iterator +rank6 [2026-10-28 11:17:15,588] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5434697668626904, 'Classification': 0.9147019968181849, 'Regression_pt': 1.0888657218962907, 'Regression_eta': 0.019410275079426356, 'Regression_sin_phi': 0.00660347732307855, 'Regression_cos_phi': 0.006484371119877323, 'Regression_energy': 1.1263945941627025}, weights={'Classification_binary': 1.0, 'Classification': 0.5941495299339294, 'Regression_pt': 0.4991154968738556, 'Regression_eta': 27.99907684326172, 'Regression_sin_phi': 82.30054473876953, 'Regression_cos_phi': 83.812255859375, 'Regression_energy': 0.48248612880706787} +rank6 [2026-10-28 11:17:17,664] INFO : Step 100/50000 rank6 | Train Loss: 5.4941 | Train Loss Unweighted: 3.3573 | Train Loss Components: Auxiliary: 1.0155 | Cardinality: 0.6417 | Classification: 0.8373 | Classification_binary: 0.4757 | HitGrouping: 0.4797 | Regression_cos_phi: 0.0063 | Regression_energy: 1.0524 | Regression_eta: 0.0130 | Regression_pt: 0.9668 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8508 | Classification_binary: 0.4743 | Regression_cos_phi: 0.0067 | Regression_energy: 1.0116 | Regression_eta: 0.0177 | Regression_pt: 0.9117 | Regression_sin_phi: 0.0058 | LR: 1.00e-03 | DataLoad Time: 0.0150s | Model Forward Time: 3.0150s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.5597 | max_loss: 0.2615 | max_backward: 1.0157 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.4654 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4647.0 | uss_MB: 3401.7 | child_rss_MB: 5567.8 | allocated_MB: 341.0 | reserved_MB: 22914.0 | inactive_split_MB: 237.2 +rank6 [2026-10-28 11:20:50,235] INFO : Step 200/50000 rank6 | Train Loss: 4.4651 | Train Loss Unweighted: 2.7232 | Train Loss Components: Auxiliary: 0.8610 | Cardinality: 0.5153 | Classification: 0.6425 | Classification_binary: 0.5162 | HitGrouping: 0.4582 | Regression_cos_phi: 0.0043 | Regression_energy: 0.7694 | Regression_eta: 0.0086 | Regression_pt: 0.7776 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3802 | Classification_binary: 0.5187 | Regression_cos_phi: 0.3366 | Regression_energy: 0.3783 | Regression_eta: 0.2877 | Regression_pt: 0.3920 | Regression_sin_phi: 0.3190 | LR: 1.00e-03 | DataLoad Time: 0.0156s | Model Forward Time: 3.7244s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7368 | max_loss: 0.2463 | max_backward: 1.7032 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5468 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4651.2 | uss_MB: 3409.7 | child_rss_MB: 5571.3 | allocated_MB: 343.8 | reserved_MB: 23264.0 | inactive_split_MB: 284.1 +rank6 [2026-10-28 11:24:24,164] INFO : Step 300/50000 rank6 | Train Loss: 3.6383 | Train Loss Unweighted: 2.1819 | Train Loss Components: Auxiliary: 0.6931 | Cardinality: 0.3447 | Classification: 0.5089 | Classification_binary: 0.4824 | HitGrouping: 0.4305 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5908 | Regression_eta: 0.0074 | Regression_pt: 0.5852 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2743 | Classification_binary: 0.5317 | Regression_cos_phi: 0.3186 | Regression_energy: 0.2884 | Regression_eta: 0.2498 | Regression_pt: 0.2983 | Regression_sin_phi: 0.3598 | LR: 1.00e-03 | DataLoad Time: 0.0151s | Model Forward Time: 3.8729s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0315 | max_forward: 0.6855 | max_loss: 0.1568 | max_backward: 1.3632 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 1.0182 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4652.7 | uss_MB: 3415.2 | child_rss_MB: 5569.6 | allocated_MB: 341.4 | reserved_MB: 23916.0 | inactive_split_MB: 289.5 +rank6 [2026-10-28 11:27:58,396] INFO : Step 400/50000 rank6 | Train Loss: 3.5064 | Train Loss Unweighted: 1.9938 | Train Loss Components: Auxiliary: 0.6582 | Cardinality: 0.3738 | Classification: 0.4591 | Classification_binary: 0.4306 | HitGrouping: 0.4041 | Regression_cos_phi: 0.0039 | Regression_energy: 0.5433 | Regression_eta: 0.0080 | Regression_pt: 0.5456 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2739 | Classification_binary: 0.4341 | Regression_cos_phi: 0.2914 | Regression_energy: 0.2275 | Regression_eta: 0.2102 | Regression_pt: 0.2426 | Regression_sin_phi: 0.2664 | LR: 1.00e-03 | DataLoad Time: 0.0142s | Model Forward Time: 3.7432s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.6100 | max_loss: 0.1486 | max_backward: 1.2396 | max_optimizer_step: 0.1474 | max_tensorboard_logging: 1.0157 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4650.3 | uss_MB: 3416.8 | child_rss_MB: 5584.9 | allocated_MB: 342.9 | reserved_MB: 23982.0 | inactive_split_MB: 297.5 +rank6 [2026-10-28 11:31:35,335] INFO : Step 500/50000 rank6 | Train Loss: 3.8899 | Train Loss Unweighted: 2.1856 | Train Loss Components: Auxiliary: 0.6982 | Cardinality: 0.3682 | Classification: 0.4707 | Classification_binary: 0.4836 | HitGrouping: 0.4071 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6048 | Regression_eta: 0.0103 | Regression_pt: 0.6070 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2738 | Classification_binary: 0.4637 | Regression_cos_phi: 0.3533 | Regression_energy: 0.3008 | Regression_eta: 0.2361 | Regression_pt: 0.3166 | Regression_sin_phi: 0.2837 | LR: 1.00e-03 | DataLoad Time: 0.0165s | Model Forward Time: 4.1818s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.7460 | max_loss: 0.2622 | max_backward: 1.7372 | max_optimizer_step: 0.1478 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 4651.3 | uss_MB: 3418.2 | child_rss_MB: 5567.0 | allocated_MB: 344.2 | reserved_MB: 24988.0 | inactive_split_MB: 288.4 +rank6 [2026-10-28 11:35:12,859] INFO : Step 600/50000 rank6 | Train Loss: 3.5648 | Train Loss Unweighted: 2.0654 | Train Loss Components: Auxiliary: 0.6756 | Cardinality: 0.3628 | Classification: 0.4159 | Classification_binary: 0.5246 | HitGrouping: 0.3632 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5539 | Regression_eta: 0.0077 | Regression_pt: 0.5557 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2365 | Classification_binary: 0.5281 | Regression_cos_phi: 0.2488 | Regression_energy: 0.2758 | Regression_eta: 0.1559 | Regression_pt: 0.2832 | Regression_sin_phi: 0.2536 | LR: 1.00e-03 | DataLoad Time: 0.0135s | Model Forward Time: 4.1944s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0071 | max_forward: 0.7483 | max_loss: 0.2574 | max_backward: 1.6305 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.9709 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 4647.1 | uss_MB: 3418.2 | child_rss_MB: 5567.3 | allocated_MB: 344.6 | reserved_MB: 24988.0 | inactive_split_MB: 289.5 +rank6 [2026-10-28 11:38:47,058] INFO : Step 700/50000 rank6 | Train Loss: 3.0543 | Train Loss Unweighted: 1.8237 | Train Loss Components: Auxiliary: 0.6036 | Cardinality: 0.3094 | Classification: 0.3407 | Classification_binary: 0.4142 | HitGrouping: 0.3367 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5254 | Regression_eta: 0.0060 | Regression_pt: 0.5314 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2143 | Classification_binary: 0.4277 | Regression_cos_phi: 0.2119 | Regression_energy: 0.2288 | Regression_eta: 0.2106 | Regression_pt: 0.2351 | Regression_sin_phi: 0.2739 | LR: 1.00e-03 | DataLoad Time: 0.0174s | Model Forward Time: 3.4868s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.6445 | max_loss: 0.2136 | max_backward: 1.2179 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.7893 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4647.0 | uss_MB: 3416.3 | child_rss_MB: 5577.3 | allocated_MB: 341.4 | reserved_MB: 24988.0 | inactive_split_MB: 289.7 +rank6 [2026-10-28 11:42:25,056] INFO : Step 800/50000 rank6 | Train Loss: 3.1854 | Train Loss Unweighted: 1.9126 | Train Loss Components: Auxiliary: 0.6154 | Cardinality: 0.2934 | Classification: 0.4077 | Classification_binary: 0.4086 | HitGrouping: 0.3317 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5404 | Regression_eta: 0.0067 | Regression_pt: 0.5423 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2547 | Classification_binary: 0.4100 | Regression_cos_phi: 0.3063 | Regression_energy: 0.2314 | Regression_eta: 0.1694 | Regression_pt: 0.2414 | Regression_sin_phi: 0.3070 | LR: 9.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.3173s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.6526 | max_loss: 0.2001 | max_backward: 1.2539 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5713 | max_tensorboard_flush: 0.0138 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 4642.6 | uss_MB: 3416.3 | child_rss_MB: 5579.0 | allocated_MB: 346.1 | reserved_MB: 24988.0 | inactive_split_MB: 222.7 +rank6 [2026-10-28 11:46:07,652] INFO : Step 900/50000 rank6 | Train Loss: 3.4950 | Train Loss Unweighted: 2.0678 | Train Loss Components: Auxiliary: 0.6566 | Cardinality: 0.3469 | Classification: 0.4160 | Classification_binary: 0.4953 | HitGrouping: 0.3295 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5663 | Regression_eta: 0.0083 | Regression_pt: 0.5742 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2652 | Classification_binary: 0.4884 | Regression_cos_phi: 0.3060 | Regression_energy: 0.2924 | Regression_eta: 0.2137 | Regression_pt: 0.3112 | Regression_sin_phi: 0.2927 | LR: 9.99e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.8984s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8131 | max_loss: 0.2548 | max_backward: 1.8396 | max_optimizer_step: 0.1492 | max_tensorboard_logging: 0.4938 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 4643.4 | uss_MB: 3417.5 | child_rss_MB: 5588.2 | allocated_MB: 344.2 | reserved_MB: 26006.0 | inactive_split_MB: 273.7 +rank6 [2026-10-28 11:49:58,411] INFO : Step 1000/50000 rank6 | Train Loss: 3.0581 | Train Loss Unweighted: 1.8801 | Train Loss Components: Auxiliary: 0.5942 | Cardinality: 0.2760 | Classification: 0.3603 | Classification_binary: 0.4232 | HitGrouping: 0.3177 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5411 | Regression_eta: 0.0066 | Regression_pt: 0.5426 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2214 | Classification_binary: 0.3995 | Regression_cos_phi: 0.2022 | Regression_energy: 0.2754 | Regression_eta: 0.1378 | Regression_pt: 0.2802 | Regression_sin_phi: 0.2397 | LR: 9.99e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2148s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1006 | max_forward: 0.7513 | max_loss: 0.1459 | max_backward: 1.3658 | max_optimizer_step: 0.1511 | max_tensorboard_logging: 1.2506 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4639.1 | uss_MB: 3415.3 | child_rss_MB: 5584.2 | allocated_MB: 344.4 | reserved_MB: 26006.0 | inactive_split_MB: 271.1 +rank6 [2026-10-28 11:49:59,499] INFO : Running validation on rank6 +rank6 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 11:50:52,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:50:53,226] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 11:51:19,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 11:51:20,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 11:51:21,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 11:51:22,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 11:51:32,566] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 11:51:32,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:51:33,322] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 11:51:49,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 11:51:50,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 11:51:51,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 11:51:52,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 11:52:02,504] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 11:52:02,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:52:03,475] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 11:52:18,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 11:52:19,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 11:52:20,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 11:52:20,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 11:52:30,866] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 11:56:35,453] INFO : Step 1100/50000 rank6 | Train Loss: 2.9181 | Train Loss Unweighted: 1.8556 | Train Loss Components: Auxiliary: 0.5873 | Cardinality: 0.2540 | Classification: 0.3370 | Classification_binary: 0.3789 | HitGrouping: 0.3247 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5584 | Regression_eta: 0.0064 | Regression_pt: 0.5695 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2167 | Classification_binary: 0.4230 | Regression_cos_phi: 0.2790 | Regression_energy: 0.2638 | Regression_eta: 0.1941 | Regression_pt: 0.2807 | Regression_sin_phi: 0.2215 | LR: 9.99e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9457s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8528 | max_loss: 0.1390 | max_backward: 1.4220 | max_optimizer_step: 0.2689 | max_tensorboard_logging: 0.9135 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 6058.3 | uss_MB: 4835.8 | child_rss_MB: 5619.2 | allocated_MB: 344.1 | reserved_MB: 21080.0 | inactive_split_MB: 227.9 +rank6 [2026-10-28 12:00:25,221] INFO : Step 1200/50000 rank6 | Train Loss: 3.1885 | Train Loss Unweighted: 1.9202 | Train Loss Components: Auxiliary: 0.6265 | Cardinality: 0.3516 | Classification: 0.3718 | Classification_binary: 0.4294 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5507 | Regression_eta: 0.0065 | Regression_pt: 0.5556 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2536 | Classification_binary: 0.4190 | Regression_cos_phi: 0.3819 | Regression_energy: 0.2437 | Regression_eta: 0.2063 | Regression_pt: 0.2575 | Regression_sin_phi: 0.2929 | LR: 9.99e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.4091s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0076 | max_forward: 0.7370 | max_loss: 0.3921 | max_backward: 1.7945 | max_optimizer_step: 0.1639 | max_tensorboard_logging: 1.0479 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 6061.1 | uss_MB: 4838.4 | child_rss_MB: 5619.8 | allocated_MB: 344.9 | reserved_MB: 23662.0 | inactive_split_MB: 241.9 +rank6 [2026-10-28 12:04:16,198] INFO : Step 1300/50000 rank6 | Train Loss: 3.2694 | Train Loss Unweighted: 1.9280 | Train Loss Components: Auxiliary: 0.6193 | Cardinality: 0.3461 | Classification: 0.3601 | Classification_binary: 0.4731 | HitGrouping: 0.3156 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5359 | Regression_eta: 0.0069 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2348 | Classification_binary: 0.5062 | Regression_cos_phi: 0.2900 | Regression_energy: 0.2577 | Regression_eta: 0.2134 | Regression_pt: 0.2700 | Regression_sin_phi: 0.3302 | LR: 9.98e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.5231s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0540 | max_forward: 0.9909 | max_loss: 0.3234 | max_backward: 2.2964 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 1.1252 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5498.9 | uss_MB: 4193.7 | child_rss_MB: 5645.6 | allocated_MB: 344.9 | reserved_MB: 24052.0 | inactive_split_MB: 299.1 +rank6 [2026-10-28 12:07:55,471] INFO : Step 1400/50000 rank6 | Train Loss: 2.8491 | Train Loss Unweighted: 1.7889 | Train Loss Components: Auxiliary: 0.5645 | Cardinality: 0.2711 | Classification: 0.3226 | Classification_binary: 0.3768 | HitGrouping: 0.3109 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5376 | Regression_eta: 0.0061 | Regression_pt: 0.5407 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1825 | Classification_binary: 0.4414 | Regression_cos_phi: 0.2244 | Regression_energy: 0.2553 | Regression_eta: 0.1497 | Regression_pt: 0.2714 | Regression_sin_phi: 0.2061 | LR: 9.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7569s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0094 | max_forward: 0.6378 | max_loss: 0.4388 | max_backward: 1.2886 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 0.9631 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 5501.2 | uss_MB: 4277.0 | child_rss_MB: 5652.8 | allocated_MB: 350.4 | reserved_MB: 24916.0 | inactive_split_MB: 216.1 +rank6 [2026-10-28 12:11:42,089] INFO : Step 1500/50000 rank6 | Train Loss: 2.8928 | Train Loss Unweighted: 1.7749 | Train Loss Components: Auxiliary: 0.5690 | Cardinality: 0.2892 | Classification: 0.3566 | Classification_binary: 0.3704 | HitGrouping: 0.3078 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5147 | Regression_eta: 0.0058 | Regression_pt: 0.5216 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2320 | Classification_binary: 0.3636 | Regression_cos_phi: 0.2518 | Regression_energy: 0.2386 | Regression_eta: 0.1869 | Regression_pt: 0.2466 | Regression_sin_phi: 0.2405 | LR: 9.98e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5596s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.6094 | max_loss: 0.4244 | max_backward: 1.4479 | max_optimizer_step: 0.1602 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5488.8 | uss_MB: 4270.8 | child_rss_MB: 5621.5 | allocated_MB: 342.2 | reserved_MB: 24916.0 | inactive_split_MB: 182.9 +rank6 [2026-10-28 12:15:27,599] INFO : Step 1600/50000 rank6 | Train Loss: 2.9805 | Train Loss Unweighted: 1.8512 | Train Loss Components: Auxiliary: 0.5725 | Cardinality: 0.2336 | Classification: 0.3605 | Classification_binary: 0.4404 | HitGrouping: 0.3159 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5142 | Regression_eta: 0.0065 | Regression_pt: 0.5235 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2107 | Classification_binary: 0.4129 | Regression_cos_phi: 0.2960 | Regression_energy: 0.2468 | Regression_eta: 0.1789 | Regression_pt: 0.2602 | Regression_sin_phi: 0.2300 | LR: 9.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7169s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7382 | max_loss: 0.1451 | max_backward: 1.5359 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.4679 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5499.3 | uss_MB: 4281.1 | child_rss_MB: 5637.2 | allocated_MB: 343.2 | reserved_MB: 25046.0 | inactive_split_MB: 260.8 +rank6 [2026-10-28 12:19:19,666] INFO : Step 1700/50000 rank6 | Train Loss: 2.9423 | Train Loss Unweighted: 1.7603 | Train Loss Components: Auxiliary: 0.5618 | Cardinality: 0.2834 | Classification: 0.3350 | Classification_binary: 0.4141 | HitGrouping: 0.3155 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4976 | Regression_eta: 0.0064 | Regression_pt: 0.5011 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1896 | Classification_binary: 0.3980 | Regression_cos_phi: 0.2314 | Regression_energy: 0.2610 | Regression_eta: 0.1410 | Regression_pt: 0.2744 | Regression_sin_phi: 0.2102 | LR: 9.97e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2245s | Phase Times: max_pre_step_barrier: 0.0070 | max_batch_to_device: 0.0065 | max_forward: 0.8351 | max_loss: 0.2466 | max_backward: 1.7292 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0857 | Memory: rss_MB: 5499.7 | uss_MB: 4254.7 | child_rss_MB: 5622.9 | allocated_MB: 342.0 | reserved_MB: 25046.0 | inactive_split_MB: 198.9 +rank6 [2026-10-28 12:23:11,815] INFO : Step 1800/50000 rank6 | Train Loss: 2.7182 | Train Loss Unweighted: 1.6765 | Train Loss Components: Auxiliary: 0.5305 | Cardinality: 0.2556 | Classification: 0.3337 | Classification_binary: 0.3613 | HitGrouping: 0.3103 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4855 | Regression_eta: 0.0054 | Regression_pt: 0.4854 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1861 | Classification_binary: 0.3490 | Regression_cos_phi: 0.2327 | Regression_energy: 0.2125 | Regression_eta: 0.1834 | Regression_pt: 0.2214 | Regression_sin_phi: 0.2450 | LR: 9.97e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0928s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6573 | max_loss: 0.2402 | max_backward: 1.4111 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 1.0385 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0959 | Memory: rss_MB: 5501.3 | uss_MB: 4283.5 | child_rss_MB: 5636.1 | allocated_MB: 340.6 | reserved_MB: 25046.0 | inactive_split_MB: 181.6 +rank6 [2026-10-28 12:26:53,741] INFO : Step 1900/50000 rank6 | Train Loss: 2.9893 | Train Loss Unweighted: 1.8606 | Train Loss Components: Auxiliary: 0.5839 | Cardinality: 0.2900 | Classification: 0.3568 | Classification_binary: 0.4041 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5399 | Regression_eta: 0.0063 | Regression_pt: 0.5477 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2130 | Classification_binary: 0.4068 | Regression_cos_phi: 0.3040 | Regression_energy: 0.3021 | Regression_eta: 0.1775 | Regression_pt: 0.3156 | Regression_sin_phi: 0.1855 | LR: 9.97e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.8708s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.6296 | max_loss: 0.1482 | max_backward: 1.2931 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 1.0537 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 5503.8 | uss_MB: 4286.6 | child_rss_MB: 5619.9 | allocated_MB: 344.8 | reserved_MB: 25048.0 | inactive_split_MB: 225.2 +rank6 [2026-10-28 12:30:53,516] INFO : Step 2000/50000 rank6 | Train Loss: 3.0007 | Train Loss Unweighted: 1.8040 | Train Loss Components: Auxiliary: 0.5673 | Cardinality: 0.3091 | Classification: 0.3519 | Classification_binary: 0.4482 | HitGrouping: 0.3137 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4922 | Regression_eta: 0.0062 | Regression_pt: 0.4996 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2388 | Classification_binary: 0.4832 | Regression_cos_phi: 0.3058 | Regression_energy: 0.2151 | Regression_eta: 0.2035 | Regression_pt: 0.2268 | Regression_sin_phi: 0.2712 | LR: 9.96e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1670s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6813 | max_loss: 0.2416 | max_backward: 1.5165 | max_optimizer_step: 0.1635 | max_tensorboard_logging: 0.9842 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5502.9 | uss_MB: 4287.2 | child_rss_MB: 5669.7 | allocated_MB: 346.7 | reserved_MB: 25696.0 | inactive_split_MB: 287.8 +rank6 [2026-10-28 12:30:54,414] INFO : Running validation on rank6 +rank6 [2026-10-28 12:30:54,414] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 12:30:54,414] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 12:31:46,080] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:31:46,628] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 12:32:13,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 12:32:14,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 12:32:15,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 12:32:16,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 12:32:26,740] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 12:32:27,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:32:27,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 12:32:43,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 12:32:44,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 12:32:45,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 12:32:46,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 12:32:56,807] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 12:32:57,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:32:57,629] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 12:33:13,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 12:33:14,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 12:33:14,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 12:33:15,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 12:33:25,466] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 12:37:37,593] INFO : Step 2100/50000 rank6 | Train Loss: 2.6299 | Train Loss Unweighted: 1.6429 | Train Loss Components: Auxiliary: 0.5172 | Cardinality: 0.2489 | Classification: 0.3119 | Classification_binary: 0.3591 | HitGrouping: 0.3047 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4785 | Regression_eta: 0.0053 | Regression_pt: 0.4834 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1722 | Classification_binary: 0.3969 | Regression_cos_phi: 0.1890 | Regression_energy: 0.2715 | Regression_eta: 0.1311 | Regression_pt: 0.2861 | Regression_sin_phi: 0.1833 | LR: 9.96e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.0198s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0685 | max_forward: 0.7343 | max_loss: 0.1422 | max_backward: 1.3363 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.0015 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6447.9 | uss_MB: 5231.1 | child_rss_MB: 5629.3 | allocated_MB: 341.1 | reserved_MB: 22546.0 | inactive_split_MB: 269.0 +rank6 [2026-10-28 12:41:24,842] INFO : Step 2200/50000 rank6 | Train Loss: 2.8483 | Train Loss Unweighted: 1.7068 | Train Loss Components: Auxiliary: 0.5372 | Cardinality: 0.2547 | Classification: 0.3411 | Classification_binary: 0.3679 | HitGrouping: 0.3030 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4876 | Regression_eta: 0.0061 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2187 | Classification_binary: 0.3744 | Regression_cos_phi: 0.3992 | Regression_energy: 0.2090 | Regression_eta: 0.2121 | Regression_pt: 0.2183 | Regression_sin_phi: 0.2247 | LR: 9.96e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.4917s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6352 | max_loss: 0.1414 | max_backward: 1.3119 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.4703 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6448.9 | uss_MB: 5233.9 | child_rss_MB: 5636.2 | allocated_MB: 341.4 | reserved_MB: 22598.0 | inactive_split_MB: 313.4 +rank6 [2026-10-28 12:45:27,860] INFO : Step 2300/50000 rank6 | Train Loss: 3.1410 | Train Loss Unweighted: 1.8301 | Train Loss Components: Auxiliary: 0.5918 | Cardinality: 0.3583 | Classification: 0.3517 | Classification_binary: 0.4348 | HitGrouping: 0.3083 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5121 | Regression_eta: 0.0065 | Regression_pt: 0.5184 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2277 | Classification_binary: 0.4258 | Regression_cos_phi: 0.3869 | Regression_energy: 0.2704 | Regression_eta: 0.2461 | Regression_pt: 0.2783 | Regression_sin_phi: 0.3324 | LR: 9.95e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8595s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.7258 | max_loss: 0.1520 | max_backward: 1.6288 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.6027 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0166 | Memory: rss_MB: 5633.9 | uss_MB: 4422.4 | child_rss_MB: 5644.1 | allocated_MB: 346.5 | reserved_MB: 23596.0 | inactive_split_MB: 273.5 +rank6 [2026-10-28 12:49:22,411] INFO : Step 2400/50000 rank6 | Train Loss: 2.8063 | Train Loss Unweighted: 1.7109 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.2683 | Classification: 0.3276 | Classification_binary: 0.3929 | HitGrouping: 0.3025 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4873 | Regression_eta: 0.0054 | Regression_pt: 0.4919 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2285 | Classification_binary: 0.4248 | Regression_cos_phi: 0.3102 | Regression_energy: 0.2700 | Regression_eta: 0.1560 | Regression_pt: 0.2815 | Regression_sin_phi: 0.2796 | LR: 9.95e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.5885s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0314 | max_forward: 0.7331 | max_loss: 0.1502 | max_backward: 1.4174 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.5071 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0157 | Memory: rss_MB: 5635.8 | uss_MB: 4426.4 | child_rss_MB: 5648.9 | allocated_MB: 344.4 | reserved_MB: 23596.0 | inactive_split_MB: 276.8 +rank6 [2026-10-28 12:53:10,911] INFO : Step 2500/50000 rank6 | Train Loss: 2.5300 | Train Loss Unweighted: 1.6034 | Train Loss Components: Auxiliary: 0.5018 | Cardinality: 0.2387 | Classification: 0.2872 | Classification_binary: 0.3313 | HitGrouping: 0.3000 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4857 | Regression_eta: 0.0042 | Regression_pt: 0.4904 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1824 | Classification_binary: 0.3485 | Regression_cos_phi: 0.2191 | Regression_energy: 0.2275 | Regression_eta: 0.1222 | Regression_pt: 0.2402 | Regression_sin_phi: 0.1730 | LR: 9.94e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.0234s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0177 | max_forward: 0.6104 | max_loss: 0.1394 | max_backward: 1.2397 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 1.1029 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5639.0 | uss_MB: 4422.1 | child_rss_MB: 5639.4 | allocated_MB: 342.5 | reserved_MB: 23596.0 | inactive_split_MB: 302.8 +rank6 [2026-10-28 12:56:55,919] INFO : Step 2600/50000 rank6 | Train Loss: 2.8992 | Train Loss Unweighted: 1.7075 | Train Loss Components: Auxiliary: 0.5471 | Cardinality: 0.3219 | Classification: 0.3420 | Classification_binary: 0.4005 | HitGrouping: 0.3027 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4737 | Regression_eta: 0.0057 | Regression_pt: 0.4796 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2044 | Classification_binary: 0.3848 | Regression_cos_phi: 0.2341 | Regression_energy: 0.2219 | Regression_eta: 0.1654 | Regression_pt: 0.2288 | Regression_sin_phi: 0.2615 | LR: 9.94e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.7059s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8849 | max_loss: 0.1495 | max_backward: 1.9136 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1254 | Memory: rss_MB: 5638.3 | uss_MB: 4434.4 | child_rss_MB: 5644.3 | allocated_MB: 343.4 | reserved_MB: 23596.0 | inactive_split_MB: 290.6 +rank6 [2026-10-28 13:00:46,527] INFO : Step 2700/50000 rank6 | Train Loss: 2.8481 | Train Loss Unweighted: 1.7200 | Train Loss Components: Auxiliary: 0.5468 | Cardinality: 0.2918 | Classification: 0.3199 | Classification_binary: 0.4344 | HitGrouping: 0.3029 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4756 | Regression_eta: 0.0057 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2097 | Classification_binary: 0.4528 | Regression_cos_phi: 0.3200 | Regression_energy: 0.2422 | Regression_eta: 0.1761 | Regression_pt: 0.2518 | Regression_sin_phi: 0.2586 | LR: 9.94e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.3262s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0064 | max_forward: 0.7554 | max_loss: 0.2027 | max_backward: 1.6333 | max_optimizer_step: 0.1633 | max_tensorboard_logging: 0.9978 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5637.9 | uss_MB: 4434.7 | child_rss_MB: 5643.6 | allocated_MB: 343.8 | reserved_MB: 23598.0 | inactive_split_MB: 284.2 +rank6 [2026-10-28 13:04:40,295] INFO : Step 2800/50000 rank6 | Train Loss: 2.6381 | Train Loss Unweighted: 1.5955 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2387 | Classification: 0.2848 | Classification_binary: 0.3420 | HitGrouping: 0.2974 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4746 | Regression_eta: 0.0056 | Regression_pt: 0.4829 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1756 | Classification_binary: 0.3947 | Regression_cos_phi: 0.2670 | Regression_energy: 0.2282 | Regression_eta: 0.1818 | Regression_pt: 0.2376 | Regression_sin_phi: 0.2151 | LR: 9.93e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.7368s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6805 | max_loss: 0.2368 | max_backward: 1.1707 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5635.7 | uss_MB: 4432.9 | child_rss_MB: 5638.4 | allocated_MB: 341.2 | reserved_MB: 24554.0 | inactive_split_MB: 308.0 +rank6 [2026-10-28 13:08:36,459] INFO : Step 2900/50000 rank6 | Train Loss: 2.7831 | Train Loss Unweighted: 1.6528 | Train Loss Components: Auxiliary: 0.5323 | Cardinality: 0.3031 | Classification: 0.3069 | Classification_binary: 0.3681 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4798 | Regression_eta: 0.0056 | Regression_pt: 0.4869 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1963 | Classification_binary: 0.3656 | Regression_cos_phi: 0.2388 | Regression_energy: 0.2330 | Regression_eta: 0.1472 | Regression_pt: 0.2416 | Regression_sin_phi: 0.2040 | LR: 9.93e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.7843s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6242 | max_loss: 0.1475 | max_backward: 1.3573 | max_optimizer_step: 0.2831 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5637.0 | uss_MB: 4433.8 | child_rss_MB: 5645.5 | allocated_MB: 341.4 | reserved_MB: 25570.0 | inactive_split_MB: 292.0 +rank6 [2026-10-28 13:12:15,920] INFO : Step 3000/50000 rank6 | Train Loss: 2.9336 | Train Loss Unweighted: 1.7260 | Train Loss Components: Auxiliary: 0.5466 | Cardinality: 0.2829 | Classification: 0.3524 | Classification_binary: 0.4205 | HitGrouping: 0.3085 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4673 | Regression_eta: 0.0061 | Regression_pt: 0.4733 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1969 | Classification_binary: 0.3886 | Regression_cos_phi: 0.2277 | Regression_energy: 0.2106 | Regression_eta: 0.1446 | Regression_pt: 0.2199 | Regression_sin_phi: 0.2563 | LR: 9.92e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1506s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0386 | max_forward: 0.8057 | max_loss: 0.1488 | max_backward: 1.9376 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5635.9 | uss_MB: 4440.1 | child_rss_MB: 5640.1 | allocated_MB: 346.5 | reserved_MB: 25570.0 | inactive_split_MB: 281.4 +rank6 [2026-10-28 13:12:16,814] INFO : Running validation on rank6 +rank6 [2026-10-28 13:12:16,814] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 13:12:16,815] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 13:13:09,907] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:13:10,420] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 13:13:30,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 13:13:31,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 13:13:32,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 13:13:33,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 13:13:43,029] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:13:43,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:13:44,017] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 13:14:00,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 13:14:01,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 13:14:02,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 13:14:03,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 13:14:13,156] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:14:13,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:14:14,221] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 13:14:29,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 13:14:30,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 13:14:31,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 13:14:31,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 13:14:41,816] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 13:18:52,945] INFO : Step 3100/50000 rank6 | Train Loss: 2.7478 | Train Loss Unweighted: 1.6734 | Train Loss Components: Auxiliary: 0.5216 | Cardinality: 0.2531 | Classification: 0.2981 | Classification_binary: 0.3781 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4881 | Regression_eta: 0.0057 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1677 | Classification_binary: 0.3867 | Regression_cos_phi: 0.2102 | Regression_energy: 0.2585 | Regression_eta: 0.1400 | Regression_pt: 0.2711 | Regression_sin_phi: 0.1739 | LR: 9.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5537s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.6511 | max_loss: 0.3988 | max_backward: 1.4306 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 1.2631 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5749.3 | uss_MB: 4552.2 | child_rss_MB: 5658.3 | allocated_MB: 342.5 | reserved_MB: 23120.0 | inactive_split_MB: 194.6 +rank6 [2026-10-28 13:22:38,494] INFO : Step 3200/50000 rank6 | Train Loss: 2.5254 | Train Loss Unweighted: 1.5710 | Train Loss Components: Auxiliary: 0.4888 | Cardinality: 0.2129 | Classification: 0.2845 | Classification_binary: 0.3271 | HitGrouping: 0.2964 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4770 | Regression_eta: 0.0050 | Regression_pt: 0.4723 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1630 | Classification_binary: 0.2997 | Regression_cos_phi: 0.1486 | Regression_energy: 0.2592 | Regression_eta: 0.1032 | Regression_pt: 0.2578 | Regression_sin_phi: 0.1317 | LR: 9.91e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.5501s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6134 | max_loss: 0.2441 | max_backward: 1.2833 | max_optimizer_step: 0.2753 | max_tensorboard_logging: 0.5629 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5733.9 | uss_MB: 4558.8 | child_rss_MB: 5628.8 | allocated_MB: 343.6 | reserved_MB: 23340.0 | inactive_split_MB: 191.9 +rank6 [2026-10-28 13:26:30,566] INFO : Step 3300/50000 rank6 | Train Loss: 2.8037 | Train Loss Unweighted: 1.6635 | Train Loss Components: Auxiliary: 0.5345 | Cardinality: 0.2841 | Classification: 0.3310 | Classification_binary: 0.3857 | HitGrouping: 0.3002 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4633 | Regression_eta: 0.0058 | Regression_pt: 0.4718 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1887 | Classification_binary: 0.3354 | Regression_cos_phi: 0.2344 | Regression_energy: 0.2104 | Regression_eta: 0.1359 | Regression_pt: 0.2194 | Regression_sin_phi: 0.2082 | LR: 9.90e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2956s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9147 | max_loss: 0.1535 | max_backward: 1.9100 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.6631 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5735.1 | uss_MB: 4562.9 | child_rss_MB: 5661.7 | allocated_MB: 344.0 | reserved_MB: 23618.0 | inactive_split_MB: 250.0 +rank6 [2026-10-28 13:30:24,442] INFO : Step 3400/50000 rank6 | Train Loss: 2.8855 | Train Loss Unweighted: 1.7067 | Train Loss Components: Auxiliary: 0.5454 | Cardinality: 0.2859 | Classification: 0.3252 | Classification_binary: 0.4155 | HitGrouping: 0.2965 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4731 | Regression_eta: 0.0067 | Regression_pt: 0.4802 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1809 | Classification_binary: 0.4051 | Regression_cos_phi: 0.1896 | Regression_energy: 0.2313 | Regression_eta: 0.1353 | Regression_pt: 0.2446 | Regression_sin_phi: 0.2932 | LR: 9.90e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.5648s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0438 | max_forward: 0.7271 | max_loss: 0.1506 | max_backward: 1.3643 | max_optimizer_step: 0.1716 | max_tensorboard_logging: 0.4827 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5729.2 | uss_MB: 4562.1 | child_rss_MB: 5658.8 | allocated_MB: 346.0 | reserved_MB: 23618.0 | inactive_split_MB: 235.6 +rank6 [2026-10-28 13:34:14,208] INFO : Step 3500/50000 rank6 | Train Loss: 2.4422 | Train Loss Unweighted: 1.4809 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.2120 | Classification: 0.2615 | Classification_binary: 0.3296 | HitGrouping: 0.2898 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4369 | Regression_eta: 0.0054 | Regression_pt: 0.4427 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1474 | Classification_binary: 0.3693 | Regression_cos_phi: 0.1710 | Regression_energy: 0.1838 | Regression_eta: 0.1146 | Regression_pt: 0.1915 | Regression_sin_phi: 0.2058 | LR: 9.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7292s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0058 | max_forward: 0.7056 | max_loss: 0.1971 | max_backward: 1.3359 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.6307 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5720.3 | uss_MB: 4563.4 | child_rss_MB: 5630.0 | allocated_MB: 339.0 | reserved_MB: 24610.0 | inactive_split_MB: 280.8 +rank6 [2026-10-28 13:38:02,721] INFO : Step 3600/50000 rank6 | Train Loss: 2.5780 | Train Loss Unweighted: 1.5250 | Train Loss Components: Auxiliary: 0.4892 | Cardinality: 0.2463 | Classification: 0.3079 | Classification_binary: 0.3446 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4295 | Regression_eta: 0.0052 | Regression_pt: 0.4324 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1805 | Classification_binary: 0.3149 | Regression_cos_phi: 0.1948 | Regression_energy: 0.2094 | Regression_eta: 0.1708 | Regression_pt: 0.2174 | Regression_sin_phi: 0.2153 | LR: 9.89e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.4628s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1453 | max_backward: 1.4106 | max_optimizer_step: 0.1681 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0126 | Memory: rss_MB: 5726.7 | uss_MB: 4567.6 | child_rss_MB: 5631.8 | allocated_MB: 343.9 | reserved_MB: 24610.0 | inactive_split_MB: 244.5 +rank6 [2026-10-28 13:41:50,383] INFO : Step 3700/50000 rank6 | Train Loss: 2.9213 | Train Loss Unweighted: 1.6952 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.3153 | Classification: 0.3418 | Classification_binary: 0.4227 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4570 | Regression_eta: 0.0063 | Regression_pt: 0.4612 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2308 | Classification_binary: 0.4077 | Regression_cos_phi: 0.2602 | Regression_energy: 0.1952 | Regression_eta: 0.2020 | Regression_pt: 0.2031 | Regression_sin_phi: 0.2389 | LR: 9.88e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2194s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7710 | max_loss: 0.1507 | max_backward: 1.6430 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.8888 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0145 | Memory: rss_MB: 5726.2 | uss_MB: 4569.4 | child_rss_MB: 5661.5 | allocated_MB: 345.4 | reserved_MB: 24610.0 | inactive_split_MB: 268.9 +rank6 [2026-10-28 13:45:46,299] INFO : Step 3800/50000 rank6 | Train Loss: 2.6063 | Train Loss Unweighted: 1.5289 | Train Loss Components: Auxiliary: 0.4933 | Cardinality: 0.2661 | Classification: 0.2839 | Classification_binary: 0.3652 | HitGrouping: 0.2879 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4319 | Regression_eta: 0.0058 | Regression_pt: 0.4369 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1702 | Classification_binary: 0.3637 | Regression_cos_phi: 0.1764 | Regression_energy: 0.2055 | Regression_eta: 0.1611 | Regression_pt: 0.2174 | Regression_sin_phi: 0.2245 | LR: 9.87e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1723s | Phase Times: max_pre_step_barrier: 0.1134 | max_batch_to_device: 0.0063 | max_forward: 0.6784 | max_loss: 0.1451 | max_backward: 1.4269 | max_optimizer_step: 0.2795 | max_tensorboard_logging: 0.9060 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5726.3 | uss_MB: 4569.5 | child_rss_MB: 5664.0 | allocated_MB: 343.8 | reserved_MB: 24610.0 | inactive_split_MB: 202.2 +rank6 [2026-10-28 13:49:32,922] INFO : Step 3900/50000 rank6 | Train Loss: 2.2761 | Train Loss Unweighted: 1.4165 | Train Loss Components: Auxiliary: 0.4517 | Cardinality: 0.1938 | Classification: 0.2553 | Classification_binary: 0.2930 | HitGrouping: 0.2882 | Regression_cos_phi: 0.0020 | Regression_energy: 0.4264 | Regression_eta: 0.0045 | Regression_pt: 0.4331 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1546 | Classification_binary: 0.3155 | Regression_cos_phi: 0.1663 | Regression_energy: 0.2137 | Regression_eta: 0.1822 | Regression_pt: 0.2236 | Regression_sin_phi: 0.2281 | LR: 9.87e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8902s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6197 | max_loss: 0.1501 | max_backward: 1.2810 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.0075 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 5727.5 | uss_MB: 3764.5 | child_rss_MB: 5661.9 | allocated_MB: 343.5 | reserved_MB: 24610.0 | inactive_split_MB: 286.6 +rank6 [2026-10-28 13:53:23,686] INFO : Step 4000/50000 rank6 | Train Loss: 2.6760 | Train Loss Unweighted: 1.5903 | Train Loss Components: Auxiliary: 0.5112 | Cardinality: 0.2445 | Classification: 0.3070 | Classification_binary: 0.3812 | HitGrouping: 0.2906 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4413 | Regression_eta: 0.0062 | Regression_pt: 0.4492 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1921 | Classification_binary: 0.3686 | Regression_cos_phi: 0.2081 | Regression_energy: 0.2175 | Regression_eta: 0.1479 | Regression_pt: 0.2292 | Regression_sin_phi: 0.2132 | LR: 9.86e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.0144s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0492 | max_forward: 0.7834 | max_loss: 0.1586 | max_backward: 1.6903 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.5589 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5731.5 | uss_MB: 4577.2 | child_rss_MB: 5661.6 | allocated_MB: 345.5 | reserved_MB: 24610.0 | inactive_split_MB: 252.6 +rank6 [2026-10-28 13:53:24,557] INFO : Running validation on rank6 +rank6 [2026-10-28 13:53:24,557] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 13:53:24,558] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 13:54:13,892] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:54:14,419] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 13:54:40,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 13:54:41,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 13:54:42,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 13:54:43,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 13:54:53,324] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 13:54:53,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:54:53,949] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 13:55:10,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 13:55:11,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 13:55:12,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 13:55:13,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 13:55:23,079] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:55:23,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:55:23,752] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 13:55:38,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 13:55:39,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 13:55:40,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 13:55:40,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 13:55:50,812] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 13:59:55,048] INFO : Step 4100/50000 rank6 | Train Loss: 2.6923 | Train Loss Unweighted: 1.5640 | Train Loss Components: Auxiliary: 0.5058 | Cardinality: 0.2671 | Classification: 0.2981 | Classification_binary: 0.4073 | HitGrouping: 0.2911 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4210 | Regression_eta: 0.0055 | Regression_pt: 0.4265 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1906 | Classification_binary: 0.4249 | Regression_cos_phi: 0.2891 | Regression_energy: 0.1907 | Regression_eta: 0.1793 | Regression_pt: 0.2014 | Regression_sin_phi: 0.2393 | LR: 9.85e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1489s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0141 | max_forward: 0.7422 | max_loss: 0.3919 | max_backward: 1.3629 | max_optimizer_step: 0.1890 | max_tensorboard_logging: 0.9903 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6680.3 | uss_MB: 5525.0 | child_rss_MB: 5670.9 | allocated_MB: 345.2 | reserved_MB: 23430.0 | inactive_split_MB: 294.4 +rank6 [2026-10-28 14:03:47,899] INFO : Step 4200/50000 rank6 | Train Loss: 2.3415 | Train Loss Unweighted: 1.4424 | Train Loss Components: Auxiliary: 0.4542 | Cardinality: 0.2058 | Classification: 0.2540 | Classification_binary: 0.3143 | HitGrouping: 0.2846 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4310 | Regression_eta: 0.0050 | Regression_pt: 0.4337 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1717 | Classification_binary: 0.3468 | Regression_cos_phi: 0.2218 | Regression_energy: 0.2156 | Regression_eta: 0.1475 | Regression_pt: 0.2236 | Regression_sin_phi: 0.1931 | LR: 9.84e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.7396s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0168 | max_forward: 0.8335 | max_loss: 0.2425 | max_backward: 1.5586 | max_optimizer_step: 0.1744 | max_tensorboard_logging: 0.4781 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5904.9 | uss_MB: 4751.1 | child_rss_MB: 5672.4 | allocated_MB: 341.2 | reserved_MB: 24406.0 | inactive_split_MB: 273.9 +rank6 [2026-10-28 14:07:35,175] INFO : Step 4300/50000 rank6 | Train Loss: 2.5102 | Train Loss Unweighted: 1.4857 | Train Loss Components: Auxiliary: 0.4757 | Cardinality: 0.2309 | Classification: 0.2786 | Classification_binary: 0.3543 | HitGrouping: 0.2938 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4177 | Regression_eta: 0.0054 | Regression_pt: 0.4246 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1345 | Classification_binary: 0.3227 | Regression_cos_phi: 0.1766 | Regression_energy: 0.2043 | Regression_eta: 0.1360 | Regression_pt: 0.2165 | Regression_sin_phi: 0.1741 | LR: 9.84e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.5980s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6520 | max_loss: 0.2453 | max_backward: 1.4084 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.5540 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5910.3 | uss_MB: 4757.7 | child_rss_MB: 5668.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 239.6 +rank6 [2026-10-28 14:11:23,916] INFO : Step 4400/50000 rank6 | Train Loss: 2.8258 | Train Loss Unweighted: 1.5691 | Train Loss Components: Auxiliary: 0.5223 | Cardinality: 0.3313 | Classification: 0.2877 | Classification_binary: 0.4120 | HitGrouping: 0.2902 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4255 | Regression_eta: 0.0063 | Regression_pt: 0.4316 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1747 | Classification_binary: 0.3766 | Regression_cos_phi: 0.2166 | Regression_energy: 0.2193 | Regression_eta: 0.2235 | Regression_pt: 0.2292 | Regression_sin_phi: 0.2558 | LR: 9.83e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.0439s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.7635 | max_loss: 0.2580 | max_backward: 1.7456 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.5221 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 5909.1 | uss_MB: 4757.0 | child_rss_MB: 5674.0 | allocated_MB: 344.6 | reserved_MB: 24406.0 | inactive_split_MB: 261.0 +rank6 [2026-10-28 14:15:17,053] INFO : Step 4500/50000 rank6 | Train Loss: 2.5622 | Train Loss Unweighted: 1.4664 | Train Loss Components: Auxiliary: 0.4942 | Cardinality: 0.3132 | Classification: 0.2685 | Classification_binary: 0.3695 | HitGrouping: 0.2876 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4070 | Regression_eta: 0.0048 | Regression_pt: 0.4117 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1691 | Classification_binary: 0.3887 | Regression_cos_phi: 0.2158 | Regression_energy: 0.1816 | Regression_eta: 0.1443 | Regression_pt: 0.1940 | Regression_sin_phi: 0.2188 | LR: 9.82e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.2139s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0462 | max_forward: 1.1330 | max_loss: 0.3526 | max_backward: 2.1744 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5571 | max_tensorboard_flush: 0.0016 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 5912.8 | uss_MB: 4736.1 | child_rss_MB: 5674.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 288.8 +rank6 [2026-10-28 14:19:09,439] INFO : Step 4600/50000 rank6 | Train Loss: 2.2516 | Train Loss Unweighted: 1.3548 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2151 | Classification: 0.2428 | Classification_binary: 0.2886 | HitGrouping: 0.2811 | Regression_cos_phi: 0.0019 | Regression_energy: 0.4059 | Regression_eta: 0.0042 | Regression_pt: 0.4088 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1396 | Classification_binary: 0.2911 | Regression_cos_phi: 0.1996 | Regression_energy: 0.2158 | Regression_eta: 0.1504 | Regression_pt: 0.2278 | Regression_sin_phi: 0.2612 | LR: 9.81e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7003s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0445 | max_forward: 0.8897 | max_loss: 0.2214 | max_backward: 1.8748 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5913.2 | uss_MB: 4741.7 | child_rss_MB: 5653.3 | allocated_MB: 341.6 | reserved_MB: 25392.0 | inactive_split_MB: 298.4 +rank6 [2026-10-28 14:23:11,224] INFO : Step 4700/50000 rank6 | Train Loss: 2.5991 | Train Loss Unweighted: 1.5142 | Train Loss Components: Auxiliary: 0.4914 | Cardinality: 0.2679 | Classification: 0.2997 | Classification_binary: 0.3514 | HitGrouping: 0.2867 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4228 | Regression_eta: 0.0061 | Regression_pt: 0.4290 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1897 | Classification_binary: 0.3453 | Regression_cos_phi: 0.2375 | Regression_energy: 0.2041 | Regression_eta: 0.1647 | Regression_pt: 0.2110 | Regression_sin_phi: 0.1734 | LR: 9.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8898s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7362 | max_loss: 0.2547 | max_backward: 1.8331 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5918.9 | uss_MB: 4764.8 | child_rss_MB: 5675.1 | allocated_MB: 343.1 | reserved_MB: 26384.0 | inactive_split_MB: 292.4 +rank6 [2026-10-28 14:27:13,048] INFO : Step 4800/50000 rank6 | Train Loss: 2.7077 | Train Loss Unweighted: 1.5200 | Train Loss Components: Auxiliary: 0.5118 | Cardinality: 0.3360 | Classification: 0.2606 | Classification_binary: 0.3918 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4264 | Regression_eta: 0.0053 | Regression_pt: 0.4305 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1547 | Classification_binary: 0.3798 | Regression_cos_phi: 0.1664 | Regression_energy: 0.2118 | Regression_eta: 0.1258 | Regression_pt: 0.2226 | Regression_sin_phi: 0.2160 | LR: 9.80e-04 | DataLoad Time: 0.0139s | Model Forward Time: 5.2505s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7154 | max_loss: 0.1552 | max_backward: 1.4398 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.8979 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0854 | Memory: rss_MB: 5917.3 | uss_MB: 4226.4 | child_rss_MB: 5678.8 | allocated_MB: 346.1 | reserved_MB: 26384.0 | inactive_split_MB: 284.0 +rank6 [2026-10-28 14:31:09,414] INFO : Step 4900/50000 rank6 | Train Loss: 2.3682 | Train Loss Unweighted: 1.4242 | Train Loss Components: Auxiliary: 0.4616 | Cardinality: 0.2176 | Classification: 0.2570 | Classification_binary: 0.3149 | HitGrouping: 0.2900 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4191 | Regression_eta: 0.0047 | Regression_pt: 0.4239 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1393 | Classification_binary: 0.3203 | Regression_cos_phi: 0.1278 | Regression_energy: 0.2149 | Regression_eta: 0.1235 | Regression_pt: 0.2257 | Regression_sin_phi: 0.1687 | LR: 9.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.4633s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0516 | max_forward: 0.6608 | max_loss: 0.1430 | max_backward: 1.2657 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5916.2 | uss_MB: 4765.4 | child_rss_MB: 5677.6 | allocated_MB: 343.8 | reserved_MB: 26384.0 | inactive_split_MB: 201.9 +rank6 [2026-10-28 14:35:19,045] INFO : Step 5000/50000 rank6 | Train Loss: 2.4060 | Train Loss Unweighted: 1.4130 | Train Loss Components: Auxiliary: 0.4756 | Cardinality: 0.2573 | Classification: 0.2748 | Classification_binary: 0.3198 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4020 | Regression_eta: 0.0043 | Regression_pt: 0.4074 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1766 | Classification_binary: 0.3201 | Regression_cos_phi: 0.2249 | Regression_energy: 0.2058 | Regression_eta: 0.1425 | Regression_pt: 0.2176 | Regression_sin_phi: 0.2083 | LR: 9.78e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8081s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6153 | max_loss: 0.1441 | max_backward: 1.2119 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.9741 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5914.8 | uss_MB: 4767.1 | child_rss_MB: 5669.5 | allocated_MB: 342.5 | reserved_MB: 27388.0 | inactive_split_MB: 218.4 +rank6 [2026-10-28 14:35:19,963] INFO : Running validation on rank6 +rank6 [2026-10-28 14:35:19,963] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 14:35:19,963] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 14:36:10,799] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:36:11,318] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 14:36:36,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 14:36:37,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 14:36:38,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 14:36:39,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 14:36:49,655] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 14:36:50,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:36:50,780] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 14:37:06,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 14:37:07,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 14:37:08,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 14:37:09,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 14:37:19,737] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 14:37:20,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:37:20,582] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 14:37:35,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 14:37:36,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 14:37:37,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 14:37:37,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 14:37:47,892] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 14:42:02,659] INFO : Step 5100/50000 rank6 | Train Loss: 2.7059 | Train Loss Unweighted: 1.5631 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2507 | Classification: 0.3068 | Classification_binary: 0.4110 | HitGrouping: 0.2884 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4148 | Regression_eta: 0.0060 | Regression_pt: 0.4186 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1559 | Classification_binary: 0.3853 | Regression_cos_phi: 0.2864 | Regression_energy: 0.1940 | Regression_eta: 0.1482 | Regression_pt: 0.2013 | Regression_sin_phi: 0.2513 | LR: 9.77e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4418s | Phase Times: max_pre_step_barrier: 0.1071 | max_batch_to_device: 0.0070 | max_forward: 0.7192 | max_loss: 0.2509 | max_backward: 1.6066 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.8859 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7032.2 | uss_MB: 5882.9 | child_rss_MB: 5662.0 | allocated_MB: 343.2 | reserved_MB: 23542.0 | inactive_split_MB: 236.2 +rank6 [2026-10-28 14:45:56,260] INFO : Step 5200/50000 rank6 | Train Loss: 2.4117 | Train Loss Unweighted: 1.4393 | Train Loss Components: Auxiliary: 0.4667 | Cardinality: 0.2215 | Classification: 0.2426 | Classification_binary: 0.3524 | HitGrouping: 0.2783 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4149 | Regression_eta: 0.0050 | Regression_pt: 0.4196 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1300 | Classification_binary: 0.3542 | Regression_cos_phi: 0.1763 | Regression_energy: 0.2056 | Regression_eta: 0.1160 | Regression_pt: 0.2168 | Regression_sin_phi: 0.1840 | LR: 9.76e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9019s | Phase Times: max_pre_step_barrier: 0.1151 | max_batch_to_device: 0.0079 | max_forward: 0.8909 | max_loss: 0.1450 | max_backward: 1.3654 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.5791 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 6020.7 | uss_MB: 4875.4 | child_rss_MB: 5666.4 | allocated_MB: 346.1 | reserved_MB: 23690.0 | inactive_split_MB: 197.9 +rank6 [2026-10-28 14:50:03,850] INFO : Step 5300/50000 rank6 | Train Loss: 2.2417 | Train Loss Unweighted: 1.3238 | Train Loss Components: Auxiliary: 0.4353 | Cardinality: 0.2226 | Classification: 0.2387 | Classification_binary: 0.3027 | HitGrouping: 0.2808 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3844 | Regression_eta: 0.0041 | Regression_pt: 0.3895 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1271 | Classification_binary: 0.3194 | Regression_cos_phi: 0.1909 | Regression_energy: 0.1900 | Regression_eta: 0.1183 | Regression_pt: 0.1994 | Regression_sin_phi: 0.1840 | LR: 9.75e-04 | DataLoad Time: 0.0232s | Model Forward Time: 4.1963s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0155 | max_forward: 1.3549 | max_loss: 0.4272 | max_backward: 1.7517 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 6024.3 | uss_MB: 4878.1 | child_rss_MB: 5664.8 | allocated_MB: 341.7 | reserved_MB: 25496.0 | inactive_split_MB: 195.4 +rank6 [2026-10-28 14:54:26,616] INFO : Step 5400/50000 rank6 | Train Loss: 2.5857 | Train Loss Unweighted: 1.4820 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2715 | Classification: 0.2898 | Classification_binary: 0.3613 | HitGrouping: 0.2829 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4076 | Regression_eta: 0.0053 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1784 | Classification_binary: 0.3422 | Regression_cos_phi: 0.3257 | Regression_energy: 0.2089 | Regression_eta: 0.1772 | Regression_pt: 0.2188 | Regression_sin_phi: 0.2575 | LR: 9.74e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.8624s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7472 | max_loss: 0.1664 | max_backward: 1.6216 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4956 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6028.8 | uss_MB: 4880.2 | child_rss_MB: 5668.3 | allocated_MB: 345.8 | reserved_MB: 25496.0 | inactive_split_MB: 226.8 +rank6 [2026-10-28 14:58:29,775] INFO : Step 5500/50000 rank6 | Train Loss: 2.4383 | Train Loss Unweighted: 1.4361 | Train Loss Components: Auxiliary: 0.4636 | Cardinality: 0.2531 | Classification: 0.2547 | Classification_binary: 0.3846 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3900 | Regression_eta: 0.0045 | Regression_pt: 0.3974 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1835 | Classification_binary: 0.4149 | Regression_cos_phi: 0.1828 | Regression_energy: 0.1976 | Regression_eta: 0.1626 | Regression_pt: 0.2091 | Regression_sin_phi: 0.1950 | LR: 9.73e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3508s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7547 | max_loss: 0.2915 | max_backward: 1.8143 | max_optimizer_step: 0.2819 | max_tensorboard_logging: 0.5958 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6029.1 | uss_MB: 4882.8 | child_rss_MB: 5667.9 | allocated_MB: 342.5 | reserved_MB: 26496.0 | inactive_split_MB: 471.5 +rank6 [2026-10-28 15:02:37,794] INFO : Step 5600/50000 rank6 | Train Loss: 2.2622 | Train Loss Unweighted: 1.3177 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.1990 | Classification: 0.2257 | Classification_binary: 0.3268 | HitGrouping: 0.2800 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3758 | Regression_eta: 0.0051 | Regression_pt: 0.3799 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1369 | Classification_binary: 0.3542 | Regression_cos_phi: 0.2158 | Regression_energy: 0.1777 | Regression_eta: 0.1221 | Regression_pt: 0.1901 | Regression_sin_phi: 0.1770 | LR: 9.72e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.1696s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0347 | max_forward: 1.1910 | max_loss: 0.2555 | max_backward: 1.5431 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 6024.4 | uss_MB: 4885.9 | child_rss_MB: 5669.8 | allocated_MB: 340.8 | reserved_MB: 26496.0 | inactive_split_MB: 191.2 +rank6 [2026-10-28 15:06:24,089] INFO : Step 5700/50000 rank6 | Train Loss: 2.4588 | Train Loss Unweighted: 1.4026 | Train Loss Components: Auxiliary: 0.4666 | Cardinality: 0.2348 | Classification: 0.2768 | Classification_binary: 0.3365 | HitGrouping: 0.2916 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3864 | Regression_eta: 0.0055 | Regression_pt: 0.3923 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1658 | Classification_binary: 0.3206 | Regression_cos_phi: 0.1917 | Regression_energy: 0.2023 | Regression_eta: 0.1558 | Regression_pt: 0.2148 | Regression_sin_phi: 0.2306 | LR: 9.71e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5059s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6547 | max_loss: 0.1874 | max_backward: 1.2748 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6027.1 | uss_MB: 4889.2 | child_rss_MB: 5674.0 | allocated_MB: 344.4 | reserved_MB: 26496.0 | inactive_split_MB: 224.5 +rank6 [2026-10-28 15:10:14,851] INFO : Step 5800/50000 rank6 | Train Loss: 2.4947 | Train Loss Unweighted: 1.4616 | Train Loss Components: Auxiliary: 0.4752 | Cardinality: 0.2715 | Classification: 0.2623 | Classification_binary: 0.3729 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4032 | Regression_eta: 0.0047 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1493 | Classification_binary: 0.3861 | Regression_cos_phi: 0.1869 | Regression_energy: 0.1915 | Regression_eta: 0.1468 | Regression_pt: 0.2017 | Regression_sin_phi: 0.1730 | LR: 9.70e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7376s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.9768 | max_loss: 0.2512 | max_backward: 2.0573 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.6978 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6026.0 | uss_MB: 4048.8 | child_rss_MB: 5650.8 | allocated_MB: 343.4 | reserved_MB: 27494.0 | inactive_split_MB: 198.3 +rank6 [2026-10-28 15:14:12,320] INFO : Step 5900/50000 rank6 | Train Loss: 2.3800 | Train Loss Unweighted: 1.3838 | Train Loss Components: Auxiliary: 0.4549 | Cardinality: 0.2203 | Classification: 0.2438 | Classification_binary: 0.3435 | HitGrouping: 0.2757 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3900 | Regression_eta: 0.0050 | Regression_pt: 0.3965 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1653 | Classification_binary: 0.3864 | Regression_cos_phi: 0.2006 | Regression_energy: 0.1695 | Regression_eta: 0.1751 | Regression_pt: 0.1773 | Regression_sin_phi: 0.2692 | LR: 9.69e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.7147s | Phase Times: max_pre_step_barrier: 0.0158 | max_batch_to_device: 0.0069 | max_forward: 1.0355 | max_loss: 0.2311 | max_backward: 2.2749 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5131 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 6029.8 | uss_MB: 3868.6 | child_rss_MB: 5658.5 | allocated_MB: 342.1 | reserved_MB: 27494.0 | inactive_split_MB: 281.2 +rank6 [2026-10-28 15:18:13,895] INFO : Step 6000/50000 rank6 | Train Loss: 2.1372 | Train Loss Unweighted: 1.3015 | Train Loss Components: Auxiliary: 0.4179 | Cardinality: 0.1916 | Classification: 0.2091 | Classification_binary: 0.2848 | HitGrouping: 0.2746 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3966 | Regression_eta: 0.0041 | Regression_pt: 0.4029 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1251 | Classification_binary: 0.3078 | Regression_cos_phi: 0.1257 | Regression_energy: 0.2035 | Regression_eta: 0.1094 | Regression_pt: 0.2127 | Regression_sin_phi: 0.2136 | LR: 9.68e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.6783s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6457 | max_loss: 0.2157 | max_backward: 1.4984 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.5590 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6033.8 | uss_MB: 4891.5 | child_rss_MB: 5658.2 | allocated_MB: 343.1 | reserved_MB: 27494.0 | inactive_split_MB: 260.7 +rank6 [2026-10-28 15:18:14,998] INFO : Running validation on rank6 +rank6 [2026-10-28 15:18:14,999] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 15:18:14,999] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 15:19:17,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:19:17,849] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 15:19:43,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 15:19:44,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 15:19:45,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 15:19:46,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 15:19:56,952] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 15:19:57,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:19:58,136] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 15:20:14,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 15:20:15,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 15:20:16,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 15:20:17,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 15:20:27,270] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 15:20:27,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:20:28,075] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 15:20:43,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 15:20:44,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 15:20:45,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 15:20:45,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 15:20:55,919] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 15:25:29,845] INFO : Step 6100/50000 rank6 | Train Loss: 2.4040 | Train Loss Unweighted: 1.3907 | Train Loss Components: Auxiliary: 0.4661 | Cardinality: 0.2610 | Classification: 0.2427 | Classification_binary: 0.3401 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3968 | Regression_eta: 0.0047 | Regression_pt: 0.4016 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1684 | Classification_binary: 0.3809 | Regression_cos_phi: 0.1940 | Regression_energy: 0.1699 | Regression_eta: 0.1498 | Regression_pt: 0.1785 | Regression_sin_phi: 0.2509 | LR: 9.67e-04 | DataLoad Time: 0.0203s | Model Forward Time: 6.4026s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0073 | max_forward: 1.0567 | max_loss: 0.2452 | max_backward: 2.4106 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.9836 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 7214.6 | uss_MB: 5292.3 | child_rss_MB: 5654.8 | allocated_MB: 344.4 | reserved_MB: 23648.0 | inactive_split_MB: 194.8 +rank6 [2026-10-28 15:29:38,612] INFO : Step 6200/50000 rank6 | Train Loss: 2.4942 | Train Loss Unweighted: 1.4287 | Train Loss Components: Auxiliary: 0.4744 | Cardinality: 0.2537 | Classification: 0.2580 | Classification_binary: 0.3817 | HitGrouping: 0.2748 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3867 | Regression_eta: 0.0050 | Regression_pt: 0.3920 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1434 | Classification_binary: 0.3953 | Regression_cos_phi: 0.1746 | Regression_energy: 0.1824 | Regression_eta: 0.1384 | Regression_pt: 0.1933 | Regression_sin_phi: 0.1842 | LR: 9.66e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0639s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7838 | max_loss: 0.1532 | max_backward: 1.6314 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6093.8 | uss_MB: 4537.3 | child_rss_MB: 5656.7 | allocated_MB: 343.8 | reserved_MB: 23830.0 | inactive_split_MB: 215.7 +rank6 [2026-10-28 15:34:14,250] INFO : Step 6300/50000 rank6 | Train Loss: 2.2204 | Train Loss Unweighted: 1.2763 | Train Loss Components: Auxiliary: 0.4285 | Cardinality: 0.2506 | Classification: 0.2258 | Classification_binary: 0.2983 | HitGrouping: 0.2710 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3704 | Regression_eta: 0.0045 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1119 | Classification_binary: 0.2887 | Regression_cos_phi: 0.1996 | Regression_energy: 0.1738 | Regression_eta: 0.1179 | Regression_pt: 0.1820 | Regression_sin_phi: 0.1315 | LR: 9.65e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.6722s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.5854 | max_loss: 0.1923 | max_backward: 1.0848 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.8586 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6096.2 | uss_MB: 4957.2 | child_rss_MB: 5651.4 | allocated_MB: 339.7 | reserved_MB: 24826.0 | inactive_split_MB: 251.6 +rank6 [2026-10-28 15:38:39,607] INFO : Step 6400/50000 rank6 | Train Loss: 2.2882 | Train Loss Unweighted: 1.3014 | Train Loss Components: Auxiliary: 0.4348 | Cardinality: 0.2271 | Classification: 0.2431 | Classification_binary: 0.3058 | HitGrouping: 0.2702 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3679 | Regression_eta: 0.0046 | Regression_pt: 0.3751 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1844 | Classification_binary: 0.2919 | Regression_cos_phi: 0.1640 | Regression_energy: 0.1725 | Regression_eta: 0.1573 | Regression_pt: 0.1835 | Regression_sin_phi: 0.2346 | LR: 9.64e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.8555s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7489 | max_loss: 0.3608 | max_backward: 2.5327 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6095.6 | uss_MB: 4594.4 | child_rss_MB: 5649.8 | allocated_MB: 345.8 | reserved_MB: 24826.0 | inactive_split_MB: 253.0 +rank6 [2026-10-28 15:42:45,600] INFO : Step 6500/50000 rank6 | Train Loss: 2.4626 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.3022 | Classification: 0.2393 | Classification_binary: 0.3805 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3783 | Regression_eta: 0.0049 | Regression_pt: 0.3831 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1742 | Classification_binary: 0.3720 | Regression_cos_phi: 0.2282 | Regression_energy: 0.1836 | Regression_eta: 0.1207 | Regression_pt: 0.1932 | Regression_sin_phi: 0.1834 | LR: 9.63e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1887s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7600 | max_loss: 0.1555 | max_backward: 1.3910 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6094.6 | uss_MB: 4960.0 | child_rss_MB: 5656.1 | allocated_MB: 344.4 | reserved_MB: 24826.0 | inactive_split_MB: 269.0 +rank6 [2026-10-28 15:46:47,732] INFO : Step 6600/50000 rank6 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3617 | Train Loss Components: Auxiliary: 0.4493 | Cardinality: 0.2405 | Classification: 0.2289 | Classification_binary: 0.3388 | HitGrouping: 0.2650 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3914 | Regression_eta: 0.0045 | Regression_pt: 0.3935 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1662 | Classification_binary: 0.3664 | Regression_cos_phi: 0.2593 | Regression_energy: 0.2102 | Regression_eta: 0.1589 | Regression_pt: 0.2163 | Regression_sin_phi: 0.2615 | LR: 9.62e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7762s | Phase Times: max_pre_step_barrier: 0.1106 | max_batch_to_device: 0.0061 | max_forward: 0.7587 | max_loss: 0.2869 | max_backward: 2.2641 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1321 | Memory: rss_MB: 6092.2 | uss_MB: 4956.2 | child_rss_MB: 5651.6 | allocated_MB: 343.8 | reserved_MB: 25846.0 | inactive_split_MB: 267.9 +rank6 [2026-10-28 15:51:24,435] INFO : Step 6700/50000 rank6 | Train Loss: 2.1225 | Train Loss Unweighted: 1.2385 | Train Loss Components: Auxiliary: 0.4092 | Cardinality: 0.2100 | Classification: 0.2079 | Classification_binary: 0.2916 | HitGrouping: 0.2751 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3623 | Regression_eta: 0.0041 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1437 | Classification_binary: 0.3166 | Regression_cos_phi: 0.1716 | Regression_energy: 0.1670 | Regression_eta: 0.0939 | Regression_pt: 0.1768 | Regression_sin_phi: 0.1534 | LR: 9.61e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9907s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.6452 | max_loss: 0.1453 | max_backward: 1.2984 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.0269 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6098.1 | uss_MB: 4965.1 | child_rss_MB: 5657.8 | allocated_MB: 345.0 | reserved_MB: 25846.0 | inactive_split_MB: 258.8 +rank6 [2026-10-28 15:55:32,973] INFO : Step 6800/50000 rank6 | Train Loss: 2.4601 | Train Loss Unweighted: 1.3772 | Train Loss Components: Auxiliary: 0.4589 | Cardinality: 0.2427 | Classification: 0.2631 | Classification_binary: 0.3490 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3747 | Regression_eta: 0.0054 | Regression_pt: 0.3793 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1817 | Classification_binary: 0.3439 | Regression_cos_phi: 0.2034 | Regression_energy: 0.1853 | Regression_eta: 0.1348 | Regression_pt: 0.1931 | Regression_sin_phi: 0.2358 | LR: 9.60e-04 | DataLoad Time: 0.0259s | Model Forward Time: 5.6131s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0662 | max_forward: 1.6558 | max_loss: 0.3855 | max_backward: 2.1309 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 1.0709 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6092.6 | uss_MB: 4961.3 | child_rss_MB: 5647.9 | allocated_MB: 345.8 | reserved_MB: 26622.0 | inactive_split_MB: 194.2 +rank6 [2026-10-28 15:59:35,211] INFO : Step 6900/50000 rank6 | Train Loss: 2.3793 | Train Loss Unweighted: 1.3337 | Train Loss Components: Auxiliary: 0.4495 | Cardinality: 0.2757 | Classification: 0.2300 | Classification_binary: 0.3711 | HitGrouping: 0.2647 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3605 | Regression_eta: 0.0045 | Regression_pt: 0.3628 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1373 | Classification_binary: 0.3837 | Regression_cos_phi: 0.2257 | Regression_energy: 0.1666 | Regression_eta: 0.1496 | Regression_pt: 0.1763 | Regression_sin_phi: 0.2252 | LR: 9.58e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2970s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1120 | max_forward: 0.8190 | max_loss: 0.1496 | max_backward: 1.4356 | max_optimizer_step: 0.2847 | max_tensorboard_logging: 0.8440 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6096.9 | uss_MB: 4962.9 | child_rss_MB: 5656.8 | allocated_MB: 345.4 | reserved_MB: 26622.0 | inactive_split_MB: 300.5 +rank6 [2026-10-28 16:03:56,961] INFO : Step 7000/50000 rank6 | Train Loss: 2.2410 | Train Loss Unweighted: 1.2707 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2746 | Classification: 0.1970 | Classification_binary: 0.3002 | HitGrouping: 0.2655 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3795 | Regression_eta: 0.0045 | Regression_pt: 0.3852 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1269 | Classification_binary: 0.3327 | Regression_cos_phi: 0.1835 | Regression_energy: 0.1651 | Regression_eta: 0.1227 | Regression_pt: 0.1756 | Regression_sin_phi: 0.1605 | LR: 9.57e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.3851s | Phase Times: max_pre_step_barrier: 0.0229 | max_batch_to_device: 0.0154 | max_forward: 1.0393 | max_loss: 0.2251 | max_backward: 2.0451 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.9425 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6097.4 | uss_MB: 4965.6 | child_rss_MB: 5656.9 | allocated_MB: 345.5 | reserved_MB: 26622.0 | inactive_split_MB: 296.5 +rank6 [2026-10-28 16:03:58,355] INFO : Running validation on rank6 +rank6 [2026-10-28 16:03:58,355] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 16:03:58,355] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 16:04:51,886] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:04:52,381] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 16:05:18,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 16:05:19,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 16:05:20,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 16:05:21,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 16:05:31,571] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 16:05:32,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:05:32,553] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 16:05:49,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 16:05:50,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 16:05:51,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 16:05:52,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 16:06:02,070] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 16:06:02,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:06:02,706] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 16:06:18,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 16:06:19,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 16:06:19,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 16:06:20,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 16:06:30,524] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 16:10:49,042] INFO : Step 7100/50000 rank6 | Train Loss: 2.1086 | Train Loss Unweighted: 1.2525 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.1821 | Classification: 0.2186 | Classification_binary: 0.2903 | HitGrouping: 0.2613 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3655 | Regression_eta: 0.0041 | Regression_pt: 0.3696 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1414 | Classification_binary: 0.2735 | Regression_cos_phi: 0.2054 | Regression_energy: 0.1799 | Regression_eta: 0.1098 | Regression_pt: 0.1864 | Regression_sin_phi: 0.2426 | LR: 9.56e-04 | DataLoad Time: 0.0198s | Model Forward Time: 3.9379s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6376 | max_loss: 0.1424 | max_backward: 1.3436 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8454 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 7191.1 | uss_MB: 6057.4 | child_rss_MB: 5686.3 | allocated_MB: 345.6 | reserved_MB: 22618.0 | inactive_split_MB: 206.0 +rank6 [2026-10-28 16:15:05,713] INFO : Step 7200/50000 rank6 | Train Loss: 2.4589 | Train Loss Unweighted: 1.3873 | Train Loss Components: Auxiliary: 0.4658 | Cardinality: 0.2721 | Classification: 0.2648 | Classification_binary: 0.3806 | HitGrouping: 0.2633 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3642 | Regression_eta: 0.0051 | Regression_pt: 0.3675 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1479 | Classification_binary: 0.3674 | Regression_cos_phi: 0.1985 | Regression_energy: 0.1539 | Regression_eta: 0.1093 | Regression_pt: 0.1589 | Regression_sin_phi: 0.1750 | LR: 9.55e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.3040s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 1.0191 | max_loss: 0.1674 | max_backward: 1.7813 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6183.3 | uss_MB: 5050.5 | child_rss_MB: 5662.0 | allocated_MB: 343.5 | reserved_MB: 22618.0 | inactive_split_MB: 315.3 +rank6 [2026-10-28 16:19:17,625] INFO : Step 7300/50000 rank6 | Train Loss: 2.1617 | Train Loss Unweighted: 1.2404 | Train Loss Components: Auxiliary: 0.4141 | Cardinality: 0.2258 | Classification: 0.1979 | Classification_binary: 0.3107 | HitGrouping: 0.2671 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3613 | Regression_eta: 0.0041 | Regression_pt: 0.3621 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1322 | Classification_binary: 0.3025 | Regression_cos_phi: 0.2606 | Regression_energy: 0.1689 | Regression_eta: 0.1276 | Regression_pt: 0.1745 | Regression_sin_phi: 0.1746 | LR: 9.53e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.7526s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0084 | max_forward: 0.7227 | max_loss: 0.1491 | max_backward: 1.4232 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.5902 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6180.9 | uss_MB: 5051.2 | child_rss_MB: 5696.9 | allocated_MB: 345.0 | reserved_MB: 23608.0 | inactive_split_MB: 295.0 +rank6 [2026-10-28 16:23:25,990] INFO : Step 7400/50000 rank6 | Train Loss: 2.0741 | Train Loss Unweighted: 1.2014 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2287 | Classification: 0.2035 | Classification_binary: 0.2781 | HitGrouping: 0.2646 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3549 | Regression_eta: 0.0040 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1299 | Classification_binary: 0.2831 | Regression_cos_phi: 0.1565 | Regression_energy: 0.1484 | Regression_eta: 0.1392 | Regression_pt: 0.1509 | Regression_sin_phi: 0.1367 | LR: 9.52e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6549s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6129 | max_loss: 0.2127 | max_backward: 1.2944 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6450 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6180.9 | uss_MB: 5049.5 | child_rss_MB: 5695.6 | allocated_MB: 343.8 | reserved_MB: 23608.0 | inactive_split_MB: 229.5 +rank6 [2026-10-28 16:27:18,559] INFO : Step 7500/50000 rank6 | Train Loss: 2.4640 | Train Loss Unweighted: 1.3686 | Train Loss Components: Auxiliary: 0.4637 | Cardinality: 0.2640 | Classification: 0.2570 | Classification_binary: 0.3594 | HitGrouping: 0.2687 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3684 | Regression_eta: 0.0054 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1814 | Classification_binary: 0.3700 | Regression_cos_phi: 0.1999 | Regression_energy: 0.1942 | Regression_eta: 0.1420 | Regression_pt: 0.2040 | Regression_sin_phi: 0.2004 | LR: 9.51e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1725s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0887 | max_forward: 0.7717 | max_loss: 0.1508 | max_backward: 1.3590 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.9634 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6179.4 | uss_MB: 5053.3 | child_rss_MB: 5684.8 | allocated_MB: 348.6 | reserved_MB: 23608.0 | inactive_split_MB: 283.7 +rank6 [2026-10-28 16:31:22,304] INFO : Step 7600/50000 rank6 | Train Loss: 2.3745 | Train Loss Unweighted: 1.3447 | Train Loss Components: Auxiliary: 0.4509 | Cardinality: 0.2616 | Classification: 0.2519 | Classification_binary: 0.3743 | HitGrouping: 0.2627 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3532 | Regression_eta: 0.0050 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1722 | Classification_binary: 0.3986 | Regression_cos_phi: 0.2100 | Regression_energy: 0.1888 | Regression_eta: 0.1485 | Regression_pt: 0.1968 | Regression_sin_phi: 0.2164 | LR: 9.50e-04 | DataLoad Time: 0.0142s | Model Forward Time: 16.0046s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.7345 | max_loss: 0.1617 | max_backward: 1.4897 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 12.6867 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6179.5 | uss_MB: 5051.2 | child_rss_MB: 5695.8 | allocated_MB: 345.3 | reserved_MB: 23608.0 | inactive_split_MB: 284.8 +rank6 [2026-10-28 16:35:23,036] INFO : Step 7700/50000 rank6 | Train Loss: 2.0023 | Train Loss Unweighted: 1.1475 | Train Loss Components: Auxiliary: 0.3866 | Cardinality: 0.2005 | Classification: 0.1968 | Classification_binary: 0.2784 | HitGrouping: 0.2562 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3315 | Regression_eta: 0.0040 | Regression_pt: 0.3329 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1388 | Classification_binary: 0.3154 | Regression_cos_phi: 0.2018 | Regression_energy: 0.1676 | Regression_eta: 0.1318 | Regression_pt: 0.1708 | Regression_sin_phi: 0.1972 | LR: 9.48e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.0198s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7891 | max_loss: 0.1927 | max_backward: 1.4291 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6182.9 | uss_MB: 5055.0 | child_rss_MB: 5702.0 | allocated_MB: 343.9 | reserved_MB: 23608.0 | inactive_split_MB: 279.7 +rank6 [2026-10-28 16:39:23,042] INFO : Step 7800/50000 rank6 | Train Loss: 2.0473 | Train Loss Unweighted: 1.2037 | Train Loss Components: Auxiliary: 0.4013 | Cardinality: 0.1891 | Classification: 0.2102 | Classification_binary: 0.2760 | HitGrouping: 0.2604 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3526 | Regression_eta: 0.0038 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1251 | Classification_binary: 0.2878 | Regression_cos_phi: 0.1917 | Regression_energy: 0.1859 | Regression_eta: 0.1264 | Regression_pt: 0.1975 | Regression_sin_phi: 0.2083 | LR: 9.47e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.4334s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.7326 | max_loss: 0.2464 | max_backward: 1.8124 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9395 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6182.4 | uss_MB: 5059.4 | child_rss_MB: 5675.6 | allocated_MB: 343.5 | reserved_MB: 24612.0 | inactive_split_MB: 224.8 +rank6 [2026-10-28 16:43:21,636] INFO : Step 7900/50000 rank6 | Train Loss: 2.3872 | Train Loss Unweighted: 1.3279 | Train Loss Components: Auxiliary: 0.4525 | Cardinality: 0.2722 | Classification: 0.2401 | Classification_binary: 0.3580 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3573 | Regression_eta: 0.0051 | Regression_pt: 0.3627 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1376 | Classification_binary: 0.3060 | Regression_cos_phi: 0.1591 | Regression_energy: 0.1671 | Regression_eta: 0.0925 | Regression_pt: 0.1742 | Regression_sin_phi: 0.1500 | LR: 9.46e-04 | DataLoad Time: 0.0143s | Model Forward Time: 3.7845s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7032 | max_loss: 0.2052 | max_backward: 1.5183 | max_optimizer_step: 0.1922 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6185.2 | uss_MB: 5060.7 | child_rss_MB: 5695.7 | allocated_MB: 347.1 | reserved_MB: 24612.0 | inactive_split_MB: 279.0 +rank6 [2026-10-28 16:47:21,752] INFO : Step 8000/50000 rank6 | Train Loss: 2.1303 | Train Loss Unweighted: 1.2019 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2451 | Classification: 0.2031 | Classification_binary: 0.3087 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3382 | Regression_eta: 0.0041 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1367 | Classification_binary: 0.3320 | Regression_cos_phi: 0.1861 | Regression_energy: 0.1663 | Regression_eta: 0.1240 | Regression_pt: 0.1790 | Regression_sin_phi: 0.1665 | LR: 9.44e-04 | DataLoad Time: 0.0218s | Model Forward Time: 3.7961s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0554 | max_forward: 0.7608 | max_loss: 0.1617 | max_backward: 1.3810 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 6186.5 | uss_MB: 4730.8 | child_rss_MB: 5692.5 | allocated_MB: 344.2 | reserved_MB: 24612.0 | inactive_split_MB: 224.1 +rank6 [2026-10-28 16:47:22,746] INFO : Running validation on rank6 +rank6 [2026-10-28 16:47:22,746] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 16:47:22,746] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 16:48:13,817] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:48:14,392] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 16:48:40,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 16:48:41,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 16:48:42,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 16:48:43,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 16:48:53,585] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 16:48:53,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:48:54,469] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 16:49:10,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 16:49:11,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 16:49:12,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 16:49:13,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 16:49:23,763] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 16:49:24,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:49:24,963] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 16:49:40,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 16:49:41,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 16:49:41,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 16:49:42,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 16:49:52,551] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 16:54:09,126] INFO : Step 8100/50000 rank6 | Train Loss: 2.0320 | Train Loss Unweighted: 1.1744 | Train Loss Components: Auxiliary: 0.3893 | Cardinality: 0.1986 | Classification: 0.1938 | Classification_binary: 0.2691 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3510 | Regression_eta: 0.0039 | Regression_pt: 0.3525 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1324 | Classification_binary: 0.2420 | Regression_cos_phi: 0.2289 | Regression_energy: 0.1565 | Regression_eta: 0.1415 | Regression_pt: 0.1626 | Regression_sin_phi: 0.1770 | LR: 9.43e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0432s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6251 | max_loss: 0.2465 | max_backward: 1.3564 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8507 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7244.9 | uss_MB: 6118.4 | child_rss_MB: 5660.8 | allocated_MB: 342.9 | reserved_MB: 23510.0 | inactive_split_MB: 258.1 +rank6 [2026-10-28 16:58:00,710] INFO : Step 8200/50000 rank6 | Train Loss: 2.1917 | Train Loss Unweighted: 1.2329 | Train Loss Components: Auxiliary: 0.4200 | Cardinality: 0.2405 | Classification: 0.2359 | Classification_binary: 0.3328 | HitGrouping: 0.2582 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3251 | Regression_eta: 0.0043 | Regression_pt: 0.3305 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1599 | Classification_binary: 0.2928 | Regression_cos_phi: 0.1996 | Regression_energy: 0.1617 | Regression_eta: 0.0978 | Regression_pt: 0.1709 | Regression_sin_phi: 0.1892 | LR: 9.42e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2794s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0064 | max_forward: 0.7679 | max_loss: 0.2484 | max_backward: 1.6620 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.8049 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1289 | Memory: rss_MB: 6232.6 | uss_MB: 5106.1 | child_rss_MB: 5673.2 | allocated_MB: 343.2 | reserved_MB: 23510.0 | inactive_split_MB: 292.4 +rank6 [2026-10-28 17:02:03,650] INFO : Step 8300/50000 rank6 | Train Loss: 2.1924 | Train Loss Unweighted: 1.2399 | Train Loss Components: Auxiliary: 0.4122 | Cardinality: 0.2039 | Classification: 0.2083 | Classification_binary: 0.3661 | HitGrouping: 0.2599 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3243 | Regression_eta: 0.0043 | Regression_pt: 0.3323 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1238 | Classification_binary: 0.3798 | Regression_cos_phi: 0.1850 | Regression_energy: 0.1694 | Regression_eta: 0.1179 | Regression_pt: 0.1790 | Regression_sin_phi: 0.2037 | LR: 9.40e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8422s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0061 | max_forward: 0.9407 | max_loss: 0.2296 | max_backward: 1.9622 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6228.1 | uss_MB: 5055.3 | child_rss_MB: 5669.2 | allocated_MB: 344.6 | reserved_MB: 24580.0 | inactive_split_MB: 267.5 +rank6 [2026-10-28 17:06:07,748] INFO : Step 8400/50000 rank6 | Train Loss: 1.9542 | Train Loss Unweighted: 1.1189 | Train Loss Components: Auxiliary: 0.3756 | Cardinality: 0.1979 | Classification: 0.1926 | Classification_binary: 0.2783 | HitGrouping: 0.2585 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3198 | Regression_eta: 0.0035 | Regression_pt: 0.3209 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1067 | Classification_binary: 0.3028 | Regression_cos_phi: 0.1621 | Regression_energy: 0.1453 | Regression_eta: 0.0972 | Regression_pt: 0.1486 | Regression_sin_phi: 0.1318 | LR: 9.39e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2375s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6939 | max_loss: 0.1474 | max_backward: 1.5357 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 0.9892 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 6216.9 | uss_MB: 5103.3 | child_rss_MB: 5675.6 | allocated_MB: 341.1 | reserved_MB: 24580.0 | inactive_split_MB: 178.9 +rank6 [2026-10-28 17:10:17,609] INFO : Step 8500/50000 rank6 | Train Loss: 2.0642 | Train Loss Unweighted: 1.1724 | Train Loss Components: Auxiliary: 0.3964 | Cardinality: 0.2023 | Classification: 0.2152 | Classification_binary: 0.2897 | HitGrouping: 0.2607 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3289 | Regression_eta: 0.0041 | Regression_pt: 0.3302 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1154 | Classification_binary: 0.2707 | Regression_cos_phi: 0.1194 | Regression_energy: 0.1502 | Regression_eta: 0.1087 | Regression_pt: 0.1532 | Regression_sin_phi: 0.1622 | LR: 9.37e-04 | DataLoad Time: 0.0186s | Model Forward Time: 3.7363s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0069 | max_forward: 0.6424 | max_loss: 0.1972 | max_backward: 1.2711 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.7327 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6218.9 | uss_MB: 5108.3 | child_rss_MB: 5667.6 | allocated_MB: 344.4 | reserved_MB: 25586.0 | inactive_split_MB: 317.0 +rank6 [2026-10-28 17:14:24,634] INFO : Step 8600/50000 rank6 | Train Loss: 2.2820 | Train Loss Unweighted: 1.2960 | Train Loss Components: Auxiliary: 0.4378 | Cardinality: 0.2546 | Classification: 0.2294 | Classification_binary: 0.3478 | HitGrouping: 0.2584 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3521 | Regression_eta: 0.0046 | Regression_pt: 0.3576 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1678 | Classification_binary: 0.3586 | Regression_cos_phi: 0.2341 | Regression_energy: 0.1602 | Regression_eta: 0.1766 | Regression_pt: 0.1671 | Regression_sin_phi: 0.2237 | LR: 9.36e-04 | DataLoad Time: 0.0210s | Model Forward Time: 3.9599s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0069 | max_forward: 0.7396 | max_loss: 0.1527 | max_backward: 1.6088 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5007 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6218.9 | uss_MB: 5112.5 | child_rss_MB: 5664.7 | allocated_MB: 344.8 | reserved_MB: 25586.0 | inactive_split_MB: 283.2 +rank6 [2026-10-28 17:18:21,031] INFO : Step 8700/50000 rank6 | Train Loss: 2.0581 | Train Loss Unweighted: 1.1705 | Train Loss Components: Auxiliary: 0.4038 | Cardinality: 0.2463 | Classification: 0.2066 | Classification_binary: 0.3017 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3267 | Regression_eta: 0.0035 | Regression_pt: 0.3282 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1264 | Classification_binary: 0.3293 | Regression_cos_phi: 0.1915 | Regression_energy: 0.1490 | Regression_eta: 0.1209 | Regression_pt: 0.1560 | Regression_sin_phi: 0.1725 | LR: 9.34e-04 | DataLoad Time: 0.0187s | Model Forward Time: 3.7165s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6873 | max_loss: 0.2444 | max_backward: 1.2985 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6214.6 | uss_MB: 5113.2 | child_rss_MB: 5650.1 | allocated_MB: 343.7 | reserved_MB: 25586.0 | inactive_split_MB: 260.4 +rank6 [2026-10-28 17:22:21,713] INFO : Step 8800/50000 rank6 | Train Loss: 1.8415 | Train Loss Unweighted: 1.0591 | Train Loss Components: Auxiliary: 0.3597 | Cardinality: 0.1811 | Classification: 0.1734 | Classification_binary: 0.2490 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3144 | Regression_eta: 0.0034 | Regression_pt: 0.3153 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0938 | Classification_binary: 0.2405 | Regression_cos_phi: 0.1244 | Regression_energy: 0.1374 | Regression_eta: 0.0803 | Regression_pt: 0.1420 | Regression_sin_phi: 0.1151 | LR: 9.33e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5718s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6737 | max_loss: 0.2385 | max_backward: 1.3073 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.5819 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6215.6 | uss_MB: 5114.6 | child_rss_MB: 5649.0 | allocated_MB: 344.1 | reserved_MB: 25586.0 | inactive_split_MB: 205.9 +rank6 [2026-10-28 17:26:43,764] INFO : Step 8900/50000 rank6 | Train Loss: 2.1706 | Train Loss Unweighted: 1.1768 | Train Loss Components: Auxiliary: 0.4167 | Cardinality: 0.3000 | Classification: 0.2064 | Classification_binary: 0.3037 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3284 | Regression_eta: 0.0039 | Regression_pt: 0.3304 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1063 | Classification_binary: 0.3163 | Regression_cos_phi: 0.2137 | Regression_energy: 0.1470 | Regression_eta: 0.1158 | Regression_pt: 0.1505 | Regression_sin_phi: 0.1842 | LR: 9.31e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.8160s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0073 | max_forward: 0.9973 | max_loss: 0.4040 | max_backward: 3.4126 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6194.6 | uss_MB: 5119.9 | child_rss_MB: 5654.4 | allocated_MB: 347.1 | reserved_MB: 25586.0 | inactive_split_MB: 275.1 +rank6 [2026-10-28 17:30:46,803] INFO : Step 9000/50000 rank6 | Train Loss: 2.1357 | Train Loss Unweighted: 1.1883 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.2351 | Classification: 0.2003 | Classification_binary: 0.3467 | HitGrouping: 0.2532 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3143 | Regression_eta: 0.0041 | Regression_pt: 0.3186 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1513 | Classification_binary: 0.4130 | Regression_cos_phi: 0.2030 | Regression_energy: 0.1514 | Regression_eta: 0.1390 | Regression_pt: 0.1586 | Regression_sin_phi: 0.1913 | LR: 9.30e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.7214s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.7873 | max_loss: 0.2178 | max_backward: 1.5407 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.3125 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 6192.6 | uss_MB: 5121.2 | child_rss_MB: 5660.4 | allocated_MB: 344.6 | reserved_MB: 25586.0 | inactive_split_MB: 289.9 +rank6 [2026-10-28 17:30:47,949] INFO : Running validation on rank6 +rank6 [2026-10-28 17:30:47,950] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 17:30:47,950] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 17:31:40,963] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:31:41,477] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 17:32:08,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 17:32:10,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 17:32:11,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 17:32:12,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 17:32:22,114] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 17:32:22,657] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:32:23,201] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 17:32:39,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 17:32:40,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 17:32:41,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 17:32:42,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 17:32:52,533] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 17:32:52,552] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:32:53,102] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 17:33:08,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 17:33:09,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 17:33:10,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 17:33:10,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 17:33:20,714] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 17:37:41,205] INFO : Step 9100/50000 rank6 | Train Loss: 1.8448 | Train Loss Unweighted: 1.0695 | Train Loss Components: Auxiliary: 0.3538 | Cardinality: 0.1722 | Classification: 0.1878 | Classification_binary: 0.2661 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3012 | Regression_eta: 0.0033 | Regression_pt: 0.3075 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1243 | Classification_binary: 0.2949 | Regression_cos_phi: 0.1333 | Regression_energy: 0.1565 | Regression_eta: 0.1069 | Regression_pt: 0.1614 | Regression_sin_phi: 0.1848 | LR: 9.28e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4311s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0153 | max_forward: 1.2776 | max_loss: 0.3881 | max_backward: 1.3802 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9203 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7188.5 | uss_MB: 6128.3 | child_rss_MB: 5636.7 | allocated_MB: 345.3 | reserved_MB: 22582.0 | inactive_split_MB: 275.2 +rank6 [2026-10-28 17:41:50,228] INFO : Step 9200/50000 rank6 | Train Loss: 1.9883 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3829 | Cardinality: 0.2086 | Classification: 0.1812 | Classification_binary: 0.2802 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3207 | Regression_eta: 0.0035 | Regression_pt: 0.3257 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0998 | Classification_binary: 0.2374 | Regression_cos_phi: 0.1771 | Regression_energy: 0.1443 | Regression_eta: 0.0949 | Regression_pt: 0.1537 | Regression_sin_phi: 0.1513 | LR: 9.27e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5145s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0073 | max_forward: 0.6839 | max_loss: 0.2414 | max_backward: 1.6336 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 1.0636 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6214.5 | uss_MB: 5161.4 | child_rss_MB: 5636.3 | allocated_MB: 345.6 | reserved_MB: 23224.0 | inactive_split_MB: 190.1 +rank6 [2026-10-28 17:45:52,428] INFO : Step 9300/50000 rank6 | Train Loss: 2.0749 | Train Loss Unweighted: 1.1591 | Train Loss Components: Auxiliary: 0.4049 | Cardinality: 0.2573 | Classification: 0.2065 | Classification_binary: 0.3180 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3111 | Regression_eta: 0.0036 | Regression_pt: 0.3160 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1400 | Classification_binary: 0.3256 | Regression_cos_phi: 0.1514 | Regression_energy: 0.1263 | Regression_eta: 0.1123 | Regression_pt: 0.1313 | Regression_sin_phi: 0.1926 | LR: 9.25e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2159s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8329 | max_loss: 0.1499 | max_backward: 1.4059 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.8766 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6214.5 | uss_MB: 5164.3 | child_rss_MB: 5654.0 | allocated_MB: 344.8 | reserved_MB: 24586.0 | inactive_split_MB: 287.2 +rank6 [2026-10-28 17:50:16,942] INFO : Step 9400/50000 rank6 | Train Loss: 1.8467 | Train Loss Unweighted: 1.0417 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1520 | Classification: 0.1837 | Classification_binary: 0.2977 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2742 | Regression_eta: 0.0040 | Regression_pt: 0.2784 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1061 | Classification_binary: 0.3312 | Regression_cos_phi: 0.1462 | Regression_energy: 0.1539 | Regression_eta: 0.0894 | Regression_pt: 0.1615 | Regression_sin_phi: 0.1411 | LR: 9.24e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.5872s | Phase Times: max_pre_step_barrier: 0.0180 | max_batch_to_device: 0.0062 | max_forward: 1.5076 | max_loss: 0.3748 | max_backward: 2.6247 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.5247 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6206.4 | uss_MB: 4649.5 | child_rss_MB: 5650.0 | allocated_MB: 343.9 | reserved_MB: 24586.0 | inactive_split_MB: 267.3 +rank6 [2026-10-28 17:54:18,173] INFO : Step 9500/50000 rank6 | Train Loss: 1.8199 | Train Loss Unweighted: 0.9995 | Train Loss Components: Auxiliary: 0.3464 | Cardinality: 0.1875 | Classification: 0.1651 | Classification_binary: 0.2558 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2851 | Regression_eta: 0.0035 | Regression_pt: 0.2864 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1121 | Classification_binary: 0.2742 | Regression_cos_phi: 0.1999 | Regression_energy: 0.1323 | Regression_eta: 0.1075 | Regression_pt: 0.1383 | Regression_sin_phi: 0.1869 | LR: 9.22e-04 | DataLoad Time: 0.0198s | Model Forward Time: 3.8169s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6532 | max_loss: 0.2385 | max_backward: 1.0043 | max_optimizer_step: 0.3001 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6205.2 | uss_MB: 5165.5 | child_rss_MB: 5654.5 | allocated_MB: 341.2 | reserved_MB: 24586.0 | inactive_split_MB: 180.8 +rank6 [2026-10-28 17:58:23,634] INFO : Step 9600/50000 rank6 | Train Loss: 2.0900 | Train Loss Unweighted: 1.1491 | Train Loss Components: Auxiliary: 0.4030 | Cardinality: 0.2464 | Classification: 0.2089 | Classification_binary: 0.3189 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3034 | Regression_eta: 0.0042 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1487 | Classification_binary: 0.3158 | Regression_cos_phi: 0.1938 | Regression_energy: 0.1552 | Regression_eta: 0.1355 | Regression_pt: 0.1631 | Regression_sin_phi: 0.2010 | LR: 9.21e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8842s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1083 | max_forward: 0.7381 | max_loss: 0.1457 | max_backward: 1.4890 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4970 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6206.2 | uss_MB: 5167.2 | child_rss_MB: 5637.2 | allocated_MB: 345.2 | reserved_MB: 25594.0 | inactive_split_MB: 190.6 +rank6 [2026-10-28 18:02:35,091] INFO : Step 9700/50000 rank6 | Train Loss: 2.1397 | Train Loss Unweighted: 1.1700 | Train Loss Components: Auxiliary: 0.4082 | Cardinality: 0.2377 | Classification: 0.1942 | Classification_binary: 0.3460 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3095 | Regression_eta: 0.0043 | Regression_pt: 0.3118 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1308 | Classification_binary: 0.3796 | Regression_cos_phi: 0.1715 | Regression_energy: 0.1606 | Regression_eta: 0.1297 | Regression_pt: 0.1675 | Regression_sin_phi: 0.2088 | LR: 9.19e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.1042s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7744 | max_loss: 0.1497 | max_backward: 1.6846 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4710 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 6209.3 | uss_MB: 5170.1 | child_rss_MB: 5650.1 | allocated_MB: 345.9 | reserved_MB: 25594.0 | inactive_split_MB: 288.1 +rank6 [2026-10-28 18:06:42,227] INFO : Step 9800/50000 rank6 | Train Loss: 1.8589 | Train Loss Unweighted: 1.0245 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2213 | Classification: 0.1590 | Classification_binary: 0.2751 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2908 | Regression_eta: 0.0031 | Regression_pt: 0.2931 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1085 | Classification_binary: 0.3195 | Regression_cos_phi: 0.1685 | Regression_energy: 0.1396 | Regression_eta: 0.0830 | Regression_pt: 0.1457 | Regression_sin_phi: 0.1620 | LR: 9.17e-04 | DataLoad Time: 0.0234s | Model Forward Time: 4.1205s | Phase Times: max_pre_step_barrier: 0.1099 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.1713 | max_backward: 1.7200 | max_optimizer_step: 0.1898 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6188.3 | uss_MB: 5169.7 | child_rss_MB: 5642.5 | allocated_MB: 342.5 | reserved_MB: 26610.0 | inactive_split_MB: 264.5 +rank6 [2026-10-28 18:10:58,312] INFO : Step 9900/50000 rank6 | Train Loss: 1.8366 | Train Loss Unweighted: 1.0077 | Train Loss Components: Auxiliary: 0.3557 | Cardinality: 0.2091 | Classification: 0.1810 | Classification_binary: 0.2609 | HitGrouping: 0.2462 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2775 | Regression_eta: 0.0035 | Regression_pt: 0.2814 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1506 | Classification_binary: 0.2218 | Regression_cos_phi: 0.1280 | Regression_energy: 0.1123 | Regression_eta: 0.0971 | Regression_pt: 0.1154 | Regression_sin_phi: 0.1629 | LR: 9.16e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5791s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6346 | max_loss: 0.1692 | max_backward: 1.3542 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6188.4 | uss_MB: 5170.9 | child_rss_MB: 5645.5 | allocated_MB: 343.7 | reserved_MB: 26610.0 | inactive_split_MB: 205.7 +rank6 [2026-10-28 18:15:06,635] INFO : Step 10000/50000 rank6 | Train Loss: 2.0706 | Train Loss Unweighted: 1.1295 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2648 | Classification: 0.1819 | Classification_binary: 0.3225 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3077 | Regression_eta: 0.0037 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1072 | Classification_binary: 0.3345 | Regression_cos_phi: 0.1838 | Regression_energy: 0.1479 | Regression_eta: 0.1031 | Regression_pt: 0.1559 | Regression_sin_phi: 0.1786 | LR: 9.14e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.7023s | Phase Times: max_pre_step_barrier: 0.0136 | max_batch_to_device: 0.0708 | max_forward: 1.8842 | max_loss: 0.5484 | max_backward: 2.6620 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.5314 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6190.4 | uss_MB: 5174.2 | child_rss_MB: 5642.5 | allocated_MB: 345.2 | reserved_MB: 26610.0 | inactive_split_MB: 288.8 +rank6 [2026-10-28 18:15:07,627] INFO : Running validation on rank6 +rank6 [2026-10-28 18:15:07,627] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 18:15:07,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 18:15:55,923] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:15:56,538] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 18:16:22,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 18:16:23,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 18:16:24,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 18:16:25,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 18:16:35,813] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 18:16:36,561] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:16:37,046] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 18:16:53,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 18:16:55,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 18:16:58,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 18:16:59,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 18:17:09,176] INFO : Time taken to make predictions on device 6 is: 0.30 min +rank6 [2026-10-28 18:17:09,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:17:10,078] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 18:17:25,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 18:17:26,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 18:17:27,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 18:17:27,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 18:17:37,810] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 18:22:01,596] INFO : Step 10100/50000 rank6 | Train Loss: 1.8115 | Train Loss Unweighted: 1.0221 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.1999 | Classification: 0.1589 | Classification_binary: 0.2815 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2868 | Regression_eta: 0.0032 | Regression_pt: 0.2886 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1009 | Classification_binary: 0.2985 | Regression_cos_phi: 0.1160 | Regression_energy: 0.1455 | Regression_eta: 0.0773 | Regression_pt: 0.1523 | Regression_sin_phi: 0.1136 | LR: 9.12e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9763s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7090 | max_loss: 0.1507 | max_backward: 1.2179 | max_optimizer_step: 0.3146 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 7166.8 | uss_MB: 6150.2 | child_rss_MB: 5645.8 | allocated_MB: 343.1 | reserved_MB: 21794.0 | inactive_split_MB: 243.8 +rank6 [2026-10-28 18:26:06,876] INFO : Step 10200/50000 rank6 | Train Loss: 1.6749 | Train Loss Unweighted: 0.9660 | Train Loss Components: Auxiliary: 0.3310 | Cardinality: 0.1773 | Classification: 0.1544 | Classification_binary: 0.2377 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2827 | Regression_eta: 0.0027 | Regression_pt: 0.2857 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0960 | Classification_binary: 0.2142 | Regression_cos_phi: 0.1264 | Regression_energy: 0.1345 | Regression_eta: 0.0920 | Regression_pt: 0.1396 | Regression_sin_phi: 0.0997 | LR: 9.11e-04 | DataLoad Time: 0.0101s | Model Forward Time: 3.9921s | Phase Times: max_pre_step_barrier: 0.0352 | max_batch_to_device: 0.0241 | max_forward: 0.6625 | max_loss: 0.1442 | max_backward: 1.3835 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7009 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1256 | Memory: rss_MB: 7164.8 | uss_MB: 6150.1 | child_rss_MB: 5645.0 | allocated_MB: 345.9 | reserved_MB: 21794.0 | inactive_split_MB: 292.4 +rank6 [2026-10-28 18:30:14,208] INFO : Step 10300/50000 rank6 | Train Loss: 1.9113 | Train Loss Unweighted: 1.0360 | Train Loss Components: Auxiliary: 0.3660 | Cardinality: 0.2108 | Classification: 0.1830 | Classification_binary: 0.2931 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0033 | Regression_pt: 0.2786 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1167 | Classification_binary: 0.3021 | Regression_cos_phi: 0.1359 | Regression_energy: 0.1328 | Regression_eta: 0.0933 | Regression_pt: 0.1393 | Regression_sin_phi: 0.1699 | LR: 9.09e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9099s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7205 | max_loss: 0.1496 | max_backward: 1.6083 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 6287.2 | uss_MB: 5273.5 | child_rss_MB: 5650.9 | allocated_MB: 343.8 | reserved_MB: 22764.0 | inactive_split_MB: 257.9 +rank6 [2026-10-28 18:34:19,878] INFO : Step 10400/50000 rank6 | Train Loss: 1.9303 | Train Loss Unweighted: 1.0569 | Train Loss Components: Auxiliary: 0.3739 | Cardinality: 0.2318 | Classification: 0.1704 | Classification_binary: 0.3342 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2718 | Regression_eta: 0.0033 | Regression_pt: 0.2738 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1136 | Classification_binary: 0.3388 | Regression_cos_phi: 0.1582 | Regression_energy: 0.1467 | Regression_eta: 0.0947 | Regression_pt: 0.1528 | Regression_sin_phi: 0.1238 | LR: 9.07e-04 | DataLoad Time: 0.0148s | Model Forward Time: 3.7366s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7465 | max_loss: 0.1499 | max_backward: 1.4212 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.4712 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6287.2 | uss_MB: 5272.0 | child_rss_MB: 5660.4 | allocated_MB: 345.6 | reserved_MB: 22792.0 | inactive_split_MB: 259.8 +rank6 [2026-10-28 18:38:24,242] INFO : Step 10500/50000 rank6 | Train Loss: 1.6601 | Train Loss Unweighted: 0.9268 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1892 | Classification: 0.1410 | Classification_binary: 0.2342 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2693 | Regression_eta: 0.0029 | Regression_pt: 0.2766 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.2293 | Regression_cos_phi: 0.1164 | Regression_energy: 0.1225 | Regression_eta: 0.0649 | Regression_pt: 0.1309 | Regression_sin_phi: 0.1170 | LR: 9.06e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.5589s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 0.6364 | max_loss: 0.2260 | max_backward: 1.1629 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6452 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6284.6 | uss_MB: 5269.3 | child_rss_MB: 5635.7 | allocated_MB: 340.4 | reserved_MB: 22792.0 | inactive_split_MB: 230.3 +rank6 [2026-10-28 18:42:22,546] INFO : Step 10600/50000 rank6 | Train Loss: 1.9388 | Train Loss Unweighted: 1.0719 | Train Loss Components: Auxiliary: 0.3737 | Cardinality: 0.2008 | Classification: 0.1863 | Classification_binary: 0.2709 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3047 | Regression_eta: 0.0042 | Regression_pt: 0.3019 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1194 | Classification_binary: 0.2296 | Regression_cos_phi: 0.1381 | Regression_energy: 0.2178 | Regression_eta: 0.1204 | Regression_pt: 0.2145 | Regression_sin_phi: 0.1270 | LR: 9.04e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9682s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6880 | max_loss: 0.2458 | max_backward: 1.4354 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7481 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 6285.6 | uss_MB: 5273.7 | child_rss_MB: 5653.3 | allocated_MB: 344.6 | reserved_MB: 22798.0 | inactive_split_MB: 286.4 +rank6 [2026-10-28 18:46:28,871] INFO : Step 10700/50000 rank6 | Train Loss: 1.8955 | Train Loss Unweighted: 1.0230 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.2139 | Classification: 0.1752 | Classification_binary: 0.3186 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2598 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1160 | Classification_binary: 0.3565 | Regression_cos_phi: 0.1542 | Regression_energy: 0.1323 | Regression_eta: 0.0898 | Regression_pt: 0.1364 | Regression_sin_phi: 0.1828 | LR: 9.02e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.1515s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0072 | max_forward: 1.0363 | max_loss: 0.1757 | max_backward: 2.3255 | max_optimizer_step: 0.1834 | max_tensorboard_logging: 0.7232 | max_tensorboard_flush: 0.0051 | max_post_step_barrier: 0.1576 | Memory: rss_MB: 6287.7 | uss_MB: 5274.9 | child_rss_MB: 5649.7 | allocated_MB: 342.1 | reserved_MB: 22798.0 | inactive_split_MB: 282.9 +rank6 [2026-10-28 18:50:28,322] INFO : Step 10800/50000 rank6 | Train Loss: 1.7679 | Train Loss Unweighted: 0.9563 | Train Loss Components: Auxiliary: 0.3446 | Cardinality: 0.2074 | Classification: 0.1518 | Classification_binary: 0.2766 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2590 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0795 | Classification_binary: 0.2744 | Regression_cos_phi: 0.0769 | Regression_energy: 0.1270 | Regression_eta: 0.0741 | Regression_pt: 0.1364 | Regression_sin_phi: 0.1379 | LR: 9.00e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7793s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0995 | max_forward: 0.7884 | max_loss: 0.2225 | max_backward: 2.0144 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.5230 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6288.7 | uss_MB: 5271.1 | child_rss_MB: 5654.7 | allocated_MB: 343.6 | reserved_MB: 23784.0 | inactive_split_MB: 286.4 +rank6 [2026-10-28 18:54:31,737] INFO : Step 10900/50000 rank6 | Train Loss: 1.6025 | Train Loss Unweighted: 0.8934 | Train Loss Components: Auxiliary: 0.3208 | Cardinality: 0.1729 | Classification: 0.1446 | Classification_binary: 0.2284 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2574 | Regression_eta: 0.0027 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1032 | Classification_binary: 0.2341 | Regression_cos_phi: 0.1241 | Regression_energy: 0.1212 | Regression_eta: 0.0754 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0738 | LR: 8.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.4463s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.5765 | max_loss: 0.1490 | max_backward: 1.1882 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6288.0 | uss_MB: 5275.5 | child_rss_MB: 5639.7 | allocated_MB: 342.1 | reserved_MB: 23796.0 | inactive_split_MB: 272.7 +rank6 [2026-10-28 18:58:45,418] INFO : Step 11000/50000 rank6 | Train Loss: 1.9315 | Train Loss Unweighted: 1.0466 | Train Loss Components: Auxiliary: 0.3721 | Cardinality: 0.2078 | Classification: 0.1806 | Classification_binary: 0.3087 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0037 | Regression_pt: 0.2758 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0870 | Classification_binary: 0.2864 | Regression_cos_phi: 0.1650 | Regression_energy: 0.1567 | Regression_eta: 0.0818 | Regression_pt: 0.1634 | Regression_sin_phi: 0.1154 | LR: 8.97e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.1701s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7424 | max_loss: 0.1704 | max_backward: 1.5866 | max_optimizer_step: 0.2906 | max_tensorboard_logging: 0.6287 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6277.0 | uss_MB: 5281.8 | child_rss_MB: 5663.8 | allocated_MB: 347.8 | reserved_MB: 23796.0 | inactive_split_MB: 262.8 +rank6 [2026-10-28 18:58:46,360] INFO : Running validation on rank6 +rank6 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 18:59:40,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:59:40,900] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 19:00:07,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 19:00:08,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 19:00:10,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 19:00:11,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 19:00:21,179] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 19:00:21,678] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:00:22,257] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 19:00:38,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 19:00:39,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 19:00:40,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 19:00:41,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 19:00:51,588] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 19:00:52,166] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:00:52,895] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 19:01:08,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 19:01:08,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 19:01:09,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 19:01:10,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 19:01:20,226] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 19:05:41,392] INFO : Step 11100/50000 rank6 | Train Loss: 1.9118 | Train Loss Unweighted: 1.0064 | Train Loss Components: Auxiliary: 0.3723 | Cardinality: 0.2793 | Classification: 0.1563 | Classification_binary: 0.3082 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2660 | Regression_eta: 0.0034 | Regression_pt: 0.2693 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0936 | Classification_binary: 0.3119 | Regression_cos_phi: 0.1570 | Regression_energy: 0.1231 | Regression_eta: 0.0825 | Regression_pt: 0.1273 | Regression_sin_phi: 0.1480 | LR: 8.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0413s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.9217 | max_loss: 0.2304 | max_backward: 1.5570 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 7169.9 | uss_MB: 6179.3 | child_rss_MB: 5664.2 | allocated_MB: 343.2 | reserved_MB: 22146.0 | inactive_split_MB: 281.7 +rank6 [2026-10-28 19:09:39,137] INFO : Step 11200/50000 rank6 | Train Loss: 1.7366 | Train Loss Unweighted: 0.9397 | Train Loss Components: Auxiliary: 0.3529 | Cardinality: 0.2466 | Classification: 0.1384 | Classification_binary: 0.2548 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2689 | Regression_eta: 0.0027 | Regression_pt: 0.2722 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.2615 | Regression_cos_phi: 0.1204 | Regression_energy: 0.1403 | Regression_eta: 0.0666 | Regression_pt: 0.1496 | Regression_sin_phi: 0.1132 | LR: 8.93e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7181s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7793 | max_loss: 0.1825 | max_backward: 1.3743 | max_optimizer_step: 0.2128 | max_tensorboard_logging: 0.4676 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7167.9 | uss_MB: 6179.2 | child_rss_MB: 5641.7 | allocated_MB: 344.6 | reserved_MB: 22972.0 | inactive_split_MB: 273.4 +rank6 [2026-10-28 19:13:41,796] INFO : Step 11300/50000 rank6 | Train Loss: 1.6876 | Train Loss Unweighted: 0.9423 | Train Loss Components: Auxiliary: 0.3334 | Cardinality: 0.1862 | Classification: 0.1581 | Classification_binary: 0.2713 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2524 | Regression_eta: 0.0027 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1054 | Classification_binary: 0.2494 | Regression_cos_phi: 0.0772 | Regression_energy: 0.1188 | Regression_eta: 0.0654 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0923 | LR: 8.91e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6804s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0223 | max_forward: 0.9315 | max_loss: 0.2308 | max_backward: 1.9859 | max_optimizer_step: 0.1928 | max_tensorboard_logging: 0.5681 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6388.7 | uss_MB: 5273.7 | child_rss_MB: 5658.4 | allocated_MB: 345.5 | reserved_MB: 24368.0 | inactive_split_MB: 269.6 +rank6 [2026-10-28 19:17:45,332] INFO : Step 11400/50000 rank6 | Train Loss: 1.8563 | Train Loss Unweighted: 0.9980 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.2376 | Classification: 0.1627 | Classification_binary: 0.3164 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2551 | Regression_eta: 0.0029 | Regression_pt: 0.2577 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0906 | Classification_binary: 0.2923 | Regression_cos_phi: 0.1118 | Regression_energy: 0.1198 | Regression_eta: 0.0600 | Regression_pt: 0.1256 | Regression_sin_phi: 0.1083 | LR: 8.89e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.0867s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.7684 | max_loss: 0.1511 | max_backward: 1.6966 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5034 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 6391.7 | uss_MB: 5402.6 | child_rss_MB: 5664.5 | allocated_MB: 343.2 | reserved_MB: 25642.0 | inactive_split_MB: 251.6 +rank6 [2026-10-28 19:21:55,043] INFO : Step 11500/50000 rank6 | Train Loss: 1.6323 | Train Loss Unweighted: 0.8976 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1755 | Classification: 0.1375 | Classification_binary: 0.2717 | HitGrouping: 0.2346 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2408 | Regression_eta: 0.0028 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2850 | Regression_cos_phi: 0.0973 | Regression_energy: 0.1071 | Regression_eta: 0.0923 | Regression_pt: 0.1118 | Regression_sin_phi: 0.1072 | LR: 8.88e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6939s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7232 | max_loss: 0.1557 | max_backward: 1.4770 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6390.5 | uss_MB: 5401.3 | child_rss_MB: 5644.3 | allocated_MB: 345.3 | reserved_MB: 25642.0 | inactive_split_MB: 274.7 +rank6 [2026-10-28 19:25:54,593] INFO : Step 11600/50000 rank6 | Train Loss: 1.6133 | Train Loss Unweighted: 0.9062 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1718 | Classification: 0.1502 | Classification_binary: 0.2356 | HitGrouping: 0.2377 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2569 | Regression_eta: 0.0025 | Regression_pt: 0.2581 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2153 | Regression_cos_phi: 0.0973 | Regression_energy: 0.1013 | Regression_eta: 0.0544 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0909 | LR: 8.86e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.6497s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6646 | max_loss: 0.1710 | max_backward: 1.3208 | max_optimizer_step: 0.1963 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6391.5 | uss_MB: 5404.2 | child_rss_MB: 5665.3 | allocated_MB: 344.1 | reserved_MB: 26630.0 | inactive_split_MB: 209.9 +rank6 [2026-10-28 19:29:50,732] INFO : Step 11700/50000 rank6 | Train Loss: 1.7739 | Train Loss Unweighted: 0.9706 | Train Loss Components: Auxiliary: 0.3531 | Cardinality: 0.2418 | Classification: 0.1518 | Classification_binary: 0.2821 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2638 | Regression_eta: 0.0028 | Regression_pt: 0.2671 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0960 | Classification_binary: 0.2919 | Regression_cos_phi: 0.0935 | Regression_energy: 0.1228 | Regression_eta: 0.0928 | Regression_pt: 0.1286 | Regression_sin_phi: 0.1456 | LR: 8.84e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.4604s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8789 | max_loss: 0.2057 | max_backward: 2.3846 | max_optimizer_step: 0.1875 | max_tensorboard_logging: 1.0403 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6389.5 | uss_MB: 4921.9 | child_rss_MB: 5640.9 | allocated_MB: 343.8 | reserved_MB: 27624.0 | inactive_split_MB: 186.6 +rank6 [2026-10-28 19:33:54,004] INFO : Step 11800/50000 rank6 | Train Loss: 1.8223 | Train Loss Unweighted: 0.9745 | Train Loss Components: Auxiliary: 0.3609 | Cardinality: 0.2329 | Classification: 0.1589 | Classification_binary: 0.3100 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2473 | Regression_eta: 0.0028 | Regression_pt: 0.2523 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0884 | Classification_binary: 0.3359 | Regression_cos_phi: 0.1279 | Regression_energy: 0.1256 | Regression_eta: 0.0812 | Regression_pt: 0.1328 | Regression_sin_phi: 0.1310 | LR: 8.82e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.3829s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0070 | max_forward: 1.2302 | max_loss: 0.2767 | max_backward: 2.3745 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6390.5 | uss_MB: 5402.7 | child_rss_MB: 5661.5 | allocated_MB: 346.0 | reserved_MB: 27624.0 | inactive_split_MB: 280.5 +rank6 [2026-10-28 19:37:48,966] INFO : Step 11900/50000 rank6 | Train Loss: 1.5797 | Train Loss Unweighted: 0.8638 | Train Loss Components: Auxiliary: 0.3225 | Cardinality: 0.1935 | Classification: 0.1358 | Classification_binary: 0.2351 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2427 | Regression_eta: 0.0025 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0820 | Classification_binary: 0.2369 | Regression_cos_phi: 0.1479 | Regression_energy: 0.1181 | Regression_eta: 0.0623 | Regression_pt: 0.1227 | Regression_sin_phi: 0.1114 | LR: 8.80e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.1379s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0787 | max_forward: 0.7011 | max_loss: 0.1484 | max_backward: 1.2002 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.9463 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 6390.3 | uss_MB: 5402.5 | child_rss_MB: 5642.2 | allocated_MB: 340.1 | reserved_MB: 28620.0 | inactive_split_MB: 264.1 +rank6 [2026-10-28 19:41:43,168] INFO : Step 12000/50000 rank6 | Train Loss: 1.6036 | Train Loss Unweighted: 0.8706 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1972 | Classification: 0.1481 | Classification_binary: 0.2459 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2330 | Regression_eta: 0.0025 | Regression_pt: 0.2385 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1010 | Classification_binary: 0.2861 | Regression_cos_phi: 0.1207 | Regression_energy: 0.1107 | Regression_eta: 0.0869 | Regression_pt: 0.1165 | Regression_sin_phi: 0.1276 | LR: 8.78e-04 | DataLoad Time: 0.0139s | Model Forward Time: 3.6998s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0072 | max_forward: 0.6194 | max_loss: 0.1546 | max_backward: 1.4628 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5143 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6383.3 | uss_MB: 5409.4 | child_rss_MB: 5646.2 | allocated_MB: 342.9 | reserved_MB: 28620.0 | inactive_split_MB: 215.8 +rank6 [2026-10-28 19:41:44,125] INFO : Running validation on rank6 +rank6 [2026-10-28 19:41:44,125] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 19:41:44,125] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 19:42:35,085] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:42:35,719] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 19:43:02,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 19:43:04,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 19:43:05,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 19:43:06,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 19:43:16,500] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 19:43:17,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:43:17,650] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 19:43:33,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 19:43:34,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 19:43:35,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 19:43:36,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 19:43:46,708] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 19:43:48,085] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:43:48,650] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 19:44:03,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 19:44:04,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 19:44:05,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 19:44:05,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 19:44:15,914] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 19:48:34,082] INFO : Step 12100/50000 rank6 | Train Loss: 1.8149 | Train Loss Unweighted: 0.9793 | Train Loss Components: Auxiliary: 0.3632 | Cardinality: 0.2324 | Classification: 0.1506 | Classification_binary: 0.3149 | HitGrouping: 0.2337 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2518 | Regression_eta: 0.0029 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0812 | Classification_binary: 0.3002 | Regression_cos_phi: 0.0881 | Regression_energy: 0.1138 | Regression_eta: 0.0715 | Regression_pt: 0.1220 | Regression_sin_phi: 0.1116 | LR: 8.76e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5266s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8305 | max_loss: 0.1678 | max_backward: 1.6498 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8610 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7131.3 | uss_MB: 6150.6 | child_rss_MB: 5646.8 | allocated_MB: 345.8 | reserved_MB: 23512.0 | inactive_split_MB: 299.6 +rank6 [2026-10-28 19:52:54,669] INFO : Step 12200/50000 rank6 | Train Loss: 1.6619 | Train Loss Unweighted: 0.9043 | Train Loss Components: Auxiliary: 0.3290 | Cardinality: 0.1906 | Classification: 0.1445 | Classification_binary: 0.2674 | HitGrouping: 0.2303 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2435 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1000 | Classification_binary: 0.3009 | Regression_cos_phi: 0.1533 | Regression_energy: 0.1161 | Regression_eta: 0.1548 | Regression_pt: 0.1206 | Regression_sin_phi: 0.1282 | LR: 8.74e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8233s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.7862 | max_loss: 0.1438 | max_backward: 1.5991 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.8843 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7133.3 | uss_MB: 5953.1 | child_rss_MB: 5653.1 | allocated_MB: 341.2 | reserved_MB: 23512.0 | inactive_split_MB: 206.8 +rank6 [2026-10-28 19:56:47,337] INFO : Step 12300/50000 rank6 | Train Loss: 1.4699 | Train Loss Unweighted: 0.8181 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1474 | Classification: 0.1404 | Classification_binary: 0.2280 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2222 | Regression_eta: 0.0028 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0797 | Classification_binary: 0.2342 | Regression_cos_phi: 0.1123 | Regression_energy: 0.1073 | Regression_eta: 0.0729 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0978 | LR: 8.72e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7228s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6177 | max_loss: 0.2232 | max_backward: 1.1179 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 0.7974 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6497.9 | uss_MB: 5527.4 | child_rss_MB: 5654.1 | allocated_MB: 343.2 | reserved_MB: 24286.0 | inactive_split_MB: 290.2 +rank6 [2026-10-28 20:00:36,190] INFO : Step 12400/50000 rank6 | Train Loss: 1.6135 | Train Loss Unweighted: 0.8782 | Train Loss Components: Auxiliary: 0.3265 | Cardinality: 0.2056 | Classification: 0.1546 | Classification_binary: 0.2540 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2308 | Regression_eta: 0.0025 | Regression_pt: 0.2337 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1054 | Classification_binary: 0.2760 | Regression_cos_phi: 0.1249 | Regression_energy: 0.1060 | Regression_eta: 0.0846 | Regression_pt: 0.1118 | Regression_sin_phi: 0.1305 | LR: 8.70e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.9934s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6928 | max_loss: 0.1486 | max_backward: 1.3860 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6497.9 | uss_MB: 5528.1 | child_rss_MB: 5671.7 | allocated_MB: 344.8 | reserved_MB: 24510.0 | inactive_split_MB: 295.5 +rank6 [2026-10-28 20:04:30,060] INFO : Step 12500/50000 rank6 | Train Loss: 1.6527 | Train Loss Unweighted: 0.8797 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1890 | Classification: 0.1391 | Classification_binary: 0.2779 | HitGrouping: 0.2315 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2273 | Regression_eta: 0.0030 | Regression_pt: 0.2295 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0874 | Classification_binary: 0.2821 | Regression_cos_phi: 0.0847 | Regression_energy: 0.1117 | Regression_eta: 0.0546 | Regression_pt: 0.1162 | Regression_sin_phi: 0.1100 | LR: 8.68e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.1670s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0535 | max_forward: 0.7747 | max_loss: 0.1534 | max_backward: 1.3339 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6494.1 | uss_MB: 5528.8 | child_rss_MB: 5649.7 | allocated_MB: 347.2 | reserved_MB: 24510.0 | inactive_split_MB: 456.8 +rank6 [2026-10-28 20:08:30,520] INFO : Step 12600/50000 rank6 | Train Loss: 1.4969 | Train Loss Unweighted: 0.8185 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1595 | Classification: 0.1210 | Classification_binary: 0.2357 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2279 | Regression_eta: 0.0024 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1009 | Classification_binary: 0.2730 | Regression_cos_phi: 0.1122 | Regression_energy: 0.1200 | Regression_eta: 0.0769 | Regression_pt: 0.1240 | Regression_sin_phi: 0.1359 | LR: 8.66e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7596s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0062 | max_forward: 0.6571 | max_loss: 0.1429 | max_backward: 1.4364 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6482.1 | uss_MB: 5531.3 | child_rss_MB: 5638.2 | allocated_MB: 342.9 | reserved_MB: 25526.0 | inactive_split_MB: 234.6 +rank6 [2026-10-28 20:12:34,583] INFO : Step 12700/50000 rank6 | Train Loss: 1.6234 | Train Loss Unweighted: 0.8661 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.2004 | Classification: 0.1459 | Classification_binary: 0.2387 | HitGrouping: 0.2403 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2360 | Regression_eta: 0.0027 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0941 | Classification_binary: 0.2362 | Regression_cos_phi: 0.1002 | Regression_energy: 0.1339 | Regression_eta: 0.0888 | Regression_pt: 0.1392 | Regression_sin_phi: 0.1473 | LR: 8.64e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.4794s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6221 | max_loss: 0.1846 | max_backward: 1.1554 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6469.4 | uss_MB: 5531.3 | child_rss_MB: 5648.7 | allocated_MB: 345.9 | reserved_MB: 26286.0 | inactive_split_MB: 241.7 +rank6 [2026-10-28 20:16:22,943] INFO : Step 12800/50000 rank6 | Train Loss: 1.7877 | Train Loss Unweighted: 0.9398 | Train Loss Components: Auxiliary: 0.3651 | Cardinality: 0.2327 | Classification: 0.1495 | Classification_binary: 0.3094 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2357 | Regression_eta: 0.0027 | Regression_pt: 0.2394 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.3038 | Regression_cos_phi: 0.1011 | Regression_energy: 0.1105 | Regression_eta: 0.0738 | Regression_pt: 0.1163 | Regression_sin_phi: 0.1247 | LR: 8.62e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.2584s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.8468 | max_loss: 0.1534 | max_backward: 1.7420 | max_optimizer_step: 0.3013 | max_tensorboard_logging: 0.5725 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6458.4 | uss_MB: 5535.9 | child_rss_MB: 5651.5 | allocated_MB: 345.6 | reserved_MB: 26286.0 | inactive_split_MB: 284.4 +rank6 [2026-10-28 20:20:13,787] INFO : Step 12900/50000 rank6 | Train Loss: 1.5881 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.3191 | Cardinality: 0.2029 | Classification: 0.1304 | Classification_binary: 0.2564 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2291 | Regression_eta: 0.0023 | Regression_pt: 0.2312 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2436 | Regression_cos_phi: 0.0846 | Regression_energy: 0.1211 | Regression_eta: 0.0527 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0899 | LR: 8.60e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9614s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7180 | max_loss: 0.2380 | max_backward: 1.4488 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6932 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6444.6 | uss_MB: 5531.5 | child_rss_MB: 5646.1 | allocated_MB: 341.8 | reserved_MB: 26286.0 | inactive_split_MB: 252.2 +rank6 [2026-10-28 20:24:06,548] INFO : Step 13000/50000 rank6 | Train Loss: 1.5217 | Train Loss Unweighted: 0.8547 | Train Loss Components: Auxiliary: 0.3141 | Cardinality: 0.1693 | Classification: 0.1382 | Classification_binary: 0.2135 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2470 | Regression_eta: 0.0024 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0967 | Classification_binary: 0.2183 | Regression_cos_phi: 0.1156 | Regression_energy: 0.1178 | Regression_eta: 0.0849 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0913 | LR: 8.58e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8288s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7101 | max_loss: 0.1763 | max_backward: 1.1845 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 0.8282 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6429.6 | uss_MB: 5536.9 | child_rss_MB: 5653.9 | allocated_MB: 339.8 | reserved_MB: 26286.0 | inactive_split_MB: 213.4 +rank6 [2026-10-28 20:24:07,450] INFO : Running validation on rank6 +rank6 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 20:24:58,458] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:24:59,079] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 20:25:24,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 20:25:25,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 20:25:26,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 20:25:27,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 20:25:37,774] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 20:25:38,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:25:38,650] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 20:25:54,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 20:25:55,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 20:25:56,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 20:25:57,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 20:26:07,660] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 20:26:08,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:26:09,008] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 20:26:24,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 20:26:25,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 20:26:25,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 20:26:26,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 20:26:36,485] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 20:30:47,340] INFO : Step 13100/50000 rank6 | Train Loss: 1.6168 | Train Loss Unweighted: 0.8649 | Train Loss Components: Auxiliary: 0.3213 | Cardinality: 0.1869 | Classification: 0.1467 | Classification_binary: 0.2696 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2210 | Regression_eta: 0.0027 | Regression_pt: 0.2221 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0873 | Classification_binary: 0.2547 | Regression_cos_phi: 0.1066 | Regression_energy: 0.1103 | Regression_eta: 0.0934 | Regression_pt: 0.1152 | Regression_sin_phi: 0.1374 | LR: 8.56e-04 | DataLoad Time: 0.0108s | Model Forward Time: 6.3685s | Phase Times: max_pre_step_barrier: 0.0337 | max_batch_to_device: 0.0136 | max_forward: 1.2842 | max_loss: 0.2583 | max_backward: 3.1281 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.9677 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7302.9 | uss_MB: 6406.9 | child_rss_MB: 5646.9 | allocated_MB: 341.8 | reserved_MB: 22940.0 | inactive_split_MB: 270.2 +rank6 [2026-10-28 20:34:56,292] INFO : Step 13200/50000 rank6 | Train Loss: 1.6996 | Train Loss Unweighted: 0.8855 | Train Loss Components: Auxiliary: 0.3386 | Cardinality: 0.2474 | Classification: 0.1410 | Classification_binary: 0.2769 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2299 | Regression_eta: 0.0027 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0871 | Classification_binary: 0.2910 | Regression_cos_phi: 0.1380 | Regression_energy: 0.1013 | Regression_eta: 0.0668 | Regression_pt: 0.1064 | Regression_sin_phi: 0.1094 | LR: 8.54e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1524s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0072 | max_forward: 0.7543 | max_loss: 0.1532 | max_backward: 1.3317 | max_optimizer_step: 0.3180 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7300.0 | uss_MB: 6407.2 | child_rss_MB: 5649.5 | allocated_MB: 343.4 | reserved_MB: 23942.0 | inactive_split_MB: 249.6 +rank6 [2026-10-28 20:38:53,363] INFO : Step 13300/50000 rank6 | Train Loss: 1.4876 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2970 | Cardinality: 0.2018 | Classification: 0.1100 | Classification_binary: 0.2192 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2205 | Regression_eta: 0.0026 | Regression_pt: 0.2230 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0751 | Classification_binary: 0.2288 | Regression_cos_phi: 0.1047 | Regression_energy: 0.1116 | Regression_eta: 0.0802 | Regression_pt: 0.1170 | Regression_sin_phi: 0.1178 | LR: 8.52e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.6362s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0259 | max_forward: 0.9208 | max_loss: 0.2245 | max_backward: 1.9523 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6473.0 | uss_MB: 4937.1 | child_rss_MB: 5535.0 | allocated_MB: 341.7 | reserved_MB: 24474.0 | inactive_split_MB: 265.4 +rank6 [2026-10-28 20:42:52,943] INFO : Step 13400/50000 rank6 | Train Loss: 1.4253 | Train Loss Unweighted: 0.7944 | Train Loss Components: Auxiliary: 0.2889 | Cardinality: 0.1585 | Classification: 0.1406 | Classification_binary: 0.2158 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2158 | Regression_eta: 0.0022 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1275 | Classification_binary: 0.2137 | Regression_cos_phi: 0.0908 | Regression_energy: 0.1219 | Regression_eta: 0.0718 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0892 | LR: 8.50e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7370s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6533 | max_loss: 0.2500 | max_backward: 1.4458 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.5454 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6474.0 | uss_MB: 5610.2 | child_rss_MB: 5538.3 | allocated_MB: 341.6 | reserved_MB: 25334.0 | inactive_split_MB: 266.6 +rank6 [2026-10-28 20:47:05,653] INFO : Step 13500/50000 rank6 | Train Loss: 1.6021 | Train Loss Unweighted: 0.8749 | Train Loss Components: Auxiliary: 0.3241 | Cardinality: 0.1832 | Classification: 0.1473 | Classification_binary: 0.2801 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2200 | Regression_eta: 0.0024 | Regression_pt: 0.2225 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0802 | Classification_binary: 0.2733 | Regression_cos_phi: 0.1240 | Regression_energy: 0.1100 | Regression_eta: 0.0637 | Regression_pt: 0.1138 | Regression_sin_phi: 0.1239 | LR: 8.48e-04 | DataLoad Time: 0.0166s | Model Forward Time: 6.1045s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.7859 | max_loss: 0.3303 | max_backward: 2.5111 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4974 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6412.7 | uss_MB: 5395.8 | child_rss_MB: 5343.5 | allocated_MB: 344.3 | reserved_MB: 25574.0 | inactive_split_MB: 284.1 +rank6 [2026-10-28 20:51:04,210] INFO : Step 13600/50000 rank6 | Train Loss: 1.5296 | Train Loss Unweighted: 0.8114 | Train Loss Components: Auxiliary: 0.3050 | Cardinality: 0.1872 | Classification: 0.1291 | Classification_binary: 0.2450 | HitGrouping: 0.2218 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2158 | Regression_eta: 0.0025 | Regression_pt: 0.2165 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.2790 | Regression_cos_phi: 0.1109 | Regression_energy: 0.1162 | Regression_eta: 0.0624 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0869 | LR: 8.45e-04 | DataLoad Time: 0.0221s | Model Forward Time: 3.6633s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.7508 | max_loss: 0.1507 | max_backward: 1.2891 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6404.7 | uss_MB: 5608.6 | child_rss_MB: 5283.3 | allocated_MB: 345.0 | reserved_MB: 26608.0 | inactive_split_MB: 257.3 +rank6 [2026-10-28 20:55:12,337] INFO : Step 13700/50000 rank6 | Train Loss: 1.4515 | Train Loss Unweighted: 0.7895 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1575 | Classification: 0.1261 | Classification_binary: 0.2121 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2219 | Regression_eta: 0.0023 | Regression_pt: 0.2245 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1077 | Classification_binary: 0.2280 | Regression_cos_phi: 0.1012 | Regression_energy: 0.0975 | Regression_eta: 0.0611 | Regression_pt: 0.1029 | Regression_sin_phi: 0.1183 | LR: 8.43e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.8533s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.6804 | max_loss: 0.2128 | max_backward: 1.1331 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6398.1 | uss_MB: 5609.6 | child_rss_MB: 5245.6 | allocated_MB: 342.5 | reserved_MB: 26608.0 | inactive_split_MB: 241.7 +rank6 [2026-10-28 20:59:19,100] INFO : Step 13800/50000 rank6 | Train Loss: 1.5060 | Train Loss Unweighted: 0.8320 | Train Loss Components: Auxiliary: 0.3084 | Cardinality: 0.1762 | Classification: 0.1368 | Classification_binary: 0.2567 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2156 | Regression_eta: 0.0023 | Regression_pt: 0.2184 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.2487 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0981 | Regression_eta: 0.0583 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0978 | LR: 8.41e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4124s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7967 | max_loss: 0.1491 | max_backward: 1.5976 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6392.1 | uss_MB: 5613.3 | child_rss_MB: 5216.1 | allocated_MB: 342.1 | reserved_MB: 26608.0 | inactive_split_MB: 241.3 +rank6 [2026-10-28 21:03:21,775] INFO : Step 13900/50000 rank6 | Train Loss: 1.6108 | Train Loss Unweighted: 0.8469 | Train Loss Components: Auxiliary: 0.3204 | Cardinality: 0.1942 | Classification: 0.1303 | Classification_binary: 0.2751 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2168 | Regression_eta: 0.0028 | Regression_pt: 0.2192 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0948 | Classification_binary: 0.2634 | Regression_cos_phi: 0.0772 | Regression_energy: 0.1094 | Regression_eta: 0.0697 | Regression_pt: 0.1156 | Regression_sin_phi: 0.1071 | LR: 8.39e-04 | DataLoad Time: 0.0188s | Model Forward Time: 6.7332s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8010 | max_loss: 0.1552 | max_backward: 2.5526 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 1.2862 | max_tensorboard_flush: 0.0041 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 6389.1 | uss_MB: 5603.5 | child_rss_MB: 5217.8 | allocated_MB: 345.6 | reserved_MB: 26608.0 | inactive_split_MB: 292.8 +rank6 [2026-10-28 21:07:29,378] INFO : Step 14000/50000 rank6 | Train Loss: 1.3659 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1707 | Classification: 0.0974 | Classification_binary: 0.1997 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2149 | Regression_eta: 0.0023 | Regression_pt: 0.2181 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2177 | Regression_cos_phi: 0.1033 | Regression_energy: 0.1011 | Regression_eta: 0.0926 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0807 | LR: 8.37e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.4502s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7574 | max_loss: 0.1908 | max_backward: 1.5870 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6393.2 | uss_MB: 4632.1 | child_rss_MB: 5217.4 | allocated_MB: 342.3 | reserved_MB: 26608.0 | inactive_split_MB: 238.4 +rank6 [2026-10-28 21:07:30,361] INFO : Running validation on rank6 +rank6 [2026-10-28 21:07:30,361] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 21:07:30,361] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 21:08:21,199] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:08:21,764] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 21:08:49,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 21:08:50,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 21:08:51,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 21:08:52,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 21:09:02,615] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 21:09:05,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:09:06,443] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 21:09:22,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 21:09:23,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 21:09:24,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 21:09:25,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 21:09:35,349] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 21:09:36,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:09:36,854] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 21:09:52,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 21:09:52,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 21:09:53,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 21:09:54,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 21:10:04,108] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 21:14:16,750] INFO : Step 14100/50000 rank6 | Train Loss: 1.5416 | Train Loss Unweighted: 0.8176 | Train Loss Components: Auxiliary: 0.3012 | Cardinality: 0.1751 | Classification: 0.1363 | Classification_binary: 0.2297 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2177 | Regression_eta: 0.0028 | Regression_pt: 0.2283 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.2028 | Regression_cos_phi: 0.1011 | Regression_energy: 0.1074 | Regression_eta: 0.0681 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0996 | LR: 8.35e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.2092s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7718 | max_loss: 0.1710 | max_backward: 1.6376 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 7412.7 | uss_MB: 6170.5 | child_rss_MB: 5197.6 | allocated_MB: 347.1 | reserved_MB: 23970.0 | inactive_split_MB: 244.9 +rank6 [2026-10-28 21:18:17,876] INFO : Step 14200/50000 rank6 | Train Loss: 1.6164 | Train Loss Unweighted: 0.8542 | Train Loss Components: Auxiliary: 0.3190 | Cardinality: 0.1950 | Classification: 0.1478 | Classification_binary: 0.2794 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2094 | Regression_eta: 0.0027 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.2470 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0945 | Regression_eta: 0.0626 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0873 | LR: 8.32e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.3712s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1153 | max_forward: 0.8788 | max_loss: 0.1615 | max_backward: 1.5286 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7413.4 | uss_MB: 6634.1 | child_rss_MB: 5195.0 | allocated_MB: 344.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.9 +rank6 [2026-10-28 21:22:21,673] INFO : Step 14300/50000 rank6 | Train Loss: 1.4246 | Train Loss Unweighted: 0.7744 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1685 | Classification: 0.1198 | Classification_binary: 0.2357 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2084 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2645 | Regression_cos_phi: 0.1238 | Regression_energy: 0.0950 | Regression_eta: 0.0669 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0799 | LR: 8.30e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.8791s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7183 | max_loss: 0.1494 | max_backward: 1.3371 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6454.4 | uss_MB: 5677.1 | child_rss_MB: 5184.2 | allocated_MB: 341.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.1 +rank6 [2026-10-28 21:26:25,336] INFO : Step 14400/50000 rank6 | Train Loss: 1.3539 | Train Loss Unweighted: 0.7267 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1402 | Classification: 0.1303 | Classification_binary: 0.2015 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1957 | Regression_eta: 0.0022 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.2203 | Regression_cos_phi: 0.0901 | Regression_energy: 0.0965 | Regression_eta: 0.0681 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0901 | LR: 8.28e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.6674s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6350 | max_loss: 0.2071 | max_backward: 1.2597 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6458.7 | uss_MB: 5681.9 | child_rss_MB: 5188.8 | allocated_MB: 346.1 | reserved_MB: 24962.0 | inactive_split_MB: 231.6 +rank6 [2026-10-28 21:30:23,756] INFO : Step 14500/50000 rank6 | Train Loss: 1.5651 | Train Loss Unweighted: 0.7967 | Train Loss Components: Auxiliary: 0.3070 | Cardinality: 0.2312 | Classification: 0.1364 | Classification_binary: 0.2565 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1984 | Regression_eta: 0.0026 | Regression_pt: 0.2005 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2936 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0936 | Regression_eta: 0.0728 | Regression_pt: 0.0984 | Regression_sin_phi: 0.1277 | LR: 8.26e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.4654s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0069 | max_forward: 0.8745 | max_loss: 0.1717 | max_backward: 1.8485 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 1.4922 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6443.5 | uss_MB: 5682.8 | child_rss_MB: 5113.1 | allocated_MB: 344.2 | reserved_MB: 25374.0 | inactive_split_MB: 206.9 +rank6 [2026-10-28 21:34:34,947] INFO : Step 14600/50000 rank6 | Train Loss: 1.6485 | Train Loss Unweighted: 0.8413 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.2375 | Classification: 0.1249 | Classification_binary: 0.2902 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2104 | Regression_eta: 0.0023 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0874 | Classification_binary: 0.3461 | Regression_cos_phi: 0.1167 | Regression_energy: 0.1023 | Regression_eta: 0.0735 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0914 | LR: 8.24e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.3664s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0068 | max_forward: 0.7866 | max_loss: 0.1515 | max_backward: 1.5469 | max_optimizer_step: 0.3128 | max_tensorboard_logging: 0.7882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6427.1 | uss_MB: 5683.8 | child_rss_MB: 5049.1 | allocated_MB: 345.7 | reserved_MB: 25374.0 | inactive_split_MB: 292.3 +rank6 [2026-10-28 21:38:39,705] INFO : Step 14700/50000 rank6 | Train Loss: 1.3645 | Train Loss Unweighted: 0.7326 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1507 | Classification: 0.1080 | Classification_binary: 0.2034 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0679 | Regression_energy: 0.1271 | Regression_eta: 0.0565 | Regression_pt: 0.1352 | Regression_sin_phi: 0.1141 | LR: 8.21e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9109s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6571 | max_loss: 0.1494 | max_backward: 1.2978 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7751 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0922 | Memory: rss_MB: 6423.6 | uss_MB: 5684.9 | child_rss_MB: 5040.9 | allocated_MB: 341.1 | reserved_MB: 25374.0 | inactive_split_MB: 264.9 +rank6 [2026-10-28 21:42:53,742] INFO : Step 14800/50000 rank6 | Train Loss: 1.3987 | Train Loss Unweighted: 0.7335 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1827 | Classification: 0.1136 | Classification_binary: 0.2244 | HitGrouping: 0.2204 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1949 | Regression_eta: 0.0022 | Regression_pt: 0.1963 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0953 | Regression_eta: 0.0553 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0578 | LR: 8.19e-04 | DataLoad Time: 0.0227s | Model Forward Time: 5.2137s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.1679 | max_loss: 0.2171 | max_backward: 2.8051 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 6420.4 | uss_MB: 4434.9 | child_rss_MB: 5026.4 | allocated_MB: 345.2 | reserved_MB: 25374.0 | inactive_split_MB: 222.5 +rank6 [2026-10-28 21:47:06,021] INFO : Step 14900/50000 rank6 | Train Loss: 1.5046 | Train Loss Unweighted: 0.7991 | Train Loss Components: Auxiliary: 0.3055 | Cardinality: 0.1834 | Classification: 0.1311 | Classification_binary: 0.2643 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1982 | Regression_eta: 0.0023 | Regression_pt: 0.2008 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2547 | Regression_cos_phi: 0.1068 | Regression_energy: 0.0880 | Regression_eta: 0.0645 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0819 | LR: 8.17e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.0334s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0071 | max_forward: 0.7642 | max_loss: 0.1561 | max_backward: 1.5975 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6421.6 | uss_MB: 5685.9 | child_rss_MB: 5030.2 | allocated_MB: 345.4 | reserved_MB: 25374.0 | inactive_split_MB: 289.7 +rank6 [2026-10-28 21:51:16,625] INFO : Step 15000/50000 rank6 | Train Loss: 1.3836 | Train Loss Unweighted: 0.7341 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1614 | Classification: 0.1182 | Classification_binary: 0.2252 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1930 | Regression_eta: 0.0023 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.2493 | Regression_cos_phi: 0.1175 | Regression_energy: 0.0960 | Regression_eta: 0.0697 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0908 | LR: 8.15e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.7142s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0066 | max_forward: 0.6828 | max_loss: 0.1453 | max_backward: 1.3064 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1064 | Memory: rss_MB: 6423.8 | uss_MB: 5686.3 | child_rss_MB: 5021.1 | allocated_MB: 342.6 | reserved_MB: 25374.0 | inactive_split_MB: 262.7 +rank6 [2026-10-28 21:51:17,569] INFO : Running validation on rank6 +rank6 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 21:52:06,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:52:07,025] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 21:52:33,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 21:52:34,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 21:52:35,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 21:52:36,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 21:52:46,265] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 21:52:46,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:52:47,271] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 21:53:03,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 21:53:04,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 21:53:05,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 21:53:06,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 21:53:16,384] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 21:53:16,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:53:17,343] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 21:53:32,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 21:53:33,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 21:53:33,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 21:53:34,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 21:53:44,625] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 21:57:58,525] INFO : Step 15100/50000 rank6 | Train Loss: 1.3706 | Train Loss Unweighted: 0.7159 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1535 | Classification: 0.1271 | Classification_binary: 0.1954 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1940 | Regression_eta: 0.0022 | Regression_pt: 0.1948 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0932 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0808 | Regression_eta: 0.0711 | Regression_pt: 0.0816 | Regression_sin_phi: 0.1181 | LR: 8.12e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.0244s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.6658 | max_loss: 0.2401 | max_backward: 1.3105 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8277 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7654.5 | uss_MB: 6910.5 | child_rss_MB: 5047.2 | allocated_MB: 341.1 | reserved_MB: 22096.0 | inactive_split_MB: 215.5 +rank6 [2026-10-28 22:01:59,130] INFO : Step 15200/50000 rank6 | Train Loss: 1.5024 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3072 | Cardinality: 0.2080 | Classification: 0.1273 | Classification_binary: 0.2385 | HitGrouping: 0.2164 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1989 | Regression_eta: 0.0022 | Regression_pt: 0.2007 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.2251 | Regression_cos_phi: 0.1214 | Regression_energy: 0.0907 | Regression_eta: 0.0544 | Regression_pt: 0.0947 | Regression_sin_phi: 0.0903 | LR: 8.10e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.5435s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7502 | max_loss: 0.1534 | max_backward: 1.8002 | max_optimizer_step: 0.2270 | max_tensorboard_logging: 0.8682 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 7649.1 | uss_MB: 6910.3 | child_rss_MB: 5045.3 | allocated_MB: 345.6 | reserved_MB: 22420.0 | inactive_split_MB: 257.7 +rank6 [2026-10-28 22:05:42,232] INFO : Step 15300/50000 rank6 | Train Loss: 1.4920 | Train Loss Unweighted: 0.7880 | Train Loss Components: Auxiliary: 0.3065 | Cardinality: 0.1970 | Classification: 0.1161 | Classification_binary: 0.2589 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2026 | Regression_eta: 0.0022 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2525 | Regression_cos_phi: 0.1050 | Regression_energy: 0.0887 | Regression_eta: 0.0556 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0974 | LR: 8.08e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5779s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0192 | max_forward: 0.7642 | max_loss: 0.1530 | max_backward: 1.6292 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 0.9125 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6524.0 | uss_MB: 5797.3 | child_rss_MB: 5044.3 | allocated_MB: 341.2 | reserved_MB: 23096.0 | inactive_split_MB: 271.7 +rank6 [2026-10-28 22:09:28,150] INFO : Step 15400/50000 rank6 | Train Loss: 1.3549 | Train Loss Unweighted: 0.7210 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1572 | Classification: 0.1148 | Classification_binary: 0.2052 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1983 | Regression_eta: 0.0022 | Regression_pt: 0.1984 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0810 | Regression_eta: 0.0533 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0817 | LR: 8.05e-04 | DataLoad Time: 0.0220s | Model Forward Time: 3.7021s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6594 | max_loss: 0.2185 | max_backward: 1.2416 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6511.0 | uss_MB: 5790.6 | child_rss_MB: 5047.2 | allocated_MB: 340.8 | reserved_MB: 23394.0 | inactive_split_MB: 166.6 +rank6 [2026-10-28 22:13:12,489] INFO : Step 15500/50000 rank6 | Train Loss: 1.4313 | Train Loss Unweighted: 0.7527 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1666 | Classification: 0.1321 | Classification_binary: 0.2182 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1979 | Regression_eta: 0.0025 | Regression_pt: 0.1996 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0761 | Classification_binary: 0.1738 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0878 | Regression_eta: 0.0698 | Regression_pt: 0.0927 | Regression_sin_phi: 0.1069 | LR: 8.03e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2548s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0792 | max_forward: 0.7874 | max_loss: 0.1514 | max_backward: 1.8676 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5380 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6511.0 | uss_MB: 5805.3 | child_rss_MB: 5052.2 | allocated_MB: 345.6 | reserved_MB: 24092.0 | inactive_split_MB: 260.5 +rank6 [2026-10-28 22:17:01,248] INFO : Step 15600/50000 rank6 | Train Loss: 1.5305 | Train Loss Unweighted: 0.8109 | Train Loss Components: Auxiliary: 0.3137 | Cardinality: 0.1790 | Classification: 0.1304 | Classification_binary: 0.2721 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2011 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 0.2765 | Regression_cos_phi: 0.0889 | Regression_energy: 0.1041 | Regression_eta: 0.0656 | Regression_pt: 0.1092 | Regression_sin_phi: 0.1033 | LR: 8.01e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2094s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7645 | max_loss: 0.2230 | max_backward: 1.7015 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1398 | Memory: rss_MB: 6508.8 | uss_MB: 5803.4 | child_rss_MB: 5052.3 | allocated_MB: 344.1 | reserved_MB: 24402.0 | inactive_split_MB: 287.9 +rank6 [2026-10-28 22:20:46,545] INFO : Step 15700/50000 rank6 | Train Loss: 1.4083 | Train Loss Unweighted: 0.7205 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1775 | Classification: 0.1238 | Classification_binary: 0.2322 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1785 | Regression_eta: 0.0024 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0814 | Classification_binary: 0.2543 | Regression_cos_phi: 0.1067 | Regression_energy: 0.0802 | Regression_eta: 0.0766 | Regression_pt: 0.0845 | Regression_sin_phi: 0.1227 | LR: 7.98e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.0300s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2392 | max_backward: 1.6606 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6506.8 | uss_MB: 5802.2 | child_rss_MB: 5055.5 | allocated_MB: 344.4 | reserved_MB: 25380.0 | inactive_split_MB: 228.2 +rank6 [2026-10-28 22:24:39,044] INFO : Step 15800/50000 rank6 | Train Loss: 1.2988 | Train Loss Unweighted: 0.6968 | Train Loss Components: Auxiliary: 0.2671 | Cardinality: 0.1436 | Classification: 0.1112 | Classification_binary: 0.1916 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1933 | Regression_eta: 0.0020 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1991 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0856 | Regression_eta: 0.0541 | Regression_pt: 0.0868 | Regression_sin_phi: 0.1184 | LR: 7.96e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7266s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6109 | max_loss: 0.2015 | max_backward: 1.1291 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6506.5 | uss_MB: 5803.6 | child_rss_MB: 5051.3 | allocated_MB: 341.6 | reserved_MB: 25380.0 | inactive_split_MB: 263.9 +rank6 [2026-10-28 22:28:35,066] INFO : Step 15900/50000 rank6 | Train Loss: 1.4020 | Train Loss Unweighted: 0.7322 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1800 | Classification: 0.1295 | Classification_binary: 0.2346 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1816 | Regression_eta: 0.0023 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0886 | Regression_eta: 0.0339 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0663 | LR: 7.94e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2693s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0933 | max_forward: 0.7823 | max_loss: 0.1445 | max_backward: 1.4740 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6508.5 | uss_MB: 5807.0 | child_rss_MB: 5047.2 | allocated_MB: 342.9 | reserved_MB: 25380.0 | inactive_split_MB: 283.5 +rank6 [2026-10-28 22:32:40,675] INFO : Step 16000/50000 rank6 | Train Loss: 1.4978 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3126 | Cardinality: 0.2093 | Classification: 0.1190 | Classification_binary: 0.2580 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1920 | Regression_eta: 0.0021 | Regression_pt: 0.1969 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2633 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0816 | Regression_eta: 0.0544 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0854 | LR: 7.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4903s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.7474 | max_loss: 0.1892 | max_backward: 1.4114 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.7981 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6509.3 | uss_MB: 5675.5 | child_rss_MB: 5048.3 | allocated_MB: 342.7 | reserved_MB: 25380.0 | inactive_split_MB: 292.9 +rank6 [2026-10-28 22:32:41,713] INFO : Running validation on rank6 +rank6 [2026-10-28 22:32:41,713] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 22:32:41,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 22:33:34,622] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:33:35,151] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 22:34:01,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 22:34:02,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 22:34:03,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 22:34:04,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 22:34:14,165] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 22:34:14,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:34:14,938] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 22:34:30,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 22:34:31,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 22:34:32,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 22:34:33,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 22:34:43,848] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 22:34:44,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:34:44,852] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 22:35:00,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 22:35:00,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 22:35:01,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 22:35:02,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 22:35:12,098] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 22:39:22,135] INFO : Step 16100/50000 rank6 | Train Loss: 1.3441 | Train Loss Unweighted: 0.7192 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1631 | Classification: 0.1197 | Classification_binary: 0.2058 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1931 | Regression_eta: 0.0020 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.2042 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0913 | Regression_eta: 0.0459 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0923 | LR: 7.89e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1514s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0584 | max_forward: 0.7257 | max_loss: 0.1421 | max_backward: 1.2590 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.0197 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6577.1 | uss_MB: 5871.7 | child_rss_MB: 5053.6 | allocated_MB: 341.3 | reserved_MB: 23342.0 | inactive_split_MB: 221.6 +rank6 [2026-10-28 22:43:09,260] INFO : Step 16200/50000 rank6 | Train Loss: 1.3737 | Train Loss Unweighted: 0.7229 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1695 | Classification: 0.1213 | Classification_binary: 0.2167 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2005 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0819 | Regression_eta: 0.0535 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0703 | LR: 7.86e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0500s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.1062 | max_forward: 0.7366 | max_loss: 0.1469 | max_backward: 1.4999 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.6015 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6580.7 | uss_MB: 5882.6 | child_rss_MB: 5053.8 | allocated_MB: 342.5 | reserved_MB: 24304.0 | inactive_split_MB: 205.9 +rank6 [2026-10-28 22:46:54,662] INFO : Step 16300/50000 rank6 | Train Loss: 1.4671 | Train Loss Unweighted: 0.7592 | Train Loss Components: Auxiliary: 0.3008 | Cardinality: 0.1775 | Classification: 0.1342 | Classification_binary: 0.2702 | HitGrouping: 0.2148 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0023 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0824 | Classification_binary: 0.2733 | Regression_cos_phi: 0.0905 | Regression_energy: 0.0894 | Regression_eta: 0.0762 | Regression_pt: 0.0922 | Regression_sin_phi: 0.1022 | LR: 7.84e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4589s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7718 | max_loss: 0.1628 | max_backward: 1.7017 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.8513 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6574.7 | uss_MB: 5880.8 | child_rss_MB: 5053.1 | allocated_MB: 345.8 | reserved_MB: 24304.0 | inactive_split_MB: 281.0 +rank6 [2026-10-28 22:50:39,546] INFO : Step 16400/50000 rank6 | Train Loss: 1.4200 | Train Loss Unweighted: 0.7161 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.2086 | Classification: 0.1206 | Classification_binary: 0.2158 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0024 | Regression_pt: 0.1885 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2288 | Regression_cos_phi: 0.1154 | Regression_energy: 0.0935 | Regression_eta: 0.0615 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0779 | LR: 7.81e-04 | DataLoad Time: 0.0184s | Model Forward Time: 3.8039s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7570 | max_loss: 0.1546 | max_backward: 1.2568 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5199 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1436 | Memory: rss_MB: 6571.7 | uss_MB: 5879.8 | child_rss_MB: 5049.7 | allocated_MB: 342.7 | reserved_MB: 25268.0 | inactive_split_MB: 213.6 +rank6 [2026-10-28 22:54:25,074] INFO : Step 16500/50000 rank6 | Train Loss: 1.2201 | Train Loss Unweighted: 0.6592 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1195 | Classification: 0.1029 | Classification_binary: 0.1835 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1836 | Regression_eta: 0.0021 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1754 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0731 | Regression_eta: 0.0344 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0743 | LR: 7.79e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3165s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0128 | max_forward: 0.7866 | max_loss: 0.2200 | max_backward: 1.9590 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.4913 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6572.2 | uss_MB: 5885.7 | child_rss_MB: 5046.8 | allocated_MB: 343.6 | reserved_MB: 26238.0 | inactive_split_MB: 248.8 +rank6 [2026-10-28 22:58:25,288] INFO : Step 16600/50000 rank6 | Train Loss: 1.4335 | Train Loss Unweighted: 0.7657 | Train Loss Components: Auxiliary: 0.2949 | Cardinality: 0.1698 | Classification: 0.1386 | Classification_binary: 0.2312 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1941 | Regression_eta: 0.0021 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0981 | Classification_binary: 0.2158 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0817 | Regression_eta: 0.0773 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0596 | LR: 7.77e-04 | DataLoad Time: 0.0132s | Model Forward Time: 5.9896s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0139 | max_forward: 1.0329 | max_loss: 0.3279 | max_backward: 2.5489 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.5244 | max_tensorboard_flush: 0.0014 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6575.3 | uss_MB: 4925.2 | child_rss_MB: 5045.0 | allocated_MB: 344.2 | reserved_MB: 27228.0 | inactive_split_MB: 219.6 +rank6 [2026-10-28 23:02:38,763] INFO : Step 16700/50000 rank6 | Train Loss: 1.4062 | Train Loss Unweighted: 0.7378 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1840 | Classification: 0.1203 | Classification_binary: 0.2439 | HitGrouping: 0.2108 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1839 | Regression_eta: 0.0020 | Regression_pt: 0.1855 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.2700 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0824 | Regression_eta: 0.0688 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0892 | LR: 7.74e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0461s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7336 | max_loss: 0.2460 | max_backward: 1.6747 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6577.4 | uss_MB: 5888.0 | child_rss_MB: 5049.7 | allocated_MB: 344.6 | reserved_MB: 27228.0 | inactive_split_MB: 279.4 +rank6 [2026-10-28 23:06:47,050] INFO : Step 16800/50000 rank6 | Train Loss: 1.3172 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1533 | Classification: 0.1173 | Classification_binary: 0.1949 | HitGrouping: 0.2175 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1819 | Regression_eta: 0.0027 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.1922 | Regression_cos_phi: 0.1012 | Regression_energy: 0.0773 | Regression_eta: 0.0831 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0866 | LR: 7.72e-04 | DataLoad Time: 0.0207s | Model Forward Time: 3.6839s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0962 | max_forward: 0.7772 | max_loss: 0.1423 | max_backward: 1.3121 | max_optimizer_step: 0.2117 | max_tensorboard_logging: 0.4865 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6580.1 | uss_MB: 5886.7 | child_rss_MB: 5043.1 | allocated_MB: 342.1 | reserved_MB: 27250.0 | inactive_split_MB: 200.8 +rank6 [2026-10-28 23:11:05,115] INFO : Step 16900/50000 rank6 | Train Loss: 1.3132 | Train Loss Unweighted: 0.6868 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1474 | Classification: 0.1260 | Classification_binary: 0.1988 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1779 | Regression_eta: 0.0021 | Regression_pt: 0.1798 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0918 | Regression_eta: 0.0465 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0759 | LR: 7.69e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.7600s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7203 | max_loss: 0.1660 | max_backward: 1.1458 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6581.3 | uss_MB: 5889.6 | child_rss_MB: 5044.4 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 193.9 +rank6 [2026-10-28 23:15:09,202] INFO : Step 17000/50000 rank6 | Train Loss: 1.3575 | Train Loss Unweighted: 0.7115 | Train Loss Components: Auxiliary: 0.2906 | Cardinality: 0.1781 | Classification: 0.1179 | Classification_binary: 0.2325 | HitGrouping: 0.2089 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0018 | Regression_pt: 0.1792 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.2391 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0847 | Regression_eta: 0.0553 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0787 | LR: 7.67e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4161s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.9008 | max_loss: 0.1474 | max_backward: 1.9229 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6575.6 | uss_MB: 5894.0 | child_rss_MB: 5025.6 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 278.9 +rank6 [2026-10-28 23:15:10,041] INFO : Running validation on rank6 +rank6 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 23:16:00,503] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:16:01,060] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 23:16:27,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 23:16:28,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 23:16:29,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 23:16:31,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 23:16:41,235] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 23:16:42,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:16:42,521] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 23:16:58,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 23:16:59,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 23:17:00,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 23:17:01,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 23:17:11,246] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 23:17:11,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:17:12,024] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 23:17:27,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 23:17:28,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 23:17:28,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 23:17:29,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 23:17:39,533] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-28 23:21:59,514] INFO : Step 17100/50000 rank6 | Train Loss: 1.3790 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1704 | Classification: 0.1078 | Classification_binary: 0.2095 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1968 | Regression_eta: 0.0020 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.2361 | Regression_cos_phi: 0.1166 | Regression_energy: 0.1013 | Regression_eta: 0.0618 | Regression_pt: 0.1062 | Regression_sin_phi: 0.1181 | LR: 7.64e-04 | DataLoad Time: 0.0138s | Model Forward Time: 4.9105s | Phase Times: max_pre_step_barrier: 0.0234 | max_batch_to_device: 0.0065 | max_forward: 0.8292 | max_loss: 0.1455 | max_backward: 1.4833 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6666.8 | uss_MB: 5307.9 | child_rss_MB: 5009.6 | allocated_MB: 343.8 | reserved_MB: 22688.0 | inactive_split_MB: 254.6 +rank6 [2026-10-28 23:26:06,862] INFO : Step 17200/50000 rank6 | Train Loss: 1.2786 | Train Loss Unweighted: 0.6598 | Train Loss Components: Auxiliary: 0.2576 | Cardinality: 0.1378 | Classification: 0.1161 | Classification_binary: 0.1910 | HitGrouping: 0.2191 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0025 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1991 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0802 | Regression_eta: 0.1109 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0620 | LR: 7.62e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.8615s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6470 | max_loss: 0.2505 | max_backward: 1.4380 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6320 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6664.1 | uss_MB: 5982.7 | child_rss_MB: 5028.0 | allocated_MB: 344.1 | reserved_MB: 22688.0 | inactive_split_MB: 295.9 +rank6 [2026-10-28 23:30:05,514] INFO : Step 17300/50000 rank6 | Train Loss: 1.3571 | Train Loss Unweighted: 0.6976 | Train Loss Components: Auxiliary: 0.2789 | Cardinality: 0.1665 | Classification: 0.1193 | Classification_binary: 0.2287 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1724 | Regression_eta: 0.0021 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.2213 | Regression_cos_phi: 0.0908 | Regression_energy: 0.0787 | Regression_eta: 0.0467 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0709 | LR: 7.59e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5412s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7276 | max_loss: 0.2520 | max_backward: 1.8579 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8525 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6666.1 | uss_MB: 5121.5 | child_rss_MB: 5026.9 | allocated_MB: 344.2 | reserved_MB: 23616.0 | inactive_split_MB: 266.5 +rank6 [2026-10-28 23:34:25,424] INFO : Step 17400/50000 rank6 | Train Loss: 1.3668 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2811 | Cardinality: 0.1760 | Classification: 0.1195 | Classification_binary: 0.2351 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1732 | Regression_eta: 0.0019 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.2414 | Regression_cos_phi: 0.1267 | Regression_energy: 0.1038 | Regression_eta: 0.0839 | Regression_pt: 0.1084 | Regression_sin_phi: 0.1488 | LR: 7.57e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4833s | Phase Times: max_pre_step_barrier: 0.1038 | max_batch_to_device: 0.0306 | max_forward: 0.7782 | max_loss: 0.2429 | max_backward: 1.3743 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.8348 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0757 | Memory: rss_MB: 6667.1 | uss_MB: 5984.6 | child_rss_MB: 5026.4 | allocated_MB: 343.3 | reserved_MB: 24604.0 | inactive_split_MB: 262.4 +rank6 [2026-10-28 23:38:51,400] INFO : Step 17500/50000 rank6 | Train Loss: 1.3114 | Train Loss Unweighted: 0.6526 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.1481 | Classification: 0.1145 | Classification_binary: 0.1884 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1714 | Regression_eta: 0.0024 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0875 | Classification_binary: 0.2082 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0815 | Regression_eta: 0.0738 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0878 | LR: 7.54e-04 | DataLoad Time: 0.0228s | Model Forward Time: 3.9130s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.6394 | max_loss: 0.1427 | max_backward: 1.2079 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.8375 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 6667.1 | uss_MB: 5986.2 | child_rss_MB: 5023.0 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 240.0 +rank6 [2026-10-28 23:43:03,144] INFO : Step 17600/50000 rank6 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2688 | Cardinality: 0.1362 | Classification: 0.1273 | Classification_binary: 0.2065 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0019 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2188 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0745 | Regression_eta: 0.0580 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0869 | LR: 7.52e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6054s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.1120 | max_forward: 1.2883 | max_loss: 0.2624 | max_backward: 1.5380 | max_optimizer_step: 0.2147 | max_tensorboard_logging: 0.7644 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 6665.9 | uss_MB: 5985.5 | child_rss_MB: 5028.5 | allocated_MB: 344.2 | reserved_MB: 25596.0 | inactive_split_MB: 237.9 +rank6 [2026-10-28 23:47:02,148] INFO : Step 17700/50000 rank6 | Train Loss: 1.3491 | Train Loss Unweighted: 0.6939 | Train Loss Components: Auxiliary: 0.2835 | Cardinality: 0.1622 | Classification: 0.1186 | Classification_binary: 0.2310 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0022 | Regression_pt: 0.1701 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.2287 | Regression_cos_phi: 0.0911 | Regression_energy: 0.0833 | Regression_eta: 0.0598 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0708 | LR: 7.49e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6527s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0704 | max_forward: 0.9841 | max_loss: 0.1994 | max_backward: 1.8911 | max_optimizer_step: 0.2227 | max_tensorboard_logging: 0.5849 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6667.9 | uss_MB: 5989.0 | child_rss_MB: 5031.2 | allocated_MB: 345.5 | reserved_MB: 25596.0 | inactive_split_MB: 287.6 +rank6 [2026-10-28 23:51:17,533] INFO : Step 17800/50000 rank6 | Train Loss: 1.3063 | Train Loss Unweighted: 0.6692 | Train Loss Components: Auxiliary: 0.2664 | Cardinality: 0.1579 | Classification: 0.1042 | Classification_binary: 0.2130 | HitGrouping: 0.2150 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1741 | Regression_eta: 0.0019 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2130 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0837 | Regression_eta: 0.0480 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0904 | LR: 7.47e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8683s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.7351 | max_loss: 0.1686 | max_backward: 1.4616 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5023 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 6666.0 | uss_MB: 5988.8 | child_rss_MB: 5032.9 | allocated_MB: 341.0 | reserved_MB: 25596.0 | inactive_split_MB: 197.8 +rank6 [2026-10-28 23:55:22,790] INFO : Step 17900/50000 rank6 | Train Loss: 1.2213 | Train Loss Unweighted: 0.6433 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1427 | Classification: 0.1147 | Classification_binary: 0.1725 | HitGrouping: 0.2070 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0921 | Regression_eta: 0.0437 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0450 | LR: 7.44e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8169s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7249 | max_loss: 0.1579 | max_backward: 1.1981 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6654.3 | uss_MB: 5990.9 | child_rss_MB: 5033.3 | allocated_MB: 341.6 | reserved_MB: 25596.0 | inactive_split_MB: 297.9 +rank6 [2026-10-28 23:59:20,282] INFO : Step 18000/50000 rank6 | Train Loss: 1.3597 | Train Loss Unweighted: 0.7083 | Train Loss Components: Auxiliary: 0.2781 | Cardinality: 0.1599 | Classification: 0.1247 | Classification_binary: 0.2275 | HitGrouping: 0.2122 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0021 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.1717 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0905 | Regression_eta: 0.0442 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0556 | LR: 7.42e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6422s | Phase Times: max_pre_step_barrier: 0.0816 | max_batch_to_device: 0.0068 | max_forward: 0.8980 | max_loss: 0.1586 | max_backward: 1.9185 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.7546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6654.3 | uss_MB: 5995.0 | child_rss_MB: 5035.3 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 254.8 +rank6 [2026-10-28 23:59:21,239] INFO : Running validation on rank6 +rank6 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 00:00:12,210] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:00:12,739] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 00:00:39,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 00:00:40,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 00:00:41,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 00:00:42,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 00:00:52,368] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 00:00:52,803] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:00:53,359] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 00:01:09,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 00:01:10,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 00:01:11,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 00:01:12,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 00:01:22,253] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 00:01:22,746] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:01:23,290] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 00:01:38,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 00:01:39,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 00:01:39,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 00:01:40,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 00:01:50,648] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 00:05:57,253] INFO : Step 18100/50000 rank6 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7216 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1666 | Classification: 0.1199 | Classification_binary: 0.2340 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1807 | Regression_eta: 0.0024 | Regression_pt: 0.1824 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0874 | Classification_binary: 0.2760 | Regression_cos_phi: 0.1087 | Regression_energy: 0.0751 | Regression_eta: 0.0937 | Regression_pt: 0.0792 | Regression_sin_phi: 0.1345 | LR: 7.39e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.5277s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.9127 | max_loss: 0.4546 | max_backward: 1.7406 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.7463 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7789.9 | uss_MB: 7130.3 | child_rss_MB: 5031.5 | allocated_MB: 344.6 | reserved_MB: 22256.0 | inactive_split_MB: 291.4 +rank6 [2026-10-29 00:09:47,528] INFO : Step 18200/50000 rank6 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6468 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1359 | Classification: 0.1208 | Classification_binary: 0.1803 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1700 | Regression_eta: 0.0020 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1727 | Regression_cos_phi: 0.0833 | Regression_energy: 0.0948 | Regression_eta: 0.0410 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0700 | LR: 7.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7595s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7114 | max_loss: 0.2369 | max_backward: 1.1051 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6784.1 | uss_MB: 6119.0 | child_rss_MB: 5031.9 | allocated_MB: 341.2 | reserved_MB: 23772.0 | inactive_split_MB: 170.7 +rank6 [2026-10-29 00:13:32,738] INFO : Step 18300/50000 rank6 | Train Loss: 1.3100 | Train Loss Unweighted: 0.6891 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1522 | Classification: 0.1230 | Classification_binary: 0.1938 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1834 | Regression_eta: 0.0019 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.1830 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0738 | Regression_eta: 0.0457 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0993 | LR: 7.34e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1459s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7219 | max_loss: 0.1491 | max_backward: 1.5825 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.7390 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 6786.1 | uss_MB: 6124.1 | child_rss_MB: 5031.6 | allocated_MB: 344.4 | reserved_MB: 24778.0 | inactive_split_MB: 234.4 +rank6 [2026-10-29 00:17:16,846] INFO : Step 18400/50000 rank6 | Train Loss: 1.3667 | Train Loss Unweighted: 0.7337 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1448 | Classification: 0.1277 | Classification_binary: 0.2393 | HitGrouping: 0.2095 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1805 | Regression_eta: 0.0022 | Regression_pt: 0.1818 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2286 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0829 | Regression_eta: 0.0505 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0742 | LR: 7.31e-04 | DataLoad Time: 0.0138s | Model Forward Time: 4.4878s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1744 | max_backward: 1.7308 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6782.0 | uss_MB: 6123.4 | child_rss_MB: 5027.3 | allocated_MB: 344.6 | reserved_MB: 24778.0 | inactive_split_MB: 281.4 +rank6 [2026-10-29 00:21:03,775] INFO : Step 18500/50000 rank6 | Train Loss: 1.2528 | Train Loss Unweighted: 0.6816 | Train Loss Components: Auxiliary: 0.2631 | Cardinality: 0.1350 | Classification: 0.1140 | Classification_binary: 0.1973 | HitGrouping: 0.2066 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1821 | Regression_eta: 0.0017 | Regression_pt: 0.1847 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1921 | Regression_cos_phi: 0.0883 | Regression_energy: 0.0900 | Regression_eta: 0.0561 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0924 | LR: 7.29e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.2631s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0113 | max_forward: 0.8281 | max_loss: 0.2418 | max_backward: 1.4683 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7561 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1431 | Memory: rss_MB: 6783.0 | uss_MB: 6122.8 | child_rss_MB: 5026.7 | allocated_MB: 343.3 | reserved_MB: 24778.0 | inactive_split_MB: 226.5 +rank6 [2026-10-29 00:24:44,690] INFO : Step 18600/50000 rank6 | Train Loss: 1.1881 | Train Loss Unweighted: 0.6335 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1331 | Classification: 0.1084 | Classification_binary: 0.1723 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1746 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0802 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0868 | Regression_eta: 0.0388 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0496 | LR: 7.26e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8509s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7299 | max_loss: 0.2279 | max_backward: 1.0799 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8080 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6782.4 | uss_MB: 6120.8 | child_rss_MB: 5022.4 | allocated_MB: 341.0 | reserved_MB: 24778.0 | inactive_split_MB: 229.3 +rank6 [2026-10-29 00:28:24,927] INFO : Step 18700/50000 rank6 | Train Loss: 1.3122 | Train Loss Unweighted: 0.6534 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1756 | Classification: 0.1225 | Classification_binary: 0.2098 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1571 | Regression_eta: 0.0020 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.1939 | Regression_cos_phi: 0.1019 | Regression_energy: 0.0738 | Regression_eta: 0.0542 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0773 | LR: 7.24e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.6245s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9129 | max_loss: 0.1606 | max_backward: 1.9397 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6785.3 | uss_MB: 6128.9 | child_rss_MB: 5021.8 | allocated_MB: 344.8 | reserved_MB: 24778.0 | inactive_split_MB: 239.8 +rank6 [2026-10-29 00:32:05,524] INFO : Step 18800/50000 rank6 | Train Loss: 1.2696 | Train Loss Unweighted: 0.6643 | Train Loss Components: Auxiliary: 0.2674 | Cardinality: 0.1602 | Classification: 0.1070 | Classification_binary: 0.2200 | HitGrouping: 0.2058 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1652 | Regression_eta: 0.0018 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2180 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0775 | Regression_eta: 0.0299 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0435 | LR: 7.21e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6462s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7444 | max_loss: 0.2527 | max_backward: 1.7167 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.9324 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 6785.1 | uss_MB: 6124.1 | child_rss_MB: 5021.8 | allocated_MB: 345.4 | reserved_MB: 25044.0 | inactive_split_MB: 284.4 +rank6 [2026-10-29 00:35:49,544] INFO : Step 18900/50000 rank6 | Train Loss: 1.1718 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1382 | Classification: 0.0996 | Classification_binary: 0.1741 | HitGrouping: 0.2036 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1652 | Regression_eta: 0.0017 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2049 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0937 | Regression_eta: 0.0449 | Regression_pt: 0.0976 | Regression_sin_phi: 0.0594 | LR: 7.18e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0758s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0056 | max_forward: 0.6830 | max_loss: 0.1421 | max_backward: 1.2812 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.8931 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6786.1 | uss_MB: 6126.4 | child_rss_MB: 5006.2 | allocated_MB: 338.7 | reserved_MB: 25044.0 | inactive_split_MB: 232.6 +rank6 [2026-10-29 00:39:31,232] INFO : Step 19000/50000 rank6 | Train Loss: 1.2397 | Train Loss Unweighted: 0.6317 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1277 | Classification: 0.1188 | Classification_binary: 0.1866 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1600 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.1791 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0852 | Regression_eta: 0.0733 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0988 | LR: 7.16e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2159s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6593 | max_loss: 0.2467 | max_backward: 1.6203 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7732 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6787.7 | uss_MB: 6128.3 | child_rss_MB: 5026.8 | allocated_MB: 342.2 | reserved_MB: 25790.0 | inactive_split_MB: 229.6 +rank6 [2026-10-29 00:39:32,174] INFO : Running validation on rank6 +rank6 [2026-10-29 00:39:32,175] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 00:39:32,175] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 00:40:23,808] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:40:24,368] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 00:40:50,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 00:40:51,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 00:40:52,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 00:40:53,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 00:41:03,111] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 00:41:03,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:41:04,088] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 00:41:20,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 00:41:20,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 00:41:21,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 00:41:22,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 00:41:32,884] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 00:41:33,292] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:41:33,790] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 00:41:49,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 00:41:49,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 00:41:50,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 00:41:51,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 00:42:01,260] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 00:46:01,995] INFO : Step 19100/50000 rank6 | Train Loss: 1.3562 | Train Loss Unweighted: 0.6933 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1644 | Classification: 0.1204 | Classification_binary: 0.2426 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1627 | Regression_eta: 0.0023 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0802 | Classification_binary: 0.2551 | Regression_cos_phi: 0.1017 | Regression_energy: 0.0786 | Regression_eta: 0.1260 | Regression_pt: 0.0794 | Regression_sin_phi: 0.1444 | LR: 7.13e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.5905s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7796 | max_loss: 0.1634 | max_backward: 1.6942 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.8232 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 8027.6 | uss_MB: 7365.9 | child_rss_MB: 5027.2 | allocated_MB: 345.2 | reserved_MB: 23900.0 | inactive_split_MB: 240.8 +rank6 [2026-10-29 00:49:48,721] INFO : Step 19200/50000 rank6 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5946 | Train Loss Components: Auxiliary: 0.2384 | Cardinality: 0.1391 | Classification: 0.0965 | Classification_binary: 0.1984 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1472 | Regression_eta: 0.0018 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2158 | Regression_cos_phi: 0.0868 | Regression_energy: 0.0825 | Regression_eta: 0.0454 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0791 | LR: 7.10e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.9566s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7138 | max_loss: 0.2391 | max_backward: 1.2379 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.7395 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6876.1 | uss_MB: 6216.1 | child_rss_MB: 5025.0 | allocated_MB: 341.3 | reserved_MB: 23900.0 | inactive_split_MB: 206.8 +rank6 [2026-10-29 00:53:30,059] INFO : Step 19300/50000 rank6 | Train Loss: 1.1768 | Train Loss Unweighted: 0.6083 | Train Loss Components: Auxiliary: 0.2422 | Cardinality: 0.1345 | Classification: 0.1035 | Classification_binary: 0.1709 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0018 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1559 | Regression_cos_phi: 0.1147 | Regression_energy: 0.0948 | Regression_eta: 0.0500 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0500 | LR: 7.08e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.8975s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0503 | max_forward: 0.7163 | max_loss: 0.1442 | max_backward: 1.3745 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6878.1 | uss_MB: 6217.4 | child_rss_MB: 5020.2 | allocated_MB: 341.0 | reserved_MB: 23900.0 | inactive_split_MB: 218.1 +rank6 [2026-10-29 00:57:11,120] INFO : Step 19400/50000 rank6 | Train Loss: 1.4590 | Train Loss Unweighted: 0.7109 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.2328 | Classification: 0.1237 | Classification_binary: 0.2265 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1770 | Regression_eta: 0.0021 | Regression_pt: 0.1794 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.2030 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0873 | Regression_eta: 0.0440 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0771 | LR: 7.05e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.4229s | Phase Times: max_pre_step_barrier: 0.1118 | max_batch_to_device: 0.0070 | max_forward: 0.7578 | max_loss: 0.2459 | max_backward: 1.7975 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6876.3 | uss_MB: 6217.1 | child_rss_MB: 5016.1 | allocated_MB: 347.7 | reserved_MB: 24186.0 | inactive_split_MB: 276.3 +rank6 [2026-10-29 01:00:48,549] INFO : Step 19500/50000 rank6 | Train Loss: 1.3753 | Train Loss Unweighted: 0.6794 | Train Loss Components: Auxiliary: 0.2810 | Cardinality: 0.1865 | Classification: 0.1153 | Classification_binary: 0.2310 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1637 | Regression_eta: 0.0023 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.2318 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0715 | Regression_eta: 0.0536 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0782 | LR: 7.02e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8982s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7197 | max_loss: 0.2164 | max_backward: 1.3937 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.4987 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6876.3 | uss_MB: 6219.5 | child_rss_MB: 5017.7 | allocated_MB: 345.2 | reserved_MB: 24186.0 | inactive_split_MB: 275.2 +rank6 [2026-10-29 01:04:42,200] INFO : Step 19600/50000 rank6 | Train Loss: 1.2086 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1557 | Classification: 0.0942 | Classification_binary: 0.1768 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1735 | Regression_eta: 0.0015 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.2395 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0843 | Regression_eta: 0.0526 | Regression_pt: 0.0881 | Regression_sin_phi: 0.0782 | LR: 7.00e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7336s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6937 | max_loss: 0.1942 | max_backward: 1.2727 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6877.2 | uss_MB: 6219.0 | child_rss_MB: 5004.9 | allocated_MB: 340.6 | reserved_MB: 24186.0 | inactive_split_MB: 277.9 +rank6 [2026-10-29 01:08:29,168] INFO : Step 19700/50000 rank6 | Train Loss: 1.2115 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1478 | Classification: 0.1094 | Classification_binary: 0.1919 | HitGrouping: 0.2027 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0848 | Regression_eta: 0.0385 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0634 | LR: 6.97e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8668s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0914 | max_forward: 0.7091 | max_loss: 0.1429 | max_backward: 1.4681 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0652 | Memory: rss_MB: 6879.2 | uss_MB: 6222.3 | child_rss_MB: 5016.1 | allocated_MB: 344.0 | reserved_MB: 24630.0 | inactive_split_MB: 143.8 +rank6 [2026-10-29 01:12:36,831] INFO : Step 19800/50000 rank6 | Train Loss: 1.2919 | Train Loss Unweighted: 0.6584 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1453 | Classification: 0.1132 | Classification_binary: 0.2375 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0020 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.1965 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0666 | Regression_eta: 0.0495 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0732 | LR: 6.94e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.4508s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0074 | max_forward: 0.7828 | max_loss: 0.2508 | max_backward: 1.8604 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6879.1 | uss_MB: 6223.6 | child_rss_MB: 5015.7 | allocated_MB: 348.0 | reserved_MB: 25634.0 | inactive_split_MB: 277.9 +rank6 [2026-10-29 01:16:27,498] INFO : Step 19900/50000 rank6 | Train Loss: 1.2010 | Train Loss Unweighted: 0.6167 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1346 | Classification: 0.0885 | Classification_binary: 0.1961 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1633 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0673 | Regression_eta: 0.0406 | Regression_pt: 0.0717 | Regression_sin_phi: 0.1098 | LR: 6.92e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.9730s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6671 | max_loss: 0.2548 | max_backward: 1.5862 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6881.1 | uss_MB: 6224.6 | child_rss_MB: 5016.9 | allocated_MB: 340.3 | reserved_MB: 25634.0 | inactive_split_MB: 201.7 +rank6 [2026-10-29 01:20:16,835] INFO : Step 20000/50000 rank6 | Train Loss: 1.2077 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1487 | Classification: 0.1030 | Classification_binary: 0.1729 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1639 | Regression_eta: 0.0019 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0801 | Regression_eta: 0.0395 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0496 | LR: 6.89e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8138s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.6570 | max_loss: 0.1550 | max_backward: 1.4848 | max_optimizer_step: 0.2205 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6884.1 | uss_MB: 6226.8 | child_rss_MB: 5009.1 | allocated_MB: 341.8 | reserved_MB: 26642.0 | inactive_split_MB: 237.7 +rank6 [2026-10-29 01:20:17,835] INFO : Running validation on rank6 +rank6 [2026-10-29 01:20:17,836] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 01:20:17,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 01:21:07,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:21:08,249] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 01:21:34,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 01:21:35,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 01:21:36,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 01:21:37,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 01:21:47,773] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 01:21:48,206] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:21:48,733] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 01:22:04,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 01:22:05,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 01:22:06,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 01:22:07,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 01:22:17,212] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 01:22:18,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:22:18,871] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 01:22:34,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 01:22:35,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 01:22:35,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 01:22:36,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 01:22:46,642] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 01:26:57,040] INFO : Step 20100/50000 rank6 | Train Loss: 1.3825 | Train Loss Unweighted: 0.6739 | Train Loss Components: Auxiliary: 0.2830 | Cardinality: 0.1854 | Classification: 0.1189 | Classification_binary: 0.2349 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1561 | Regression_eta: 0.0022 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2046 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0749 | Regression_eta: 0.0476 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0806 | LR: 6.86e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2467s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.9496 | max_loss: 0.1965 | max_backward: 1.6425 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.5117 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1294 | Memory: rss_MB: 8120.6 | uss_MB: 7458.3 | child_rss_MB: 5010.7 | allocated_MB: 344.8 | reserved_MB: 21328.0 | inactive_split_MB: 198.7 +rank6 [2026-10-29 01:30:57,229] INFO : Step 20200/50000 rank6 | Train Loss: 1.3220 | Train Loss Unweighted: 0.6624 | Train Loss Components: Auxiliary: 0.2728 | Cardinality: 0.1545 | Classification: 0.1109 | Classification_binary: 0.2232 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1618 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.2342 | Regression_cos_phi: 0.0853 | Regression_energy: 0.0761 | Regression_eta: 0.0607 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0858 | LR: 6.84e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8616s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.9337 | max_loss: 0.2287 | max_backward: 1.4619 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6969.7 | uss_MB: 6305.7 | child_rss_MB: 5014.0 | allocated_MB: 344.4 | reserved_MB: 21328.0 | inactive_split_MB: 287.4 +rank6 [2026-10-29 01:34:46,195] INFO : Step 20300/50000 rank6 | Train Loss: 1.2742 | Train Loss Unweighted: 0.6629 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1615 | Classification: 0.1132 | Classification_binary: 0.1937 | HitGrouping: 0.2129 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1761 | Regression_eta: 0.0018 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.2214 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0835 | Regression_eta: 0.0428 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0489 | LR: 6.81e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8199s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6857 | max_loss: 0.2423 | max_backward: 1.2822 | max_optimizer_step: 0.2837 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0419 | Memory: rss_MB: 6967.2 | uss_MB: 6303.7 | child_rss_MB: 5019.2 | allocated_MB: 340.8 | reserved_MB: 21826.0 | inactive_split_MB: 239.4 +rank6 [2026-10-29 01:38:43,604] INFO : Step 20400/50000 rank6 | Train Loss: 1.2743 | Train Loss Unweighted: 0.6521 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1507 | Classification: 0.1279 | Classification_binary: 0.1990 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1604 | Regression_eta: 0.0019 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.2013 | Regression_cos_phi: 0.0882 | Regression_energy: 0.0768 | Regression_eta: 0.0446 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0501 | LR: 6.78e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7396s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0283 | max_forward: 1.3648 | max_loss: 0.1816 | max_backward: 1.7875 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 6969.3 | uss_MB: 6306.4 | child_rss_MB: 5010.2 | allocated_MB: 343.0 | reserved_MB: 22154.0 | inactive_split_MB: 243.5 +rank6 [2026-10-29 01:42:48,352] INFO : Step 20500/50000 rank6 | Train Loss: 1.2712 | Train Loss Unweighted: 0.6509 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1409 | Classification: 0.1147 | Classification_binary: 0.2202 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1558 | Regression_eta: 0.0019 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.2249 | Regression_cos_phi: 0.0853 | Regression_energy: 0.0804 | Regression_eta: 0.0543 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0788 | LR: 6.76e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2549s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7205 | max_loss: 0.2283 | max_backward: 1.6623 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.6254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0704 | Memory: rss_MB: 6969.0 | uss_MB: 6306.9 | child_rss_MB: 5000.2 | allocated_MB: 345.0 | reserved_MB: 22154.0 | inactive_split_MB: 208.2 +rank6 [2026-10-29 01:46:45,621] INFO : Step 20600/50000 rank6 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6338 | Train Loss Components: Auxiliary: 0.2625 | Cardinality: 0.1623 | Classification: 0.1101 | Classification_binary: 0.1900 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0018 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.2045 | Regression_cos_phi: 0.1094 | Regression_energy: 0.0872 | Regression_eta: 0.0606 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0700 | LR: 6.73e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.9326s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 1.1520 | max_loss: 0.2849 | max_backward: 2.1471 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.6141 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6963.6 | uss_MB: 6301.3 | child_rss_MB: 5002.5 | allocated_MB: 341.4 | reserved_MB: 22154.0 | inactive_split_MB: 183.5 +rank6 [2026-10-29 01:50:54,186] INFO : Step 20700/50000 rank6 | Train Loss: 1.1613 | Train Loss Unweighted: 0.6066 | Train Loss Components: Auxiliary: 0.2395 | Cardinality: 0.1327 | Classification: 0.1061 | Classification_binary: 0.1677 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0016 | Regression_pt: 0.1666 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0679 | Regression_eta: 0.0369 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0486 | LR: 6.70e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5660s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6276 | max_loss: 0.2330 | max_backward: 1.4954 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6965.5 | uss_MB: 5936.8 | child_rss_MB: 5004.9 | allocated_MB: 341.0 | reserved_MB: 22154.0 | inactive_split_MB: 285.0 +rank6 [2026-10-29 01:55:01,769] INFO : Step 20800/50000 rank6 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1493 | Classification: 0.1205 | Classification_binary: 0.2020 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.1952 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0812 | Regression_eta: 0.0632 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0770 | LR: 6.67e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2088s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.7679 | max_loss: 0.1604 | max_backward: 1.7694 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.4914 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6972.0 | uss_MB: 6311.7 | child_rss_MB: 5027.1 | allocated_MB: 344.1 | reserved_MB: 22154.0 | inactive_split_MB: 258.7 +rank6 [2026-10-29 01:59:06,402] INFO : Step 20900/50000 rank6 | Train Loss: 1.2934 | Train Loss Unweighted: 0.6525 | Train Loss Components: Auxiliary: 0.2719 | Cardinality: 0.1620 | Classification: 0.1167 | Classification_binary: 0.2121 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1591 | Regression_eta: 0.0021 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 0.2630 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0890 | Regression_eta: 0.0607 | Regression_pt: 0.0933 | Regression_sin_phi: 0.0884 | LR: 6.65e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1347s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7787 | max_loss: 0.1747 | max_backward: 1.6678 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 6974.0 | uss_MB: 5772.6 | child_rss_MB: 5037.2 | allocated_MB: 343.6 | reserved_MB: 23208.0 | inactive_split_MB: 296.4 +rank6 [2026-10-29 02:03:08,876] INFO : Step 21000/50000 rank6 | Train Loss: 1.1402 | Train Loss Unweighted: 0.5874 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1236 | Classification: 0.0967 | Classification_binary: 0.1701 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1583 | Regression_eta: 0.0017 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1827 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0851 | Regression_eta: 0.0523 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0805 | LR: 6.62e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.8983s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6629 | max_loss: 0.1833 | max_backward: 1.4689 | max_optimizer_step: 0.2206 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6973.2 | uss_MB: 6313.8 | child_rss_MB: 5014.5 | allocated_MB: 340.4 | reserved_MB: 23208.0 | inactive_split_MB: 264.1 +rank6 [2026-10-29 02:03:09,907] INFO : Running validation on rank6 +rank6 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 02:04:00,486] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:04:01,073] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 02:04:27,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 02:04:28,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 02:04:29,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 02:04:30,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 02:04:40,240] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 02:04:40,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:04:41,352] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 02:04:57,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 02:04:58,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 02:04:59,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 02:05:00,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 02:05:10,320] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 02:05:10,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:05:11,207] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 02:05:27,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 02:05:27,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 02:05:28,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 02:05:30,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 02:05:40,844] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 02:10:06,178] INFO : Step 21100/50000 rank6 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6632 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1574 | Classification: 0.1277 | Classification_binary: 0.2026 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0024 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0734 | Regression_eta: 0.0716 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0680 | LR: 6.59e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2635s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7159 | max_loss: 0.1411 | max_backward: 1.5805 | max_optimizer_step: 0.3396 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8247.8 | uss_MB: 7584.2 | child_rss_MB: 5046.5 | allocated_MB: 343.8 | reserved_MB: 21372.0 | inactive_split_MB: 255.4 +rank6 [2026-10-29 02:10:38,255] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank6 [2026-10-29 02:10:38,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-29 02:14:15,482] INFO : Step 21200/50000 rank6 | Train Loss: 1.2354 | Train Loss Unweighted: 0.6366 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1422 | Classification: 0.1056 | Classification_binary: 0.1876 | HitGrouping: 0.2030 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0017 | Regression_pt: 0.1709 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0926 | Regression_energy: 0.0854 | Regression_eta: 0.0394 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0608 | LR: 6.56e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.1587s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7385 | max_loss: 0.2183 | max_backward: 1.4276 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.7483 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7137.9 | uss_MB: 6469.9 | child_rss_MB: 5019.5 | allocated_MB: 342.5 | reserved_MB: 23286.0 | inactive_split_MB: 273.0 +rank6 [2026-10-29 02:18:17,990] INFO : Step 21300/50000 rank6 | Train Loss: 1.1511 | Train Loss Unweighted: 0.5782 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1240 | Classification: 0.1021 | Classification_binary: 0.1706 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1505 | Regression_eta: 0.0017 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0915 | Regression_energy: 0.0815 | Regression_eta: 0.0544 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0729 | LR: 6.54e-04 | DataLoad Time: 0.0230s | Model Forward Time: 4.4902s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0290 | max_forward: 1.3238 | max_loss: 0.2105 | max_backward: 1.6672 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.6878 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7140.4 | uss_MB: 6475.6 | child_rss_MB: 5012.2 | allocated_MB: 343.8 | reserved_MB: 23286.0 | inactive_split_MB: 272.8 +rank6 [2026-10-29 02:22:14,588] INFO : Step 21400/50000 rank6 | Train Loss: 1.2993 | Train Loss Unweighted: 0.6458 | Train Loss Components: Auxiliary: 0.2724 | Cardinality: 0.1796 | Classification: 0.1204 | Classification_binary: 0.2158 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1518 | Regression_eta: 0.0018 | Regression_pt: 0.1542 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2075 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0737 | Regression_eta: 0.0339 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0508 | LR: 6.51e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5448s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8696 | max_loss: 0.1572 | max_backward: 1.8002 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 0.6992 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7114.7 | uss_MB: 6081.0 | child_rss_MB: 5015.7 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 201.9 +rank6 [2026-10-29 02:26:14,444] INFO : Step 21500/50000 rank6 | Train Loss: 1.2839 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2693 | Cardinality: 0.1478 | Classification: 0.1058 | Classification_binary: 0.2162 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2174 | Regression_cos_phi: 0.0908 | Regression_energy: 0.0708 | Regression_eta: 0.0530 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0789 | LR: 6.48e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7760s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.8684 | max_loss: 0.1577 | max_backward: 1.9355 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.7593 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0648 | Memory: rss_MB: 7121.8 | uss_MB: 6458.0 | child_rss_MB: 5014.4 | allocated_MB: 341.9 | reserved_MB: 25248.0 | inactive_split_MB: 284.1 +rank6 [2026-10-29 02:30:00,712] INFO : Step 21600/50000 rank6 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1290 | Classification: 0.1062 | Classification_binary: 0.1618 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1486 | Regression_eta: 0.0018 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.1937 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0738 | Regression_eta: 0.0528 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0844 | LR: 6.45e-04 | DataLoad Time: 0.0215s | Model Forward Time: 3.9640s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0061 | max_forward: 0.7663 | max_loss: 0.1883 | max_backward: 1.4257 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.4997 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7120.6 | uss_MB: 6456.9 | child_rss_MB: 5032.2 | allocated_MB: 340.5 | reserved_MB: 25248.0 | inactive_split_MB: 234.8 +rank6 [2026-10-29 02:33:57,426] INFO : Step 21700/50000 rank6 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6283 | Train Loss Components: Auxiliary: 0.2577 | Cardinality: 0.1575 | Classification: 0.1186 | Classification_binary: 0.1853 | HitGrouping: 0.2068 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1594 | Regression_eta: 0.0022 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0991 | Regression_energy: 0.0839 | Regression_eta: 0.0462 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0685 | LR: 6.43e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7312s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.6796 | max_loss: 0.1417 | max_backward: 1.3881 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7122.2 | uss_MB: 6458.0 | child_rss_MB: 5034.2 | allocated_MB: 343.8 | reserved_MB: 25248.0 | inactive_split_MB: 244.5 +rank6 [2026-10-29 02:37:54,439] INFO : Step 21800/50000 rank6 | Train Loss: 1.2260 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2691 | Cardinality: 0.1439 | Classification: 0.1105 | Classification_binary: 0.2043 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1576 | Regression_eta: 0.0016 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1742 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0675 | Regression_eta: 0.0383 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0648 | LR: 6.40e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.9203s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 0.8043 | max_loss: 0.1895 | max_backward: 2.4027 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.9442 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7124.3 | uss_MB: 5827.3 | child_rss_MB: 5037.0 | allocated_MB: 344.5 | reserved_MB: 25248.0 | inactive_split_MB: 277.8 +rank6 [2026-10-29 02:41:42,512] INFO : Step 21900/50000 rank6 | Train Loss: 1.2119 | Train Loss Unweighted: 0.5952 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1861 | Classification: 0.1033 | Classification_binary: 0.1865 | HitGrouping: 0.1989 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0015 | Regression_pt: 0.1511 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1977 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0710 | Regression_eta: 0.0407 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0647 | LR: 6.37e-04 | DataLoad Time: 0.0136s | Model Forward Time: 4.5486s | Phase Times: max_pre_step_barrier: 0.0174 | max_batch_to_device: 0.0163 | max_forward: 0.9684 | max_loss: 0.1448 | max_backward: 1.6354 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.8990 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 7124.6 | uss_MB: 6460.4 | child_rss_MB: 5033.8 | allocated_MB: 342.7 | reserved_MB: 25248.0 | inactive_split_MB: 192.3 +rank6 [2026-10-29 02:45:41,731] INFO : Step 22000/50000 rank6 | Train Loss: 1.1094 | Train Loss Unweighted: 0.5651 | Train Loss Components: Auxiliary: 0.2344 | Cardinality: 0.1248 | Classification: 0.1037 | Classification_binary: 0.1579 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1497 | Regression_eta: 0.0017 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1387 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0822 | Regression_eta: 0.0426 | Regression_pt: 0.0837 | Regression_sin_phi: 0.1034 | LR: 6.34e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.4198s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.6825 | max_loss: 0.1653 | max_backward: 1.4396 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.0256 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 7124.1 | uss_MB: 6463.7 | child_rss_MB: 5009.1 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 247.3 +rank6 [2026-10-29 02:45:42,811] INFO : Running validation on rank6 +rank6 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 02:46:37,520] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:46:38,050] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 02:46:57,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 02:46:58,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 02:46:59,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 02:47:00,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 02:47:10,852] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 02:47:11,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:47:11,962] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 02:47:27,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 02:47:28,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 02:47:29,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 02:47:30,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 02:47:40,585] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 02:47:41,460] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:47:42,035] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 02:47:57,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 02:47:58,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 02:47:58,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 02:47:59,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 02:48:09,656] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 02:52:24,843] INFO : Step 22100/50000 rank6 | Train Loss: 1.2514 | Train Loss Unweighted: 0.6219 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1530 | Classification: 0.1140 | Classification_binary: 0.2076 | HitGrouping: 0.2031 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1470 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1992 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0679 | Regression_eta: 0.0467 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0731 | LR: 6.32e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5389s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7250 | max_loss: 0.2514 | max_backward: 1.5956 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.8632 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0587 | Memory: rss_MB: 8032.9 | uss_MB: 7365.3 | child_rss_MB: 5019.4 | allocated_MB: 347.3 | reserved_MB: 23386.0 | inactive_split_MB: 278.6 +rank6 [2026-10-29 02:56:29,091] INFO : Step 22200/50000 rank6 | Train Loss: 1.2727 | Train Loss Unweighted: 0.6485 | Train Loss Components: Auxiliary: 0.2709 | Cardinality: 0.1498 | Classification: 0.1104 | Classification_binary: 0.2184 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1574 | Regression_eta: 0.0018 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0791 | Classification_binary: 0.2212 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0675 | Regression_eta: 0.0512 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0769 | LR: 6.29e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.9378s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0074 | max_forward: 0.7776 | max_loss: 0.1472 | max_backward: 1.7604 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.3305 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7158.4 | uss_MB: 5092.5 | child_rss_MB: 5016.1 | allocated_MB: 342.3 | reserved_MB: 23578.0 | inactive_split_MB: 277.9 +rank6 [2026-10-29 03:00:35,124] INFO : Step 22300/50000 rank6 | Train Loss: 1.1242 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2347 | Cardinality: 0.1367 | Classification: 0.1013 | Classification_binary: 0.1688 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1587 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0726 | Regression_eta: 0.0377 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0599 | LR: 6.26e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.7783s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7697 | max_loss: 0.1538 | max_backward: 1.3553 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.4896 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 7162.9 | uss_MB: 6498.9 | child_rss_MB: 4987.3 | allocated_MB: 341.2 | reserved_MB: 23578.0 | inactive_split_MB: 233.8 +rank6 [2026-10-29 03:04:32,773] INFO : Step 22400/50000 rank6 | Train Loss: 1.2357 | Train Loss Unweighted: 0.6255 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1493 | Classification: 0.1171 | Classification_binary: 0.1885 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1575 | Regression_eta: 0.0021 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2023 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0808 | Regression_eta: 0.0705 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0890 | LR: 6.23e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9142s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0067 | max_forward: 0.6425 | max_loss: 0.1530 | max_backward: 1.4136 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.5876 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 7158.4 | uss_MB: 6496.8 | child_rss_MB: 5021.9 | allocated_MB: 342.5 | reserved_MB: 23578.0 | inactive_split_MB: 251.5 +rank6 [2026-10-29 03:08:26,683] INFO : Step 22500/50000 rank6 | Train Loss: 1.2633 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2638 | Cardinality: 0.1676 | Classification: 0.0954 | Classification_binary: 0.2181 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0020 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1952 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0741 | Regression_eta: 0.0483 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0899 | LR: 6.20e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.4282s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1522 | max_backward: 1.5631 | max_optimizer_step: 0.3261 | max_tensorboard_logging: 0.7552 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 7160.2 | uss_MB: 6502.4 | child_rss_MB: 5011.7 | allocated_MB: 342.9 | reserved_MB: 24052.0 | inactive_split_MB: 295.1 +rank6 [2026-10-29 03:12:18,754] INFO : Step 22600/50000 rank6 | Train Loss: 1.1636 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1329 | Classification: 0.0997 | Classification_binary: 0.1889 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1493 | Regression_eta: 0.0020 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0789 | Classification_binary: 0.2261 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0804 | Regression_eta: 0.0620 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0970 | LR: 6.18e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9316s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7375 | max_loss: 0.2403 | max_backward: 1.4584 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7163.5 | uss_MB: 6503.3 | child_rss_MB: 5024.4 | allocated_MB: 344.2 | reserved_MB: 24590.0 | inactive_split_MB: 288.5 +rank6 [2026-10-29 03:16:20,064] INFO : Step 22700/50000 rank6 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1212 | Classification: 0.0976 | Classification_binary: 0.1566 | HitGrouping: 0.2101 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1455 | Regression_eta: 0.0015 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0710 | Regression_eta: 0.0412 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0836 | LR: 6.15e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.9676s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.6124 | max_loss: 0.1376 | max_backward: 1.1766 | max_optimizer_step: 0.2237 | max_tensorboard_logging: 3.8568 | max_tensorboard_flush: 0.0169 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 7164.2 | uss_MB: 6160.1 | child_rss_MB: 5007.8 | allocated_MB: 341.6 | reserved_MB: 25058.0 | inactive_split_MB: 256.4 +rank6 [2026-10-29 03:20:29,739] INFO : Step 22800/50000 rank6 | Train Loss: 1.1840 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2533 | Cardinality: 0.1443 | Classification: 0.1191 | Classification_binary: 0.1924 | HitGrouping: 0.1959 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1444 | Regression_eta: 0.0016 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.1736 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0668 | Regression_eta: 0.0304 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0810 | LR: 6.12e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4528s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7010 | max_loss: 0.2312 | max_backward: 1.6853 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8726 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 7164.4 | uss_MB: 6502.8 | child_rss_MB: 5021.2 | allocated_MB: 341.7 | reserved_MB: 25058.0 | inactive_split_MB: 281.3 +rank6 [2026-10-29 03:24:21,160] INFO : Step 22900/50000 rank6 | Train Loss: 1.2680 | Train Loss Unweighted: 0.6318 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1792 | Classification: 0.1020 | Classification_binary: 0.2074 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1580 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.2321 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0923 | Regression_eta: 0.0489 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0817 | LR: 6.09e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9982s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 1.1637 | max_loss: 0.4031 | max_backward: 1.8442 | max_optimizer_step: 0.3320 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7165.6 | uss_MB: 6506.2 | child_rss_MB: 5023.4 | allocated_MB: 344.8 | reserved_MB: 25058.0 | inactive_split_MB: 300.1 +rank6 [2026-10-29 03:28:05,592] INFO : Step 23000/50000 rank6 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5484 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1129 | Classification: 0.0836 | Classification_binary: 0.1519 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1547 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0697 | Regression_eta: 0.0487 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0572 | LR: 6.06e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7521s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0063 | max_forward: 0.6963 | max_loss: 0.2382 | max_backward: 1.2870 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 7162.6 | uss_MB: 6506.3 | child_rss_MB: 5008.7 | allocated_MB: 340.3 | reserved_MB: 26066.0 | inactive_split_MB: 218.4 +rank6 [2026-10-29 03:28:06,534] INFO : Running validation on rank6 +rank6 [2026-10-29 03:28:06,534] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 03:28:06,534] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 03:28:56,135] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:28:56,638] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 03:29:22,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 03:29:23,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 03:29:24,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 03:29:25,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 03:29:35,571] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 03:29:35,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:29:36,301] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 03:29:52,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 03:29:53,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 03:29:54,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 03:29:54,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 03:30:04,952] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 03:30:05,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:30:05,703] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 03:30:21,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 03:30:21,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 03:30:22,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 03:30:23,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 03:30:33,254] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 03:34:45,380] INFO : Step 23100/50000 rank6 | Train Loss: 1.2422 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1379 | Classification: 0.1246 | Classification_binary: 0.1795 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1482 | Regression_eta: 0.0023 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1636 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0686 | Regression_eta: 0.0417 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0604 | LR: 6.04e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8243s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7203 | max_loss: 0.1522 | max_backward: 1.1000 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.6183 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 8419.6 | uss_MB: 7758.7 | child_rss_MB: 5039.2 | allocated_MB: 344.7 | reserved_MB: 22598.0 | inactive_split_MB: 244.5 +rank6 [2026-10-29 03:38:33,524] INFO : Step 23200/50000 rank6 | Train Loss: 1.2507 | Train Loss Unweighted: 0.6359 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1400 | Classification: 0.1121 | Classification_binary: 0.2208 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1488 | Regression_eta: 0.0020 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0843 | Classification_binary: 0.2492 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0955 | Regression_eta: 0.0711 | Regression_pt: 0.1004 | Regression_sin_phi: 0.1163 | LR: 6.01e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.5955s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0727 | max_forward: 0.9486 | max_loss: 0.1744 | max_backward: 1.7031 | max_optimizer_step: 0.2405 | max_tensorboard_logging: 0.6717 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1109 | Memory: rss_MB: 7225.3 | uss_MB: 6564.6 | child_rss_MB: 5046.8 | allocated_MB: 343.7 | reserved_MB: 23792.0 | inactive_split_MB: 223.2 +rank6 [2026-10-29 03:42:24,564] INFO : Step 23300/50000 rank6 | Train Loss: 1.1880 | Train Loss Unweighted: 0.5927 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1464 | Classification: 0.0987 | Classification_binary: 0.1851 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1524 | Regression_eta: 0.0019 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0690 | Regression_eta: 0.0559 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0762 | LR: 5.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8597s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7557 | max_loss: 0.1467 | max_backward: 1.4708 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.4854 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 7231.3 | uss_MB: 6572.4 | child_rss_MB: 5045.1 | allocated_MB: 342.6 | reserved_MB: 24534.0 | inactive_split_MB: 271.6 +rank6 [2026-10-29 03:46:11,296] INFO : Step 23400/50000 rank6 | Train Loss: 1.1136 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2402 | Cardinality: 0.1202 | Classification: 0.1070 | Classification_binary: 0.1497 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1644 | Regression_eta: 0.0017 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0603 | Regression_eta: 0.0466 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0459 | LR: 5.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.6665s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.5743 | max_loss: 0.2297 | max_backward: 1.0127 | max_optimizer_step: 0.3321 | max_tensorboard_logging: 0.6182 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 7232.3 | uss_MB: 6571.7 | child_rss_MB: 5046.9 | allocated_MB: 340.5 | reserved_MB: 24848.0 | inactive_split_MB: 241.3 +rank6 [2026-10-29 03:50:07,300] INFO : Step 23500/50000 rank6 | Train Loss: 1.1601 | Train Loss Unweighted: 0.5799 | Train Loss Components: Auxiliary: 0.2477 | Cardinality: 0.1161 | Classification: 0.1057 | Classification_binary: 0.1852 | HitGrouping: 0.2029 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1416 | Regression_eta: 0.0017 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0853 | Regression_energy: 0.0689 | Regression_eta: 0.0612 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0682 | LR: 5.92e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4401s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8864 | max_loss: 0.1573 | max_backward: 1.9036 | max_optimizer_step: 0.2294 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7232.1 | uss_MB: 6574.4 | child_rss_MB: 5036.9 | allocated_MB: 341.9 | reserved_MB: 24848.0 | inactive_split_MB: 233.9 +rank6 [2026-10-29 03:53:52,162] INFO : Step 23600/50000 rank6 | Train Loss: 1.2402 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1447 | Classification: 0.1136 | Classification_binary: 0.2049 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1534 | Regression_eta: 0.0020 | Regression_pt: 0.1550 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0802 | Classification_binary: 0.2184 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0681 | Regression_eta: 0.0682 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0852 | LR: 5.90e-04 | DataLoad Time: 0.0217s | Model Forward Time: 3.9412s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8397 | max_loss: 0.1470 | max_backward: 1.4395 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7234.1 | uss_MB: 6575.7 | child_rss_MB: 5045.6 | allocated_MB: 342.5 | reserved_MB: 24848.0 | inactive_split_MB: 215.5 +rank6 [2026-10-29 03:57:35,779] INFO : Step 23700/50000 rank6 | Train Loss: 1.1009 | Train Loss Unweighted: 0.5682 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1218 | Classification: 0.0954 | Classification_binary: 0.1599 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1528 | Regression_eta: 0.0014 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0594 | Regression_eta: 0.0615 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0608 | LR: 5.87e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6814s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6589 | max_loss: 0.1387 | max_backward: 1.2152 | max_optimizer_step: 0.2251 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7230.0 | uss_MB: 6574.3 | child_rss_MB: 5051.4 | allocated_MB: 342.5 | reserved_MB: 25508.0 | inactive_split_MB: 226.3 +rank6 [2026-10-29 04:01:35,998] INFO : Step 23800/50000 rank6 | Train Loss: 1.1390 | Train Loss Unweighted: 0.5882 | Train Loss Components: Auxiliary: 0.2427 | Cardinality: 0.1196 | Classification: 0.1035 | Classification_binary: 0.1735 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0820 | Regression_eta: 0.0358 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0481 | LR: 5.84e-04 | DataLoad Time: 0.0222s | Model Forward Time: 3.7609s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.7283 | max_loss: 0.1547 | max_backward: 1.2415 | max_optimizer_step: 0.2259 | max_tensorboard_logging: 0.5270 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7231.0 | uss_MB: 6110.2 | child_rss_MB: 5050.1 | allocated_MB: 345.2 | reserved_MB: 25508.0 | inactive_split_MB: 241.8 +rank6 [2026-10-29 04:05:42,354] INFO : Step 23900/50000 rank6 | Train Loss: 1.2349 | Train Loss Unweighted: 0.6078 | Train Loss Components: Auxiliary: 0.2621 | Cardinality: 0.1596 | Classification: 0.1044 | Classification_binary: 0.2077 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1451 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2000 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0779 | Regression_eta: 0.0420 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0672 | LR: 5.81e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.4876s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.9077 | max_loss: 0.1538 | max_backward: 1.9088 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7234.7 | uss_MB: 6579.6 | child_rss_MB: 5044.4 | allocated_MB: 342.1 | reserved_MB: 27474.0 | inactive_split_MB: 282.4 +rank6 [2026-10-29 04:09:40,154] INFO : Step 24000/50000 rank6 | Train Loss: 1.1074 | Train Loss Unweighted: 0.5714 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1170 | Classification: 0.1007 | Classification_binary: 0.1792 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1431 | Regression_eta: 0.0015 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0948 | Regression_energy: 0.0690 | Regression_eta: 0.0356 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0906 | LR: 5.78e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.2188s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 1.1428 | max_loss: 0.2613 | max_backward: 2.5767 | max_optimizer_step: 0.2298 | max_tensorboard_logging: 0.5998 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7236.7 | uss_MB: 6577.2 | child_rss_MB: 5038.1 | allocated_MB: 342.1 | reserved_MB: 27476.0 | inactive_split_MB: 218.6 +rank6 [2026-10-29 04:09:41,046] INFO : Running validation on rank6 +rank6 [2026-10-29 04:09:41,047] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 04:09:41,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 04:10:38,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:10:39,335] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 04:11:05,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 04:11:06,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 04:11:09,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 04:11:10,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 04:11:20,658] INFO : Time taken to make predictions on device 6 is: 0.29 min +rank6 [2026-10-29 04:11:20,671] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:11:21,254] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 04:11:37,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 04:11:38,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 04:11:39,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 04:11:40,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 04:11:50,170] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:11:50,532] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:11:51,026] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 04:12:06,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 04:12:06,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 04:12:07,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 04:12:08,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 04:12:18,204] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 04:16:38,286] INFO : Step 24100/50000 rank6 | Train Loss: 1.1244 | Train Loss Unweighted: 0.5584 | Train Loss Components: Auxiliary: 0.2373 | Cardinality: 0.1285 | Classification: 0.0978 | Classification_binary: 0.1606 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.1507 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0705 | Regression_eta: 0.0373 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0561 | LR: 5.75e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1982s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0860 | max_forward: 0.7361 | max_loss: 0.1432 | max_backward: 1.4268 | max_optimizer_step: 0.3451 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 8228.7 | uss_MB: 7570.1 | child_rss_MB: 5001.0 | allocated_MB: 343.2 | reserved_MB: 21608.0 | inactive_split_MB: 269.9 +rank6 [2026-10-29 04:20:36,776] INFO : Step 24200/50000 rank6 | Train Loss: 1.3067 | Train Loss Unweighted: 0.6341 | Train Loss Components: Auxiliary: 0.2739 | Cardinality: 0.1760 | Classification: 0.1145 | Classification_binary: 0.2101 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0018 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.2176 | Regression_cos_phi: 0.1148 | Regression_energy: 0.0664 | Regression_eta: 0.0568 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0657 | LR: 5.73e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.0789s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1066 | max_forward: 1.0531 | max_loss: 0.2796 | max_backward: 2.3820 | max_optimizer_step: 0.2441 | max_tensorboard_logging: 0.6187 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 8233.6 | uss_MB: 7574.6 | child_rss_MB: 5010.3 | allocated_MB: 345.0 | reserved_MB: 23864.0 | inactive_split_MB: 203.0 +rank6 [2026-10-29 04:24:33,011] INFO : Step 24300/50000 rank6 | Train Loss: 1.2207 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1582 | Classification: 0.1151 | Classification_binary: 0.2013 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1450 | Regression_eta: 0.0017 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.2068 | Regression_cos_phi: 0.0934 | Regression_energy: 0.0694 | Regression_eta: 0.0654 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0889 | LR: 5.70e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.3080s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7395 | max_loss: 0.2490 | max_backward: 1.4969 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.6555 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7355.3 | uss_MB: 6697.1 | child_rss_MB: 5007.1 | allocated_MB: 344.8 | reserved_MB: 23864.0 | inactive_split_MB: 284.8 +rank6 [2026-10-29 04:28:19,172] INFO : Step 24400/50000 rank6 | Train Loss: 1.1288 | Train Loss Unweighted: 0.5921 | Train Loss Components: Auxiliary: 0.2439 | Cardinality: 0.1229 | Classification: 0.1017 | Classification_binary: 0.1613 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1631 | Regression_eta: 0.0015 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0682 | Regression_eta: 0.0339 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0458 | LR: 5.67e-04 | DataLoad Time: 0.0185s | Model Forward Time: 3.9870s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.6685 | max_loss: 0.1564 | max_backward: 1.4736 | max_optimizer_step: 0.2223 | max_tensorboard_logging: 0.5369 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7357.1 | uss_MB: 6698.7 | child_rss_MB: 5016.5 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 267.2 +rank6 [2026-10-29 04:32:03,440] INFO : Step 24500/50000 rank6 | Train Loss: 1.2319 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2586 | Cardinality: 0.1511 | Classification: 0.1107 | Classification_binary: 0.1876 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1540 | Regression_eta: 0.0018 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1930 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0722 | Regression_eta: 0.0499 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0478 | LR: 5.64e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1611s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0677 | max_forward: 0.7219 | max_loss: 0.1439 | max_backward: 1.4253 | max_optimizer_step: 0.3416 | max_tensorboard_logging: 0.5859 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 7356.5 | uss_MB: 6698.2 | child_rss_MB: 5010.5 | allocated_MB: 343.1 | reserved_MB: 23864.0 | inactive_split_MB: 275.8 +rank6 [2026-10-29 04:36:01,551] INFO : Step 24600/50000 rank6 | Train Loss: 1.2474 | Train Loss Unweighted: 0.6241 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1630 | Classification: 0.1074 | Classification_binary: 0.2102 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.2122 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0669 | Regression_eta: 0.0475 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0804 | LR: 5.61e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5127s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7876 | max_loss: 0.1528 | max_backward: 1.7437 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.6204 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7358.2 | uss_MB: 6700.7 | child_rss_MB: 5006.8 | allocated_MB: 344.3 | reserved_MB: 23864.0 | inactive_split_MB: 285.7 +rank6 [2026-10-29 04:39:51,297] INFO : Step 24700/50000 rank6 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1193 | Classification: 0.0990 | Classification_binary: 0.1723 | HitGrouping: 0.2019 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1471 | Regression_eta: 0.0016 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0607 | Regression_eta: 0.0459 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0475 | LR: 5.58e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9505s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6978 | max_loss: 0.2493 | max_backward: 1.2925 | max_optimizer_step: 0.2314 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7361.3 | uss_MB: 6700.3 | child_rss_MB: 5009.8 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 223.9 +rank6 [2026-10-29 04:43:37,807] INFO : Step 24800/50000 rank6 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5554 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1264 | Classification: 0.0932 | Classification_binary: 0.1583 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0752 | Regression_eta: 0.0375 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0614 | LR: 5.56e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.2760s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6582 | max_loss: 0.2533 | max_backward: 1.5858 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.6923 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1086 | Memory: rss_MB: 7365.2 | uss_MB: 6703.3 | child_rss_MB: 5005.7 | allocated_MB: 342.9 | reserved_MB: 23864.0 | inactive_split_MB: 278.6 +rank6 [2026-10-29 04:47:36,371] INFO : Step 24900/50000 rank6 | Train Loss: 1.1348 | Train Loss Unweighted: 0.5816 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1381 | Classification: 0.1126 | Classification_binary: 0.1845 | HitGrouping: 0.1937 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1403 | Regression_eta: 0.0015 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1592 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0647 | Regression_eta: 0.0380 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0512 | LR: 5.53e-04 | DataLoad Time: 0.0152s | Model Forward Time: 8.7029s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0067 | max_forward: 0.7276 | max_loss: 0.2411 | max_backward: 1.3958 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 5.1754 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1038 | Memory: rss_MB: 7366.2 | uss_MB: 6703.5 | child_rss_MB: 5007.2 | allocated_MB: 343.9 | reserved_MB: 23864.0 | inactive_split_MB: 239.7 +rank6 [2026-10-29 04:51:28,372] INFO : Step 25000/50000 rank6 | Train Loss: 1.1995 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1590 | Classification: 0.0891 | Classification_binary: 0.2009 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1421 | Regression_eta: 0.0018 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.2142 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0641 | Regression_eta: 0.0362 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0731 | LR: 5.50e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2026s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7692 | max_loss: 0.2091 | max_backward: 1.3616 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6652 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1392 | Memory: rss_MB: 7366.4 | uss_MB: 6702.5 | child_rss_MB: 5014.2 | allocated_MB: 343.5 | reserved_MB: 23864.0 | inactive_split_MB: 293.8 +rank6 [2026-10-29 04:51:29,310] INFO : Running validation on rank6 +rank6 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 04:52:22,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:52:22,822] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 04:52:49,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 04:52:50,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 04:52:51,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 04:52:52,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 04:53:02,388] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:53:02,764] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:53:03,282] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 04:53:19,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 04:53:20,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 04:53:20,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 04:53:21,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 04:53:31,927] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:53:32,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:53:32,992] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 04:53:48,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 04:53:49,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 04:53:49,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 04:53:50,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 04:54:00,484] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 04:58:07,811] INFO : Step 25100/50000 rank6 | Train Loss: 1.0833 | Train Loss Unweighted: 0.5514 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1317 | Classification: 0.0920 | Classification_binary: 0.1568 | HitGrouping: 0.2008 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1491 | Regression_eta: 0.0016 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1720 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0760 | Regression_eta: 0.0485 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0703 | LR: 5.47e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7876s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7295 | max_loss: 0.1401 | max_backward: 1.2439 | max_optimizer_step: 0.2490 | max_tensorboard_logging: 0.4940 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8607.6 | uss_MB: 7944.9 | child_rss_MB: 5042.3 | allocated_MB: 342.3 | reserved_MB: 23608.0 | inactive_split_MB: 222.3 +rank6 [2026-10-29 05:01:53,204] INFO : Step 25200/50000 rank6 | Train Loss: 1.1886 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1523 | Classification: 0.1158 | Classification_binary: 0.1800 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1512 | Regression_eta: 0.0016 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.1710 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0798 | Regression_eta: 0.0469 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0799 | LR: 5.44e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.7315s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8134 | max_loss: 0.1416 | max_backward: 1.5095 | max_optimizer_step: 0.3471 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 8606.1 | uss_MB: 7945.9 | child_rss_MB: 5039.5 | allocated_MB: 343.6 | reserved_MB: 24590.0 | inactive_split_MB: 260.9 +rank6 [2026-10-29 05:05:52,266] INFO : Step 25300/50000 rank6 | Train Loss: 1.2031 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2624 | Cardinality: 0.1455 | Classification: 0.1145 | Classification_binary: 0.1966 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1448 | Regression_eta: 0.0016 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0952 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0714 | Regression_eta: 0.0623 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0970 | LR: 5.42e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5350s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.8219 | max_loss: 0.1491 | max_backward: 1.6406 | max_optimizer_step: 0.2735 | max_tensorboard_logging: 0.7204 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0488 | Memory: rss_MB: 7456.6 | uss_MB: 6796.5 | child_rss_MB: 5036.7 | allocated_MB: 341.8 | reserved_MB: 25572.0 | inactive_split_MB: 253.5 +rank6 [2026-10-29 05:09:44,970] INFO : Step 25400/50000 rank6 | Train Loss: 1.1194 | Train Loss Unweighted: 0.5655 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1346 | Classification: 0.0983 | Classification_binary: 0.1754 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1438 | Regression_eta: 0.0015 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.1794 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0607 | Regression_eta: 0.0354 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0684 | LR: 5.39e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.1068s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.6986 | max_loss: 0.1530 | max_backward: 1.2606 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.8427 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7454.9 | uss_MB: 6795.9 | child_rss_MB: 5040.6 | allocated_MB: 345.2 | reserved_MB: 25572.0 | inactive_split_MB: 223.8 +rank6 [2026-10-29 05:13:34,054] INFO : Step 25500/50000 rank6 | Train Loss: 1.0289 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1068 | Classification: 0.1034 | Classification_binary: 0.1444 | HitGrouping: 0.1913 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1484 | Regression_eta: 0.0014 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1446 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0801 | Regression_eta: 0.0278 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0394 | LR: 5.36e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.1583s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0554 | max_forward: 0.7008 | max_loss: 0.1433 | max_backward: 1.4571 | max_optimizer_step: 0.3349 | max_tensorboard_logging: 0.6083 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 7460.4 | uss_MB: 6802.0 | child_rss_MB: 5038.5 | allocated_MB: 345.4 | reserved_MB: 25572.0 | inactive_split_MB: 240.6 +rank6 [2026-10-29 05:17:32,215] INFO : Step 25600/50000 rank6 | Train Loss: 1.2028 | Train Loss Unweighted: 0.5968 | Train Loss Components: Auxiliary: 0.2590 | Cardinality: 0.1607 | Classification: 0.1135 | Classification_binary: 0.1926 | HitGrouping: 0.1978 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1429 | Regression_eta: 0.0016 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0682 | Regression_eta: 0.0319 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0586 | LR: 5.33e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6817s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8800 | max_loss: 0.1578 | max_backward: 1.8564 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 7459.9 | uss_MB: 6802.4 | child_rss_MB: 5035.4 | allocated_MB: 345.5 | reserved_MB: 26054.0 | inactive_split_MB: 284.5 +rank6 [2026-10-29 05:21:26,001] INFO : Step 25700/50000 rank6 | Train Loss: 1.1800 | Train Loss Unweighted: 0.5964 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1298 | Classification: 0.1086 | Classification_binary: 0.1961 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0019 | Regression_pt: 0.1444 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2529 | Regression_cos_phi: 0.1366 | Regression_energy: 0.0662 | Regression_eta: 0.0877 | Regression_pt: 0.0693 | Regression_sin_phi: 0.1319 | LR: 5.30e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5186s | Phase Times: max_pre_step_barrier: 0.0086 | max_batch_to_device: 0.0137 | max_forward: 1.0340 | max_loss: 0.2483 | max_backward: 1.6704 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 0.7612 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7459.7 | uss_MB: 6801.4 | child_rss_MB: 5033.4 | allocated_MB: 343.5 | reserved_MB: 26054.0 | inactive_split_MB: 270.8 +rank6 [2026-10-29 05:25:16,514] INFO : Step 25800/50000 rank6 | Train Loss: 1.1337 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2388 | Cardinality: 0.1264 | Classification: 0.0898 | Classification_binary: 0.1617 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0019 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0593 | Regression_energy: 0.1041 | Regression_eta: 0.0485 | Regression_pt: 0.1075 | Regression_sin_phi: 0.0725 | LR: 5.27e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9352s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0986 | max_forward: 0.8129 | max_loss: 0.1662 | max_backward: 1.2979 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1103 | Memory: rss_MB: 7457.8 | uss_MB: 6799.8 | child_rss_MB: 5030.2 | allocated_MB: 341.8 | reserved_MB: 26054.0 | inactive_split_MB: 273.2 +rank6 [2026-10-29 05:29:08,211] INFO : Step 25900/50000 rank6 | Train Loss: 1.1019 | Train Loss Unweighted: 0.5481 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1165 | Classification: 0.0977 | Classification_binary: 0.1594 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1434 | Regression_eta: 0.0016 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1736 | Regression_cos_phi: 0.1095 | Regression_energy: 0.0839 | Regression_eta: 0.0496 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0901 | LR: 5.25e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5197s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7034 | max_loss: 0.2469 | max_backward: 1.7130 | max_optimizer_step: 0.2363 | max_tensorboard_logging: 0.8382 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 7454.8 | uss_MB: 6800.6 | child_rss_MB: 5037.1 | allocated_MB: 343.0 | reserved_MB: 26054.0 | inactive_split_MB: 187.0 +rank6 [2026-10-29 05:33:07,468] INFO : Step 26000/50000 rank6 | Train Loss: 1.1708 | Train Loss Unweighted: 0.5680 | Train Loss Components: Auxiliary: 0.2498 | Cardinality: 0.1352 | Classification: 0.0999 | Classification_binary: 0.2021 | HitGrouping: 0.1994 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1304 | Regression_eta: 0.0021 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2165 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0615 | Regression_eta: 0.0475 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0580 | LR: 5.22e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2246s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.7877 | max_loss: 0.1637 | max_backward: 1.6550 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.4954 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 7454.4 | uss_MB: 6802.1 | child_rss_MB: 5032.6 | allocated_MB: 345.0 | reserved_MB: 27098.0 | inactive_split_MB: 285.6 +rank6 [2026-10-29 05:33:08,394] INFO : Running validation on rank6 +rank6 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 05:33:56,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:33:57,248] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 05:34:24,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 05:34:25,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 05:34:27,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 05:34:28,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 05:34:38,057] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 05:34:38,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:34:38,537] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 05:34:55,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 05:34:56,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 05:34:56,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 05:34:57,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 05:35:07,908] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 05:35:08,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:35:08,510] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 05:35:24,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 05:35:24,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 05:35:25,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 05:35:26,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 05:35:36,133] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 05:39:49,626] INFO : Step 26100/50000 rank6 | Train Loss: 1.0974 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2307 | Cardinality: 0.1119 | Classification: 0.0956 | Classification_binary: 0.1689 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1513 | Regression_eta: 0.0014 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0755 | Regression_eta: 0.0505 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0708 | LR: 5.19e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1590s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0117 | max_forward: 0.7200 | max_loss: 0.1501 | max_backward: 1.2286 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.8284 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 8630.6 | uss_MB: 7973.7 | child_rss_MB: 5008.3 | allocated_MB: 340.1 | reserved_MB: 21762.0 | inactive_split_MB: 215.7 +rank6 [2026-10-29 05:43:56,539] INFO : Step 26200/50000 rank6 | Train Loss: 1.0876 | Train Loss Unweighted: 0.5502 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1065 | Classification: 0.0968 | Classification_binary: 0.1527 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1476 | Regression_eta: 0.0018 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0634 | Regression_eta: 0.0441 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0670 | LR: 5.16e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3253s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6530 | max_loss: 0.1421 | max_backward: 1.3973 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.8695 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8630.6 | uss_MB: 7973.8 | child_rss_MB: 5031.7 | allocated_MB: 342.3 | reserved_MB: 22678.0 | inactive_split_MB: 242.5 +rank6 [2026-10-29 05:47:48,030] INFO : Step 26300/50000 rank6 | Train Loss: 1.2051 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1512 | Classification: 0.1193 | Classification_binary: 0.1877 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1458 | Regression_eta: 0.0017 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.1617 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0640 | Regression_eta: 0.0347 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0735 | LR: 5.13e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4678s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0072 | max_forward: 0.7601 | max_loss: 0.2267 | max_backward: 1.8902 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7522.0 | uss_MB: 6871.8 | child_rss_MB: 4996.1 | allocated_MB: 345.4 | reserved_MB: 22678.0 | inactive_split_MB: 270.9 +rank6 [2026-10-29 05:51:47,278] INFO : Step 26400/50000 rank6 | Train Loss: 1.2256 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1648 | Classification: 0.1020 | Classification_binary: 0.1981 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1520 | Regression_eta: 0.0015 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2414 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0677 | Regression_eta: 0.0410 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0856 | LR: 5.10e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3844s | Phase Times: max_pre_step_barrier: 0.0351 | max_batch_to_device: 0.0070 | max_forward: 0.8129 | max_loss: 0.1477 | max_backward: 1.4015 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8384 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 7520.2 | uss_MB: 6869.2 | child_rss_MB: 4985.0 | allocated_MB: 342.8 | reserved_MB: 22678.0 | inactive_split_MB: 196.9 +rank6 [2026-10-29 05:55:40,758] INFO : Step 26500/50000 rank6 | Train Loss: 1.1110 | Train Loss Unweighted: 0.5356 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1488 | Classification: 0.0891 | Classification_binary: 0.1546 | HitGrouping: 0.2012 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0650 | Regression_eta: 0.0446 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0488 | LR: 5.08e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0222s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6841 | max_loss: 0.2421 | max_backward: 1.1616 | max_optimizer_step: 0.2383 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7505.4 | uss_MB: 6872.6 | child_rss_MB: 4957.6 | allocated_MB: 340.6 | reserved_MB: 22678.0 | inactive_split_MB: 269.9 +rank6 [2026-10-29 05:59:33,068] INFO : Step 26600/50000 rank6 | Train Loss: 1.0895 | Train Loss Unweighted: 0.5529 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1261 | Classification: 0.1083 | Classification_binary: 0.1603 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1386 | Regression_eta: 0.0015 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1674 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0634 | Regression_eta: 0.0454 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0696 | LR: 5.05e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.4525s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0696 | max_forward: 0.6926 | max_loss: 0.1459 | max_backward: 1.4149 | max_optimizer_step: 0.2385 | max_tensorboard_logging: 0.8927 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7477.7 | uss_MB: 6873.1 | child_rss_MB: 4831.1 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 232.0 +rank6 [2026-10-29 06:03:23,754] INFO : Step 26700/50000 rank6 | Train Loss: 1.2592 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1717 | Classification: 0.1100 | Classification_binary: 0.2137 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1420 | Regression_eta: 0.0017 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2108 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0679 | Regression_eta: 0.0444 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0687 | LR: 5.02e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.6743s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7750 | max_loss: 0.1526 | max_backward: 1.6427 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.8866 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7338.5 | uss_MB: 6135.7 | child_rss_MB: 4788.2 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 267.4 +rank6 [2026-10-29 06:07:16,644] INFO : Step 26800/50000 rank6 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5792 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1288 | Classification: 0.0911 | Classification_binary: 0.1633 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1600 | Regression_eta: 0.0015 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1752 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0751 | Regression_eta: 0.0413 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0634 | LR: 4.99e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0411s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7108 | max_loss: 0.1467 | max_backward: 1.2855 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7173 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 7253.0 | uss_MB: 6872.2 | child_rss_MB: 4809.0 | allocated_MB: 340.8 | reserved_MB: 22846.0 | inactive_split_MB: 253.2 +rank6 [2026-10-29 06:11:09,652] INFO : Step 26900/50000 rank6 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.0996 | Classification: 0.0867 | Classification_binary: 0.1428 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1483 | Regression_eta: 0.0018 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0941 | Regression_eta: 0.0364 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0676 | LR: 4.96e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8394s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0336 | max_forward: 0.6534 | max_loss: 0.1408 | max_backward: 1.3898 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0912 | Memory: rss_MB: 7214.0 | uss_MB: 6872.7 | child_rss_MB: 4790.0 | allocated_MB: 342.5 | reserved_MB: 22846.0 | inactive_split_MB: 233.5 +rank6 [2026-10-29 06:15:05,368] INFO : Step 27000/50000 rank6 | Train Loss: 1.0945 | Train Loss Unweighted: 0.5396 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1196 | Classification: 0.0957 | Classification_binary: 0.1912 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1247 | Regression_eta: 0.0015 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.2052 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0632 | Regression_eta: 0.0520 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0743 | LR: 4.94e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.1529s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7531 | max_loss: 0.1689 | max_backward: 1.5554 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6017 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7214.1 | uss_MB: 6877.0 | child_rss_MB: 4792.2 | allocated_MB: 345.3 | reserved_MB: 23694.0 | inactive_split_MB: 251.5 +rank6 [2026-10-29 06:15:06,393] INFO : Running validation on rank6 +rank6 [2026-10-29 06:15:06,394] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 06:15:06,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 06:16:00,133] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:16:00,711] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 06:16:29,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 06:16:30,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 06:16:31,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 06:16:32,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 06:16:42,860] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 06:16:43,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:16:43,720] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 06:16:59,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 06:17:00,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 06:17:01,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 06:17:02,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 06:17:12,339] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 06:17:13,249] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:17:13,778] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 06:17:29,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 06:17:29,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 06:17:30,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 06:17:30,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 06:17:40,964] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 06:21:51,976] INFO : Step 27100/50000 rank6 | Train Loss: 1.1577 | Train Loss Unweighted: 0.5595 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1497 | Classification: 0.0884 | Classification_binary: 0.1879 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1399 | Regression_eta: 0.0018 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1896 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0617 | Regression_eta: 0.0445 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0488 | LR: 4.91e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.0503s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8543 | max_loss: 0.1477 | max_backward: 1.8762 | max_optimizer_step: 0.2540 | max_tensorboard_logging: 0.9462 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 8306.7 | uss_MB: 7966.7 | child_rss_MB: 4814.1 | allocated_MB: 342.9 | reserved_MB: 21640.0 | inactive_split_MB: 268.0 +rank6 [2026-10-29 06:25:39,973] INFO : Step 27200/50000 rank6 | Train Loss: 1.0028 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1065 | Classification: 0.0834 | Classification_binary: 0.1354 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1443 | Regression_eta: 0.0015 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0705 | Regression_eta: 0.0297 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0504 | LR: 4.88e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.0897s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6839 | max_loss: 0.1724 | max_backward: 1.0628 | max_optimizer_step: 0.2515 | max_tensorboard_logging: 0.9641 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0689 | Memory: rss_MB: 8305.7 | uss_MB: 7964.4 | child_rss_MB: 4815.8 | allocated_MB: 340.5 | reserved_MB: 22612.0 | inactive_split_MB: 267.2 +rank6 [2026-10-29 06:29:18,756] INFO : Step 27300/50000 rank6 | Train Loss: 1.1913 | Train Loss Unweighted: 0.5857 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1563 | Classification: 0.1169 | Classification_binary: 0.1769 | HitGrouping: 0.1965 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1426 | Regression_eta: 0.0019 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.1919 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0737 | Regression_eta: 0.1020 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0994 | LR: 4.85e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2490s | Phase Times: max_pre_step_barrier: 0.0244 | max_batch_to_device: 0.0064 | max_forward: 0.6813 | max_loss: 0.1454 | max_backward: 1.4618 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7699 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 7290.8 | uss_MB: 6952.0 | child_rss_MB: 4811.5 | allocated_MB: 345.6 | reserved_MB: 23586.0 | inactive_split_MB: 267.9 +rank6 [2026-10-29 06:33:03,427] INFO : Step 27400/50000 rank6 | Train Loss: 1.1723 | Train Loss Unweighted: 0.5888 | Train Loss Components: Auxiliary: 0.2572 | Cardinality: 0.1400 | Classification: 0.1088 | Classification_binary: 0.2016 | HitGrouping: 0.1987 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1370 | Regression_eta: 0.0017 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1785 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0865 | Regression_eta: 0.0348 | Regression_pt: 0.0901 | Regression_sin_phi: 0.0573 | LR: 4.82e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6657s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8048 | max_loss: 0.1475 | max_backward: 1.6655 | max_optimizer_step: 0.2279 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7292.0 | uss_MB: 6951.9 | child_rss_MB: 4814.0 | allocated_MB: 344.5 | reserved_MB: 23586.0 | inactive_split_MB: 279.5 +rank6 [2026-10-29 06:37:01,482] INFO : Step 27500/50000 rank6 | Train Loss: 1.0739 | Train Loss Unweighted: 0.5273 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1251 | Classification: 0.0933 | Classification_binary: 0.1655 | HitGrouping: 0.1957 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0632 | Regression_eta: 0.0581 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0744 | LR: 4.80e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2224s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7686 | max_loss: 0.1542 | max_backward: 1.3750 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.8628 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 7288.8 | uss_MB: 6945.7 | child_rss_MB: 4804.4 | allocated_MB: 343.6 | reserved_MB: 23586.0 | inactive_split_MB: 241.4 +rank6 [2026-10-29 06:40:55,531] INFO : Step 27600/50000 rank6 | Train Loss: 1.0507 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1163 | Classification: 0.1024 | Classification_binary: 0.1467 | HitGrouping: 0.1992 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1210 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0640 | Regression_eta: 0.0281 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0576 | LR: 4.77e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4899s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0060 | max_forward: 0.6744 | max_loss: 0.2492 | max_backward: 1.6867 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.5257 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 7290.8 | uss_MB: 6853.7 | child_rss_MB: 4808.0 | allocated_MB: 341.2 | reserved_MB: 24480.0 | inactive_split_MB: 234.8 +rank6 [2026-10-29 06:44:47,870] INFO : Step 27700/50000 rank6 | Train Loss: 1.1881 | Train Loss Unweighted: 0.5834 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1544 | Classification: 0.1001 | Classification_binary: 0.1885 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1444 | Regression_eta: 0.0015 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0778 | Regression_eta: 0.0615 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0774 | LR: 4.74e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.2899s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7716 | max_loss: 0.1527 | max_backward: 1.4941 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 0.7272 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7291.1 | uss_MB: 6952.7 | child_rss_MB: 4804.6 | allocated_MB: 344.5 | reserved_MB: 24480.0 | inactive_split_MB: 194.2 +rank6 [2026-10-29 06:48:38,928] INFO : Step 27800/50000 rank6 | Train Loss: 1.1903 | Train Loss Unweighted: 0.5862 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1435 | Classification: 0.1032 | Classification_binary: 0.2006 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1387 | Regression_eta: 0.0018 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1796 | Regression_cos_phi: 0.0802 | Regression_energy: 0.0594 | Regression_eta: 0.0860 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0618 | LR: 4.71e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1869s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8137 | max_loss: 0.1509 | max_backward: 1.4748 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6041 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7291.1 | uss_MB: 6953.1 | child_rss_MB: 4808.7 | allocated_MB: 348.4 | reserved_MB: 24592.0 | inactive_split_MB: 226.8 +rank6 [2026-10-29 06:52:39,105] INFO : Step 27900/50000 rank6 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5521 | Train Loss Components: Auxiliary: 0.2284 | Cardinality: 0.1139 | Classification: 0.0902 | Classification_binary: 0.1544 | HitGrouping: 0.2057 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1519 | Regression_eta: 0.0017 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0650 | Regression_eta: 0.0693 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0430 | LR: 4.68e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6306s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6397 | max_loss: 0.2456 | max_backward: 1.5234 | max_optimizer_step: 0.2276 | max_tensorboard_logging: 1.1626 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 7291.1 | uss_MB: 6955.2 | child_rss_MB: 4805.2 | allocated_MB: 342.4 | reserved_MB: 25612.0 | inactive_split_MB: 259.0 +rank6 [2026-10-29 06:56:30,759] INFO : Step 28000/50000 rank6 | Train Loss: 1.1111 | Train Loss Unweighted: 0.5401 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1274 | Classification: 0.1035 | Classification_binary: 0.1699 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0754 | Regression_eta: 0.0293 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0572 | LR: 4.66e-04 | DataLoad Time: 0.0222s | Model Forward Time: 5.0933s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.7503 | max_loss: 0.1492 | max_backward: 1.7566 | max_optimizer_step: 0.2764 | max_tensorboard_logging: 1.2295 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 7291.2 | uss_MB: 5984.8 | child_rss_MB: 4794.8 | allocated_MB: 344.1 | reserved_MB: 25612.0 | inactive_split_MB: 243.2 +rank6 [2026-10-29 06:56:31,846] INFO : Running validation on rank6 +rank6 [2026-10-29 06:56:31,846] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 06:56:31,846] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 06:57:25,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:57:25,561] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 06:57:52,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 06:57:53,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 06:57:54,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 06:57:55,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 06:58:05,195] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 06:58:05,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:58:06,179] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 06:58:22,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 06:58:23,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 06:58:23,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 06:58:24,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 06:58:34,949] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 06:58:35,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:58:35,800] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 06:58:51,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 06:58:51,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 06:58:52,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 06:58:53,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 06:59:03,081] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 07:03:11,252] INFO : Step 28100/50000 rank6 | Train Loss: 1.1727 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1644 | Classification: 0.0934 | Classification_binary: 0.1929 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1365 | Regression_eta: 0.0015 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.2193 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0757 | Regression_eta: 0.0668 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0852 | LR: 4.63e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9498s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8425 | max_loss: 0.2571 | max_backward: 1.8157 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 1.0134 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 8340.2 | uss_MB: 8005.6 | child_rss_MB: 4802.9 | allocated_MB: 344.8 | reserved_MB: 22140.0 | inactive_split_MB: 293.2 +rank6 [2026-10-29 07:07:01,163] INFO : Step 28200/50000 rank6 | Train Loss: 1.0520 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2296 | Cardinality: 0.1203 | Classification: 0.0934 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1466 | Regression_eta: 0.0014 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0649 | Regression_eta: 0.0258 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0500 | LR: 4.60e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3555s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7943 | max_loss: 0.1496 | max_backward: 1.4503 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 0.7872 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 8343.2 | uss_MB: 8006.4 | child_rss_MB: 4796.0 | allocated_MB: 341.0 | reserved_MB: 24034.0 | inactive_split_MB: 210.4 +rank6 [2026-10-29 07:10:52,728] INFO : Step 28300/50000 rank6 | Train Loss: 1.0602 | Train Loss Unweighted: 0.5453 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1043 | Classification: 0.0923 | Classification_binary: 0.1501 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1496 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1352 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0942 | Regression_eta: 0.0400 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0468 | LR: 4.57e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2441s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6482 | max_loss: 0.2266 | max_backward: 1.5658 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.6870 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0836 | Memory: rss_MB: 7393.0 | uss_MB: 7059.5 | child_rss_MB: 4787.3 | allocated_MB: 342.7 | reserved_MB: 25016.0 | inactive_split_MB: 243.9 +rank6 [2026-10-29 07:14:40,573] INFO : Step 28400/50000 rank6 | Train Loss: 1.1507 | Train Loss Unweighted: 0.5391 | Train Loss Components: Auxiliary: 0.2436 | Cardinality: 0.1500 | Classification: 0.0965 | Classification_binary: 0.1896 | HitGrouping: 0.1911 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1241 | Regression_eta: 0.0018 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1636 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0527 | Regression_eta: 0.0465 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0447 | LR: 4.55e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6288s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7664 | max_loss: 0.1532 | max_backward: 1.7342 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.8455 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7394.0 | uss_MB: 7059.9 | child_rss_MB: 4785.1 | allocated_MB: 344.2 | reserved_MB: 25016.0 | inactive_split_MB: 278.9 +rank6 [2026-10-29 07:18:29,001] INFO : Step 28500/50000 rank6 | Train Loss: 1.1595 | Train Loss Unweighted: 0.5640 | Train Loss Components: Auxiliary: 0.2492 | Cardinality: 0.1556 | Classification: 0.0927 | Classification_binary: 0.1908 | HitGrouping: 0.1954 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1384 | Regression_eta: 0.0015 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1983 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0785 | Regression_eta: 0.0395 | Regression_pt: 0.0821 | Regression_sin_phi: 0.0740 | LR: 4.52e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2720s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.8033 | max_loss: 0.1665 | max_backward: 1.5231 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.6178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0709 | Memory: rss_MB: 7393.5 | uss_MB: 7056.4 | child_rss_MB: 4808.7 | allocated_MB: 347.4 | reserved_MB: 26000.0 | inactive_split_MB: 283.2 +rank6 [2026-10-29 07:22:17,773] INFO : Step 28600/50000 rank6 | Train Loss: 1.0672 | Train Loss Unweighted: 0.5222 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1172 | Classification: 0.0893 | Classification_binary: 0.1544 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.1687 | Regression_cos_phi: 0.1143 | Regression_energy: 0.0676 | Regression_eta: 0.0492 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0792 | LR: 4.49e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.0211s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6704 | max_loss: 0.1829 | max_backward: 1.2353 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.8074 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7396.5 | uss_MB: 7057.8 | child_rss_MB: 4793.1 | allocated_MB: 342.3 | reserved_MB: 26000.0 | inactive_split_MB: 243.7 +rank6 [2026-10-29 07:26:10,155] INFO : Step 28700/50000 rank6 | Train Loss: 1.1535 | Train Loss Unweighted: 0.5811 | Train Loss Components: Auxiliary: 0.2469 | Cardinality: 0.1254 | Classification: 0.1220 | Classification_binary: 0.1661 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1442 | Regression_eta: 0.0021 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0642 | Regression_eta: 0.0430 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0592 | LR: 4.46e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5823s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6905 | max_loss: 0.2072 | max_backward: 1.6876 | max_optimizer_step: 0.2353 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 7397.5 | uss_MB: 7064.1 | child_rss_MB: 4802.4 | allocated_MB: 343.5 | reserved_MB: 26000.0 | inactive_split_MB: 234.1 +rank6 [2026-10-29 07:30:12,151] INFO : Step 28800/50000 rank6 | Train Loss: 1.2131 | Train Loss Unweighted: 0.5739 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1788 | Classification: 0.1057 | Classification_binary: 0.2127 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2049 | Regression_cos_phi: 0.1052 | Regression_energy: 0.0624 | Regression_eta: 0.0418 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0816 | LR: 4.44e-04 | DataLoad Time: 0.0105s | Model Forward Time: 6.8371s | Phase Times: max_pre_step_barrier: 0.0263 | max_batch_to_device: 0.0149 | max_forward: 1.2928 | max_loss: 0.2713 | max_backward: 3.0161 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.8895 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 7398.3 | uss_MB: 5722.3 | child_rss_MB: 4796.9 | allocated_MB: 343.8 | reserved_MB: 26000.0 | inactive_split_MB: 273.3 +rank6 [2026-10-29 07:34:11,570] INFO : Step 28900/50000 rank6 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1252 | Classification: 0.0967 | Classification_binary: 0.1637 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1509 | Regression_eta: 0.0015 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0871 | Regression_eta: 0.0406 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0778 | LR: 4.41e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.1529s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7512 | max_loss: 0.2189 | max_backward: 1.6229 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.9330 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7399.6 | uss_MB: 5634.1 | child_rss_MB: 4800.1 | allocated_MB: 339.6 | reserved_MB: 26000.0 | inactive_split_MB: 223.0 +rank6 [2026-10-29 07:38:06,080] INFO : Step 29000/50000 rank6 | Train Loss: 1.0238 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1070 | Classification: 0.0861 | Classification_binary: 0.1439 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1389 | Regression_eta: 0.0014 | Regression_pt: 0.1414 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1440 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0618 | Regression_eta: 0.0451 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0634 | LR: 4.38e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9822s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.6636 | max_loss: 0.2429 | max_backward: 1.5394 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.5216 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7400.5 | uss_MB: 7067.2 | child_rss_MB: 4794.8 | allocated_MB: 345.2 | reserved_MB: 26984.0 | inactive_split_MB: 235.2 +rank6 [2026-10-29 07:38:07,029] INFO : Running validation on rank6 +rank6 [2026-10-29 07:38:07,030] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 07:38:07,030] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 07:38:57,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:38:58,451] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 07:39:24,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 07:39:25,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 07:39:26,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 07:39:27,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 07:39:37,735] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 07:39:38,026] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:39:38,577] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 07:39:54,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 07:39:55,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 07:39:56,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 07:39:57,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 07:40:07,608] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 07:40:07,779] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:40:08,277] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 07:40:23,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 07:40:24,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 07:40:24,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 07:40:25,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 07:40:35,453] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 07:45:01,997] INFO : Step 29100/50000 rank6 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5808 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1279 | Classification: 0.1113 | Classification_binary: 0.1973 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1339 | Regression_eta: 0.0017 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0809 | Regression_eta: 0.0550 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0931 | LR: 4.35e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8720s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.8060 | max_loss: 0.1503 | max_backward: 1.9210 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 8379.4 | uss_MB: 8043.2 | child_rss_MB: 4787.2 | allocated_MB: 344.6 | reserved_MB: 24404.0 | inactive_split_MB: 274.9 +rank6 [2026-10-29 07:48:58,054] INFO : Step 29200/50000 rank6 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5633 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1265 | Classification: 0.0969 | Classification_binary: 0.1883 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1368 | Regression_eta: 0.0014 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2154 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0624 | Regression_eta: 0.0406 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0720 | LR: 4.33e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.8377s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0742 | max_forward: 1.0303 | max_loss: 0.2183 | max_backward: 1.8745 | max_optimizer_step: 0.2501 | max_tensorboard_logging: 0.5570 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8379.4 | uss_MB: 8042.7 | child_rss_MB: 4798.0 | allocated_MB: 343.1 | reserved_MB: 24404.0 | inactive_split_MB: 228.1 +rank6 [2026-10-29 07:52:51,792] INFO : Step 29300/50000 rank6 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2330 | Cardinality: 0.1231 | Classification: 0.0898 | Classification_binary: 0.1516 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1428 | Regression_eta: 0.0017 | Regression_pt: 0.1453 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1772 | Regression_cos_phi: 0.1025 | Regression_energy: 0.0697 | Regression_eta: 0.0537 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0738 | LR: 4.30e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.7358s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0067 | max_forward: 0.7139 | max_loss: 0.1578 | max_backward: 1.5096 | max_optimizer_step: 0.2586 | max_tensorboard_logging: 1.1324 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7473.7 | uss_MB: 6607.6 | child_rss_MB: 4797.1 | allocated_MB: 341.0 | reserved_MB: 24404.0 | inactive_split_MB: 247.0 +rank6 [2026-10-29 07:56:44,366] INFO : Step 29400/50000 rank6 | Train Loss: 1.1400 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1377 | Classification: 0.1135 | Classification_binary: 0.1661 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1386 | Regression_eta: 0.0019 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1550 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0822 | Regression_eta: 0.0395 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0861 | LR: 4.27e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.2872s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.6285 | max_loss: 0.1485 | max_backward: 1.3787 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.9179 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7474.7 | uss_MB: 7139.2 | child_rss_MB: 4788.9 | allocated_MB: 343.3 | reserved_MB: 24404.0 | inactive_split_MB: 277.1 +rank6 [2026-10-29 08:00:58,644] INFO : Step 29500/50000 rank6 | Train Loss: 1.1987 | Train Loss Unweighted: 0.6106 | Train Loss Components: Auxiliary: 0.2551 | Cardinality: 0.1277 | Classification: 0.1201 | Classification_binary: 0.2013 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1419 | Regression_eta: 0.0019 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0883 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0735 | Regression_eta: 0.0515 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0644 | LR: 4.24e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.8970s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0749 | max_forward: 0.8153 | max_loss: 0.3465 | max_backward: 2.8565 | max_optimizer_step: 0.3442 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7477.7 | uss_MB: 7142.6 | child_rss_MB: 4792.1 | allocated_MB: 343.9 | reserved_MB: 24404.0 | inactive_split_MB: 271.5 +rank6 [2026-10-29 08:05:08,852] INFO : Step 29600/50000 rank6 | Train Loss: 1.1065 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2391 | Cardinality: 0.1179 | Classification: 0.0927 | Classification_binary: 0.1666 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0018 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1698 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0548 | Regression_eta: 0.0422 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0756 | LR: 4.22e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4744s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8025 | max_loss: 0.1485 | max_backward: 1.4674 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.8982 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 7476.5 | uss_MB: 7138.1 | child_rss_MB: 4790.4 | allocated_MB: 340.8 | reserved_MB: 24404.0 | inactive_split_MB: 290.7 +rank6 [2026-10-29 08:09:13,801] INFO : Step 29700/50000 rank6 | Train Loss: 1.0488 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2273 | Cardinality: 0.1414 | Classification: 0.0952 | Classification_binary: 0.1459 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1231 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0598 | Regression_eta: 0.0276 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0305 | LR: 4.19e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5177s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0057 | max_forward: 0.7342 | max_loss: 0.1652 | max_backward: 1.4907 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.9743 | max_tensorboard_flush: 0.0140 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7477.5 | uss_MB: 6998.2 | child_rss_MB: 4799.0 | allocated_MB: 342.3 | reserved_MB: 24404.0 | inactive_split_MB: 236.6 +rank6 [2026-10-29 08:13:32,078] INFO : Step 29800/50000 rank6 | Train Loss: 1.1686 | Train Loss Unweighted: 0.5644 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1551 | Classification: 0.0952 | Classification_binary: 0.1922 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1357 | Regression_eta: 0.0014 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0834 | Classification_binary: 0.2122 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0727 | Regression_eta: 0.0377 | Regression_pt: 0.0752 | Regression_sin_phi: 0.1000 | LR: 4.16e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6765s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7294 | max_loss: 0.1685 | max_backward: 1.6547 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9409 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7478.5 | uss_MB: 7143.5 | child_rss_MB: 4794.4 | allocated_MB: 345.0 | reserved_MB: 24404.0 | inactive_split_MB: 265.1 +rank6 [2026-10-29 08:17:37,374] INFO : Step 29900/50000 rank6 | Train Loss: 1.1305 | Train Loss Unweighted: 0.5666 | Train Loss Components: Auxiliary: 0.2471 | Cardinality: 0.1380 | Classification: 0.0944 | Classification_binary: 0.1750 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1464 | Regression_eta: 0.0015 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1985 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0811 | Regression_eta: 0.0576 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0704 | LR: 4.14e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3505s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0611 | max_forward: 0.8060 | max_loss: 0.1494 | max_backward: 1.3251 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 7479.5 | uss_MB: 7143.0 | child_rss_MB: 4800.7 | allocated_MB: 344.7 | reserved_MB: 26422.0 | inactive_split_MB: 298.4 +rank6 [2026-10-29 08:22:04,988] INFO : Step 30000/50000 rank6 | Train Loss: 1.0854 | Train Loss Unweighted: 0.5586 | Train Loss Components: Auxiliary: 0.2416 | Cardinality: 0.1337 | Classification: 0.1000 | Classification_binary: 0.1541 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1499 | Regression_eta: 0.0014 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1324 | Regression_cos_phi: 0.0300 | Regression_energy: 0.0662 | Regression_eta: 0.0168 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0273 | LR: 4.11e-04 | DataLoad Time: 0.0195s | Model Forward Time: 7.0740s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0172 | max_forward: 0.9685 | max_loss: 0.2166 | max_backward: 4.0112 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.8430 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7461.9 | uss_MB: 7142.5 | child_rss_MB: 4797.1 | allocated_MB: 338.0 | reserved_MB: 26422.0 | inactive_split_MB: 288.0 +rank6 [2026-10-29 08:22:05,952] INFO : Running validation on rank6 +rank6 [2026-10-29 08:22:05,952] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 08:22:05,953] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 08:22:57,875] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:22:58,358] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 08:23:24,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 08:23:25,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 08:23:26,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 08:23:28,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 08:23:38,085] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 08:23:38,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:23:38,901] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 08:23:55,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 08:23:56,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 08:23:56,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 08:23:57,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 08:24:07,900] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 08:24:08,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:24:08,615] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 08:24:24,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 08:24:24,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 08:24:25,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 08:24:26,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 08:24:36,228] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 08:29:00,406] INFO : Step 30100/50000 rank6 | Train Loss: 1.0762 | Train Loss Unweighted: 0.5358 | Train Loss Components: Auxiliary: 0.2378 | Cardinality: 0.1106 | Classification: 0.1097 | Classification_binary: 0.1679 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0946 | Regression_energy: 0.0631 | Regression_eta: 0.0474 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0942 | LR: 4.08e-04 | DataLoad Time: 0.0171s | Model Forward Time: 8.2909s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7352 | max_loss: 0.1443 | max_backward: 1.7375 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 1.7731 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.3497 | Memory: rss_MB: 7528.6 | uss_MB: 6331.6 | child_rss_MB: 4801.2 | allocated_MB: 344.1 | reserved_MB: 22922.0 | inactive_split_MB: 248.8 +rank6 [2026-10-29 08:33:16,460] INFO : Step 30200/50000 rank6 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5304 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1421 | Classification: 0.0934 | Classification_binary: 0.1839 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0016 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1635 | Regression_cos_phi: 0.0317 | Regression_energy: 0.0539 | Regression_eta: 0.0296 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0414 | LR: 4.06e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5996s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8008 | max_loss: 0.1560 | max_backward: 1.6706 | max_optimizer_step: 0.2663 | max_tensorboard_logging: 0.7725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0555 | Memory: rss_MB: 7535.6 | uss_MB: 7212.9 | child_rss_MB: 4775.8 | allocated_MB: 344.1 | reserved_MB: 23882.0 | inactive_split_MB: 274.8 +rank6 [2026-10-29 08:37:30,482] INFO : Step 30300/50000 rank6 | Train Loss: 1.0429 | Train Loss Unweighted: 0.5252 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1223 | Classification: 0.0922 | Classification_binary: 0.1566 | HitGrouping: 0.1929 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1806 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0634 | Regression_eta: 0.0488 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0806 | LR: 4.03e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.6276s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0060 | max_forward: 1.1719 | max_loss: 0.2884 | max_backward: 2.4994 | max_optimizer_step: 0.2255 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7536.6 | uss_MB: 7213.9 | child_rss_MB: 4771.6 | allocated_MB: 341.8 | reserved_MB: 23882.0 | inactive_split_MB: 270.7 +rank6 [2026-10-29 08:41:40,106] INFO : Step 30400/50000 rank6 | Train Loss: 1.0543 | Train Loss Unweighted: 0.5182 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1318 | Classification: 0.0889 | Classification_binary: 0.1487 | HitGrouping: 0.1973 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1382 | Regression_eta: 0.0014 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1393 | Regression_cos_phi: 0.0347 | Regression_energy: 0.0849 | Regression_eta: 0.0376 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0378 | LR: 4.00e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.0643s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0150 | max_forward: 0.8845 | max_loss: 0.2482 | max_backward: 2.1745 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.5112 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7452.2 | uss_MB: 7214.6 | child_rss_MB: 4475.9 | allocated_MB: 344.7 | reserved_MB: 24842.0 | inactive_split_MB: 245.3 +rank6 [2026-10-29 08:45:49,048] INFO : Step 30500/50000 rank6 | Train Loss: 1.1589 | Train Loss Unweighted: 0.5596 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1407 | Classification: 0.1085 | Classification_binary: 0.1952 | HitGrouping: 0.1960 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1801 | Regression_cos_phi: 0.0968 | Regression_energy: 0.0595 | Regression_eta: 0.0532 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0816 | LR: 3.98e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.7054s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7791 | max_loss: 0.2528 | max_backward: 1.9433 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.5210 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0672 | Memory: rss_MB: 7432.7 | uss_MB: 7008.9 | child_rss_MB: 4374.7 | allocated_MB: 346.2 | reserved_MB: 26806.0 | inactive_split_MB: 227.8 +rank6 [2026-10-29 08:49:58,495] INFO : Step 30600/50000 rank6 | Train Loss: 1.1269 | Train Loss Unweighted: 0.5561 | Train Loss Components: Auxiliary: 0.2414 | Cardinality: 0.1456 | Classification: 0.0973 | Classification_binary: 0.1804 | HitGrouping: 0.1953 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1373 | Regression_eta: 0.0014 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0624 | Regression_eta: 0.0517 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0751 | LR: 3.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4773s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1011 | max_forward: 0.8824 | max_loss: 0.1534 | max_backward: 1.4258 | max_optimizer_step: 0.3067 | max_tensorboard_logging: 0.7925 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 7378.2 | uss_MB: 7215.1 | child_rss_MB: 4177.7 | allocated_MB: 345.8 | reserved_MB: 27800.0 | inactive_split_MB: 283.5 +rank6 [2026-10-29 08:54:17,062] INFO : Step 30700/50000 rank6 | Train Loss: 1.0989 | Train Loss Unweighted: 0.5515 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1377 | Classification: 0.0890 | Classification_binary: 0.1568 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0013 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0324 | Classification_binary: 0.1585 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0643 | Regression_eta: 0.0334 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0640 | LR: 3.92e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2937s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0056 | max_forward: 0.7546 | max_loss: 0.2396 | max_backward: 1.3389 | max_optimizer_step: 0.2508 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0572 | Memory: rss_MB: 7380.2 | uss_MB: 7216.5 | child_rss_MB: 4178.9 | allocated_MB: 340.3 | reserved_MB: 27800.0 | inactive_split_MB: 223.2 +rank6 [2026-10-29 08:58:31,923] INFO : Step 30800/50000 rank6 | Train Loss: 1.1241 | Train Loss Unweighted: 0.5488 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1377 | Classification: 0.1081 | Classification_binary: 0.1717 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1317 | Regression_eta: 0.0015 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1471 | Regression_cos_phi: 0.0390 | Regression_energy: 0.0564 | Regression_eta: 0.0395 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0567 | LR: 3.90e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.8020s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7775 | max_loss: 0.2735 | max_backward: 1.9298 | max_optimizer_step: 0.3489 | max_tensorboard_logging: 0.8756 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0726 | Memory: rss_MB: 7357.4 | uss_MB: 5945.6 | child_rss_MB: 4116.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 275.1 +rank6 [2026-10-29 09:02:40,910] INFO : Step 30900/50000 rank6 | Train Loss: 1.1501 | Train Loss Unweighted: 0.5762 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1300 | Classification: 0.0976 | Classification_binary: 0.1996 | HitGrouping: 0.2000 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1374 | Regression_eta: 0.0017 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1890 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0590 | Regression_eta: 0.0421 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0562 | LR: 3.87e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.5953s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.3511 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 1.0306 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 7360.4 | uss_MB: 7217.8 | child_rss_MB: 4118.5 | allocated_MB: 342.2 | reserved_MB: 27800.0 | inactive_split_MB: 198.3 +rank6 [2026-10-29 09:06:46,924] INFO : Step 31000/50000 rank6 | Train Loss: 1.1054 | Train Loss Unweighted: 0.5218 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1446 | Classification: 0.0888 | Classification_binary: 0.1693 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0016 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0592 | Regression_eta: 0.0464 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0670 | LR: 3.84e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.7143s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.8505 | max_loss: 0.2491 | max_backward: 1.6460 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9731 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7337.9 | uss_MB: 6674.8 | child_rss_MB: 4046.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 190.7 +rank6 [2026-10-29 09:06:47,903] INFO : Running validation on rank6 +rank6 [2026-10-29 09:06:47,903] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 09:06:47,903] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 09:07:39,008] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:07:39,556] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 09:08:08,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 09:08:09,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 09:08:10,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 09:08:11,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 09:08:21,544] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 09:08:22,017] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:08:22,479] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 09:08:38,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 09:08:40,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 09:08:41,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 09:08:42,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 09:08:52,824] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 09:08:52,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:08:53,432] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 09:09:08,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 09:09:09,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 09:09:10,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 09:09:10,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 09:09:20,794] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 09:13:45,022] INFO : Step 31100/50000 rank6 | Train Loss: 1.0306 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2182 | Cardinality: 0.1238 | Classification: 0.0932 | Classification_binary: 0.1415 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1339 | Regression_cos_phi: 0.0344 | Regression_energy: 0.0680 | Regression_eta: 0.0273 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0610 | LR: 3.82e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.5118s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1068 | max_forward: 0.8067 | max_loss: 0.1436 | max_backward: 1.1427 | max_optimizer_step: 0.2576 | max_tensorboard_logging: 1.1475 | max_tensorboard_flush: 0.0054 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 8196.3 | uss_MB: 5921.9 | child_rss_MB: 4032.2 | allocated_MB: 343.8 | reserved_MB: 24492.0 | inactive_split_MB: 293.3 +rank6 [2026-10-29 09:17:44,911] INFO : Step 31200/50000 rank6 | Train Loss: 1.1102 | Train Loss Unweighted: 0.5369 | Train Loss Components: Auxiliary: 0.2472 | Cardinality: 0.1495 | Classification: 0.0959 | Classification_binary: 0.1708 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1328 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1645 | Regression_cos_phi: 0.0996 | Regression_energy: 0.0708 | Regression_eta: 0.0546 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0404 | LR: 3.79e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.1830s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7127 | max_loss: 0.1476 | max_backward: 1.5707 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7428.7 | uss_MB: 7313.8 | child_rss_MB: 4003.5 | allocated_MB: 344.5 | reserved_MB: 24648.0 | inactive_split_MB: 235.5 +rank6 [2026-10-29 09:21:47,087] INFO : Step 31300/50000 rank6 | Train Loss: 1.1865 | Train Loss Unweighted: 0.5472 | Train Loss Components: Auxiliary: 0.2511 | Cardinality: 0.1748 | Classification: 0.0979 | Classification_binary: 0.1851 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0019 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1975 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0581 | Regression_eta: 0.0433 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0558 | LR: 3.76e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2306s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0064 | max_forward: 0.7815 | max_loss: 0.1495 | max_backward: 1.6330 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.5234 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7366.1 | uss_MB: 7313.9 | child_rss_MB: 3706.8 | allocated_MB: 344.4 | reserved_MB: 25640.0 | inactive_split_MB: 233.6 +rank6 [2026-10-29 09:25:42,888] INFO : Step 31400/50000 rank6 | Train Loss: 1.0272 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1191 | Classification: 0.0784 | Classification_binary: 0.1451 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0574 | Regression_eta: 0.0285 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0468 | LR: 3.74e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.5256s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0063 | max_forward: 0.7252 | max_loss: 0.1478 | max_backward: 1.6173 | max_optimizer_step: 0.2404 | max_tensorboard_logging: 0.9541 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7335.5 | uss_MB: 7312.2 | child_rss_MB: 3625.5 | allocated_MB: 342.1 | reserved_MB: 25640.0 | inactive_split_MB: 296.4 +rank6 [2026-10-29 09:29:44,710] INFO : Step 31500/50000 rank6 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5247 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1206 | Classification: 0.1037 | Classification_binary: 0.1583 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1297 | Regression_eta: 0.0014 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0812 | Regression_eta: 0.0435 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0624 | LR: 3.71e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3656s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6643 | max_loss: 0.2487 | max_backward: 1.5648 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.8248 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7336.0 | uss_MB: 7313.0 | child_rss_MB: 3638.4 | allocated_MB: 344.4 | reserved_MB: 26092.0 | inactive_split_MB: 238.2 +rank6 [2026-10-29 09:33:42,078] INFO : Step 31600/50000 rank6 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5471 | Train Loss Components: Auxiliary: 0.2424 | Cardinality: 0.1284 | Classification: 0.1033 | Classification_binary: 0.1828 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1282 | Regression_eta: 0.0017 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0831 | Classification_binary: 0.1921 | Regression_cos_phi: 0.0847 | Regression_energy: 0.0570 | Regression_eta: 0.0634 | Regression_pt: 0.0602 | Regression_sin_phi: 0.1192 | LR: 3.69e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.0244s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7603 | max_loss: 0.1471 | max_backward: 2.0475 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.9076 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7328.4 | uss_MB: 7314.4 | child_rss_MB: 3585.6 | allocated_MB: 341.5 | reserved_MB: 26640.0 | inactive_split_MB: 287.0 +rank6 [2026-10-29 09:37:47,825] INFO : Step 31700/50000 rank6 | Train Loss: 1.0533 | Train Loss Unweighted: 0.5009 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1416 | Classification: 0.0871 | Classification_binary: 0.1632 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1931 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0640 | Regression_eta: 0.0613 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0889 | LR: 3.66e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1120s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7635 | max_loss: 0.1543 | max_backward: 1.2075 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7330.4 | uss_MB: 7312.9 | child_rss_MB: 3557.0 | allocated_MB: 341.9 | reserved_MB: 26640.0 | inactive_split_MB: 274.1 +rank6 [2026-10-29 09:45:40,561] INFO : Step 31800/50000 rank6 | Train Loss: 1.0385 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1148 | Classification: 0.0877 | Classification_binary: 0.1481 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1418 | Regression_eta: 0.0012 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0668 | Regression_eta: 0.0275 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0337 | LR: 3.64e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0347s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0072 | max_forward: 0.6606 | max_loss: 0.1610 | max_backward: 1.1159 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.8258 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1417 | Memory: rss_MB: 7328.9 | uss_MB: 7312.1 | child_rss_MB: 3561.3 | allocated_MB: 345.5 | reserved_MB: 27080.0 | inactive_split_MB: 260.5 +rank6 [2026-10-29 09:49:44,089] INFO : Step 31900/50000 rank6 | Train Loss: 1.1464 | Train Loss Unweighted: 0.5585 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1372 | Classification: 0.1078 | Classification_binary: 0.1815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0017 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0679 | Regression_eta: 0.0394 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0620 | LR: 3.61e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.5141s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7718 | max_loss: 0.1553 | max_backward: 1.7857 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.6721 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7333.9 | uss_MB: 7320.8 | child_rss_MB: 3569.3 | allocated_MB: 343.8 | reserved_MB: 27080.0 | inactive_split_MB: 280.2 +rank6 [2026-10-29 09:53:38,294] INFO : Step 32000/50000 rank6 | Train Loss: 1.1115 | Train Loss Unweighted: 0.5351 | Train Loss Components: Auxiliary: 0.2393 | Cardinality: 0.1406 | Classification: 0.0954 | Classification_binary: 0.1748 | HitGrouping: 0.1912 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1302 | Regression_eta: 0.0018 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1768 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0724 | Regression_eta: 0.0275 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0453 | LR: 3.58e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.2443s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.9767 | max_loss: 0.6919 | max_backward: 2.1955 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.9277 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 7329.3 | uss_MB: 7313.9 | child_rss_MB: 3566.4 | allocated_MB: 344.2 | reserved_MB: 27080.0 | inactive_split_MB: 265.7 +rank6 [2026-10-29 09:53:39,348] INFO : Running validation on rank6 +rank6 [2026-10-29 09:53:39,349] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 09:53:39,349] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 09:54:28,723] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:54:29,292] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 09:54:57,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 09:54:58,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 09:54:59,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 09:55:00,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 09:55:10,753] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 09:55:10,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:55:11,414] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 09:55:27,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 09:55:28,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 09:55:29,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 09:55:30,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 09:55:40,745] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 09:55:41,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:55:41,777] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 09:55:57,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 09:55:57,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 09:55:58,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 09:55:59,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 09:56:09,133] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 10:00:19,239] INFO : Step 32100/50000 rank6 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1218 | Classification: 0.0914 | Classification_binary: 0.1509 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1333 | Regression_eta: 0.0016 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1522 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0557 | Regression_eta: 0.0781 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0536 | LR: 3.56e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.3621s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7098 | max_loss: 0.1454 | max_backward: 1.1394 | max_optimizer_step: 0.2650 | max_tensorboard_logging: 0.9832 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 8404.9 | uss_MB: 8386.6 | child_rss_MB: 3573.9 | allocated_MB: 344.0 | reserved_MB: 22148.0 | inactive_split_MB: 276.8 +rank6 [2026-10-29 10:04:33,375] INFO : Step 32200/50000 rank6 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1241 | Classification: 0.1035 | Classification_binary: 0.1630 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1279 | Regression_eta: 0.0014 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0564 | Regression_eta: 0.0368 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0492 | LR: 3.53e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6703s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0891 | max_forward: 0.7517 | max_loss: 0.1441 | max_backward: 1.7835 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.7808 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7418.2 | uss_MB: 7403.1 | child_rss_MB: 3576.2 | allocated_MB: 345.5 | reserved_MB: 23122.0 | inactive_split_MB: 233.4 +rank6 [2026-10-29 10:08:31,625] INFO : Step 32300/50000 rank6 | Train Loss: 1.1438 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2535 | Cardinality: 0.1335 | Classification: 0.1042 | Classification_binary: 0.1958 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1303 | Regression_eta: 0.0015 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1962 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0642 | Regression_eta: 0.0435 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0505 | LR: 3.51e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.7821s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7857 | max_loss: 0.1780 | max_backward: 1.7580 | max_optimizer_step: 0.2428 | max_tensorboard_logging: 0.8954 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 7421.3 | uss_MB: 7404.5 | child_rss_MB: 3576.8 | allocated_MB: 343.9 | reserved_MB: 24108.0 | inactive_split_MB: 280.1 +rank6 [2026-10-29 10:12:37,147] INFO : Step 32400/50000 rank6 | Train Loss: 1.0306 | Train Loss Unweighted: 0.5063 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1168 | Classification: 0.0867 | Classification_binary: 0.1599 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0013 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0825 | Regression_eta: 0.0448 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0678 | LR: 3.48e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.3213s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.1456 | max_backward: 1.3401 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.8752 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7420.4 | uss_MB: 7401.9 | child_rss_MB: 3574.4 | allocated_MB: 344.8 | reserved_MB: 24108.0 | inactive_split_MB: 271.0 +rank6 [2026-10-29 10:16:48,485] INFO : Step 32500/50000 rank6 | Train Loss: 1.0033 | Train Loss Unweighted: 0.5126 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.0995 | Classification: 0.1015 | Classification_binary: 0.1402 | HitGrouping: 0.1898 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1332 | Regression_eta: 0.0015 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.1742 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0628 | Regression_eta: 0.0718 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0758 | LR: 3.46e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9946s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6088 | max_loss: 0.1480 | max_backward: 1.0729 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.8045 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7420.4 | uss_MB: 7401.8 | child_rss_MB: 3575.0 | allocated_MB: 341.9 | reserved_MB: 25096.0 | inactive_split_MB: 268.1 +rank6 [2026-10-29 10:21:04,066] INFO : Step 32600/50000 rank6 | Train Loss: 1.0981 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1271 | Classification: 0.1053 | Classification_binary: 0.1773 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1306 | Regression_eta: 0.0014 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.1658 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0649 | Regression_eta: 0.0401 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0635 | LR: 3.43e-04 | DataLoad Time: 0.0156s | Model Forward Time: 6.2967s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8529 | max_loss: 0.3369 | max_backward: 2.9831 | max_optimizer_step: 0.2386 | max_tensorboard_logging: 0.5557 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7420.7 | uss_MB: 5748.3 | child_rss_MB: 3585.6 | allocated_MB: 344.1 | reserved_MB: 25096.0 | inactive_split_MB: 287.9 +rank6 [2026-10-29 10:25:23,203] INFO : Step 32700/50000 rank6 | Train Loss: 1.0956 | Train Loss Unweighted: 0.5307 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.1282 | Classification: 0.1011 | Classification_binary: 0.1731 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0547 | Regression_eta: 0.0797 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0720 | LR: 3.41e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.1086s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0062 | max_forward: 0.8233 | max_loss: 0.1478 | max_backward: 1.5120 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0564 | Memory: rss_MB: 7420.7 | uss_MB: 7403.4 | child_rss_MB: 3588.8 | allocated_MB: 342.9 | reserved_MB: 25096.0 | inactive_split_MB: 272.6 +rank6 [2026-10-29 10:29:23,500] INFO : Step 32800/50000 rank6 | Train Loss: 1.0857 | Train Loss Unweighted: 0.5241 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1299 | Classification: 0.1011 | Classification_binary: 0.1551 | HitGrouping: 0.1982 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1327 | Regression_eta: 0.0016 | Regression_pt: 0.1319 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0910 | Regression_eta: 0.0362 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0515 | LR: 3.38e-04 | DataLoad Time: 0.0236s | Model Forward Time: 4.2378s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0179 | max_forward: 0.9787 | max_loss: 0.1782 | max_backward: 1.5075 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 0.5808 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.1577 | Memory: rss_MB: 7419.5 | uss_MB: 7401.5 | child_rss_MB: 3585.9 | allocated_MB: 345.1 | reserved_MB: 25096.0 | inactive_split_MB: 263.9 +rank6 [2026-10-29 10:33:26,397] INFO : Step 32900/50000 rank6 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1116 | Classification: 0.1066 | Classification_binary: 0.1631 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1300 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0397 | Regression_energy: 0.0629 | Regression_eta: 0.0826 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0391 | LR: 3.36e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.7005s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 1.3284 | max_loss: 0.5826 | max_backward: 1.9391 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7423.7 | uss_MB: 7407.1 | child_rss_MB: 3589.0 | allocated_MB: 344.3 | reserved_MB: 26086.0 | inactive_split_MB: 212.1 +rank6 [2026-10-29 10:37:13,221] INFO : Step 33000/50000 rank6 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5373 | Train Loss Components: Auxiliary: 0.2310 | Cardinality: 0.1017 | Classification: 0.1099 | Classification_binary: 0.1844 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0689 | Regression_eta: 0.0359 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0585 | LR: 3.33e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.3122s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 0.7919 | max_loss: 0.2231 | max_backward: 1.5724 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.5025 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1340 | Memory: rss_MB: 7423.5 | uss_MB: 7406.2 | child_rss_MB: 3581.4 | allocated_MB: 344.2 | reserved_MB: 26086.0 | inactive_split_MB: 269.8 +rank6 [2026-10-29 10:37:14,208] INFO : Running validation on rank6 +rank6 [2026-10-29 10:37:14,208] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 10:37:14,209] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 10:38:03,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:38:04,390] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 10:38:31,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 10:38:33,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 10:38:34,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 10:38:35,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 10:38:45,589] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 10:38:45,833] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:38:46,377] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 10:39:02,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 10:39:03,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 10:39:04,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 10:39:05,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 10:39:15,399] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 10:39:15,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:39:16,281] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 10:39:31,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 10:39:32,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 10:39:33,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 10:39:33,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 10:39:43,693] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 10:43:56,310] INFO : Step 33100/50000 rank6 | Train Loss: 1.1040 | Train Loss Unweighted: 0.5268 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1427 | Classification: 0.0921 | Classification_binary: 0.1655 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1319 | Regression_eta: 0.0015 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1592 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0597 | Regression_eta: 0.0366 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0505 | LR: 3.31e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3485s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0325 | max_forward: 0.7755 | max_loss: 0.1478 | max_backward: 1.3916 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.6610 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0760 | Memory: rss_MB: 8784.9 | uss_MB: 8766.5 | child_rss_MB: 3557.4 | allocated_MB: 343.8 | reserved_MB: 23356.0 | inactive_split_MB: 267.3 +rank6 [2026-10-29 10:47:56,312] INFO : Step 33200/50000 rank6 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4955 | Train Loss Components: Auxiliary: 0.2164 | Cardinality: 0.1253 | Classification: 0.1001 | Classification_binary: 0.1407 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0019 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1178 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0555 | Regression_eta: 0.0289 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0276 | LR: 3.28e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.0964s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0401 | max_forward: 0.9954 | max_loss: 0.3116 | max_backward: 2.2139 | max_optimizer_step: 0.2657 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7500.4 | uss_MB: 7483.0 | child_rss_MB: 3560.3 | allocated_MB: 344.1 | reserved_MB: 23356.0 | inactive_split_MB: 241.6 +rank6 [2026-10-29 10:51:55,084] INFO : Step 33300/50000 rank6 | Train Loss: 1.0288 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2302 | Cardinality: 0.1166 | Classification: 0.0975 | Classification_binary: 0.1653 | HitGrouping: 0.1885 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1220 | Regression_eta: 0.0012 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0464 | Regression_eta: 0.0284 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0698 | LR: 3.26e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5474s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7054 | max_loss: 0.2213 | max_backward: 1.6633 | max_optimizer_step: 0.2474 | max_tensorboard_logging: 0.8237 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0708 | Memory: rss_MB: 7500.4 | uss_MB: 7483.1 | child_rss_MB: 3556.5 | allocated_MB: 343.5 | reserved_MB: 23356.0 | inactive_split_MB: 261.9 +rank6 [2026-10-29 10:55:54,910] INFO : Step 33400/50000 rank6 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5337 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1249 | Classification: 0.0970 | Classification_binary: 0.1720 | HitGrouping: 0.1947 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0013 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.2182 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0657 | Regression_eta: 0.0470 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0622 | LR: 3.23e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.4575s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0073 | max_forward: 1.2444 | max_loss: 0.2025 | max_backward: 2.5109 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7502.5 | uss_MB: 7485.1 | child_rss_MB: 3549.5 | allocated_MB: 347.6 | reserved_MB: 23358.0 | inactive_split_MB: 303.0 +rank6 [2026-10-29 10:59:53,594] INFO : Step 33500/50000 rank6 | Train Loss: 1.0164 | Train Loss Unweighted: 0.4956 | Train Loss Components: Auxiliary: 0.2212 | Cardinality: 0.1274 | Classification: 0.0875 | Classification_binary: 0.1421 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1309 | Regression_eta: 0.0013 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0650 | Regression_eta: 0.0264 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0309 | LR: 3.21e-04 | DataLoad Time: 0.0094s | Model Forward Time: 4.6156s | Phase Times: max_pre_step_barrier: 0.0271 | max_batch_to_device: 0.0558 | max_forward: 0.9437 | max_loss: 0.1437 | max_backward: 1.5429 | max_optimizer_step: 0.2447 | max_tensorboard_logging: 0.8833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7505.9 | uss_MB: 7486.7 | child_rss_MB: 3558.2 | allocated_MB: 341.2 | reserved_MB: 23358.0 | inactive_split_MB: 237.5 +rank6 [2026-10-29 11:05:22,800] INFO : Step 33600/50000 rank6 | Train Loss: 1.0074 | Train Loss Unweighted: 0.4988 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1109 | Classification: 0.0969 | Classification_binary: 0.1509 | HitGrouping: 0.1901 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0012 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1182 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0589 | Regression_eta: 0.0256 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0613 | LR: 3.19e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1865s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0970 | max_forward: 0.7667 | max_loss: 0.1478 | max_backward: 1.4192 | max_optimizer_step: 0.2426 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7500.7 | uss_MB: 7485.1 | child_rss_MB: 3550.7 | allocated_MB: 344.5 | reserved_MB: 24528.0 | inactive_split_MB: 238.9 +rank6 [2026-10-29 11:09:17,747] INFO : Step 33700/50000 rank6 | Train Loss: 1.1891 | Train Loss Unweighted: 0.5658 | Train Loss Components: Auxiliary: 0.2602 | Cardinality: 0.1587 | Classification: 0.1116 | Classification_binary: 0.1953 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1270 | Regression_eta: 0.0015 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.2132 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0613 | Regression_eta: 0.0587 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0906 | LR: 3.16e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8096s | Phase Times: max_pre_step_barrier: 0.0253 | max_batch_to_device: 0.0066 | max_forward: 1.3051 | max_loss: 0.1764 | max_backward: 1.7331 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.5706 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0895 | Memory: rss_MB: 7502.0 | uss_MB: 7487.9 | child_rss_MB: 3548.7 | allocated_MB: 347.7 | reserved_MB: 24528.0 | inactive_split_MB: 288.7 +rank6 [2026-10-29 11:13:14,784] INFO : Step 33800/50000 rank6 | Train Loss: 1.0442 | Train Loss Unweighted: 0.4970 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1421 | Classification: 0.0764 | Classification_binary: 0.1525 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1318 | Regression_eta: 0.0014 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1804 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0616 | Regression_eta: 0.0382 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0471 | LR: 3.14e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2993s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7370 | max_loss: 0.2458 | max_backward: 1.5902 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.6292 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1331 | Memory: rss_MB: 7496.1 | uss_MB: 7482.2 | child_rss_MB: 3539.4 | allocated_MB: 344.2 | reserved_MB: 24528.0 | inactive_split_MB: 271.5 +rank6 [2026-10-29 11:17:10,798] INFO : Step 33900/50000 rank6 | Train Loss: 1.0341 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1352 | Classification: 0.0865 | Classification_binary: 0.1441 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1254 | Regression_eta: 0.0016 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1284 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0607 | Regression_eta: 0.0443 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0470 | LR: 3.11e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.3842s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.6305 | max_loss: 0.1507 | max_backward: 1.4015 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9471 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0605 | Memory: rss_MB: 7498.1 | uss_MB: 7488.3 | child_rss_MB: 3543.5 | allocated_MB: 344.1 | reserved_MB: 24528.0 | inactive_split_MB: 268.7 +rank6 [2026-10-29 11:21:05,085] INFO : Step 34000/50000 rank6 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1309 | Classification: 0.0985 | Classification_binary: 0.1707 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1170 | Regression_eta: 0.0014 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0478 | Regression_eta: 0.0396 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0605 | LR: 3.09e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.6359s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7414 | max_loss: 0.1458 | max_backward: 1.7218 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 7499.1 | uss_MB: 7491.4 | child_rss_MB: 3548.4 | allocated_MB: 342.9 | reserved_MB: 24528.0 | inactive_split_MB: 277.5 +rank6 [2026-10-29 11:21:06,163] INFO : Running validation on rank6 +rank6 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 11:21:56,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:21:57,376] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 11:22:17,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 11:22:18,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 11:22:19,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 11:22:21,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 11:22:31,441] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-29 11:22:32,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:22:32,506] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 11:22:48,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 11:22:49,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 11:22:50,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 11:22:51,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 11:23:01,400] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 11:23:02,011] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:23:02,514] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 11:23:17,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 11:23:18,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 11:23:19,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 11:23:19,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 11:23:29,800] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 11:27:45,672] INFO : Step 34100/50000 rank6 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5582 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1263 | Classification: 0.1031 | Classification_binary: 0.1801 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1358 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0688 | Regression_eta: 0.0291 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0556 | LR: 3.06e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8615s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9025 | max_loss: 0.1870 | max_backward: 1.7732 | max_optimizer_step: 0.2826 | max_tensorboard_logging: 0.8215 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0738 | Memory: rss_MB: 9109.5 | uss_MB: 8140.4 | child_rss_MB: 3555.5 | allocated_MB: 342.7 | reserved_MB: 22124.0 | inactive_split_MB: 211.6 +rank6 [2026-10-29 11:31:40,885] INFO : Step 34200/50000 rank6 | Train Loss: 0.9931 | Train Loss Unweighted: 0.4794 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1232 | Classification: 0.0771 | Classification_binary: 0.1377 | HitGrouping: 0.1996 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1301 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1206 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0578 | Regression_eta: 0.0482 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0667 | LR: 3.04e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.7205s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.8809 | max_loss: 0.3036 | max_backward: 1.8297 | max_optimizer_step: 0.2897 | max_tensorboard_logging: 0.6835 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7608.6 | uss_MB: 6291.1 | child_rss_MB: 3559.3 | allocated_MB: 339.0 | reserved_MB: 22508.0 | inactive_split_MB: 220.4 +rank6 [2026-10-29 11:35:37,830] INFO : Step 34300/50000 rank6 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4920 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1192 | Classification: 0.0965 | Classification_binary: 0.1484 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1293 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0597 | Regression_eta: 0.0325 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0314 | LR: 3.02e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.3199s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7255 | max_loss: 0.1427 | max_backward: 1.3652 | max_optimizer_step: 0.2496 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7610.6 | uss_MB: 7596.5 | child_rss_MB: 3557.3 | allocated_MB: 345.0 | reserved_MB: 22508.0 | inactive_split_MB: 216.4 +rank6 [2026-10-29 11:39:28,026] INFO : Step 34400/50000 rank6 | Train Loss: 1.0662 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2383 | Cardinality: 0.1289 | Classification: 0.0974 | Classification_binary: 0.1749 | HitGrouping: 0.1864 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1281 | Regression_eta: 0.0014 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1669 | Regression_cos_phi: 0.0436 | Regression_energy: 0.0633 | Regression_eta: 0.0433 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0510 | LR: 2.99e-04 | DataLoad Time: 0.0190s | Model Forward Time: 6.1242s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9438 | max_loss: 0.1560 | max_backward: 2.9359 | max_optimizer_step: 0.2526 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7613.7 | uss_MB: 6304.5 | child_rss_MB: 3550.6 | allocated_MB: 345.3 | reserved_MB: 23132.0 | inactive_split_MB: 282.7 +rank6 [2026-10-29 11:43:17,765] INFO : Step 34500/50000 rank6 | Train Loss: 1.0328 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1103 | Classification: 0.0867 | Classification_binary: 0.1518 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0014 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0746 | Regression_eta: 0.0350 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0592 | LR: 2.97e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.6253s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.8153 | max_loss: 0.1429 | max_backward: 1.8299 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.5315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7615.7 | uss_MB: 6643.2 | child_rss_MB: 3549.5 | allocated_MB: 341.4 | reserved_MB: 23132.0 | inactive_split_MB: 238.6 +rank6 [2026-10-29 11:47:05,942] INFO : Step 34600/50000 rank6 | Train Loss: 1.0317 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1201 | Classification: 0.0877 | Classification_binary: 0.1434 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1359 | Regression_eta: 0.0014 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0775 | Regression_eta: 0.0442 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0575 | LR: 2.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2906s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.6882 | max_loss: 0.1360 | max_backward: 1.3984 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5787 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7607.9 | uss_MB: 6553.3 | child_rss_MB: 3582.8 | allocated_MB: 343.8 | reserved_MB: 23132.0 | inactive_split_MB: 233.1 +rank6 [2026-10-29 11:51:12,922] INFO : Step 34700/50000 rank6 | Train Loss: 1.1152 | Train Loss Unweighted: 0.5285 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1612 | Classification: 0.0937 | Classification_binary: 0.1734 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1283 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0659 | Regression_eta: 0.0597 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0559 | LR: 2.92e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5243s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7859 | max_loss: 0.2571 | max_backward: 1.9119 | max_optimizer_step: 0.2516 | max_tensorboard_logging: 0.5032 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7610.6 | uss_MB: 7599.4 | child_rss_MB: 3559.6 | allocated_MB: 343.9 | reserved_MB: 23132.0 | inactive_split_MB: 446.8 +rank6 [2026-10-29 11:54:56,885] INFO : Step 34800/50000 rank6 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5142 | Train Loss Components: Auxiliary: 0.2371 | Cardinality: 0.1583 | Classification: 0.0857 | Classification_binary: 0.1750 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1255 | Regression_eta: 0.0013 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0380 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0582 | Regression_eta: 0.0377 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0737 | LR: 2.90e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2205s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0423 | max_forward: 0.8214 | max_loss: 0.1472 | max_backward: 1.4019 | max_optimizer_step: 0.3234 | max_tensorboard_logging: 0.5950 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7613.6 | uss_MB: 7602.4 | child_rss_MB: 3541.0 | allocated_MB: 344.5 | reserved_MB: 23132.0 | inactive_split_MB: 201.5 +rank6 [2026-10-29 11:58:48,855] INFO : Step 34900/50000 rank6 | Train Loss: 0.9512 | Train Loss Unweighted: 0.4678 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1120 | Classification: 0.0788 | Classification_binary: 0.1318 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1262 | Regression_eta: 0.0014 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1307 | Regression_cos_phi: 0.0297 | Regression_energy: 0.0628 | Regression_eta: 0.0303 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0418 | LR: 2.88e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2617s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0986 | max_forward: 0.7397 | max_loss: 0.1421 | max_backward: 1.3918 | max_optimizer_step: 0.3560 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7614.6 | uss_MB: 7602.0 | child_rss_MB: 3543.9 | allocated_MB: 341.5 | reserved_MB: 23500.0 | inactive_split_MB: 262.9 +rank6 [2026-10-29 12:02:43,663] INFO : Step 35000/50000 rank6 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1401 | Classification: 0.0852 | Classification_binary: 0.1555 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0012 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1116 | Regression_cos_phi: 0.0304 | Regression_energy: 0.0552 | Regression_eta: 0.0194 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0371 | LR: 2.85e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7029s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0159 | max_forward: 0.7961 | max_loss: 0.2615 | max_backward: 2.0104 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6382 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7618.3 | uss_MB: 7606.3 | child_rss_MB: 3544.7 | allocated_MB: 345.2 | reserved_MB: 23500.0 | inactive_split_MB: 284.5 +rank6 [2026-10-29 12:02:44,719] INFO : Running validation on rank6 +rank6 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 12:03:33,663] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 12:03:34,206] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 12:04:01,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 12:04:03,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 12:04:04,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 12:04:05,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 12:04:15,626] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 12:04:18,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 12:04:18,509] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 12:04:34,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 12:04:35,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 12:04:36,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 12:04:37,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 12:04:47,691] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 12:04:48,304] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 12:04:48,824] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 12:05:04,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 12:05:05,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 12:05:05,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 12:05:06,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 12:05:16,449] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 12:09:34,473] INFO : Step 35100/50000 rank6 | Train Loss: 1.1060 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1387 | Classification: 0.0926 | Classification_binary: 0.1848 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0014 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1783 | Regression_cos_phi: 0.1002 | Regression_energy: 0.0607 | Regression_eta: 0.0467 | Regression_pt: 0.0638 | Regression_sin_phi: 0.1002 | LR: 2.83e-04 | DataLoad Time: 0.0144s | Model Forward Time: 6.6812s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.9070 | max_loss: 0.1513 | max_backward: 2.1746 | max_optimizer_step: 0.2850 | max_tensorboard_logging: 1.1439 | max_tensorboard_flush: 0.0024 | max_post_step_barrier: 0.0768 | Memory: rss_MB: 9217.3 | uss_MB: 7564.1 | child_rss_MB: 3570.6 | allocated_MB: 344.3 | reserved_MB: 22622.0 | inactive_split_MB: 852.9 +rank6 [2026-10-29 12:13:24,279] INFO : Step 35200/50000 rank6 | Train Loss: 1.0386 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1293 | Classification: 0.0826 | Classification_binary: 0.1559 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1247 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1501 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0500 | Regression_eta: 0.0327 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0476 | LR: 2.81e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.5324s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.8193 | max_loss: 0.1608 | max_backward: 1.3752 | max_optimizer_step: 0.2535 | max_tensorboard_logging: 0.9185 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0700 | Memory: rss_MB: 7738.0 | uss_MB: 7717.0 | child_rss_MB: 3546.7 | allocated_MB: 341.8 | reserved_MB: 25614.0 | inactive_split_MB: 192.0 +rank6 [2026-10-29 12:17:18,033] INFO : Step 35300/50000 rank6 | Train Loss: 0.9736 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1059 | Classification: 0.0970 | Classification_binary: 0.1371 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0016 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1281 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0602 | Regression_eta: 0.0319 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0387 | LR: 2.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6484s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6434 | max_loss: 0.2341 | max_backward: 1.6834 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0588 | Memory: rss_MB: 7740.0 | uss_MB: 7297.3 | child_rss_MB: 3543.2 | allocated_MB: 343.2 | reserved_MB: 25614.0 | inactive_split_MB: 226.8 +rank6 [2026-10-29 12:21:06,024] INFO : Step 35400/50000 rank6 | Train Loss: 1.1186 | Train Loss Unweighted: 0.5363 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1393 | Classification: 0.1069 | Classification_binary: 0.1767 | HitGrouping: 0.1943 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0016 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0565 | Regression_eta: 0.0402 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0481 | LR: 2.76e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.8145s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0060 | max_forward: 0.7591 | max_loss: 0.1562 | max_backward: 1.6040 | max_optimizer_step: 0.2543 | max_tensorboard_logging: 0.9624 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.0563 | Memory: rss_MB: 7735.5 | uss_MB: 7248.0 | child_rss_MB: 3540.1 | allocated_MB: 345.3 | reserved_MB: 25762.0 | inactive_split_MB: 277.5 +rank6 [2026-10-29 12:24:54,118] INFO : Step 35500/50000 rank6 | Train Loss: 1.1216 | Train Loss Unweighted: 0.5466 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1527 | Classification: 0.0970 | Classification_binary: 0.1767 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1342 | Regression_eta: 0.0014 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.1939 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0831 | Regression_eta: 0.0479 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0933 | LR: 2.74e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4068s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0648 | max_forward: 0.8748 | max_loss: 0.1544 | max_backward: 1.6387 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 0.5130 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0632 | Memory: rss_MB: 7738.5 | uss_MB: 7721.7 | child_rss_MB: 3543.4 | allocated_MB: 343.4 | reserved_MB: 25762.0 | inactive_split_MB: 284.4 +rank6 [2026-10-29 12:28:50,968] INFO : Step 35600/50000 rank6 | Train Loss: 1.0380 | Train Loss Unweighted: 0.5039 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1124 | Classification: 0.0922 | Classification_binary: 0.1405 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0553 | Regression_eta: 0.0336 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0537 | LR: 2.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8977s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0559 | max_forward: 0.7376 | max_loss: 0.1415 | max_backward: 1.1754 | max_optimizer_step: 0.2524 | max_tensorboard_logging: 0.5225 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1603 | Memory: rss_MB: 7739.4 | uss_MB: 7724.9 | child_rss_MB: 3557.9 | allocated_MB: 343.8 | reserved_MB: 25762.0 | inactive_split_MB: 233.6 +rank6 [2026-10-29 12:32:54,484] INFO : Step 35700/50000 rank6 | Train Loss: 1.0352 | Train Loss Unweighted: 0.5000 | Train Loss Components: Auxiliary: 0.2288 | Cardinality: 0.1199 | Classification: 0.1043 | Classification_binary: 0.1492 | HitGrouping: 0.1850 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1212 | Regression_eta: 0.0015 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0641 | Regression_eta: 0.0329 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0515 | LR: 2.70e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3745s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0360 | max_forward: 0.7358 | max_loss: 0.1419 | max_backward: 1.6761 | max_optimizer_step: 0.2531 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7741.1 | uss_MB: 7726.8 | child_rss_MB: 3543.5 | allocated_MB: 344.2 | reserved_MB: 26034.0 | inactive_split_MB: 271.8 +rank6 [2026-10-29 12:36:36,038] INFO : Step 35800/50000 rank6 | Train Loss: 1.0929 | Train Loss Unweighted: 0.5323 | Train Loss Components: Auxiliary: 0.2446 | Cardinality: 0.1241 | Classification: 0.0986 | Classification_binary: 0.1801 | HitGrouping: 0.1908 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0014 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.1715 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0564 | Regression_eta: 0.0443 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0869 | LR: 2.68e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1509s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7732 | max_loss: 0.1506 | max_backward: 1.4048 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0581 | Memory: rss_MB: 7740.3 | uss_MB: 7206.4 | child_rss_MB: 3548.3 | allocated_MB: 344.1 | reserved_MB: 26034.0 | inactive_split_MB: 276.1 +rank6 [2026-10-29 14:48:22,186] INFO : Initializing model with seed=12345; process seed=12351 +rank6 [2026-10-29 14:48:22,198] INFO : configured dtype=torch.bfloat16 for autocast +rank6 [2026-10-29 14:48:45,046] INFO : Instantiating model +rank6 [2026-10-29 14:48:45,046] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank6 [2026-10-29 14:48:45,046] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank6 [2026-10-29 14:48:45,064] INFO : Input encoding initialization took 0.02s +rank6 [2026-10-29 14:48:45,064] INFO : nn0 parameters: 171776 +rank6 [2026-10-29 14:48:45,064] INFO : Initializing shared backbone layers, num_convs=6 +rank6 [2026-10-29 14:48:45,064] INFO : Initializing backbone layer 0 +rank6 [2026-10-29 14:48:45,064] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:48:45,068] INFO : Initializing backbone layer 1 +rank6 [2026-10-29 14:48:45,068] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:48:45,071] INFO : Initializing backbone layer 2 +rank6 [2026-10-29 14:48:45,071] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:48:45,074] INFO : Initializing backbone layer 3 +rank6 [2026-10-29 14:48:45,074] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:48:45,077] INFO : Initializing backbone layer 4 +rank6 [2026-10-29 14:48:45,077] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:48:45,080] INFO : Initializing backbone layer 5 +rank6 [2026-10-29 14:48:45,080] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank6 [2026-10-29 14:48:45,083] INFO : Backbone initialization took 0.02s +rank6 [2026-10-29 14:48:45,084] INFO : backbone parameters: 2374656 +rank6 [2026-10-29 14:48:45,084] INFO : Initializing output DNNs +rank6 [2026-10-29 14:48:45,132] INFO : Output DNNs initialization took 0.05s +rank6 [2026-10-29 14:48:45,132] INFO : backbone_mode=BackboneMode.SHARED +rank6 [2026-10-29 14:48:45,132] INFO : jagged_attention enabled=True +rank6 [2026-10-29 14:48:45,132] INFO : flash_attn_varlen enabled=True +rank6 [2026-10-29 14:48:45,132] INFO : task_queries enabled=False +rank6 [2026-10-29 14:48:45,132] INFO : classification_norm parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : regression_norm parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : classification_query parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : regression_query parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : classification_readout parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : regression_readout parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : output_mode=set +rank6 [2026-10-29 14:48:45,133] INFO : set_decoder parameters: 4713812 +rank6 [2026-10-29 14:48:45,133] INFO : nn_binary_particle parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : nn_pid parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : nn_pt parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : nn_eta parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : nn_sin_phi parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : nn_cos_phi parameters: 0 +rank6 [2026-10-29 14:48:45,133] INFO : nn_energy parameters: 0 +rank6 [2026-10-29 14:48:45,134] INFO : Total MLPF parameters: 7260244 +rank6 [2026-10-29 14:48:45,134] INFO : MLPF __init__ done +rank6 [2026-10-29 14:48:45,136] INFO : Instantiated model +rank6 [2026-10-29 14:48:45,136] INFO : Moving model to device rank=6 +rank6 [2026-10-29 14:48:45,155] INFO : Moved model to device rank=6 +rank6 [2026-10-29 14:48:45,157] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank6 [2026-10-29 14:48:45,497] INFO : Restoring training from step 35001 +rank6 [2026-10-29 14:48:45,503] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth +rank6 [2026-10-29 14:48:45,504] INFO : Configured model for SyncBatchNorm rank=6 +rank6 [2026-10-29 14:48:45,760] INFO : Configured model for DistributedDataParallel rank=6 +rank6 [2026-10-29 14:48:45,875] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank6 [2026-10-29 14:48:45,875] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank6 [2026-10-29 14:48:45,877] INFO : Trainable parameters: 7260244 +rank6 [2026-10-29 14:48:45,877] INFO : Non-trainable parameters: 0 +rank6 [2026-10-29 14:48:45,877] INFO : Total parameters: 7260244 +rank6 [2026-10-29 14:48:45,879] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank6 [2026-10-29 14:48:45,880] INFO : Getting dataloaders +rank6 [2026-10-29 14:48:45,880] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:48:51,642] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:48:52,730] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:48:53,925] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-29 14:48:53,926] INFO : Creating WeightedMixIterator with 1 data loaders. +rank6 [2026-10-29 14:48:53,926] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:48:54,211] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:48:54,468] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:48:54,718] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-29 14:48:54,719] INFO : Creating InterleavedIterator with 1 data loaders. +rank6 [2026-10-29 14:49:08,544] INFO : Got dataloaders +rank6 [2026-10-29 14:49:08,545] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13888] of epochs [1] +rank6 [2026-10-29 14:49:08,545] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank6 [2026-10-29 14:49:08,545] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank6 [2026-10-29 14:49:08,545] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank6 [2026-10-29 14:49:09,995] INFO : loader split=train rank=6 len=21112 +rank6 [2026-10-29 14:49:09,995] INFO : loader split=valid rank=6 len=12 +rank6 [2026-10-29 14:49:09,995] INFO : Starting training from step 35001 to 50000 on rank 6 of 8 +rank6 [2026-10-29 14:49:09,995] INFO : Creating train_iterator +rank6 [2026-10-29 14:49:09,996] INFO : ResumableSampler physical:train loading state: start_index=222208 +rank6 [2026-10-29 14:49:24,056] INFO : Created train_iterator +rank6 [2026-10-29 14:53:12,251] INFO : Step 35100/50000 rank6 | Train Loss: 1.0826 | Train Loss Unweighted: 0.5185 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1393 | Classification: 0.0961 | Classification_binary: 0.1822 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1828 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0617 | Regression_eta: 0.0609 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0832 | LR: 2.83e-04 | DataLoad Time: 0.0139s | Model Forward Time: 3.4935s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.7340 | max_loss: 0.1651 | max_backward: 1.4152 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4761 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4625.8 | uss_MB: 3388.6 | child_rss_MB: 5578.9 | allocated_MB: 372.6 | reserved_MB: 22674.0 | inactive_split_MB: 823.1 +rank6 [2026-10-29 14:56:45,511] INFO : Step 35200/50000 rank6 | Train Loss: 1.0562 | Train Loss Unweighted: 0.5125 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1301 | Classification: 0.0878 | Classification_binary: 0.1540 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1331 | Regression_eta: 0.0017 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0531 | Regression_eta: 0.0469 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0694 | LR: 2.81e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.1453s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6042 | max_loss: 0.1411 | max_backward: 1.1580 | max_optimizer_step: 0.1447 | max_tensorboard_logging: 0.5108 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4625.0 | uss_MB: 3391.6 | child_rss_MB: 5591.1 | allocated_MB: 368.1 | reserved_MB: 25668.0 | inactive_split_MB: 192.7 +rank6 [2026-10-29 15:00:19,321] INFO : Step 35300/50000 rank6 | Train Loss: 0.9962 | Train Loss Unweighted: 0.4725 | Train Loss Components: Auxiliary: 0.2160 | Cardinality: 0.1172 | Classification: 0.0927 | Classification_binary: 0.1341 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1234 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0524 | Regression_eta: 0.0275 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0425 | LR: 2.79e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.0281s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.5940 | max_loss: 0.1358 | max_backward: 1.1911 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0027 | Memory: rss_MB: 4636.4 | uss_MB: 3402.6 | child_rss_MB: 5598.9 | allocated_MB: 371.5 | reserved_MB: 25668.0 | inactive_split_MB: 207.6 +rank6 [2026-10-29 15:03:53,851] INFO : Step 35400/50000 rank6 | Train Loss: 1.1270 | Train Loss Unweighted: 0.5445 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1356 | Classification: 0.0990 | Classification_binary: 0.1805 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1297 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0656 | Regression_eta: 0.0363 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0581 | LR: 2.76e-04 | DataLoad Time: 0.0149s | Model Forward Time: 3.6497s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1507 | max_backward: 1.5023 | max_optimizer_step: 0.2459 | max_tensorboard_logging: 0.5679 | max_tensorboard_flush: 0.0064 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4646.3 | uss_MB: 3410.8 | child_rss_MB: 5599.4 | allocated_MB: 371.4 | reserved_MB: 25816.0 | inactive_split_MB: 263.6 +rank6 [2026-10-29 15:07:33,751] INFO : Step 35500/50000 rank6 | Train Loss: 1.0502 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1302 | Classification: 0.0904 | Classification_binary: 0.1703 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1250 | Regression_eta: 0.0013 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1953 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0694 | Regression_eta: 0.0458 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0574 | LR: 2.74e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.6099s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0737 | max_forward: 0.7546 | max_loss: 0.1581 | max_backward: 1.4977 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4644.7 | uss_MB: 3412.9 | child_rss_MB: 5568.4 | allocated_MB: 371.1 | reserved_MB: 25816.0 | inactive_split_MB: 263.4 +rank6 [2026-10-29 15:11:19,390] INFO : Step 35600/50000 rank6 | Train Loss: 1.0258 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2226 | Cardinality: 0.1119 | Classification: 0.0922 | Classification_binary: 0.1418 | HitGrouping: 0.1950 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1286 | Regression_eta: 0.0015 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0540 | Regression_eta: 0.0237 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0986 | LR: 2.72e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.2540s | Phase Times: max_pre_step_barrier: 0.0929 | max_batch_to_device: 0.0064 | max_forward: 0.5696 | max_loss: 0.1430 | max_backward: 1.1627 | max_optimizer_step: 0.1475 | max_tensorboard_logging: 0.5805 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4646.9 | uss_MB: 3412.7 | child_rss_MB: 5602.7 | allocated_MB: 370.5 | reserved_MB: 25816.0 | inactive_split_MB: 212.3 +rank6 [2026-10-29 15:15:08,570] INFO : Step 35700/50000 rank6 | Train Loss: 1.0162 | Train Loss Unweighted: 0.5013 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1185 | Classification: 0.0946 | Classification_binary: 0.1537 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1248 | Regression_eta: 0.0015 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1462 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0592 | Regression_eta: 0.0292 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0418 | LR: 2.70e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5944s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7397 | max_loss: 0.1413 | max_backward: 1.6688 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4646.6 | uss_MB: 3411.7 | child_rss_MB: 5604.2 | allocated_MB: 372.2 | reserved_MB: 26084.0 | inactive_split_MB: 249.8 +rank6 [2026-10-29 15:18:49,408] INFO : Step 35800/50000 rank6 | Train Loss: 1.0884 | Train Loss Unweighted: 0.5265 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1296 | Classification: 0.0948 | Classification_binary: 0.1803 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.1780 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0587 | Regression_eta: 0.0441 | Regression_pt: 0.0617 | Regression_sin_phi: 0.1001 | LR: 2.68e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2014s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0199 | max_forward: 0.9402 | max_loss: 0.2389 | max_backward: 1.7889 | max_optimizer_step: 0.2466 | max_tensorboard_logging: 0.7071 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4643.4 | uss_MB: 3414.5 | child_rss_MB: 5602.6 | allocated_MB: 371.6 | reserved_MB: 26084.0 | inactive_split_MB: 257.5 +rank6 [2026-10-29 15:22:30,828] INFO : Step 35900/50000 rank6 | Train Loss: 1.0760 | Train Loss Unweighted: 0.5299 | Train Loss Components: Auxiliary: 0.2328 | Cardinality: 0.1277 | Classification: 0.0983 | Classification_binary: 0.1568 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1349 | Regression_eta: 0.0016 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0897 | Regression_eta: 0.0357 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0559 | LR: 2.65e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.2908s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6268 | max_loss: 0.2373 | max_backward: 1.2570 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.4934 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0016 | Memory: rss_MB: 4639.4 | uss_MB: 3412.6 | child_rss_MB: 5605.5 | allocated_MB: 369.7 | reserved_MB: 26800.0 | inactive_split_MB: 294.3 +rank6 [2026-10-29 15:26:13,159] INFO : Step 36000/50000 rank6 | Train Loss: 1.0831 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1276 | Classification: 0.1031 | Classification_binary: 0.1434 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1452 | Regression_eta: 0.0017 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1867 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0649 | Regression_eta: 0.0456 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0901 | LR: 2.63e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.2184s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6266 | max_loss: 0.1437 | max_backward: 1.3209 | max_optimizer_step: 0.1464 | max_tensorboard_logging: 0.4977 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4620.1 | uss_MB: 3411.5 | child_rss_MB: 5582.3 | allocated_MB: 368.7 | reserved_MB: 26800.0 | inactive_split_MB: 270.9 +rank6 [2026-10-29 15:26:14,080] INFO : Running validation on rank6 +rank6 [2026-10-29 15:26:14,080] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 15:26:14,080] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 15:27:03,348] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:27:03,871] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 15:27:29,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 15:27:30,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 15:27:31,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 15:27:32,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 15:27:42,331] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 15:27:42,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:27:43,314] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 15:27:59,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 15:28:00,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 15:28:01,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 15:28:02,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 15:28:12,044] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 15:28:12,651] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:28:13,127] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 15:28:28,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 15:28:29,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 15:28:29,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 15:28:30,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 15:28:40,469] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 15:32:50,524] INFO : Step 36100/50000 rank6 | Train Loss: 1.0137 | Train Loss Unweighted: 0.5026 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1079 | Classification: 0.1014 | Classification_binary: 0.1522 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1216 | Regression_eta: 0.0013 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0616 | Regression_eta: 0.0353 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0555 | LR: 2.61e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0465s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0065 | max_forward: 0.6968 | max_loss: 0.2782 | max_backward: 1.4278 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.9130 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6101.3 | uss_MB: 4907.2 | child_rss_MB: 5609.1 | allocated_MB: 372.5 | reserved_MB: 23174.0 | inactive_split_MB: 246.6 +rank6 [2026-10-29 15:36:33,883] INFO : Step 36200/50000 rank6 | Train Loss: 1.0439 | Train Loss Unweighted: 0.5147 | Train Loss Components: Auxiliary: 0.2338 | Cardinality: 0.1304 | Classification: 0.0909 | Classification_binary: 0.1697 | HitGrouping: 0.1917 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1255 | Regression_eta: 0.0011 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1840 | Regression_cos_phi: 0.0438 | Regression_energy: 0.0566 | Regression_eta: 0.0329 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0702 | LR: 2.59e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.5424s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7440 | max_loss: 0.1544 | max_backward: 1.4037 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 6101.4 | uss_MB: 4909.6 | child_rss_MB: 5587.9 | allocated_MB: 367.8 | reserved_MB: 23174.0 | inactive_split_MB: 282.8 +rank6 [2026-10-29 15:40:20,925] INFO : Step 36300/50000 rank6 | Train Loss: 1.0379 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1216 | Classification: 0.0886 | Classification_binary: 0.1447 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1266 | Regression_eta: 0.0016 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1399 | Regression_cos_phi: 0.1158 | Regression_energy: 0.0673 | Regression_eta: 0.0432 | Regression_pt: 0.0714 | Regression_sin_phi: 0.1014 | LR: 2.57e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.3323s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6106 | max_loss: 0.2412 | max_backward: 1.3407 | max_optimizer_step: 0.1535 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1063 | Memory: rss_MB: 5450.1 | uss_MB: 4269.0 | child_rss_MB: 5611.1 | allocated_MB: 370.5 | reserved_MB: 23174.0 | inactive_split_MB: 274.2 +rank6 [2026-10-29 15:44:06,294] INFO : Step 36400/50000 rank6 | Train Loss: 1.0182 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.0976 | Classification: 0.0939 | Classification_binary: 0.1520 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1167 | Regression_eta: 0.0016 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1521 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0548 | Regression_eta: 0.0346 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0531 | LR: 2.55e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.5206s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.6772 | max_loss: 0.1711 | max_backward: 1.4058 | max_optimizer_step: 0.1518 | max_tensorboard_logging: 0.5460 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0089 | Memory: rss_MB: 5451.9 | uss_MB: 4272.1 | child_rss_MB: 5615.2 | allocated_MB: 371.8 | reserved_MB: 23382.0 | inactive_split_MB: 230.6 +rank6 [2026-10-29 15:47:54,065] INFO : Step 36500/50000 rank6 | Train Loss: 1.0953 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1370 | Classification: 0.0895 | Classification_binary: 0.1838 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1893 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0641 | Regression_eta: 0.0385 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0516 | LR: 2.52e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9825s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7475 | max_loss: 0.2537 | max_backward: 1.7352 | max_optimizer_step: 0.1520 | max_tensorboard_logging: 0.6267 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5440.5 | uss_MB: 4262.4 | child_rss_MB: 5614.9 | allocated_MB: 372.3 | reserved_MB: 24160.0 | inactive_split_MB: 275.1 +rank6 [2026-10-29 15:51:40,079] INFO : Step 36600/50000 rank6 | Train Loss: 0.9967 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1165 | Classification: 0.0804 | Classification_binary: 0.1438 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1366 | Regression_eta: 0.0013 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1799 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0706 | Regression_eta: 0.0333 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0488 | LR: 2.50e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6299s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6592 | max_loss: 0.2501 | max_backward: 1.4213 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5953 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5450.1 | uss_MB: 4272.8 | child_rss_MB: 5617.4 | allocated_MB: 367.8 | reserved_MB: 27180.0 | inactive_split_MB: 225.4 +rank6 [2026-10-29 15:55:26,197] INFO : Step 36700/50000 rank6 | Train Loss: 0.9770 | Train Loss Unweighted: 0.4817 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1070 | Classification: 0.0901 | Classification_binary: 0.1347 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0316 | Regression_energy: 0.0732 | Regression_eta: 0.0260 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0450 | LR: 2.48e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.2748s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0062 | max_forward: 0.5676 | max_loss: 0.2381 | max_backward: 1.1343 | max_optimizer_step: 0.2417 | max_tensorboard_logging: 0.4693 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5448.5 | uss_MB: 4274.1 | child_rss_MB: 5620.1 | allocated_MB: 370.8 | reserved_MB: 27180.0 | inactive_split_MB: 235.2 +rank6 [2026-10-29 15:59:20,983] INFO : Step 36800/50000 rank6 | Train Loss: 1.0234 | Train Loss Unweighted: 0.4967 | Train Loss Components: Auxiliary: 0.2267 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1662 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1197 | Regression_eta: 0.0013 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1592 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0488 | Regression_eta: 0.0364 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0457 | LR: 2.46e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.5118s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 1.1394 | max_loss: 0.1518 | max_backward: 2.3088 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5450.3 | uss_MB: 4278.9 | child_rss_MB: 5629.5 | allocated_MB: 371.9 | reserved_MB: 27180.0 | inactive_split_MB: 230.7 +rank6 [2026-10-29 16:03:11,411] INFO : Step 36900/50000 rank6 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2268 | Cardinality: 0.1211 | Classification: 0.0855 | Classification_binary: 0.1637 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1219 | Regression_eta: 0.0015 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0581 | Regression_eta: 0.0379 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0450 | LR: 2.44e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.5012s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0071 | max_forward: 0.7005 | max_loss: 0.1465 | max_backward: 1.2640 | max_optimizer_step: 0.1579 | max_tensorboard_logging: 0.4728 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5451.1 | uss_MB: 4280.1 | child_rss_MB: 5636.6 | allocated_MB: 372.3 | reserved_MB: 27180.0 | inactive_split_MB: 201.7 +rank6 [2026-10-29 16:07:15,145] INFO : Step 37000/50000 rank6 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2194 | Cardinality: 0.1268 | Classification: 0.0926 | Classification_binary: 0.1402 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0014 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0530 | Regression_eta: 0.0343 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0374 | LR: 2.42e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.5217s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6075 | max_loss: 0.2475 | max_backward: 1.2956 | max_optimizer_step: 0.1585 | max_tensorboard_logging: 0.7613 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5445.1 | uss_MB: 4277.2 | child_rss_MB: 5637.5 | allocated_MB: 368.1 | reserved_MB: 27180.0 | inactive_split_MB: 226.3 +rank6 [2026-10-29 16:07:16,060] INFO : Running validation on rank6 +rank6 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 16:08:14,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:08:14,583] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 16:08:39,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 16:08:40,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 16:08:41,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 16:08:42,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 16:08:52,725] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 16:08:53,058] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:08:53,528] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 16:09:09,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 16:09:10,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 16:09:11,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 16:09:12,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 16:09:22,786] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 16:09:23,588] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:09:24,059] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 16:09:39,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 16:09:40,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 16:09:40,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 16:09:41,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 16:09:51,480] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 16:13:56,322] INFO : Step 37100/50000 rank6 | Train Loss: 1.1150 | Train Loss Unweighted: 0.5431 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1226 | Classification: 0.1105 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1340 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0949 | Classification_binary: 0.1509 | Regression_cos_phi: 0.1261 | Regression_energy: 0.0638 | Regression_eta: 0.0688 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0746 | LR: 2.40e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.8598s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7539 | max_loss: 0.3988 | max_backward: 1.7903 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4747 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6593.7 | uss_MB: 5426.2 | child_rss_MB: 5642.5 | allocated_MB: 372.5 | reserved_MB: 23160.0 | inactive_split_MB: 265.5 +rank6 [2026-10-29 16:17:46,448] INFO : Step 37200/50000 rank6 | Train Loss: 1.0752 | Train Loss Unweighted: 0.5181 | Train Loss Components: Auxiliary: 0.2406 | Cardinality: 0.1258 | Classification: 0.0924 | Classification_binary: 0.1812 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1200 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0744 | Regression_eta: 0.0447 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0636 | LR: 2.38e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.1712s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0549 | max_forward: 0.8058 | max_loss: 0.1510 | max_backward: 1.8476 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5280 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6591.2 | uss_MB: 5426.9 | child_rss_MB: 5644.6 | allocated_MB: 371.5 | reserved_MB: 24140.0 | inactive_split_MB: 271.0 +rank6 [2026-10-29 16:21:30,618] INFO : Step 37300/50000 rank6 | Train Loss: 1.0230 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1563 | HitGrouping: 0.1981 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1213 | Regression_eta: 0.0016 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1652 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0577 | Regression_eta: 0.0417 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0562 | LR: 2.36e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0282s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.6853 | max_loss: 0.1502 | max_backward: 1.3245 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 1.0657 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5580.1 | uss_MB: 4415.2 | child_rss_MB: 5668.9 | allocated_MB: 370.9 | reserved_MB: 24140.0 | inactive_split_MB: 261.7 +rank6 [2026-10-29 16:25:14,029] INFO : Step 37400/50000 rank6 | Train Loss: 1.0130 | Train Loss Unweighted: 0.4929 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1180 | Classification: 0.0961 | Classification_binary: 0.1390 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1273 | Regression_eta: 0.0015 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1016 | Regression_cos_phi: 0.0200 | Regression_energy: 0.0511 | Regression_eta: 0.0392 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0445 | LR: 2.34e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.5046s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.5424 | max_loss: 0.1464 | max_backward: 1.0792 | max_optimizer_step: 0.1675 | max_tensorboard_logging: 0.8668 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5585.6 | uss_MB: 4420.8 | child_rss_MB: 5670.0 | allocated_MB: 368.8 | reserved_MB: 25122.0 | inactive_split_MB: 232.7 +rank6 [2026-10-29 16:30:41,706] INFO : Step 37500/50000 rank6 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1315 | Classification: 0.0992 | Classification_binary: 0.1679 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1176 | Regression_eta: 0.0013 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1618 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0569 | Regression_eta: 0.0380 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0590 | LR: 2.32e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1479s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.7240 | max_loss: 0.1472 | max_backward: 1.5381 | max_optimizer_step: 0.2707 | max_tensorboard_logging: 0.8554 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5585.8 | uss_MB: 4422.3 | child_rss_MB: 5642.8 | allocated_MB: 370.2 | reserved_MB: 26134.0 | inactive_split_MB: 179.7 +rank6 [2026-10-29 16:34:38,362] INFO : Step 37600/50000 rank6 | Train Loss: 1.0243 | Train Loss Unweighted: 0.4867 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1182 | Classification: 0.0808 | Classification_binary: 0.1619 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0592 | Regression_eta: 0.0355 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0704 | LR: 2.30e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.1681s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.7069 | max_loss: 0.1543 | max_backward: 1.5139 | max_optimizer_step: 0.1704 | max_tensorboard_logging: 1.0159 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5587.9 | uss_MB: 4422.7 | child_rss_MB: 5672.0 | allocated_MB: 370.8 | reserved_MB: 26846.0 | inactive_split_MB: 273.3 +rank6 [2026-10-29 16:38:29,295] INFO : Step 37700/50000 rank6 | Train Loss: 0.9610 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2102 | Cardinality: 0.1049 | Classification: 0.0974 | Classification_binary: 0.1317 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0012 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0572 | Regression_eta: 0.0358 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0912 | LR: 2.28e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5171s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6233 | max_loss: 0.2433 | max_backward: 1.3306 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5644 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5591.0 | uss_MB: 4425.5 | child_rss_MB: 5643.2 | allocated_MB: 369.4 | reserved_MB: 28822.0 | inactive_split_MB: 231.5 +rank6 [2026-10-29 16:42:35,307] INFO : Step 37800/50000 rank6 | Train Loss: 0.9994 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1105 | Classification: 0.0952 | Classification_binary: 0.1458 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1161 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0653 | Regression_eta: 0.0371 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0337 | LR: 2.26e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8906s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.6659 | max_loss: 0.1473 | max_backward: 1.3840 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.9558 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5593.5 | uss_MB: 4427.0 | child_rss_MB: 5671.1 | allocated_MB: 371.5 | reserved_MB: 28822.0 | inactive_split_MB: 170.1 +rank6 [2026-10-29 16:46:40,061] INFO : Step 37900/50000 rank6 | Train Loss: 1.0837 | Train Loss Unweighted: 0.5094 | Train Loss Components: Auxiliary: 0.2372 | Cardinality: 0.1419 | Classification: 0.0937 | Classification_binary: 0.1795 | HitGrouping: 0.1848 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1160 | Regression_eta: 0.0015 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0602 | Regression_eta: 0.0478 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0540 | LR: 2.24e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2209s | Phase Times: max_pre_step_barrier: 0.1076 | max_batch_to_device: 0.0071 | max_forward: 0.7572 | max_loss: 0.1547 | max_backward: 1.4632 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 0.9205 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 5593.0 | uss_MB: 4429.3 | child_rss_MB: 5670.1 | allocated_MB: 370.2 | reserved_MB: 28822.0 | inactive_split_MB: 265.3 +rank6 [2026-10-29 16:50:31,520] INFO : Step 38000/50000 rank6 | Train Loss: 0.9811 | Train Loss Unweighted: 0.4782 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1213 | Classification: 0.0781 | Classification_binary: 0.1413 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1277 | Regression_eta: 0.0012 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0657 | Regression_eta: 0.0329 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0392 | LR: 2.22e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.5102s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.6120 | max_loss: 0.2171 | max_backward: 1.2526 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5836 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5590.4 | uss_MB: 4430.0 | child_rss_MB: 5667.2 | allocated_MB: 368.7 | reserved_MB: 28822.0 | inactive_split_MB: 223.4 +rank6 [2026-10-29 16:50:32,415] INFO : Running validation on rank6 +rank6 [2026-10-29 16:50:32,415] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 16:50:32,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 16:51:23,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:51:24,264] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 16:51:51,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 16:51:52,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 16:51:53,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 16:51:54,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 16:52:04,127] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 16:52:04,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:52:05,008] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 16:52:20,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 16:52:21,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 16:52:22,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 16:52:23,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 16:52:33,475] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 16:52:34,974] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:52:35,418] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 16:52:50,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 16:52:51,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 16:52:52,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 16:52:52,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 16:53:02,872] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 16:57:14,095] INFO : Step 38100/50000 rank6 | Train Loss: 1.0158 | Train Loss Unweighted: 0.4912 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1220 | Classification: 0.0982 | Classification_binary: 0.1455 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1224 | Regression_eta: 0.0013 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1131 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0664 | Regression_eta: 0.0209 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0336 | LR: 2.20e-04 | DataLoad Time: 0.0252s | Model Forward Time: 4.9406s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8406 | max_loss: 0.1820 | max_backward: 1.8536 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.3354 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0179 | Memory: rss_MB: 5714.6 | uss_MB: 4482.4 | child_rss_MB: 5669.8 | allocated_MB: 370.5 | reserved_MB: 24292.0 | inactive_split_MB: 257.8 +rank6 [2026-10-29 17:01:01,944] INFO : Step 38200/50000 rank6 | Train Loss: 1.0273 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2292 | Cardinality: 0.1060 | Classification: 0.0997 | Classification_binary: 0.1650 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1399 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0706 | Regression_eta: 0.0252 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0702 | LR: 2.18e-04 | DataLoad Time: 0.0148s | Model Forward Time: 3.8909s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7758 | max_loss: 0.1794 | max_backward: 1.7969 | max_optimizer_step: 0.1713 | max_tensorboard_logging: 0.4698 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5717.4 | uss_MB: 4558.1 | child_rss_MB: 5664.1 | allocated_MB: 372.9 | reserved_MB: 26212.0 | inactive_split_MB: 221.1 +rank6 [2026-10-29 17:05:07,130] INFO : Step 38300/50000 rank6 | Train Loss: 1.0498 | Train Loss Unweighted: 0.5122 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1249 | Classification: 0.0913 | Classification_binary: 0.1583 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1294 | Regression_eta: 0.0013 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0582 | Regression_eta: 0.0327 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0680 | LR: 2.16e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7376s | Phase Times: max_pre_step_barrier: 0.1103 | max_batch_to_device: 0.0086 | max_forward: 0.6730 | max_loss: 0.1509 | max_backward: 1.3787 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.4782 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5715.4 | uss_MB: 4556.5 | child_rss_MB: 5665.1 | allocated_MB: 369.2 | reserved_MB: 26212.0 | inactive_split_MB: 171.8 +rank6 [2026-10-29 17:09:08,354] INFO : Step 38400/50000 rank6 | Train Loss: 0.9354 | Train Loss Unweighted: 0.4735 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.0968 | Classification: 0.0760 | Classification_binary: 0.1246 | HitGrouping: 0.1889 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1340 | Regression_eta: 0.0013 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1131 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0603 | Regression_eta: 0.0302 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0412 | LR: 2.14e-04 | DataLoad Time: 0.0214s | Model Forward Time: 3.6833s | Phase Times: max_pre_step_barrier: 0.1107 | max_batch_to_device: 0.0062 | max_forward: 0.6635 | max_loss: 0.1419 | max_backward: 1.4771 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 0.4619 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5721.1 | uss_MB: 4559.4 | child_rss_MB: 5662.2 | allocated_MB: 370.2 | reserved_MB: 26212.0 | inactive_split_MB: 267.8 +rank6 [2026-10-29 17:13:06,091] INFO : Step 38500/50000 rank6 | Train Loss: 1.0482 | Train Loss Unweighted: 0.5077 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1228 | Classification: 0.1147 | Classification_binary: 0.1473 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1208 | Regression_eta: 0.0016 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0975 | Classification_binary: 0.1297 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0541 | Regression_eta: 0.0442 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0708 | LR: 2.12e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.4594s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.6383 | max_loss: 0.1393 | max_backward: 1.3136 | max_optimizer_step: 0.1695 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 5717.0 | uss_MB: 4562.8 | child_rss_MB: 5658.1 | allocated_MB: 370.2 | reserved_MB: 27188.0 | inactive_split_MB: 210.0 +rank6 [2026-10-29 17:17:04,838] INFO : Step 38600/50000 rank6 | Train Loss: 1.0321 | Train Loss Unweighted: 0.4950 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1157 | Classification: 0.0910 | Classification_binary: 0.1735 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1130 | Regression_eta: 0.0014 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0557 | Regression_eta: 0.0449 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0618 | LR: 2.11e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.0846s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0127 | max_forward: 0.7516 | max_loss: 0.1551 | max_backward: 1.6059 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.7250 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5714.3 | uss_MB: 4563.8 | child_rss_MB: 5662.6 | allocated_MB: 372.6 | reserved_MB: 28172.0 | inactive_split_MB: 283.9 +rank6 [2026-10-29 17:20:59,240] INFO : Step 38700/50000 rank6 | Train Loss: 0.9420 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1147 | Classification: 0.0828 | Classification_binary: 0.1348 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1155 | Regression_eta: 0.0013 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1364 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0555 | Regression_eta: 0.0278 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0372 | LR: 2.09e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.6815s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.8052 | max_loss: 0.2133 | max_backward: 1.3215 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5712.0 | uss_MB: 4410.4 | child_rss_MB: 5659.5 | allocated_MB: 367.4 | reserved_MB: 28172.0 | inactive_split_MB: 206.7 +rank6 [2026-10-29 17:25:08,219] INFO : Step 38800/50000 rank6 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4825 | Train Loss Components: Auxiliary: 0.2201 | Cardinality: 0.1186 | Classification: 0.0965 | Classification_binary: 0.1429 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1198 | Regression_eta: 0.0012 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0669 | Regression_eta: 0.0382 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0902 | LR: 2.07e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.6551s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.9011 | max_loss: 0.1880 | max_backward: 1.7487 | max_optimizer_step: 0.2166 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5710.9 | uss_MB: 4431.1 | child_rss_MB: 5666.7 | allocated_MB: 371.5 | reserved_MB: 28172.0 | inactive_split_MB: 220.5 +rank6 [2026-10-29 17:29:07,491] INFO : Step 38900/50000 rank6 | Train Loss: 0.9986 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2211 | Cardinality: 0.1078 | Classification: 0.0988 | Classification_binary: 0.1578 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0566 | Regression_eta: 0.0446 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0411 | LR: 2.05e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.4406s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0073 | max_forward: 0.9800 | max_loss: 0.2159 | max_backward: 2.2158 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0079 | Memory: rss_MB: 5714.2 | uss_MB: 3754.5 | child_rss_MB: 5665.7 | allocated_MB: 373.3 | reserved_MB: 29164.0 | inactive_split_MB: 189.2 +rank6 [2026-10-29 17:33:02,382] INFO : Step 39000/50000 rank6 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4758 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1227 | Classification: 0.0847 | Classification_binary: 0.1518 | HitGrouping: 0.1853 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1179 | Regression_eta: 0.0013 | Regression_pt: 0.1189 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0700 | Regression_eta: 0.0239 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0326 | LR: 2.03e-04 | DataLoad Time: 0.0189s | Model Forward Time: 3.7824s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0515 | max_forward: 0.7410 | max_loss: 0.1790 | max_backward: 1.4655 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5713.4 | uss_MB: 4569.9 | child_rss_MB: 5663.9 | allocated_MB: 370.5 | reserved_MB: 29164.0 | inactive_split_MB: 192.2 +rank6 [2026-10-29 17:33:03,294] INFO : Running validation on rank6 +rank6 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 17:33:56,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:33:56,829] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 17:34:24,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 17:34:25,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 17:34:26,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 17:34:27,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 17:34:37,548] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 17:34:38,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:34:38,632] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 17:34:54,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 17:34:55,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 17:34:56,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 17:34:57,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 17:35:07,463] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 17:35:07,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:35:08,264] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 17:35:23,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 17:35:24,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 17:35:25,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 17:35:25,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 17:35:35,840] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 17:39:50,704] INFO : Step 39100/50000 rank6 | Train Loss: 1.0043 | Train Loss Unweighted: 0.4886 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1149 | Classification: 0.0878 | Classification_binary: 0.1320 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1329 | Regression_eta: 0.0012 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1433 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0988 | Regression_eta: 0.0351 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0869 | LR: 2.01e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.6386s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0736 | max_forward: 0.7590 | max_loss: 0.4062 | max_backward: 1.4330 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.4941 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6810.5 | uss_MB: 5664.9 | child_rss_MB: 5641.2 | allocated_MB: 371.1 | reserved_MB: 23656.0 | inactive_split_MB: 194.8 +rank6 [2026-10-29 17:43:44,583] INFO : Step 39200/50000 rank6 | Train Loss: 1.0325 | Train Loss Unweighted: 0.5174 | Train Loss Components: Auxiliary: 0.2315 | Cardinality: 0.1117 | Classification: 0.1086 | Classification_binary: 0.1481 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0636 | Regression_eta: 0.0422 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0660 | LR: 2.00e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.6953s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0102 | max_forward: 0.7546 | max_loss: 0.4268 | max_backward: 1.4370 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5801.6 | uss_MB: 4660.4 | child_rss_MB: 5666.2 | allocated_MB: 371.2 | reserved_MB: 23934.0 | inactive_split_MB: 196.6 +rank6 [2026-10-29 17:47:41,749] INFO : Step 39300/50000 rank6 | Train Loss: 1.0373 | Train Loss Unweighted: 0.5050 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1241 | Classification: 0.0956 | Classification_binary: 0.1745 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1150 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1735 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0466 | Regression_eta: 0.0332 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0487 | LR: 1.98e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.2674s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0844 | max_forward: 0.8662 | max_loss: 0.2199 | max_backward: 1.7416 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5805.0 | uss_MB: 4500.0 | child_rss_MB: 5642.3 | allocated_MB: 373.1 | reserved_MB: 23934.0 | inactive_split_MB: 256.8 +rank6 [2026-10-29 17:51:43,103] INFO : Step 39400/50000 rank6 | Train Loss: 0.9741 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2189 | Cardinality: 0.1078 | Classification: 0.0808 | Classification_binary: 0.1461 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1741 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0503 | Regression_eta: 0.0311 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0555 | LR: 1.96e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5608s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.6778 | max_loss: 0.1515 | max_backward: 1.4156 | max_optimizer_step: 0.1740 | max_tensorboard_logging: 0.4653 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5807.5 | uss_MB: 4663.4 | child_rss_MB: 5665.4 | allocated_MB: 368.0 | reserved_MB: 23934.0 | inactive_split_MB: 199.1 +rank6 [2026-10-29 17:55:40,096] INFO : Step 39500/50000 rank6 | Train Loss: 0.9809 | Train Loss Unweighted: 0.4664 | Train Loss Components: Auxiliary: 0.2175 | Cardinality: 0.1105 | Classification: 0.0963 | Classification_binary: 0.1347 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1158 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1386 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0615 | Regression_eta: 0.0674 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0726 | LR: 1.94e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5546s | Phase Times: max_pre_step_barrier: 0.0259 | max_batch_to_device: 0.1050 | max_forward: 0.6813 | max_loss: 0.1541 | max_backward: 1.3449 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 0.5375 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0111 | Memory: rss_MB: 5806.7 | uss_MB: 4606.4 | child_rss_MB: 5664.0 | allocated_MB: 367.2 | reserved_MB: 24970.0 | inactive_split_MB: 230.9 +rank6 [2026-10-29 17:59:42,116] INFO : Step 39600/50000 rank6 | Train Loss: 1.0814 | Train Loss Unweighted: 0.5099 | Train Loss Components: Auxiliary: 0.2411 | Cardinality: 0.1344 | Classification: 0.0978 | Classification_binary: 0.1731 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0571 | Regression_eta: 0.0520 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0797 | LR: 1.93e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1484s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8839 | max_loss: 0.1464 | max_backward: 1.8057 | max_optimizer_step: 0.2568 | max_tensorboard_logging: 0.5748 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5805.3 | uss_MB: 4666.8 | child_rss_MB: 5646.1 | allocated_MB: 370.6 | reserved_MB: 24970.0 | inactive_split_MB: 243.4 +rank6 [2026-10-29 18:03:55,339] INFO : Step 39700/50000 rank6 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4877 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1185 | Classification: 0.0909 | Classification_binary: 0.1674 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1134 | Regression_eta: 0.0013 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.1990 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0477 | Regression_eta: 0.0416 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0809 | LR: 1.91e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.8052s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.6954 | max_loss: 0.2515 | max_backward: 1.5651 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.5599 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5806.9 | uss_MB: 4667.8 | child_rss_MB: 5665.2 | allocated_MB: 371.1 | reserved_MB: 24970.0 | inactive_split_MB: 244.9 +rank6 [2026-10-29 18:07:56,647] INFO : Step 39800/50000 rank6 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1022 | Classification: 0.0711 | Classification_binary: 0.1322 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1510 | Regression_cos_phi: 0.0350 | Regression_energy: 0.0508 | Regression_eta: 0.0309 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0470 | LR: 1.89e-04 | DataLoad Time: 0.0131s | Model Forward Time: 4.9319s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0063 | max_forward: 1.0012 | max_loss: 0.2859 | max_backward: 2.1063 | max_optimizer_step: 0.1800 | max_tensorboard_logging: 0.4881 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5810.9 | uss_MB: 3839.1 | child_rss_MB: 5666.9 | allocated_MB: 368.3 | reserved_MB: 24970.0 | inactive_split_MB: 241.7 +rank6 [2026-10-29 18:11:54,160] INFO : Step 39900/50000 rank6 | Train Loss: 0.9667 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.2188 | Cardinality: 0.1055 | Classification: 0.0897 | Classification_binary: 0.1526 | HitGrouping: 0.1883 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1101 | Regression_eta: 0.0012 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0270 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0556 | Regression_eta: 0.0230 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0368 | LR: 1.88e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.9943s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.3186 | max_loss: 0.4377 | max_backward: 3.6189 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.6123 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 5815.3 | uss_MB: 4217.6 | child_rss_MB: 5667.9 | allocated_MB: 370.5 | reserved_MB: 24970.0 | inactive_split_MB: 229.4 +rank6 [2026-10-29 18:15:59,749] INFO : Step 40000/50000 rank6 | Train Loss: 1.0232 | Train Loss Unweighted: 0.5131 | Train Loss Components: Auxiliary: 0.2348 | Cardinality: 0.1100 | Classification: 0.0962 | Classification_binary: 0.1712 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1206 | Regression_eta: 0.0013 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0523 | Regression_eta: 0.0259 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0441 | LR: 1.86e-04 | DataLoad Time: 0.0190s | Model Forward Time: 7.4616s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0077 | max_forward: 0.8168 | max_loss: 0.2161 | max_backward: 1.5709 | max_optimizer_step: 0.1733 | max_tensorboard_logging: 1.1681 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.1427 | Memory: rss_MB: 5816.6 | uss_MB: 3786.6 | child_rss_MB: 5665.4 | allocated_MB: 375.8 | reserved_MB: 24970.0 | inactive_split_MB: 259.2 +rank6 [2026-10-29 18:16:02,308] INFO : Running validation on rank6 +rank6 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 18:16:02,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 18:16:53,651] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:16:54,181] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 18:17:19,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 18:17:20,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 18:17:22,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 18:17:23,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 18:17:33,094] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 18:17:33,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:17:33,662] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 18:17:49,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 18:17:50,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 18:17:51,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 18:17:52,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 18:18:02,368] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:18:03,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:18:03,904] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 18:18:19,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 18:18:19,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 18:18:20,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 18:18:21,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 18:18:31,250] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 18:22:46,459] INFO : Step 40100/50000 rank6 | Train Loss: 0.9349 | Train Loss Unweighted: 0.4412 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1220 | Classification: 0.0762 | Classification_binary: 0.1328 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1148 | Regression_eta: 0.0012 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0605 | Regression_eta: 0.0292 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0420 | LR: 1.84e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.6897s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.7093 | max_loss: 0.3974 | max_backward: 1.3310 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.5555 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 7103.1 | uss_MB: 5953.7 | child_rss_MB: 5672.5 | allocated_MB: 367.6 | reserved_MB: 22178.0 | inactive_split_MB: 265.9 +rank6 [2026-10-29 18:26:52,186] INFO : Step 40200/50000 rank6 | Train Loss: 0.9570 | Train Loss Unweighted: 0.4556 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1119 | Classification: 0.0905 | Classification_binary: 0.1293 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0013 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1201 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0510 | Regression_eta: 0.0271 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0479 | LR: 1.83e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.5890s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6287 | max_loss: 0.4129 | max_backward: 1.3901 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 0.5609 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5918.3 | uss_MB: 4768.5 | child_rss_MB: 5665.4 | allocated_MB: 370.8 | reserved_MB: 22294.0 | inactive_split_MB: 233.2 +rank6 [2026-10-29 18:30:59,535] INFO : Step 40300/50000 rank6 | Train Loss: 1.0209 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1232 | Classification: 0.0991 | Classification_binary: 0.1688 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1068 | Regression_eta: 0.0015 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0461 | Regression_eta: 0.0375 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0505 | LR: 1.81e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9654s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0749 | max_forward: 0.7563 | max_loss: 0.1523 | max_backward: 1.5452 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5632 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5917.0 | uss_MB: 4772.2 | child_rss_MB: 5652.3 | allocated_MB: 371.8 | reserved_MB: 23296.0 | inactive_split_MB: 234.2 +rank6 [2026-10-29 18:35:20,931] INFO : Step 40400/50000 rank6 | Train Loss: 1.0486 | Train Loss Unweighted: 0.5003 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1357 | Classification: 0.0888 | Classification_binary: 0.1690 | HitGrouping: 0.1879 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1196 | Regression_eta: 0.0015 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.2048 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0653 | Regression_eta: 0.0543 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0703 | LR: 1.79e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.7347s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0072 | max_forward: 0.7368 | max_loss: 0.2443 | max_backward: 3.1596 | max_optimizer_step: 0.1851 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5913.1 | uss_MB: 3802.2 | child_rss_MB: 5663.8 | allocated_MB: 371.2 | reserved_MB: 23386.0 | inactive_split_MB: 254.8 +rank6 [2026-10-29 18:39:23,871] INFO : Step 40500/50000 rank6 | Train Loss: 0.9309 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.0955 | Classification: 0.0829 | Classification_binary: 0.1355 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0014 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0595 | Regression_eta: 0.0256 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0472 | LR: 1.78e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7227s | Phase Times: max_pre_step_barrier: 0.0186 | max_batch_to_device: 0.0076 | max_forward: 0.6863 | max_loss: 0.2393 | max_backward: 1.3065 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.7004 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5916.1 | uss_MB: 4771.6 | child_rss_MB: 5669.9 | allocated_MB: 369.2 | reserved_MB: 24370.0 | inactive_split_MB: 237.3 +rank6 [2026-10-29 18:43:23,924] INFO : Step 40600/50000 rank6 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1087 | Classification: 0.0887 | Classification_binary: 0.1521 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1580 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0600 | Regression_eta: 0.0291 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0609 | LR: 1.76e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3455s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.1102 | max_forward: 0.7412 | max_loss: 0.1464 | max_backward: 1.4782 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 1.0719 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5917.7 | uss_MB: 4772.7 | child_rss_MB: 5655.3 | allocated_MB: 372.3 | reserved_MB: 24370.0 | inactive_split_MB: 234.4 +rank6 [2026-10-29 18:47:14,843] INFO : Step 40700/50000 rank6 | Train Loss: 1.0611 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1374 | Classification: 0.0904 | Classification_binary: 0.1773 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1190 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1595 | Regression_cos_phi: 0.0303 | Regression_energy: 0.0518 | Regression_eta: 0.0269 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0388 | LR: 1.75e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2469s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7422 | max_loss: 0.1589 | max_backward: 1.5298 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5918.6 | uss_MB: 4774.9 | child_rss_MB: 5665.5 | allocated_MB: 372.7 | reserved_MB: 24370.0 | inactive_split_MB: 255.3 +rank6 [2026-10-29 18:51:03,370] INFO : Step 40800/50000 rank6 | Train Loss: 0.9701 | Train Loss Unweighted: 0.4785 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.0983 | Classification: 0.0901 | Classification_binary: 0.1373 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1242 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0693 | Regression_eta: 0.0428 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0563 | LR: 1.73e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7271s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5843 | max_loss: 0.1422 | max_backward: 1.0661 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.0389 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5915.8 | uss_MB: 4772.0 | child_rss_MB: 5661.2 | allocated_MB: 367.5 | reserved_MB: 24658.0 | inactive_split_MB: 205.4 +rank6 [2026-10-29 18:54:43,249] INFO : Step 40900/50000 rank6 | Train Loss: 0.9731 | Train Loss Unweighted: 0.4636 | Train Loss Components: Auxiliary: 0.2169 | Cardinality: 0.1073 | Classification: 0.0883 | Classification_binary: 0.1426 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1140 | Regression_eta: 0.0013 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1259 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0509 | Regression_eta: 0.0300 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0571 | LR: 1.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8718s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.1047 | max_forward: 0.7107 | max_loss: 0.1679 | max_backward: 1.3343 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.8664 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5919.0 | uss_MB: 4778.3 | child_rss_MB: 5681.4 | allocated_MB: 372.0 | reserved_MB: 25640.0 | inactive_split_MB: 260.9 +rank6 [2026-10-29 18:58:25,754] INFO : Step 41000/50000 rank6 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1031 | Classification: 0.0930 | Classification_binary: 0.1618 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1164 | Regression_eta: 0.0014 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0500 | Regression_eta: 0.0402 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0552 | LR: 1.70e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4161s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.7604 | max_loss: 0.2558 | max_backward: 1.7603 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.8399 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 5926.0 | uss_MB: 4782.0 | child_rss_MB: 5654.3 | allocated_MB: 371.9 | reserved_MB: 25640.0 | inactive_split_MB: 195.8 +rank6 [2026-10-29 18:58:26,765] INFO : Running validation on rank6 +rank6 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 18:59:14,712] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:59:15,274] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 18:59:40,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 18:59:41,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 18:59:42,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 18:59:43,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 18:59:53,158] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:59:53,544] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:59:54,025] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 19:00:09,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 19:00:10,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 19:00:11,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 19:00:12,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 19:00:22,412] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 19:00:23,208] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:00:23,684] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 19:00:39,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 19:00:40,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 19:00:40,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 19:00:41,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 19:00:51,441] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 19:04:58,255] INFO : Step 41100/50000 rank6 | Train Loss: 1.0684 | Train Loss Unweighted: 0.5053 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1438 | Classification: 0.0856 | Classification_binary: 0.1616 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1271 | Regression_eta: 0.0014 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1518 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0600 | Regression_eta: 0.0315 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0410 | LR: 1.69e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.3324s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0066 | max_forward: 0.8006 | max_loss: 0.1726 | max_backward: 1.6699 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.6214 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0566 | Memory: rss_MB: 7184.9 | uss_MB: 5901.7 | child_rss_MB: 5647.6 | allocated_MB: 371.5 | reserved_MB: 22898.0 | inactive_split_MB: 258.4 +rank6 [2026-10-29 19:08:41,623] INFO : Step 41200/50000 rank6 | Train Loss: 0.9455 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1027 | Classification: 0.0887 | Classification_binary: 0.1242 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1223 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1117 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0555 | Regression_eta: 0.0918 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0823 | LR: 1.67e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.4120s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0589 | max_forward: 0.6517 | max_loss: 0.1401 | max_backward: 1.1761 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5685 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5984.6 | uss_MB: 4847.1 | child_rss_MB: 5628.1 | allocated_MB: 370.2 | reserved_MB: 24878.0 | inactive_split_MB: 199.0 +rank6 [2026-10-29 19:12:28,247] INFO : Step 41300/50000 rank6 | Train Loss: 0.9373 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1011 | Classification: 0.0853 | Classification_binary: 0.1448 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0013 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1517 | Regression_cos_phi: 0.0404 | Regression_energy: 0.0474 | Regression_eta: 0.0252 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0535 | LR: 1.66e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8978s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.8726 | max_loss: 0.1617 | max_backward: 1.6974 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5417 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5987.6 | uss_MB: 4852.4 | child_rss_MB: 5623.6 | allocated_MB: 370.7 | reserved_MB: 24878.0 | inactive_split_MB: 233.6 +rank6 [2026-10-29 19:16:15,464] INFO : Step 41400/50000 rank6 | Train Loss: 1.0355 | Train Loss Unweighted: 0.4965 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1363 | Classification: 0.0957 | Classification_binary: 0.1654 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1162 | Regression_eta: 0.0012 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0532 | Regression_eta: 0.0331 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0423 | LR: 1.64e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.9201s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7129 | max_loss: 0.2460 | max_backward: 1.6340 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5967.8 | uss_MB: 4849.9 | child_rss_MB: 5563.5 | allocated_MB: 372.0 | reserved_MB: 24878.0 | inactive_split_MB: 305.4 +rank6 [2026-10-29 19:19:52,653] INFO : Step 41500/50000 rank6 | Train Loss: 0.9689 | Train Loss Unweighted: 0.4689 | Train Loss Components: Auxiliary: 0.2118 | Cardinality: 0.1040 | Classification: 0.0862 | Classification_binary: 0.1340 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1227 | Regression_eta: 0.0012 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0519 | Regression_eta: 0.0325 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0490 | LR: 1.63e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.4453s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.6415 | max_loss: 0.1502 | max_backward: 1.1845 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5935.8 | uss_MB: 4853.4 | child_rss_MB: 5403.5 | allocated_MB: 368.5 | reserved_MB: 24878.0 | inactive_split_MB: 173.5 +rank6 [2026-10-29 19:23:40,393] INFO : Step 41600/50000 rank6 | Train Loss: 0.9559 | Train Loss Unweighted: 0.4625 | Train Loss Components: Auxiliary: 0.2141 | Cardinality: 0.1097 | Classification: 0.0872 | Classification_binary: 0.1298 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1211 | Regression_eta: 0.0012 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0542 | Regression_eta: 0.0317 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0453 | LR: 1.61e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.5875s | Phase Times: max_pre_step_barrier: 0.0755 | max_batch_to_device: 0.0062 | max_forward: 0.6849 | max_loss: 0.2047 | max_backward: 1.1646 | max_optimizer_step: 0.2235 | max_tensorboard_logging: 0.5806 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5909.3 | uss_MB: 4827.6 | child_rss_MB: 5308.7 | allocated_MB: 370.4 | reserved_MB: 24878.0 | inactive_split_MB: 287.4 +rank6 [2026-10-29 19:27:23,743] INFO : Step 41700/50000 rank6 | Train Loss: 1.0204 | Train Loss Unweighted: 0.4905 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1165 | Classification: 0.0860 | Classification_binary: 0.1552 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0690 | Regression_eta: 0.0515 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0877 | LR: 1.60e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.0315s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0773 | max_forward: 0.7754 | max_loss: 0.1486 | max_backward: 1.5759 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 0.5773 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5907.3 | uss_MB: 4857.3 | child_rss_MB: 5302.2 | allocated_MB: 370.8 | reserved_MB: 24878.0 | inactive_split_MB: 238.3 +rank6 [2026-10-29 19:31:24,299] INFO : Step 41800/50000 rank6 | Train Loss: 1.0101 | Train Loss Unweighted: 0.4703 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1282 | Classification: 0.0818 | Classification_binary: 0.1618 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1861 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0415 | Regression_eta: 0.0334 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0636 | LR: 1.58e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9756s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8147 | max_loss: 0.4259 | max_backward: 1.4886 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1200 | Memory: rss_MB: 5899.0 | uss_MB: 4855.0 | child_rss_MB: 5276.1 | allocated_MB: 371.9 | reserved_MB: 25802.0 | inactive_split_MB: 266.5 +rank6 [2026-10-29 19:35:32,346] INFO : Step 41900/50000 rank6 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4672 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1185 | Classification: 0.0878 | Classification_binary: 0.1282 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1241 | Regression_eta: 0.0011 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0625 | Regression_eta: 0.0230 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0423 | LR: 1.57e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.4626s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0473 | max_forward: 0.6783 | max_loss: 0.1431 | max_backward: 1.2347 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.5634 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5868.0 | uss_MB: 4858.2 | child_rss_MB: 5156.0 | allocated_MB: 370.8 | reserved_MB: 25802.0 | inactive_split_MB: 166.5 +rank6 [2026-10-29 19:39:32,016] INFO : Step 42000/50000 rank6 | Train Loss: 0.9837 | Train Loss Unweighted: 0.4719 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.1093 | Classification: 0.1015 | Classification_binary: 0.1413 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1128 | Regression_eta: 0.0014 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1172 | Regression_cos_phi: 0.0319 | Regression_energy: 0.0476 | Regression_eta: 0.0393 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0428 | LR: 1.56e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.6386s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6848 | max_loss: 0.2509 | max_backward: 1.3746 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5866.0 | uss_MB: 4860.4 | child_rss_MB: 5161.1 | allocated_MB: 370.2 | reserved_MB: 25802.0 | inactive_split_MB: 219.9 +rank6 [2026-10-29 19:39:32,941] INFO : Running validation on rank6 +rank6 [2026-10-29 19:39:32,941] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 19:39:32,941] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 19:40:23,394] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:40:23,892] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 19:40:52,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 19:40:53,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 19:40:54,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 19:40:55,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 19:41:05,823] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 19:41:06,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:41:06,794] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 19:41:23,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 19:41:25,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 19:41:26,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 19:41:27,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 19:41:37,756] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-29 19:41:38,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:41:38,675] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 19:41:54,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 19:41:54,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 19:41:55,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 19:41:56,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 19:42:06,034] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 19:46:29,430] INFO : Step 42100/50000 rank6 | Train Loss: 1.0639 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1434 | Classification: 0.0999 | Classification_binary: 0.1737 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1097 | Regression_eta: 0.0012 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1974 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0589 | Regression_eta: 0.0347 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0723 | LR: 1.54e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.0379s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.7424 | max_loss: 0.1586 | max_backward: 1.4434 | max_optimizer_step: 0.3014 | max_tensorboard_logging: 0.6771 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 7018.7 | uss_MB: 6010.1 | child_rss_MB: 5164.1 | allocated_MB: 370.8 | reserved_MB: 23298.0 | inactive_split_MB: 265.3 +rank6 [2026-10-29 19:50:21,662] INFO : Step 42200/50000 rank6 | Train Loss: 1.0152 | Train Loss Unweighted: 0.4831 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1169 | Classification: 0.0913 | Classification_binary: 0.1515 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1190 | Regression_eta: 0.0015 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1433 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0540 | Regression_eta: 0.0402 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0638 | LR: 1.53e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.5450s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.6567 | max_loss: 0.2000 | max_backward: 1.2379 | max_optimizer_step: 0.1803 | max_tensorboard_logging: 0.5782 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5943.7 | uss_MB: 4948.2 | child_rss_MB: 5171.4 | allocated_MB: 370.9 | reserved_MB: 24984.0 | inactive_split_MB: 272.4 +rank6 [2026-10-29 19:51:24,069] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank6 [2026-10-29 19:51:24,069] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-29 19:54:15,429] INFO : Step 42300/50000 rank6 | Train Loss: 1.0224 | Train Loss Unweighted: 0.4869 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.1261 | Classification: 0.0943 | Classification_binary: 0.1704 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0543 | Regression_eta: 0.0300 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0487 | LR: 1.52e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.7700s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7065 | max_loss: 0.2237 | max_backward: 1.4902 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.5353 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6010.9 | uss_MB: 5019.0 | child_rss_MB: 5170.2 | allocated_MB: 371.9 | reserved_MB: 24984.0 | inactive_split_MB: 183.6 +rank6 [2026-10-29 19:58:13,284] INFO : Step 42400/50000 rank6 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4895 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1253 | Classification: 0.0872 | Classification_binary: 0.1616 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1184 | Regression_eta: 0.0013 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1804 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0455 | Regression_eta: 0.0531 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0717 | LR: 1.50e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7980s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7436 | max_loss: 0.2477 | max_backward: 1.5279 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5169 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5933.8 | uss_MB: 4947.5 | child_rss_MB: 5164.9 | allocated_MB: 373.0 | reserved_MB: 24984.0 | inactive_split_MB: 275.6 +rank6 [2026-10-29 20:02:13,261] INFO : Step 42500/50000 rank6 | Train Loss: 0.9211 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1056 | Classification: 0.0789 | Classification_binary: 0.1296 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1162 | Regression_eta: 0.0011 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1200 | Regression_cos_phi: 0.0345 | Regression_energy: 0.0497 | Regression_eta: 0.0271 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0253 | LR: 1.49e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.5542s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.6196 | max_loss: 0.2440 | max_backward: 1.3403 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 2.3828 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5926.8 | uss_MB: 4949.7 | child_rss_MB: 5158.5 | allocated_MB: 371.3 | reserved_MB: 25980.0 | inactive_split_MB: 274.9 +rank6 [2026-10-29 20:06:17,859] INFO : Step 42600/50000 rank6 | Train Loss: 0.9688 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1101 | Classification: 0.0874 | Classification_binary: 0.1472 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0012 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0946 | Regression_energy: 0.0637 | Regression_eta: 0.0430 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0973 | LR: 1.48e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9495s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6353 | max_loss: 0.2547 | max_backward: 1.3961 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 5918.3 | uss_MB: 4947.3 | child_rss_MB: 5167.3 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 272.5 +rank6 [2026-10-29 20:10:16,914] INFO : Step 42700/50000 rank6 | Train Loss: 1.0476 | Train Loss Unweighted: 0.4808 | Train Loss Components: Auxiliary: 0.2377 | Cardinality: 0.1284 | Classification: 0.0883 | Classification_binary: 0.1739 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1071 | Regression_eta: 0.0014 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1833 | Regression_cos_phi: 0.0448 | Regression_energy: 0.0581 | Regression_eta: 0.0336 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0760 | LR: 1.47e-04 | DataLoad Time: 0.0111s | Model Forward Time: 5.7543s | Phase Times: max_pre_step_barrier: 0.0320 | max_batch_to_device: 0.0114 | max_forward: 2.1726 | max_loss: 0.5101 | max_backward: 2.7494 | max_optimizer_step: 0.2679 | max_tensorboard_logging: 0.6520 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5921.8 | uss_MB: 4952.4 | child_rss_MB: 5151.8 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 482.9 +rank6 [2026-10-29 20:14:17,050] INFO : Step 42800/50000 rank6 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4792 | Train Loss Components: Auxiliary: 0.2262 | Cardinality: 0.1277 | Classification: 0.0841 | Classification_binary: 0.1487 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1209 | Regression_eta: 0.0012 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0570 | Regression_eta: 0.0449 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0525 | LR: 1.45e-04 | DataLoad Time: 0.0182s | Model Forward Time: 6.2791s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0175 | max_forward: 1.3870 | max_loss: 0.5106 | max_backward: 1.9221 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 2.4017 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5924.3 | uss_MB: 4952.5 | child_rss_MB: 5165.0 | allocated_MB: 372.6 | reserved_MB: 25980.0 | inactive_split_MB: 234.4 +rank6 [2026-10-29 20:18:09,439] INFO : Step 42900/50000 rank6 | Train Loss: 0.8847 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.0993 | Classification: 0.0847 | Classification_binary: 0.1207 | HitGrouping: 0.1828 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1069 | Regression_eta: 0.0011 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1269 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0550 | Regression_eta: 0.0328 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0766 | LR: 1.44e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.7401s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.6133 | max_loss: 0.1478 | max_backward: 1.3445 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.7551 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0762 | Memory: rss_MB: 5924.9 | uss_MB: 4957.1 | child_rss_MB: 5179.7 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 276.9 +rank6 [2026-10-29 20:22:02,171] INFO : Step 43000/50000 rank6 | Train Loss: 0.9800 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1096 | Classification: 0.0987 | Classification_binary: 0.1606 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0012 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0355 | Regression_energy: 0.0467 | Regression_eta: 0.0225 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0360 | LR: 1.43e-04 | DataLoad Time: 0.0219s | Model Forward Time: 3.9018s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0071 | max_forward: 0.7477 | max_loss: 0.1477 | max_backward: 1.5663 | max_optimizer_step: 0.1600 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 5929.7 | uss_MB: 4961.8 | child_rss_MB: 5171.4 | allocated_MB: 369.1 | reserved_MB: 25980.0 | inactive_split_MB: 209.4 +rank6 [2026-10-29 20:22:03,144] INFO : Running validation on rank6 +rank6 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 20:22:51,801] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:22:52,330] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 20:23:18,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 20:23:19,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 20:23:20,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 20:23:21,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 20:23:31,555] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 20:23:31,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:23:32,371] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 20:23:48,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 20:23:49,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 20:23:50,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 20:23:51,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 20:24:01,123] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 20:24:01,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:24:02,214] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 20:24:17,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 20:24:18,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 20:24:18,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 20:24:19,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 20:24:29,371] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 20:28:29,103] INFO : Step 43100/50000 rank6 | Train Loss: 0.9692 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1151 | Classification: 0.0766 | Classification_binary: 0.1586 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1062 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0415 | Regression_eta: 0.0452 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0692 | LR: 1.42e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7257s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.8051 | max_loss: 0.1438 | max_backward: 1.3224 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 7068.8 | uss_MB: 6101.3 | child_rss_MB: 5191.0 | allocated_MB: 370.5 | reserved_MB: 21096.0 | inactive_split_MB: 247.5 +rank6 [2026-10-29 20:32:30,812] INFO : Step 43200/50000 rank6 | Train Loss: 0.9431 | Train Loss Unweighted: 0.4458 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1175 | Classification: 0.0811 | Classification_binary: 0.1294 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0012 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0536 | Regression_eta: 0.0468 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0387 | LR: 1.40e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5216s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0120 | max_forward: 0.6342 | max_loss: 0.1417 | max_backward: 1.2752 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6068.4 | uss_MB: 5101.7 | child_rss_MB: 5194.2 | allocated_MB: 370.4 | reserved_MB: 22140.0 | inactive_split_MB: 198.0 +rank6 [2026-10-29 20:36:16,428] INFO : Step 43300/50000 rank6 | Train Loss: 1.0382 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1221 | Classification: 0.1031 | Classification_binary: 0.1555 | HitGrouping: 0.1894 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1173 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0539 | Regression_eta: 0.0341 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0622 | LR: 1.39e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2395s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0266 | max_forward: 0.7527 | max_loss: 0.1476 | max_backward: 1.6633 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.7701 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6071.5 | uss_MB: 5107.2 | child_rss_MB: 5175.9 | allocated_MB: 371.1 | reserved_MB: 23082.0 | inactive_split_MB: 236.4 +rank6 [2026-10-29 20:40:05,495] INFO : Step 43400/50000 rank6 | Train Loss: 1.0532 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1292 | Classification: 0.0981 | Classification_binary: 0.1785 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0015 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1756 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0504 | Regression_eta: 0.0434 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0534 | LR: 1.38e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2967s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0691 | max_forward: 0.8057 | max_loss: 0.1503 | max_backward: 1.8319 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.6703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6067.0 | uss_MB: 5102.4 | child_rss_MB: 5192.5 | allocated_MB: 371.5 | reserved_MB: 23082.0 | inactive_split_MB: 776.0 +rank6 [2026-10-29 20:43:57,009] INFO : Step 43500/50000 rank6 | Train Loss: 0.9308 | Train Loss Unweighted: 0.4531 | Train Loss Components: Auxiliary: 0.2084 | Cardinality: 0.0966 | Classification: 0.0846 | Classification_binary: 0.1384 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1135 | Regression_eta: 0.0013 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0581 | Regression_eta: 0.0408 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0685 | LR: 1.37e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.6269s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0171 | max_forward: 0.6718 | max_loss: 0.1455 | max_backward: 1.4141 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 6070.4 | uss_MB: 5108.5 | child_rss_MB: 5191.0 | allocated_MB: 372.0 | reserved_MB: 23082.0 | inactive_split_MB: 228.0 +rank6 [2026-10-29 20:47:52,743] INFO : Step 43600/50000 rank6 | Train Loss: 0.9195 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.0926 | Classification: 0.0847 | Classification_binary: 0.1298 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1175 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1199 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0831 | Regression_eta: 0.0294 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0912 | LR: 1.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5151s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.6453 | max_loss: 0.1399 | max_backward: 1.3514 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0942 | Memory: rss_MB: 6067.9 | uss_MB: 5106.0 | child_rss_MB: 5190.1 | allocated_MB: 367.6 | reserved_MB: 23082.0 | inactive_split_MB: 270.4 +rank6 [2026-10-29 20:51:54,645] INFO : Step 43700/50000 rank6 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1181 | Classification: 0.0916 | Classification_binary: 0.1553 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1009 | Regression_eta: 0.0014 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0448 | Regression_eta: 0.0322 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0563 | LR: 1.35e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.0772s | Phase Times: max_pre_step_barrier: 0.0199 | max_batch_to_device: 0.0121 | max_forward: 0.8005 | max_loss: 0.2526 | max_backward: 1.9159 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 6075.4 | uss_MB: 5110.5 | child_rss_MB: 5168.5 | allocated_MB: 371.6 | reserved_MB: 23674.0 | inactive_split_MB: 256.4 +rank6 [2026-10-29 20:55:51,187] INFO : Step 43800/50000 rank6 | Train Loss: 1.0325 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1341 | Classification: 0.0892 | Classification_binary: 0.1542 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0012 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1578 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0492 | Regression_eta: 0.0292 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0531 | LR: 1.34e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6844s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7618 | max_loss: 0.1715 | max_backward: 1.3818 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6073.5 | uss_MB: 5110.6 | child_rss_MB: 5186.9 | allocated_MB: 371.7 | reserved_MB: 23674.0 | inactive_split_MB: 282.3 +rank6 [2026-10-29 20:59:46,231] INFO : Step 43900/50000 rank6 | Train Loss: 0.9104 | Train Loss Unweighted: 0.4325 | Train Loss Components: Auxiliary: 0.2009 | Cardinality: 0.1078 | Classification: 0.0725 | Classification_binary: 0.1272 | HitGrouping: 0.1855 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1146 | Regression_eta: 0.0011 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1206 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0407 | Regression_eta: 0.0210 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0337 | LR: 1.33e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.3855s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.6145 | max_loss: 0.1412 | max_backward: 1.1415 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5764 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6071.8 | uss_MB: 5111.9 | child_rss_MB: 5178.4 | allocated_MB: 368.3 | reserved_MB: 23674.0 | inactive_split_MB: 193.1 +rank6 [2026-10-29 21:03:43,513] INFO : Step 44000/50000 rank6 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4573 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1078 | Classification: 0.0956 | Classification_binary: 0.1421 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1083 | Regression_eta: 0.0011 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1302 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0669 | Regression_eta: 0.0255 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0377 | LR: 1.32e-04 | DataLoad Time: 0.0185s | Model Forward Time: 3.9501s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0061 | max_forward: 1.1697 | max_loss: 0.2455 | max_backward: 1.7366 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6074.9 | uss_MB: 5118.1 | child_rss_MB: 5179.2 | allocated_MB: 370.7 | reserved_MB: 23674.0 | inactive_split_MB: 207.3 +rank6 [2026-10-29 21:03:44,527] INFO : Running validation on rank6 +rank6 [2026-10-29 21:03:44,527] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 21:03:44,528] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 21:04:32,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:04:33,227] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 21:04:58,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 21:04:59,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 21:05:01,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 21:05:02,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 21:05:12,032] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 21:05:12,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:05:12,998] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 21:05:28,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 21:05:29,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 21:05:30,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 21:05:31,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 21:05:41,724] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 21:05:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:05:42,732] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 21:05:58,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 21:05:58,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 21:05:59,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 21:06:00,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 21:06:10,194] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 21:10:20,128] INFO : Step 44100/50000 rank6 | Train Loss: 1.0009 | Train Loss Unweighted: 0.4605 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1400 | Classification: 0.0841 | Classification_binary: 0.1644 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1044 | Regression_eta: 0.0011 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0456 | Regression_eta: 0.0300 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0493 | LR: 1.31e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.5856s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 1.2763 | max_loss: 0.5349 | max_backward: 2.2676 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 1.2755 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 7076.5 | uss_MB: 6129.8 | child_rss_MB: 5164.2 | allocated_MB: 372.3 | reserved_MB: 24224.0 | inactive_split_MB: 272.2 +rank6 [2026-10-29 21:14:15,311] INFO : Step 44200/50000 rank6 | Train Loss: 0.9801 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1146 | Classification: 0.0882 | Classification_binary: 0.1482 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1167 | Regression_eta: 0.0014 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0371 | Regression_energy: 0.0524 | Regression_eta: 0.0274 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0421 | LR: 1.30e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.0052s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0132 | max_forward: 0.6993 | max_loss: 0.1574 | max_backward: 1.3862 | max_optimizer_step: 0.2915 | max_tensorboard_logging: 0.7724 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6137.2 | uss_MB: 5191.5 | child_rss_MB: 5174.4 | allocated_MB: 370.2 | reserved_MB: 26182.0 | inactive_split_MB: 176.5 +rank6 [2026-10-29 21:17:58,578] INFO : Step 44300/50000 rank6 | Train Loss: 0.8787 | Train Loss Unweighted: 0.4332 | Train Loss Components: Auxiliary: 0.1952 | Cardinality: 0.0950 | Classification: 0.0869 | Classification_binary: 0.1153 | HitGrouping: 0.1807 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0011 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0297 | Regression_energy: 0.0554 | Regression_eta: 0.0489 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0436 | LR: 1.29e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.7426s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0142 | max_forward: 0.7699 | max_loss: 0.2088 | max_backward: 1.4049 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 0.4839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 6137.2 | uss_MB: 4195.8 | child_rss_MB: 5178.8 | allocated_MB: 367.2 | reserved_MB: 27204.0 | inactive_split_MB: 205.8 +rank6 [2026-10-29 21:21:59,794] INFO : Step 44400/50000 rank6 | Train Loss: 0.9558 | Train Loss Unweighted: 0.4680 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.0905 | Classification: 0.0911 | Classification_binary: 0.1587 | HitGrouping: 0.1858 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1383 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0418 | Regression_eta: 0.0351 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0386 | LR: 1.28e-04 | DataLoad Time: 0.0215s | Model Forward Time: 6.3267s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0370 | max_forward: 0.7829 | max_loss: 0.1727 | max_backward: 1.7862 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 2.7361 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6138.4 | uss_MB: 5206.0 | child_rss_MB: 5161.4 | allocated_MB: 373.1 | reserved_MB: 27586.0 | inactive_split_MB: 176.3 +rank6 [2026-10-29 21:26:11,867] INFO : Step 44500/50000 rank6 | Train Loss: 0.9739 | Train Loss Unweighted: 0.4610 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1139 | Classification: 0.0849 | Classification_binary: 0.1578 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1077 | Regression_eta: 0.0012 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1708 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0531 | Regression_eta: 0.0352 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0572 | LR: 1.27e-04 | DataLoad Time: 0.0096s | Model Forward Time: 3.9088s | Phase Times: max_pre_step_barrier: 0.0250 | max_batch_to_device: 0.0065 | max_forward: 0.7930 | max_loss: 0.1668 | max_backward: 1.3363 | max_optimizer_step: 0.2871 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6136.3 | uss_MB: 5200.8 | child_rss_MB: 5177.3 | allocated_MB: 371.5 | reserved_MB: 27586.0 | inactive_split_MB: 195.8 +rank6 [2026-10-29 21:30:04,561] INFO : Step 44600/50000 rank6 | Train Loss: 0.8904 | Train Loss Unweighted: 0.4364 | Train Loss Components: Auxiliary: 0.2007 | Cardinality: 0.1041 | Classification: 0.0775 | Classification_binary: 0.1238 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1161 | Regression_eta: 0.0010 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1209 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0777 | Regression_eta: 0.0235 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0466 | LR: 1.26e-04 | DataLoad Time: 0.0183s | Model Forward Time: 3.5124s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.6364 | max_loss: 0.1431 | max_backward: 1.3712 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.4900 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6139.4 | uss_MB: 5205.0 | child_rss_MB: 5167.5 | allocated_MB: 368.4 | reserved_MB: 27586.0 | inactive_split_MB: 170.2 +rank6 [2026-10-29 21:33:51,596] INFO : Step 44700/50000 rank6 | Train Loss: 1.0003 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1252 | Classification: 0.0899 | Classification_binary: 0.1520 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1117 | Regression_eta: 0.0013 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0465 | Regression_eta: 0.0701 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0548 | LR: 1.25e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6374s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6952 | max_loss: 0.1455 | max_backward: 1.5015 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6145.1 | uss_MB: 5206.5 | child_rss_MB: 5162.4 | allocated_MB: 371.9 | reserved_MB: 28574.0 | inactive_split_MB: 214.1 +rank6 [2026-10-29 21:37:55,987] INFO : Step 44800/50000 rank6 | Train Loss: 0.9665 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1115 | Classification: 0.0889 | Classification_binary: 0.1605 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1042 | Regression_eta: 0.0011 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1600 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0456 | Regression_eta: 0.0275 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0605 | LR: 1.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.8732s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7593 | max_loss: 0.1924 | max_backward: 2.8086 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.6087 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 6148.4 | uss_MB: 5210.8 | child_rss_MB: 5160.0 | allocated_MB: 370.8 | reserved_MB: 28574.0 | inactive_split_MB: 257.7 +rank6 [2026-10-29 21:41:36,915] INFO : Step 44900/50000 rank6 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.0886 | Classification_binary: 0.1466 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1121 | Regression_eta: 0.0013 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0536 | Regression_eta: 0.0399 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0690 | LR: 1.23e-04 | DataLoad Time: 0.0149s | Model Forward Time: 3.5200s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0101 | max_forward: 0.6531 | max_loss: 0.1431 | max_backward: 1.3230 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6145.9 | uss_MB: 5207.3 | child_rss_MB: 5182.8 | allocated_MB: 368.1 | reserved_MB: 28574.0 | inactive_split_MB: 178.8 +rank6 [2026-10-29 21:45:50,736] INFO : Step 45000/50000 rank6 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4443 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1053 | Classification: 0.0832 | Classification_binary: 0.1311 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0363 | Classification_binary: 0.1260 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0510 | Regression_eta: 0.0428 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0602 | LR: 1.22e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1199s | Phase Times: max_pre_step_barrier: 0.0074 | max_batch_to_device: 0.0159 | max_forward: 1.1897 | max_loss: 0.2457 | max_backward: 1.2951 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.8342 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0919 | Memory: rss_MB: 6152.1 | uss_MB: 5213.7 | child_rss_MB: 5170.7 | allocated_MB: 368.8 | reserved_MB: 28576.0 | inactive_split_MB: 266.4 +rank6 [2026-10-29 21:45:51,739] INFO : Running validation on rank6 +rank6 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 21:45:51,739] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 21:46:40,563] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:46:41,117] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 21:47:08,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 21:47:10,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 21:47:11,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 21:47:12,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 21:47:22,755] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 21:47:23,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:47:23,723] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 21:47:39,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 21:47:40,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 21:47:41,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 21:47:42,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 21:47:52,631] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 21:47:53,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:47:53,965] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 21:48:10,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 21:48:10,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 21:48:11,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 21:48:12,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 21:48:22,243] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 21:52:56,452] INFO : Step 45100/50000 rank6 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1151 | Classification: 0.1010 | Classification_binary: 0.1562 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1118 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0356 | Regression_energy: 0.0497 | Regression_eta: 0.0193 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0314 | LR: 1.21e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.3045s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7307 | max_loss: 0.4409 | max_backward: 1.5823 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 7066.7 | uss_MB: 6122.4 | child_rss_MB: 5160.9 | allocated_MB: 373.1 | reserved_MB: 23280.0 | inactive_split_MB: 227.2 +rank6 [2026-10-29 21:56:57,436] INFO : Step 45200/50000 rank6 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1108 | Classification: 0.0815 | Classification_binary: 0.1593 | HitGrouping: 0.1884 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1754 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0464 | Regression_eta: 0.0370 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0591 | LR: 1.20e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.2202s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.9482 | max_loss: 0.4476 | max_backward: 1.7165 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.4656 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7064.1 | uss_MB: 6120.3 | child_rss_MB: 5157.7 | allocated_MB: 373.0 | reserved_MB: 23338.0 | inactive_split_MB: 277.4 +rank6 [2026-10-29 22:01:28,909] INFO : Step 45300/50000 rank6 | Train Loss: 0.9312 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1141 | Classification: 0.0882 | Classification_binary: 0.1299 | HitGrouping: 0.1891 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0010 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1303 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0534 | Regression_eta: 0.0272 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0511 | LR: 1.19e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.4277s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6106 | max_loss: 0.2394 | max_backward: 1.1660 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 6271.7 | uss_MB: 5329.4 | child_rss_MB: 5157.3 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 223.2 +rank6 [2026-10-29 22:05:37,387] INFO : Step 45400/50000 rank6 | Train Loss: 0.9251 | Train Loss Unweighted: 0.4554 | Train Loss Components: Auxiliary: 0.2112 | Cardinality: 0.1034 | Classification: 0.0967 | Classification_binary: 0.1405 | HitGrouping: 0.1801 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1072 | Regression_eta: 0.0012 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1527 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0567 | Regression_eta: 0.0350 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0443 | LR: 1.19e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.9532s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7158 | max_loss: 0.2520 | max_backward: 1.7727 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6276.9 | uss_MB: 5335.5 | child_rss_MB: 5162.7 | allocated_MB: 372.6 | reserved_MB: 23338.0 | inactive_split_MB: 212.6 +rank6 [2026-10-29 22:09:49,791] INFO : Step 45500/50000 rank6 | Train Loss: 1.0136 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1238 | Classification: 0.0902 | Classification_binary: 0.1661 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1060 | Regression_eta: 0.0013 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1844 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0553 | Regression_eta: 0.0512 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0524 | LR: 1.18e-04 | DataLoad Time: 0.0116s | Model Forward Time: 6.1533s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0147 | max_forward: 1.9023 | max_loss: 0.2979 | max_backward: 3.5478 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5661 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 6272.0 | uss_MB: 4281.7 | child_rss_MB: 5159.4 | allocated_MB: 372.4 | reserved_MB: 23338.0 | inactive_split_MB: 256.1 +rank6 [2026-10-29 22:14:16,023] INFO : Step 45600/50000 rank6 | Train Loss: 0.9675 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2172 | Cardinality: 0.1193 | Classification: 0.0849 | Classification_binary: 0.1400 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1218 | Regression_eta: 0.0010 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0620 | Regression_eta: 0.0375 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0536 | LR: 1.17e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7207s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6781 | max_loss: 0.1576 | max_backward: 1.5089 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 0.5447 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6272.0 | uss_MB: 5337.5 | child_rss_MB: 5161.4 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 216.2 +rank6 [2026-10-29 22:18:15,631] INFO : Step 45700/50000 rank6 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1961 | Cardinality: 0.0908 | Classification: 0.0837 | Classification_binary: 0.1223 | HitGrouping: 0.1794 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1108 | Regression_eta: 0.0011 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1229 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0489 | Regression_eta: 0.0355 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0330 | LR: 1.16e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.4720s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6216 | max_loss: 0.1658 | max_backward: 1.1797 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6234.4 | uss_MB: 5336.4 | child_rss_MB: 5162.7 | allocated_MB: 370.8 | reserved_MB: 23338.0 | inactive_split_MB: 226.6 +rank6 [2026-10-29 22:22:19,535] INFO : Step 45800/50000 rank6 | Train Loss: 0.9372 | Train Loss Unweighted: 0.4491 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1073 | Classification: 0.0880 | Classification_binary: 0.1481 | HitGrouping: 0.1781 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1242 | Regression_cos_phi: 0.0337 | Regression_energy: 0.0357 | Regression_eta: 0.0209 | Regression_pt: 0.0376 | Regression_sin_phi: 0.0325 | LR: 1.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2358s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8017 | max_loss: 0.2428 | max_backward: 1.9917 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0978 | Memory: rss_MB: 6240.7 | uss_MB: 5347.0 | child_rss_MB: 5162.1 | allocated_MB: 375.2 | reserved_MB: 23338.0 | inactive_split_MB: 268.8 +rank6 [2026-10-29 22:26:26,930] INFO : Step 45900/50000 rank6 | Train Loss: 0.9600 | Train Loss Unweighted: 0.4592 | Train Loss Components: Auxiliary: 0.2179 | Cardinality: 0.1027 | Classification: 0.0816 | Classification_binary: 0.1590 | HitGrouping: 0.1875 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0510 | Regression_eta: 0.0288 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0587 | LR: 1.15e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.8426s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 1.3210 | max_loss: 0.4294 | max_backward: 3.2770 | max_optimizer_step: 0.2740 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 6235.6 | uss_MB: 5343.7 | child_rss_MB: 5161.3 | allocated_MB: 370.5 | reserved_MB: 23338.0 | inactive_split_MB: 259.6 +rank6 [2026-10-29 22:30:40,887] INFO : Step 46000/50000 rank6 | Train Loss: 0.8490 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.0938 | Classification: 0.0817 | Classification_binary: 0.1110 | HitGrouping: 0.1819 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0010 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1163 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0619 | Regression_eta: 0.0260 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0498 | LR: 1.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.5927s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.5955 | max_loss: 0.1430 | max_backward: 1.2483 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 0.6657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 6228.9 | uss_MB: 5344.9 | child_rss_MB: 5164.3 | allocated_MB: 365.7 | reserved_MB: 23338.0 | inactive_split_MB: 222.2 +rank6 [2026-10-29 22:30:41,828] INFO : Running validation on rank6 +rank6 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 22:31:32,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:31:33,519] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 22:31:59,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 22:32:01,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 22:32:03,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 22:32:05,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 22:32:15,055] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-29 22:32:15,390] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:32:16,223] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 22:32:32,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 22:32:33,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 22:32:34,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 22:32:35,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 22:32:45,408] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 22:32:45,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:32:46,015] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 22:33:01,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 22:33:01,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 22:33:02,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 22:33:03,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 22:33:13,095] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-29 22:37:44,432] INFO : Step 46100/50000 rank6 | Train Loss: 0.9299 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1082 | Classification: 0.0899 | Classification_binary: 0.1357 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1081 | Regression_eta: 0.0010 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0529 | Regression_eta: 0.0309 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0688 | LR: 1.13e-04 | DataLoad Time: 0.0193s | Model Forward Time: 6.9693s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 2.5146 | max_loss: 0.4579 | max_backward: 2.4958 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1014 | Memory: rss_MB: 6980.2 | uss_MB: 6096.7 | child_rss_MB: 5172.0 | allocated_MB: 371.5 | reserved_MB: 22784.0 | inactive_split_MB: 245.0 +rank6 [2026-10-29 22:41:43,511] INFO : Step 46200/50000 rank6 | Train Loss: 0.9944 | Train Loss Unweighted: 0.4707 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1286 | Classification: 0.0940 | Classification_binary: 0.1623 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1052 | Regression_eta: 0.0011 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0593 | Regression_eta: 0.0274 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0647 | LR: 1.13e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2034s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7685 | max_loss: 0.2359 | max_backward: 1.6855 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1253 | Memory: rss_MB: 6976.0 | uss_MB: 6094.5 | child_rss_MB: 5141.0 | allocated_MB: 373.1 | reserved_MB: 22784.0 | inactive_split_MB: 264.3 +rank6 [2026-10-29 22:45:39,134] INFO : Step 46300/50000 rank6 | Train Loss: 0.9564 | Train Loss Unweighted: 0.4515 | Train Loss Components: Auxiliary: 0.2114 | Cardinality: 0.1124 | Classification: 0.0869 | Classification_binary: 0.1435 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0453 | Regression_eta: 0.0322 | Regression_pt: 0.0455 | Regression_sin_phi: 0.0730 | LR: 1.12e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8634s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6709 | max_loss: 0.2410 | max_backward: 1.2189 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.8303 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6276.5 | uss_MB: 5413.4 | child_rss_MB: 5109.4 | allocated_MB: 372.3 | reserved_MB: 23784.0 | inactive_split_MB: 228.2 +rank6 [2026-10-29 22:49:34,649] INFO : Step 46400/50000 rank6 | Train Loss: 0.9514 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1147 | Classification: 0.0822 | Classification_binary: 0.1248 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0010 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0390 | Classification_binary: 0.1130 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0728 | Regression_eta: 0.0449 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0465 | LR: 1.11e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.9537s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.6748 | max_loss: 0.4371 | max_backward: 1.4077 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.8211 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6270.0 | uss_MB: 5413.5 | child_rss_MB: 5075.5 | allocated_MB: 374.2 | reserved_MB: 23988.0 | inactive_split_MB: 241.8 +rank6 [2026-10-29 22:53:41,742] INFO : Step 46500/50000 rank6 | Train Loss: 0.9517 | Train Loss Unweighted: 0.4566 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.0937 | Classification: 0.0928 | Classification_binary: 0.1533 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1031 | Regression_eta: 0.0013 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0502 | Regression_eta: 0.0255 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0530 | LR: 1.11e-04 | DataLoad Time: 0.0145s | Model Forward Time: 3.9663s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0087 | max_forward: 0.7136 | max_loss: 0.1507 | max_backward: 1.5229 | max_optimizer_step: 0.2767 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6272.9 | uss_MB: 5416.3 | child_rss_MB: 5079.6 | allocated_MB: 372.0 | reserved_MB: 23988.0 | inactive_split_MB: 248.1 +rank6 [2026-10-29 22:57:41,614] INFO : Step 46600/50000 rank6 | Train Loss: 0.9541 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.2166 | Cardinality: 0.1078 | Classification: 0.0915 | Classification_binary: 0.1560 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0555 | Regression_eta: 0.0304 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0606 | LR: 1.10e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.5741s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7127 | max_loss: 0.1604 | max_backward: 1.3079 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4722 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 6261.6 | uss_MB: 5413.8 | child_rss_MB: 5045.9 | allocated_MB: 370.5 | reserved_MB: 23988.0 | inactive_split_MB: 179.8 +rank6 [2026-10-29 23:02:40,690] INFO : Step 46700/50000 rank6 | Train Loss: 0.9378 | Train Loss Unweighted: 0.4497 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.0977 | Classification: 0.0857 | Classification_binary: 0.1264 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0015 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0481 | Regression_eta: 0.0368 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0443 | LR: 1.10e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.7483s | Phase Times: max_pre_step_barrier: 0.0197 | max_batch_to_device: 0.0481 | max_forward: 0.6699 | max_loss: 0.2593 | max_backward: 1.4356 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5914 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6261.6 | uss_MB: 5415.0 | child_rss_MB: 5040.6 | allocated_MB: 368.9 | reserved_MB: 24784.0 | inactive_split_MB: 229.2 +rank6 [2026-10-29 23:06:36,577] INFO : Step 46800/50000 rank6 | Train Loss: 0.9152 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.2022 | Cardinality: 0.1011 | Classification: 0.0910 | Classification_binary: 0.1376 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0988 | Regression_eta: 0.0014 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0446 | Regression_eta: 0.0205 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0319 | LR: 1.09e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6021s | Phase Times: max_pre_step_barrier: 0.1216 | max_batch_to_device: 0.0144 | max_forward: 1.0361 | max_loss: 0.2737 | max_backward: 1.7951 | max_optimizer_step: 0.2978 | max_tensorboard_logging: 0.5843 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6263.1 | uss_MB: 5417.5 | child_rss_MB: 5044.7 | allocated_MB: 368.9 | reserved_MB: 24994.0 | inactive_split_MB: 204.7 +rank6 [2026-10-29 23:10:45,824] INFO : Step 46900/50000 rank6 | Train Loss: 0.9278 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2149 | Cardinality: 0.1073 | Classification: 0.0827 | Classification_binary: 0.1607 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1027 | Regression_eta: 0.0010 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1688 | Regression_cos_phi: 0.0371 | Regression_energy: 0.0514 | Regression_eta: 0.0296 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0304 | LR: 1.09e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0946s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0446 | max_forward: 0.7815 | max_loss: 0.1553 | max_backward: 1.4732 | max_optimizer_step: 0.2929 | max_tensorboard_logging: 0.6668 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6256.7 | uss_MB: 5416.7 | child_rss_MB: 5041.4 | allocated_MB: 369.3 | reserved_MB: 24994.0 | inactive_split_MB: 274.7 +rank6 [2026-10-29 23:14:40,386] INFO : Step 47000/50000 rank6 | Train Loss: 0.9023 | Train Loss Unweighted: 0.4500 | Train Loss Components: Auxiliary: 0.2093 | Cardinality: 0.0953 | Classification: 0.0802 | Classification_binary: 0.1325 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1175 | Regression_eta: 0.0010 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1287 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0498 | Regression_eta: 0.0142 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0469 | LR: 1.08e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5083s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6624 | max_loss: 0.1440 | max_backward: 1.2028 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 6259.7 | uss_MB: 5421.0 | child_rss_MB: 5043.0 | allocated_MB: 369.4 | reserved_MB: 25784.0 | inactive_split_MB: 224.5 +rank6 [2026-10-29 23:14:41,333] INFO : Running validation on rank6 +rank6 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 23:15:32,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:15:33,149] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 23:16:03,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 23:16:04,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 23:16:05,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 23:16:06,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 23:16:16,017] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 23:16:16,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:16:16,906] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 23:16:33,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 23:16:34,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 23:16:35,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 23:16:36,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 23:16:46,175] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 23:16:47,374] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:16:47,863] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 23:17:03,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 23:17:04,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 23:17:05,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 23:17:05,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 23:17:15,957] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 23:21:36,067] INFO : Step 47100/50000 rank6 | Train Loss: 0.9138 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.2058 | Cardinality: 0.0938 | Classification: 0.0946 | Classification_binary: 0.1243 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1103 | Regression_eta: 0.0011 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1189 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0541 | Regression_eta: 0.0285 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0661 | LR: 1.07e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1756s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0063 | max_forward: 0.8048 | max_loss: 0.2433 | max_backward: 1.3159 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 1.0353 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 6952.3 | uss_MB: 6113.6 | child_rss_MB: 5037.2 | allocated_MB: 368.1 | reserved_MB: 22106.0 | inactive_split_MB: 269.0 +rank6 [2026-10-29 23:25:45,623] INFO : Step 47200/50000 rank6 | Train Loss: 0.9875 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2263 | Cardinality: 0.1259 | Classification: 0.0903 | Classification_binary: 0.1586 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1110 | Regression_eta: 0.0012 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0530 | Regression_eta: 0.0276 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0532 | LR: 1.07e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2833s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0486 | max_forward: 0.7581 | max_loss: 0.1506 | max_backward: 1.5914 | max_optimizer_step: 0.2300 | max_tensorboard_logging: 0.7953 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 6951.3 | uss_MB: 6117.8 | child_rss_MB: 5032.9 | allocated_MB: 370.5 | reserved_MB: 23104.0 | inactive_split_MB: 195.8 +rank6 [2026-10-29 23:29:53,763] INFO : Step 47300/50000 rank6 | Train Loss: 0.9350 | Train Loss Unweighted: 0.4467 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.0958 | Classification: 0.0857 | Classification_binary: 0.1512 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1517 | Regression_cos_phi: 0.0360 | Regression_energy: 0.0474 | Regression_eta: 0.0252 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0498 | LR: 1.06e-04 | DataLoad Time: 0.0226s | Model Forward Time: 6.3999s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0184 | max_forward: 1.9890 | max_loss: 0.4771 | max_backward: 3.1656 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6309.1 | uss_MB: 5475.6 | child_rss_MB: 5028.4 | allocated_MB: 370.8 | reserved_MB: 23104.0 | inactive_split_MB: 271.1 +rank6 [2026-10-29 23:33:58,356] INFO : Step 47400/50000 rank6 | Train Loss: 0.8593 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.0807 | Classification: 0.0812 | Classification_binary: 0.1182 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1114 | Regression_eta: 0.0011 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0503 | Regression_eta: 0.0274 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0425 | LR: 1.06e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.0130s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0125 | max_forward: 0.7544 | max_loss: 0.1826 | max_backward: 1.4299 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8027 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6311.1 | uss_MB: 5475.9 | child_rss_MB: 5031.1 | allocated_MB: 369.9 | reserved_MB: 23104.0 | inactive_split_MB: 253.6 +rank6 [2026-10-29 23:37:53,327] INFO : Step 47500/50000 rank6 | Train Loss: 0.9089 | Train Loss Unweighted: 0.4235 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1000 | Classification: 0.0799 | Classification_binary: 0.1391 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1008 | Regression_eta: 0.0012 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0422 | Regression_eta: 0.0585 | Regression_pt: 0.0439 | Regression_sin_phi: 0.0467 | LR: 1.06e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.2723s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0068 | max_forward: 0.7329 | max_loss: 0.1827 | max_backward: 1.5814 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9161 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6306.2 | uss_MB: 5478.3 | child_rss_MB: 5026.1 | allocated_MB: 371.7 | reserved_MB: 23104.0 | inactive_split_MB: 243.6 +rank6 [2026-10-29 23:41:48,170] INFO : Step 47600/50000 rank6 | Train Loss: 0.9781 | Train Loss Unweighted: 0.4635 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1226 | Classification: 0.0910 | Classification_binary: 0.1607 | HitGrouping: 0.1784 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1531 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0597 | Regression_eta: 0.0344 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0483 | LR: 1.05e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.4262s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8435 | max_loss: 0.1581 | max_backward: 1.8989 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5341 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6310.2 | uss_MB: 5483.0 | child_rss_MB: 5033.7 | allocated_MB: 370.6 | reserved_MB: 24122.0 | inactive_split_MB: 257.4 +rank6 [2026-10-29 23:46:00,212] INFO : Step 47700/50000 rank6 | Train Loss: 0.9311 | Train Loss Unweighted: 0.4523 | Train Loss Components: Auxiliary: 0.2105 | Cardinality: 0.0992 | Classification: 0.0889 | Classification_binary: 0.1321 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1138 | Regression_eta: 0.0012 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0489 | Regression_eta: 0.0330 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0416 | LR: 1.05e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.5055s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0068 | max_forward: 0.6708 | max_loss: 0.1613 | max_backward: 1.2326 | max_optimizer_step: 0.1946 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6306.7 | uss_MB: 5481.0 | child_rss_MB: 5031.8 | allocated_MB: 368.1 | reserved_MB: 24122.0 | inactive_split_MB: 276.0 +rank6 [2026-10-29 23:50:02,911] INFO : Step 47800/50000 rank6 | Train Loss: 0.9063 | Train Loss Unweighted: 0.4379 | Train Loss Components: Auxiliary: 0.2020 | Cardinality: 0.0932 | Classification: 0.0920 | Classification_binary: 0.1284 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0011 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1389 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0610 | Regression_eta: 0.0396 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0635 | LR: 1.04e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.8442s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0610 | max_forward: 0.6626 | max_loss: 0.1349 | max_backward: 1.5091 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6302.7 | uss_MB: 5485.7 | child_rss_MB: 5029.8 | allocated_MB: 369.5 | reserved_MB: 24122.0 | inactive_split_MB: 236.5 +rank6 [2026-10-29 23:54:07,643] INFO : Step 47900/50000 rank6 | Train Loss: 0.9591 | Train Loss Unweighted: 0.4478 | Train Loss Components: Auxiliary: 0.2152 | Cardinality: 0.1022 | Classification: 0.0882 | Classification_binary: 0.1545 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1003 | Regression_eta: 0.0014 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1508 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0390 | Regression_eta: 0.0280 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0505 | LR: 1.04e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.1684s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8744 | max_loss: 0.2241 | max_backward: 1.6370 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 6302.2 | uss_MB: 5483.9 | child_rss_MB: 5030.3 | allocated_MB: 370.5 | reserved_MB: 24122.0 | inactive_split_MB: 218.9 +rank6 [2026-10-30 00:00:28,989] INFO : Step 48000/50000 rank6 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1221 | Classification: 0.0823 | Classification_binary: 0.1525 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1079 | Regression_eta: 0.0017 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0543 | Regression_eta: 0.0250 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0578 | LR: 1.04e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.1293s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0057 | max_forward: 0.7069 | max_loss: 0.2447 | max_backward: 1.4090 | max_optimizer_step: 0.1933 | max_tensorboard_logging: 0.8132 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1250 | Memory: rss_MB: 6301.2 | uss_MB: 5485.0 | child_rss_MB: 5024.5 | allocated_MB: 368.8 | reserved_MB: 24122.0 | inactive_split_MB: 269.6 +rank6 [2026-10-30 00:00:29,931] INFO : Running validation on rank6 +rank6 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 00:01:19,557] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:01:20,065] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 00:01:46,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 00:01:48,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 00:01:49,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 00:01:50,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 00:02:00,538] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-30 00:02:01,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:02:01,859] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 00:02:17,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 00:02:18,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 00:02:19,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 00:02:20,524] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 00:02:30,534] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 00:02:32,510] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:02:32,991] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 00:02:48,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 00:02:48,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 00:02:49,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 00:02:50,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 00:03:00,142] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-30 00:07:25,203] INFO : Step 48100/50000 rank6 | Train Loss: 0.9084 | Train Loss Unweighted: 0.4360 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1162 | Classification: 0.0769 | Classification_binary: 0.1204 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1183 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1293 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0630 | Regression_eta: 0.0353 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0379 | LR: 1.03e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9238s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1372 | max_backward: 1.2024 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.9158 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7374.2 | uss_MB: 6557.5 | child_rss_MB: 5038.7 | allocated_MB: 367.7 | reserved_MB: 24830.0 | inactive_split_MB: 222.3 +rank6 [2026-10-30 00:11:33,472] INFO : Step 48200/50000 rank6 | Train Loss: 0.9585 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2151 | Cardinality: 0.1199 | Classification: 0.0839 | Classification_binary: 0.1503 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1339 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0436 | Regression_eta: 0.0252 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0410 | LR: 1.03e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4373s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7766 | max_loss: 0.1498 | max_backward: 1.7414 | max_optimizer_step: 0.3242 | max_tensorboard_logging: 0.7504 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 7374.2 | uss_MB: 6560.6 | child_rss_MB: 5031.5 | allocated_MB: 370.4 | reserved_MB: 24830.0 | inactive_split_MB: 179.3 +rank6 [2026-10-30 00:15:41,697] INFO : Step 48300/50000 rank6 | Train Loss: 1.0575 | Train Loss Unweighted: 0.4963 | Train Loss Components: Auxiliary: 0.2408 | Cardinality: 0.1341 | Classification: 0.0958 | Classification_binary: 0.1687 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1138 | Regression_eta: 0.0013 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0502 | Regression_eta: 0.0359 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0399 | LR: 1.03e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2031s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7497 | max_loss: 0.2201 | max_backward: 1.7233 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6378.2 | uss_MB: 5566.5 | child_rss_MB: 5032.3 | allocated_MB: 372.3 | reserved_MB: 24830.0 | inactive_split_MB: 269.6 +rank6 [2026-10-30 00:19:56,564] INFO : Step 48400/50000 rank6 | Train Loss: 0.9108 | Train Loss Unweighted: 0.4349 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1093 | Classification: 0.0691 | Classification_binary: 0.1311 | HitGrouping: 0.1830 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1153 | Regression_eta: 0.0013 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1163 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0591 | Regression_eta: 0.0598 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0703 | LR: 1.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8543s | Phase Times: max_pre_step_barrier: 0.1141 | max_batch_to_device: 0.0064 | max_forward: 0.6489 | max_loss: 0.1390 | max_backward: 1.1801 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6705 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6380.2 | uss_MB: 5566.3 | child_rss_MB: 5030.1 | allocated_MB: 370.1 | reserved_MB: 24830.0 | inactive_split_MB: 261.5 +rank6 [2026-10-30 00:23:53,558] INFO : Step 48500/50000 rank6 | Train Loss: 0.9694 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1004 | Classification: 0.0844 | Classification_binary: 0.1367 | HitGrouping: 0.1928 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1194 | Regression_eta: 0.0012 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1216 | Regression_cos_phi: 0.0404 | Regression_energy: 0.0503 | Regression_eta: 0.0220 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0469 | LR: 1.02e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.7554s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 1.7512 | max_loss: 0.2678 | max_backward: 1.9538 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.4804 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 6374.4 | uss_MB: 5565.1 | child_rss_MB: 5023.6 | allocated_MB: 371.5 | reserved_MB: 26842.0 | inactive_split_MB: 176.5 +rank6 [2026-10-30 00:27:51,108] INFO : Step 48600/50000 rank6 | Train Loss: 0.9724 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1148 | Classification: 0.0854 | Classification_binary: 0.1551 | HitGrouping: 0.1844 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0327 | Classification_binary: 0.1199 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0428 | Regression_eta: 0.0257 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0526 | LR: 1.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0074s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0086 | max_forward: 0.6984 | max_loss: 0.2525 | max_backward: 1.6611 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6375.4 | uss_MB: 5569.8 | child_rss_MB: 5026.2 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 234.8 +rank6 [2026-10-30 00:31:52,432] INFO : Step 48700/50000 rank6 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4863 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1212 | Classification: 0.0968 | Classification_binary: 0.1569 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1143 | Regression_eta: 0.0012 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0481 | Regression_eta: 0.0310 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0405 | LR: 1.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9639s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7410 | max_loss: 0.2472 | max_backward: 1.4647 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.6585 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6316.1 | uss_MB: 5565.7 | child_rss_MB: 4823.4 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 275.8 +rank6 [2026-10-30 00:35:40,393] INFO : Step 48800/50000 rank6 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1981 | Cardinality: 0.0898 | Classification: 0.0814 | Classification_binary: 0.1196 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1094 | Regression_eta: 0.0012 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1175 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0480 | Regression_eta: 0.0420 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0507 | LR: 1.01e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7704s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.6122 | max_loss: 0.1775 | max_backward: 1.1932 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6316.1 | uss_MB: 5568.8 | child_rss_MB: 4813.9 | allocated_MB: 368.0 | reserved_MB: 26842.0 | inactive_split_MB: 263.7 +rank6 [2026-10-30 00:39:23,571] INFO : Step 48900/50000 rank6 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1005 | Classification: 0.0873 | Classification_binary: 0.1401 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0494 | Regression_eta: 0.0441 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0687 | LR: 1.01e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.9257s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0809 | max_forward: 0.7397 | max_loss: 0.1455 | max_backward: 1.5027 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6303.5 | uss_MB: 5491.8 | child_rss_MB: 4768.8 | allocated_MB: 373.0 | reserved_MB: 26842.0 | inactive_split_MB: 191.3 +rank6 [2026-10-30 00:43:08,795] INFO : Step 49000/50000 rank6 | Train Loss: 1.0019 | Train Loss Unweighted: 0.4657 | Train Loss Components: Auxiliary: 0.2334 | Cardinality: 0.1354 | Classification: 0.0893 | Classification_binary: 0.1587 | HitGrouping: 0.1838 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0010 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1466 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0434 | Regression_eta: 0.0291 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0465 | LR: 1.01e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4543s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1085 | max_forward: 0.8072 | max_loss: 0.1558 | max_backward: 1.7809 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.6082 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6305.0 | uss_MB: 5577.9 | child_rss_MB: 4749.4 | allocated_MB: 370.9 | reserved_MB: 26842.0 | inactive_split_MB: 271.3 +rank6 [2026-10-30 00:43:09,869] INFO : Running validation on rank6 +rank6 [2026-10-30 00:43:09,869] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 00:43:09,869] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 00:43:59,277] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:43:59,788] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 00:44:27,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 00:44:28,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 00:44:29,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 00:44:30,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 00:44:40,247] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 00:44:40,646] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:44:41,121] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 00:44:56,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 00:44:57,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 00:44:58,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 00:44:59,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 00:45:09,627] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 00:45:10,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:45:10,743] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 00:45:26,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 00:45:26,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 00:45:27,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 00:45:28,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 00:45:38,257] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-30 00:49:42,469] INFO : Step 49100/50000 rank6 | Train Loss: 0.9797 | Train Loss Unweighted: 0.4561 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1125 | Classification: 0.0877 | Classification_binary: 0.1433 | HitGrouping: 0.1905 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0013 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0603 | Regression_eta: 0.0373 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0553 | LR: 1.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7309s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6869 | max_loss: 0.1436 | max_backward: 1.2639 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.4741 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0455 | Memory: rss_MB: 7470.2 | uss_MB: 6734.9 | child_rss_MB: 4778.3 | allocated_MB: 370.4 | reserved_MB: 22116.0 | inactive_split_MB: 274.9 +rank6 [2026-10-30 00:53:47,121] INFO : Step 49200/50000 rank6 | Train Loss: 0.8929 | Train Loss Unweighted: 0.4393 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1020 | Classification: 0.0903 | Classification_binary: 0.1250 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0009 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1101 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0556 | Regression_eta: 0.0260 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0411 | LR: 1.01e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.7533s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6890 | max_loss: 0.2430 | max_backward: 1.3427 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.6347 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7470.5 | uss_MB: 6738.6 | child_rss_MB: 4771.6 | allocated_MB: 368.3 | reserved_MB: 24092.0 | inactive_split_MB: 187.3 +rank6 [2026-10-30 00:57:33,941] INFO : Step 49300/50000 rank6 | Train Loss: 1.0122 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1210 | Classification: 0.0951 | Classification_binary: 0.1621 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1066 | Regression_eta: 0.0015 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0502 | Regression_eta: 0.0444 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0577 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8968s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1495 | max_backward: 1.5906 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6415.4 | uss_MB: 5684.6 | child_rss_MB: 4746.9 | allocated_MB: 370.9 | reserved_MB: 24092.0 | inactive_split_MB: 226.9 +rank6 [2026-10-30 01:01:33,225] INFO : Step 49400/50000 rank6 | Train Loss: 0.9790 | Train Loss Unweighted: 0.4695 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1137 | Classification: 0.0883 | Classification_binary: 0.1513 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1131 | Regression_eta: 0.0013 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0626 | Regression_eta: 0.0484 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0624 | LR: 1.00e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.0355s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2433 | max_backward: 1.8823 | max_optimizer_step: 0.3081 | max_tensorboard_logging: 0.5951 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6413.4 | uss_MB: 5608.6 | child_rss_MB: 4765.4 | allocated_MB: 372.2 | reserved_MB: 24092.0 | inactive_split_MB: 278.9 +rank6 [2026-10-30 01:05:37,233] INFO : Step 49500/50000 rank6 | Train Loss: 0.9706 | Train Loss Unweighted: 0.4581 | Train Loss Components: Auxiliary: 0.2120 | Cardinality: 0.1158 | Classification: 0.0879 | Classification_binary: 0.1279 | HitGrouping: 0.1920 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0448 | Regression_energy: 0.0592 | Regression_eta: 0.0421 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0439 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9630s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6307 | max_loss: 0.1505 | max_backward: 1.3337 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.8584 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6409.4 | uss_MB: 5683.7 | child_rss_MB: 4759.9 | allocated_MB: 370.4 | reserved_MB: 24614.0 | inactive_split_MB: 171.1 +rank6 [2026-10-30 01:09:49,675] INFO : Step 49600/50000 rank6 | Train Loss: 0.9965 | Train Loss Unweighted: 0.4752 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1098 | Classification: 0.1016 | Classification_binary: 0.1437 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1133 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0598 | Regression_eta: 0.0364 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0702 | LR: 1.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1630s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.6933 | max_loss: 0.1496 | max_backward: 1.5639 | max_optimizer_step: 0.3077 | max_tensorboard_logging: 0.7702 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6412.2 | uss_MB: 5687.4 | child_rss_MB: 4755.4 | allocated_MB: 371.3 | reserved_MB: 24614.0 | inactive_split_MB: 250.0 +rank6 [2026-10-30 01:13:44,418] INFO : Step 49700/50000 rank6 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4775 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1108 | Classification: 0.1000 | Classification_binary: 0.1702 | HitGrouping: 0.1796 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1019 | Regression_eta: 0.0014 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1723 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0474 | Regression_eta: 0.0335 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0510 | LR: 1.00e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.2334s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7591 | max_loss: 0.1933 | max_backward: 1.7410 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 6407.4 | uss_MB: 5675.2 | child_rss_MB: 4703.7 | allocated_MB: 372.4 | reserved_MB: 24614.0 | inactive_split_MB: 259.5 +rank6 [2026-10-30 01:17:46,084] INFO : Step 49800/50000 rank6 | Train Loss: 0.9472 | Train Loss Unweighted: 0.4401 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1188 | Classification: 0.0821 | Classification_binary: 0.1427 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1703 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0556 | Regression_eta: 0.0265 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0581 | LR: 1.00e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.7711s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7430 | max_loss: 0.2153 | max_backward: 1.0846 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.7647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6388.3 | uss_MB: 5552.5 | child_rss_MB: 4630.2 | allocated_MB: 368.0 | reserved_MB: 24614.0 | inactive_split_MB: 273.7 +rank6 [2026-10-30 01:21:39,661] INFO : Step 49900/50000 rank6 | Train Loss: 0.9411 | Train Loss Unweighted: 0.4586 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.0996 | Classification: 0.0961 | Classification_binary: 0.1284 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1145 | Regression_eta: 0.0013 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1291 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0577 | Regression_eta: 0.0415 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0761 | LR: 1.00e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0434s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.6992 | max_loss: 0.1527 | max_backward: 1.3836 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8576 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6363.3 | uss_MB: 5692.1 | child_rss_MB: 4528.3 | allocated_MB: 368.7 | reserved_MB: 24614.0 | inactive_split_MB: 241.3 +rank6 [2026-10-30 01:25:29,377] INFO : Step 50000/50000 rank6 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2297 | Cardinality: 0.1196 | Classification: 0.0902 | Classification_binary: 0.1583 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.1090 | Regression_cos_phi: 0.0285 | Regression_energy: 0.0646 | Regression_eta: 0.0238 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0312 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2518s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.7146 | max_loss: 0.1481 | max_backward: 1.6207 | max_optimizer_step: 0.2721 | max_tensorboard_logging: 0.8195 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6364.5 | uss_MB: 5690.8 | child_rss_MB: 4519.4 | allocated_MB: 371.5 | reserved_MB: 25088.0 | inactive_split_MB: 268.9 +rank6 [2026-10-30 01:25:30,314] INFO : Running validation on rank6 +rank6 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 01:26:21,850] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:26:22,339] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 01:26:50,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 01:26:51,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 01:26:52,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 01:26:53,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 01:27:03,868] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-30 01:27:03,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:27:04,490] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 01:27:20,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 01:27:21,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 01:27:22,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 01:27:23,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 01:27:33,173] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 01:27:34,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:27:34,756] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 01:27:50,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 01:27:50,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 01:27:51,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 01:27:52,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 01:28:02,075] INFO : Time taken to make predictions on device 6 is: 0.22 min +rank6 [2026-10-30 01:28:03,386] INFO : Training completed. Total time on device 6: 638.890min diff --git a/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.7 b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.7 new file mode 100644 index 0000000000000000000000000000000000000000..2ef329537bdf00b3c6eedcf4d596ccae162b56c5 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_attention_seed12345_20261028_111201_979187/train.log.7 @@ -0,0 +1,2484 @@ +rank7 [2026-10-28 11:12:33,117] INFO : Initializing model with seed=12345; process seed=12352 +rank7 [2026-10-28 11:12:33,118] INFO : configured dtype=torch.bfloat16 for autocast +rank7 [2026-10-28 11:12:54,339] INFO : Instantiating model +rank7 [2026-10-28 11:12:54,339] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank7 [2026-10-28 11:12:54,339] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank7 [2026-10-28 11:12:54,358] INFO : Input encoding initialization took 0.02s +rank7 [2026-10-28 11:12:54,359] INFO : nn0 parameters: 171776 +rank7 [2026-10-28 11:12:54,359] INFO : Initializing shared backbone layers, num_convs=6 +rank7 [2026-10-28 11:12:54,359] INFO : Initializing backbone layer 0 +rank7 [2026-10-28 11:12:54,359] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:54,365] INFO : Initializing backbone layer 1 +rank7 [2026-10-28 11:12:54,365] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:54,370] INFO : Initializing backbone layer 2 +rank7 [2026-10-28 11:12:54,370] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:54,375] INFO : Initializing backbone layer 3 +rank7 [2026-10-28 11:12:54,375] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:54,380] INFO : Initializing backbone layer 4 +rank7 [2026-10-28 11:12:54,380] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:54,386] INFO : Initializing backbone layer 5 +rank7 [2026-10-28 11:12:54,386] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-28 11:12:54,391] INFO : Backbone initialization took 0.03s +rank7 [2026-10-28 11:12:54,391] INFO : backbone parameters: 2374656 +rank7 [2026-10-28 11:12:54,391] INFO : Initializing output DNNs +rank7 [2026-10-28 11:12:54,427] INFO : Output DNNs initialization took 0.04s +rank7 [2026-10-28 11:12:54,427] INFO : backbone_mode=BackboneMode.SHARED +rank7 [2026-10-28 11:12:54,427] INFO : jagged_attention enabled=True +rank7 [2026-10-28 11:12:54,427] INFO : flash_attn_varlen enabled=True +rank7 [2026-10-28 11:12:54,427] INFO : task_queries enabled=False +rank7 [2026-10-28 11:12:54,427] INFO : classification_norm parameters: 0 +rank7 [2026-10-28 11:12:54,427] INFO : regression_norm parameters: 0 +rank7 [2026-10-28 11:12:54,427] INFO : classification_query parameters: 0 +rank7 [2026-10-28 11:12:54,427] INFO : regression_query parameters: 0 +rank7 [2026-10-28 11:12:54,428] INFO : classification_readout parameters: 0 +rank7 [2026-10-28 11:12:54,428] INFO : regression_readout parameters: 0 +rank7 [2026-10-28 11:12:54,428] INFO : output_mode=set +rank7 [2026-10-28 11:12:54,428] INFO : set_decoder parameters: 4713812 +rank7 [2026-10-28 11:12:54,428] INFO : nn_binary_particle parameters: 0 +rank7 [2026-10-28 11:12:54,428] INFO : nn_pid parameters: 0 +rank7 [2026-10-28 11:12:54,428] INFO : nn_pt parameters: 0 +rank7 [2026-10-28 11:12:54,428] INFO : nn_eta parameters: 0 +rank7 [2026-10-28 11:12:54,428] INFO : nn_sin_phi parameters: 0 +rank7 [2026-10-28 11:12:54,428] INFO : nn_cos_phi parameters: 0 +rank7 [2026-10-28 11:12:54,428] INFO : nn_energy parameters: 0 +rank7 [2026-10-28 11:12:54,429] INFO : Total MLPF parameters: 7260244 +rank7 [2026-10-28 11:12:54,429] INFO : MLPF __init__ done +rank7 [2026-10-28 11:12:54,430] INFO : Instantiated model +rank7 [2026-10-28 11:12:54,430] INFO : Moving model to device rank=7 +rank7 [2026-10-28 11:12:54,449] INFO : Moved model to device rank=7 +rank7 [2026-10-28 11:12:54,450] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank7 [2026-10-28 11:12:54,451] INFO : Configured model for SyncBatchNorm rank=7 +rank7 [2026-10-28 11:12:54,646] INFO : Configured model for DistributedDataParallel rank=7 +rank7 [2026-10-28 11:12:54,778] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank7 [2026-10-28 11:12:54,778] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank7 [2026-10-28 11:12:54,779] INFO : Trainable parameters: 7260244 +rank7 [2026-10-28 11:12:54,779] INFO : Non-trainable parameters: 0 +rank7 [2026-10-28 11:12:54,779] INFO : Total parameters: 7260244 +rank7 [2026-10-28 11:12:54,782] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank7 [2026-10-28 11:12:54,782] INFO : Getting dataloaders +rank7 [2026-10-28 11:12:54,782] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:03,737] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:04,808] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,147] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-28 11:13:06,147] INFO : Creating WeightedMixIterator with 1 data loaders. +rank7 [2026-10-28 11:13:06,147] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,415] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,671] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,925] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-28 11:13:06,925] INFO : Creating InterleavedIterator with 1 data loaders. +rank7 [2026-10-28 11:13:20,804] INFO : Got dataloaders +rank7 [2026-10-28 11:13:20,804] INFO : loader split=train rank=7 len=21112 +rank7 [2026-10-28 11:13:20,804] INFO : loader split=valid rank=7 len=12 +rank7 [2026-10-28 11:13:20,805] INFO : Starting training from step 1 to 50000 on rank 7 of 8 +rank7 [2026-10-28 11:13:20,805] INFO : Creating train_iterator +rank7 [2026-10-28 11:13:20,805] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-28 11:13:35,143] INFO : Created train_iterator +rank7 [2026-10-28 11:17:15,588] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5434697668626904, 'Classification': 0.9147019968181849, 'Regression_pt': 1.0888657218962907, 'Regression_eta': 0.019410275079426356, 'Regression_sin_phi': 0.00660347732307855, 'Regression_cos_phi': 0.006484371119877323, 'Regression_energy': 1.1263945941627025}, weights={'Classification_binary': 1.0, 'Classification': 0.5941495299339294, 'Regression_pt': 0.4991154968738556, 'Regression_eta': 27.99907684326172, 'Regression_sin_phi': 82.30054473876953, 'Regression_cos_phi': 83.812255859375, 'Regression_energy': 0.48248612880706787} +rank7 [2026-10-28 11:17:17,663] INFO : Step 100/50000 rank7 | Train Loss: 5.4941 | Train Loss Unweighted: 3.3573 | Train Loss Components: Auxiliary: 1.0155 | Cardinality: 0.6417 | Classification: 0.8373 | Classification_binary: 0.4757 | HitGrouping: 0.4797 | Regression_cos_phi: 0.0063 | Regression_energy: 1.0524 | Regression_eta: 0.0130 | Regression_pt: 0.9668 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.8164 | Classification_binary: 0.4499 | Regression_cos_phi: 0.0071 | Regression_energy: 0.9729 | Regression_eta: 0.0129 | Regression_pt: 0.8388 | Regression_sin_phi: 0.0059 | LR: 1.00e-03 | DataLoad Time: 0.0152s | Model Forward Time: 3.0142s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.5597 | max_loss: 0.2615 | max_backward: 1.0157 | max_optimizer_step: 0.1470 | max_tensorboard_logging: 0.4654 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0031 | Memory: rss_MB: 4647.0 | uss_MB: 3401.7 | child_rss_MB: 5567.8 | allocated_MB: 341.0 | reserved_MB: 22914.0 | inactive_split_MB: 237.2 +rank7 [2026-10-28 11:20:50,235] INFO : Step 200/50000 rank7 | Train Loss: 4.4651 | Train Loss Unweighted: 2.7232 | Train Loss Components: Auxiliary: 0.8610 | Cardinality: 0.5153 | Classification: 0.6425 | Classification_binary: 0.5162 | HitGrouping: 0.4582 | Regression_cos_phi: 0.0043 | Regression_energy: 0.7694 | Regression_eta: 0.0086 | Regression_pt: 0.7776 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3866 | Classification_binary: 0.4510 | Regression_cos_phi: 0.2835 | Regression_energy: 0.4208 | Regression_eta: 0.2295 | Regression_pt: 0.4400 | Regression_sin_phi: 0.3508 | LR: 1.00e-03 | DataLoad Time: 0.0151s | Model Forward Time: 3.7250s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.7368 | max_loss: 0.2463 | max_backward: 1.7032 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5468 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0021 | Memory: rss_MB: 4651.2 | uss_MB: 3409.7 | child_rss_MB: 5571.3 | allocated_MB: 343.8 | reserved_MB: 23264.0 | inactive_split_MB: 284.1 +rank7 [2026-10-28 11:24:24,165] INFO : Step 300/50000 rank7 | Train Loss: 3.6383 | Train Loss Unweighted: 2.1819 | Train Loss Components: Auxiliary: 0.6931 | Cardinality: 0.3447 | Classification: 0.5089 | Classification_binary: 0.4824 | HitGrouping: 0.4305 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5908 | Regression_eta: 0.0074 | Regression_pt: 0.5852 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3311 | Classification_binary: 0.5333 | Regression_cos_phi: 0.3712 | Regression_energy: 0.3192 | Regression_eta: 0.2260 | Regression_pt: 0.3281 | Regression_sin_phi: 0.3205 | LR: 1.00e-03 | DataLoad Time: 0.0142s | Model Forward Time: 3.8752s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0315 | max_forward: 0.6855 | max_loss: 0.1568 | max_backward: 1.3632 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 1.0182 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0032 | Memory: rss_MB: 4652.7 | uss_MB: 3415.2 | child_rss_MB: 5569.6 | allocated_MB: 341.4 | reserved_MB: 23916.0 | inactive_split_MB: 289.5 +rank7 [2026-10-28 11:27:58,396] INFO : Step 400/50000 rank7 | Train Loss: 3.5064 | Train Loss Unweighted: 1.9938 | Train Loss Components: Auxiliary: 0.6582 | Cardinality: 0.3738 | Classification: 0.4591 | Classification_binary: 0.4306 | HitGrouping: 0.4041 | Regression_cos_phi: 0.0039 | Regression_energy: 0.5433 | Regression_eta: 0.0080 | Regression_pt: 0.5456 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2780 | Classification_binary: 0.4404 | Regression_cos_phi: 0.4258 | Regression_energy: 0.2603 | Regression_eta: 0.2456 | Regression_pt: 0.2719 | Regression_sin_phi: 0.2551 | LR: 1.00e-03 | DataLoad Time: 0.0158s | Model Forward Time: 3.7419s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.6100 | max_loss: 0.1486 | max_backward: 1.2396 | max_optimizer_step: 0.1474 | max_tensorboard_logging: 1.0157 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4650.3 | uss_MB: 3416.8 | child_rss_MB: 5584.9 | allocated_MB: 342.9 | reserved_MB: 23982.0 | inactive_split_MB: 297.5 +rank7 [2026-10-28 11:31:35,335] INFO : Step 500/50000 rank7 | Train Loss: 3.8899 | Train Loss Unweighted: 2.1856 | Train Loss Components: Auxiliary: 0.6982 | Cardinality: 0.3682 | Classification: 0.4707 | Classification_binary: 0.4836 | HitGrouping: 0.4071 | Regression_cos_phi: 0.0046 | Regression_energy: 0.6048 | Regression_eta: 0.0103 | Regression_pt: 0.6070 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.3151 | Classification_binary: 0.4435 | Regression_cos_phi: 0.4031 | Regression_energy: 0.2932 | Regression_eta: 0.3255 | Regression_pt: 0.3006 | Regression_sin_phi: 0.4835 | LR: 1.00e-03 | DataLoad Time: 0.0154s | Model Forward Time: 4.1835s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0074 | max_forward: 0.7460 | max_loss: 0.2622 | max_backward: 1.7372 | max_optimizer_step: 0.1478 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 4651.3 | uss_MB: 3418.2 | child_rss_MB: 5567.0 | allocated_MB: 344.2 | reserved_MB: 24988.0 | inactive_split_MB: 288.4 +rank7 [2026-10-28 11:35:12,859] INFO : Step 600/50000 rank7 | Train Loss: 3.5648 | Train Loss Unweighted: 2.0654 | Train Loss Components: Auxiliary: 0.6756 | Cardinality: 0.3628 | Classification: 0.4159 | Classification_binary: 0.5246 | HitGrouping: 0.3632 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5539 | Regression_eta: 0.0077 | Regression_pt: 0.5557 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2618 | Classification_binary: 0.5188 | Regression_cos_phi: 0.3033 | Regression_energy: 0.2709 | Regression_eta: 0.2353 | Regression_pt: 0.2804 | Regression_sin_phi: 0.2704 | LR: 1.00e-03 | DataLoad Time: 0.0139s | Model Forward Time: 4.1939s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0071 | max_forward: 0.7483 | max_loss: 0.2574 | max_backward: 1.6305 | max_optimizer_step: 0.1443 | max_tensorboard_logging: 0.9709 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0456 | Memory: rss_MB: 4647.1 | uss_MB: 3418.2 | child_rss_MB: 5567.3 | allocated_MB: 344.6 | reserved_MB: 24988.0 | inactive_split_MB: 289.5 +rank7 [2026-10-28 11:38:47,057] INFO : Step 700/50000 rank7 | Train Loss: 3.0543 | Train Loss Unweighted: 1.8237 | Train Loss Components: Auxiliary: 0.6036 | Cardinality: 0.3094 | Classification: 0.3407 | Classification_binary: 0.4142 | HitGrouping: 0.3367 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5254 | Regression_eta: 0.0060 | Regression_pt: 0.5314 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2173 | Classification_binary: 0.4618 | Regression_cos_phi: 0.3427 | Regression_energy: 0.2898 | Regression_eta: 0.2225 | Regression_pt: 0.3029 | Regression_sin_phi: 0.2795 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 3.4870s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.6445 | max_loss: 0.2136 | max_backward: 1.2179 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.7893 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0042 | Memory: rss_MB: 4647.0 | uss_MB: 3416.3 | child_rss_MB: 5577.3 | allocated_MB: 341.4 | reserved_MB: 24988.0 | inactive_split_MB: 289.7 +rank7 [2026-10-28 11:42:25,056] INFO : Step 800/50000 rank7 | Train Loss: 3.1854 | Train Loss Unweighted: 1.9126 | Train Loss Components: Auxiliary: 0.6154 | Cardinality: 0.2934 | Classification: 0.4077 | Classification_binary: 0.4086 | HitGrouping: 0.3317 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5404 | Regression_eta: 0.0067 | Regression_pt: 0.5423 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2346 | Classification_binary: 0.4217 | Regression_cos_phi: 0.2916 | Regression_energy: 0.2657 | Regression_eta: 0.2214 | Regression_pt: 0.2757 | Regression_sin_phi: 0.2844 | LR: 9.99e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.3169s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.6526 | max_loss: 0.2001 | max_backward: 1.2539 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5713 | max_tensorboard_flush: 0.0138 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 4642.6 | uss_MB: 3416.3 | child_rss_MB: 5579.0 | allocated_MB: 346.1 | reserved_MB: 24988.0 | inactive_split_MB: 222.7 +rank7 [2026-10-28 11:46:07,652] INFO : Step 900/50000 rank7 | Train Loss: 3.4950 | Train Loss Unweighted: 2.0678 | Train Loss Components: Auxiliary: 0.6566 | Cardinality: 0.3469 | Classification: 0.4160 | Classification_binary: 0.4953 | HitGrouping: 0.3295 | Regression_cos_phi: 0.0036 | Regression_energy: 0.5663 | Regression_eta: 0.0083 | Regression_pt: 0.5742 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2295 | Classification_binary: 0.4302 | Regression_cos_phi: 0.2047 | Regression_energy: 0.2462 | Regression_eta: 0.2114 | Regression_pt: 0.2662 | Regression_sin_phi: 0.2726 | LR: 9.99e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.8979s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.8131 | max_loss: 0.2548 | max_backward: 1.8396 | max_optimizer_step: 0.1492 | max_tensorboard_logging: 0.4938 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0036 | Memory: rss_MB: 4643.4 | uss_MB: 3417.5 | child_rss_MB: 5588.2 | allocated_MB: 344.2 | reserved_MB: 26006.0 | inactive_split_MB: 273.7 +rank7 [2026-10-28 11:49:58,411] INFO : Step 1000/50000 rank7 | Train Loss: 3.0581 | Train Loss Unweighted: 1.8801 | Train Loss Components: Auxiliary: 0.5942 | Cardinality: 0.2760 | Classification: 0.3603 | Classification_binary: 0.4232 | HitGrouping: 0.3177 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5411 | Regression_eta: 0.0066 | Regression_pt: 0.5426 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2534 | Classification_binary: 0.4689 | Regression_cos_phi: 0.2826 | Regression_energy: 0.2433 | Regression_eta: 0.1989 | Regression_pt: 0.2534 | Regression_sin_phi: 0.3540 | LR: 9.99e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2150s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1006 | max_forward: 0.7513 | max_loss: 0.1459 | max_backward: 1.3658 | max_optimizer_step: 0.1511 | max_tensorboard_logging: 1.2506 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0037 | Memory: rss_MB: 4639.1 | uss_MB: 3415.3 | child_rss_MB: 5584.2 | allocated_MB: 344.4 | reserved_MB: 26006.0 | inactive_split_MB: 271.1 +rank7 [2026-10-28 11:49:59,499] INFO : Running validation on rank7 +rank7 [2026-10-28 11:49:59,499] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 11:49:59,500] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 11:50:52,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:50:53,227] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 11:51:19,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 11:51:20,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 11:51:21,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 11:51:22,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 11:51:32,272] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 11:51:32,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:51:33,326] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 11:51:49,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 11:51:50,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 11:51:51,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 11:51:52,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 11:52:02,708] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 11:52:02,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:52:03,476] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 11:52:18,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 11:52:19,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 11:52:20,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 11:52:20,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_1000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 11:52:30,936] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 11:56:35,453] INFO : Step 1100/50000 rank7 | Train Loss: 2.9181 | Train Loss Unweighted: 1.8556 | Train Loss Components: Auxiliary: 0.5873 | Cardinality: 0.2540 | Classification: 0.3370 | Classification_binary: 0.3789 | HitGrouping: 0.3247 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5584 | Regression_eta: 0.0064 | Regression_pt: 0.5695 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1895 | Classification_binary: 0.3972 | Regression_cos_phi: 0.2380 | Regression_energy: 0.2521 | Regression_eta: 0.1955 | Regression_pt: 0.2653 | Regression_sin_phi: 0.2241 | LR: 9.99e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.9456s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8528 | max_loss: 0.1390 | max_backward: 1.4220 | max_optimizer_step: 0.2689 | max_tensorboard_logging: 0.9135 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0136 | Memory: rss_MB: 6058.3 | uss_MB: 4835.8 | child_rss_MB: 5619.2 | allocated_MB: 344.1 | reserved_MB: 21080.0 | inactive_split_MB: 227.9 +rank7 [2026-10-28 12:00:25,221] INFO : Step 1200/50000 rank7 | Train Loss: 3.1885 | Train Loss Unweighted: 1.9202 | Train Loss Components: Auxiliary: 0.6265 | Cardinality: 0.3516 | Classification: 0.3718 | Classification_binary: 0.4294 | HitGrouping: 0.3258 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5507 | Regression_eta: 0.0065 | Regression_pt: 0.5556 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2074 | Classification_binary: 0.4028 | Regression_cos_phi: 0.1968 | Regression_energy: 0.3027 | Regression_eta: 0.1270 | Regression_pt: 0.3171 | Regression_sin_phi: 0.1746 | LR: 9.99e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.4096s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0076 | max_forward: 0.7370 | max_loss: 0.3921 | max_backward: 1.7945 | max_optimizer_step: 0.1639 | max_tensorboard_logging: 1.0479 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 6061.1 | uss_MB: 4838.4 | child_rss_MB: 5619.8 | allocated_MB: 344.9 | reserved_MB: 23662.0 | inactive_split_MB: 241.9 +rank7 [2026-10-28 12:04:16,198] INFO : Step 1300/50000 rank7 | Train Loss: 3.2694 | Train Loss Unweighted: 1.9280 | Train Loss Components: Auxiliary: 0.6193 | Cardinality: 0.3461 | Classification: 0.3601 | Classification_binary: 0.4731 | HitGrouping: 0.3156 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5359 | Regression_eta: 0.0069 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1817 | Classification_binary: 0.4668 | Regression_cos_phi: 0.2053 | Regression_energy: 0.2588 | Regression_eta: 0.1620 | Regression_pt: 0.2682 | Regression_sin_phi: 0.1794 | LR: 9.98e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.5227s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0540 | max_forward: 0.9909 | max_loss: 0.3234 | max_backward: 2.2964 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 1.1252 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5498.9 | uss_MB: 4193.7 | child_rss_MB: 5645.6 | allocated_MB: 344.9 | reserved_MB: 24052.0 | inactive_split_MB: 299.1 +rank7 [2026-10-28 12:07:55,471] INFO : Step 1400/50000 rank7 | Train Loss: 2.8491 | Train Loss Unweighted: 1.7889 | Train Loss Components: Auxiliary: 0.5645 | Cardinality: 0.2711 | Classification: 0.3226 | Classification_binary: 0.3768 | HitGrouping: 0.3109 | Regression_cos_phi: 0.0027 | Regression_energy: 0.5376 | Regression_eta: 0.0061 | Regression_pt: 0.5407 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2175 | Classification_binary: 0.3982 | Regression_cos_phi: 0.2428 | Regression_energy: 0.2791 | Regression_eta: 0.1442 | Regression_pt: 0.2908 | Regression_sin_phi: 0.2123 | LR: 9.98e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.7560s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0094 | max_forward: 0.6378 | max_loss: 0.4388 | max_backward: 1.2886 | max_optimizer_step: 0.1596 | max_tensorboard_logging: 0.9631 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0107 | Memory: rss_MB: 5501.2 | uss_MB: 4277.0 | child_rss_MB: 5652.8 | allocated_MB: 350.4 | reserved_MB: 24916.0 | inactive_split_MB: 216.1 +rank7 [2026-10-28 12:11:42,090] INFO : Step 1500/50000 rank7 | Train Loss: 2.8928 | Train Loss Unweighted: 1.7749 | Train Loss Components: Auxiliary: 0.5690 | Cardinality: 0.2892 | Classification: 0.3566 | Classification_binary: 0.3704 | HitGrouping: 0.3078 | Regression_cos_phi: 0.0030 | Regression_energy: 0.5147 | Regression_eta: 0.0058 | Regression_pt: 0.5216 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1605 | Classification_binary: 0.3532 | Regression_cos_phi: 0.2967 | Regression_energy: 0.2333 | Regression_eta: 0.1878 | Regression_pt: 0.2481 | Regression_sin_phi: 0.2353 | LR: 9.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.5612s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.6094 | max_loss: 0.4244 | max_backward: 1.4479 | max_optimizer_step: 0.1602 | max_tensorboard_logging: 0.5745 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0102 | Memory: rss_MB: 5488.8 | uss_MB: 4270.8 | child_rss_MB: 5621.5 | allocated_MB: 342.2 | reserved_MB: 24916.0 | inactive_split_MB: 182.9 +rank7 [2026-10-28 12:15:27,598] INFO : Step 1600/50000 rank7 | Train Loss: 2.9805 | Train Loss Unweighted: 1.8512 | Train Loss Components: Auxiliary: 0.5725 | Cardinality: 0.2336 | Classification: 0.3605 | Classification_binary: 0.4404 | HitGrouping: 0.3159 | Regression_cos_phi: 0.0031 | Regression_energy: 0.5142 | Regression_eta: 0.0065 | Regression_pt: 0.5235 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2400 | Classification_binary: 0.4438 | Regression_cos_phi: 0.3545 | Regression_energy: 0.2430 | Regression_eta: 0.2530 | Regression_pt: 0.2541 | Regression_sin_phi: 0.2923 | LR: 9.98e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.7173s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7382 | max_loss: 0.1451 | max_backward: 1.5359 | max_optimizer_step: 0.1583 | max_tensorboard_logging: 0.4679 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5499.3 | uss_MB: 4281.1 | child_rss_MB: 5637.2 | allocated_MB: 343.2 | reserved_MB: 25046.0 | inactive_split_MB: 260.8 +rank7 [2026-10-28 12:19:19,666] INFO : Step 1700/50000 rank7 | Train Loss: 2.9423 | Train Loss Unweighted: 1.7603 | Train Loss Components: Auxiliary: 0.5618 | Cardinality: 0.2834 | Classification: 0.3350 | Classification_binary: 0.4141 | HitGrouping: 0.3155 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4976 | Regression_eta: 0.0064 | Regression_pt: 0.5011 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1947 | Classification_binary: 0.4286 | Regression_cos_phi: 0.2756 | Regression_energy: 0.2514 | Regression_eta: 0.1708 | Regression_pt: 0.2634 | Regression_sin_phi: 0.1884 | LR: 9.97e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2311s | Phase Times: max_pre_step_barrier: 0.0070 | max_batch_to_device: 0.0065 | max_forward: 0.8351 | max_loss: 0.2466 | max_backward: 1.7292 | max_optimizer_step: 0.1641 | max_tensorboard_logging: 0.4994 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0857 | Memory: rss_MB: 5499.7 | uss_MB: 4254.7 | child_rss_MB: 5622.9 | allocated_MB: 342.0 | reserved_MB: 25046.0 | inactive_split_MB: 198.9 +rank7 [2026-10-28 12:23:11,816] INFO : Step 1800/50000 rank7 | Train Loss: 2.7182 | Train Loss Unweighted: 1.6765 | Train Loss Components: Auxiliary: 0.5305 | Cardinality: 0.2556 | Classification: 0.3337 | Classification_binary: 0.3613 | HitGrouping: 0.3103 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4855 | Regression_eta: 0.0054 | Regression_pt: 0.4854 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1942 | Classification_binary: 0.3727 | Regression_cos_phi: 0.2596 | Regression_energy: 0.2380 | Regression_eta: 0.1684 | Regression_pt: 0.2426 | Regression_sin_phi: 0.2198 | LR: 9.97e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0935s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6573 | max_loss: 0.2402 | max_backward: 1.4111 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 1.0385 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0959 | Memory: rss_MB: 5501.3 | uss_MB: 4283.5 | child_rss_MB: 5636.1 | allocated_MB: 340.6 | reserved_MB: 25046.0 | inactive_split_MB: 181.6 +rank7 [2026-10-28 12:26:53,741] INFO : Step 1900/50000 rank7 | Train Loss: 2.9893 | Train Loss Unweighted: 1.8606 | Train Loss Components: Auxiliary: 0.5839 | Cardinality: 0.2900 | Classification: 0.3568 | Classification_binary: 0.4041 | HitGrouping: 0.3105 | Regression_cos_phi: 0.0029 | Regression_energy: 0.5399 | Regression_eta: 0.0063 | Regression_pt: 0.5477 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2194 | Classification_binary: 0.3817 | Regression_cos_phi: 0.2199 | Regression_energy: 0.2350 | Regression_eta: 0.1607 | Regression_pt: 0.2429 | Regression_sin_phi: 0.2828 | LR: 9.97e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8699s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.6296 | max_loss: 0.1482 | max_backward: 1.2931 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 1.0537 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0048 | Memory: rss_MB: 5503.8 | uss_MB: 4286.6 | child_rss_MB: 5619.9 | allocated_MB: 344.8 | reserved_MB: 25048.0 | inactive_split_MB: 225.2 +rank7 [2026-10-28 12:30:53,516] INFO : Step 2000/50000 rank7 | Train Loss: 3.0007 | Train Loss Unweighted: 1.8040 | Train Loss Components: Auxiliary: 0.5673 | Cardinality: 0.3091 | Classification: 0.3519 | Classification_binary: 0.4482 | HitGrouping: 0.3137 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4922 | Regression_eta: 0.0062 | Regression_pt: 0.4996 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2320 | Classification_binary: 0.4704 | Regression_cos_phi: 0.2911 | Regression_energy: 0.2399 | Regression_eta: 0.1599 | Regression_pt: 0.2539 | Regression_sin_phi: 0.2501 | LR: 9.96e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1668s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.6813 | max_loss: 0.2416 | max_backward: 1.5165 | max_optimizer_step: 0.1635 | max_tensorboard_logging: 0.9842 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5502.9 | uss_MB: 4287.2 | child_rss_MB: 5669.7 | allocated_MB: 346.7 | reserved_MB: 25696.0 | inactive_split_MB: 287.8 +rank7 [2026-10-28 12:30:54,414] INFO : Running validation on rank7 +rank7 [2026-10-28 12:30:54,414] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 12:30:54,414] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 12:31:46,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:31:46,628] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 12:32:13,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 12:32:14,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 12:32:15,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 12:32:16,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 12:32:26,398] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 12:32:27,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:32:27,705] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 12:32:44,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 12:32:45,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 12:32:45,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 12:32:46,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 12:32:56,856] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 12:32:57,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:32:57,630] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 12:33:13,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 12:33:14,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 12:33:15,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 12:33:15,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_2000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 12:33:25,900] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 12:37:37,593] INFO : Step 2100/50000 rank7 | Train Loss: 2.6299 | Train Loss Unweighted: 1.6429 | Train Loss Components: Auxiliary: 0.5172 | Cardinality: 0.2489 | Classification: 0.3119 | Classification_binary: 0.3591 | HitGrouping: 0.3047 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4785 | Regression_eta: 0.0053 | Regression_pt: 0.4834 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1733 | Classification_binary: 0.3888 | Regression_cos_phi: 0.2335 | Regression_energy: 0.2385 | Regression_eta: 0.1769 | Regression_pt: 0.2467 | Regression_sin_phi: 0.2295 | LR: 9.96e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0185s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0685 | max_forward: 0.7343 | max_loss: 0.1422 | max_backward: 1.3363 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 1.0015 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6447.9 | uss_MB: 5231.1 | child_rss_MB: 5629.3 | allocated_MB: 341.1 | reserved_MB: 22546.0 | inactive_split_MB: 269.0 +rank7 [2026-10-28 12:41:24,842] INFO : Step 2200/50000 rank7 | Train Loss: 2.8483 | Train Loss Unweighted: 1.7068 | Train Loss Components: Auxiliary: 0.5372 | Cardinality: 0.2547 | Classification: 0.3411 | Classification_binary: 0.3679 | HitGrouping: 0.3030 | Regression_cos_phi: 0.0033 | Regression_energy: 0.4876 | Regression_eta: 0.0061 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2069 | Classification_binary: 0.3615 | Regression_cos_phi: 0.2571 | Regression_energy: 0.2423 | Regression_eta: 0.1443 | Regression_pt: 0.2572 | Regression_sin_phi: 0.2193 | LR: 9.96e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.4918s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6352 | max_loss: 0.1414 | max_backward: 1.3119 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.4703 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6448.9 | uss_MB: 5233.9 | child_rss_MB: 5636.2 | allocated_MB: 341.4 | reserved_MB: 22598.0 | inactive_split_MB: 313.4 +rank7 [2026-10-28 12:45:27,860] INFO : Step 2300/50000 rank7 | Train Loss: 3.1410 | Train Loss Unweighted: 1.8301 | Train Loss Components: Auxiliary: 0.5918 | Cardinality: 0.3583 | Classification: 0.3517 | Classification_binary: 0.4348 | HitGrouping: 0.3083 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5121 | Regression_eta: 0.0065 | Regression_pt: 0.5184 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1953 | Classification_binary: 0.3716 | Regression_cos_phi: 0.2246 | Regression_energy: 0.2329 | Regression_eta: 0.1529 | Regression_pt: 0.2451 | Regression_sin_phi: 0.2343 | LR: 9.95e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8592s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0080 | max_forward: 0.7258 | max_loss: 0.1520 | max_backward: 1.6288 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.6027 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0166 | Memory: rss_MB: 5633.9 | uss_MB: 4422.4 | child_rss_MB: 5644.1 | allocated_MB: 346.5 | reserved_MB: 23596.0 | inactive_split_MB: 273.5 +rank7 [2026-10-28 12:49:22,412] INFO : Step 2400/50000 rank7 | Train Loss: 2.8063 | Train Loss Unweighted: 1.7109 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.2683 | Classification: 0.3276 | Classification_binary: 0.3929 | HitGrouping: 0.3025 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4873 | Regression_eta: 0.0054 | Regression_pt: 0.4919 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2208 | Classification_binary: 0.4387 | Regression_cos_phi: 0.2970 | Regression_energy: 0.2372 | Regression_eta: 0.1406 | Regression_pt: 0.2508 | Regression_sin_phi: 0.2641 | LR: 9.95e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.5897s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0314 | max_forward: 0.7331 | max_loss: 0.1502 | max_backward: 1.4174 | max_optimizer_step: 0.1680 | max_tensorboard_logging: 0.5071 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0157 | Memory: rss_MB: 5635.8 | uss_MB: 4426.4 | child_rss_MB: 5648.9 | allocated_MB: 344.4 | reserved_MB: 23596.0 | inactive_split_MB: 276.8 +rank7 [2026-10-28 12:53:10,911] INFO : Step 2500/50000 rank7 | Train Loss: 2.5300 | Train Loss Unweighted: 1.6034 | Train Loss Components: Auxiliary: 0.5018 | Cardinality: 0.2387 | Classification: 0.2872 | Classification_binary: 0.3313 | HitGrouping: 0.3000 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4857 | Regression_eta: 0.0042 | Regression_pt: 0.4904 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2232 | Classification_binary: 0.3317 | Regression_cos_phi: 0.2922 | Regression_energy: 0.2496 | Regression_eta: 0.1527 | Regression_pt: 0.2583 | Regression_sin_phi: 0.1766 | LR: 9.94e-04 | DataLoad Time: 0.0229s | Model Forward Time: 4.0229s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0177 | max_forward: 0.6104 | max_loss: 0.1394 | max_backward: 1.2397 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 1.1029 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 5639.0 | uss_MB: 4422.1 | child_rss_MB: 5639.4 | allocated_MB: 342.5 | reserved_MB: 23596.0 | inactive_split_MB: 302.8 +rank7 [2026-10-28 12:56:55,919] INFO : Step 2600/50000 rank7 | Train Loss: 2.8992 | Train Loss Unweighted: 1.7075 | Train Loss Components: Auxiliary: 0.5471 | Cardinality: 0.3219 | Classification: 0.3420 | Classification_binary: 0.4005 | HitGrouping: 0.3027 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4737 | Regression_eta: 0.0057 | Regression_pt: 0.4796 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1860 | Classification_binary: 0.3686 | Regression_cos_phi: 0.1773 | Regression_energy: 0.2020 | Regression_eta: 0.1366 | Regression_pt: 0.2131 | Regression_sin_phi: 0.1829 | LR: 9.94e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.7056s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.8849 | max_loss: 0.1495 | max_backward: 1.9136 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1254 | Memory: rss_MB: 5638.3 | uss_MB: 4434.4 | child_rss_MB: 5644.3 | allocated_MB: 343.4 | reserved_MB: 23596.0 | inactive_split_MB: 290.6 +rank7 [2026-10-28 13:00:46,528] INFO : Step 2700/50000 rank7 | Train Loss: 2.8481 | Train Loss Unweighted: 1.7200 | Train Loss Components: Auxiliary: 0.5468 | Cardinality: 0.2918 | Classification: 0.3199 | Classification_binary: 0.4344 | HitGrouping: 0.3029 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4756 | Regression_eta: 0.0057 | Regression_pt: 0.4789 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1991 | Classification_binary: 0.4025 | Regression_cos_phi: 0.2042 | Regression_energy: 0.2679 | Regression_eta: 0.1473 | Regression_pt: 0.2794 | Regression_sin_phi: 0.1923 | LR: 9.94e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.3249s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0064 | max_forward: 0.7554 | max_loss: 0.2027 | max_backward: 1.6333 | max_optimizer_step: 0.1633 | max_tensorboard_logging: 0.9978 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5637.9 | uss_MB: 4434.7 | child_rss_MB: 5643.6 | allocated_MB: 343.8 | reserved_MB: 23598.0 | inactive_split_MB: 284.2 +rank7 [2026-10-28 13:04:40,295] INFO : Step 2800/50000 rank7 | Train Loss: 2.6381 | Train Loss Unweighted: 1.5955 | Train Loss Components: Auxiliary: 0.5049 | Cardinality: 0.2387 | Classification: 0.2848 | Classification_binary: 0.3420 | HitGrouping: 0.2974 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4746 | Regression_eta: 0.0056 | Regression_pt: 0.4829 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1836 | Classification_binary: 0.3640 | Regression_cos_phi: 0.1667 | Regression_energy: 0.2264 | Regression_eta: 0.1631 | Regression_pt: 0.2360 | Regression_sin_phi: 0.2443 | LR: 9.93e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.7381s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.6805 | max_loss: 0.2368 | max_backward: 1.1707 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.9186 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5635.7 | uss_MB: 4432.9 | child_rss_MB: 5638.4 | allocated_MB: 341.2 | reserved_MB: 24554.0 | inactive_split_MB: 308.0 +rank7 [2026-10-28 13:08:36,459] INFO : Step 2900/50000 rank7 | Train Loss: 2.7831 | Train Loss Unweighted: 1.6528 | Train Loss Components: Auxiliary: 0.5323 | Cardinality: 0.3031 | Classification: 0.3069 | Classification_binary: 0.3681 | HitGrouping: 0.3071 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4798 | Regression_eta: 0.0056 | Regression_pt: 0.4869 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1572 | Classification_binary: 0.3627 | Regression_cos_phi: 0.2012 | Regression_energy: 0.2293 | Regression_eta: 0.1193 | Regression_pt: 0.2437 | Regression_sin_phi: 0.1923 | LR: 9.93e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.7848s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.6242 | max_loss: 0.1475 | max_backward: 1.3573 | max_optimizer_step: 0.2831 | max_tensorboard_logging: 0.6063 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 5637.0 | uss_MB: 4433.8 | child_rss_MB: 5645.5 | allocated_MB: 341.4 | reserved_MB: 25570.0 | inactive_split_MB: 292.0 +rank7 [2026-10-28 13:12:15,921] INFO : Step 3000/50000 rank7 | Train Loss: 2.9336 | Train Loss Unweighted: 1.7260 | Train Loss Components: Auxiliary: 0.5466 | Cardinality: 0.2829 | Classification: 0.3524 | Classification_binary: 0.4205 | HitGrouping: 0.3085 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4673 | Regression_eta: 0.0061 | Regression_pt: 0.4733 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1995 | Classification_binary: 0.3874 | Regression_cos_phi: 0.2664 | Regression_energy: 0.2152 | Regression_eta: 0.1433 | Regression_pt: 0.2208 | Regression_sin_phi: 0.2602 | LR: 9.92e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1519s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0386 | max_forward: 0.8057 | max_loss: 0.1488 | max_backward: 1.9376 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5635.9 | uss_MB: 4440.1 | child_rss_MB: 5640.1 | allocated_MB: 346.5 | reserved_MB: 25570.0 | inactive_split_MB: 281.4 +rank7 [2026-10-28 13:12:16,814] INFO : Running validation on rank7 +rank7 [2026-10-28 13:12:16,815] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 13:12:16,815] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 13:13:09,907] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:13:10,420] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 13:13:30,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 13:13:31,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 13:13:32,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 13:13:32,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 13:13:42,961] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 13:13:43,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:13:44,017] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 13:14:00,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 13:14:01,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 13:14:02,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 13:14:03,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 13:14:13,444] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 13:14:13,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:14:14,219] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 13:14:29,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 13:14:30,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 13:14:31,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 13:14:31,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_3000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 13:14:41,759] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 13:18:52,945] INFO : Step 3100/50000 rank7 | Train Loss: 2.7478 | Train Loss Unweighted: 1.6734 | Train Loss Components: Auxiliary: 0.5216 | Cardinality: 0.2531 | Classification: 0.2981 | Classification_binary: 0.3781 | HitGrouping: 0.2957 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4881 | Regression_eta: 0.0057 | Regression_pt: 0.4977 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2091 | Classification_binary: 0.4076 | Regression_cos_phi: 0.2880 | Regression_energy: 0.2352 | Regression_eta: 0.2158 | Regression_pt: 0.2492 | Regression_sin_phi: 0.3445 | LR: 9.91e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5552s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.6511 | max_loss: 0.3988 | max_backward: 1.4306 | max_optimizer_step: 0.2172 | max_tensorboard_logging: 1.2631 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5749.3 | uss_MB: 4552.2 | child_rss_MB: 5658.3 | allocated_MB: 342.5 | reserved_MB: 23120.0 | inactive_split_MB: 194.6 +rank7 [2026-10-28 13:22:38,494] INFO : Step 3200/50000 rank7 | Train Loss: 2.5254 | Train Loss Unweighted: 1.5710 | Train Loss Components: Auxiliary: 0.4888 | Cardinality: 0.2129 | Classification: 0.2845 | Classification_binary: 0.3271 | HitGrouping: 0.2964 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4770 | Regression_eta: 0.0050 | Regression_pt: 0.4723 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1569 | Classification_binary: 0.3218 | Regression_cos_phi: 0.1489 | Regression_energy: 0.2198 | Regression_eta: 0.1228 | Regression_pt: 0.2264 | Regression_sin_phi: 0.1608 | LR: 9.91e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.5505s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.6134 | max_loss: 0.2441 | max_backward: 1.2833 | max_optimizer_step: 0.2753 | max_tensorboard_logging: 0.5629 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5733.9 | uss_MB: 4558.8 | child_rss_MB: 5628.8 | allocated_MB: 343.6 | reserved_MB: 23340.0 | inactive_split_MB: 191.9 +rank7 [2026-10-28 13:26:30,566] INFO : Step 3300/50000 rank7 | Train Loss: 2.8037 | Train Loss Unweighted: 1.6635 | Train Loss Components: Auxiliary: 0.5345 | Cardinality: 0.2841 | Classification: 0.3310 | Classification_binary: 0.3857 | HitGrouping: 0.3002 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4633 | Regression_eta: 0.0058 | Regression_pt: 0.4718 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1869 | Classification_binary: 0.3501 | Regression_cos_phi: 0.2097 | Regression_energy: 0.2129 | Regression_eta: 0.1656 | Regression_pt: 0.2288 | Regression_sin_phi: 0.2433 | LR: 9.90e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2952s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.9147 | max_loss: 0.1535 | max_backward: 1.9100 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.6631 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5735.1 | uss_MB: 4562.9 | child_rss_MB: 5661.7 | allocated_MB: 344.0 | reserved_MB: 23618.0 | inactive_split_MB: 250.0 +rank7 [2026-10-28 13:30:24,443] INFO : Step 3400/50000 rank7 | Train Loss: 2.8855 | Train Loss Unweighted: 1.7067 | Train Loss Components: Auxiliary: 0.5454 | Cardinality: 0.2859 | Classification: 0.3252 | Classification_binary: 0.4155 | HitGrouping: 0.2965 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4731 | Regression_eta: 0.0067 | Regression_pt: 0.4802 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1950 | Classification_binary: 0.4120 | Regression_cos_phi: 0.2171 | Regression_energy: 0.2324 | Regression_eta: 0.1619 | Regression_pt: 0.2436 | Regression_sin_phi: 0.1959 | LR: 9.90e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.5649s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0438 | max_forward: 0.7271 | max_loss: 0.1506 | max_backward: 1.3643 | max_optimizer_step: 0.1716 | max_tensorboard_logging: 0.4827 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5729.2 | uss_MB: 4562.1 | child_rss_MB: 5658.8 | allocated_MB: 346.0 | reserved_MB: 23618.0 | inactive_split_MB: 235.6 +rank7 [2026-10-28 13:34:14,209] INFO : Step 3500/50000 rank7 | Train Loss: 2.4422 | Train Loss Unweighted: 1.4809 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.2120 | Classification: 0.2615 | Classification_binary: 0.3296 | HitGrouping: 0.2898 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4369 | Regression_eta: 0.0054 | Regression_pt: 0.4427 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2048 | Classification_binary: 0.4065 | Regression_cos_phi: 0.2426 | Regression_energy: 0.2238 | Regression_eta: 0.1958 | Regression_pt: 0.2329 | Regression_sin_phi: 0.2612 | LR: 9.89e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.7305s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0058 | max_forward: 0.7056 | max_loss: 0.1971 | max_backward: 1.3359 | max_optimizer_step: 0.1720 | max_tensorboard_logging: 0.6307 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5720.3 | uss_MB: 4563.4 | child_rss_MB: 5630.0 | allocated_MB: 339.0 | reserved_MB: 24610.0 | inactive_split_MB: 280.8 +rank7 [2026-10-28 13:38:02,721] INFO : Step 3600/50000 rank7 | Train Loss: 2.5780 | Train Loss Unweighted: 1.5250 | Train Loss Components: Auxiliary: 0.4892 | Cardinality: 0.2463 | Classification: 0.3079 | Classification_binary: 0.3446 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4295 | Regression_eta: 0.0052 | Regression_pt: 0.4324 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1953 | Classification_binary: 0.3483 | Regression_cos_phi: 0.2904 | Regression_energy: 0.1955 | Regression_eta: 0.1712 | Regression_pt: 0.2032 | Regression_sin_phi: 0.3317 | LR: 9.89e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.4639s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1453 | max_backward: 1.4106 | max_optimizer_step: 0.1681 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0126 | Memory: rss_MB: 5726.7 | uss_MB: 4567.6 | child_rss_MB: 5631.8 | allocated_MB: 343.9 | reserved_MB: 24610.0 | inactive_split_MB: 244.5 +rank7 [2026-10-28 13:41:50,384] INFO : Step 3700/50000 rank7 | Train Loss: 2.9213 | Train Loss Unweighted: 1.6952 | Train Loss Components: Auxiliary: 0.5392 | Cardinality: 0.3153 | Classification: 0.3418 | Classification_binary: 0.4227 | HitGrouping: 0.2968 | Regression_cos_phi: 0.0032 | Regression_energy: 0.4570 | Regression_eta: 0.0063 | Regression_pt: 0.4612 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.2014 | Classification_binary: 0.3730 | Regression_cos_phi: 0.2651 | Regression_energy: 0.2517 | Regression_eta: 0.1700 | Regression_pt: 0.2610 | Regression_sin_phi: 0.1815 | LR: 9.88e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2197s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7710 | max_loss: 0.1507 | max_backward: 1.6430 | max_optimizer_step: 0.1687 | max_tensorboard_logging: 0.8888 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0145 | Memory: rss_MB: 5726.2 | uss_MB: 4569.4 | child_rss_MB: 5661.5 | allocated_MB: 345.4 | reserved_MB: 24610.0 | inactive_split_MB: 268.9 +rank7 [2026-10-28 13:45:46,299] INFO : Step 3800/50000 rank7 | Train Loss: 2.6063 | Train Loss Unweighted: 1.5289 | Train Loss Components: Auxiliary: 0.4933 | Cardinality: 0.2661 | Classification: 0.2839 | Classification_binary: 0.3652 | HitGrouping: 0.2879 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4319 | Regression_eta: 0.0058 | Regression_pt: 0.4369 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1721 | Classification_binary: 0.3979 | Regression_cos_phi: 0.2395 | Regression_energy: 0.2225 | Regression_eta: 0.1400 | Regression_pt: 0.2349 | Regression_sin_phi: 0.2373 | LR: 9.87e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1720s | Phase Times: max_pre_step_barrier: 0.1134 | max_batch_to_device: 0.0063 | max_forward: 0.6784 | max_loss: 0.1451 | max_backward: 1.4269 | max_optimizer_step: 0.2795 | max_tensorboard_logging: 0.9060 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5726.3 | uss_MB: 4569.5 | child_rss_MB: 5664.0 | allocated_MB: 343.8 | reserved_MB: 24610.0 | inactive_split_MB: 202.2 +rank7 [2026-10-28 13:49:32,923] INFO : Step 3900/50000 rank7 | Train Loss: 2.2761 | Train Loss Unweighted: 1.4165 | Train Loss Components: Auxiliary: 0.4517 | Cardinality: 0.1938 | Classification: 0.2553 | Classification_binary: 0.2930 | HitGrouping: 0.2882 | Regression_cos_phi: 0.0020 | Regression_energy: 0.4264 | Regression_eta: 0.0045 | Regression_pt: 0.4331 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1044 | Classification_binary: 0.2685 | Regression_cos_phi: 0.1364 | Regression_energy: 0.2310 | Regression_eta: 0.0970 | Regression_pt: 0.2447 | Regression_sin_phi: 0.1499 | LR: 9.87e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8902s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6197 | max_loss: 0.1501 | max_backward: 1.2810 | max_optimizer_step: 0.1739 | max_tensorboard_logging: 1.0075 | max_tensorboard_flush: 0.0144 | max_post_step_barrier: 0.0053 | Memory: rss_MB: 5727.5 | uss_MB: 3764.5 | child_rss_MB: 5661.9 | allocated_MB: 343.5 | reserved_MB: 24610.0 | inactive_split_MB: 286.6 +rank7 [2026-10-28 13:53:23,686] INFO : Step 4000/50000 rank7 | Train Loss: 2.6760 | Train Loss Unweighted: 1.5903 | Train Loss Components: Auxiliary: 0.5112 | Cardinality: 0.2445 | Classification: 0.3070 | Classification_binary: 0.3812 | HitGrouping: 0.2906 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4413 | Regression_eta: 0.0062 | Regression_pt: 0.4492 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1813 | Classification_binary: 0.3345 | Regression_cos_phi: 0.2441 | Regression_energy: 0.2166 | Regression_eta: 0.1626 | Regression_pt: 0.2226 | Regression_sin_phi: 0.2215 | LR: 9.86e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.0137s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0492 | max_forward: 0.7834 | max_loss: 0.1586 | max_backward: 1.6903 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 0.5589 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5731.5 | uss_MB: 4577.2 | child_rss_MB: 5661.6 | allocated_MB: 345.5 | reserved_MB: 24610.0 | inactive_split_MB: 252.6 +rank7 [2026-10-28 13:53:24,557] INFO : Running validation on rank7 +rank7 [2026-10-28 13:53:24,557] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 13:53:24,558] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 13:54:13,891] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:54:14,418] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 13:54:39,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 13:54:40,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 13:54:41,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 13:54:42,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 13:54:52,652] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 13:54:53,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:54:53,946] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 13:55:09,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 13:55:10,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 13:55:11,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 13:55:12,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 13:55:22,760] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 13:55:23,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:55:23,756] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 13:55:39,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 13:55:40,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 13:55:40,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 13:55:41,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_4000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 13:55:51,386] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 13:59:55,049] INFO : Step 4100/50000 rank7 | Train Loss: 2.6923 | Train Loss Unweighted: 1.5640 | Train Loss Components: Auxiliary: 0.5058 | Cardinality: 0.2671 | Classification: 0.2981 | Classification_binary: 0.4073 | HitGrouping: 0.2911 | Regression_cos_phi: 0.0029 | Regression_energy: 0.4210 | Regression_eta: 0.0055 | Regression_pt: 0.4265 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1820 | Classification_binary: 0.4112 | Regression_cos_phi: 0.2485 | Regression_energy: 0.1960 | Regression_eta: 0.1698 | Regression_pt: 0.2061 | Regression_sin_phi: 0.2443 | LR: 9.85e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.1513s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0141 | max_forward: 0.7422 | max_loss: 0.3919 | max_backward: 1.3629 | max_optimizer_step: 0.1890 | max_tensorboard_logging: 0.9903 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6680.3 | uss_MB: 5525.0 | child_rss_MB: 5670.9 | allocated_MB: 345.2 | reserved_MB: 23430.0 | inactive_split_MB: 294.4 +rank7 [2026-10-28 14:03:47,900] INFO : Step 4200/50000 rank7 | Train Loss: 2.3415 | Train Loss Unweighted: 1.4424 | Train Loss Components: Auxiliary: 0.4542 | Cardinality: 0.2058 | Classification: 0.2540 | Classification_binary: 0.3143 | HitGrouping: 0.2846 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4310 | Regression_eta: 0.0050 | Regression_pt: 0.4337 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1545 | Classification_binary: 0.4257 | Regression_cos_phi: 0.2347 | Regression_energy: 0.2016 | Regression_eta: 0.1693 | Regression_pt: 0.2129 | Regression_sin_phi: 0.1953 | LR: 9.84e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.7414s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0168 | max_forward: 0.8335 | max_loss: 0.2425 | max_backward: 1.5586 | max_optimizer_step: 0.1744 | max_tensorboard_logging: 0.4781 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5904.9 | uss_MB: 4751.1 | child_rss_MB: 5672.4 | allocated_MB: 341.2 | reserved_MB: 24406.0 | inactive_split_MB: 273.9 +rank7 [2026-10-28 14:07:35,176] INFO : Step 4300/50000 rank7 | Train Loss: 2.5102 | Train Loss Unweighted: 1.4857 | Train Loss Components: Auxiliary: 0.4757 | Cardinality: 0.2309 | Classification: 0.2786 | Classification_binary: 0.3543 | HitGrouping: 0.2938 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4177 | Regression_eta: 0.0054 | Regression_pt: 0.4246 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1401 | Classification_binary: 0.3323 | Regression_cos_phi: 0.1910 | Regression_energy: 0.1988 | Regression_eta: 0.1186 | Regression_pt: 0.2061 | Regression_sin_phi: 0.2141 | LR: 9.84e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5989s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6520 | max_loss: 0.2453 | max_backward: 1.4084 | max_optimizer_step: 0.1863 | max_tensorboard_logging: 0.5540 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0208 | Memory: rss_MB: 5910.3 | uss_MB: 4757.7 | child_rss_MB: 5668.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 239.6 +rank7 [2026-10-28 14:11:23,917] INFO : Step 4400/50000 rank7 | Train Loss: 2.8258 | Train Loss Unweighted: 1.5691 | Train Loss Components: Auxiliary: 0.5223 | Cardinality: 0.3313 | Classification: 0.2877 | Classification_binary: 0.4120 | HitGrouping: 0.2902 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4255 | Regression_eta: 0.0063 | Regression_pt: 0.4316 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1578 | Classification_binary: 0.3861 | Regression_cos_phi: 0.2591 | Regression_energy: 0.1910 | Regression_eta: 0.1563 | Regression_pt: 0.2010 | Regression_sin_phi: 0.2059 | LR: 9.83e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.0440s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 0.7635 | max_loss: 0.2580 | max_backward: 1.7456 | max_optimizer_step: 0.1777 | max_tensorboard_logging: 1.5221 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 5909.1 | uss_MB: 4757.0 | child_rss_MB: 5674.0 | allocated_MB: 344.6 | reserved_MB: 24406.0 | inactive_split_MB: 261.0 +rank7 [2026-10-28 14:15:17,053] INFO : Step 4500/50000 rank7 | Train Loss: 2.5622 | Train Loss Unweighted: 1.4664 | Train Loss Components: Auxiliary: 0.4942 | Cardinality: 0.3132 | Classification: 0.2685 | Classification_binary: 0.3695 | HitGrouping: 0.2876 | Regression_cos_phi: 0.0023 | Regression_energy: 0.4070 | Regression_eta: 0.0048 | Regression_pt: 0.4117 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1859 | Classification_binary: 0.3913 | Regression_cos_phi: 0.2444 | Regression_energy: 0.2093 | Regression_eta: 0.1588 | Regression_pt: 0.2182 | Regression_sin_phi: 0.2476 | LR: 9.82e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.2151s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0462 | max_forward: 1.1330 | max_loss: 0.3526 | max_backward: 2.1744 | max_optimizer_step: 0.1888 | max_tensorboard_logging: 0.5571 | max_tensorboard_flush: 0.0016 | max_post_step_barrier: 0.0680 | Memory: rss_MB: 5912.8 | uss_MB: 4736.1 | child_rss_MB: 5674.9 | allocated_MB: 343.9 | reserved_MB: 24406.0 | inactive_split_MB: 288.8 +rank7 [2026-10-28 14:19:09,440] INFO : Step 4600/50000 rank7 | Train Loss: 2.2516 | Train Loss Unweighted: 1.3548 | Train Loss Components: Auxiliary: 0.4408 | Cardinality: 0.2151 | Classification: 0.2428 | Classification_binary: 0.2886 | HitGrouping: 0.2811 | Regression_cos_phi: 0.0019 | Regression_energy: 0.4059 | Regression_eta: 0.0042 | Regression_pt: 0.4088 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1730 | Classification_binary: 0.2803 | Regression_cos_phi: 0.1781 | Regression_energy: 0.1838 | Regression_eta: 0.1233 | Regression_pt: 0.1882 | Regression_sin_phi: 0.2073 | LR: 9.81e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7001s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0445 | max_forward: 0.8897 | max_loss: 0.2214 | max_backward: 1.8748 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.5636 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 5913.2 | uss_MB: 4741.7 | child_rss_MB: 5653.3 | allocated_MB: 341.6 | reserved_MB: 25392.0 | inactive_split_MB: 298.4 +rank7 [2026-10-28 14:23:11,224] INFO : Step 4700/50000 rank7 | Train Loss: 2.5991 | Train Loss Unweighted: 1.5142 | Train Loss Components: Auxiliary: 0.4914 | Cardinality: 0.2679 | Classification: 0.2997 | Classification_binary: 0.3514 | HitGrouping: 0.2867 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4228 | Regression_eta: 0.0061 | Regression_pt: 0.4290 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1475 | Classification_binary: 0.3230 | Regression_cos_phi: 0.1702 | Regression_energy: 0.2325 | Regression_eta: 0.1503 | Regression_pt: 0.2451 | Regression_sin_phi: 0.2119 | LR: 9.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8897s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7362 | max_loss: 0.2547 | max_backward: 1.8331 | max_optimizer_step: 0.1734 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5918.9 | uss_MB: 4764.8 | child_rss_MB: 5675.1 | allocated_MB: 343.1 | reserved_MB: 26384.0 | inactive_split_MB: 292.4 +rank7 [2026-10-28 14:27:13,069] INFO : Step 4800/50000 rank7 | Train Loss: 2.7077 | Train Loss Unweighted: 1.5200 | Train Loss Components: Auxiliary: 0.5118 | Cardinality: 0.3360 | Classification: 0.2606 | Classification_binary: 0.3918 | HitGrouping: 0.2871 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4264 | Regression_eta: 0.0053 | Regression_pt: 0.4305 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1639 | Classification_binary: 0.4126 | Regression_cos_phi: 0.2265 | Regression_energy: 0.1931 | Regression_eta: 0.1509 | Regression_pt: 0.2018 | Regression_sin_phi: 0.1949 | LR: 9.80e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.2494s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0074 | max_forward: 0.7154 | max_loss: 0.1552 | max_backward: 1.4398 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.8979 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0854 | Memory: rss_MB: 5917.3 | uss_MB: 4226.4 | child_rss_MB: 5678.8 | allocated_MB: 346.1 | reserved_MB: 26384.0 | inactive_split_MB: 284.0 +rank7 [2026-10-28 14:31:09,416] INFO : Step 4900/50000 rank7 | Train Loss: 2.3682 | Train Loss Unweighted: 1.4242 | Train Loss Components: Auxiliary: 0.4616 | Cardinality: 0.2176 | Classification: 0.2570 | Classification_binary: 0.3149 | HitGrouping: 0.2900 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4191 | Regression_eta: 0.0047 | Regression_pt: 0.4239 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1375 | Classification_binary: 0.3241 | Regression_cos_phi: 0.1955 | Regression_energy: 0.2206 | Regression_eta: 0.1144 | Regression_pt: 0.2345 | Regression_sin_phi: 0.2183 | LR: 9.79e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.4650s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0516 | max_forward: 0.6608 | max_loss: 0.1430 | max_backward: 1.2657 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.5696 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0187 | Memory: rss_MB: 5916.2 | uss_MB: 4765.4 | child_rss_MB: 5677.6 | allocated_MB: 343.8 | reserved_MB: 26384.0 | inactive_split_MB: 201.9 +rank7 [2026-10-28 14:35:19,045] INFO : Step 5000/50000 rank7 | Train Loss: 2.4060 | Train Loss Unweighted: 1.4130 | Train Loss Components: Auxiliary: 0.4756 | Cardinality: 0.2573 | Classification: 0.2748 | Classification_binary: 0.3198 | HitGrouping: 0.2768 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4020 | Regression_eta: 0.0043 | Regression_pt: 0.4074 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1185 | Classification_binary: 0.2778 | Regression_cos_phi: 0.1489 | Regression_energy: 0.2167 | Regression_eta: 0.0759 | Regression_pt: 0.2254 | Regression_sin_phi: 0.1167 | LR: 9.78e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8087s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.6153 | max_loss: 0.1441 | max_backward: 1.2119 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.9741 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5914.8 | uss_MB: 4767.1 | child_rss_MB: 5669.5 | allocated_MB: 342.5 | reserved_MB: 27388.0 | inactive_split_MB: 218.4 +rank7 [2026-10-28 14:35:19,963] INFO : Running validation on rank7 +rank7 [2026-10-28 14:35:19,963] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 14:35:19,963] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 14:36:10,799] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:36:11,318] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 14:36:36,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 14:36:37,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 14:36:38,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 14:36:39,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 14:36:49,348] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 14:36:50,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:36:50,780] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 14:37:06,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 14:37:07,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 14:37:08,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 14:37:09,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 14:37:19,600] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 14:37:20,104] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:37:20,581] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 14:37:36,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 14:37:37,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 14:37:37,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 14:37:38,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_5000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 14:37:48,410] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 14:42:02,659] INFO : Step 5100/50000 rank7 | Train Loss: 2.7059 | Train Loss Unweighted: 1.5631 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2507 | Classification: 0.3068 | Classification_binary: 0.4110 | HitGrouping: 0.2884 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4148 | Regression_eta: 0.0060 | Regression_pt: 0.4186 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1834 | Classification_binary: 0.4226 | Regression_cos_phi: 0.2545 | Regression_energy: 0.1991 | Regression_eta: 0.2002 | Regression_pt: 0.2077 | Regression_sin_phi: 0.2784 | LR: 9.77e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4420s | Phase Times: max_pre_step_barrier: 0.1071 | max_batch_to_device: 0.0070 | max_forward: 0.7192 | max_loss: 0.2509 | max_backward: 1.6066 | max_optimizer_step: 0.1929 | max_tensorboard_logging: 0.8859 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7032.2 | uss_MB: 5882.9 | child_rss_MB: 5662.0 | allocated_MB: 343.2 | reserved_MB: 23542.0 | inactive_split_MB: 236.2 +rank7 [2026-10-28 14:45:56,261] INFO : Step 5200/50000 rank7 | Train Loss: 2.4117 | Train Loss Unweighted: 1.4393 | Train Loss Components: Auxiliary: 0.4667 | Cardinality: 0.2215 | Classification: 0.2426 | Classification_binary: 0.3524 | HitGrouping: 0.2783 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4149 | Regression_eta: 0.0050 | Regression_pt: 0.4196 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1485 | Classification_binary: 0.3755 | Regression_cos_phi: 0.2308 | Regression_energy: 0.1912 | Regression_eta: 0.1319 | Regression_pt: 0.2004 | Regression_sin_phi: 0.2043 | LR: 9.76e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9045s | Phase Times: max_pre_step_barrier: 0.1151 | max_batch_to_device: 0.0079 | max_forward: 0.8909 | max_loss: 0.1450 | max_backward: 1.3654 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.5791 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0216 | Memory: rss_MB: 6020.7 | uss_MB: 4875.4 | child_rss_MB: 5666.4 | allocated_MB: 346.1 | reserved_MB: 23690.0 | inactive_split_MB: 197.9 +rank7 [2026-10-28 14:50:03,850] INFO : Step 5300/50000 rank7 | Train Loss: 2.2417 | Train Loss Unweighted: 1.3238 | Train Loss Components: Auxiliary: 0.4353 | Cardinality: 0.2226 | Classification: 0.2387 | Classification_binary: 0.3027 | HitGrouping: 0.2808 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3844 | Regression_eta: 0.0041 | Regression_pt: 0.3895 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1337 | Classification_binary: 0.2982 | Regression_cos_phi: 0.1658 | Regression_energy: 0.1815 | Regression_eta: 0.1133 | Regression_pt: 0.1861 | Regression_sin_phi: 0.1810 | LR: 9.75e-04 | DataLoad Time: 0.0235s | Model Forward Time: 4.1969s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0155 | max_forward: 1.3549 | max_loss: 0.4272 | max_backward: 1.7517 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5867 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 6024.3 | uss_MB: 4878.1 | child_rss_MB: 5664.8 | allocated_MB: 341.7 | reserved_MB: 25496.0 | inactive_split_MB: 195.4 +rank7 [2026-10-28 14:54:26,616] INFO : Step 5400/50000 rank7 | Train Loss: 2.5857 | Train Loss Unweighted: 1.4820 | Train Loss Components: Auxiliary: 0.4850 | Cardinality: 0.2715 | Classification: 0.2898 | Classification_binary: 0.3613 | HitGrouping: 0.2829 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4076 | Regression_eta: 0.0053 | Regression_pt: 0.4125 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1652 | Classification_binary: 0.3204 | Regression_cos_phi: 0.2288 | Regression_energy: 0.1934 | Regression_eta: 0.1408 | Regression_pt: 0.2022 | Regression_sin_phi: 0.2442 | LR: 9.74e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.8616s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7472 | max_loss: 0.1664 | max_backward: 1.6216 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4956 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 6028.8 | uss_MB: 4880.2 | child_rss_MB: 5668.3 | allocated_MB: 345.8 | reserved_MB: 25496.0 | inactive_split_MB: 226.8 +rank7 [2026-10-28 14:58:29,775] INFO : Step 5500/50000 rank7 | Train Loss: 2.4383 | Train Loss Unweighted: 1.4361 | Train Loss Components: Auxiliary: 0.4636 | Cardinality: 0.2531 | Classification: 0.2547 | Classification_binary: 0.3846 | HitGrouping: 0.2764 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3900 | Regression_eta: 0.0045 | Regression_pt: 0.3974 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1420 | Classification_binary: 0.3892 | Regression_cos_phi: 0.1990 | Regression_energy: 0.1854 | Regression_eta: 0.1166 | Regression_pt: 0.1955 | Regression_sin_phi: 0.2098 | LR: 9.73e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3526s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0073 | max_forward: 0.7547 | max_loss: 0.2915 | max_backward: 1.8143 | max_optimizer_step: 0.2819 | max_tensorboard_logging: 0.5958 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6029.1 | uss_MB: 4882.8 | child_rss_MB: 5667.9 | allocated_MB: 342.5 | reserved_MB: 26496.0 | inactive_split_MB: 471.5 +rank7 [2026-10-28 15:02:37,793] INFO : Step 5600/50000 rank7 | Train Loss: 2.2622 | Train Loss Unweighted: 1.3177 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.1990 | Classification: 0.2257 | Classification_binary: 0.3268 | HitGrouping: 0.2800 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3758 | Regression_eta: 0.0051 | Regression_pt: 0.3799 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1575 | Classification_binary: 0.3793 | Regression_cos_phi: 0.2671 | Regression_energy: 0.1725 | Regression_eta: 0.1688 | Regression_pt: 0.1744 | Regression_sin_phi: 0.2778 | LR: 9.72e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.1688s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0347 | max_forward: 1.1910 | max_loss: 0.2555 | max_backward: 1.5431 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.4639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0235 | Memory: rss_MB: 6024.4 | uss_MB: 4885.9 | child_rss_MB: 5669.8 | allocated_MB: 340.8 | reserved_MB: 26496.0 | inactive_split_MB: 191.2 +rank7 [2026-10-28 15:06:24,089] INFO : Step 5700/50000 rank7 | Train Loss: 2.4588 | Train Loss Unweighted: 1.4026 | Train Loss Components: Auxiliary: 0.4666 | Cardinality: 0.2348 | Classification: 0.2768 | Classification_binary: 0.3365 | HitGrouping: 0.2916 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3864 | Regression_eta: 0.0055 | Regression_pt: 0.3923 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1720 | Classification_binary: 0.3453 | Regression_cos_phi: 0.2703 | Regression_energy: 0.2013 | Regression_eta: 0.1641 | Regression_pt: 0.2070 | Regression_sin_phi: 0.2027 | LR: 9.71e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.5064s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.6547 | max_loss: 0.1874 | max_backward: 1.2748 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6027.1 | uss_MB: 4889.2 | child_rss_MB: 5674.0 | allocated_MB: 344.4 | reserved_MB: 26496.0 | inactive_split_MB: 224.5 +rank7 [2026-10-28 15:10:14,851] INFO : Step 5800/50000 rank7 | Train Loss: 2.4947 | Train Loss Unweighted: 1.4616 | Train Loss Components: Auxiliary: 0.4752 | Cardinality: 0.2715 | Classification: 0.2623 | Classification_binary: 0.3729 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0025 | Regression_energy: 0.4032 | Regression_eta: 0.0047 | Regression_pt: 0.4136 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1473 | Classification_binary: 0.3383 | Regression_cos_phi: 0.1846 | Regression_energy: 0.2030 | Regression_eta: 0.1135 | Regression_pt: 0.2156 | Regression_sin_phi: 0.1759 | LR: 9.70e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7378s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.9768 | max_loss: 0.2512 | max_backward: 2.0573 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.6978 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6026.0 | uss_MB: 4048.8 | child_rss_MB: 5650.8 | allocated_MB: 343.4 | reserved_MB: 27494.0 | inactive_split_MB: 198.3 +rank7 [2026-10-28 15:14:12,312] INFO : Step 5900/50000 rank7 | Train Loss: 2.3800 | Train Loss Unweighted: 1.3838 | Train Loss Components: Auxiliary: 0.4549 | Cardinality: 0.2203 | Classification: 0.2438 | Classification_binary: 0.3435 | HitGrouping: 0.2757 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3900 | Regression_eta: 0.0050 | Regression_pt: 0.3965 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1810 | Classification_binary: 0.3726 | Regression_cos_phi: 0.2382 | Regression_energy: 0.1852 | Regression_eta: 0.1432 | Regression_pt: 0.1966 | Regression_sin_phi: 0.2529 | LR: 9.69e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7142s | Phase Times: max_pre_step_barrier: 0.0158 | max_batch_to_device: 0.0069 | max_forward: 1.0355 | max_loss: 0.2311 | max_backward: 2.2749 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.5131 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0141 | Memory: rss_MB: 6029.8 | uss_MB: 3868.6 | child_rss_MB: 5658.5 | allocated_MB: 342.1 | reserved_MB: 27494.0 | inactive_split_MB: 281.2 +rank7 [2026-10-28 15:18:13,894] INFO : Step 6000/50000 rank7 | Train Loss: 2.1372 | Train Loss Unweighted: 1.3015 | Train Loss Components: Auxiliary: 0.4179 | Cardinality: 0.1916 | Classification: 0.2091 | Classification_binary: 0.2848 | HitGrouping: 0.2746 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3966 | Regression_eta: 0.0041 | Regression_pt: 0.4029 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1218 | Classification_binary: 0.2841 | Regression_cos_phi: 0.1277 | Regression_energy: 0.1646 | Regression_eta: 0.0924 | Regression_pt: 0.1724 | Regression_sin_phi: 0.1915 | LR: 9.68e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.6780s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.6457 | max_loss: 0.2157 | max_backward: 1.4984 | max_optimizer_step: 0.1794 | max_tensorboard_logging: 0.5590 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6033.8 | uss_MB: 4891.5 | child_rss_MB: 5658.2 | allocated_MB: 343.1 | reserved_MB: 27494.0 | inactive_split_MB: 260.7 +rank7 [2026-10-28 15:18:14,999] INFO : Running validation on rank7 +rank7 [2026-10-28 15:18:14,999] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 15:18:14,999] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 15:19:17,313] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:19:17,849] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 15:19:43,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 15:19:44,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 15:19:45,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 15:19:46,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 15:19:56,444] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 15:19:57,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:19:58,136] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 15:20:14,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 15:20:15,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 15:20:16,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 15:20:16,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 15:20:26,992] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 15:20:27,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:20:28,076] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 15:20:43,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 15:20:44,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 15:20:45,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 15:20:46,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_6000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 15:20:56,210] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 15:25:29,846] INFO : Step 6100/50000 rank7 | Train Loss: 2.4040 | Train Loss Unweighted: 1.3907 | Train Loss Components: Auxiliary: 0.4661 | Cardinality: 0.2610 | Classification: 0.2427 | Classification_binary: 0.3401 | HitGrouping: 0.2699 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3968 | Regression_eta: 0.0047 | Regression_pt: 0.4016 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1026 | Classification_binary: 0.2658 | Regression_cos_phi: 0.1220 | Regression_energy: 0.1769 | Regression_eta: 0.0805 | Regression_pt: 0.1883 | Regression_sin_phi: 0.1568 | LR: 9.67e-04 | DataLoad Time: 0.0189s | Model Forward Time: 6.4046s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0073 | max_forward: 1.0567 | max_loss: 0.2452 | max_backward: 2.4106 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.9836 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0323 | Memory: rss_MB: 7214.6 | uss_MB: 5292.3 | child_rss_MB: 5654.8 | allocated_MB: 344.4 | reserved_MB: 23648.0 | inactive_split_MB: 194.8 +rank7 [2026-10-28 15:29:38,624] INFO : Step 6200/50000 rank7 | Train Loss: 2.4942 | Train Loss Unweighted: 1.4287 | Train Loss Components: Auxiliary: 0.4744 | Cardinality: 0.2537 | Classification: 0.2580 | Classification_binary: 0.3817 | HitGrouping: 0.2748 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3867 | Regression_eta: 0.0050 | Regression_pt: 0.3920 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1532 | Classification_binary: 0.3931 | Regression_cos_phi: 0.2260 | Regression_energy: 0.1978 | Regression_eta: 0.1407 | Regression_pt: 0.2068 | Regression_sin_phi: 0.2054 | LR: 9.66e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0640s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.7838 | max_loss: 0.1532 | max_backward: 1.6314 | max_optimizer_step: 0.1841 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6093.8 | uss_MB: 4537.3 | child_rss_MB: 5656.7 | allocated_MB: 343.8 | reserved_MB: 23830.0 | inactive_split_MB: 215.7 +rank7 [2026-10-28 15:34:14,251] INFO : Step 6300/50000 rank7 | Train Loss: 2.2204 | Train Loss Unweighted: 1.2763 | Train Loss Components: Auxiliary: 0.4285 | Cardinality: 0.2506 | Classification: 0.2258 | Classification_binary: 0.2983 | HitGrouping: 0.2710 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3704 | Regression_eta: 0.0045 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1212 | Classification_binary: 0.3187 | Regression_cos_phi: 0.2019 | Regression_energy: 0.1802 | Regression_eta: 0.1279 | Regression_pt: 0.1915 | Regression_sin_phi: 0.1476 | LR: 9.65e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.6717s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.5854 | max_loss: 0.1923 | max_backward: 1.0848 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.8586 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6096.2 | uss_MB: 4957.2 | child_rss_MB: 5651.4 | allocated_MB: 339.7 | reserved_MB: 24826.0 | inactive_split_MB: 251.6 +rank7 [2026-10-28 15:38:39,607] INFO : Step 6400/50000 rank7 | Train Loss: 2.2882 | Train Loss Unweighted: 1.3014 | Train Loss Components: Auxiliary: 0.4348 | Cardinality: 0.2271 | Classification: 0.2431 | Classification_binary: 0.3058 | HitGrouping: 0.2702 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3679 | Regression_eta: 0.0046 | Regression_pt: 0.3751 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1149 | Classification_binary: 0.2869 | Regression_cos_phi: 0.1486 | Regression_energy: 0.1601 | Regression_eta: 0.1011 | Regression_pt: 0.1681 | Regression_sin_phi: 0.1684 | LR: 9.64e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.8560s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7489 | max_loss: 0.3608 | max_backward: 2.5327 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 0.9886 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0400 | Memory: rss_MB: 6095.6 | uss_MB: 4594.4 | child_rss_MB: 5649.8 | allocated_MB: 345.8 | reserved_MB: 24826.0 | inactive_split_MB: 253.0 +rank7 [2026-10-28 15:42:45,601] INFO : Step 6500/50000 rank7 | Train Loss: 2.4626 | Train Loss Unweighted: 1.3908 | Train Loss Components: Auxiliary: 0.4705 | Cardinality: 0.3022 | Classification: 0.2393 | Classification_binary: 0.3805 | HitGrouping: 0.2651 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3783 | Regression_eta: 0.0049 | Regression_pt: 0.3831 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1411 | Classification_binary: 0.3903 | Regression_cos_phi: 0.2485 | Regression_energy: 0.1836 | Regression_eta: 0.1749 | Regression_pt: 0.1939 | Regression_sin_phi: 0.2560 | LR: 9.63e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1896s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0074 | max_forward: 0.7600 | max_loss: 0.1555 | max_backward: 1.3910 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6094.6 | uss_MB: 4960.0 | child_rss_MB: 5656.1 | allocated_MB: 344.4 | reserved_MB: 24826.0 | inactive_split_MB: 269.0 +rank7 [2026-10-28 15:46:47,733] INFO : Step 6600/50000 rank7 | Train Loss: 2.3121 | Train Loss Unweighted: 1.3617 | Train Loss Components: Auxiliary: 0.4493 | Cardinality: 0.2405 | Classification: 0.2289 | Classification_binary: 0.3388 | HitGrouping: 0.2650 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3914 | Regression_eta: 0.0045 | Regression_pt: 0.3935 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1189 | Classification_binary: 0.3557 | Regression_cos_phi: 0.1436 | Regression_energy: 0.1968 | Regression_eta: 0.0935 | Regression_pt: 0.2054 | Regression_sin_phi: 0.1732 | LR: 9.62e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7778s | Phase Times: max_pre_step_barrier: 0.1106 | max_batch_to_device: 0.0061 | max_forward: 0.7587 | max_loss: 0.2869 | max_backward: 2.2641 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1321 | Memory: rss_MB: 6092.2 | uss_MB: 4956.2 | child_rss_MB: 5651.6 | allocated_MB: 343.8 | reserved_MB: 25846.0 | inactive_split_MB: 267.9 +rank7 [2026-10-28 15:51:24,436] INFO : Step 6700/50000 rank7 | Train Loss: 2.1225 | Train Loss Unweighted: 1.2385 | Train Loss Components: Auxiliary: 0.4092 | Cardinality: 0.2100 | Classification: 0.2079 | Classification_binary: 0.2916 | HitGrouping: 0.2751 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3623 | Regression_eta: 0.0041 | Regression_pt: 0.3686 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0948 | Classification_binary: 0.3026 | Regression_cos_phi: 0.1625 | Regression_energy: 0.1717 | Regression_eta: 0.0969 | Regression_pt: 0.1794 | Regression_sin_phi: 0.1522 | LR: 9.61e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9918s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.6452 | max_loss: 0.1453 | max_backward: 1.2984 | max_optimizer_step: 0.1872 | max_tensorboard_logging: 1.0269 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6098.1 | uss_MB: 4965.1 | child_rss_MB: 5657.8 | allocated_MB: 345.0 | reserved_MB: 25846.0 | inactive_split_MB: 258.8 +rank7 [2026-10-28 15:55:32,974] INFO : Step 6800/50000 rank7 | Train Loss: 2.4601 | Train Loss Unweighted: 1.3772 | Train Loss Components: Auxiliary: 0.4589 | Cardinality: 0.2427 | Classification: 0.2631 | Classification_binary: 0.3490 | HitGrouping: 0.2666 | Regression_cos_phi: 0.0027 | Regression_energy: 0.3747 | Regression_eta: 0.0054 | Regression_pt: 0.3793 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1627 | Classification_binary: 0.3396 | Regression_cos_phi: 0.2258 | Regression_energy: 0.1780 | Regression_eta: 0.1720 | Regression_pt: 0.1883 | Regression_sin_phi: 0.2114 | LR: 9.60e-04 | DataLoad Time: 0.0247s | Model Forward Time: 5.6146s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0662 | max_forward: 1.6558 | max_loss: 0.3855 | max_backward: 2.1309 | max_optimizer_step: 0.1880 | max_tensorboard_logging: 1.0709 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6092.6 | uss_MB: 4961.3 | child_rss_MB: 5647.9 | allocated_MB: 345.8 | reserved_MB: 26622.0 | inactive_split_MB: 194.2 +rank7 [2026-10-28 15:59:35,211] INFO : Step 6900/50000 rank7 | Train Loss: 2.3793 | Train Loss Unweighted: 1.3337 | Train Loss Components: Auxiliary: 0.4495 | Cardinality: 0.2757 | Classification: 0.2300 | Classification_binary: 0.3711 | HitGrouping: 0.2647 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3605 | Regression_eta: 0.0045 | Regression_pt: 0.3628 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1767 | Classification_binary: 0.3826 | Regression_cos_phi: 0.2269 | Regression_energy: 0.1487 | Regression_eta: 0.1856 | Regression_pt: 0.1565 | Regression_sin_phi: 0.2566 | LR: 9.58e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2972s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1120 | max_forward: 0.8190 | max_loss: 0.1496 | max_backward: 1.4356 | max_optimizer_step: 0.2847 | max_tensorboard_logging: 0.8440 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6096.9 | uss_MB: 4962.9 | child_rss_MB: 5656.8 | allocated_MB: 345.4 | reserved_MB: 26622.0 | inactive_split_MB: 300.5 +rank7 [2026-10-28 16:03:56,960] INFO : Step 7000/50000 rank7 | Train Loss: 2.2410 | Train Loss Unweighted: 1.2707 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.2746 | Classification: 0.1970 | Classification_binary: 0.3002 | HitGrouping: 0.2655 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3795 | Regression_eta: 0.0045 | Regression_pt: 0.3852 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1274 | Classification_binary: 0.3196 | Regression_cos_phi: 0.1495 | Regression_energy: 0.2053 | Regression_eta: 0.1114 | Regression_pt: 0.2158 | Regression_sin_phi: 0.1820 | LR: 9.57e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.3852s | Phase Times: max_pre_step_barrier: 0.0229 | max_batch_to_device: 0.0154 | max_forward: 1.0393 | max_loss: 0.2251 | max_backward: 2.0451 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.9425 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6097.4 | uss_MB: 4965.6 | child_rss_MB: 5656.9 | allocated_MB: 345.5 | reserved_MB: 26622.0 | inactive_split_MB: 296.5 +rank7 [2026-10-28 16:03:58,354] INFO : Running validation on rank7 +rank7 [2026-10-28 16:03:58,354] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 16:03:58,354] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 16:04:51,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:04:52,381] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 16:05:18,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 16:05:19,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 16:05:20,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 16:05:21,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 16:05:31,248] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 16:05:32,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:05:32,550] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 16:05:48,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 16:05:49,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 16:05:50,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 16:05:51,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 16:06:01,356] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 16:06:02,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:06:02,710] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 16:06:18,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 16:06:19,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 16:06:19,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 16:06:20,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_7000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 16:06:30,605] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 16:10:49,043] INFO : Step 7100/50000 rank7 | Train Loss: 2.1086 | Train Loss Unweighted: 1.2525 | Train Loss Components: Auxiliary: 0.4099 | Cardinality: 0.1821 | Classification: 0.2186 | Classification_binary: 0.2903 | HitGrouping: 0.2613 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3655 | Regression_eta: 0.0041 | Regression_pt: 0.3696 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0889 | Classification_binary: 0.2926 | Regression_cos_phi: 0.1434 | Regression_energy: 0.1669 | Regression_eta: 0.0703 | Regression_pt: 0.1736 | Regression_sin_phi: 0.1140 | LR: 9.56e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.9389s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.6376 | max_loss: 0.1424 | max_backward: 1.3436 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8454 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 7191.1 | uss_MB: 6057.4 | child_rss_MB: 5686.3 | allocated_MB: 345.6 | reserved_MB: 22618.0 | inactive_split_MB: 206.0 +rank7 [2026-10-28 16:15:05,704] INFO : Step 7200/50000 rank7 | Train Loss: 2.4589 | Train Loss Unweighted: 1.3873 | Train Loss Components: Auxiliary: 0.4658 | Cardinality: 0.2721 | Classification: 0.2648 | Classification_binary: 0.3806 | HitGrouping: 0.2633 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3642 | Regression_eta: 0.0051 | Regression_pt: 0.3675 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1479 | Classification_binary: 0.3517 | Regression_cos_phi: 0.2310 | Regression_energy: 0.1846 | Regression_eta: 0.1742 | Regression_pt: 0.1889 | Regression_sin_phi: 0.2010 | LR: 9.55e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3061s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 1.0191 | max_loss: 0.1674 | max_backward: 1.7813 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5868 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6183.3 | uss_MB: 5050.5 | child_rss_MB: 5662.0 | allocated_MB: 343.5 | reserved_MB: 22618.0 | inactive_split_MB: 315.3 +rank7 [2026-10-28 16:19:17,626] INFO : Step 7300/50000 rank7 | Train Loss: 2.1617 | Train Loss Unweighted: 1.2404 | Train Loss Components: Auxiliary: 0.4141 | Cardinality: 0.2258 | Classification: 0.1979 | Classification_binary: 0.3107 | HitGrouping: 0.2671 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3613 | Regression_eta: 0.0041 | Regression_pt: 0.3621 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1234 | Classification_binary: 0.3364 | Regression_cos_phi: 0.1850 | Regression_energy: 0.1609 | Regression_eta: 0.1084 | Regression_pt: 0.1652 | Regression_sin_phi: 0.1605 | LR: 9.53e-04 | DataLoad Time: 0.0202s | Model Forward Time: 3.7540s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0084 | max_forward: 0.7227 | max_loss: 0.1491 | max_backward: 1.4232 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.5902 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6180.9 | uss_MB: 5051.2 | child_rss_MB: 5696.9 | allocated_MB: 345.0 | reserved_MB: 23608.0 | inactive_split_MB: 295.0 +rank7 [2026-10-28 16:23:25,991] INFO : Step 7400/50000 rank7 | Train Loss: 2.0741 | Train Loss Unweighted: 1.2014 | Train Loss Components: Auxiliary: 0.4055 | Cardinality: 0.2287 | Classification: 0.2035 | Classification_binary: 0.2781 | HitGrouping: 0.2646 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3549 | Regression_eta: 0.0040 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1176 | Classification_binary: 0.2600 | Regression_cos_phi: 0.1400 | Regression_energy: 0.1800 | Regression_eta: 0.1113 | Regression_pt: 0.1833 | Regression_sin_phi: 0.1777 | LR: 9.52e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.6550s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6129 | max_loss: 0.2127 | max_backward: 1.2944 | max_optimizer_step: 0.1887 | max_tensorboard_logging: 0.6450 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6180.9 | uss_MB: 5049.5 | child_rss_MB: 5695.6 | allocated_MB: 343.8 | reserved_MB: 23608.0 | inactive_split_MB: 229.5 +rank7 [2026-10-28 16:27:18,559] INFO : Step 7500/50000 rank7 | Train Loss: 2.4640 | Train Loss Unweighted: 1.3686 | Train Loss Components: Auxiliary: 0.4637 | Cardinality: 0.2640 | Classification: 0.2570 | Classification_binary: 0.3594 | HitGrouping: 0.2687 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3684 | Regression_eta: 0.0054 | Regression_pt: 0.3731 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1672 | Classification_binary: 0.3615 | Regression_cos_phi: 0.3078 | Regression_energy: 0.1866 | Regression_eta: 0.1983 | Regression_pt: 0.1966 | Regression_sin_phi: 0.2598 | LR: 9.51e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1725s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0887 | max_forward: 0.7717 | max_loss: 0.1508 | max_backward: 1.3590 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.9634 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6179.4 | uss_MB: 5053.3 | child_rss_MB: 5684.8 | allocated_MB: 348.6 | reserved_MB: 23608.0 | inactive_split_MB: 283.7 +rank7 [2026-10-28 16:31:22,305] INFO : Step 7600/50000 rank7 | Train Loss: 2.3745 | Train Loss Unweighted: 1.3447 | Train Loss Components: Auxiliary: 0.4509 | Cardinality: 0.2616 | Classification: 0.2519 | Classification_binary: 0.3743 | HitGrouping: 0.2627 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3532 | Regression_eta: 0.0050 | Regression_pt: 0.3557 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1573 | Classification_binary: 0.3923 | Regression_cos_phi: 0.2157 | Regression_energy: 0.1728 | Regression_eta: 0.1564 | Regression_pt: 0.1840 | Regression_sin_phi: 0.1929 | LR: 9.50e-04 | DataLoad Time: 0.0149s | Model Forward Time: 16.0048s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0074 | max_forward: 0.7345 | max_loss: 0.1617 | max_backward: 1.4897 | max_optimizer_step: 0.1910 | max_tensorboard_logging: 12.6867 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6179.5 | uss_MB: 5051.2 | child_rss_MB: 5695.8 | allocated_MB: 345.3 | reserved_MB: 23608.0 | inactive_split_MB: 284.8 +rank7 [2026-10-28 16:35:23,037] INFO : Step 7700/50000 rank7 | Train Loss: 2.0023 | Train Loss Unweighted: 1.1475 | Train Loss Components: Auxiliary: 0.3866 | Cardinality: 0.2005 | Classification: 0.1968 | Classification_binary: 0.2784 | HitGrouping: 0.2562 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3315 | Regression_eta: 0.0040 | Regression_pt: 0.3329 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1531 | Classification_binary: 0.3574 | Regression_cos_phi: 0.2519 | Regression_energy: 0.1839 | Regression_eta: 0.1405 | Regression_pt: 0.1924 | Regression_sin_phi: 0.2100 | LR: 9.48e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.0205s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7891 | max_loss: 0.1927 | max_backward: 1.4291 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0448 | Memory: rss_MB: 6182.9 | uss_MB: 5055.0 | child_rss_MB: 5702.0 | allocated_MB: 343.9 | reserved_MB: 23608.0 | inactive_split_MB: 279.7 +rank7 [2026-10-28 16:39:23,043] INFO : Step 7800/50000 rank7 | Train Loss: 2.0473 | Train Loss Unweighted: 1.2037 | Train Loss Components: Auxiliary: 0.4013 | Cardinality: 0.1891 | Classification: 0.2102 | Classification_binary: 0.2760 | HitGrouping: 0.2604 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3526 | Regression_eta: 0.0038 | Regression_pt: 0.3571 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1139 | Classification_binary: 0.2401 | Regression_cos_phi: 0.1301 | Regression_energy: 0.1666 | Regression_eta: 0.0840 | Regression_pt: 0.1713 | Regression_sin_phi: 0.1487 | LR: 9.47e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.4333s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0073 | max_forward: 0.7326 | max_loss: 0.2464 | max_backward: 1.8124 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.9395 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6182.4 | uss_MB: 5059.4 | child_rss_MB: 5675.6 | allocated_MB: 343.5 | reserved_MB: 24612.0 | inactive_split_MB: 224.8 +rank7 [2026-10-28 16:43:21,636] INFO : Step 7900/50000 rank7 | Train Loss: 2.3872 | Train Loss Unweighted: 1.3279 | Train Loss Components: Auxiliary: 0.4525 | Cardinality: 0.2722 | Classification: 0.2401 | Classification_binary: 0.3580 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3573 | Regression_eta: 0.0051 | Regression_pt: 0.3627 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1409 | Classification_binary: 0.3444 | Regression_cos_phi: 0.1790 | Regression_energy: 0.1593 | Regression_eta: 0.1126 | Regression_pt: 0.1673 | Regression_sin_phi: 0.1623 | LR: 9.46e-04 | DataLoad Time: 0.0151s | Model Forward Time: 3.7843s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0076 | max_forward: 0.7032 | max_loss: 0.2052 | max_backward: 1.5183 | max_optimizer_step: 0.1922 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6185.2 | uss_MB: 5060.7 | child_rss_MB: 5695.7 | allocated_MB: 347.1 | reserved_MB: 24612.0 | inactive_split_MB: 279.0 +rank7 [2026-10-28 16:47:21,752] INFO : Step 8000/50000 rank7 | Train Loss: 2.1303 | Train Loss Unweighted: 1.2019 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2451 | Classification: 0.2031 | Classification_binary: 0.3087 | HitGrouping: 0.2587 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3382 | Regression_eta: 0.0041 | Regression_pt: 0.3438 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1102 | Classification_binary: 0.3134 | Regression_cos_phi: 0.1395 | Regression_energy: 0.1964 | Regression_eta: 0.1149 | Regression_pt: 0.2028 | Regression_sin_phi: 0.2301 | LR: 9.44e-04 | DataLoad Time: 0.0220s | Model Forward Time: 3.7965s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0554 | max_forward: 0.7608 | max_loss: 0.1617 | max_backward: 1.3810 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.4889 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 6186.5 | uss_MB: 4730.8 | child_rss_MB: 5692.5 | allocated_MB: 344.2 | reserved_MB: 24612.0 | inactive_split_MB: 224.1 +rank7 [2026-10-28 16:47:22,746] INFO : Running validation on rank7 +rank7 [2026-10-28 16:47:22,746] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 16:47:22,746] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 16:48:13,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:48:14,393] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 16:48:40,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 16:48:41,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 16:48:42,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 16:48:43,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 16:48:53,204] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 16:48:53,946] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:48:54,468] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 16:49:10,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 16:49:11,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 16:49:13,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 16:49:14,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 16:49:24,233] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 16:49:24,438] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:49:24,963] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 16:49:40,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 16:49:41,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 16:49:41,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 16:49:42,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_8000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 16:49:52,515] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 16:54:09,126] INFO : Step 8100/50000 rank7 | Train Loss: 2.0320 | Train Loss Unweighted: 1.1744 | Train Loss Components: Auxiliary: 0.3893 | Cardinality: 0.1986 | Classification: 0.1938 | Classification_binary: 0.2691 | HitGrouping: 0.2614 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3510 | Regression_eta: 0.0039 | Regression_pt: 0.3525 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1029 | Classification_binary: 0.2862 | Regression_cos_phi: 0.1462 | Regression_energy: 0.1559 | Regression_eta: 0.0934 | Regression_pt: 0.1626 | Regression_sin_phi: 0.1336 | LR: 9.43e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.0439s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6251 | max_loss: 0.2465 | max_backward: 1.3564 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8507 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 7244.9 | uss_MB: 6118.4 | child_rss_MB: 5660.8 | allocated_MB: 342.9 | reserved_MB: 23510.0 | inactive_split_MB: 258.1 +rank7 [2026-10-28 16:58:00,709] INFO : Step 8200/50000 rank7 | Train Loss: 2.1917 | Train Loss Unweighted: 1.2329 | Train Loss Components: Auxiliary: 0.4200 | Cardinality: 0.2405 | Classification: 0.2359 | Classification_binary: 0.3328 | HitGrouping: 0.2582 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3251 | Regression_eta: 0.0043 | Regression_pt: 0.3305 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1367 | Classification_binary: 0.2564 | Regression_cos_phi: 0.1725 | Regression_energy: 0.1518 | Regression_eta: 0.1215 | Regression_pt: 0.1596 | Regression_sin_phi: 0.1894 | LR: 9.42e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2767s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0064 | max_forward: 0.7679 | max_loss: 0.2484 | max_backward: 1.6620 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.8049 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1289 | Memory: rss_MB: 6232.6 | uss_MB: 5106.1 | child_rss_MB: 5673.2 | allocated_MB: 343.2 | reserved_MB: 23510.0 | inactive_split_MB: 292.4 +rank7 [2026-10-28 17:02:03,650] INFO : Step 8300/50000 rank7 | Train Loss: 2.1924 | Train Loss Unweighted: 1.2399 | Train Loss Components: Auxiliary: 0.4122 | Cardinality: 0.2039 | Classification: 0.2083 | Classification_binary: 0.3661 | HitGrouping: 0.2599 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3243 | Regression_eta: 0.0043 | Regression_pt: 0.3323 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1203 | Classification_binary: 0.3544 | Regression_cos_phi: 0.1844 | Regression_energy: 0.1631 | Regression_eta: 0.1210 | Regression_pt: 0.1711 | Regression_sin_phi: 0.1913 | LR: 9.40e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.8414s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0061 | max_forward: 0.9407 | max_loss: 0.2296 | max_backward: 1.9622 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 6228.1 | uss_MB: 5055.3 | child_rss_MB: 5669.2 | allocated_MB: 344.6 | reserved_MB: 24580.0 | inactive_split_MB: 267.5 +rank7 [2026-10-28 17:06:07,748] INFO : Step 8400/50000 rank7 | Train Loss: 1.9542 | Train Loss Unweighted: 1.1189 | Train Loss Components: Auxiliary: 0.3756 | Cardinality: 0.1979 | Classification: 0.1926 | Classification_binary: 0.2783 | HitGrouping: 0.2585 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3198 | Regression_eta: 0.0035 | Regression_pt: 0.3209 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1086 | Classification_binary: 0.3143 | Regression_cos_phi: 0.1287 | Regression_energy: 0.1533 | Regression_eta: 0.1042 | Regression_pt: 0.1627 | Regression_sin_phi: 0.1692 | LR: 9.39e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2374s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6939 | max_loss: 0.1474 | max_backward: 1.5357 | max_optimizer_step: 0.1936 | max_tensorboard_logging: 0.9892 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0240 | Memory: rss_MB: 6216.9 | uss_MB: 5103.3 | child_rss_MB: 5675.6 | allocated_MB: 341.1 | reserved_MB: 24580.0 | inactive_split_MB: 178.9 +rank7 [2026-10-28 17:10:17,598] INFO : Step 8500/50000 rank7 | Train Loss: 2.0642 | Train Loss Unweighted: 1.1724 | Train Loss Components: Auxiliary: 0.3964 | Cardinality: 0.2023 | Classification: 0.2152 | Classification_binary: 0.2897 | HitGrouping: 0.2607 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3289 | Regression_eta: 0.0041 | Regression_pt: 0.3302 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0945 | Classification_binary: 0.2511 | Regression_cos_phi: 0.1196 | Regression_energy: 0.1517 | Regression_eta: 0.0811 | Regression_pt: 0.1588 | Regression_sin_phi: 0.1063 | LR: 9.37e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.7370s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0069 | max_forward: 0.6424 | max_loss: 0.1972 | max_backward: 1.2711 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.7327 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0258 | Memory: rss_MB: 6218.9 | uss_MB: 5108.3 | child_rss_MB: 5667.6 | allocated_MB: 344.4 | reserved_MB: 25586.0 | inactive_split_MB: 317.0 +rank7 [2026-10-28 17:14:24,634] INFO : Step 8600/50000 rank7 | Train Loss: 2.2820 | Train Loss Unweighted: 1.2960 | Train Loss Components: Auxiliary: 0.4378 | Cardinality: 0.2546 | Classification: 0.2294 | Classification_binary: 0.3478 | HitGrouping: 0.2584 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3521 | Regression_eta: 0.0046 | Regression_pt: 0.3576 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1101 | Classification_binary: 0.2985 | Regression_cos_phi: 0.1631 | Regression_energy: 0.1814 | Regression_eta: 0.0999 | Regression_pt: 0.1916 | Regression_sin_phi: 0.1441 | LR: 9.36e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.9619s | Phase Times: max_pre_step_barrier: 0.0045 | max_batch_to_device: 0.0069 | max_forward: 0.7396 | max_loss: 0.1527 | max_backward: 1.6088 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5007 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6218.9 | uss_MB: 5112.5 | child_rss_MB: 5664.7 | allocated_MB: 344.8 | reserved_MB: 25586.0 | inactive_split_MB: 283.2 +rank7 [2026-10-28 17:18:21,031] INFO : Step 8700/50000 rank7 | Train Loss: 2.0581 | Train Loss Unweighted: 1.1705 | Train Loss Components: Auxiliary: 0.4038 | Cardinality: 0.2463 | Classification: 0.2066 | Classification_binary: 0.3017 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3267 | Regression_eta: 0.0035 | Regression_pt: 0.3282 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1394 | Classification_binary: 0.3305 | Regression_cos_phi: 0.1273 | Regression_energy: 0.1732 | Regression_eta: 0.1241 | Regression_pt: 0.1822 | Regression_sin_phi: 0.1980 | LR: 9.34e-04 | DataLoad Time: 0.0190s | Model Forward Time: 3.7163s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.6873 | max_loss: 0.2444 | max_backward: 1.2985 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.5655 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6214.6 | uss_MB: 5113.2 | child_rss_MB: 5650.1 | allocated_MB: 343.7 | reserved_MB: 25586.0 | inactive_split_MB: 260.4 +rank7 [2026-10-28 17:22:21,714] INFO : Step 8800/50000 rank7 | Train Loss: 1.8415 | Train Loss Unweighted: 1.0591 | Train Loss Components: Auxiliary: 0.3597 | Cardinality: 0.1811 | Classification: 0.1734 | Classification_binary: 0.2490 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3144 | Regression_eta: 0.0034 | Regression_pt: 0.3153 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0977 | Classification_binary: 0.2621 | Regression_cos_phi: 0.1549 | Regression_energy: 0.1634 | Regression_eta: 0.0881 | Regression_pt: 0.1708 | Regression_sin_phi: 0.1740 | LR: 9.33e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5725s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.6737 | max_loss: 0.2385 | max_backward: 1.3073 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.5819 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6215.6 | uss_MB: 5114.6 | child_rss_MB: 5649.0 | allocated_MB: 344.1 | reserved_MB: 25586.0 | inactive_split_MB: 205.9 +rank7 [2026-10-28 17:26:43,764] INFO : Step 8900/50000 rank7 | Train Loss: 2.1706 | Train Loss Unweighted: 1.1768 | Train Loss Components: Auxiliary: 0.4167 | Cardinality: 0.3000 | Classification: 0.2064 | Classification_binary: 0.3037 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3284 | Regression_eta: 0.0039 | Regression_pt: 0.3304 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0982 | Classification_binary: 0.2547 | Regression_cos_phi: 0.1117 | Regression_energy: 0.1717 | Regression_eta: 0.1120 | Regression_pt: 0.1793 | Regression_sin_phi: 0.1650 | LR: 9.31e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.8167s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0073 | max_forward: 0.9973 | max_loss: 0.4040 | max_backward: 3.4126 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6194.6 | uss_MB: 5119.9 | child_rss_MB: 5654.4 | allocated_MB: 347.1 | reserved_MB: 25586.0 | inactive_split_MB: 275.1 +rank7 [2026-10-28 17:30:46,803] INFO : Step 9000/50000 rank7 | Train Loss: 2.1357 | Train Loss Unweighted: 1.1883 | Train Loss Components: Auxiliary: 0.4064 | Cardinality: 0.2351 | Classification: 0.2003 | Classification_binary: 0.3467 | HitGrouping: 0.2532 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3143 | Regression_eta: 0.0041 | Regression_pt: 0.3186 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1271 | Classification_binary: 0.3452 | Regression_cos_phi: 0.1894 | Regression_energy: 0.1495 | Regression_eta: 0.1319 | Regression_pt: 0.1571 | Regression_sin_phi: 0.2213 | LR: 9.30e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.7222s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0070 | max_forward: 0.7873 | max_loss: 0.2178 | max_backward: 1.5407 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 1.3125 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0254 | Memory: rss_MB: 6192.6 | uss_MB: 5121.2 | child_rss_MB: 5660.4 | allocated_MB: 344.6 | reserved_MB: 25586.0 | inactive_split_MB: 289.9 +rank7 [2026-10-28 17:30:47,950] INFO : Running validation on rank7 +rank7 [2026-10-28 17:30:47,950] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 17:30:47,950] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 17:31:40,963] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:31:41,476] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 17:32:08,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 17:32:09,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 17:32:10,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 17:32:11,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 17:32:21,132] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 17:32:22,657] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:32:23,201] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 17:32:39,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 17:32:40,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 17:32:41,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 17:32:42,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 17:32:52,257] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 17:32:52,551] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:32:53,106] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 17:33:08,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 17:33:09,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 17:33:09,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 17:33:10,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_9000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 17:33:20,701] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 17:37:41,206] INFO : Step 9100/50000 rank7 | Train Loss: 1.8448 | Train Loss Unweighted: 1.0695 | Train Loss Components: Auxiliary: 0.3538 | Cardinality: 0.1722 | Classification: 0.1878 | Classification_binary: 0.2661 | HitGrouping: 0.2538 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3012 | Regression_eta: 0.0033 | Regression_pt: 0.3075 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1235 | Classification_binary: 0.2947 | Regression_cos_phi: 0.2111 | Regression_energy: 0.1435 | Regression_eta: 0.1143 | Regression_pt: 0.1521 | Regression_sin_phi: 0.1448 | LR: 9.28e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4325s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0153 | max_forward: 1.2776 | max_loss: 0.3881 | max_backward: 1.3802 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9203 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7188.5 | uss_MB: 6128.3 | child_rss_MB: 5636.7 | allocated_MB: 345.3 | reserved_MB: 22582.0 | inactive_split_MB: 275.2 +rank7 [2026-10-28 17:41:50,229] INFO : Step 9200/50000 rank7 | Train Loss: 1.9883 | Train Loss Unweighted: 1.1154 | Train Loss Components: Auxiliary: 0.3829 | Cardinality: 0.2086 | Classification: 0.1812 | Classification_binary: 0.2802 | HitGrouping: 0.2537 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3207 | Regression_eta: 0.0035 | Regression_pt: 0.3257 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1113 | Classification_binary: 0.2635 | Regression_cos_phi: 0.1837 | Regression_energy: 0.1557 | Regression_eta: 0.1058 | Regression_pt: 0.1639 | Regression_sin_phi: 0.1651 | LR: 9.27e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5155s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0073 | max_forward: 0.6839 | max_loss: 0.2414 | max_backward: 1.6336 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 1.0636 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6214.5 | uss_MB: 5161.4 | child_rss_MB: 5636.3 | allocated_MB: 345.6 | reserved_MB: 23224.0 | inactive_split_MB: 190.1 +rank7 [2026-10-28 17:45:52,428] INFO : Step 9300/50000 rank7 | Train Loss: 2.0749 | Train Loss Unweighted: 1.1591 | Train Loss Components: Auxiliary: 0.4049 | Cardinality: 0.2573 | Classification: 0.2065 | Classification_binary: 0.3180 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3111 | Regression_eta: 0.0036 | Regression_pt: 0.3160 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1179 | Classification_binary: 0.2946 | Regression_cos_phi: 0.1241 | Regression_energy: 0.1626 | Regression_eta: 0.1140 | Regression_pt: 0.1695 | Regression_sin_phi: 0.1471 | LR: 9.25e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2166s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0066 | max_forward: 0.8329 | max_loss: 0.1499 | max_backward: 1.4059 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.8766 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6214.5 | uss_MB: 5164.3 | child_rss_MB: 5654.0 | allocated_MB: 344.8 | reserved_MB: 24586.0 | inactive_split_MB: 287.2 +rank7 [2026-10-28 17:50:16,942] INFO : Step 9400/50000 rank7 | Train Loss: 1.8467 | Train Loss Unweighted: 1.0417 | Train Loss Components: Auxiliary: 0.3501 | Cardinality: 0.1520 | Classification: 0.1837 | Classification_binary: 0.2977 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2742 | Regression_eta: 0.0040 | Regression_pt: 0.2784 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1330 | Classification_binary: 0.3333 | Regression_cos_phi: 0.1619 | Regression_energy: 0.1328 | Regression_eta: 0.1395 | Regression_pt: 0.1389 | Regression_sin_phi: 0.1595 | LR: 9.24e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.5919s | Phase Times: max_pre_step_barrier: 0.0180 | max_batch_to_device: 0.0062 | max_forward: 1.5076 | max_loss: 0.3748 | max_backward: 2.6247 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.5247 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0328 | Memory: rss_MB: 6206.4 | uss_MB: 4649.5 | child_rss_MB: 5650.0 | allocated_MB: 343.9 | reserved_MB: 24586.0 | inactive_split_MB: 267.3 +rank7 [2026-10-28 17:54:18,173] INFO : Step 9500/50000 rank7 | Train Loss: 1.8199 | Train Loss Unweighted: 0.9995 | Train Loss Components: Auxiliary: 0.3464 | Cardinality: 0.1875 | Classification: 0.1651 | Classification_binary: 0.2558 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2851 | Regression_eta: 0.0035 | Regression_pt: 0.2864 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0918 | Classification_binary: 0.2565 | Regression_cos_phi: 0.1225 | Regression_energy: 0.1440 | Regression_eta: 0.0736 | Regression_pt: 0.1480 | Regression_sin_phi: 0.1130 | LR: 9.22e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.8171s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0063 | max_forward: 0.6532 | max_loss: 0.2385 | max_backward: 1.0043 | max_optimizer_step: 0.3001 | max_tensorboard_logging: 0.6010 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0464 | Memory: rss_MB: 6205.2 | uss_MB: 5165.5 | child_rss_MB: 5654.5 | allocated_MB: 341.2 | reserved_MB: 24586.0 | inactive_split_MB: 180.8 +rank7 [2026-10-28 17:58:23,634] INFO : Step 9600/50000 rank7 | Train Loss: 2.0900 | Train Loss Unweighted: 1.1491 | Train Loss Components: Auxiliary: 0.4030 | Cardinality: 0.2464 | Classification: 0.2089 | Classification_binary: 0.3189 | HitGrouping: 0.2500 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3034 | Regression_eta: 0.0042 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1133 | Classification_binary: 0.2781 | Regression_cos_phi: 0.1287 | Regression_energy: 0.1463 | Regression_eta: 0.0972 | Regression_pt: 0.1543 | Regression_sin_phi: 0.1107 | LR: 9.21e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8839s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1083 | max_forward: 0.7381 | max_loss: 0.1457 | max_backward: 1.4890 | max_optimizer_step: 0.1959 | max_tensorboard_logging: 0.4970 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6206.2 | uss_MB: 5167.2 | child_rss_MB: 5637.2 | allocated_MB: 345.2 | reserved_MB: 25594.0 | inactive_split_MB: 190.6 +rank7 [2026-10-28 18:02:35,091] INFO : Step 9700/50000 rank7 | Train Loss: 2.1397 | Train Loss Unweighted: 1.1700 | Train Loss Components: Auxiliary: 0.4082 | Cardinality: 0.2377 | Classification: 0.1942 | Classification_binary: 0.3460 | HitGrouping: 0.2574 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3095 | Regression_eta: 0.0043 | Regression_pt: 0.3118 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1008 | Classification_binary: 0.3209 | Regression_cos_phi: 0.1274 | Regression_energy: 0.1513 | Regression_eta: 0.1195 | Regression_pt: 0.1574 | Regression_sin_phi: 0.1648 | LR: 9.19e-04 | DataLoad Time: 0.0138s | Model Forward Time: 4.1050s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7744 | max_loss: 0.1497 | max_backward: 1.6846 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4710 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 6209.3 | uss_MB: 5170.1 | child_rss_MB: 5650.1 | allocated_MB: 345.9 | reserved_MB: 25594.0 | inactive_split_MB: 288.1 +rank7 [2026-10-28 18:06:42,228] INFO : Step 9800/50000 rank7 | Train Loss: 1.8589 | Train Loss Unweighted: 1.0245 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2213 | Classification: 0.1590 | Classification_binary: 0.2751 | HitGrouping: 0.2488 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2908 | Regression_eta: 0.0031 | Regression_pt: 0.2931 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0998 | Classification_binary: 0.3143 | Regression_cos_phi: 0.1510 | Regression_energy: 0.1465 | Regression_eta: 0.1041 | Regression_pt: 0.1536 | Regression_sin_phi: 0.1258 | LR: 9.17e-04 | DataLoad Time: 0.0240s | Model Forward Time: 4.1206s | Phase Times: max_pre_step_barrier: 0.1099 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.1713 | max_backward: 1.7200 | max_optimizer_step: 0.1898 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 6188.3 | uss_MB: 5169.7 | child_rss_MB: 5642.5 | allocated_MB: 342.5 | reserved_MB: 26610.0 | inactive_split_MB: 264.5 +rank7 [2026-10-28 18:10:58,313] INFO : Step 9900/50000 rank7 | Train Loss: 1.8366 | Train Loss Unweighted: 1.0077 | Train Loss Components: Auxiliary: 0.3557 | Cardinality: 0.2091 | Classification: 0.1810 | Classification_binary: 0.2609 | HitGrouping: 0.2462 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2775 | Regression_eta: 0.0035 | Regression_pt: 0.2814 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0916 | Classification_binary: 0.2259 | Regression_cos_phi: 0.1226 | Regression_energy: 0.1440 | Regression_eta: 0.1656 | Regression_pt: 0.1544 | Regression_sin_phi: 0.1474 | LR: 9.16e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5784s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6346 | max_loss: 0.1692 | max_backward: 1.3542 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.5776 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6188.4 | uss_MB: 5170.9 | child_rss_MB: 5645.5 | allocated_MB: 343.7 | reserved_MB: 26610.0 | inactive_split_MB: 205.7 +rank7 [2026-10-28 18:15:06,636] INFO : Step 10000/50000 rank7 | Train Loss: 2.0706 | Train Loss Unweighted: 1.1295 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2648 | Classification: 0.1819 | Classification_binary: 0.3225 | HitGrouping: 0.2472 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3077 | Regression_eta: 0.0037 | Regression_pt: 0.3098 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1087 | Classification_binary: 0.3035 | Regression_cos_phi: 0.1527 | Regression_energy: 0.1378 | Regression_eta: 0.1079 | Regression_pt: 0.1438 | Regression_sin_phi: 0.1856 | LR: 9.14e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.7022s | Phase Times: max_pre_step_barrier: 0.0136 | max_batch_to_device: 0.0708 | max_forward: 1.8842 | max_loss: 0.5484 | max_backward: 2.6620 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.5314 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6190.4 | uss_MB: 5174.2 | child_rss_MB: 5642.5 | allocated_MB: 345.2 | reserved_MB: 26610.0 | inactive_split_MB: 288.8 +rank7 [2026-10-28 18:15:07,627] INFO : Running validation on rank7 +rank7 [2026-10-28 18:15:07,627] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 18:15:07,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 18:15:55,923] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:15:56,538] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 18:16:22,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 18:16:23,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 18:16:24,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 18:16:25,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 18:16:35,320] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 18:16:36,561] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:16:37,047] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 18:16:52,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 18:16:53,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 18:16:55,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 18:16:56,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 18:17:06,872] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-28 18:17:09,602] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:17:10,078] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 18:17:26,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 18:17:26,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 18:17:27,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 18:17:28,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_10000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 18:17:38,257] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 18:22:01,597] INFO : Step 10100/50000 rank7 | Train Loss: 1.8115 | Train Loss Unweighted: 1.0221 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.1999 | Classification: 0.1589 | Classification_binary: 0.2815 | HitGrouping: 0.2465 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2868 | Regression_eta: 0.0032 | Regression_pt: 0.2886 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0831 | Classification_binary: 0.2702 | Regression_cos_phi: 0.1373 | Regression_energy: 0.1172 | Regression_eta: 0.1105 | Regression_pt: 0.1222 | Regression_sin_phi: 0.1488 | LR: 9.12e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9759s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7090 | max_loss: 0.1507 | max_backward: 1.2179 | max_optimizer_step: 0.3146 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 7166.8 | uss_MB: 6150.2 | child_rss_MB: 5645.8 | allocated_MB: 343.1 | reserved_MB: 21794.0 | inactive_split_MB: 243.8 +rank7 [2026-10-28 18:26:06,876] INFO : Step 10200/50000 rank7 | Train Loss: 1.6749 | Train Loss Unweighted: 0.9660 | Train Loss Components: Auxiliary: 0.3310 | Cardinality: 0.1773 | Classification: 0.1544 | Classification_binary: 0.2377 | HitGrouping: 0.2453 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2827 | Regression_eta: 0.0027 | Regression_pt: 0.2857 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0949 | Classification_binary: 0.2557 | Regression_cos_phi: 0.1360 | Regression_energy: 0.1346 | Regression_eta: 0.0684 | Regression_pt: 0.1407 | Regression_sin_phi: 0.1109 | LR: 9.11e-04 | DataLoad Time: 0.0128s | Model Forward Time: 3.9975s | Phase Times: max_pre_step_barrier: 0.0352 | max_batch_to_device: 0.0241 | max_forward: 0.6625 | max_loss: 0.1442 | max_backward: 1.3835 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7009 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1256 | Memory: rss_MB: 7164.8 | uss_MB: 6150.1 | child_rss_MB: 5645.0 | allocated_MB: 345.9 | reserved_MB: 21794.0 | inactive_split_MB: 292.4 +rank7 [2026-10-28 18:30:14,208] INFO : Step 10300/50000 rank7 | Train Loss: 1.9113 | Train Loss Unweighted: 1.0360 | Train Loss Components: Auxiliary: 0.3660 | Cardinality: 0.2108 | Classification: 0.1830 | Classification_binary: 0.2931 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0033 | Regression_pt: 0.2786 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0986 | Classification_binary: 0.2492 | Regression_cos_phi: 0.2342 | Regression_energy: 0.1226 | Regression_eta: 0.0807 | Regression_pt: 0.1274 | Regression_sin_phi: 0.1511 | LR: 9.09e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.9102s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7205 | max_loss: 0.1496 | max_backward: 1.6083 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0728 | Memory: rss_MB: 6287.2 | uss_MB: 5273.5 | child_rss_MB: 5650.9 | allocated_MB: 343.8 | reserved_MB: 22764.0 | inactive_split_MB: 257.9 +rank7 [2026-10-28 18:34:19,878] INFO : Step 10400/50000 rank7 | Train Loss: 1.9303 | Train Loss Unweighted: 1.0569 | Train Loss Components: Auxiliary: 0.3739 | Cardinality: 0.2318 | Classification: 0.1704 | Classification_binary: 0.3342 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2718 | Regression_eta: 0.0033 | Regression_pt: 0.2738 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1107 | Classification_binary: 0.3434 | Regression_cos_phi: 0.1271 | Regression_energy: 0.1251 | Regression_eta: 0.0848 | Regression_pt: 0.1306 | Regression_sin_phi: 0.1244 | LR: 9.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.7359s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7465 | max_loss: 0.1499 | max_backward: 1.4212 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.4712 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6287.2 | uss_MB: 5272.0 | child_rss_MB: 5660.4 | allocated_MB: 345.6 | reserved_MB: 22792.0 | inactive_split_MB: 259.8 +rank7 [2026-10-28 18:38:24,241] INFO : Step 10500/50000 rank7 | Train Loss: 1.6601 | Train Loss Unweighted: 0.9268 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1892 | Classification: 0.1410 | Classification_binary: 0.2342 | HitGrouping: 0.2455 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2693 | Regression_eta: 0.0029 | Regression_pt: 0.2766 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.2611 | Regression_cos_phi: 0.1041 | Regression_energy: 0.1178 | Regression_eta: 0.0871 | Regression_pt: 0.1243 | Regression_sin_phi: 0.1099 | LR: 9.06e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.5590s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0066 | max_forward: 0.6364 | max_loss: 0.2260 | max_backward: 1.1629 | max_optimizer_step: 0.1962 | max_tensorboard_logging: 0.6452 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6284.6 | uss_MB: 5269.3 | child_rss_MB: 5635.7 | allocated_MB: 340.4 | reserved_MB: 22792.0 | inactive_split_MB: 230.3 +rank7 [2026-10-28 18:42:22,546] INFO : Step 10600/50000 rank7 | Train Loss: 1.9388 | Train Loss Unweighted: 1.0719 | Train Loss Components: Auxiliary: 0.3737 | Cardinality: 0.2008 | Classification: 0.1863 | Classification_binary: 0.2709 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3047 | Regression_eta: 0.0042 | Regression_pt: 0.3019 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1179 | Classification_binary: 0.2549 | Regression_cos_phi: 0.1801 | Regression_energy: 0.1230 | Regression_eta: 0.1040 | Regression_pt: 0.1265 | Regression_sin_phi: 0.1530 | LR: 9.04e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9672s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6880 | max_loss: 0.2458 | max_backward: 1.4354 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7481 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0295 | Memory: rss_MB: 6285.6 | uss_MB: 5273.7 | child_rss_MB: 5653.3 | allocated_MB: 344.6 | reserved_MB: 22798.0 | inactive_split_MB: 286.4 +rank7 [2026-10-28 18:46:28,871] INFO : Step 10700/50000 rank7 | Train Loss: 1.8955 | Train Loss Unweighted: 1.0230 | Train Loss Components: Auxiliary: 0.3694 | Cardinality: 0.2139 | Classification: 0.1752 | Classification_binary: 0.3186 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2598 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0881 | Classification_binary: 0.2561 | Regression_cos_phi: 0.1223 | Regression_energy: 0.1293 | Regression_eta: 0.0773 | Regression_pt: 0.1381 | Regression_sin_phi: 0.1299 | LR: 9.02e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.1516s | Phase Times: max_pre_step_barrier: 0.0159 | max_batch_to_device: 0.0072 | max_forward: 1.0363 | max_loss: 0.1757 | max_backward: 2.3255 | max_optimizer_step: 0.1834 | max_tensorboard_logging: 0.7232 | max_tensorboard_flush: 0.0051 | max_post_step_barrier: 0.1576 | Memory: rss_MB: 6287.7 | uss_MB: 5274.9 | child_rss_MB: 5649.7 | allocated_MB: 342.1 | reserved_MB: 22798.0 | inactive_split_MB: 282.9 +rank7 [2026-10-28 18:50:28,323] INFO : Step 10800/50000 rank7 | Train Loss: 1.7679 | Train Loss Unweighted: 0.9563 | Train Loss Components: Auxiliary: 0.3446 | Cardinality: 0.2074 | Classification: 0.1518 | Classification_binary: 0.2766 | HitGrouping: 0.2395 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2590 | Regression_eta: 0.0033 | Regression_pt: 0.2625 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.3110 | Regression_cos_phi: 0.1055 | Regression_energy: 0.1239 | Regression_eta: 0.0721 | Regression_pt: 0.1308 | Regression_sin_phi: 0.1266 | LR: 9.00e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.7794s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0995 | max_forward: 0.7884 | max_loss: 0.2225 | max_backward: 2.0144 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.5230 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6288.7 | uss_MB: 5271.1 | child_rss_MB: 5654.7 | allocated_MB: 343.6 | reserved_MB: 23784.0 | inactive_split_MB: 286.4 +rank7 [2026-10-28 18:54:31,738] INFO : Step 10900/50000 rank7 | Train Loss: 1.6025 | Train Loss Unweighted: 0.8934 | Train Loss Components: Auxiliary: 0.3208 | Cardinality: 0.1729 | Classification: 0.1446 | Classification_binary: 0.2284 | HitGrouping: 0.2408 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2574 | Regression_eta: 0.0027 | Regression_pt: 0.2576 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2167 | Regression_cos_phi: 0.1269 | Regression_energy: 0.1569 | Regression_eta: 0.0606 | Regression_pt: 0.1623 | Regression_sin_phi: 0.0825 | LR: 8.99e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.4482s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.5765 | max_loss: 0.1490 | max_backward: 1.1882 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.5204 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6288.0 | uss_MB: 5275.5 | child_rss_MB: 5639.7 | allocated_MB: 342.1 | reserved_MB: 23796.0 | inactive_split_MB: 272.7 +rank7 [2026-10-28 18:58:45,418] INFO : Step 11000/50000 rank7 | Train Loss: 1.9315 | Train Loss Unweighted: 1.0466 | Train Loss Components: Auxiliary: 0.3721 | Cardinality: 0.2078 | Classification: 0.1806 | Classification_binary: 0.3087 | HitGrouping: 0.2467 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2740 | Regression_eta: 0.0037 | Regression_pt: 0.2758 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1151 | Classification_binary: 0.2953 | Regression_cos_phi: 0.1581 | Regression_energy: 0.1200 | Regression_eta: 0.0980 | Regression_pt: 0.1248 | Regression_sin_phi: 0.1456 | LR: 8.97e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.1702s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7424 | max_loss: 0.1704 | max_backward: 1.5866 | max_optimizer_step: 0.2906 | max_tensorboard_logging: 0.6287 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0317 | Memory: rss_MB: 6277.0 | uss_MB: 5281.8 | child_rss_MB: 5663.8 | allocated_MB: 347.8 | reserved_MB: 23796.0 | inactive_split_MB: 262.8 +rank7 [2026-10-28 18:58:46,360] INFO : Running validation on rank7 +rank7 [2026-10-28 18:58:46,360] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 18:58:46,361] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 18:59:40,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:59:40,904] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 19:00:07,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 19:00:08,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 19:00:09,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 19:00:10,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 19:00:20,721] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 19:00:21,678] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:00:22,257] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 19:00:38,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 19:00:39,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 19:00:40,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 19:00:41,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 19:00:51,097] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 19:00:52,166] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:00:52,896] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 19:01:08,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 19:01:09,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 19:01:09,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 19:01:10,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_11000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 19:01:20,652] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 19:05:41,393] INFO : Step 11100/50000 rank7 | Train Loss: 1.9118 | Train Loss Unweighted: 1.0064 | Train Loss Components: Auxiliary: 0.3723 | Cardinality: 0.2793 | Classification: 0.1563 | Classification_binary: 0.3082 | HitGrouping: 0.2374 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2660 | Regression_eta: 0.0034 | Regression_pt: 0.2693 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.2653 | Regression_cos_phi: 0.1149 | Regression_energy: 0.1104 | Regression_eta: 0.0870 | Regression_pt: 0.1154 | Regression_sin_phi: 0.1113 | LR: 8.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0417s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.9217 | max_loss: 0.2304 | max_backward: 1.5570 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.5064 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 7169.9 | uss_MB: 6179.3 | child_rss_MB: 5664.2 | allocated_MB: 343.2 | reserved_MB: 22146.0 | inactive_split_MB: 281.7 +rank7 [2026-10-28 19:09:39,138] INFO : Step 11200/50000 rank7 | Train Loss: 1.7366 | Train Loss Unweighted: 0.9397 | Train Loss Components: Auxiliary: 0.3529 | Cardinality: 0.2466 | Classification: 0.1384 | Classification_binary: 0.2548 | HitGrouping: 0.2428 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2689 | Regression_eta: 0.0027 | Regression_pt: 0.2722 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0985 | Classification_binary: 0.2949 | Regression_cos_phi: 0.1018 | Regression_energy: 0.1363 | Regression_eta: 0.0873 | Regression_pt: 0.1418 | Regression_sin_phi: 0.1316 | LR: 8.93e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.7181s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7793 | max_loss: 0.1825 | max_backward: 1.3743 | max_optimizer_step: 0.2128 | max_tensorboard_logging: 0.4676 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 7167.9 | uss_MB: 6179.2 | child_rss_MB: 5641.7 | allocated_MB: 344.6 | reserved_MB: 22972.0 | inactive_split_MB: 273.4 +rank7 [2026-10-28 19:13:41,809] INFO : Step 11300/50000 rank7 | Train Loss: 1.6876 | Train Loss Unweighted: 0.9423 | Train Loss Components: Auxiliary: 0.3334 | Cardinality: 0.1862 | Classification: 0.1581 | Classification_binary: 0.2713 | HitGrouping: 0.2437 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2524 | Regression_eta: 0.0027 | Regression_pt: 0.2550 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2534 | Regression_cos_phi: 0.1032 | Regression_energy: 0.1215 | Regression_eta: 0.0558 | Regression_pt: 0.1245 | Regression_sin_phi: 0.1111 | LR: 8.91e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6811s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0223 | max_forward: 0.9315 | max_loss: 0.2308 | max_backward: 1.9859 | max_optimizer_step: 0.1928 | max_tensorboard_logging: 0.5681 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6388.7 | uss_MB: 5273.7 | child_rss_MB: 5658.4 | allocated_MB: 345.5 | reserved_MB: 24368.0 | inactive_split_MB: 269.6 +rank7 [2026-10-28 19:17:45,333] INFO : Step 11400/50000 rank7 | Train Loss: 1.8563 | Train Loss Unweighted: 0.9980 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.2376 | Classification: 0.1627 | Classification_binary: 0.3164 | HitGrouping: 0.2394 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2551 | Regression_eta: 0.0029 | Regression_pt: 0.2577 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0820 | Classification_binary: 0.2904 | Regression_cos_phi: 0.1245 | Regression_energy: 0.1195 | Regression_eta: 0.0877 | Regression_pt: 0.1248 | Regression_sin_phi: 0.1207 | LR: 8.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0872s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0070 | max_forward: 0.7684 | max_loss: 0.1511 | max_backward: 1.6966 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5034 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 6391.7 | uss_MB: 5402.6 | child_rss_MB: 5664.5 | allocated_MB: 343.2 | reserved_MB: 25642.0 | inactive_split_MB: 251.6 +rank7 [2026-10-28 19:21:55,045] INFO : Step 11500/50000 rank7 | Train Loss: 1.6323 | Train Loss Unweighted: 0.8976 | Train Loss Components: Auxiliary: 0.3217 | Cardinality: 0.1755 | Classification: 0.1375 | Classification_binary: 0.2717 | HitGrouping: 0.2346 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2408 | Regression_eta: 0.0028 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0926 | Classification_binary: 0.3054 | Regression_cos_phi: 0.1621 | Regression_energy: 0.1225 | Regression_eta: 0.0940 | Regression_pt: 0.1287 | Regression_sin_phi: 0.1251 | LR: 8.88e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.6962s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7232 | max_loss: 0.1557 | max_backward: 1.4770 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6390.5 | uss_MB: 5401.3 | child_rss_MB: 5644.3 | allocated_MB: 345.3 | reserved_MB: 25642.0 | inactive_split_MB: 274.7 +rank7 [2026-10-28 19:25:54,595] INFO : Step 11600/50000 rank7 | Train Loss: 1.6133 | Train Loss Unweighted: 0.9062 | Train Loss Components: Auxiliary: 0.3244 | Cardinality: 0.1718 | Classification: 0.1502 | Classification_binary: 0.2356 | HitGrouping: 0.2377 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2569 | Regression_eta: 0.0025 | Regression_pt: 0.2581 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0894 | Classification_binary: 0.2209 | Regression_cos_phi: 0.1260 | Regression_energy: 0.1400 | Regression_eta: 0.0773 | Regression_pt: 0.1448 | Regression_sin_phi: 0.1252 | LR: 8.86e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.6501s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.6646 | max_loss: 0.1710 | max_backward: 1.3208 | max_optimizer_step: 0.1963 | max_tensorboard_logging: 0.5714 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6391.5 | uss_MB: 5404.2 | child_rss_MB: 5665.3 | allocated_MB: 344.1 | reserved_MB: 26630.0 | inactive_split_MB: 209.9 +rank7 [2026-10-28 19:29:50,732] INFO : Step 11700/50000 rank7 | Train Loss: 1.7739 | Train Loss Unweighted: 0.9706 | Train Loss Components: Auxiliary: 0.3531 | Cardinality: 0.2418 | Classification: 0.1518 | Classification_binary: 0.2821 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2638 | Regression_eta: 0.0028 | Regression_pt: 0.2671 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0867 | Classification_binary: 0.2483 | Regression_cos_phi: 0.1275 | Regression_energy: 0.1384 | Regression_eta: 0.0550 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0936 | LR: 8.84e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.4599s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.8789 | max_loss: 0.2057 | max_backward: 2.3846 | max_optimizer_step: 0.1875 | max_tensorboard_logging: 1.0403 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0291 | Memory: rss_MB: 6389.5 | uss_MB: 4921.9 | child_rss_MB: 5640.9 | allocated_MB: 343.8 | reserved_MB: 27624.0 | inactive_split_MB: 186.6 +rank7 [2026-10-28 19:33:54,005] INFO : Step 11800/50000 rank7 | Train Loss: 1.8223 | Train Loss Unweighted: 0.9745 | Train Loss Components: Auxiliary: 0.3609 | Cardinality: 0.2329 | Classification: 0.1589 | Classification_binary: 0.3100 | HitGrouping: 0.2361 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2473 | Regression_eta: 0.0028 | Regression_pt: 0.2523 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1092 | Classification_binary: 0.3731 | Regression_cos_phi: 0.1657 | Regression_energy: 0.1179 | Regression_eta: 0.0905 | Regression_pt: 0.1237 | Regression_sin_phi: 0.2101 | LR: 8.82e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.3847s | Phase Times: max_pre_step_barrier: 0.0103 | max_batch_to_device: 0.0070 | max_forward: 1.2302 | max_loss: 0.2767 | max_backward: 2.3745 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6390.5 | uss_MB: 5402.7 | child_rss_MB: 5661.5 | allocated_MB: 346.0 | reserved_MB: 27624.0 | inactive_split_MB: 280.5 +rank7 [2026-10-28 19:37:48,967] INFO : Step 11900/50000 rank7 | Train Loss: 1.5797 | Train Loss Unweighted: 0.8638 | Train Loss Components: Auxiliary: 0.3225 | Cardinality: 0.1935 | Classification: 0.1358 | Classification_binary: 0.2351 | HitGrouping: 0.2353 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2427 | Regression_eta: 0.0025 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0834 | Classification_binary: 0.2651 | Regression_cos_phi: 0.1092 | Regression_energy: 0.1137 | Regression_eta: 0.0637 | Regression_pt: 0.1173 | Regression_sin_phi: 0.1099 | LR: 8.80e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.1380s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0787 | max_forward: 0.7011 | max_loss: 0.1484 | max_backward: 1.2002 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.9463 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0325 | Memory: rss_MB: 6390.3 | uss_MB: 5402.5 | child_rss_MB: 5642.2 | allocated_MB: 340.1 | reserved_MB: 28620.0 | inactive_split_MB: 264.1 +rank7 [2026-10-28 19:41:43,168] INFO : Step 12000/50000 rank7 | Train Loss: 1.6036 | Train Loss Unweighted: 0.8706 | Train Loss Components: Auxiliary: 0.3181 | Cardinality: 0.1972 | Classification: 0.1481 | Classification_binary: 0.2459 | HitGrouping: 0.2286 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2330 | Regression_eta: 0.0025 | Regression_pt: 0.2385 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0844 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0977 | Regression_eta: 0.0475 | Regression_pt: 0.1043 | Regression_sin_phi: 0.0778 | LR: 8.78e-04 | DataLoad Time: 0.0145s | Model Forward Time: 3.6991s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0072 | max_forward: 0.6194 | max_loss: 0.1546 | max_backward: 1.4628 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5143 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6383.3 | uss_MB: 5409.4 | child_rss_MB: 5646.2 | allocated_MB: 342.9 | reserved_MB: 28620.0 | inactive_split_MB: 215.8 +rank7 [2026-10-28 19:41:44,125] INFO : Running validation on rank7 +rank7 [2026-10-28 19:41:44,125] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 19:41:44,125] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 19:42:35,084] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:42:35,720] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 19:43:02,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 19:43:04,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 19:43:05,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 19:43:06,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 19:43:16,406] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 19:43:17,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:43:17,653] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 19:43:34,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 19:43:35,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 19:43:35,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 19:43:36,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 19:43:46,876] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 19:43:48,085] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:43:48,651] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 19:44:03,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 19:44:04,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 19:44:05,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 19:44:05,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_12000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 19:44:15,840] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 19:48:34,083] INFO : Step 12100/50000 rank7 | Train Loss: 1.8149 | Train Loss Unweighted: 0.9793 | Train Loss Components: Auxiliary: 0.3632 | Cardinality: 0.2324 | Classification: 0.1506 | Classification_binary: 0.3149 | HitGrouping: 0.2337 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2518 | Regression_eta: 0.0029 | Regression_pt: 0.2562 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1070 | Classification_binary: 0.2976 | Regression_cos_phi: 0.0983 | Regression_energy: 0.1208 | Regression_eta: 0.0679 | Regression_pt: 0.1242 | Regression_sin_phi: 0.1579 | LR: 8.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5271s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8305 | max_loss: 0.1678 | max_backward: 1.6498 | max_optimizer_step: 0.2425 | max_tensorboard_logging: 0.8610 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 7131.3 | uss_MB: 6150.6 | child_rss_MB: 5646.8 | allocated_MB: 345.8 | reserved_MB: 23512.0 | inactive_split_MB: 299.6 +rank7 [2026-10-28 19:52:54,661] INFO : Step 12200/50000 rank7 | Train Loss: 1.6619 | Train Loss Unweighted: 0.9043 | Train Loss Components: Auxiliary: 0.3290 | Cardinality: 0.1906 | Classification: 0.1445 | Classification_binary: 0.2674 | HitGrouping: 0.2303 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2432 | Regression_eta: 0.0030 | Regression_pt: 0.2435 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.1053 | Classification_binary: 0.2928 | Regression_cos_phi: 0.1479 | Regression_energy: 0.1077 | Regression_eta: 0.0898 | Regression_pt: 0.1124 | Regression_sin_phi: 0.1442 | LR: 8.74e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8243s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0997 | max_forward: 0.7862 | max_loss: 0.1438 | max_backward: 1.5991 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.8843 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0545 | Memory: rss_MB: 7133.3 | uss_MB: 5953.1 | child_rss_MB: 5653.1 | allocated_MB: 341.2 | reserved_MB: 23512.0 | inactive_split_MB: 206.8 +rank7 [2026-10-28 19:56:47,338] INFO : Step 12300/50000 rank7 | Train Loss: 1.4699 | Train Loss Unweighted: 0.8181 | Train Loss Components: Auxiliary: 0.2935 | Cardinality: 0.1474 | Classification: 0.1404 | Classification_binary: 0.2280 | HitGrouping: 0.2250 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2222 | Regression_eta: 0.0028 | Regression_pt: 0.2223 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.2122 | Regression_cos_phi: 0.1199 | Regression_energy: 0.0964 | Regression_eta: 0.0639 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0978 | LR: 8.72e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7241s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.6177 | max_loss: 0.2232 | max_backward: 1.1179 | max_optimizer_step: 0.2702 | max_tensorboard_logging: 0.7974 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0320 | Memory: rss_MB: 6497.9 | uss_MB: 5527.4 | child_rss_MB: 5654.1 | allocated_MB: 343.2 | reserved_MB: 24286.0 | inactive_split_MB: 290.2 +rank7 [2026-10-28 20:00:36,190] INFO : Step 12400/50000 rank7 | Train Loss: 1.6135 | Train Loss Unweighted: 0.8782 | Train Loss Components: Auxiliary: 0.3265 | Cardinality: 0.2056 | Classification: 0.1546 | Classification_binary: 0.2540 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2308 | Regression_eta: 0.0025 | Regression_pt: 0.2337 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0950 | Classification_binary: 0.2629 | Regression_cos_phi: 0.1113 | Regression_energy: 0.1106 | Regression_eta: 0.0659 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0900 | LR: 8.70e-04 | DataLoad Time: 0.0216s | Model Forward Time: 3.9923s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 0.6928 | max_loss: 0.1486 | max_backward: 1.3860 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.7721 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6497.9 | uss_MB: 5528.1 | child_rss_MB: 5671.7 | allocated_MB: 344.8 | reserved_MB: 24510.0 | inactive_split_MB: 295.5 +rank7 [2026-10-28 20:04:30,060] INFO : Step 12500/50000 rank7 | Train Loss: 1.6527 | Train Loss Unweighted: 0.8797 | Train Loss Components: Auxiliary: 0.3282 | Cardinality: 0.1890 | Classification: 0.1391 | Classification_binary: 0.2779 | HitGrouping: 0.2315 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2273 | Regression_eta: 0.0030 | Regression_pt: 0.2295 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.2796 | Regression_cos_phi: 0.1290 | Regression_energy: 0.1133 | Regression_eta: 0.1006 | Regression_pt: 0.1189 | Regression_sin_phi: 0.1024 | LR: 8.68e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.1676s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0535 | max_forward: 0.7747 | max_loss: 0.1534 | max_backward: 1.3339 | max_optimizer_step: 0.1991 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6494.1 | uss_MB: 5528.8 | child_rss_MB: 5649.7 | allocated_MB: 347.2 | reserved_MB: 24510.0 | inactive_split_MB: 456.8 +rank7 [2026-10-28 20:08:30,520] INFO : Step 12600/50000 rank7 | Train Loss: 1.4969 | Train Loss Unweighted: 0.8185 | Train Loss Components: Auxiliary: 0.2999 | Cardinality: 0.1595 | Classification: 0.1210 | Classification_binary: 0.2357 | HitGrouping: 0.2384 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2279 | Regression_eta: 0.0024 | Regression_pt: 0.2291 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.2449 | Regression_cos_phi: 0.1319 | Regression_energy: 0.1057 | Regression_eta: 0.0888 | Regression_pt: 0.1092 | Regression_sin_phi: 0.1315 | LR: 8.66e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.7603s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0062 | max_forward: 0.6571 | max_loss: 0.1429 | max_backward: 1.4364 | max_optimizer_step: 0.2024 | max_tensorboard_logging: 0.4629 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1078 | Memory: rss_MB: 6482.1 | uss_MB: 5531.3 | child_rss_MB: 5638.2 | allocated_MB: 342.9 | reserved_MB: 25526.0 | inactive_split_MB: 234.6 +rank7 [2026-10-28 20:12:34,584] INFO : Step 12700/50000 rank7 | Train Loss: 1.6234 | Train Loss Unweighted: 0.8661 | Train Loss Components: Auxiliary: 0.3300 | Cardinality: 0.2004 | Classification: 0.1459 | Classification_binary: 0.2387 | HitGrouping: 0.2403 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2360 | Regression_eta: 0.0027 | Regression_pt: 0.2402 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2379 | Regression_cos_phi: 0.0878 | Regression_energy: 0.1169 | Regression_eta: 0.0608 | Regression_pt: 0.1224 | Regression_sin_phi: 0.1069 | LR: 8.64e-04 | DataLoad Time: 0.0203s | Model Forward Time: 3.4795s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6221 | max_loss: 0.1846 | max_backward: 1.1554 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.5899 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6469.4 | uss_MB: 5531.3 | child_rss_MB: 5648.7 | allocated_MB: 345.9 | reserved_MB: 26286.0 | inactive_split_MB: 241.7 +rank7 [2026-10-28 20:16:22,943] INFO : Step 12800/50000 rank7 | Train Loss: 1.7877 | Train Loss Unweighted: 0.9398 | Train Loss Components: Auxiliary: 0.3651 | Cardinality: 0.2327 | Classification: 0.1495 | Classification_binary: 0.3094 | HitGrouping: 0.2289 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2357 | Regression_eta: 0.0027 | Regression_pt: 0.2394 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0844 | Classification_binary: 0.2803 | Regression_cos_phi: 0.1012 | Regression_energy: 0.1183 | Regression_eta: 0.0923 | Regression_pt: 0.1266 | Regression_sin_phi: 0.1339 | LR: 8.62e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.2570s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0068 | max_forward: 0.8468 | max_loss: 0.1534 | max_backward: 1.7420 | max_optimizer_step: 0.3013 | max_tensorboard_logging: 0.5725 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6458.4 | uss_MB: 5535.9 | child_rss_MB: 5651.5 | allocated_MB: 345.6 | reserved_MB: 26286.0 | inactive_split_MB: 284.4 +rank7 [2026-10-28 20:20:13,787] INFO : Step 12900/50000 rank7 | Train Loss: 1.5881 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.3191 | Cardinality: 0.2029 | Classification: 0.1304 | Classification_binary: 0.2564 | HitGrouping: 0.2356 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2291 | Regression_eta: 0.0023 | Regression_pt: 0.2312 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.3070 | Regression_cos_phi: 0.1267 | Regression_energy: 0.1007 | Regression_eta: 0.0649 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0904 | LR: 8.60e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9621s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7180 | max_loss: 0.2380 | max_backward: 1.4488 | max_optimizer_step: 0.1996 | max_tensorboard_logging: 0.6932 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6444.6 | uss_MB: 5531.5 | child_rss_MB: 5646.1 | allocated_MB: 341.8 | reserved_MB: 26286.0 | inactive_split_MB: 252.2 +rank7 [2026-10-28 20:24:06,549] INFO : Step 13000/50000 rank7 | Train Loss: 1.5217 | Train Loss Unweighted: 0.8547 | Train Loss Components: Auxiliary: 0.3141 | Cardinality: 0.1693 | Classification: 0.1382 | Classification_binary: 0.2135 | HitGrouping: 0.2292 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2470 | Regression_eta: 0.0024 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0795 | Classification_binary: 0.2363 | Regression_cos_phi: 0.0920 | Regression_energy: 0.1130 | Regression_eta: 0.0739 | Regression_pt: 0.1163 | Regression_sin_phi: 0.1047 | LR: 8.58e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8287s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7101 | max_loss: 0.1763 | max_backward: 1.1845 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 0.8282 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6429.6 | uss_MB: 5536.9 | child_rss_MB: 5653.9 | allocated_MB: 339.8 | reserved_MB: 26286.0 | inactive_split_MB: 213.4 +rank7 [2026-10-28 20:24:07,450] INFO : Running validation on rank7 +rank7 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 20:24:07,450] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 20:24:58,457] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:24:59,078] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 20:25:24,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 20:25:25,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 20:25:26,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 20:25:27,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 20:25:37,336] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 20:25:38,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:25:38,655] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 20:25:54,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 20:25:55,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 20:25:56,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 20:25:57,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 20:26:07,174] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 20:26:08,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:26:09,008] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 20:26:24,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 20:26:25,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 20:26:25,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 20:26:26,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_13000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 20:26:36,682] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 20:30:47,340] INFO : Step 13100/50000 rank7 | Train Loss: 1.6168 | Train Loss Unweighted: 0.8649 | Train Loss Components: Auxiliary: 0.3213 | Cardinality: 0.1869 | Classification: 0.1467 | Classification_binary: 0.2696 | HitGrouping: 0.2253 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2210 | Regression_eta: 0.0027 | Regression_pt: 0.2221 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0930 | Classification_binary: 0.2314 | Regression_cos_phi: 0.1337 | Regression_energy: 0.1065 | Regression_eta: 0.0651 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0972 | LR: 8.56e-04 | DataLoad Time: 0.0133s | Model Forward Time: 6.3734s | Phase Times: max_pre_step_barrier: 0.0337 | max_batch_to_device: 0.0136 | max_forward: 1.2842 | max_loss: 0.2583 | max_backward: 3.1281 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.9677 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7302.9 | uss_MB: 6406.9 | child_rss_MB: 5646.9 | allocated_MB: 341.8 | reserved_MB: 22940.0 | inactive_split_MB: 270.2 +rank7 [2026-10-28 20:34:56,293] INFO : Step 13200/50000 rank7 | Train Loss: 1.6996 | Train Loss Unweighted: 0.8855 | Train Loss Components: Auxiliary: 0.3386 | Cardinality: 0.2474 | Classification: 0.1410 | Classification_binary: 0.2769 | HitGrouping: 0.2245 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2299 | Regression_eta: 0.0027 | Regression_pt: 0.2323 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0849 | Classification_binary: 0.2805 | Regression_cos_phi: 0.1101 | Regression_energy: 0.1073 | Regression_eta: 0.0751 | Regression_pt: 0.1102 | Regression_sin_phi: 0.1224 | LR: 8.54e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.1537s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0072 | max_forward: 0.7543 | max_loss: 0.1532 | max_backward: 1.3317 | max_optimizer_step: 0.3180 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7300.0 | uss_MB: 6407.2 | child_rss_MB: 5649.5 | allocated_MB: 343.4 | reserved_MB: 23942.0 | inactive_split_MB: 249.6 +rank7 [2026-10-28 20:38:53,364] INFO : Step 13300/50000 rank7 | Train Loss: 1.4876 | Train Loss Unweighted: 0.7776 | Train Loss Components: Auxiliary: 0.2970 | Cardinality: 0.2018 | Classification: 0.1100 | Classification_binary: 0.2192 | HitGrouping: 0.2251 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2205 | Regression_eta: 0.0026 | Regression_pt: 0.2230 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2490 | Regression_cos_phi: 0.1334 | Regression_energy: 0.1033 | Regression_eta: 0.0644 | Regression_pt: 0.1075 | Regression_sin_phi: 0.0780 | LR: 8.52e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.6398s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0259 | max_forward: 0.9208 | max_loss: 0.2245 | max_backward: 1.9523 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0027 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6473.0 | uss_MB: 4937.1 | child_rss_MB: 5535.0 | allocated_MB: 341.7 | reserved_MB: 24474.0 | inactive_split_MB: 265.4 +rank7 [2026-10-28 20:42:52,944] INFO : Step 13400/50000 rank7 | Train Loss: 1.4253 | Train Loss Unweighted: 0.7944 | Train Loss Components: Auxiliary: 0.2889 | Cardinality: 0.1585 | Classification: 0.1406 | Classification_binary: 0.2158 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2158 | Regression_eta: 0.0022 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0869 | Classification_binary: 0.1754 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0944 | Regression_eta: 0.0376 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0499 | LR: 8.50e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7384s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6533 | max_loss: 0.2500 | max_backward: 1.4458 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.5454 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0337 | Memory: rss_MB: 6474.0 | uss_MB: 5610.2 | child_rss_MB: 5538.3 | allocated_MB: 341.6 | reserved_MB: 25334.0 | inactive_split_MB: 266.6 +rank7 [2026-10-28 20:47:05,652] INFO : Step 13500/50000 rank7 | Train Loss: 1.6021 | Train Loss Unweighted: 0.8749 | Train Loss Components: Auxiliary: 0.3241 | Cardinality: 0.1832 | Classification: 0.1473 | Classification_binary: 0.2801 | HitGrouping: 0.2262 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2200 | Regression_eta: 0.0024 | Regression_pt: 0.2225 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0898 | Classification_binary: 0.2785 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0989 | Regression_eta: 0.0567 | Regression_pt: 0.1027 | Regression_sin_phi: 0.1120 | LR: 8.48e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.1054s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.7859 | max_loss: 0.3303 | max_backward: 2.5111 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.4974 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6412.7 | uss_MB: 5395.8 | child_rss_MB: 5343.5 | allocated_MB: 344.3 | reserved_MB: 25574.0 | inactive_split_MB: 284.1 +rank7 [2026-10-28 20:51:04,210] INFO : Step 13600/50000 rank7 | Train Loss: 1.5296 | Train Loss Unweighted: 0.8114 | Train Loss Components: Auxiliary: 0.3050 | Cardinality: 0.1872 | Classification: 0.1291 | Classification_binary: 0.2450 | HitGrouping: 0.2218 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2158 | Regression_eta: 0.0025 | Regression_pt: 0.2165 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.2494 | Regression_cos_phi: 0.1113 | Regression_energy: 0.1040 | Regression_eta: 0.0780 | Regression_pt: 0.1087 | Regression_sin_phi: 0.1031 | LR: 8.45e-04 | DataLoad Time: 0.0206s | Model Forward Time: 3.6651s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0060 | max_forward: 0.7508 | max_loss: 0.1507 | max_backward: 1.2891 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 6404.7 | uss_MB: 5608.6 | child_rss_MB: 5283.3 | allocated_MB: 345.0 | reserved_MB: 26608.0 | inactive_split_MB: 257.3 +rank7 [2026-10-28 20:55:12,337] INFO : Step 13700/50000 rank7 | Train Loss: 1.4515 | Train Loss Unweighted: 0.7895 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1575 | Classification: 0.1261 | Classification_binary: 0.2121 | HitGrouping: 0.2249 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2219 | Regression_eta: 0.0023 | Regression_pt: 0.2245 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2323 | Regression_cos_phi: 0.1001 | Regression_energy: 0.1050 | Regression_eta: 0.0680 | Regression_pt: 0.1097 | Regression_sin_phi: 0.1093 | LR: 8.43e-04 | DataLoad Time: 0.0196s | Model Forward Time: 3.8539s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.6804 | max_loss: 0.2128 | max_backward: 1.1331 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6398.1 | uss_MB: 5609.6 | child_rss_MB: 5245.6 | allocated_MB: 342.5 | reserved_MB: 26608.0 | inactive_split_MB: 241.7 +rank7 [2026-10-28 20:59:19,100] INFO : Step 13800/50000 rank7 | Train Loss: 1.5060 | Train Loss Unweighted: 0.8320 | Train Loss Components: Auxiliary: 0.3084 | Cardinality: 0.1762 | Classification: 0.1368 | Classification_binary: 0.2567 | HitGrouping: 0.2226 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2156 | Regression_eta: 0.0023 | Regression_pt: 0.2184 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0829 | Classification_binary: 0.2406 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0948 | Regression_eta: 0.0659 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0882 | LR: 8.41e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4110s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7967 | max_loss: 0.1491 | max_backward: 1.5976 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.9349 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6392.1 | uss_MB: 5613.3 | child_rss_MB: 5216.1 | allocated_MB: 342.1 | reserved_MB: 26608.0 | inactive_split_MB: 241.3 +rank7 [2026-10-28 21:03:21,781] INFO : Step 13900/50000 rank7 | Train Loss: 1.6108 | Train Loss Unweighted: 0.8469 | Train Loss Components: Auxiliary: 0.3204 | Cardinality: 0.1942 | Classification: 0.1303 | Classification_binary: 0.2751 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2168 | Regression_eta: 0.0028 | Regression_pt: 0.2192 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.2830 | Regression_cos_phi: 0.0801 | Regression_energy: 0.1118 | Regression_eta: 0.0598 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0921 | LR: 8.39e-04 | DataLoad Time: 0.0193s | Model Forward Time: 6.7330s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8010 | max_loss: 0.1552 | max_backward: 2.5526 | max_optimizer_step: 0.2181 | max_tensorboard_logging: 1.2862 | max_tensorboard_flush: 0.0041 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 6389.1 | uss_MB: 5603.5 | child_rss_MB: 5217.8 | allocated_MB: 345.6 | reserved_MB: 26608.0 | inactive_split_MB: 292.8 +rank7 [2026-10-28 21:07:29,378] INFO : Step 14000/50000 rank7 | Train Loss: 1.3659 | Train Loss Unweighted: 0.7342 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1707 | Classification: 0.0974 | Classification_binary: 0.1997 | HitGrouping: 0.2221 | Regression_cos_phi: 0.0010 | Regression_energy: 0.2149 | Regression_eta: 0.0023 | Regression_pt: 0.2181 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.2215 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0914 | Regression_eta: 0.0652 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0842 | LR: 8.37e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.4511s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7574 | max_loss: 0.1908 | max_backward: 1.5870 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.9331 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0214 | Memory: rss_MB: 6393.2 | uss_MB: 4632.1 | child_rss_MB: 5217.4 | allocated_MB: 342.3 | reserved_MB: 26608.0 | inactive_split_MB: 238.4 +rank7 [2026-10-28 21:07:30,352] INFO : Running validation on rank7 +rank7 [2026-10-28 21:07:30,352] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 21:07:30,352] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 21:08:21,198] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:08:21,763] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 21:08:49,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 21:08:50,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 21:08:51,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 21:08:52,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 21:09:02,162] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 21:09:05,935] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:09:06,443] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 21:09:22,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 21:09:23,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 21:09:24,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 21:09:25,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 21:09:35,353] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 21:09:36,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:09:36,854] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 21:09:52,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 21:09:53,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 21:09:53,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 21:09:54,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_14000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 21:10:04,415] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 21:14:16,750] INFO : Step 14100/50000 rank7 | Train Loss: 1.5416 | Train Loss Unweighted: 0.8176 | Train Loss Components: Auxiliary: 0.3012 | Cardinality: 0.1751 | Classification: 0.1363 | Classification_binary: 0.2297 | HitGrouping: 0.2237 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2177 | Regression_eta: 0.0028 | Regression_pt: 0.2283 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0661 | Regression_energy: 0.1015 | Regression_eta: 0.0316 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0451 | LR: 8.35e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.2096s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7718 | max_loss: 0.1710 | max_backward: 1.6376 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5860 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0549 | Memory: rss_MB: 7412.7 | uss_MB: 6170.5 | child_rss_MB: 5197.6 | allocated_MB: 347.1 | reserved_MB: 23970.0 | inactive_split_MB: 244.9 +rank7 [2026-10-28 21:18:17,876] INFO : Step 14200/50000 rank7 | Train Loss: 1.6164 | Train Loss Unweighted: 0.8542 | Train Loss Components: Auxiliary: 0.3190 | Cardinality: 0.1950 | Classification: 0.1478 | Classification_binary: 0.2794 | HitGrouping: 0.2256 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2094 | Regression_eta: 0.0027 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.2646 | Regression_cos_phi: 0.1084 | Regression_energy: 0.1066 | Regression_eta: 0.0713 | Regression_pt: 0.1133 | Regression_sin_phi: 0.1178 | LR: 8.32e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.3715s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1153 | max_forward: 0.8788 | max_loss: 0.1615 | max_backward: 1.5286 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0533 | Memory: rss_MB: 7413.4 | uss_MB: 6634.1 | child_rss_MB: 5195.0 | allocated_MB: 344.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.9 +rank7 [2026-10-28 21:22:21,673] INFO : Step 14300/50000 rank7 | Train Loss: 1.4246 | Train Loss Unweighted: 0.7744 | Train Loss Components: Auxiliary: 0.2862 | Cardinality: 0.1685 | Classification: 0.1198 | Classification_binary: 0.2357 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2062 | Regression_eta: 0.0021 | Regression_pt: 0.2084 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.2546 | Regression_cos_phi: 0.0939 | Regression_energy: 0.0890 | Regression_eta: 0.0516 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0868 | LR: 8.30e-04 | DataLoad Time: 0.0197s | Model Forward Time: 3.8799s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7183 | max_loss: 0.1494 | max_backward: 1.3371 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 6454.4 | uss_MB: 5677.1 | child_rss_MB: 5184.2 | allocated_MB: 341.9 | reserved_MB: 24962.0 | inactive_split_MB: 233.1 +rank7 [2026-10-28 21:26:25,335] INFO : Step 14400/50000 rank7 | Train Loss: 1.3539 | Train Loss Unweighted: 0.7267 | Train Loss Components: Auxiliary: 0.2785 | Cardinality: 0.1402 | Classification: 0.1303 | Classification_binary: 0.2015 | HitGrouping: 0.2217 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1957 | Regression_eta: 0.0022 | Regression_pt: 0.1949 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1845 | Regression_cos_phi: 0.1188 | Regression_energy: 0.0791 | Regression_eta: 0.0408 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0629 | LR: 8.28e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.6670s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.6350 | max_loss: 0.2071 | max_backward: 1.2597 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6458.7 | uss_MB: 5681.9 | child_rss_MB: 5188.8 | allocated_MB: 346.1 | reserved_MB: 24962.0 | inactive_split_MB: 231.6 +rank7 [2026-10-28 21:30:23,756] INFO : Step 14500/50000 rank7 | Train Loss: 1.5651 | Train Loss Unweighted: 0.7967 | Train Loss Components: Auxiliary: 0.3070 | Cardinality: 0.2312 | Classification: 0.1364 | Classification_binary: 0.2565 | HitGrouping: 0.2225 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1984 | Regression_eta: 0.0026 | Regression_pt: 0.2005 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0941 | Classification_binary: 0.2339 | Regression_cos_phi: 0.1092 | Regression_energy: 0.0945 | Regression_eta: 0.0595 | Regression_pt: 0.1000 | Regression_sin_phi: 0.1073 | LR: 8.26e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.4658s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0069 | max_forward: 0.8745 | max_loss: 0.1717 | max_backward: 1.8485 | max_optimizer_step: 0.2012 | max_tensorboard_logging: 1.4922 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6443.5 | uss_MB: 5682.8 | child_rss_MB: 5113.1 | allocated_MB: 344.2 | reserved_MB: 25374.0 | inactive_split_MB: 206.9 +rank7 [2026-10-28 21:34:34,947] INFO : Step 14600/50000 rank7 | Train Loss: 1.6485 | Train Loss Unweighted: 0.8413 | Train Loss Components: Auxiliary: 0.3426 | Cardinality: 0.2375 | Classification: 0.1249 | Classification_binary: 0.2902 | HitGrouping: 0.2244 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2104 | Regression_eta: 0.0023 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2659 | Regression_cos_phi: 0.0985 | Regression_energy: 0.0857 | Regression_eta: 0.0517 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0983 | LR: 8.24e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.3666s | Phase Times: max_pre_step_barrier: 0.0053 | max_batch_to_device: 0.0068 | max_forward: 0.7866 | max_loss: 0.1515 | max_backward: 1.5469 | max_optimizer_step: 0.3128 | max_tensorboard_logging: 0.7882 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6427.1 | uss_MB: 5683.8 | child_rss_MB: 5049.1 | allocated_MB: 345.7 | reserved_MB: 25374.0 | inactive_split_MB: 292.3 +rank7 [2026-10-28 21:38:39,704] INFO : Step 14700/50000 rank7 | Train Loss: 1.3645 | Train Loss Unweighted: 0.7326 | Train Loss Components: Auxiliary: 0.2783 | Cardinality: 0.1507 | Classification: 0.1080 | Classification_binary: 0.2034 | HitGrouping: 0.2228 | Regression_cos_phi: 0.0009 | Regression_energy: 0.2076 | Regression_eta: 0.0024 | Regression_pt: 0.2091 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.2055 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0963 | Regression_eta: 0.0642 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0700 | LR: 8.21e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9105s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0059 | max_forward: 0.6571 | max_loss: 0.1494 | max_backward: 1.2978 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7751 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0922 | Memory: rss_MB: 6423.6 | uss_MB: 5684.9 | child_rss_MB: 5040.9 | allocated_MB: 341.1 | reserved_MB: 25374.0 | inactive_split_MB: 264.9 +rank7 [2026-10-28 21:42:53,741] INFO : Step 14800/50000 rank7 | Train Loss: 1.3987 | Train Loss Unweighted: 0.7335 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1827 | Classification: 0.1136 | Classification_binary: 0.2244 | HitGrouping: 0.2204 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1949 | Regression_eta: 0.0022 | Regression_pt: 0.1963 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.2044 | Regression_cos_phi: 0.1045 | Regression_energy: 0.0904 | Regression_eta: 0.0743 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0526 | LR: 8.19e-04 | DataLoad Time: 0.0229s | Model Forward Time: 5.2136s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 1.1679 | max_loss: 0.2171 | max_backward: 2.8051 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 6420.4 | uss_MB: 4434.9 | child_rss_MB: 5026.4 | allocated_MB: 345.2 | reserved_MB: 25374.0 | inactive_split_MB: 222.5 +rank7 [2026-10-28 21:47:06,021] INFO : Step 14900/50000 rank7 | Train Loss: 1.5046 | Train Loss Unweighted: 0.7991 | Train Loss Components: Auxiliary: 0.3055 | Cardinality: 0.1834 | Classification: 0.1311 | Classification_binary: 0.2643 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1982 | Regression_eta: 0.0023 | Regression_pt: 0.2008 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2565 | Regression_cos_phi: 0.0906 | Regression_energy: 0.0898 | Regression_eta: 0.0606 | Regression_pt: 0.0938 | Regression_sin_phi: 0.1129 | LR: 8.17e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.0332s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0071 | max_forward: 0.7642 | max_loss: 0.1561 | max_backward: 1.5975 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6421.6 | uss_MB: 5685.9 | child_rss_MB: 5030.2 | allocated_MB: 345.4 | reserved_MB: 25374.0 | inactive_split_MB: 289.7 +rank7 [2026-10-28 21:51:16,624] INFO : Step 15000/50000 rank7 | Train Loss: 1.3836 | Train Loss Unweighted: 0.7341 | Train Loss Components: Auxiliary: 0.2827 | Cardinality: 0.1614 | Classification: 0.1182 | Classification_binary: 0.2252 | HitGrouping: 0.2202 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1930 | Regression_eta: 0.0023 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.2314 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0904 | Regression_eta: 0.0622 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0697 | LR: 8.15e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.7139s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0066 | max_forward: 0.6828 | max_loss: 0.1453 | max_backward: 1.3064 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.5031 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1064 | Memory: rss_MB: 6423.8 | uss_MB: 5686.3 | child_rss_MB: 5021.1 | allocated_MB: 342.6 | reserved_MB: 25374.0 | inactive_split_MB: 262.7 +rank7 [2026-10-28 21:51:17,569] INFO : Running validation on rank7 +rank7 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 21:51:17,569] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 21:52:06,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:52:07,026] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 21:52:33,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 21:52:34,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 21:52:34,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 21:52:35,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 21:52:45,803] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 21:52:46,745] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:52:47,271] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 21:53:03,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 21:53:04,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 21:53:05,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 21:53:06,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 21:53:16,038] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 21:53:16,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:53:17,340] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 21:53:32,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 21:53:33,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 21:53:34,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 21:53:34,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_15000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 21:53:44,784] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 21:57:58,525] INFO : Step 15100/50000 rank7 | Train Loss: 1.3706 | Train Loss Unweighted: 0.7159 | Train Loss Components: Auxiliary: 0.2774 | Cardinality: 0.1535 | Classification: 0.1271 | Classification_binary: 0.1954 | HitGrouping: 0.2229 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1940 | Regression_eta: 0.0022 | Regression_pt: 0.1948 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0777 | Regression_energy: 0.0923 | Regression_eta: 0.0413 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0812 | LR: 8.12e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.0244s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.6658 | max_loss: 0.2401 | max_backward: 1.3105 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8277 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7654.5 | uss_MB: 6910.5 | child_rss_MB: 5047.2 | allocated_MB: 341.1 | reserved_MB: 22096.0 | inactive_split_MB: 215.5 +rank7 [2026-10-28 22:01:59,131] INFO : Step 15200/50000 rank7 | Train Loss: 1.5024 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3072 | Cardinality: 0.2080 | Classification: 0.1273 | Classification_binary: 0.2385 | HitGrouping: 0.2164 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1989 | Regression_eta: 0.0022 | Regression_pt: 0.2007 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.1915 | Regression_cos_phi: 0.1148 | Regression_energy: 0.0922 | Regression_eta: 0.0559 | Regression_pt: 0.0964 | Regression_sin_phi: 0.1182 | LR: 8.10e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.5440s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7502 | max_loss: 0.1534 | max_backward: 1.8002 | max_optimizer_step: 0.2270 | max_tensorboard_logging: 0.8682 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 7649.1 | uss_MB: 6910.3 | child_rss_MB: 5045.3 | allocated_MB: 345.6 | reserved_MB: 22420.0 | inactive_split_MB: 257.7 +rank7 [2026-10-28 22:05:42,234] INFO : Step 15300/50000 rank7 | Train Loss: 1.4920 | Train Loss Unweighted: 0.7880 | Train Loss Components: Auxiliary: 0.3065 | Cardinality: 0.1970 | Classification: 0.1161 | Classification_binary: 0.2589 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.2026 | Regression_eta: 0.0022 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.2832 | Regression_cos_phi: 0.0927 | Regression_energy: 0.0894 | Regression_eta: 0.0743 | Regression_pt: 0.0952 | Regression_sin_phi: 0.1244 | LR: 8.08e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5806s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0192 | max_forward: 0.7642 | max_loss: 0.1530 | max_backward: 1.6292 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 0.9125 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6524.0 | uss_MB: 5797.3 | child_rss_MB: 5044.3 | allocated_MB: 341.2 | reserved_MB: 23096.0 | inactive_split_MB: 271.7 +rank7 [2026-10-28 22:09:28,150] INFO : Step 15400/50000 rank7 | Train Loss: 1.3549 | Train Loss Unweighted: 0.7210 | Train Loss Components: Auxiliary: 0.2726 | Cardinality: 0.1572 | Classification: 0.1148 | Classification_binary: 0.2052 | HitGrouping: 0.2179 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1983 | Regression_eta: 0.0022 | Regression_pt: 0.1984 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2492 | Regression_cos_phi: 0.1422 | Regression_energy: 0.1170 | Regression_eta: 0.0636 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0829 | LR: 8.05e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.7043s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6594 | max_loss: 0.2185 | max_backward: 1.2416 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6511.0 | uss_MB: 5790.6 | child_rss_MB: 5047.2 | allocated_MB: 340.8 | reserved_MB: 23394.0 | inactive_split_MB: 166.6 +rank7 [2026-10-28 22:13:12,491] INFO : Step 15500/50000 rank7 | Train Loss: 1.4313 | Train Loss Unweighted: 0.7527 | Train Loss Components: Auxiliary: 0.2907 | Cardinality: 0.1666 | Classification: 0.1321 | Classification_binary: 0.2182 | HitGrouping: 0.2188 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1979 | Regression_eta: 0.0025 | Regression_pt: 0.1996 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0915 | Classification_binary: 0.2105 | Regression_cos_phi: 0.0906 | Regression_energy: 0.1007 | Regression_eta: 0.0739 | Regression_pt: 0.1039 | Regression_sin_phi: 0.0790 | LR: 8.03e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2550s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0792 | max_forward: 0.7874 | max_loss: 0.1514 | max_backward: 1.8676 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.5380 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6511.0 | uss_MB: 5805.3 | child_rss_MB: 5052.2 | allocated_MB: 345.6 | reserved_MB: 24092.0 | inactive_split_MB: 260.5 +rank7 [2026-10-28 22:17:01,248] INFO : Step 15600/50000 rank7 | Train Loss: 1.5305 | Train Loss Unweighted: 0.8109 | Train Loss Components: Auxiliary: 0.3137 | Cardinality: 0.1790 | Classification: 0.1304 | Classification_binary: 0.2721 | HitGrouping: 0.2193 | Regression_cos_phi: 0.0012 | Regression_energy: 0.2011 | Regression_eta: 0.0024 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0825 | Classification_binary: 0.2463 | Regression_cos_phi: 0.1176 | Regression_energy: 0.0931 | Regression_eta: 0.0740 | Regression_pt: 0.0962 | Regression_sin_phi: 0.1198 | LR: 8.01e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2098s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0068 | max_forward: 0.7645 | max_loss: 0.2230 | max_backward: 1.7015 | max_optimizer_step: 0.2096 | max_tensorboard_logging: 0.5157 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1398 | Memory: rss_MB: 6508.8 | uss_MB: 5803.4 | child_rss_MB: 5052.3 | allocated_MB: 344.1 | reserved_MB: 24402.0 | inactive_split_MB: 287.9 +rank7 [2026-10-28 22:20:46,546] INFO : Step 15700/50000 rank7 | Train Loss: 1.4083 | Train Loss Unweighted: 0.7205 | Train Loss Components: Auxiliary: 0.2819 | Cardinality: 0.1775 | Classification: 0.1238 | Classification_binary: 0.2322 | HitGrouping: 0.2149 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1785 | Regression_eta: 0.0024 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0869 | Classification_binary: 0.2484 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0836 | Regression_eta: 0.0746 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0859 | LR: 7.98e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.0300s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.8324 | max_loss: 0.2392 | max_backward: 1.6606 | max_optimizer_step: 0.2067 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0126 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6506.8 | uss_MB: 5802.2 | child_rss_MB: 5055.5 | allocated_MB: 344.4 | reserved_MB: 25380.0 | inactive_split_MB: 228.2 +rank7 [2026-10-28 22:24:39,044] INFO : Step 15800/50000 rank7 | Train Loss: 1.2988 | Train Loss Unweighted: 0.6968 | Train Loss Components: Auxiliary: 0.2671 | Cardinality: 0.1436 | Classification: 0.1112 | Classification_binary: 0.1916 | HitGrouping: 0.2134 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1933 | Regression_eta: 0.0020 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1854 | Regression_cos_phi: 0.0955 | Regression_energy: 0.0841 | Regression_eta: 0.0575 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0794 | LR: 7.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7268s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.6109 | max_loss: 0.2015 | max_backward: 1.1291 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6506.5 | uss_MB: 5803.6 | child_rss_MB: 5051.3 | allocated_MB: 341.6 | reserved_MB: 25380.0 | inactive_split_MB: 263.9 +rank7 [2026-10-28 22:28:35,066] INFO : Step 15900/50000 rank7 | Train Loss: 1.4020 | Train Loss Unweighted: 0.7322 | Train Loss Components: Auxiliary: 0.2796 | Cardinality: 0.1800 | Classification: 0.1295 | Classification_binary: 0.2346 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1816 | Regression_eta: 0.0023 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.2329 | Regression_cos_phi: 0.0930 | Regression_energy: 0.0930 | Regression_eta: 0.0712 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0979 | LR: 7.94e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2686s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0933 | max_forward: 0.7823 | max_loss: 0.1445 | max_backward: 1.4740 | max_optimizer_step: 0.2089 | max_tensorboard_logging: 0.8747 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6508.5 | uss_MB: 5807.0 | child_rss_MB: 5047.2 | allocated_MB: 342.9 | reserved_MB: 25380.0 | inactive_split_MB: 283.5 +rank7 [2026-10-28 22:32:40,675] INFO : Step 16000/50000 rank7 | Train Loss: 1.4978 | Train Loss Unweighted: 0.7701 | Train Loss Components: Auxiliary: 0.3126 | Cardinality: 0.2093 | Classification: 0.1190 | Classification_binary: 0.2580 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1920 | Regression_eta: 0.0021 | Regression_pt: 0.1969 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.2727 | Regression_cos_phi: 0.1266 | Regression_energy: 0.1074 | Regression_eta: 0.0673 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0838 | LR: 7.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4905s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.7474 | max_loss: 0.1892 | max_backward: 1.4114 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.7981 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0507 | Memory: rss_MB: 6509.3 | uss_MB: 5675.5 | child_rss_MB: 5048.3 | allocated_MB: 342.7 | reserved_MB: 25380.0 | inactive_split_MB: 292.9 +rank7 [2026-10-28 22:32:41,713] INFO : Running validation on rank7 +rank7 [2026-10-28 22:32:41,713] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 22:32:41,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 22:33:34,621] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:33:35,151] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 22:34:01,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 22:34:02,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 22:34:03,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 22:34:03,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 22:34:13,844] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 22:34:14,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:34:14,939] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 22:34:31,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 22:34:31,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 22:34:32,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 22:34:33,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 22:34:43,792] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 22:34:44,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:34:44,850] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 22:35:00,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 22:35:00,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 22:35:01,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 22:35:02,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_16000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 22:35:12,316] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 22:39:22,136] INFO : Step 16100/50000 rank7 | Train Loss: 1.3441 | Train Loss Unweighted: 0.7192 | Train Loss Components: Auxiliary: 0.2713 | Cardinality: 0.1631 | Classification: 0.1197 | Classification_binary: 0.2058 | HitGrouping: 0.2206 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1931 | Regression_eta: 0.0020 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2308 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0969 | Regression_eta: 0.0571 | Regression_pt: 0.1015 | Regression_sin_phi: 0.1035 | LR: 7.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1528s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0584 | max_forward: 0.7257 | max_loss: 0.1421 | max_backward: 1.2590 | max_optimizer_step: 0.2226 | max_tensorboard_logging: 1.0197 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6577.1 | uss_MB: 5871.7 | child_rss_MB: 5053.6 | allocated_MB: 341.3 | reserved_MB: 23342.0 | inactive_split_MB: 221.6 +rank7 [2026-10-28 22:43:09,260] INFO : Step 16200/50000 rank7 | Train Loss: 1.3737 | Train Loss Unweighted: 0.7229 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1695 | Classification: 0.1213 | Classification_binary: 0.2167 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1895 | Regression_eta: 0.0019 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.1908 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0824 | Regression_eta: 0.0559 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0744 | LR: 7.86e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0519s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.1062 | max_forward: 0.7366 | max_loss: 0.1469 | max_backward: 1.4999 | max_optimizer_step: 0.2071 | max_tensorboard_logging: 0.6015 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6580.7 | uss_MB: 5882.6 | child_rss_MB: 5053.8 | allocated_MB: 342.5 | reserved_MB: 24304.0 | inactive_split_MB: 205.9 +rank7 [2026-10-28 22:46:54,663] INFO : Step 16300/50000 rank7 | Train Loss: 1.4671 | Train Loss Unweighted: 0.7592 | Train Loss Components: Auxiliary: 0.3008 | Cardinality: 0.1775 | Classification: 0.1342 | Classification_binary: 0.2702 | HitGrouping: 0.2148 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1748 | Regression_eta: 0.0023 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0786 | Classification_binary: 0.2122 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0968 | Regression_eta: 0.0601 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0800 | LR: 7.84e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4607s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7718 | max_loss: 0.1628 | max_backward: 1.7017 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.8513 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6574.7 | uss_MB: 5880.8 | child_rss_MB: 5053.1 | allocated_MB: 345.8 | reserved_MB: 24304.0 | inactive_split_MB: 281.0 +rank7 [2026-10-28 22:50:39,547] INFO : Step 16400/50000 rank7 | Train Loss: 1.4200 | Train Loss Unweighted: 0.7161 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.2086 | Classification: 0.1206 | Classification_binary: 0.2158 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1867 | Regression_eta: 0.0024 | Regression_pt: 0.1885 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2315 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0888 | Regression_eta: 0.0724 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0949 | LR: 7.81e-04 | DataLoad Time: 0.0198s | Model Forward Time: 3.8035s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7570 | max_loss: 0.1546 | max_backward: 1.2568 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5199 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1436 | Memory: rss_MB: 6571.7 | uss_MB: 5879.8 | child_rss_MB: 5049.7 | allocated_MB: 342.7 | reserved_MB: 25268.0 | inactive_split_MB: 213.6 +rank7 [2026-10-28 22:54:25,075] INFO : Step 16500/50000 rank7 | Train Loss: 1.2201 | Train Loss Unweighted: 0.6592 | Train Loss Components: Auxiliary: 0.2501 | Cardinality: 0.1195 | Classification: 0.1029 | Classification_binary: 0.1835 | HitGrouping: 0.2086 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1836 | Regression_eta: 0.0021 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0605 | Regression_energy: 0.1028 | Regression_eta: 0.0389 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0510 | LR: 7.79e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3175s | Phase Times: max_pre_step_barrier: 0.0162 | max_batch_to_device: 0.0128 | max_forward: 0.7866 | max_loss: 0.2200 | max_backward: 1.9590 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.4913 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6572.2 | uss_MB: 5885.7 | child_rss_MB: 5046.8 | allocated_MB: 343.6 | reserved_MB: 26238.0 | inactive_split_MB: 248.8 +rank7 [2026-10-28 22:58:25,288] INFO : Step 16600/50000 rank7 | Train Loss: 1.4335 | Train Loss Unweighted: 0.7657 | Train Loss Components: Auxiliary: 0.2949 | Cardinality: 0.1698 | Classification: 0.1386 | Classification_binary: 0.2312 | HitGrouping: 0.2151 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1941 | Regression_eta: 0.0021 | Regression_pt: 0.1975 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0835 | Classification_binary: 0.2323 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0935 | Regression_eta: 0.0452 | Regression_pt: 0.0988 | Regression_sin_phi: 0.1082 | LR: 7.77e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.9928s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0139 | max_forward: 1.0329 | max_loss: 0.3279 | max_backward: 2.5489 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.5244 | max_tensorboard_flush: 0.0014 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6575.3 | uss_MB: 4925.2 | child_rss_MB: 5045.0 | allocated_MB: 344.2 | reserved_MB: 27228.0 | inactive_split_MB: 219.6 +rank7 [2026-10-28 23:02:38,763] INFO : Step 16700/50000 rank7 | Train Loss: 1.4062 | Train Loss Unweighted: 0.7378 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1840 | Classification: 0.1203 | Classification_binary: 0.2439 | HitGrouping: 0.2108 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1839 | Regression_eta: 0.0020 | Regression_pt: 0.1855 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2609 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0915 | Regression_eta: 0.0495 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0841 | LR: 7.74e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0462s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7336 | max_loss: 0.2460 | max_backward: 1.6747 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6577.4 | uss_MB: 5888.0 | child_rss_MB: 5049.7 | allocated_MB: 344.6 | reserved_MB: 27228.0 | inactive_split_MB: 279.4 +rank7 [2026-10-28 23:06:47,051] INFO : Step 16800/50000 rank7 | Train Loss: 1.3172 | Train Loss Unweighted: 0.6826 | Train Loss Components: Auxiliary: 0.2634 | Cardinality: 0.1533 | Classification: 0.1173 | Classification_binary: 0.1949 | HitGrouping: 0.2175 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1819 | Regression_eta: 0.0027 | Regression_pt: 0.1837 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0883 | Classification_binary: 0.2183 | Regression_cos_phi: 0.0943 | Regression_energy: 0.1003 | Regression_eta: 0.0685 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0730 | LR: 7.72e-04 | DataLoad Time: 0.0212s | Model Forward Time: 3.6840s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0962 | max_forward: 0.7772 | max_loss: 0.1423 | max_backward: 1.3121 | max_optimizer_step: 0.2117 | max_tensorboard_logging: 0.4865 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6580.1 | uss_MB: 5886.7 | child_rss_MB: 5043.1 | allocated_MB: 342.1 | reserved_MB: 27250.0 | inactive_split_MB: 200.8 +rank7 [2026-10-28 23:11:05,115] INFO : Step 16900/50000 rank7 | Train Loss: 1.3132 | Train Loss Unweighted: 0.6868 | Train Loss Components: Auxiliary: 0.2695 | Cardinality: 0.1474 | Classification: 0.1260 | Classification_binary: 0.1988 | HitGrouping: 0.2104 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1779 | Regression_eta: 0.0021 | Regression_pt: 0.1798 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1897 | Regression_cos_phi: 0.1085 | Regression_energy: 0.0791 | Regression_eta: 0.0490 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0661 | LR: 7.69e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.7599s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7203 | max_loss: 0.1660 | max_backward: 1.1458 | max_optimizer_step: 0.2090 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6581.3 | uss_MB: 5889.6 | child_rss_MB: 5044.4 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 193.9 +rank7 [2026-10-28 23:15:09,202] INFO : Step 17000/50000 rank7 | Train Loss: 1.3575 | Train Loss Unweighted: 0.7115 | Train Loss Components: Auxiliary: 0.2906 | Cardinality: 0.1781 | Classification: 0.1179 | Classification_binary: 0.2325 | HitGrouping: 0.2089 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1782 | Regression_eta: 0.0018 | Regression_pt: 0.1792 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2089 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0919 | Regression_eta: 0.0477 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0976 | LR: 7.67e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4162s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0060 | max_forward: 0.9008 | max_loss: 0.1474 | max_backward: 1.9229 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5974 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6575.6 | uss_MB: 5894.0 | child_rss_MB: 5025.6 | allocated_MB: 344.8 | reserved_MB: 28222.0 | inactive_split_MB: 278.9 +rank7 [2026-10-28 23:15:10,041] INFO : Running validation on rank7 +rank7 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 23:15:10,041] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 23:16:00,503] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:16:01,060] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 23:16:27,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 23:16:27,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 23:16:28,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 23:16:30,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 23:16:40,354] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 23:16:42,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:16:42,521] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 23:16:58,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 23:16:59,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 23:16:59,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 23:17:00,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 23:17:10,901] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 23:17:11,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:17:12,023] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 23:17:27,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 23:17:28,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 23:17:28,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 23:17:29,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_17000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 23:17:39,348] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-28 23:21:59,507] INFO : Step 17100/50000 rank7 | Train Loss: 1.3790 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2868 | Cardinality: 0.1704 | Classification: 0.1078 | Classification_binary: 0.2095 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1968 | Regression_eta: 0.0020 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.2296 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0979 | Regression_eta: 0.0594 | Regression_pt: 0.1030 | Regression_sin_phi: 0.1067 | LR: 7.64e-04 | DataLoad Time: 0.0139s | Model Forward Time: 4.9036s | Phase Times: max_pre_step_barrier: 0.0234 | max_batch_to_device: 0.0065 | max_forward: 0.8292 | max_loss: 0.1455 | max_backward: 1.4833 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 1.1211 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6666.8 | uss_MB: 5307.9 | child_rss_MB: 5009.6 | allocated_MB: 343.8 | reserved_MB: 22688.0 | inactive_split_MB: 254.6 +rank7 [2026-10-28 23:26:06,863] INFO : Step 17200/50000 rank7 | Train Loss: 1.2786 | Train Loss Unweighted: 0.6598 | Train Loss Components: Auxiliary: 0.2576 | Cardinality: 0.1378 | Classification: 0.1161 | Classification_binary: 0.1910 | HitGrouping: 0.2191 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0025 | Regression_pt: 0.1733 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1814 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0964 | Regression_eta: 0.0458 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0588 | LR: 7.62e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.8624s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6470 | max_loss: 0.2505 | max_backward: 1.4380 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6320 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6664.1 | uss_MB: 5982.7 | child_rss_MB: 5028.0 | allocated_MB: 344.1 | reserved_MB: 22688.0 | inactive_split_MB: 295.9 +rank7 [2026-10-28 23:30:05,515] INFO : Step 17300/50000 rank7 | Train Loss: 1.3571 | Train Loss Unweighted: 0.6976 | Train Loss Components: Auxiliary: 0.2789 | Cardinality: 0.1665 | Classification: 0.1193 | Classification_binary: 0.2287 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1724 | Regression_eta: 0.0021 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1902 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0745 | Regression_eta: 0.0501 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0654 | LR: 7.59e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5419s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7276 | max_loss: 0.2520 | max_backward: 1.8579 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.8525 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6666.1 | uss_MB: 5121.5 | child_rss_MB: 5026.9 | allocated_MB: 344.2 | reserved_MB: 23616.0 | inactive_split_MB: 266.5 +rank7 [2026-10-28 23:34:25,421] INFO : Step 17400/50000 rank7 | Train Loss: 1.3668 | Train Loss Unweighted: 0.7091 | Train Loss Components: Auxiliary: 0.2811 | Cardinality: 0.1760 | Classification: 0.1195 | Classification_binary: 0.2351 | HitGrouping: 0.2084 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1732 | Regression_eta: 0.0019 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.2603 | Regression_cos_phi: 0.1097 | Regression_energy: 0.0826 | Regression_eta: 0.0536 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0994 | LR: 7.57e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4842s | Phase Times: max_pre_step_barrier: 0.1038 | max_batch_to_device: 0.0306 | max_forward: 0.7782 | max_loss: 0.2429 | max_backward: 1.3743 | max_optimizer_step: 0.2094 | max_tensorboard_logging: 0.8348 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0757 | Memory: rss_MB: 6667.1 | uss_MB: 5984.6 | child_rss_MB: 5026.4 | allocated_MB: 343.3 | reserved_MB: 24604.0 | inactive_split_MB: 262.4 +rank7 [2026-10-28 23:38:51,400] INFO : Step 17500/50000 rank7 | Train Loss: 1.3114 | Train Loss Unweighted: 0.6526 | Train Loss Components: Auxiliary: 0.2581 | Cardinality: 0.1481 | Classification: 0.1145 | Classification_binary: 0.1884 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1714 | Regression_eta: 0.0024 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.1825 | Regression_cos_phi: 0.0969 | Regression_energy: 0.0886 | Regression_eta: 0.0644 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0813 | LR: 7.54e-04 | DataLoad Time: 0.0209s | Model Forward Time: 3.9152s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.6394 | max_loss: 0.1427 | max_backward: 1.2079 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.8375 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0574 | Memory: rss_MB: 6667.1 | uss_MB: 5986.2 | child_rss_MB: 5023.0 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 240.0 +rank7 [2026-10-28 23:43:03,145] INFO : Step 17600/50000 rank7 | Train Loss: 1.2883 | Train Loss Unweighted: 0.6843 | Train Loss Components: Auxiliary: 0.2688 | Cardinality: 0.1362 | Classification: 0.1273 | Classification_binary: 0.2065 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0019 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.2126 | Regression_cos_phi: 0.0816 | Regression_energy: 0.1097 | Regression_eta: 0.0455 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0823 | LR: 7.52e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6072s | Phase Times: max_pre_step_barrier: 0.0051 | max_batch_to_device: 0.1120 | max_forward: 1.2883 | max_loss: 0.2624 | max_backward: 1.5380 | max_optimizer_step: 0.2147 | max_tensorboard_logging: 0.7644 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 6665.9 | uss_MB: 5985.5 | child_rss_MB: 5028.5 | allocated_MB: 344.2 | reserved_MB: 25596.0 | inactive_split_MB: 237.9 +rank7 [2026-10-28 23:47:02,149] INFO : Step 17700/50000 rank7 | Train Loss: 1.3491 | Train Loss Unweighted: 0.6939 | Train Loss Components: Auxiliary: 0.2835 | Cardinality: 0.1622 | Classification: 0.1186 | Classification_binary: 0.2310 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1699 | Regression_eta: 0.0022 | Regression_pt: 0.1701 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0777 | Classification_binary: 0.2202 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0706 | Regression_eta: 0.0618 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0693 | LR: 7.49e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.6530s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0704 | max_forward: 0.9841 | max_loss: 0.1994 | max_backward: 1.8911 | max_optimizer_step: 0.2227 | max_tensorboard_logging: 0.5849 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6667.9 | uss_MB: 5989.0 | child_rss_MB: 5031.2 | allocated_MB: 345.5 | reserved_MB: 25596.0 | inactive_split_MB: 287.6 +rank7 [2026-10-28 23:51:17,533] INFO : Step 17800/50000 rank7 | Train Loss: 1.3063 | Train Loss Unweighted: 0.6692 | Train Loss Components: Auxiliary: 0.2664 | Cardinality: 0.1579 | Classification: 0.1042 | Classification_binary: 0.2130 | HitGrouping: 0.2150 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1741 | Regression_eta: 0.0019 | Regression_pt: 0.1740 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0734 | Classification_binary: 0.2300 | Regression_cos_phi: 0.0968 | Regression_energy: 0.0783 | Regression_eta: 0.0428 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0666 | LR: 7.47e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8686s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0065 | max_forward: 0.7351 | max_loss: 0.1686 | max_backward: 1.4616 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.5023 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 6666.0 | uss_MB: 5988.8 | child_rss_MB: 5032.9 | allocated_MB: 341.0 | reserved_MB: 25596.0 | inactive_split_MB: 197.8 +rank7 [2026-10-28 23:55:22,791] INFO : Step 17900/50000 rank7 | Train Loss: 1.2213 | Train Loss Unweighted: 0.6433 | Train Loss Components: Auxiliary: 0.2530 | Cardinality: 0.1427 | Classification: 0.1147 | Classification_binary: 0.1725 | HitGrouping: 0.2070 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1756 | Regression_eta: 0.0016 | Regression_pt: 0.1770 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1584 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0967 | Regression_eta: 0.0284 | Regression_pt: 0.1026 | Regression_sin_phi: 0.0507 | LR: 7.44e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8179s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7249 | max_loss: 0.1579 | max_backward: 1.1981 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6654.3 | uss_MB: 5990.9 | child_rss_MB: 5033.3 | allocated_MB: 341.6 | reserved_MB: 25596.0 | inactive_split_MB: 297.9 +rank7 [2026-10-28 23:59:20,282] INFO : Step 18000/50000 rank7 | Train Loss: 1.3597 | Train Loss Unweighted: 0.7083 | Train Loss Components: Auxiliary: 0.2781 | Cardinality: 0.1599 | Classification: 0.1247 | Classification_binary: 0.2275 | HitGrouping: 0.2122 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1749 | Regression_eta: 0.0021 | Regression_pt: 0.1769 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.2393 | Regression_cos_phi: 0.0881 | Regression_energy: 0.0786 | Regression_eta: 0.0617 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0967 | LR: 7.42e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6421s | Phase Times: max_pre_step_barrier: 0.0816 | max_batch_to_device: 0.0068 | max_forward: 0.8980 | max_loss: 0.1586 | max_backward: 1.9185 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.7546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6654.3 | uss_MB: 5995.0 | child_rss_MB: 5035.3 | allocated_MB: 344.6 | reserved_MB: 25596.0 | inactive_split_MB: 254.8 +rank7 [2026-10-28 23:59:21,239] INFO : Running validation on rank7 +rank7 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 23:59:21,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 00:00:12,210] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:00:12,739] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 00:00:39,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 00:00:40,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 00:00:41,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 00:00:41,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 00:00:51,897] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 00:00:52,803] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:00:53,359] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 00:01:09,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 00:01:10,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 00:01:11,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 00:01:12,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 00:01:22,160] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 00:01:22,746] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:01:23,292] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 00:01:38,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 00:01:39,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 00:01:40,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 00:01:40,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_18000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 00:01:50,743] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 00:05:57,253] INFO : Step 18100/50000 rank7 | Train Loss: 1.3911 | Train Loss Unweighted: 0.7216 | Train Loss Components: Auxiliary: 0.2853 | Cardinality: 0.1666 | Classification: 0.1199 | Classification_binary: 0.2340 | HitGrouping: 0.2078 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1807 | Regression_eta: 0.0024 | Regression_pt: 0.1824 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.2290 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0885 | Regression_eta: 0.0569 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0811 | LR: 7.39e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.5279s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0073 | max_forward: 0.9127 | max_loss: 0.4546 | max_backward: 1.7406 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.7463 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0938 | Memory: rss_MB: 7789.9 | uss_MB: 7130.3 | child_rss_MB: 5031.5 | allocated_MB: 344.6 | reserved_MB: 22256.0 | inactive_split_MB: 291.4 +rank7 [2026-10-29 00:09:47,529] INFO : Step 18200/50000 rank7 | Train Loss: 1.2438 | Train Loss Unweighted: 0.6468 | Train Loss Components: Auxiliary: 0.2545 | Cardinality: 0.1359 | Classification: 0.1208 | Classification_binary: 0.1803 | HitGrouping: 0.2154 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1700 | Regression_eta: 0.0020 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2065 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0791 | Regression_eta: 0.0567 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0871 | LR: 7.36e-04 | DataLoad Time: 0.0152s | Model Forward Time: 3.7616s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.7114 | max_loss: 0.2369 | max_backward: 1.1051 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7318 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 6784.1 | uss_MB: 6119.0 | child_rss_MB: 5031.9 | allocated_MB: 341.2 | reserved_MB: 23772.0 | inactive_split_MB: 170.7 +rank7 [2026-10-29 00:13:32,740] INFO : Step 18300/50000 rank7 | Train Loss: 1.3100 | Train Loss Unweighted: 0.6891 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1522 | Classification: 0.1230 | Classification_binary: 0.1938 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1834 | Regression_eta: 0.0019 | Regression_pt: 0.1848 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1992 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0783 | Regression_eta: 0.0619 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0834 | LR: 7.34e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1470s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7219 | max_loss: 0.1491 | max_backward: 1.5825 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.7390 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0389 | Memory: rss_MB: 6786.1 | uss_MB: 6124.1 | child_rss_MB: 5031.6 | allocated_MB: 344.4 | reserved_MB: 24778.0 | inactive_split_MB: 234.4 +rank7 [2026-10-29 00:17:16,846] INFO : Step 18400/50000 rank7 | Train Loss: 1.3667 | Train Loss Unweighted: 0.7337 | Train Loss Components: Auxiliary: 0.2871 | Cardinality: 0.1448 | Classification: 0.1277 | Classification_binary: 0.2393 | HitGrouping: 0.2095 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1805 | Regression_eta: 0.0022 | Regression_pt: 0.1818 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.2296 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0692 | Regression_eta: 0.0553 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0689 | LR: 7.31e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4859s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7606 | max_loss: 0.1744 | max_backward: 1.7308 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.8312 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6782.0 | uss_MB: 6123.4 | child_rss_MB: 5027.3 | allocated_MB: 344.6 | reserved_MB: 24778.0 | inactive_split_MB: 281.4 +rank7 [2026-10-29 00:21:03,776] INFO : Step 18500/50000 rank7 | Train Loss: 1.2528 | Train Loss Unweighted: 0.6816 | Train Loss Components: Auxiliary: 0.2631 | Cardinality: 0.1350 | Classification: 0.1140 | Classification_binary: 0.1973 | HitGrouping: 0.2066 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1821 | Regression_eta: 0.0017 | Regression_pt: 0.1847 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2528 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0865 | Regression_eta: 0.0508 | Regression_pt: 0.0919 | Regression_sin_phi: 0.1146 | LR: 7.29e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.2634s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0113 | max_forward: 0.8281 | max_loss: 0.2418 | max_backward: 1.4683 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7561 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1431 | Memory: rss_MB: 6783.0 | uss_MB: 6122.8 | child_rss_MB: 5026.7 | allocated_MB: 343.3 | reserved_MB: 24778.0 | inactive_split_MB: 226.5 +rank7 [2026-10-29 00:24:44,691] INFO : Step 18600/50000 rank7 | Train Loss: 1.1881 | Train Loss Unweighted: 0.6335 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1331 | Classification: 0.1084 | Classification_binary: 0.1723 | HitGrouping: 0.2106 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1746 | Regression_eta: 0.0015 | Regression_pt: 0.1749 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1600 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0731 | Regression_eta: 0.0573 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0705 | LR: 7.26e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8528s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7299 | max_loss: 0.2279 | max_backward: 1.0799 | max_optimizer_step: 0.2793 | max_tensorboard_logging: 0.8080 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6782.4 | uss_MB: 6120.8 | child_rss_MB: 5022.4 | allocated_MB: 341.0 | reserved_MB: 24778.0 | inactive_split_MB: 229.3 +rank7 [2026-10-29 00:28:24,928] INFO : Step 18700/50000 rank7 | Train Loss: 1.3122 | Train Loss Unweighted: 0.6534 | Train Loss Components: Auxiliary: 0.2669 | Cardinality: 0.1756 | Classification: 0.1225 | Classification_binary: 0.2098 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1571 | Regression_eta: 0.0020 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2244 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0729 | Regression_eta: 0.0587 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0850 | LR: 7.24e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.6248s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.9129 | max_loss: 0.1606 | max_backward: 1.9397 | max_optimizer_step: 0.2139 | max_tensorboard_logging: 0.7921 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 6785.3 | uss_MB: 6128.9 | child_rss_MB: 5021.8 | allocated_MB: 344.8 | reserved_MB: 24778.0 | inactive_split_MB: 239.8 +rank7 [2026-10-29 00:32:05,525] INFO : Step 18800/50000 rank7 | Train Loss: 1.2696 | Train Loss Unweighted: 0.6643 | Train Loss Components: Auxiliary: 0.2674 | Cardinality: 0.1602 | Classification: 0.1070 | Classification_binary: 0.2200 | HitGrouping: 0.2058 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1652 | Regression_eta: 0.0018 | Regression_pt: 0.1686 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0732 | Regression_eta: 0.0370 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0488 | LR: 7.21e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6469s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0073 | max_forward: 0.7444 | max_loss: 0.2527 | max_backward: 1.7167 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.9324 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 6785.1 | uss_MB: 6124.1 | child_rss_MB: 5021.8 | allocated_MB: 345.4 | reserved_MB: 25044.0 | inactive_split_MB: 284.4 +rank7 [2026-10-29 00:35:49,544] INFO : Step 18900/50000 rank7 | Train Loss: 1.1718 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1382 | Classification: 0.0996 | Classification_binary: 0.1741 | HitGrouping: 0.2036 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1652 | Regression_eta: 0.0017 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.2055 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0802 | Regression_eta: 0.0422 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0655 | LR: 7.18e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.0742s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0056 | max_forward: 0.6830 | max_loss: 0.1421 | max_backward: 1.2812 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.8931 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6786.1 | uss_MB: 6126.4 | child_rss_MB: 5006.2 | allocated_MB: 338.7 | reserved_MB: 25044.0 | inactive_split_MB: 232.6 +rank7 [2026-10-29 00:39:31,234] INFO : Step 19000/50000 rank7 | Train Loss: 1.2397 | Train Loss Unweighted: 0.6317 | Train Loss Components: Auxiliary: 0.2573 | Cardinality: 0.1277 | Classification: 0.1188 | Classification_binary: 0.1866 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1600 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1840 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0726 | Regression_eta: 0.0334 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0547 | LR: 7.16e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.2164s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6593 | max_loss: 0.2467 | max_backward: 1.6203 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.7732 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6787.7 | uss_MB: 6128.3 | child_rss_MB: 5026.8 | allocated_MB: 342.2 | reserved_MB: 25790.0 | inactive_split_MB: 229.6 +rank7 [2026-10-29 00:39:32,175] INFO : Running validation on rank7 +rank7 [2026-10-29 00:39:32,175] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 00:39:32,175] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 00:40:23,807] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:40:24,368] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 00:40:50,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 00:40:50,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 00:40:51,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 00:40:52,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 00:41:02,717] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 00:41:03,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:41:04,088] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 00:41:19,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 00:41:20,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 00:41:21,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 00:41:22,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 00:41:32,716] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 00:41:33,292] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:41:33,791] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 00:41:49,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 00:41:50,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 00:41:50,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 00:41:51,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_19000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 00:42:01,358] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 00:46:01,996] INFO : Step 19100/50000 rank7 | Train Loss: 1.3562 | Train Loss Unweighted: 0.6933 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1644 | Classification: 0.1204 | Classification_binary: 0.2426 | HitGrouping: 0.2060 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1627 | Regression_eta: 0.0023 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0813 | Classification_binary: 0.2748 | Regression_cos_phi: 0.0892 | Regression_energy: 0.0811 | Regression_eta: 0.0694 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0876 | LR: 7.13e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.5907s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7796 | max_loss: 0.1634 | max_backward: 1.6942 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.8232 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 8027.6 | uss_MB: 7365.9 | child_rss_MB: 5027.2 | allocated_MB: 345.2 | reserved_MB: 23900.0 | inactive_split_MB: 240.8 +rank7 [2026-10-29 00:49:48,723] INFO : Step 19200/50000 rank7 | Train Loss: 1.1859 | Train Loss Unweighted: 0.5946 | Train Loss Components: Auxiliary: 0.2384 | Cardinality: 0.1391 | Classification: 0.0965 | Classification_binary: 0.1984 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1472 | Regression_eta: 0.0018 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2004 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0713 | Regression_eta: 0.0546 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0849 | LR: 7.10e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9587s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7138 | max_loss: 0.2391 | max_backward: 1.2379 | max_optimizer_step: 0.2683 | max_tensorboard_logging: 0.7395 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6876.1 | uss_MB: 6216.1 | child_rss_MB: 5025.0 | allocated_MB: 341.3 | reserved_MB: 23900.0 | inactive_split_MB: 206.8 +rank7 [2026-10-29 00:53:30,060] INFO : Step 19300/50000 rank7 | Train Loss: 1.1768 | Train Loss Unweighted: 0.6083 | Train Loss Components: Auxiliary: 0.2422 | Cardinality: 0.1345 | Classification: 0.1035 | Classification_binary: 0.1709 | HitGrouping: 0.2100 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0018 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0818 | Regression_eta: 0.0366 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0561 | LR: 7.08e-04 | DataLoad Time: 0.0214s | Model Forward Time: 3.8964s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0503 | max_forward: 0.7163 | max_loss: 0.1442 | max_backward: 1.3745 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6878.1 | uss_MB: 6217.4 | child_rss_MB: 5020.2 | allocated_MB: 341.0 | reserved_MB: 23900.0 | inactive_split_MB: 218.1 +rank7 [2026-10-29 00:57:11,121] INFO : Step 19400/50000 rank7 | Train Loss: 1.4590 | Train Loss Unweighted: 0.7109 | Train Loss Components: Auxiliary: 0.3106 | Cardinality: 0.2328 | Classification: 0.1237 | Classification_binary: 0.2265 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1770 | Regression_eta: 0.0021 | Regression_pt: 0.1794 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0783 | Classification_binary: 0.1902 | Regression_cos_phi: 0.0964 | Regression_energy: 0.0714 | Regression_eta: 0.0504 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0960 | LR: 7.05e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.4243s | Phase Times: max_pre_step_barrier: 0.1118 | max_batch_to_device: 0.0070 | max_forward: 0.7578 | max_loss: 0.2459 | max_backward: 1.7975 | max_optimizer_step: 0.2207 | max_tensorboard_logging: 0.4946 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6876.3 | uss_MB: 6217.1 | child_rss_MB: 5016.1 | allocated_MB: 347.7 | reserved_MB: 24186.0 | inactive_split_MB: 276.3 +rank7 [2026-10-29 01:00:48,550] INFO : Step 19500/50000 rank7 | Train Loss: 1.3753 | Train Loss Unweighted: 0.6794 | Train Loss Components: Auxiliary: 0.2810 | Cardinality: 0.1865 | Classification: 0.1153 | Classification_binary: 0.2310 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1637 | Regression_eta: 0.0023 | Regression_pt: 0.1650 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0866 | Classification_binary: 0.2324 | Regression_cos_phi: 0.1116 | Regression_energy: 0.0863 | Regression_eta: 0.0698 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0994 | LR: 7.02e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8974s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7197 | max_loss: 0.2164 | max_backward: 1.3937 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.4987 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6876.3 | uss_MB: 6219.5 | child_rss_MB: 5017.7 | allocated_MB: 345.2 | reserved_MB: 24186.0 | inactive_split_MB: 275.2 +rank7 [2026-10-29 01:04:42,200] INFO : Step 19600/50000 rank7 | Train Loss: 1.2086 | Train Loss Unweighted: 0.6229 | Train Loss Components: Auxiliary: 0.2543 | Cardinality: 0.1557 | Classification: 0.0942 | Classification_binary: 0.1768 | HitGrouping: 0.2138 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1735 | Regression_eta: 0.0015 | Regression_pt: 0.1752 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2044 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0700 | Regression_eta: 0.0541 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0721 | LR: 7.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.7343s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.6937 | max_loss: 0.1942 | max_backward: 1.2727 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.5893 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6877.2 | uss_MB: 6219.0 | child_rss_MB: 5004.9 | allocated_MB: 340.6 | reserved_MB: 24186.0 | inactive_split_MB: 277.9 +rank7 [2026-10-29 01:08:29,168] INFO : Step 19700/50000 rank7 | Train Loss: 1.2115 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1478 | Classification: 0.1094 | Classification_binary: 0.1919 | HitGrouping: 0.2027 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0746 | Regression_eta: 0.0371 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0477 | LR: 6.97e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8673s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0914 | max_forward: 0.7091 | max_loss: 0.1429 | max_backward: 1.4681 | max_optimizer_step: 0.2183 | max_tensorboard_logging: 0.4967 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0652 | Memory: rss_MB: 6879.2 | uss_MB: 6222.3 | child_rss_MB: 5016.1 | allocated_MB: 344.0 | reserved_MB: 24630.0 | inactive_split_MB: 143.8 +rank7 [2026-10-29 01:12:36,832] INFO : Step 19800/50000 rank7 | Train Loss: 1.2919 | Train Loss Unweighted: 0.6584 | Train Loss Components: Auxiliary: 0.2687 | Cardinality: 0.1453 | Classification: 0.1132 | Classification_binary: 0.2375 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1502 | Regression_eta: 0.0020 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0734 | Classification_binary: 0.2279 | Regression_cos_phi: 0.0887 | Regression_energy: 0.0804 | Regression_eta: 0.0692 | Regression_pt: 0.0846 | Regression_sin_phi: 0.1077 | LR: 6.94e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.4531s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0074 | max_forward: 0.7828 | max_loss: 0.2508 | max_backward: 1.8604 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 0.6313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 6879.1 | uss_MB: 6223.6 | child_rss_MB: 5015.7 | allocated_MB: 348.0 | reserved_MB: 25634.0 | inactive_split_MB: 277.9 +rank7 [2026-10-29 01:16:27,498] INFO : Step 19900/50000 rank7 | Train Loss: 1.2010 | Train Loss Unweighted: 0.6167 | Train Loss Components: Auxiliary: 0.2526 | Cardinality: 0.1346 | Classification: 0.0885 | Classification_binary: 0.1961 | HitGrouping: 0.2040 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1633 | Regression_eta: 0.0017 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2070 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0812 | Regression_eta: 0.0522 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0844 | LR: 6.92e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9745s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6671 | max_loss: 0.2548 | max_backward: 1.5862 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.5118 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6881.1 | uss_MB: 6224.6 | child_rss_MB: 5016.9 | allocated_MB: 340.3 | reserved_MB: 25634.0 | inactive_split_MB: 201.7 +rank7 [2026-10-29 01:20:16,835] INFO : Step 20000/50000 rank7 | Train Loss: 1.2077 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2556 | Cardinality: 0.1487 | Classification: 0.1030 | Classification_binary: 0.1729 | HitGrouping: 0.2105 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1639 | Regression_eta: 0.0019 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0930 | Regression_eta: 0.0513 | Regression_pt: 0.0957 | Regression_sin_phi: 0.0838 | LR: 6.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8147s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.6570 | max_loss: 0.1550 | max_backward: 1.4848 | max_optimizer_step: 0.2205 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6884.1 | uss_MB: 6226.8 | child_rss_MB: 5009.1 | allocated_MB: 341.8 | reserved_MB: 26642.0 | inactive_split_MB: 237.7 +rank7 [2026-10-29 01:20:17,836] INFO : Running validation on rank7 +rank7 [2026-10-29 01:20:17,836] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 01:20:17,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 01:21:07,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:21:08,246] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 01:21:34,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 01:21:35,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 01:21:36,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 01:21:37,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 01:21:47,605] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 01:21:48,206] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:21:48,733] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 01:22:05,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 01:22:06,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 01:22:06,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 01:22:07,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 01:22:17,817] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 01:22:18,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:22:18,872] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 01:22:34,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 01:22:35,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 01:22:35,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 01:22:36,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_20000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 01:22:46,691] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 01:26:57,041] INFO : Step 20100/50000 rank7 | Train Loss: 1.3825 | Train Loss Unweighted: 0.6739 | Train Loss Components: Auxiliary: 0.2830 | Cardinality: 0.1854 | Classification: 0.1189 | Classification_binary: 0.2349 | HitGrouping: 0.2141 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1561 | Regression_eta: 0.0022 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.2147 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0813 | Regression_eta: 0.0438 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0640 | LR: 6.86e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2458s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.9496 | max_loss: 0.1965 | max_backward: 1.6425 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.5117 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.1294 | Memory: rss_MB: 8120.6 | uss_MB: 7458.3 | child_rss_MB: 5010.7 | allocated_MB: 344.8 | reserved_MB: 21328.0 | inactive_split_MB: 198.7 +rank7 [2026-10-29 01:30:57,230] INFO : Step 20200/50000 rank7 | Train Loss: 1.3220 | Train Loss Unweighted: 0.6624 | Train Loss Components: Auxiliary: 0.2728 | Cardinality: 0.1545 | Classification: 0.1109 | Classification_binary: 0.2232 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1618 | Regression_eta: 0.0021 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0790 | Classification_binary: 0.2451 | Regression_cos_phi: 0.1175 | Regression_energy: 0.0720 | Regression_eta: 0.0619 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0932 | LR: 6.84e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8629s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.9337 | max_loss: 0.2287 | max_backward: 1.4619 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.4910 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6969.7 | uss_MB: 6305.7 | child_rss_MB: 5014.0 | allocated_MB: 344.4 | reserved_MB: 21328.0 | inactive_split_MB: 287.4 +rank7 [2026-10-29 01:34:46,195] INFO : Step 20300/50000 rank7 | Train Loss: 1.2742 | Train Loss Unweighted: 0.6629 | Train Loss Components: Auxiliary: 0.2710 | Cardinality: 0.1615 | Classification: 0.1132 | Classification_binary: 0.1937 | HitGrouping: 0.2129 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1761 | Regression_eta: 0.0018 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.2307 | Regression_cos_phi: 0.0868 | Regression_energy: 0.0826 | Regression_eta: 0.0616 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0719 | LR: 6.81e-04 | DataLoad Time: 0.0150s | Model Forward Time: 3.8215s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.6857 | max_loss: 0.2423 | max_backward: 1.2822 | max_optimizer_step: 0.2837 | max_tensorboard_logging: 0.5944 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0419 | Memory: rss_MB: 6967.2 | uss_MB: 6303.7 | child_rss_MB: 5019.2 | allocated_MB: 340.8 | reserved_MB: 21826.0 | inactive_split_MB: 239.4 +rank7 [2026-10-29 01:38:43,604] INFO : Step 20400/50000 rank7 | Train Loss: 1.2743 | Train Loss Unweighted: 0.6521 | Train Loss Components: Auxiliary: 0.2657 | Cardinality: 0.1507 | Classification: 0.1279 | Classification_binary: 0.1990 | HitGrouping: 0.2083 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1604 | Regression_eta: 0.0019 | Regression_pt: 0.1610 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0806 | Regression_eta: 0.0562 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0675 | LR: 6.78e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.7404s | Phase Times: max_pre_step_barrier: 0.0109 | max_batch_to_device: 0.0283 | max_forward: 1.3648 | max_loss: 0.1816 | max_backward: 1.7875 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.5271 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 6969.3 | uss_MB: 6306.4 | child_rss_MB: 5010.2 | allocated_MB: 343.0 | reserved_MB: 22154.0 | inactive_split_MB: 243.5 +rank7 [2026-10-29 01:42:48,352] INFO : Step 20500/50000 rank7 | Train Loss: 1.2712 | Train Loss Unweighted: 0.6509 | Train Loss Components: Auxiliary: 0.2684 | Cardinality: 0.1409 | Classification: 0.1147 | Classification_binary: 0.2202 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1558 | Regression_eta: 0.0019 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0689 | Regression_eta: 0.0479 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0597 | LR: 6.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2547s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7205 | max_loss: 0.2283 | max_backward: 1.6623 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.6254 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0704 | Memory: rss_MB: 6969.0 | uss_MB: 6306.9 | child_rss_MB: 5000.2 | allocated_MB: 345.0 | reserved_MB: 22154.0 | inactive_split_MB: 208.2 +rank7 [2026-10-29 01:46:45,622] INFO : Step 20600/50000 rank7 | Train Loss: 1.2460 | Train Loss Unweighted: 0.6338 | Train Loss Components: Auxiliary: 0.2625 | Cardinality: 0.1623 | Classification: 0.1101 | Classification_binary: 0.1900 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1649 | Regression_eta: 0.0018 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2112 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0642 | Regression_eta: 0.0419 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0747 | LR: 6.73e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.9336s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0061 | max_forward: 1.1520 | max_loss: 0.2849 | max_backward: 2.1471 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.6141 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6963.6 | uss_MB: 6301.3 | child_rss_MB: 5002.5 | allocated_MB: 341.4 | reserved_MB: 22154.0 | inactive_split_MB: 183.5 +rank7 [2026-10-29 01:50:54,186] INFO : Step 20700/50000 rank7 | Train Loss: 1.1613 | Train Loss Unweighted: 0.6066 | Train Loss Components: Auxiliary: 0.2395 | Cardinality: 0.1327 | Classification: 0.1061 | Classification_binary: 0.1677 | HitGrouping: 0.2090 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1629 | Regression_eta: 0.0016 | Regression_pt: 0.1666 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0829 | Regression_eta: 0.0499 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0829 | LR: 6.70e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5668s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6276 | max_loss: 0.2330 | max_backward: 1.4954 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0641 | Memory: rss_MB: 6965.5 | uss_MB: 5936.8 | child_rss_MB: 5004.9 | allocated_MB: 341.0 | reserved_MB: 22154.0 | inactive_split_MB: 285.0 +rank7 [2026-10-29 01:55:01,768] INFO : Step 20800/50000 rank7 | Train Loss: 1.2519 | Train Loss Unweighted: 0.6312 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1493 | Classification: 0.1205 | Classification_binary: 0.2020 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0020 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2039 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0728 | Regression_eta: 0.0536 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0601 | LR: 6.67e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.2087s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.7679 | max_loss: 0.1604 | max_backward: 1.7694 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.4914 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 6972.0 | uss_MB: 6311.7 | child_rss_MB: 5027.1 | allocated_MB: 344.1 | reserved_MB: 22154.0 | inactive_split_MB: 258.7 +rank7 [2026-10-29 01:59:06,393] INFO : Step 20900/50000 rank7 | Train Loss: 1.2934 | Train Loss Unweighted: 0.6525 | Train Loss Components: Auxiliary: 0.2719 | Cardinality: 0.1620 | Classification: 0.1167 | Classification_binary: 0.2121 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1591 | Regression_eta: 0.0021 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2044 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0716 | Regression_eta: 0.0386 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0566 | LR: 6.65e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1341s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7787 | max_loss: 0.1747 | max_backward: 1.6678 | max_optimizer_step: 0.2201 | max_tensorboard_logging: 0.4884 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1132 | Memory: rss_MB: 6974.0 | uss_MB: 5772.6 | child_rss_MB: 5037.2 | allocated_MB: 343.6 | reserved_MB: 23208.0 | inactive_split_MB: 296.4 +rank7 [2026-10-29 02:03:08,876] INFO : Step 21000/50000 rank7 | Train Loss: 1.1402 | Train Loss Unweighted: 0.5874 | Train Loss Components: Auxiliary: 0.2399 | Cardinality: 0.1236 | Classification: 0.0967 | Classification_binary: 0.1701 | HitGrouping: 0.2098 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1583 | Regression_eta: 0.0017 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0859 | Regression_eta: 0.0523 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0722 | LR: 6.62e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8991s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.6629 | max_loss: 0.1833 | max_backward: 1.4689 | max_optimizer_step: 0.2206 | max_tensorboard_logging: 0.5119 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 6973.2 | uss_MB: 6313.8 | child_rss_MB: 5014.5 | allocated_MB: 340.4 | reserved_MB: 23208.0 | inactive_split_MB: 264.1 +rank7 [2026-10-29 02:03:09,907] INFO : Running validation on rank7 +rank7 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 02:03:09,907] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 02:04:00,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:04:01,073] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 02:04:27,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 02:04:28,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 02:04:28,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 02:04:29,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 02:04:39,772] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 02:04:40,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:04:41,351] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 02:04:57,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 02:04:58,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 02:04:59,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 02:05:00,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 02:05:10,396] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 02:05:10,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:05:11,355] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 02:05:26,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 02:05:27,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 02:05:28,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 02:05:29,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_21000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 02:05:39,937] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 02:10:06,180] INFO : Step 21100/50000 rank7 | Train Loss: 1.3089 | Train Loss Unweighted: 0.6632 | Train Loss Components: Auxiliary: 0.2723 | Cardinality: 0.1574 | Classification: 0.1277 | Classification_binary: 0.2026 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0024 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0797 | Classification_binary: 0.1901 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0951 | Regression_eta: 0.0591 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0667 | LR: 6.59e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2653s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7159 | max_loss: 0.1411 | max_backward: 1.5805 | max_optimizer_step: 0.3396 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8247.8 | uss_MB: 7584.2 | child_rss_MB: 5046.5 | allocated_MB: 343.8 | reserved_MB: 21372.0 | inactive_split_MB: 255.4 +rank7 [2026-10-29 02:10:38,255] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank7 [2026-10-29 02:10:38,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-29 02:14:15,483] INFO : Step 21200/50000 rank7 | Train Loss: 1.2354 | Train Loss Unweighted: 0.6366 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1422 | Classification: 0.1056 | Classification_binary: 0.1876 | HitGrouping: 0.2030 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1688 | Regression_eta: 0.0017 | Regression_pt: 0.1709 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2119 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0836 | Regression_eta: 0.0458 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0719 | LR: 6.56e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.1582s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 0.7385 | max_loss: 0.2183 | max_backward: 1.4276 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.7483 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7137.9 | uss_MB: 6469.9 | child_rss_MB: 5019.5 | allocated_MB: 342.5 | reserved_MB: 23286.0 | inactive_split_MB: 273.0 +rank7 [2026-10-29 02:18:17,991] INFO : Step 21300/50000 rank7 | Train Loss: 1.1511 | Train Loss Unweighted: 0.5782 | Train Loss Components: Auxiliary: 0.2425 | Cardinality: 0.1240 | Classification: 0.1021 | Classification_binary: 0.1706 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1505 | Regression_eta: 0.0017 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1742 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0762 | Regression_eta: 0.0409 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0675 | LR: 6.54e-04 | DataLoad Time: 0.0238s | Model Forward Time: 4.4909s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0290 | max_forward: 1.3238 | max_loss: 0.2105 | max_backward: 1.6672 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.6878 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7140.4 | uss_MB: 6475.6 | child_rss_MB: 5012.2 | allocated_MB: 343.8 | reserved_MB: 23286.0 | inactive_split_MB: 272.8 +rank7 [2026-10-29 02:22:14,589] INFO : Step 21400/50000 rank7 | Train Loss: 1.2993 | Train Loss Unweighted: 0.6458 | Train Loss Components: Auxiliary: 0.2724 | Cardinality: 0.1796 | Classification: 0.1204 | Classification_binary: 0.2158 | HitGrouping: 0.2050 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1518 | Regression_eta: 0.0018 | Regression_pt: 0.1542 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0944 | Regression_energy: 0.0811 | Regression_eta: 0.0459 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0844 | LR: 6.51e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.5448s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8696 | max_loss: 0.1572 | max_backward: 1.8002 | max_optimizer_step: 0.2462 | max_tensorboard_logging: 0.6992 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7114.7 | uss_MB: 6081.0 | child_rss_MB: 5015.7 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 201.9 +rank7 [2026-10-29 02:26:14,444] INFO : Step 21500/50000 rank7 | Train Loss: 1.2839 | Train Loss Unweighted: 0.6563 | Train Loss Components: Auxiliary: 0.2693 | Cardinality: 0.1478 | Classification: 0.1058 | Classification_binary: 0.2162 | HitGrouping: 0.2091 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1639 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2166 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0786 | Regression_eta: 0.0333 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0793 | LR: 6.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7745s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.8684 | max_loss: 0.1577 | max_backward: 1.9355 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.7593 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0648 | Memory: rss_MB: 7121.8 | uss_MB: 6458.0 | child_rss_MB: 5014.4 | allocated_MB: 341.9 | reserved_MB: 25248.0 | inactive_split_MB: 284.1 +rank7 [2026-10-29 02:30:00,712] INFO : Step 21600/50000 rank7 | Train Loss: 1.1374 | Train Loss Unweighted: 0.5707 | Train Loss Components: Auxiliary: 0.2360 | Cardinality: 0.1290 | Classification: 0.1062 | Classification_binary: 0.1618 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1486 | Regression_eta: 0.0018 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1420 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0765 | Regression_eta: 0.0365 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0421 | LR: 6.45e-04 | DataLoad Time: 0.0218s | Model Forward Time: 3.9642s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0061 | max_forward: 0.7663 | max_loss: 0.1883 | max_backward: 1.4257 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.4997 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 7120.6 | uss_MB: 6456.9 | child_rss_MB: 5032.2 | allocated_MB: 340.5 | reserved_MB: 25248.0 | inactive_split_MB: 234.8 +rank7 [2026-10-29 02:33:57,426] INFO : Step 21700/50000 rank7 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6283 | Train Loss Components: Auxiliary: 0.2577 | Cardinality: 0.1575 | Classification: 0.1186 | Classification_binary: 0.1853 | HitGrouping: 0.2068 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1594 | Regression_eta: 0.0022 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0923 | Classification_binary: 0.1761 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0749 | Regression_eta: 0.0536 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0556 | LR: 6.43e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.7322s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.6796 | max_loss: 0.1417 | max_backward: 1.3881 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7122.2 | uss_MB: 6458.0 | child_rss_MB: 5034.2 | allocated_MB: 343.8 | reserved_MB: 25248.0 | inactive_split_MB: 244.5 +rank7 [2026-10-29 02:37:54,439] INFO : Step 21800/50000 rank7 | Train Loss: 1.2260 | Train Loss Unweighted: 0.6351 | Train Loss Components: Auxiliary: 0.2691 | Cardinality: 0.1439 | Classification: 0.1105 | Classification_binary: 0.2043 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1576 | Regression_eta: 0.0016 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1773 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0817 | Regression_eta: 0.0338 | Regression_pt: 0.0845 | Regression_sin_phi: 0.0661 | LR: 6.40e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.9195s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0067 | max_forward: 0.8043 | max_loss: 0.1895 | max_backward: 2.4027 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.9442 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7124.3 | uss_MB: 5827.3 | child_rss_MB: 5037.0 | allocated_MB: 344.5 | reserved_MB: 25248.0 | inactive_split_MB: 277.8 +rank7 [2026-10-29 02:41:42,512] INFO : Step 21900/50000 rank7 | Train Loss: 1.2119 | Train Loss Unweighted: 0.5952 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1861 | Classification: 0.1033 | Classification_binary: 0.1865 | HitGrouping: 0.1989 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0015 | Regression_pt: 0.1511 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0864 | Classification_binary: 0.2208 | Regression_cos_phi: 0.0945 | Regression_energy: 0.0818 | Regression_eta: 0.0567 | Regression_pt: 0.0838 | Regression_sin_phi: 0.1210 | LR: 6.37e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.5485s | Phase Times: max_pre_step_barrier: 0.0174 | max_batch_to_device: 0.0163 | max_forward: 0.9684 | max_loss: 0.1448 | max_backward: 1.6354 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.8990 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 7124.6 | uss_MB: 6460.4 | child_rss_MB: 5033.8 | allocated_MB: 342.7 | reserved_MB: 25248.0 | inactive_split_MB: 192.3 +rank7 [2026-10-29 02:45:41,732] INFO : Step 22000/50000 rank7 | Train Loss: 1.1094 | Train Loss Unweighted: 0.5651 | Train Loss Components: Auxiliary: 0.2344 | Cardinality: 0.1248 | Classification: 0.1037 | Classification_binary: 0.1579 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1497 | Regression_eta: 0.0017 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0737 | Regression_eta: 0.0432 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0566 | LR: 6.34e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.4217s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.6825 | max_loss: 0.1653 | max_backward: 1.4396 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.0256 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 7124.1 | uss_MB: 6463.7 | child_rss_MB: 5009.1 | allocated_MB: 344.2 | reserved_MB: 25248.0 | inactive_split_MB: 247.3 +rank7 [2026-10-29 02:45:42,811] INFO : Running validation on rank7 +rank7 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 02:45:42,811] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 02:46:37,519] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:46:38,045] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 02:46:57,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 02:46:58,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 02:46:59,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 02:47:00,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 02:47:10,276] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 02:47:11,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:47:11,958] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 02:47:27,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 02:47:28,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 02:47:29,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 02:47:30,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 02:47:40,613] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 02:47:41,460] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:47:42,032] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 02:47:57,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 02:47:58,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 02:47:58,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 02:47:59,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_22000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 02:48:09,672] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 02:52:24,843] INFO : Step 22100/50000 rank7 | Train Loss: 1.2514 | Train Loss Unweighted: 0.6219 | Train Loss Components: Auxiliary: 0.2649 | Cardinality: 0.1530 | Classification: 0.1140 | Classification_binary: 0.2076 | HitGrouping: 0.2031 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1470 | Regression_eta: 0.0018 | Regression_pt: 0.1495 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.2153 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0679 | Regression_eta: 0.0736 | Regression_pt: 0.0706 | Regression_sin_phi: 0.1284 | LR: 6.32e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5401s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7250 | max_loss: 0.2514 | max_backward: 1.5956 | max_optimizer_step: 0.2443 | max_tensorboard_logging: 0.8632 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0587 | Memory: rss_MB: 8032.9 | uss_MB: 7365.3 | child_rss_MB: 5019.4 | allocated_MB: 347.3 | reserved_MB: 23386.0 | inactive_split_MB: 278.6 +rank7 [2026-10-29 02:56:29,092] INFO : Step 22200/50000 rank7 | Train Loss: 1.2727 | Train Loss Unweighted: 0.6485 | Train Loss Components: Auxiliary: 0.2709 | Cardinality: 0.1498 | Classification: 0.1104 | Classification_binary: 0.2184 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1574 | Regression_eta: 0.0018 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2078 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0911 | Regression_eta: 0.0360 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0695 | LR: 6.29e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.9393s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0074 | max_forward: 0.7776 | max_loss: 0.1472 | max_backward: 1.7604 | max_optimizer_step: 0.2283 | max_tensorboard_logging: 1.3305 | max_tensorboard_flush: 0.0032 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7158.4 | uss_MB: 5092.5 | child_rss_MB: 5016.1 | allocated_MB: 342.3 | reserved_MB: 23578.0 | inactive_split_MB: 277.9 +rank7 [2026-10-29 03:00:35,124] INFO : Step 22300/50000 rank7 | Train Loss: 1.1242 | Train Loss Unweighted: 0.5879 | Train Loss Components: Auxiliary: 0.2347 | Cardinality: 0.1367 | Classification: 0.1013 | Classification_binary: 0.1688 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1566 | Regression_eta: 0.0017 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.2219 | Regression_cos_phi: 0.0960 | Regression_energy: 0.0650 | Regression_eta: 0.0460 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0960 | LR: 6.26e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.7791s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7697 | max_loss: 0.1538 | max_backward: 1.3553 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.4896 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0913 | Memory: rss_MB: 7162.9 | uss_MB: 6498.9 | child_rss_MB: 4987.3 | allocated_MB: 341.2 | reserved_MB: 23578.0 | inactive_split_MB: 233.8 +rank7 [2026-10-29 03:04:32,773] INFO : Step 22400/50000 rank7 | Train Loss: 1.2357 | Train Loss Unweighted: 0.6255 | Train Loss Components: Auxiliary: 0.2623 | Cardinality: 0.1493 | Classification: 0.1171 | Classification_binary: 0.1885 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1575 | Regression_eta: 0.0021 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.1816 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0685 | Regression_eta: 0.0999 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0683 | LR: 6.23e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9143s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0067 | max_forward: 0.6425 | max_loss: 0.1530 | max_backward: 1.4136 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.5876 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0646 | Memory: rss_MB: 7158.4 | uss_MB: 6496.8 | child_rss_MB: 5021.9 | allocated_MB: 342.5 | reserved_MB: 23578.0 | inactive_split_MB: 251.5 +rank7 [2026-10-29 03:08:26,683] INFO : Step 22500/50000 rank7 | Train Loss: 1.2633 | Train Loss Unweighted: 0.6215 | Train Loss Components: Auxiliary: 0.2638 | Cardinality: 0.1676 | Classification: 0.0954 | Classification_binary: 0.2181 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1511 | Regression_eta: 0.0020 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1939 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0676 | Regression_eta: 0.0632 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0819 | LR: 6.20e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.4273s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1522 | max_backward: 1.5631 | max_optimizer_step: 0.3261 | max_tensorboard_logging: 0.7552 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 7160.2 | uss_MB: 6502.4 | child_rss_MB: 5011.7 | allocated_MB: 342.9 | reserved_MB: 24052.0 | inactive_split_MB: 295.1 +rank7 [2026-10-29 03:12:18,754] INFO : Step 22600/50000 rank7 | Train Loss: 1.1636 | Train Loss Unweighted: 0.5935 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1329 | Classification: 0.0997 | Classification_binary: 0.1889 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1493 | Regression_eta: 0.0020 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2100 | Regression_cos_phi: 0.0853 | Regression_energy: 0.0669 | Regression_eta: 0.0578 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0665 | LR: 6.18e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9317s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7375 | max_loss: 0.2403 | max_backward: 1.4584 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7163.5 | uss_MB: 6503.3 | child_rss_MB: 5024.4 | allocated_MB: 344.2 | reserved_MB: 24590.0 | inactive_split_MB: 288.5 +rank7 [2026-10-29 03:16:20,064] INFO : Step 22700/50000 rank7 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5506 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1212 | Classification: 0.0976 | Classification_binary: 0.1566 | HitGrouping: 0.2101 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1455 | Regression_eta: 0.0015 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0658 | Regression_eta: 0.0599 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0503 | LR: 6.15e-04 | DataLoad Time: 0.0169s | Model Forward Time: 6.9678s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.6124 | max_loss: 0.1376 | max_backward: 1.1766 | max_optimizer_step: 0.2237 | max_tensorboard_logging: 3.8568 | max_tensorboard_flush: 0.0169 | max_post_step_barrier: 0.0442 | Memory: rss_MB: 7164.2 | uss_MB: 6160.1 | child_rss_MB: 5007.8 | allocated_MB: 341.6 | reserved_MB: 25058.0 | inactive_split_MB: 256.4 +rank7 [2026-10-29 03:20:29,740] INFO : Step 22800/50000 rank7 | Train Loss: 1.1840 | Train Loss Unweighted: 0.6057 | Train Loss Components: Auxiliary: 0.2533 | Cardinality: 0.1443 | Classification: 0.1191 | Classification_binary: 0.1924 | HitGrouping: 0.1959 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1444 | Regression_eta: 0.0016 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1694 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0676 | Regression_eta: 0.0500 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0600 | LR: 6.12e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4523s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7010 | max_loss: 0.2312 | max_backward: 1.6853 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8726 | max_tensorboard_flush: 0.0081 | max_post_step_barrier: 0.0438 | Memory: rss_MB: 7164.4 | uss_MB: 6502.8 | child_rss_MB: 5021.2 | allocated_MB: 341.7 | reserved_MB: 25058.0 | inactive_split_MB: 281.3 +rank7 [2026-10-29 03:24:21,160] INFO : Step 22900/50000 rank7 | Train Loss: 1.2680 | Train Loss Unweighted: 0.6318 | Train Loss Components: Auxiliary: 0.2704 | Cardinality: 0.1792 | Classification: 0.1020 | Classification_binary: 0.2074 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1580 | Regression_eta: 0.0017 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0726 | Regression_energy: 0.0908 | Regression_eta: 0.0716 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0839 | LR: 6.09e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9982s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0067 | max_forward: 1.1637 | max_loss: 0.4031 | max_backward: 1.8442 | max_optimizer_step: 0.3320 | max_tensorboard_logging: 0.7826 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7165.6 | uss_MB: 6506.2 | child_rss_MB: 5023.4 | allocated_MB: 344.8 | reserved_MB: 25058.0 | inactive_split_MB: 300.1 +rank7 [2026-10-29 03:28:05,593] INFO : Step 23000/50000 rank7 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5484 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1129 | Classification: 0.0836 | Classification_binary: 0.1519 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1547 | Regression_eta: 0.0017 | Regression_pt: 0.1551 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1727 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0877 | Regression_eta: 0.0365 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0487 | LR: 6.06e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.7530s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0063 | max_forward: 0.6963 | max_loss: 0.2382 | max_backward: 1.2870 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 7162.6 | uss_MB: 6506.3 | child_rss_MB: 5008.7 | allocated_MB: 340.3 | reserved_MB: 26066.0 | inactive_split_MB: 218.4 +rank7 [2026-10-29 03:28:06,534] INFO : Running validation on rank7 +rank7 [2026-10-29 03:28:06,534] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 03:28:06,535] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 03:28:56,134] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:28:56,638] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 03:29:22,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 03:29:23,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 03:29:24,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 03:29:24,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 03:29:34,964] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 03:29:35,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:29:36,301] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 03:29:52,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 03:29:53,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 03:29:53,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 03:29:54,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 03:30:04,896] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 03:30:05,222] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:30:05,703] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 03:30:21,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 03:30:21,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 03:30:22,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 03:30:23,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_23000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 03:30:33,185] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 03:34:45,381] INFO : Step 23100/50000 rank7 | Train Loss: 1.2422 | Train Loss Unweighted: 0.6061 | Train Loss Components: Auxiliary: 0.2557 | Cardinality: 0.1379 | Classification: 0.1246 | Classification_binary: 0.1795 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1482 | Regression_eta: 0.0023 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0721 | Regression_eta: 0.0352 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0612 | LR: 6.04e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8241s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7203 | max_loss: 0.1522 | max_backward: 1.1000 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.6183 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0577 | Memory: rss_MB: 8419.6 | uss_MB: 7758.7 | child_rss_MB: 5039.2 | allocated_MB: 344.7 | reserved_MB: 22598.0 | inactive_split_MB: 244.5 +rank7 [2026-10-29 03:38:33,524] INFO : Step 23200/50000 rank7 | Train Loss: 1.2507 | Train Loss Unweighted: 0.6359 | Train Loss Components: Auxiliary: 0.2653 | Cardinality: 0.1400 | Classification: 0.1121 | Classification_binary: 0.2208 | HitGrouping: 0.2013 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1488 | Regression_eta: 0.0020 | Regression_pt: 0.1505 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0677 | Regression_eta: 0.0457 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0491 | LR: 6.01e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.5951s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0727 | max_forward: 0.9486 | max_loss: 0.1744 | max_backward: 1.7031 | max_optimizer_step: 0.2405 | max_tensorboard_logging: 0.6717 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1109 | Memory: rss_MB: 7225.3 | uss_MB: 6564.6 | child_rss_MB: 5046.8 | allocated_MB: 343.7 | reserved_MB: 23792.0 | inactive_split_MB: 223.2 +rank7 [2026-10-29 03:42:24,565] INFO : Step 23300/50000 rank7 | Train Loss: 1.1880 | Train Loss Unweighted: 0.5927 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1464 | Classification: 0.0987 | Classification_binary: 0.1851 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1524 | Regression_eta: 0.0019 | Regression_pt: 0.1529 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0831 | Regression_eta: 0.0392 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0690 | LR: 5.98e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8605s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7557 | max_loss: 0.1467 | max_backward: 1.4708 | max_optimizer_step: 0.2232 | max_tensorboard_logging: 0.4854 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 7231.3 | uss_MB: 6572.4 | child_rss_MB: 5045.1 | allocated_MB: 342.6 | reserved_MB: 24534.0 | inactive_split_MB: 271.6 +rank7 [2026-10-29 03:46:11,305] INFO : Step 23400/50000 rank7 | Train Loss: 1.1136 | Train Loss Unweighted: 0.5909 | Train Loss Components: Auxiliary: 0.2402 | Cardinality: 0.1202 | Classification: 0.1070 | Classification_binary: 0.1497 | HitGrouping: 0.2053 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1644 | Regression_eta: 0.0017 | Regression_pt: 0.1665 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0795 | Regression_eta: 0.0366 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0666 | LR: 5.95e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.6665s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.5743 | max_loss: 0.2297 | max_backward: 1.0127 | max_optimizer_step: 0.3321 | max_tensorboard_logging: 0.6182 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0538 | Memory: rss_MB: 7232.3 | uss_MB: 6571.7 | child_rss_MB: 5046.9 | allocated_MB: 340.5 | reserved_MB: 24848.0 | inactive_split_MB: 241.3 +rank7 [2026-10-29 03:50:07,300] INFO : Step 23500/50000 rank7 | Train Loss: 1.1601 | Train Loss Unweighted: 0.5799 | Train Loss Components: Auxiliary: 0.2477 | Cardinality: 0.1161 | Classification: 0.1057 | Classification_binary: 0.1852 | HitGrouping: 0.2029 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1416 | Regression_eta: 0.0017 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1814 | Regression_cos_phi: 0.0922 | Regression_energy: 0.0671 | Regression_eta: 0.0505 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0573 | LR: 5.92e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4403s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8864 | max_loss: 0.1573 | max_backward: 1.9036 | max_optimizer_step: 0.2294 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7232.1 | uss_MB: 6574.4 | child_rss_MB: 5036.9 | allocated_MB: 341.9 | reserved_MB: 24848.0 | inactive_split_MB: 233.9 +rank7 [2026-10-29 03:53:52,163] INFO : Step 23600/50000 rank7 | Train Loss: 1.2402 | Train Loss Unweighted: 0.6307 | Train Loss Components: Auxiliary: 0.2607 | Cardinality: 0.1447 | Classification: 0.1136 | Classification_binary: 0.2049 | HitGrouping: 0.2039 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1534 | Regression_eta: 0.0020 | Regression_pt: 0.1550 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.2282 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0917 | Regression_eta: 0.0611 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0764 | LR: 5.90e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.9445s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8397 | max_loss: 0.1470 | max_backward: 1.4395 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.4763 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 7234.1 | uss_MB: 6575.7 | child_rss_MB: 5045.6 | allocated_MB: 342.5 | reserved_MB: 24848.0 | inactive_split_MB: 215.5 +rank7 [2026-10-29 03:57:35,779] INFO : Step 23700/50000 rank7 | Train Loss: 1.1009 | Train Loss Unweighted: 0.5682 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1218 | Classification: 0.0954 | Classification_binary: 0.1599 | HitGrouping: 0.2041 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1528 | Regression_eta: 0.0014 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0892 | Regression_energy: 0.0823 | Regression_eta: 0.0419 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0763 | LR: 5.87e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.6817s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6589 | max_loss: 0.1387 | max_backward: 1.2152 | max_optimizer_step: 0.2251 | max_tensorboard_logging: 0.5355 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0484 | Memory: rss_MB: 7230.0 | uss_MB: 6574.3 | child_rss_MB: 5051.4 | allocated_MB: 342.5 | reserved_MB: 25508.0 | inactive_split_MB: 226.3 +rank7 [2026-10-29 04:01:35,999] INFO : Step 23800/50000 rank7 | Train Loss: 1.1390 | Train Loss Unweighted: 0.5882 | Train Loss Components: Auxiliary: 0.2427 | Cardinality: 0.1196 | Classification: 0.1035 | Classification_binary: 0.1735 | HitGrouping: 0.2044 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1531 | Regression_eta: 0.0017 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0866 | Regression_eta: 0.0583 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0764 | LR: 5.84e-04 | DataLoad Time: 0.0220s | Model Forward Time: 3.7620s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.7283 | max_loss: 0.1547 | max_backward: 1.2415 | max_optimizer_step: 0.2259 | max_tensorboard_logging: 0.5270 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 7231.0 | uss_MB: 6110.2 | child_rss_MB: 5050.1 | allocated_MB: 345.2 | reserved_MB: 25508.0 | inactive_split_MB: 241.8 +rank7 [2026-10-29 04:05:42,354] INFO : Step 23900/50000 rank7 | Train Loss: 1.2349 | Train Loss Unweighted: 0.6078 | Train Loss Components: Auxiliary: 0.2621 | Cardinality: 0.1596 | Classification: 0.1044 | Classification_binary: 0.2077 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1451 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1857 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0677 | Regression_eta: 0.0448 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0757 | LR: 5.81e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.4884s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.9077 | max_loss: 0.1538 | max_backward: 1.9088 | max_optimizer_step: 0.2275 | max_tensorboard_logging: 0.5788 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7234.7 | uss_MB: 6579.6 | child_rss_MB: 5044.4 | allocated_MB: 342.1 | reserved_MB: 27474.0 | inactive_split_MB: 282.4 +rank7 [2026-10-29 04:09:40,153] INFO : Step 24000/50000 rank7 | Train Loss: 1.1074 | Train Loss Unweighted: 0.5714 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1170 | Classification: 0.1007 | Classification_binary: 0.1792 | HitGrouping: 0.2022 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1431 | Regression_eta: 0.0015 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.2035 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0650 | Regression_eta: 0.0638 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0784 | LR: 5.78e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.2186s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 1.1428 | max_loss: 0.2613 | max_backward: 2.5767 | max_optimizer_step: 0.2298 | max_tensorboard_logging: 0.5998 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 7236.7 | uss_MB: 6577.2 | child_rss_MB: 5038.1 | allocated_MB: 342.1 | reserved_MB: 27476.0 | inactive_split_MB: 218.6 +rank7 [2026-10-29 04:09:41,047] INFO : Running validation on rank7 +rank7 [2026-10-29 04:09:41,047] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 04:09:41,047] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 04:10:38,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:10:39,335] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 04:11:05,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 04:11:06,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 04:11:07,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 04:11:10,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 04:11:20,021] INFO : Time taken to make predictions on device 7 is: 0.28 min +rank7 [2026-10-29 04:11:20,671] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:11:21,258] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 04:11:37,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 04:11:38,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 04:11:39,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 04:11:40,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 04:11:50,019] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 04:11:50,532] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:11:51,026] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 04:12:06,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 04:12:07,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 04:12:07,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 04:12:08,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_24000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 04:12:18,660] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 04:16:38,287] INFO : Step 24100/50000 rank7 | Train Loss: 1.1244 | Train Loss Unweighted: 0.5584 | Train Loss Components: Auxiliary: 0.2373 | Cardinality: 0.1285 | Classification: 0.0978 | Classification_binary: 0.1606 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.1503 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0759 | Regression_eta: 0.0525 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0927 | LR: 5.75e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1992s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0860 | max_forward: 0.7361 | max_loss: 0.1432 | max_backward: 1.4268 | max_optimizer_step: 0.3451 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0629 | Memory: rss_MB: 8228.7 | uss_MB: 7570.1 | child_rss_MB: 5001.0 | allocated_MB: 343.2 | reserved_MB: 21608.0 | inactive_split_MB: 269.9 +rank7 [2026-10-29 04:20:36,776] INFO : Step 24200/50000 rank7 | Train Loss: 1.3067 | Train Loss Unweighted: 0.6341 | Train Loss Components: Auxiliary: 0.2739 | Cardinality: 0.1760 | Classification: 0.1145 | Classification_binary: 0.2101 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0018 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0848 | Classification_binary: 0.1922 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0702 | Regression_eta: 0.0432 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0733 | LR: 5.73e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0786s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.1066 | max_forward: 1.0531 | max_loss: 0.2796 | max_backward: 2.3820 | max_optimizer_step: 0.2441 | max_tensorboard_logging: 0.6187 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 8233.6 | uss_MB: 7574.6 | child_rss_MB: 5010.3 | allocated_MB: 345.0 | reserved_MB: 23864.0 | inactive_split_MB: 203.0 +rank7 [2026-10-29 04:24:33,010] INFO : Step 24300/50000 rank7 | Train Loss: 1.2207 | Train Loss Unweighted: 0.6113 | Train Loss Components: Auxiliary: 0.2566 | Cardinality: 0.1582 | Classification: 0.1151 | Classification_binary: 0.2013 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1450 | Regression_eta: 0.0017 | Regression_pt: 0.1465 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.2175 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0681 | Regression_eta: 0.0450 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0719 | LR: 5.70e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3055s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0066 | max_forward: 0.7395 | max_loss: 0.2490 | max_backward: 1.4969 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.6555 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7355.3 | uss_MB: 6697.1 | child_rss_MB: 5007.1 | allocated_MB: 344.8 | reserved_MB: 23864.0 | inactive_split_MB: 284.8 +rank7 [2026-10-29 04:28:19,172] INFO : Step 24400/50000 rank7 | Train Loss: 1.1288 | Train Loss Unweighted: 0.5921 | Train Loss Components: Auxiliary: 0.2439 | Cardinality: 0.1229 | Classification: 0.1017 | Classification_binary: 0.1613 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1631 | Regression_eta: 0.0015 | Regression_pt: 0.1628 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1624 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0708 | Regression_eta: 0.0318 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0606 | LR: 5.67e-04 | DataLoad Time: 0.0186s | Model Forward Time: 3.9873s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.6685 | max_loss: 0.1564 | max_backward: 1.4736 | max_optimizer_step: 0.2223 | max_tensorboard_logging: 0.5369 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 7357.1 | uss_MB: 6698.7 | child_rss_MB: 5016.5 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 267.2 +rank7 [2026-10-29 04:32:03,440] INFO : Step 24500/50000 rank7 | Train Loss: 1.2319 | Train Loss Unweighted: 0.6122 | Train Loss Components: Auxiliary: 0.2586 | Cardinality: 0.1511 | Classification: 0.1107 | Classification_binary: 0.1876 | HitGrouping: 0.2103 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1540 | Regression_eta: 0.0018 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1729 | Regression_cos_phi: 0.1204 | Regression_energy: 0.0711 | Regression_eta: 0.0615 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0744 | LR: 5.64e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1615s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0677 | max_forward: 0.7219 | max_loss: 0.1439 | max_backward: 1.4253 | max_optimizer_step: 0.3416 | max_tensorboard_logging: 0.5859 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 7356.5 | uss_MB: 6698.2 | child_rss_MB: 5010.5 | allocated_MB: 343.1 | reserved_MB: 23864.0 | inactive_split_MB: 275.8 +rank7 [2026-10-29 04:36:01,552] INFO : Step 24600/50000 rank7 | Train Loss: 1.2474 | Train Loss Unweighted: 0.6241 | Train Loss Components: Auxiliary: 0.2689 | Cardinality: 0.1630 | Classification: 0.1074 | Classification_binary: 0.2102 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.2424 | Regression_cos_phi: 0.1155 | Regression_energy: 0.0796 | Regression_eta: 0.0604 | Regression_pt: 0.0838 | Regression_sin_phi: 0.1052 | LR: 5.61e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5136s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7876 | max_loss: 0.1528 | max_backward: 1.7437 | max_optimizer_step: 0.2329 | max_tensorboard_logging: 0.6204 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0460 | Memory: rss_MB: 7358.2 | uss_MB: 6700.7 | child_rss_MB: 5006.8 | allocated_MB: 344.3 | reserved_MB: 23864.0 | inactive_split_MB: 285.7 +rank7 [2026-10-29 04:39:51,298] INFO : Step 24700/50000 rank7 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1193 | Classification: 0.0990 | Classification_binary: 0.1723 | HitGrouping: 0.2019 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1471 | Regression_eta: 0.0016 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0891 | Regression_eta: 0.0570 | Regression_pt: 0.0949 | Regression_sin_phi: 0.0745 | LR: 5.58e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9508s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0065 | max_forward: 0.6978 | max_loss: 0.2493 | max_backward: 1.2925 | max_optimizer_step: 0.2314 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7361.3 | uss_MB: 6700.3 | child_rss_MB: 5009.8 | allocated_MB: 344.1 | reserved_MB: 23864.0 | inactive_split_MB: 223.9 +rank7 [2026-10-29 04:43:37,808] INFO : Step 24800/50000 rank7 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5554 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1264 | Classification: 0.0932 | Classification_binary: 0.1583 | HitGrouping: 0.2054 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0016 | Regression_pt: 0.1503 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1438 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0797 | Regression_eta: 0.0396 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0611 | LR: 5.56e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.2773s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0062 | max_forward: 0.6582 | max_loss: 0.2533 | max_backward: 1.5858 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.6923 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.1086 | Memory: rss_MB: 7365.2 | uss_MB: 6703.3 | child_rss_MB: 5005.7 | allocated_MB: 342.9 | reserved_MB: 23864.0 | inactive_split_MB: 278.6 +rank7 [2026-10-29 04:47:36,372] INFO : Step 24900/50000 rank7 | Train Loss: 1.1348 | Train Loss Unweighted: 0.5816 | Train Loss Components: Auxiliary: 0.2466 | Cardinality: 0.1381 | Classification: 0.1126 | Classification_binary: 0.1845 | HitGrouping: 0.1937 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1403 | Regression_eta: 0.0015 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0667 | Regression_eta: 0.0350 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0446 | LR: 5.53e-04 | DataLoad Time: 0.0154s | Model Forward Time: 8.7035s | Phase Times: max_pre_step_barrier: 0.1112 | max_batch_to_device: 0.0067 | max_forward: 0.7276 | max_loss: 0.2411 | max_backward: 1.3958 | max_optimizer_step: 0.2268 | max_tensorboard_logging: 5.1754 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1038 | Memory: rss_MB: 7366.2 | uss_MB: 6703.5 | child_rss_MB: 5007.2 | allocated_MB: 343.9 | reserved_MB: 23864.0 | inactive_split_MB: 239.7 +rank7 [2026-10-29 04:51:28,373] INFO : Step 25000/50000 rank7 | Train Loss: 1.1995 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1590 | Classification: 0.0891 | Classification_binary: 0.2009 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1421 | Regression_eta: 0.0018 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.2133 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0683 | Regression_eta: 0.0637 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0821 | LR: 5.50e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2046s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7692 | max_loss: 0.2091 | max_backward: 1.3616 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6652 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1392 | Memory: rss_MB: 7366.4 | uss_MB: 6702.5 | child_rss_MB: 5014.2 | allocated_MB: 343.5 | reserved_MB: 23864.0 | inactive_split_MB: 293.8 +rank7 [2026-10-29 04:51:29,310] INFO : Running validation on rank7 +rank7 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 04:51:29,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 04:52:22,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:52:22,822] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 04:52:49,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 04:52:50,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 04:52:51,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 04:52:52,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 04:53:02,147] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:53:02,764] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:53:03,282] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 04:53:19,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 04:53:20,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 04:53:21,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 04:53:22,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 04:53:32,272] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:53:32,528] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:53:32,991] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 04:53:49,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 04:53:49,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 04:53:50,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 04:53:51,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_25000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 04:54:01,084] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 04:58:07,812] INFO : Step 25100/50000 rank7 | Train Loss: 1.0833 | Train Loss Unweighted: 0.5514 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1317 | Classification: 0.0920 | Classification_binary: 0.1568 | HitGrouping: 0.2008 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1491 | Regression_eta: 0.0016 | Regression_pt: 0.1506 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0656 | Regression_eta: 0.0338 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0774 | LR: 5.47e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.7895s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 0.7295 | max_loss: 0.1401 | max_backward: 1.2439 | max_optimizer_step: 0.2490 | max_tensorboard_logging: 0.4940 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 8607.6 | uss_MB: 7944.9 | child_rss_MB: 5042.3 | allocated_MB: 342.3 | reserved_MB: 23608.0 | inactive_split_MB: 222.3 +rank7 [2026-10-29 05:01:53,205] INFO : Step 25200/50000 rank7 | Train Loss: 1.1886 | Train Loss Unweighted: 0.6033 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1523 | Classification: 0.1158 | Classification_binary: 0.1800 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1512 | Regression_eta: 0.0016 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0350 | Regression_energy: 0.0694 | Regression_eta: 0.0373 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0620 | LR: 5.44e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7326s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.8134 | max_loss: 0.1416 | max_backward: 1.5095 | max_optimizer_step: 0.3471 | max_tensorboard_logging: 0.5759 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0803 | Memory: rss_MB: 8606.1 | uss_MB: 7945.9 | child_rss_MB: 5039.5 | allocated_MB: 343.6 | reserved_MB: 24590.0 | inactive_split_MB: 260.9 +rank7 [2026-10-29 05:05:52,267] INFO : Step 25300/50000 rank7 | Train Loss: 1.2031 | Train Loss Unweighted: 0.6064 | Train Loss Components: Auxiliary: 0.2624 | Cardinality: 0.1455 | Classification: 0.1145 | Classification_binary: 0.1966 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1448 | Regression_eta: 0.0016 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.1938 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0790 | Regression_eta: 0.0520 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0603 | LR: 5.42e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.5365s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0061 | max_forward: 0.8219 | max_loss: 0.1491 | max_backward: 1.6406 | max_optimizer_step: 0.2735 | max_tensorboard_logging: 0.7204 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0488 | Memory: rss_MB: 7456.6 | uss_MB: 6796.5 | child_rss_MB: 5036.7 | allocated_MB: 341.8 | reserved_MB: 25572.0 | inactive_split_MB: 253.5 +rank7 [2026-10-29 05:09:44,971] INFO : Step 25400/50000 rank7 | Train Loss: 1.1194 | Train Loss Unweighted: 0.5655 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1346 | Classification: 0.0983 | Classification_binary: 0.1754 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1438 | Regression_eta: 0.0015 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1940 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0626 | Regression_eta: 0.0554 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0966 | LR: 5.39e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.1080s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 0.6986 | max_loss: 0.1530 | max_backward: 1.2606 | max_optimizer_step: 0.2325 | max_tensorboard_logging: 0.8427 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7454.9 | uss_MB: 6795.9 | child_rss_MB: 5040.6 | allocated_MB: 345.2 | reserved_MB: 25572.0 | inactive_split_MB: 223.8 +rank7 [2026-10-29 05:13:34,055] INFO : Step 25500/50000 rank7 | Train Loss: 1.0289 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1068 | Classification: 0.1034 | Classification_binary: 0.1444 | HitGrouping: 0.1913 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1484 | Regression_eta: 0.0014 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1270 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0586 | Regression_eta: 0.0285 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0513 | LR: 5.36e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1586s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0554 | max_forward: 0.7008 | max_loss: 0.1433 | max_backward: 1.4571 | max_optimizer_step: 0.3349 | max_tensorboard_logging: 0.6083 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 7460.4 | uss_MB: 6802.0 | child_rss_MB: 5038.5 | allocated_MB: 345.4 | reserved_MB: 25572.0 | inactive_split_MB: 240.6 +rank7 [2026-10-29 05:17:32,215] INFO : Step 25600/50000 rank7 | Train Loss: 1.2028 | Train Loss Unweighted: 0.5968 | Train Loss Components: Auxiliary: 0.2590 | Cardinality: 0.1607 | Classification: 0.1135 | Classification_binary: 0.1926 | HitGrouping: 0.1978 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1429 | Regression_eta: 0.0016 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1729 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0720 | Regression_eta: 0.0499 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0597 | LR: 5.33e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6838s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8800 | max_loss: 0.1578 | max_backward: 1.8564 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 7459.9 | uss_MB: 6802.4 | child_rss_MB: 5035.4 | allocated_MB: 345.5 | reserved_MB: 26054.0 | inactive_split_MB: 284.5 +rank7 [2026-10-29 05:21:26,002] INFO : Step 25700/50000 rank7 | Train Loss: 1.1800 | Train Loss Unweighted: 0.5964 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1298 | Classification: 0.1086 | Classification_binary: 0.1961 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0019 | Regression_pt: 0.1444 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0799 | Classification_binary: 0.2422 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0668 | Regression_eta: 0.0601 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0682 | LR: 5.30e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5196s | Phase Times: max_pre_step_barrier: 0.0086 | max_batch_to_device: 0.0137 | max_forward: 1.0340 | max_loss: 0.2483 | max_backward: 1.6704 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 0.7612 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7459.7 | uss_MB: 6801.4 | child_rss_MB: 5033.4 | allocated_MB: 343.5 | reserved_MB: 26054.0 | inactive_split_MB: 270.8 +rank7 [2026-10-29 05:25:16,515] INFO : Step 25800/50000 rank7 | Train Loss: 1.1337 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2388 | Cardinality: 0.1264 | Classification: 0.0898 | Classification_binary: 0.1617 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1589 | Regression_eta: 0.0019 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1949 | Regression_cos_phi: 0.0718 | Regression_energy: 0.1021 | Regression_eta: 0.0567 | Regression_pt: 0.1043 | Regression_sin_phi: 0.0670 | LR: 5.27e-04 | DataLoad Time: 0.0213s | Model Forward Time: 3.9365s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0986 | max_forward: 0.8129 | max_loss: 0.1662 | max_backward: 1.2979 | max_optimizer_step: 0.2342 | max_tensorboard_logging: 0.4920 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1103 | Memory: rss_MB: 7457.8 | uss_MB: 6799.8 | child_rss_MB: 5030.2 | allocated_MB: 341.8 | reserved_MB: 26054.0 | inactive_split_MB: 273.2 +rank7 [2026-10-29 05:29:08,211] INFO : Step 25900/50000 rank7 | Train Loss: 1.1019 | Train Loss Unweighted: 0.5481 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1165 | Classification: 0.0977 | Classification_binary: 0.1594 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1434 | Regression_eta: 0.0016 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0374 | Regression_energy: 0.0667 | Regression_eta: 0.0249 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0560 | LR: 5.25e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5198s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.7034 | max_loss: 0.2469 | max_backward: 1.7130 | max_optimizer_step: 0.2363 | max_tensorboard_logging: 0.8382 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 7454.8 | uss_MB: 6800.6 | child_rss_MB: 5037.1 | allocated_MB: 343.0 | reserved_MB: 26054.0 | inactive_split_MB: 187.0 +rank7 [2026-10-29 05:33:07,469] INFO : Step 26000/50000 rank7 | Train Loss: 1.1708 | Train Loss Unweighted: 0.5680 | Train Loss Components: Auxiliary: 0.2498 | Cardinality: 0.1352 | Classification: 0.0999 | Classification_binary: 0.2021 | HitGrouping: 0.1994 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1304 | Regression_eta: 0.0021 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.2127 | Regression_cos_phi: 0.0873 | Regression_energy: 0.0595 | Regression_eta: 0.0706 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0796 | LR: 5.22e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2243s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.7877 | max_loss: 0.1637 | max_backward: 1.6550 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.4954 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 7454.4 | uss_MB: 6802.1 | child_rss_MB: 5032.6 | allocated_MB: 345.0 | reserved_MB: 27098.0 | inactive_split_MB: 285.6 +rank7 [2026-10-29 05:33:08,394] INFO : Running validation on rank7 +rank7 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 05:33:08,394] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 05:33:56,766] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:33:57,246] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 05:34:24,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 05:34:25,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 05:34:26,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 05:34:26,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 05:34:36,925] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 05:34:38,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:34:38,538] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 05:34:55,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 05:34:55,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 05:34:56,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 05:34:57,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 05:35:07,790] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 05:35:08,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:35:08,510] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 05:35:24,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 05:35:25,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 05:35:25,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 05:35:26,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_26000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 05:35:36,584] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 05:39:49,628] INFO : Step 26100/50000 rank7 | Train Loss: 1.0974 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2307 | Cardinality: 0.1119 | Classification: 0.0956 | Classification_binary: 0.1689 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1513 | Regression_eta: 0.0014 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.1709 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0583 | Regression_eta: 0.0296 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0497 | LR: 5.19e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1616s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0117 | max_forward: 0.7200 | max_loss: 0.1501 | max_backward: 1.2286 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.8284 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0613 | Memory: rss_MB: 8630.6 | uss_MB: 7973.7 | child_rss_MB: 5008.3 | allocated_MB: 340.1 | reserved_MB: 21762.0 | inactive_split_MB: 215.7 +rank7 [2026-10-29 05:43:56,539] INFO : Step 26200/50000 rank7 | Train Loss: 1.0876 | Train Loss Unweighted: 0.5502 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1065 | Classification: 0.0968 | Classification_binary: 0.1527 | HitGrouping: 0.2042 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1476 | Regression_eta: 0.0018 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0593 | Regression_eta: 0.0652 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0704 | LR: 5.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3250s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6530 | max_loss: 0.1421 | max_backward: 1.3973 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.8695 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0570 | Memory: rss_MB: 8630.6 | uss_MB: 7973.8 | child_rss_MB: 5031.7 | allocated_MB: 342.3 | reserved_MB: 22678.0 | inactive_split_MB: 242.5 +rank7 [2026-10-29 05:47:48,030] INFO : Step 26300/50000 rank7 | Train Loss: 1.2051 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2585 | Cardinality: 0.1512 | Classification: 0.1193 | Classification_binary: 0.1877 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1458 | Regression_eta: 0.0017 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.1964 | Regression_cos_phi: 0.0777 | Regression_energy: 0.0840 | Regression_eta: 0.0498 | Regression_pt: 0.0900 | Regression_sin_phi: 0.0737 | LR: 5.13e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4685s | Phase Times: max_pre_step_barrier: 0.1111 | max_batch_to_device: 0.0072 | max_forward: 0.7601 | max_loss: 0.2267 | max_backward: 1.8902 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.4853 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 7522.0 | uss_MB: 6871.8 | child_rss_MB: 4996.1 | allocated_MB: 345.4 | reserved_MB: 22678.0 | inactive_split_MB: 270.9 +rank7 [2026-10-29 05:51:47,278] INFO : Step 26400/50000 rank7 | Train Loss: 1.2256 | Train Loss Unweighted: 0.6081 | Train Loss Components: Auxiliary: 0.2628 | Cardinality: 0.1648 | Classification: 0.1020 | Classification_binary: 0.1981 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1520 | Regression_eta: 0.0015 | Regression_pt: 0.1526 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1849 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0722 | Regression_eta: 0.0391 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0922 | LR: 5.10e-04 | DataLoad Time: 0.0139s | Model Forward Time: 4.3865s | Phase Times: max_pre_step_barrier: 0.0351 | max_batch_to_device: 0.0070 | max_forward: 0.8129 | max_loss: 0.1477 | max_backward: 1.4015 | max_optimizer_step: 0.2357 | max_tensorboard_logging: 0.8384 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 7520.2 | uss_MB: 6869.2 | child_rss_MB: 4985.0 | allocated_MB: 342.8 | reserved_MB: 22678.0 | inactive_split_MB: 196.9 +rank7 [2026-10-29 05:55:40,759] INFO : Step 26500/50000 rank7 | Train Loss: 1.1110 | Train Loss Unweighted: 0.5356 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1488 | Classification: 0.0891 | Classification_binary: 0.1546 | HitGrouping: 0.2012 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1829 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0640 | Regression_eta: 0.0566 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0872 | LR: 5.08e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0225s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.6841 | max_loss: 0.2421 | max_backward: 1.1616 | max_optimizer_step: 0.2383 | max_tensorboard_logging: 0.8648 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0535 | Memory: rss_MB: 7505.4 | uss_MB: 6872.6 | child_rss_MB: 4957.6 | allocated_MB: 340.6 | reserved_MB: 22678.0 | inactive_split_MB: 269.9 +rank7 [2026-10-29 05:59:33,069] INFO : Step 26600/50000 rank7 | Train Loss: 1.0895 | Train Loss Unweighted: 0.5529 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1261 | Classification: 0.1083 | Classification_binary: 0.1603 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1386 | Regression_eta: 0.0015 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0709 | Regression_eta: 0.0429 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0592 | LR: 5.05e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4539s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0696 | max_forward: 0.6926 | max_loss: 0.1459 | max_backward: 1.4149 | max_optimizer_step: 0.2385 | max_tensorboard_logging: 0.8927 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7477.7 | uss_MB: 6873.1 | child_rss_MB: 4831.1 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 232.0 +rank7 [2026-10-29 06:03:23,754] INFO : Step 26700/50000 rank7 | Train Loss: 1.2592 | Train Loss Unweighted: 0.6131 | Train Loss Components: Auxiliary: 0.2727 | Cardinality: 0.1717 | Classification: 0.1100 | Classification_binary: 0.2137 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1420 | Regression_eta: 0.0017 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.1928 | Regression_cos_phi: 0.1187 | Regression_energy: 0.0608 | Regression_eta: 0.0694 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0654 | LR: 5.02e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.6749s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7750 | max_loss: 0.1526 | max_backward: 1.6427 | max_optimizer_step: 0.2461 | max_tensorboard_logging: 0.8866 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 7338.5 | uss_MB: 6135.7 | child_rss_MB: 4788.2 | allocated_MB: 345.0 | reserved_MB: 22846.0 | inactive_split_MB: 267.4 +rank7 [2026-10-29 06:07:16,645] INFO : Step 26800/50000 rank7 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5792 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1288 | Classification: 0.0911 | Classification_binary: 0.1633 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1600 | Regression_eta: 0.0015 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.1822 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0772 | Regression_eta: 0.0586 | Regression_pt: 0.0804 | Regression_sin_phi: 0.1061 | LR: 4.99e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.0433s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7108 | max_loss: 0.1467 | max_backward: 1.2855 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7173 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0761 | Memory: rss_MB: 7253.0 | uss_MB: 6872.2 | child_rss_MB: 4809.0 | allocated_MB: 340.8 | reserved_MB: 22846.0 | inactive_split_MB: 253.2 +rank7 [2026-10-29 06:11:09,652] INFO : Step 26900/50000 rank7 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5310 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.0996 | Classification: 0.0867 | Classification_binary: 0.1428 | HitGrouping: 0.1962 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1483 | Regression_eta: 0.0018 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0684 | Regression_eta: 0.0911 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0727 | LR: 4.96e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8402s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0336 | max_forward: 0.6534 | max_loss: 0.1408 | max_backward: 1.3898 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 0.4807 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0912 | Memory: rss_MB: 7214.0 | uss_MB: 6872.7 | child_rss_MB: 4790.0 | allocated_MB: 342.5 | reserved_MB: 22846.0 | inactive_split_MB: 233.5 +rank7 [2026-10-29 06:15:05,368] INFO : Step 27000/50000 rank7 | Train Loss: 1.0945 | Train Loss Unweighted: 0.5396 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1196 | Classification: 0.0957 | Classification_binary: 0.1912 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1247 | Regression_eta: 0.0015 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1769 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0572 | Regression_eta: 0.0537 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0482 | LR: 4.94e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1531s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7531 | max_loss: 0.1689 | max_backward: 1.5554 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6017 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0500 | Memory: rss_MB: 7214.1 | uss_MB: 6877.0 | child_rss_MB: 4792.2 | allocated_MB: 345.3 | reserved_MB: 23694.0 | inactive_split_MB: 251.5 +rank7 [2026-10-29 06:15:06,393] INFO : Running validation on rank7 +rank7 [2026-10-29 06:15:06,393] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 06:15:06,393] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 06:16:00,133] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:16:00,711] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 06:16:29,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 06:16:30,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 06:16:31,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 06:16:32,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 06:16:42,485] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 06:16:43,160] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:16:43,720] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 06:17:00,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 06:17:00,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 06:17:01,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 06:17:02,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 06:17:12,796] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 06:17:13,249] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:17:13,778] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 06:17:29,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 06:17:30,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 06:17:30,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 06:17:31,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_27000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 06:17:41,587] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 06:21:51,976] INFO : Step 27100/50000 rank7 | Train Loss: 1.1577 | Train Loss Unweighted: 0.5595 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1497 | Classification: 0.0884 | Classification_binary: 0.1879 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1399 | Regression_eta: 0.0018 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1827 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0661 | Regression_eta: 0.0512 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0553 | LR: 4.91e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.0521s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.8543 | max_loss: 0.1477 | max_backward: 1.8762 | max_optimizer_step: 0.2540 | max_tensorboard_logging: 0.9462 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 8306.7 | uss_MB: 7966.7 | child_rss_MB: 4814.1 | allocated_MB: 342.9 | reserved_MB: 21640.0 | inactive_split_MB: 268.0 +rank7 [2026-10-29 06:25:39,974] INFO : Step 27200/50000 rank7 | Train Loss: 1.0028 | Train Loss Unweighted: 0.5117 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1065 | Classification: 0.0834 | Classification_binary: 0.1354 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1443 | Regression_eta: 0.0015 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1256 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0696 | Regression_eta: 0.0343 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0603 | LR: 4.88e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.0907s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.6839 | max_loss: 0.1724 | max_backward: 1.0628 | max_optimizer_step: 0.2515 | max_tensorboard_logging: 0.9641 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0689 | Memory: rss_MB: 8305.7 | uss_MB: 7964.4 | child_rss_MB: 4815.8 | allocated_MB: 340.5 | reserved_MB: 22612.0 | inactive_split_MB: 267.2 +rank7 [2026-10-29 06:29:18,757] INFO : Step 27300/50000 rank7 | Train Loss: 1.1913 | Train Loss Unweighted: 0.5857 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1563 | Classification: 0.1169 | Classification_binary: 0.1769 | HitGrouping: 0.1965 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1426 | Regression_eta: 0.0019 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.1749 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0756 | Regression_eta: 0.0513 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0918 | LR: 4.85e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.2521s | Phase Times: max_pre_step_barrier: 0.0244 | max_batch_to_device: 0.0064 | max_forward: 0.6813 | max_loss: 0.1454 | max_backward: 1.4618 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.7699 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0503 | Memory: rss_MB: 7290.8 | uss_MB: 6952.0 | child_rss_MB: 4811.5 | allocated_MB: 345.6 | reserved_MB: 23586.0 | inactive_split_MB: 267.9 +rank7 [2026-10-29 06:33:03,427] INFO : Step 27400/50000 rank7 | Train Loss: 1.1723 | Train Loss Unweighted: 0.5888 | Train Loss Components: Auxiliary: 0.2572 | Cardinality: 0.1400 | Classification: 0.1088 | Classification_binary: 0.2016 | HitGrouping: 0.1987 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1370 | Regression_eta: 0.0017 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0895 | Classification_binary: 0.2185 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0719 | Regression_eta: 0.0549 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0610 | LR: 4.82e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6649s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8048 | max_loss: 0.1475 | max_backward: 1.6655 | max_optimizer_step: 0.2279 | max_tensorboard_logging: 0.8939 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7292.0 | uss_MB: 6951.9 | child_rss_MB: 4814.0 | allocated_MB: 344.5 | reserved_MB: 23586.0 | inactive_split_MB: 279.5 +rank7 [2026-10-29 06:37:01,482] INFO : Step 27500/50000 rank7 | Train Loss: 1.0739 | Train Loss Unweighted: 0.5273 | Train Loss Components: Auxiliary: 0.2251 | Cardinality: 0.1251 | Classification: 0.0933 | Classification_binary: 0.1655 | HitGrouping: 0.1957 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1328 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0666 | Regression_eta: 0.0486 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0648 | LR: 4.80e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2212s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7686 | max_loss: 0.1542 | max_backward: 1.3750 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.8628 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0458 | Memory: rss_MB: 7288.8 | uss_MB: 6945.7 | child_rss_MB: 4804.4 | allocated_MB: 343.6 | reserved_MB: 23586.0 | inactive_split_MB: 241.4 +rank7 [2026-10-29 06:40:55,522] INFO : Step 27600/50000 rank7 | Train Loss: 1.0507 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1163 | Classification: 0.1024 | Classification_binary: 0.1467 | HitGrouping: 0.1992 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1324 | Regression_eta: 0.0017 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0544 | Regression_eta: 0.0304 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0814 | LR: 4.77e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4909s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0060 | max_forward: 0.6744 | max_loss: 0.2492 | max_backward: 1.6867 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.5257 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0558 | Memory: rss_MB: 7290.8 | uss_MB: 6853.7 | child_rss_MB: 4808.0 | allocated_MB: 341.2 | reserved_MB: 24480.0 | inactive_split_MB: 234.8 +rank7 [2026-10-29 06:44:47,871] INFO : Step 27700/50000 rank7 | Train Loss: 1.1881 | Train Loss Unweighted: 0.5834 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1544 | Classification: 0.1001 | Classification_binary: 0.1885 | HitGrouping: 0.1970 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1444 | Regression_eta: 0.0015 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1855 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0618 | Regression_eta: 0.0340 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0734 | LR: 4.74e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.2904s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0062 | max_forward: 0.7716 | max_loss: 0.1527 | max_backward: 1.4941 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 0.7272 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0469 | Memory: rss_MB: 7291.1 | uss_MB: 6952.7 | child_rss_MB: 4804.6 | allocated_MB: 344.5 | reserved_MB: 24480.0 | inactive_split_MB: 194.2 +rank7 [2026-10-29 06:48:38,929] INFO : Step 27800/50000 rank7 | Train Loss: 1.1903 | Train Loss Unweighted: 0.5862 | Train Loss Components: Auxiliary: 0.2553 | Cardinality: 0.1435 | Classification: 0.1032 | Classification_binary: 0.2006 | HitGrouping: 0.1991 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1387 | Regression_eta: 0.0018 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.2066 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0604 | Regression_eta: 0.0365 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0503 | LR: 4.71e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1892s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.8137 | max_loss: 0.1509 | max_backward: 1.4748 | max_optimizer_step: 0.2334 | max_tensorboard_logging: 0.6041 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0529 | Memory: rss_MB: 7291.1 | uss_MB: 6953.1 | child_rss_MB: 4808.7 | allocated_MB: 348.4 | reserved_MB: 24592.0 | inactive_split_MB: 226.8 +rank7 [2026-10-29 06:52:39,106] INFO : Step 27900/50000 rank7 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5521 | Train Loss Components: Auxiliary: 0.2284 | Cardinality: 0.1139 | Classification: 0.0902 | Classification_binary: 0.1544 | HitGrouping: 0.2057 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1519 | Regression_eta: 0.0017 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0684 | Regression_eta: 0.0409 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0519 | LR: 4.68e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6327s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.6397 | max_loss: 0.2456 | max_backward: 1.5234 | max_optimizer_step: 0.2276 | max_tensorboard_logging: 1.1626 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0427 | Memory: rss_MB: 7291.1 | uss_MB: 6955.2 | child_rss_MB: 4805.2 | allocated_MB: 342.4 | reserved_MB: 25612.0 | inactive_split_MB: 259.0 +rank7 [2026-10-29 06:56:30,759] INFO : Step 28000/50000 rank7 | Train Loss: 1.1111 | Train Loss Unweighted: 0.5401 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1274 | Classification: 0.1035 | Classification_binary: 0.1699 | HitGrouping: 0.2004 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1674 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0641 | Regression_eta: 0.0441 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0752 | LR: 4.66e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.0939s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.7503 | max_loss: 0.1492 | max_backward: 1.7566 | max_optimizer_step: 0.2764 | max_tensorboard_logging: 1.2295 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0339 | Memory: rss_MB: 7291.2 | uss_MB: 5984.8 | child_rss_MB: 4794.8 | allocated_MB: 344.1 | reserved_MB: 25612.0 | inactive_split_MB: 243.2 +rank7 [2026-10-29 06:56:31,846] INFO : Running validation on rank7 +rank7 [2026-10-29 06:56:31,846] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 06:56:31,846] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 06:57:24,999] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:57:25,560] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 06:57:52,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 06:57:53,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 06:57:54,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 06:57:54,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 06:58:04,896] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 06:58:05,603] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:58:06,180] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 06:58:22,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 06:58:23,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 06:58:23,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 06:58:24,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 06:58:34,833] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 06:58:35,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:58:35,800] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 06:58:51,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 06:58:52,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 06:58:52,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 06:58:53,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_28000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 06:59:03,674] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 07:03:11,251] INFO : Step 28100/50000 rank7 | Train Loss: 1.1727 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1644 | Classification: 0.0934 | Classification_binary: 0.1929 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1365 | Regression_eta: 0.0015 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1901 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0661 | Regression_eta: 0.0430 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0559 | LR: 4.63e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.9506s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8425 | max_loss: 0.2571 | max_backward: 1.8157 | max_optimizer_step: 0.2529 | max_tensorboard_logging: 1.0134 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0693 | Memory: rss_MB: 8340.2 | uss_MB: 8005.6 | child_rss_MB: 4802.9 | allocated_MB: 344.8 | reserved_MB: 22140.0 | inactive_split_MB: 293.2 +rank7 [2026-10-29 07:07:01,163] INFO : Step 28200/50000 rank7 | Train Loss: 1.0520 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2296 | Cardinality: 0.1203 | Classification: 0.0934 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1466 | Regression_eta: 0.0014 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1804 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0582 | Regression_eta: 0.0370 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0619 | LR: 4.60e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3560s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7943 | max_loss: 0.1496 | max_backward: 1.4503 | max_optimizer_step: 0.2472 | max_tensorboard_logging: 0.7872 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0669 | Memory: rss_MB: 8343.2 | uss_MB: 8006.4 | child_rss_MB: 4796.0 | allocated_MB: 341.0 | reserved_MB: 24034.0 | inactive_split_MB: 210.4 +rank7 [2026-10-29 07:10:52,728] INFO : Step 28300/50000 rank7 | Train Loss: 1.0602 | Train Loss Unweighted: 0.5453 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1043 | Classification: 0.0923 | Classification_binary: 0.1501 | HitGrouping: 0.2005 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1496 | Regression_eta: 0.0016 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0728 | Regression_eta: 0.0676 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0610 | LR: 4.57e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2447s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.6482 | max_loss: 0.2266 | max_backward: 1.5658 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.6870 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0836 | Memory: rss_MB: 7393.0 | uss_MB: 7059.5 | child_rss_MB: 4787.3 | allocated_MB: 342.7 | reserved_MB: 25016.0 | inactive_split_MB: 243.9 +rank7 [2026-10-29 07:14:40,575] INFO : Step 28400/50000 rank7 | Train Loss: 1.1507 | Train Loss Unweighted: 0.5391 | Train Loss Components: Auxiliary: 0.2436 | Cardinality: 0.1500 | Classification: 0.0965 | Classification_binary: 0.1896 | HitGrouping: 0.1911 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1241 | Regression_eta: 0.0018 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1685 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0536 | Regression_eta: 0.0578 | Regression_pt: 0.0551 | Regression_sin_phi: 0.1067 | LR: 4.55e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6304s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7664 | max_loss: 0.1532 | max_backward: 1.7342 | max_optimizer_step: 0.2366 | max_tensorboard_logging: 0.8455 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7394.0 | uss_MB: 7059.9 | child_rss_MB: 4785.1 | allocated_MB: 344.2 | reserved_MB: 25016.0 | inactive_split_MB: 278.9 +rank7 [2026-10-29 07:18:29,002] INFO : Step 28500/50000 rank7 | Train Loss: 1.1595 | Train Loss Unweighted: 0.5640 | Train Loss Components: Auxiliary: 0.2492 | Cardinality: 0.1556 | Classification: 0.0927 | Classification_binary: 0.1908 | HitGrouping: 0.1954 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1384 | Regression_eta: 0.0015 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.2096 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0675 | Regression_eta: 0.0527 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0752 | LR: 4.52e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2733s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0080 | max_forward: 0.8033 | max_loss: 0.1665 | max_backward: 1.5231 | max_optimizer_step: 0.2389 | max_tensorboard_logging: 0.6178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0709 | Memory: rss_MB: 7393.5 | uss_MB: 7056.4 | child_rss_MB: 4808.7 | allocated_MB: 347.4 | reserved_MB: 26000.0 | inactive_split_MB: 283.2 +rank7 [2026-10-29 07:22:17,774] INFO : Step 28600/50000 rank7 | Train Loss: 1.0672 | Train Loss Unweighted: 0.5222 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1172 | Classification: 0.0893 | Classification_binary: 0.1544 | HitGrouping: 0.2051 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1645 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0635 | Regression_eta: 0.0559 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0631 | LR: 4.49e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0230s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.6704 | max_loss: 0.1829 | max_backward: 1.2353 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.8074 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7396.5 | uss_MB: 7057.8 | child_rss_MB: 4793.1 | allocated_MB: 342.3 | reserved_MB: 26000.0 | inactive_split_MB: 243.7 +rank7 [2026-10-29 07:26:10,155] INFO : Step 28700/50000 rank7 | Train Loss: 1.1535 | Train Loss Unweighted: 0.5811 | Train Loss Components: Auxiliary: 0.2469 | Cardinality: 0.1254 | Classification: 0.1220 | Classification_binary: 0.1661 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1442 | Regression_eta: 0.0021 | Regression_pt: 0.1450 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0686 | Regression_eta: 0.0615 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0568 | LR: 4.46e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5821s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.6905 | max_loss: 0.2072 | max_backward: 1.6876 | max_optimizer_step: 0.2353 | max_tensorboard_logging: 0.9405 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0493 | Memory: rss_MB: 7397.5 | uss_MB: 7064.1 | child_rss_MB: 4802.4 | allocated_MB: 343.5 | reserved_MB: 26000.0 | inactive_split_MB: 234.1 +rank7 [2026-10-29 07:30:12,141] INFO : Step 28800/50000 rank7 | Train Loss: 1.2131 | Train Loss Unweighted: 0.5739 | Train Loss Components: Auxiliary: 0.2574 | Cardinality: 0.1788 | Classification: 0.1057 | Classification_binary: 0.2127 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1857 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0622 | Regression_eta: 0.0478 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0735 | LR: 4.44e-04 | DataLoad Time: 0.0114s | Model Forward Time: 6.8412s | Phase Times: max_pre_step_barrier: 0.0263 | max_batch_to_device: 0.0149 | max_forward: 1.2928 | max_loss: 0.2713 | max_backward: 3.0161 | max_optimizer_step: 0.2675 | max_tensorboard_logging: 0.8895 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0657 | Memory: rss_MB: 7398.3 | uss_MB: 5722.3 | child_rss_MB: 4796.9 | allocated_MB: 343.8 | reserved_MB: 26000.0 | inactive_split_MB: 273.3 +rank7 [2026-10-29 07:34:11,570] INFO : Step 28900/50000 rank7 | Train Loss: 1.1052 | Train Loss Unweighted: 0.5690 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1252 | Classification: 0.0967 | Classification_binary: 0.1637 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1509 | Regression_eta: 0.0015 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.1668 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0756 | Regression_eta: 0.0412 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0752 | LR: 4.41e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.1522s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7512 | max_loss: 0.2189 | max_backward: 1.6229 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.9330 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7399.6 | uss_MB: 5634.1 | child_rss_MB: 4800.1 | allocated_MB: 339.6 | reserved_MB: 26000.0 | inactive_split_MB: 223.0 +rank7 [2026-10-29 07:38:06,080] INFO : Step 29000/50000 rank7 | Train Loss: 1.0238 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1070 | Classification: 0.0861 | Classification_binary: 0.1439 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1389 | Regression_eta: 0.0014 | Regression_pt: 0.1414 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0441 | Regression_energy: 0.0757 | Regression_eta: 0.0335 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0757 | LR: 4.38e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.9819s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.6636 | max_loss: 0.2429 | max_backward: 1.5394 | max_optimizer_step: 0.2367 | max_tensorboard_logging: 0.5216 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7400.5 | uss_MB: 7067.2 | child_rss_MB: 4794.8 | allocated_MB: 345.2 | reserved_MB: 26984.0 | inactive_split_MB: 235.2 +rank7 [2026-10-29 07:38:07,030] INFO : Running validation on rank7 +rank7 [2026-10-29 07:38:07,030] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 07:38:07,030] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 07:38:57,930] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:38:58,451] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 07:39:24,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 07:39:25,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 07:39:26,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 07:39:27,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 07:39:37,248] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 07:39:38,026] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:39:38,577] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 07:39:54,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 07:39:55,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 07:39:56,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 07:39:57,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 07:40:07,268] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 07:40:07,779] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:40:08,277] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 07:40:24,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 07:40:24,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 07:40:25,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 07:40:26,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_29000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 07:40:36,268] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 07:45:01,998] INFO : Step 29100/50000 rank7 | Train Loss: 1.1570 | Train Loss Unweighted: 0.5808 | Train Loss Components: Auxiliary: 0.2465 | Cardinality: 0.1279 | Classification: 0.1113 | Classification_binary: 0.1973 | HitGrouping: 0.1972 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1339 | Regression_eta: 0.0017 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1817 | Regression_cos_phi: 0.0867 | Regression_energy: 0.0779 | Regression_eta: 0.0485 | Regression_pt: 0.0806 | Regression_sin_phi: 0.0707 | LR: 4.35e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8728s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.8060 | max_loss: 0.1503 | max_backward: 1.9210 | max_optimizer_step: 0.2533 | max_tensorboard_logging: 0.8167 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0627 | Memory: rss_MB: 8379.4 | uss_MB: 8043.2 | child_rss_MB: 4787.2 | allocated_MB: 344.6 | reserved_MB: 24404.0 | inactive_split_MB: 274.9 +rank7 [2026-10-29 07:48:58,055] INFO : Step 29200/50000 rank7 | Train Loss: 1.1137 | Train Loss Unweighted: 0.5633 | Train Loss Components: Auxiliary: 0.2432 | Cardinality: 0.1265 | Classification: 0.0969 | Classification_binary: 0.1883 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1368 | Regression_eta: 0.0014 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0607 | Regression_eta: 0.0366 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0777 | LR: 4.33e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8379s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0742 | max_forward: 1.0303 | max_loss: 0.2183 | max_backward: 1.8745 | max_optimizer_step: 0.2501 | max_tensorboard_logging: 0.5570 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8379.4 | uss_MB: 8042.7 | child_rss_MB: 4798.0 | allocated_MB: 343.1 | reserved_MB: 24404.0 | inactive_split_MB: 228.1 +rank7 [2026-10-29 07:52:51,792] INFO : Step 29300/50000 rank7 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2330 | Cardinality: 0.1231 | Classification: 0.0898 | Classification_binary: 0.1516 | HitGrouping: 0.1998 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1428 | Regression_eta: 0.0017 | Regression_pt: 0.1453 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1756 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0743 | Regression_eta: 0.0547 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0702 | LR: 4.30e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.7358s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0067 | max_forward: 0.7139 | max_loss: 0.1578 | max_backward: 1.5096 | max_optimizer_step: 0.2586 | max_tensorboard_logging: 1.1324 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7473.7 | uss_MB: 6607.6 | child_rss_MB: 4797.1 | allocated_MB: 341.0 | reserved_MB: 24404.0 | inactive_split_MB: 247.0 +rank7 [2026-10-29 07:56:44,367] INFO : Step 29400/50000 rank7 | Train Loss: 1.1400 | Train Loss Unweighted: 0.5622 | Train Loss Components: Auxiliary: 0.2442 | Cardinality: 0.1377 | Classification: 0.1135 | Classification_binary: 0.1661 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1386 | Regression_eta: 0.0019 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0661 | Regression_eta: 0.0317 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0394 | LR: 4.27e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.2877s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.6285 | max_loss: 0.1485 | max_backward: 1.3787 | max_optimizer_step: 0.2390 | max_tensorboard_logging: 0.9179 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7474.7 | uss_MB: 7139.2 | child_rss_MB: 4788.9 | allocated_MB: 343.3 | reserved_MB: 24404.0 | inactive_split_MB: 277.1 +rank7 [2026-10-29 08:00:58,645] INFO : Step 29500/50000 rank7 | Train Loss: 1.1987 | Train Loss Unweighted: 0.6106 | Train Loss Components: Auxiliary: 0.2551 | Cardinality: 0.1277 | Classification: 0.1201 | Classification_binary: 0.2013 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1419 | Regression_eta: 0.0019 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0901 | Classification_binary: 0.2182 | Regression_cos_phi: 0.0876 | Regression_energy: 0.0678 | Regression_eta: 0.0750 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0994 | LR: 4.24e-04 | DataLoad Time: 0.0140s | Model Forward Time: 5.8995s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0749 | max_forward: 0.8153 | max_loss: 0.3465 | max_backward: 2.8565 | max_optimizer_step: 0.3442 | max_tensorboard_logging: 0.8315 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0526 | Memory: rss_MB: 7477.7 | uss_MB: 7142.6 | child_rss_MB: 4792.1 | allocated_MB: 343.9 | reserved_MB: 24404.0 | inactive_split_MB: 271.5 +rank7 [2026-10-29 08:05:08,852] INFO : Step 29600/50000 rank7 | Train Loss: 1.1065 | Train Loss Unweighted: 0.5576 | Train Loss Components: Auxiliary: 0.2391 | Cardinality: 0.1179 | Classification: 0.0927 | Classification_binary: 0.1666 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0018 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0650 | Regression_eta: 0.0533 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0613 | LR: 4.22e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4752s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.8025 | max_loss: 0.1485 | max_backward: 1.4674 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 0.8982 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 7476.5 | uss_MB: 7138.1 | child_rss_MB: 4790.4 | allocated_MB: 340.8 | reserved_MB: 24404.0 | inactive_split_MB: 290.7 +rank7 [2026-10-29 08:09:13,801] INFO : Step 29700/50000 rank7 | Train Loss: 1.0488 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2273 | Cardinality: 0.1414 | Classification: 0.0952 | Classification_binary: 0.1459 | HitGrouping: 0.1925 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1440 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0680 | Regression_eta: 0.0324 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0662 | LR: 4.19e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5178s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0057 | max_forward: 0.7342 | max_loss: 0.1652 | max_backward: 1.4907 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.9743 | max_tensorboard_flush: 0.0140 | max_post_step_barrier: 0.0556 | Memory: rss_MB: 7477.5 | uss_MB: 6998.2 | child_rss_MB: 4799.0 | allocated_MB: 342.3 | reserved_MB: 24404.0 | inactive_split_MB: 236.6 +rank7 [2026-10-29 08:13:32,078] INFO : Step 29800/50000 rank7 | Train Loss: 1.1686 | Train Loss Unweighted: 0.5644 | Train Loss Components: Auxiliary: 0.2578 | Cardinality: 0.1551 | Classification: 0.0952 | Classification_binary: 0.1922 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1357 | Regression_eta: 0.0014 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1822 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0712 | Regression_eta: 0.0364 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0835 | LR: 4.16e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6776s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7294 | max_loss: 0.1685 | max_backward: 1.6547 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9409 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0546 | Memory: rss_MB: 7478.5 | uss_MB: 7143.5 | child_rss_MB: 4794.4 | allocated_MB: 345.0 | reserved_MB: 24404.0 | inactive_split_MB: 265.1 +rank7 [2026-10-29 08:17:37,375] INFO : Step 29900/50000 rank7 | Train Loss: 1.1305 | Train Loss Unweighted: 0.5666 | Train Loss Components: Auxiliary: 0.2471 | Cardinality: 0.1380 | Classification: 0.0944 | Classification_binary: 0.1750 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1464 | Regression_eta: 0.0015 | Regression_pt: 0.1476 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1906 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0696 | Regression_eta: 0.0355 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0531 | LR: 4.14e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3508s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0611 | max_forward: 0.8060 | max_loss: 0.1494 | max_backward: 1.3251 | max_optimizer_step: 0.2411 | max_tensorboard_logging: 0.8986 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0544 | Memory: rss_MB: 7479.5 | uss_MB: 7143.0 | child_rss_MB: 4800.7 | allocated_MB: 344.7 | reserved_MB: 26422.0 | inactive_split_MB: 298.4 +rank7 [2026-10-29 08:22:04,988] INFO : Step 30000/50000 rank7 | Train Loss: 1.0854 | Train Loss Unweighted: 0.5586 | Train Loss Components: Auxiliary: 0.2416 | Cardinality: 0.1337 | Classification: 0.1000 | Classification_binary: 0.1541 | HitGrouping: 0.1983 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1499 | Regression_eta: 0.0014 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.1331 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0901 | Regression_eta: 0.0426 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0529 | LR: 4.11e-04 | DataLoad Time: 0.0199s | Model Forward Time: 7.0745s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0172 | max_forward: 0.9685 | max_loss: 0.2166 | max_backward: 4.0112 | max_optimizer_step: 0.2521 | max_tensorboard_logging: 0.8430 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0537 | Memory: rss_MB: 7461.9 | uss_MB: 7142.5 | child_rss_MB: 4797.1 | allocated_MB: 338.0 | reserved_MB: 26422.0 | inactive_split_MB: 288.0 +rank7 [2026-10-29 08:22:05,952] INFO : Running validation on rank7 +rank7 [2026-10-29 08:22:05,953] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 08:22:05,953] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 08:22:57,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:22:58,364] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 08:23:24,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 08:23:25,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 08:23:26,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 08:23:27,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 08:23:37,480] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 08:23:38,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:23:38,901] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 08:23:55,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 08:23:56,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 08:23:56,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 08:23:57,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 08:24:07,826] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 08:24:08,114] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:24:08,615] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 08:24:24,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 08:24:24,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 08:24:25,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 08:24:26,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_30000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 08:24:36,550] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 08:29:00,406] INFO : Step 30100/50000 rank7 | Train Loss: 1.0762 | Train Loss Unweighted: 0.5358 | Train Loss Components: Auxiliary: 0.2378 | Cardinality: 0.1106 | Classification: 0.1097 | Classification_binary: 0.1679 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.1240 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0562 | Regression_eta: 0.0472 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0538 | LR: 4.08e-04 | DataLoad Time: 0.0161s | Model Forward Time: 8.2924s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7352 | max_loss: 0.1443 | max_backward: 1.7375 | max_optimizer_step: 0.2574 | max_tensorboard_logging: 1.7731 | max_tensorboard_flush: 0.0049 | max_post_step_barrier: 0.3497 | Memory: rss_MB: 7528.6 | uss_MB: 6331.6 | child_rss_MB: 4801.2 | allocated_MB: 344.1 | reserved_MB: 22922.0 | inactive_split_MB: 248.8 +rank7 [2026-10-29 08:33:16,462] INFO : Step 30200/50000 rank7 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5304 | Train Loss Components: Auxiliary: 0.2412 | Cardinality: 0.1421 | Classification: 0.0934 | Classification_binary: 0.1839 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0016 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1796 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0555 | Regression_eta: 0.0428 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0690 | LR: 4.06e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6004s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8008 | max_loss: 0.1560 | max_backward: 1.6706 | max_optimizer_step: 0.2663 | max_tensorboard_logging: 0.7725 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0555 | Memory: rss_MB: 7535.6 | uss_MB: 7212.9 | child_rss_MB: 4775.8 | allocated_MB: 344.1 | reserved_MB: 23882.0 | inactive_split_MB: 274.8 +rank7 [2026-10-29 08:37:30,482] INFO : Step 30300/50000 rank7 | Train Loss: 1.0429 | Train Loss Unweighted: 0.5252 | Train Loss Components: Auxiliary: 0.2280 | Cardinality: 0.1223 | Classification: 0.0922 | Classification_binary: 0.1566 | HitGrouping: 0.1929 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1363 | Regression_eta: 0.0013 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0667 | Regression_eta: 0.0407 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0540 | LR: 4.03e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.6259s | Phase Times: max_pre_step_barrier: 0.1135 | max_batch_to_device: 0.0060 | max_forward: 1.1719 | max_loss: 0.2884 | max_backward: 2.4994 | max_optimizer_step: 0.2255 | max_tensorboard_logging: 0.9297 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7536.6 | uss_MB: 7213.9 | child_rss_MB: 4771.6 | allocated_MB: 341.8 | reserved_MB: 23882.0 | inactive_split_MB: 270.7 +rank7 [2026-10-29 08:41:40,106] INFO : Step 30400/50000 rank7 | Train Loss: 1.0543 | Train Loss Unweighted: 0.5182 | Train Loss Components: Auxiliary: 0.2271 | Cardinality: 0.1318 | Classification: 0.0889 | Classification_binary: 0.1487 | HitGrouping: 0.1973 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1382 | Regression_eta: 0.0014 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1208 | Regression_cos_phi: 0.0332 | Regression_energy: 0.0695 | Regression_eta: 0.0222 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0520 | LR: 4.00e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.0667s | Phase Times: max_pre_step_barrier: 0.0154 | max_batch_to_device: 0.0150 | max_forward: 0.8845 | max_loss: 0.2482 | max_backward: 2.1745 | max_optimizer_step: 0.2418 | max_tensorboard_logging: 0.5112 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7452.2 | uss_MB: 7214.6 | child_rss_MB: 4475.9 | allocated_MB: 344.7 | reserved_MB: 24842.0 | inactive_split_MB: 245.3 +rank7 [2026-10-29 08:45:49,047] INFO : Step 30500/50000 rank7 | Train Loss: 1.1589 | Train Loss Unweighted: 0.5596 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1407 | Classification: 0.1085 | Classification_binary: 0.1952 | HitGrouping: 0.1960 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1873 | Regression_cos_phi: 0.0966 | Regression_energy: 0.0729 | Regression_eta: 0.0517 | Regression_pt: 0.0747 | Regression_sin_phi: 0.1057 | LR: 3.98e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.7055s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7791 | max_loss: 0.2528 | max_backward: 1.9433 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.5210 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0672 | Memory: rss_MB: 7432.7 | uss_MB: 7008.9 | child_rss_MB: 4374.7 | allocated_MB: 346.2 | reserved_MB: 26806.0 | inactive_split_MB: 227.8 +rank7 [2026-10-29 08:49:58,495] INFO : Step 30600/50000 rank7 | Train Loss: 1.1269 | Train Loss Unweighted: 0.5561 | Train Loss Components: Auxiliary: 0.2414 | Cardinality: 0.1456 | Classification: 0.0973 | Classification_binary: 0.1804 | HitGrouping: 0.1953 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1373 | Regression_eta: 0.0014 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.2069 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0625 | Regression_eta: 0.0415 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0610 | LR: 3.95e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4781s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1011 | max_forward: 0.8824 | max_loss: 0.1534 | max_backward: 1.4258 | max_optimizer_step: 0.3067 | max_tensorboard_logging: 0.7925 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0509 | Memory: rss_MB: 7378.2 | uss_MB: 7215.1 | child_rss_MB: 4177.7 | allocated_MB: 345.8 | reserved_MB: 27800.0 | inactive_split_MB: 283.5 +rank7 [2026-10-29 08:54:17,062] INFO : Step 30700/50000 rank7 | Train Loss: 1.0989 | Train Loss Unweighted: 0.5515 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1377 | Classification: 0.0890 | Classification_binary: 0.1568 | HitGrouping: 0.2021 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1504 | Regression_eta: 0.0013 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1616 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0879 | Regression_eta: 0.0367 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0510 | LR: 3.92e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.2930s | Phase Times: max_pre_step_barrier: 0.1120 | max_batch_to_device: 0.0056 | max_forward: 0.7546 | max_loss: 0.2396 | max_backward: 1.3389 | max_optimizer_step: 0.2508 | max_tensorboard_logging: 0.8263 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0572 | Memory: rss_MB: 7380.2 | uss_MB: 7216.5 | child_rss_MB: 4178.9 | allocated_MB: 340.3 | reserved_MB: 27800.0 | inactive_split_MB: 223.2 +rank7 [2026-10-29 08:58:31,929] INFO : Step 30800/50000 rank7 | Train Loss: 1.1241 | Train Loss Unweighted: 0.5488 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1377 | Classification: 0.1081 | Classification_binary: 0.1717 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1317 | Regression_eta: 0.0015 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0842 | Classification_binary: 0.1644 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0653 | Regression_eta: 0.0349 | Regression_pt: 0.0691 | Regression_sin_phi: 0.1014 | LR: 3.90e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.8017s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 0.7775 | max_loss: 0.2735 | max_backward: 1.9298 | max_optimizer_step: 0.3489 | max_tensorboard_logging: 0.8756 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0726 | Memory: rss_MB: 7357.4 | uss_MB: 5945.6 | child_rss_MB: 4116.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 275.1 +rank7 [2026-10-29 09:02:40,911] INFO : Step 30900/50000 rank7 | Train Loss: 1.1501 | Train Loss Unweighted: 0.5762 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1300 | Classification: 0.0976 | Classification_binary: 0.1996 | HitGrouping: 0.2000 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1374 | Regression_eta: 0.0017 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1866 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0644 | Regression_eta: 0.0430 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0575 | LR: 3.87e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.5952s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.3511 | max_optimizer_step: 0.2292 | max_tensorboard_logging: 1.0306 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0665 | Memory: rss_MB: 7360.4 | uss_MB: 7217.8 | child_rss_MB: 4118.5 | allocated_MB: 342.2 | reserved_MB: 27800.0 | inactive_split_MB: 198.3 +rank7 [2026-10-29 09:06:46,925] INFO : Step 31000/50000 rank7 | Train Loss: 1.1054 | Train Loss Unweighted: 0.5218 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1446 | Classification: 0.0888 | Classification_binary: 0.1693 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0016 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0622 | Regression_eta: 0.0544 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0527 | LR: 3.84e-04 | DataLoad Time: 0.0225s | Model Forward Time: 4.7146s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0058 | max_forward: 0.8505 | max_loss: 0.2491 | max_backward: 1.6460 | max_optimizer_step: 0.2421 | max_tensorboard_logging: 0.9731 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7337.9 | uss_MB: 6674.8 | child_rss_MB: 4046.1 | allocated_MB: 344.8 | reserved_MB: 27800.0 | inactive_split_MB: 190.7 +rank7 [2026-10-29 09:06:47,903] INFO : Running validation on rank7 +rank7 [2026-10-29 09:06:47,903] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 09:06:47,903] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 09:07:39,008] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:07:39,555] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 09:08:08,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 09:08:09,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 09:08:10,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 09:08:11,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 09:08:21,300] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 09:08:22,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:08:22,479] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 09:08:39,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 09:08:40,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 09:08:41,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 09:08:42,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 09:08:52,633] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 09:08:52,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:08:53,433] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 09:09:09,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 09:09:09,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 09:09:10,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 09:09:11,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_31000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 09:09:21,184] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 09:13:45,045] INFO : Step 31100/50000 rank7 | Train Loss: 1.0306 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2182 | Cardinality: 0.1238 | Classification: 0.0932 | Classification_binary: 0.1415 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0015 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0610 | Regression_eta: 0.0727 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0673 | LR: 3.82e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.5344s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1068 | max_forward: 0.8067 | max_loss: 0.1436 | max_backward: 1.1427 | max_optimizer_step: 0.2576 | max_tensorboard_logging: 1.1475 | max_tensorboard_flush: 0.0054 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 8196.3 | uss_MB: 5921.9 | child_rss_MB: 4032.2 | allocated_MB: 343.8 | reserved_MB: 24492.0 | inactive_split_MB: 293.3 +rank7 [2026-10-29 09:17:44,912] INFO : Step 31200/50000 rank7 | Train Loss: 1.1102 | Train Loss Unweighted: 0.5369 | Train Loss Components: Auxiliary: 0.2472 | Cardinality: 0.1495 | Classification: 0.0959 | Classification_binary: 0.1708 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1328 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1223 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0542 | Regression_eta: 0.0270 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0474 | LR: 3.79e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.1841s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7127 | max_loss: 0.1476 | max_backward: 1.5707 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7428.7 | uss_MB: 7313.8 | child_rss_MB: 4003.5 | allocated_MB: 344.5 | reserved_MB: 24648.0 | inactive_split_MB: 235.5 +rank7 [2026-10-29 09:21:47,088] INFO : Step 31300/50000 rank7 | Train Loss: 1.1865 | Train Loss Unweighted: 0.5472 | Train Loss Components: Auxiliary: 0.2511 | Cardinality: 0.1748 | Classification: 0.0979 | Classification_binary: 0.1851 | HitGrouping: 0.1932 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1290 | Regression_eta: 0.0019 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1772 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0661 | Regression_eta: 0.0344 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0419 | LR: 3.76e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.2306s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0064 | max_forward: 0.7815 | max_loss: 0.1495 | max_backward: 1.6330 | max_optimizer_step: 0.2373 | max_tensorboard_logging: 0.5234 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0596 | Memory: rss_MB: 7366.1 | uss_MB: 7313.9 | child_rss_MB: 3706.8 | allocated_MB: 344.4 | reserved_MB: 25640.0 | inactive_split_MB: 233.6 +rank7 [2026-10-29 09:25:42,888] INFO : Step 31400/50000 rank7 | Train Loss: 1.0272 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1191 | Classification: 0.0784 | Classification_binary: 0.1451 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1683 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0569 | Regression_eta: 0.0358 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0556 | LR: 3.74e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.5262s | Phase Times: max_pre_step_barrier: 0.0057 | max_batch_to_device: 0.0063 | max_forward: 0.7252 | max_loss: 0.1478 | max_backward: 1.6173 | max_optimizer_step: 0.2404 | max_tensorboard_logging: 0.9541 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0540 | Memory: rss_MB: 7335.5 | uss_MB: 7312.2 | child_rss_MB: 3625.5 | allocated_MB: 342.1 | reserved_MB: 25640.0 | inactive_split_MB: 296.4 +rank7 [2026-10-29 09:29:44,711] INFO : Step 31500/50000 rank7 | Train Loss: 1.0343 | Train Loss Unweighted: 0.5247 | Train Loss Components: Auxiliary: 0.2222 | Cardinality: 0.1206 | Classification: 0.1037 | Classification_binary: 0.1583 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1297 | Regression_eta: 0.0014 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1196 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0464 | Regression_eta: 0.0278 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0500 | LR: 3.71e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3661s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0060 | max_forward: 0.6643 | max_loss: 0.2487 | max_backward: 1.5648 | max_optimizer_step: 0.2369 | max_tensorboard_logging: 0.8248 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7336.0 | uss_MB: 7313.0 | child_rss_MB: 3638.4 | allocated_MB: 344.4 | reserved_MB: 26092.0 | inactive_split_MB: 238.2 +rank7 [2026-10-29 09:33:42,078] INFO : Step 31600/50000 rank7 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5471 | Train Loss Components: Auxiliary: 0.2424 | Cardinality: 0.1284 | Classification: 0.1033 | Classification_binary: 0.1828 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1282 | Regression_eta: 0.0017 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0651 | Regression_eta: 0.0320 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0612 | LR: 3.69e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.0261s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7603 | max_loss: 0.1471 | max_backward: 2.0475 | max_optimizer_step: 0.2208 | max_tensorboard_logging: 0.9076 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7328.4 | uss_MB: 7314.4 | child_rss_MB: 3585.6 | allocated_MB: 341.5 | reserved_MB: 26640.0 | inactive_split_MB: 287.0 +rank7 [2026-10-29 09:37:47,826] INFO : Step 31700/50000 rank7 | Train Loss: 1.0533 | Train Loss Unweighted: 0.5009 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1416 | Classification: 0.0871 | Classification_binary: 0.1632 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0813 | Classification_binary: 0.1764 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0583 | Regression_eta: 0.0356 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0589 | LR: 3.66e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.1122s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7635 | max_loss: 0.1543 | max_backward: 1.2075 | max_optimizer_step: 0.2406 | max_tensorboard_logging: 0.8209 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 7330.4 | uss_MB: 7312.9 | child_rss_MB: 3557.0 | allocated_MB: 341.9 | reserved_MB: 26640.0 | inactive_split_MB: 274.1 +rank7 [2026-10-29 09:45:40,560] INFO : Step 31800/50000 rank7 | Train Loss: 1.0385 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1148 | Classification: 0.0877 | Classification_binary: 0.1481 | HitGrouping: 0.2028 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1418 | Regression_eta: 0.0012 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0384 | Regression_energy: 0.0791 | Regression_eta: 0.0293 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0537 | LR: 3.64e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0343s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0072 | max_forward: 0.6606 | max_loss: 0.1610 | max_backward: 1.1159 | max_optimizer_step: 0.2370 | max_tensorboard_logging: 0.8258 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1417 | Memory: rss_MB: 7328.9 | uss_MB: 7312.1 | child_rss_MB: 3561.3 | allocated_MB: 345.5 | reserved_MB: 27080.0 | inactive_split_MB: 260.5 +rank7 [2026-10-29 09:49:44,089] INFO : Step 31900/50000 rank7 | Train Loss: 1.1464 | Train Loss Unweighted: 0.5585 | Train Loss Components: Auxiliary: 0.2485 | Cardinality: 0.1372 | Classification: 0.1078 | Classification_binary: 0.1815 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0017 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0715 | Regression_eta: 0.0399 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0726 | LR: 3.61e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.5144s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.7718 | max_loss: 0.1553 | max_backward: 1.7857 | max_optimizer_step: 0.2430 | max_tensorboard_logging: 0.6721 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7333.9 | uss_MB: 7320.8 | child_rss_MB: 3569.3 | allocated_MB: 343.8 | reserved_MB: 27080.0 | inactive_split_MB: 280.2 +rank7 [2026-10-29 09:53:38,294] INFO : Step 32000/50000 rank7 | Train Loss: 1.1115 | Train Loss Unweighted: 0.5351 | Train Loss Components: Auxiliary: 0.2393 | Cardinality: 0.1406 | Classification: 0.0954 | Classification_binary: 0.1748 | HitGrouping: 0.1912 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1302 | Regression_eta: 0.0018 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1612 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0518 | Regression_eta: 0.0332 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0709 | LR: 3.58e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.2447s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.9767 | max_loss: 0.6919 | max_backward: 2.1955 | max_optimizer_step: 0.2416 | max_tensorboard_logging: 0.9277 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0543 | Memory: rss_MB: 7329.3 | uss_MB: 7313.9 | child_rss_MB: 3566.4 | allocated_MB: 344.2 | reserved_MB: 27080.0 | inactive_split_MB: 265.7 +rank7 [2026-10-29 09:53:39,346] INFO : Running validation on rank7 +rank7 [2026-10-29 09:53:39,347] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 09:53:39,347] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 09:54:28,723] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:54:29,292] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 09:54:57,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 09:54:58,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 09:54:59,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 09:55:00,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 09:55:10,177] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 09:55:10,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:55:11,414] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 09:55:27,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 09:55:28,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 09:55:30,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 09:55:31,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 09:55:41,187] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 09:55:41,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:55:41,777] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 09:55:57,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 09:55:57,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 09:55:58,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 09:55:59,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_32000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 09:56:09,309] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 10:00:19,250] INFO : Step 32100/50000 rank7 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5134 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1218 | Classification: 0.0914 | Classification_binary: 0.1509 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1333 | Regression_eta: 0.0016 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0631 | Regression_eta: 0.0376 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0433 | LR: 3.56e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.3628s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7098 | max_loss: 0.1454 | max_backward: 1.1394 | max_optimizer_step: 0.2650 | max_tensorboard_logging: 0.9832 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 8404.9 | uss_MB: 8386.6 | child_rss_MB: 3573.9 | allocated_MB: 344.0 | reserved_MB: 22148.0 | inactive_split_MB: 276.8 +rank7 [2026-10-29 10:04:33,375] INFO : Step 32200/50000 rank7 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1241 | Classification: 0.1035 | Classification_binary: 0.1630 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1279 | Regression_eta: 0.0014 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1463 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0584 | Regression_eta: 0.0424 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0454 | LR: 3.53e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.6702s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0891 | max_forward: 0.7517 | max_loss: 0.1441 | max_backward: 1.7835 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.7808 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0599 | Memory: rss_MB: 7418.2 | uss_MB: 7403.1 | child_rss_MB: 3576.2 | allocated_MB: 345.5 | reserved_MB: 23122.0 | inactive_split_MB: 233.4 +rank7 [2026-10-29 10:08:31,627] INFO : Step 32300/50000 rank7 | Train Loss: 1.1438 | Train Loss Unweighted: 0.5643 | Train Loss Components: Auxiliary: 0.2535 | Cardinality: 0.1335 | Classification: 0.1042 | Classification_binary: 0.1958 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1303 | Regression_eta: 0.0015 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.2044 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0686 | Regression_eta: 0.0430 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0737 | LR: 3.51e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7834s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7857 | max_loss: 0.1780 | max_backward: 1.7580 | max_optimizer_step: 0.2428 | max_tensorboard_logging: 0.8954 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0617 | Memory: rss_MB: 7421.3 | uss_MB: 7404.5 | child_rss_MB: 3576.8 | allocated_MB: 343.9 | reserved_MB: 24108.0 | inactive_split_MB: 280.1 +rank7 [2026-10-29 10:12:37,147] INFO : Step 32400/50000 rank7 | Train Loss: 1.0306 | Train Loss Unweighted: 0.5063 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1168 | Classification: 0.0867 | Classification_binary: 0.1599 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0013 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1847 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0674 | Regression_eta: 0.0255 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0695 | LR: 3.48e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.3236s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.1456 | max_backward: 1.3401 | max_optimizer_step: 0.2444 | max_tensorboard_logging: 0.8752 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0569 | Memory: rss_MB: 7420.4 | uss_MB: 7401.9 | child_rss_MB: 3574.4 | allocated_MB: 344.8 | reserved_MB: 24108.0 | inactive_split_MB: 271.0 +rank7 [2026-10-29 10:16:48,486] INFO : Step 32500/50000 rank7 | Train Loss: 1.0033 | Train Loss Unweighted: 0.5126 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.0995 | Classification: 0.1015 | Classification_binary: 0.1402 | HitGrouping: 0.1898 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1332 | Regression_eta: 0.0015 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1309 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0565 | Regression_eta: 0.0361 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0588 | LR: 3.46e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9966s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6088 | max_loss: 0.1480 | max_backward: 1.0729 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.8045 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7420.4 | uss_MB: 7401.8 | child_rss_MB: 3575.0 | allocated_MB: 341.9 | reserved_MB: 25096.0 | inactive_split_MB: 268.1 +rank7 [2026-10-29 10:21:04,066] INFO : Step 32600/50000 rank7 | Train Loss: 1.0981 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1271 | Classification: 0.1053 | Classification_binary: 0.1773 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1306 | Regression_eta: 0.0014 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1521 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0634 | Regression_eta: 0.0418 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0596 | LR: 3.43e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.2965s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.8529 | max_loss: 0.3369 | max_backward: 2.9831 | max_optimizer_step: 0.2386 | max_tensorboard_logging: 0.5557 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 7420.7 | uss_MB: 5748.3 | child_rss_MB: 3585.6 | allocated_MB: 344.1 | reserved_MB: 25096.0 | inactive_split_MB: 287.9 +rank7 [2026-10-29 10:25:23,204] INFO : Step 32700/50000 rank7 | Train Loss: 1.0956 | Train Loss Unweighted: 0.5307 | Train Loss Components: Auxiliary: 0.2357 | Cardinality: 0.1282 | Classification: 0.1011 | Classification_binary: 0.1731 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1259 | Regression_eta: 0.0017 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0561 | Regression_eta: 0.0365 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0655 | LR: 3.41e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.1125s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0062 | max_forward: 0.8233 | max_loss: 0.1478 | max_backward: 1.5120 | max_optimizer_step: 0.2440 | max_tensorboard_logging: 0.4966 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0564 | Memory: rss_MB: 7420.7 | uss_MB: 7403.4 | child_rss_MB: 3588.8 | allocated_MB: 342.9 | reserved_MB: 25096.0 | inactive_split_MB: 272.6 +rank7 [2026-10-29 10:29:23,500] INFO : Step 32800/50000 rank7 | Train Loss: 1.0857 | Train Loss Unweighted: 0.5241 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1299 | Classification: 0.1011 | Classification_binary: 0.1551 | HitGrouping: 0.1982 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1327 | Regression_eta: 0.0016 | Regression_pt: 0.1319 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1545 | Regression_cos_phi: 0.0925 | Regression_energy: 0.0495 | Regression_eta: 0.0472 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0685 | LR: 3.38e-04 | DataLoad Time: 0.0245s | Model Forward Time: 4.2375s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0179 | max_forward: 0.9787 | max_loss: 0.1782 | max_backward: 1.5075 | max_optimizer_step: 0.2435 | max_tensorboard_logging: 0.5808 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.1577 | Memory: rss_MB: 7419.5 | uss_MB: 7401.5 | child_rss_MB: 3585.9 | allocated_MB: 345.1 | reserved_MB: 25096.0 | inactive_split_MB: 263.9 +rank7 [2026-10-29 10:33:26,397] INFO : Step 32900/50000 rank7 | Train Loss: 1.0832 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2342 | Cardinality: 0.1116 | Classification: 0.1066 | Classification_binary: 0.1631 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1300 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0722 | Regression_eta: 0.0364 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0952 | LR: 3.36e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.6984s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0068 | max_forward: 1.3284 | max_loss: 0.5826 | max_backward: 1.9391 | max_optimizer_step: 0.2448 | max_tensorboard_logging: 0.6116 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0562 | Memory: rss_MB: 7423.7 | uss_MB: 7407.1 | child_rss_MB: 3589.0 | allocated_MB: 344.3 | reserved_MB: 26086.0 | inactive_split_MB: 212.1 +rank7 [2026-10-29 10:37:13,221] INFO : Step 33000/50000 rank7 | Train Loss: 1.0508 | Train Loss Unweighted: 0.5373 | Train Loss Components: Auxiliary: 0.2310 | Cardinality: 0.1017 | Classification: 0.1099 | Classification_binary: 0.1844 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0428 | Regression_eta: 0.0294 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0692 | LR: 3.33e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.3121s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0068 | max_forward: 0.7919 | max_loss: 0.2231 | max_backward: 1.5724 | max_optimizer_step: 0.2468 | max_tensorboard_logging: 0.5025 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1340 | Memory: rss_MB: 7423.5 | uss_MB: 7406.2 | child_rss_MB: 3581.4 | allocated_MB: 344.2 | reserved_MB: 26086.0 | inactive_split_MB: 269.8 +rank7 [2026-10-29 10:37:14,209] INFO : Running validation on rank7 +rank7 [2026-10-29 10:37:14,209] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 10:37:14,209] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 10:38:03,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:38:04,390] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 10:38:31,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 10:38:32,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 10:38:34,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 10:38:35,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 10:38:45,048] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 10:38:45,833] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:38:46,381] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 10:39:02,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 10:39:03,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 10:39:04,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 10:39:05,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 10:39:15,321] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 10:39:15,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:39:16,281] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 10:39:31,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 10:39:32,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 10:39:33,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 10:39:34,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_33000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 10:39:44,095] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 10:43:56,310] INFO : Step 33100/50000 rank7 | Train Loss: 1.1040 | Train Loss Unweighted: 0.5268 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1427 | Classification: 0.0921 | Classification_binary: 0.1655 | HitGrouping: 0.1976 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1319 | Regression_eta: 0.0015 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1879 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0578 | Regression_eta: 0.0372 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0624 | LR: 3.31e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3472s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0325 | max_forward: 0.7755 | max_loss: 0.1478 | max_backward: 1.3916 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.6610 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0760 | Memory: rss_MB: 8784.9 | uss_MB: 8766.5 | child_rss_MB: 3557.4 | allocated_MB: 343.8 | reserved_MB: 23356.0 | inactive_split_MB: 267.3 +rank7 [2026-10-29 10:47:56,312] INFO : Step 33200/50000 rank7 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4955 | Train Loss Components: Auxiliary: 0.2164 | Cardinality: 0.1253 | Classification: 0.1001 | Classification_binary: 0.1407 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0019 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0600 | Regression_eta: 0.1607 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0752 | LR: 3.28e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.0957s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0401 | max_forward: 0.9954 | max_loss: 0.3116 | max_backward: 2.2139 | max_optimizer_step: 0.2657 | max_tensorboard_logging: 0.6683 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7500.4 | uss_MB: 7483.0 | child_rss_MB: 3560.3 | allocated_MB: 344.1 | reserved_MB: 23356.0 | inactive_split_MB: 241.6 +rank7 [2026-10-29 10:51:55,085] INFO : Step 33300/50000 rank7 | Train Loss: 1.0288 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2302 | Cardinality: 0.1166 | Classification: 0.0975 | Classification_binary: 0.1653 | HitGrouping: 0.1885 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1220 | Regression_eta: 0.0012 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1578 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0611 | Regression_eta: 0.0349 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0450 | LR: 3.26e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5477s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.7054 | max_loss: 0.2213 | max_backward: 1.6633 | max_optimizer_step: 0.2474 | max_tensorboard_logging: 0.8237 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0708 | Memory: rss_MB: 7500.4 | uss_MB: 7483.1 | child_rss_MB: 3556.5 | allocated_MB: 343.5 | reserved_MB: 23356.0 | inactive_split_MB: 261.9 +rank7 [2026-10-29 10:55:54,910] INFO : Step 33400/50000 rank7 | Train Loss: 1.0821 | Train Loss Unweighted: 0.5337 | Train Loss Components: Auxiliary: 0.2386 | Cardinality: 0.1249 | Classification: 0.0970 | Classification_binary: 0.1720 | HitGrouping: 0.1947 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1308 | Regression_eta: 0.0013 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1698 | Regression_cos_phi: 0.0739 | Regression_energy: 0.0660 | Regression_eta: 0.0364 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0483 | LR: 3.23e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.4573s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0073 | max_forward: 1.2444 | max_loss: 0.2025 | max_backward: 2.5109 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7502.5 | uss_MB: 7485.1 | child_rss_MB: 3549.5 | allocated_MB: 347.6 | reserved_MB: 23358.0 | inactive_split_MB: 303.0 +rank7 [2026-10-29 10:59:53,595] INFO : Step 33500/50000 rank7 | Train Loss: 1.0164 | Train Loss Unweighted: 0.4956 | Train Loss Components: Auxiliary: 0.2212 | Cardinality: 0.1274 | Classification: 0.0875 | Classification_binary: 0.1421 | HitGrouping: 0.2033 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1309 | Regression_eta: 0.0013 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1769 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0657 | Regression_eta: 0.0337 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0549 | LR: 3.21e-04 | DataLoad Time: 0.0123s | Model Forward Time: 4.6230s | Phase Times: max_pre_step_barrier: 0.0271 | max_batch_to_device: 0.0558 | max_forward: 0.9437 | max_loss: 0.1437 | max_backward: 1.5429 | max_optimizer_step: 0.2447 | max_tensorboard_logging: 0.8833 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7505.9 | uss_MB: 7486.7 | child_rss_MB: 3558.2 | allocated_MB: 341.2 | reserved_MB: 23358.0 | inactive_split_MB: 237.5 +rank7 [2026-10-29 11:05:22,802] INFO : Step 33600/50000 rank7 | Train Loss: 1.0074 | Train Loss Unweighted: 0.4988 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1109 | Classification: 0.0969 | Classification_binary: 0.1509 | HitGrouping: 0.1901 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1239 | Regression_eta: 0.0012 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0577 | Regression_eta: 0.0403 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0619 | LR: 3.19e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1883s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0970 | max_forward: 0.7667 | max_loss: 0.1478 | max_backward: 1.4192 | max_optimizer_step: 0.2426 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7500.7 | uss_MB: 7485.1 | child_rss_MB: 3550.7 | allocated_MB: 344.5 | reserved_MB: 24528.0 | inactive_split_MB: 238.9 +rank7 [2026-10-29 11:09:17,747] INFO : Step 33700/50000 rank7 | Train Loss: 1.1891 | Train Loss Unweighted: 0.5658 | Train Loss Components: Auxiliary: 0.2602 | Cardinality: 0.1587 | Classification: 0.1116 | Classification_binary: 0.1953 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1270 | Regression_eta: 0.0015 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0505 | Regression_eta: 0.0331 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0893 | LR: 3.16e-04 | DataLoad Time: 0.0136s | Model Forward Time: 4.8126s | Phase Times: max_pre_step_barrier: 0.0253 | max_batch_to_device: 0.0066 | max_forward: 1.3051 | max_loss: 0.1764 | max_backward: 1.7331 | max_optimizer_step: 0.2460 | max_tensorboard_logging: 0.5706 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0895 | Memory: rss_MB: 7502.0 | uss_MB: 7487.9 | child_rss_MB: 3548.7 | allocated_MB: 347.7 | reserved_MB: 24528.0 | inactive_split_MB: 288.7 +rank7 [2026-10-29 11:13:14,784] INFO : Step 33800/50000 rank7 | Train Loss: 1.0442 | Train Loss Unweighted: 0.4970 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1421 | Classification: 0.0764 | Classification_binary: 0.1525 | HitGrouping: 0.1922 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1318 | Regression_eta: 0.0014 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0588 | Regression_eta: 0.0251 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0414 | LR: 3.14e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2986s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7370 | max_loss: 0.2458 | max_backward: 1.5902 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.6292 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.1331 | Memory: rss_MB: 7496.1 | uss_MB: 7482.2 | child_rss_MB: 3539.4 | allocated_MB: 344.2 | reserved_MB: 24528.0 | inactive_split_MB: 271.5 +rank7 [2026-10-29 11:17:10,798] INFO : Step 33900/50000 rank7 | Train Loss: 1.0341 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1352 | Classification: 0.0865 | Classification_binary: 0.1441 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1254 | Regression_eta: 0.0016 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0692 | Regression_eta: 0.0366 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0647 | LR: 3.11e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3827s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.6305 | max_loss: 0.1507 | max_backward: 1.4015 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9471 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0605 | Memory: rss_MB: 7498.1 | uss_MB: 7488.3 | child_rss_MB: 3543.5 | allocated_MB: 344.1 | reserved_MB: 24528.0 | inactive_split_MB: 268.7 +rank7 [2026-10-29 11:21:05,084] INFO : Step 34000/50000 rank7 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2369 | Cardinality: 0.1309 | Classification: 0.0985 | Classification_binary: 0.1707 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1170 | Regression_eta: 0.0014 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0609 | Regression_eta: 0.0396 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0658 | LR: 3.09e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.6366s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.7414 | max_loss: 0.1458 | max_backward: 1.7218 | max_optimizer_step: 0.2478 | max_tensorboard_logging: 0.9287 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0639 | Memory: rss_MB: 7499.1 | uss_MB: 7491.4 | child_rss_MB: 3548.4 | allocated_MB: 342.9 | reserved_MB: 24528.0 | inactive_split_MB: 277.5 +rank7 [2026-10-29 11:21:06,163] INFO : Running validation on rank7 +rank7 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 11:21:06,163] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 11:21:56,885] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:21:57,376] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 11:22:16,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 11:22:18,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 11:22:19,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 11:22:20,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 11:22:30,984] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 11:22:32,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:22:32,506] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 11:22:48,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 11:22:49,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 11:22:50,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 11:22:51,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 11:23:01,072] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 11:23:02,011] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:23:02,514] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 11:23:18,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 11:23:18,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 11:23:19,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 11:23:20,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_34000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 11:23:30,277] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 11:27:45,673] INFO : Step 34100/50000 rank7 | Train Loss: 1.1180 | Train Loss Unweighted: 0.5582 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1263 | Classification: 0.1031 | Classification_binary: 0.1801 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1358 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2255 | Regression_cos_phi: 0.0988 | Regression_energy: 0.0695 | Regression_eta: 0.0741 | Regression_pt: 0.0720 | Regression_sin_phi: 0.1006 | LR: 3.06e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8624s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9025 | max_loss: 0.1870 | max_backward: 1.7732 | max_optimizer_step: 0.2826 | max_tensorboard_logging: 0.8215 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0738 | Memory: rss_MB: 9109.5 | uss_MB: 8140.4 | child_rss_MB: 3555.5 | allocated_MB: 342.7 | reserved_MB: 22124.0 | inactive_split_MB: 211.6 +rank7 [2026-10-29 11:31:40,884] INFO : Step 34200/50000 rank7 | Train Loss: 0.9931 | Train Loss Unweighted: 0.4794 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1232 | Classification: 0.0771 | Classification_binary: 0.1377 | HitGrouping: 0.1996 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1301 | Regression_eta: 0.0016 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1714 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0620 | Regression_eta: 0.0606 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0724 | LR: 3.04e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.7183s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.8809 | max_loss: 0.3036 | max_backward: 1.8297 | max_optimizer_step: 0.2897 | max_tensorboard_logging: 0.6835 | max_tensorboard_flush: 0.0071 | max_post_step_barrier: 0.0586 | Memory: rss_MB: 7608.6 | uss_MB: 6291.1 | child_rss_MB: 3559.3 | allocated_MB: 339.0 | reserved_MB: 22508.0 | inactive_split_MB: 220.4 +rank7 [2026-10-29 11:35:37,831] INFO : Step 34300/50000 rank7 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4920 | Train Loss Components: Auxiliary: 0.2248 | Cardinality: 0.1192 | Classification: 0.0965 | Classification_binary: 0.1484 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.1281 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0595 | Regression_eta: 0.0383 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0558 | LR: 3.02e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.3198s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.7255 | max_loss: 0.1427 | max_backward: 1.3652 | max_optimizer_step: 0.2496 | max_tensorboard_logging: 0.8789 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0578 | Memory: rss_MB: 7610.6 | uss_MB: 7596.5 | child_rss_MB: 3557.3 | allocated_MB: 345.0 | reserved_MB: 22508.0 | inactive_split_MB: 216.4 +rank7 [2026-10-29 11:39:28,026] INFO : Step 34400/50000 rank7 | Train Loss: 1.0662 | Train Loss Unweighted: 0.5327 | Train Loss Components: Auxiliary: 0.2383 | Cardinality: 0.1289 | Classification: 0.0974 | Classification_binary: 0.1749 | HitGrouping: 0.1864 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1281 | Regression_eta: 0.0014 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0630 | Regression_eta: 0.0352 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0645 | LR: 2.99e-04 | DataLoad Time: 0.0188s | Model Forward Time: 6.1250s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9438 | max_loss: 0.1560 | max_backward: 2.9359 | max_optimizer_step: 0.2526 | max_tensorboard_logging: 0.5496 | max_tensorboard_flush: 0.0059 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 7613.7 | uss_MB: 6304.5 | child_rss_MB: 3550.6 | allocated_MB: 345.3 | reserved_MB: 23132.0 | inactive_split_MB: 282.7 +rank7 [2026-10-29 11:43:17,761] INFO : Step 34500/50000 rank7 | Train Loss: 1.0328 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1103 | Classification: 0.0867 | Classification_binary: 0.1518 | HitGrouping: 0.1945 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1314 | Regression_eta: 0.0014 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0637 | Regression_eta: 0.0422 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0441 | LR: 2.97e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.6253s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.8153 | max_loss: 0.1429 | max_backward: 1.8299 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.5315 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7615.7 | uss_MB: 6643.2 | child_rss_MB: 3549.5 | allocated_MB: 341.4 | reserved_MB: 23132.0 | inactive_split_MB: 238.6 +rank7 [2026-10-29 11:47:05,943] INFO : Step 34600/50000 rank7 | Train Loss: 1.0317 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1201 | Classification: 0.0877 | Classification_binary: 0.1434 | HitGrouping: 0.1988 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1359 | Regression_eta: 0.0014 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0607 | Regression_eta: 0.0431 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0352 | LR: 2.95e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2914s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.6882 | max_loss: 0.1360 | max_backward: 1.3984 | max_optimizer_step: 0.2445 | max_tensorboard_logging: 0.5787 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0658 | Memory: rss_MB: 7607.9 | uss_MB: 6553.3 | child_rss_MB: 3582.8 | allocated_MB: 343.8 | reserved_MB: 23132.0 | inactive_split_MB: 233.1 +rank7 [2026-10-29 11:51:12,922] INFO : Step 34700/50000 rank7 | Train Loss: 1.1152 | Train Loss Unweighted: 0.5285 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1612 | Classification: 0.0937 | Classification_binary: 0.1734 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1283 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0652 | Regression_eta: 0.0435 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0742 | LR: 2.92e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5245s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7859 | max_loss: 0.2571 | max_backward: 1.9119 | max_optimizer_step: 0.2516 | max_tensorboard_logging: 0.5032 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0622 | Memory: rss_MB: 7610.6 | uss_MB: 7599.4 | child_rss_MB: 3559.6 | allocated_MB: 343.9 | reserved_MB: 23132.0 | inactive_split_MB: 446.8 +rank7 [2026-10-29 11:54:56,884] INFO : Step 34800/50000 rank7 | Train Loss: 1.0848 | Train Loss Unweighted: 0.5142 | Train Loss Components: Auxiliary: 0.2371 | Cardinality: 0.1583 | Classification: 0.0857 | Classification_binary: 0.1750 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1255 | Regression_eta: 0.0013 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1784 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0648 | Regression_eta: 0.0317 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0575 | LR: 2.90e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.2210s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0423 | max_forward: 0.8214 | max_loss: 0.1472 | max_backward: 1.4019 | max_optimizer_step: 0.3234 | max_tensorboard_logging: 0.5950 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0585 | Memory: rss_MB: 7613.6 | uss_MB: 7602.4 | child_rss_MB: 3541.0 | allocated_MB: 344.5 | reserved_MB: 23132.0 | inactive_split_MB: 201.5 +rank7 [2026-10-29 11:58:48,856] INFO : Step 34900/50000 rank7 | Train Loss: 0.9512 | Train Loss Unweighted: 0.4678 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1120 | Classification: 0.0788 | Classification_binary: 0.1318 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1262 | Regression_eta: 0.0014 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1304 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0526 | Regression_eta: 0.0690 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0423 | LR: 2.88e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2629s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0986 | max_forward: 0.7397 | max_loss: 0.1421 | max_backward: 1.3918 | max_optimizer_step: 0.3560 | max_tensorboard_logging: 0.6071 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7614.6 | uss_MB: 7602.0 | child_rss_MB: 3543.9 | allocated_MB: 341.5 | reserved_MB: 23500.0 | inactive_split_MB: 262.9 +rank7 [2026-10-29 12:02:43,664] INFO : Step 35000/50000 rank7 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4747 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1401 | Classification: 0.0852 | Classification_binary: 0.1555 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1151 | Regression_eta: 0.0012 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1294 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0623 | Regression_eta: 0.0258 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0477 | LR: 2.85e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7034s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0159 | max_forward: 0.7961 | max_loss: 0.2615 | max_backward: 2.0104 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6382 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 7618.3 | uss_MB: 7606.3 | child_rss_MB: 3544.7 | allocated_MB: 345.2 | reserved_MB: 23500.0 | inactive_split_MB: 284.5 +rank7 [2026-10-29 12:02:44,719] INFO : Running validation on rank7 +rank7 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 12:02:44,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 12:03:33,662] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 12:03:34,206] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 12:04:01,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 12:04:02,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 12:04:04,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 12:04:05,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 12:04:15,132] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 12:04:18,001] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 12:04:18,510] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 12:04:34,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 12:04:35,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 12:04:36,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 12:04:37,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 12:04:47,060] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 12:04:48,304] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 12:04:48,822] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 12:05:04,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 12:05:05,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 12:05:05,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 12:05:06,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_35000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 12:05:16,622] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 12:09:34,457] INFO : Step 35100/50000 rank7 | Train Loss: 1.1060 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1387 | Classification: 0.0926 | Classification_binary: 0.1848 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1211 | Regression_eta: 0.0014 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.2024 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0644 | Regression_eta: 0.0386 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0509 | LR: 2.83e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.6641s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0069 | max_forward: 0.9070 | max_loss: 0.1513 | max_backward: 2.1746 | max_optimizer_step: 0.2850 | max_tensorboard_logging: 1.1439 | max_tensorboard_flush: 0.0024 | max_post_step_barrier: 0.0768 | Memory: rss_MB: 9217.3 | uss_MB: 7564.1 | child_rss_MB: 3570.6 | allocated_MB: 344.3 | reserved_MB: 22622.0 | inactive_split_MB: 852.9 +rank7 [2026-10-29 12:13:24,280] INFO : Step 35200/50000 rank7 | Train Loss: 1.0386 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2252 | Cardinality: 0.1293 | Classification: 0.0826 | Classification_binary: 0.1559 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1247 | Regression_eta: 0.0016 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1440 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0589 | Regression_eta: 0.0315 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0364 | LR: 2.81e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5320s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.8193 | max_loss: 0.1608 | max_backward: 1.3752 | max_optimizer_step: 0.2535 | max_tensorboard_logging: 0.9185 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0700 | Memory: rss_MB: 7738.0 | uss_MB: 7717.0 | child_rss_MB: 3546.7 | allocated_MB: 341.8 | reserved_MB: 25614.0 | inactive_split_MB: 192.0 +rank7 [2026-10-29 12:17:18,033] INFO : Step 35300/50000 rank7 | Train Loss: 0.9736 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1059 | Classification: 0.0970 | Classification_binary: 0.1371 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0016 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1462 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0644 | Regression_eta: 0.0354 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0567 | LR: 2.79e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6480s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6434 | max_loss: 0.2341 | max_backward: 1.6834 | max_optimizer_step: 0.2475 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0588 | Memory: rss_MB: 7740.0 | uss_MB: 7297.3 | child_rss_MB: 3543.2 | allocated_MB: 343.2 | reserved_MB: 25614.0 | inactive_split_MB: 226.8 +rank7 [2026-10-29 12:21:06,024] INFO : Step 35400/50000 rank7 | Train Loss: 1.1186 | Train Loss Unweighted: 0.5363 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1393 | Classification: 0.1069 | Classification_binary: 0.1767 | HitGrouping: 0.1943 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0016 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0674 | Regression_eta: 0.0444 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0606 | LR: 2.76e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.8136s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0060 | max_forward: 0.7591 | max_loss: 0.1562 | max_backward: 1.6040 | max_optimizer_step: 0.2543 | max_tensorboard_logging: 0.9624 | max_tensorboard_flush: 0.0056 | max_post_step_barrier: 0.0563 | Memory: rss_MB: 7735.5 | uss_MB: 7248.0 | child_rss_MB: 3540.1 | allocated_MB: 345.3 | reserved_MB: 25762.0 | inactive_split_MB: 277.5 +rank7 [2026-10-29 12:24:54,118] INFO : Step 35500/50000 rank7 | Train Loss: 1.1216 | Train Loss Unweighted: 0.5466 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1527 | Classification: 0.0970 | Classification_binary: 0.1767 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1342 | Regression_eta: 0.0014 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.2131 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0592 | Regression_eta: 0.0472 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0697 | LR: 2.74e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4076s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0648 | max_forward: 0.8748 | max_loss: 0.1544 | max_backward: 1.6387 | max_optimizer_step: 0.2546 | max_tensorboard_logging: 0.5130 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0632 | Memory: rss_MB: 7738.5 | uss_MB: 7721.7 | child_rss_MB: 3543.4 | allocated_MB: 343.4 | reserved_MB: 25762.0 | inactive_split_MB: 284.4 +rank7 [2026-10-29 12:28:50,969] INFO : Step 35600/50000 rank7 | Train Loss: 1.0380 | Train Loss Unweighted: 0.5039 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1124 | Classification: 0.0922 | Classification_binary: 0.1405 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1335 | Regression_eta: 0.0015 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.1571 | Regression_cos_phi: 0.0887 | Regression_energy: 0.0592 | Regression_eta: 0.0678 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0686 | LR: 2.72e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8991s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0559 | max_forward: 0.7376 | max_loss: 0.1415 | max_backward: 1.1754 | max_optimizer_step: 0.2524 | max_tensorboard_logging: 0.5225 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1603 | Memory: rss_MB: 7739.4 | uss_MB: 7724.9 | child_rss_MB: 3557.9 | allocated_MB: 343.8 | reserved_MB: 25762.0 | inactive_split_MB: 233.6 +rank7 [2026-10-29 12:32:54,485] INFO : Step 35700/50000 rank7 | Train Loss: 1.0352 | Train Loss Unweighted: 0.5000 | Train Loss Components: Auxiliary: 0.2288 | Cardinality: 0.1199 | Classification: 0.1043 | Classification_binary: 0.1492 | HitGrouping: 0.1850 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1212 | Regression_eta: 0.0015 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0625 | Regression_eta: 0.0230 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0415 | LR: 2.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3744s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0360 | max_forward: 0.7358 | max_loss: 0.1419 | max_backward: 1.6761 | max_optimizer_step: 0.2531 | max_tensorboard_logging: 0.5093 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7741.1 | uss_MB: 7726.8 | child_rss_MB: 3543.5 | allocated_MB: 344.2 | reserved_MB: 26034.0 | inactive_split_MB: 271.8 +rank7 [2026-10-29 12:36:36,039] INFO : Step 35800/50000 rank7 | Train Loss: 1.0929 | Train Loss Unweighted: 0.5323 | Train Loss Components: Auxiliary: 0.2446 | Cardinality: 0.1241 | Classification: 0.0986 | Classification_binary: 0.1801 | HitGrouping: 0.1908 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1248 | Regression_eta: 0.0014 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1751 | Regression_cos_phi: 0.1104 | Regression_energy: 0.0716 | Regression_eta: 0.0507 | Regression_pt: 0.0739 | Regression_sin_phi: 0.1009 | LR: 2.68e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1520s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7732 | max_loss: 0.1506 | max_backward: 1.4048 | max_optimizer_step: 0.2539 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0581 | Memory: rss_MB: 7740.3 | uss_MB: 7206.4 | child_rss_MB: 3548.3 | allocated_MB: 344.1 | reserved_MB: 26034.0 | inactive_split_MB: 276.1 +rank7 [2026-10-29 14:48:22,003] INFO : Initializing model with seed=12345; process seed=12352 +rank7 [2026-10-29 14:48:22,007] INFO : configured dtype=torch.bfloat16 for autocast +rank7 [2026-10-29 14:48:45,046] INFO : Instantiating model +rank7 [2026-10-29 14:48:45,046] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank7 [2026-10-29 14:48:45,046] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank7 [2026-10-29 14:48:45,064] INFO : Input encoding initialization took 0.02s +rank7 [2026-10-29 14:48:45,064] INFO : nn0 parameters: 171776 +rank7 [2026-10-29 14:48:45,064] INFO : Initializing shared backbone layers, num_convs=6 +rank7 [2026-10-29 14:48:45,064] INFO : Initializing backbone layer 0 +rank7 [2026-10-29 14:48:45,065] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:48:45,068] INFO : Initializing backbone layer 1 +rank7 [2026-10-29 14:48:45,068] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:48:45,071] INFO : Initializing backbone layer 2 +rank7 [2026-10-29 14:48:45,071] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:48:45,074] INFO : Initializing backbone layer 3 +rank7 [2026-10-29 14:48:45,075] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:48:45,078] INFO : Initializing backbone layer 4 +rank7 [2026-10-29 14:48:45,078] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:48:45,081] INFO : Initializing backbone layer 5 +rank7 [2026-10-29 14:48:45,081] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank7 [2026-10-29 14:48:45,084] INFO : Backbone initialization took 0.02s +rank7 [2026-10-29 14:48:45,084] INFO : backbone parameters: 2374656 +rank7 [2026-10-29 14:48:45,084] INFO : Initializing output DNNs +rank7 [2026-10-29 14:48:45,133] INFO : Output DNNs initialization took 0.05s +rank7 [2026-10-29 14:48:45,133] INFO : backbone_mode=BackboneMode.SHARED +rank7 [2026-10-29 14:48:45,133] INFO : jagged_attention enabled=True +rank7 [2026-10-29 14:48:45,133] INFO : flash_attn_varlen enabled=True +rank7 [2026-10-29 14:48:45,133] INFO : task_queries enabled=False +rank7 [2026-10-29 14:48:45,133] INFO : classification_norm parameters: 0 +rank7 [2026-10-29 14:48:45,133] INFO : regression_norm parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : classification_query parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : regression_query parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : classification_readout parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : regression_readout parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : output_mode=set +rank7 [2026-10-29 14:48:45,134] INFO : set_decoder parameters: 4713812 +rank7 [2026-10-29 14:48:45,134] INFO : nn_binary_particle parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : nn_pid parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : nn_pt parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : nn_eta parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : nn_sin_phi parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : nn_cos_phi parameters: 0 +rank7 [2026-10-29 14:48:45,134] INFO : nn_energy parameters: 0 +rank7 [2026-10-29 14:48:45,135] INFO : Total MLPF parameters: 7260244 +rank7 [2026-10-29 14:48:45,135] INFO : MLPF __init__ done +rank7 [2026-10-29 14:48:45,136] INFO : Instantiated model +rank7 [2026-10-29 14:48:45,136] INFO : Moving model to device rank=7 +rank7 [2026-10-29 14:48:45,169] INFO : Moved model to device rank=7 +rank7 [2026-10-29 14:48:45,171] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank7 [2026-10-29 14:48:45,514] INFO : Restoring training from step 35001 +rank7 [2026-10-29 14:48:45,520] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/checkpoints/checkpoint-35000.pth +rank7 [2026-10-29 14:48:45,521] INFO : Configured model for SyncBatchNorm rank=7 +rank7 [2026-10-29 14:48:45,764] INFO : Configured model for DistributedDataParallel rank=7 +rank7 [2026-10-29 14:48:45,875] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +194 module.set_decoder.tracker_momentum_head.embed... ... 0 +195 module.set_decoder.tracker_momentum_head.hidde... ... 0 +196 module.set_decoder.tracker_momentum_head.hidde... ... 0 +197 module.set_decoder.tracker_momentum_head.outpu... ... 0 +198 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[199 rows x 3 columns] +rank7 [2026-10-29 14:48:45,876] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank7 [2026-10-29 14:48:45,877] INFO : Trainable parameters: 7260244 +rank7 [2026-10-29 14:48:45,877] INFO : Non-trainable parameters: 0 +rank7 [2026-10-29 14:48:45,877] INFO : Total parameters: 7260244 +rank7 [2026-10-29 14:48:45,880] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.mha.in_proj_weight 196608 0 + module.backbone.0.mha.in_proj_bias 768 0 + module.backbone.0.mha.out_proj.weight 65536 0 + module.backbone.0.mha.out_proj.bias 256 0 + module.backbone.0.norm0.weight 256 0 + module.backbone.0.norm0.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm1.bias 256 0 + module.backbone.0.seq.0.weight 65536 0 + module.backbone.0.seq.0.bias 256 0 + module.backbone.0.seq.2.weight 65536 0 + module.backbone.0.seq.2.bias 256 0 + module.backbone.1.mha.in_proj_weight 196608 0 + module.backbone.1.mha.in_proj_bias 768 0 + module.backbone.1.mha.out_proj.weight 65536 0 + module.backbone.1.mha.out_proj.bias 256 0 + module.backbone.1.norm0.weight 256 0 + module.backbone.1.norm0.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm1.bias 256 0 + module.backbone.1.seq.0.weight 65536 0 + module.backbone.1.seq.0.bias 256 0 + module.backbone.1.seq.2.weight 65536 0 + module.backbone.1.seq.2.bias 256 0 + module.backbone.2.mha.in_proj_weight 196608 0 + module.backbone.2.mha.in_proj_bias 768 0 + module.backbone.2.mha.out_proj.weight 65536 0 + module.backbone.2.mha.out_proj.bias 256 0 + module.backbone.2.norm0.weight 256 0 + module.backbone.2.norm0.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm1.bias 256 0 + module.backbone.2.seq.0.weight 65536 0 + module.backbone.2.seq.0.bias 256 0 + module.backbone.2.seq.2.weight 65536 0 + module.backbone.2.seq.2.bias 256 0 + module.backbone.3.mha.in_proj_weight 196608 0 + module.backbone.3.mha.in_proj_bias 768 0 + module.backbone.3.mha.out_proj.weight 65536 0 + module.backbone.3.mha.out_proj.bias 256 0 + module.backbone.3.norm0.weight 256 0 + module.backbone.3.norm0.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm1.bias 256 0 + module.backbone.3.seq.0.weight 65536 0 + module.backbone.3.seq.0.bias 256 0 + module.backbone.3.seq.2.weight 65536 0 + module.backbone.3.seq.2.bias 256 0 + module.backbone.4.mha.in_proj_weight 196608 0 + module.backbone.4.mha.in_proj_bias 768 0 + module.backbone.4.mha.out_proj.weight 65536 0 + module.backbone.4.mha.out_proj.bias 256 0 + module.backbone.4.norm0.weight 256 0 + module.backbone.4.norm0.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm1.bias 256 0 + module.backbone.4.seq.0.weight 65536 0 + module.backbone.4.seq.0.bias 256 0 + module.backbone.4.seq.2.weight 65536 0 + module.backbone.4.seq.2.bias 256 0 + module.backbone.5.mha.in_proj_weight 196608 0 + module.backbone.5.mha.in_proj_bias 768 0 + module.backbone.5.mha.out_proj.weight 65536 0 + module.backbone.5.mha.out_proj.bias 256 0 + module.backbone.5.norm0.weight 256 0 + module.backbone.5.norm0.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm1.bias 256 0 + module.backbone.5.seq.0.weight 65536 0 + module.backbone.5.seq.0.bias 256 0 + module.backbone.5.seq.2.weight 65536 0 + module.backbone.5.seq.2.bias 256 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank7 [2026-10-29 14:48:45,880] INFO : Getting dataloaders +rank7 [2026-10-29 14:48:45,880] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:48:51,646] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:48:52,725] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:48:53,949] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-29 14:48:53,949] INFO : Creating WeightedMixIterator with 1 data loaders. +rank7 [2026-10-29 14:48:53,949] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:48:54,235] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:48:54,489] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:48:54,735] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-29 14:48:54,736] INFO : Creating InterleavedIterator with 1 data loaders. +rank7 [2026-10-29 14:49:08,716] INFO : Got dataloaders +rank7 [2026-10-29 14:49:08,716] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [13888] of epochs [1] +rank7 [2026-10-29 14:49:08,716] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank7 [2026-10-29 14:49:08,716] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank7 [2026-10-29 14:49:08,716] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank7 [2026-10-29 14:49:09,899] INFO : loader split=train rank=7 len=21112 +rank7 [2026-10-29 14:49:09,899] INFO : loader split=valid rank=7 len=12 +rank7 [2026-10-29 14:49:09,899] INFO : Starting training from step 35001 to 50000 on rank 7 of 8 +rank7 [2026-10-29 14:49:09,899] INFO : Creating train_iterator +rank7 [2026-10-29 14:49:09,899] INFO : ResumableSampler physical:train loading state: start_index=222208 +rank7 [2026-10-29 14:49:23,990] INFO : Created train_iterator +rank7 [2026-10-29 14:53:12,251] INFO : Step 35100/50000 rank7 | Train Loss: 1.0826 | Train Loss Unweighted: 0.5185 | Train Loss Components: Auxiliary: 0.2413 | Cardinality: 0.1393 | Classification: 0.0961 | Classification_binary: 0.1822 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0014 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.2007 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0485 | Regression_eta: 0.0387 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0455 | LR: 2.83e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.4920s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0069 | max_forward: 0.7340 | max_loss: 0.1651 | max_backward: 1.4152 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.4761 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4625.8 | uss_MB: 3388.6 | child_rss_MB: 5578.9 | allocated_MB: 372.6 | reserved_MB: 22674.0 | inactive_split_MB: 823.1 +rank7 [2026-10-29 14:56:45,511] INFO : Step 35200/50000 rank7 | Train Loss: 1.0562 | Train Loss Unweighted: 0.5125 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1301 | Classification: 0.0878 | Classification_binary: 0.1540 | HitGrouping: 0.1924 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1331 | Regression_eta: 0.0017 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1377 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0599 | Regression_eta: 0.0360 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0425 | LR: 2.81e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.1441s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.6042 | max_loss: 0.1411 | max_backward: 1.1580 | max_optimizer_step: 0.1447 | max_tensorboard_logging: 0.5108 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4625.0 | uss_MB: 3391.6 | child_rss_MB: 5591.1 | allocated_MB: 368.1 | reserved_MB: 25668.0 | inactive_split_MB: 192.7 +rank7 [2026-10-29 15:00:19,321] INFO : Step 35300/50000 rank7 | Train Loss: 0.9962 | Train Loss Unweighted: 0.4725 | Train Loss Components: Auxiliary: 0.2160 | Cardinality: 0.1172 | Classification: 0.0927 | Classification_binary: 0.1341 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1215 | Regression_eta: 0.0015 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0614 | Regression_eta: 0.0299 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0599 | LR: 2.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.0276s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.5940 | max_loss: 0.1358 | max_backward: 1.1911 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0027 | Memory: rss_MB: 4636.4 | uss_MB: 3402.6 | child_rss_MB: 5598.9 | allocated_MB: 371.5 | reserved_MB: 25668.0 | inactive_split_MB: 207.6 +rank7 [2026-10-29 15:03:53,852] INFO : Step 35400/50000 rank7 | Train Loss: 1.1270 | Train Loss Unweighted: 0.5445 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1356 | Classification: 0.0990 | Classification_binary: 0.1805 | HitGrouping: 0.1940 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1297 | Regression_eta: 0.0016 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0624 | Regression_eta: 0.0474 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0628 | LR: 2.76e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.6492s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1507 | max_backward: 1.5023 | max_optimizer_step: 0.2459 | max_tensorboard_logging: 0.5679 | max_tensorboard_flush: 0.0064 | max_post_step_barrier: 0.0041 | Memory: rss_MB: 4646.3 | uss_MB: 3410.8 | child_rss_MB: 5599.4 | allocated_MB: 371.4 | reserved_MB: 25816.0 | inactive_split_MB: 263.6 +rank7 [2026-10-29 15:07:33,751] INFO : Step 35500/50000 rank7 | Train Loss: 1.0502 | Train Loss Unweighted: 0.5157 | Train Loss Components: Auxiliary: 0.2363 | Cardinality: 0.1302 | Classification: 0.0904 | Classification_binary: 0.1703 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1250 | Regression_eta: 0.0013 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0545 | Regression_eta: 0.0598 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0841 | LR: 2.74e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.6105s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0737 | max_forward: 0.7546 | max_loss: 0.1581 | max_backward: 1.4977 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4644.7 | uss_MB: 3412.9 | child_rss_MB: 5568.4 | allocated_MB: 371.1 | reserved_MB: 25816.0 | inactive_split_MB: 263.4 +rank7 [2026-10-29 15:11:19,390] INFO : Step 35600/50000 rank7 | Train Loss: 1.0258 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2226 | Cardinality: 0.1119 | Classification: 0.0922 | Classification_binary: 0.1418 | HitGrouping: 0.1950 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1286 | Regression_eta: 0.0015 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0541 | Regression_eta: 0.0756 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0760 | LR: 2.72e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.2543s | Phase Times: max_pre_step_barrier: 0.0929 | max_batch_to_device: 0.0064 | max_forward: 0.5696 | max_loss: 0.1430 | max_backward: 1.1627 | max_optimizer_step: 0.1475 | max_tensorboard_logging: 0.5805 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4646.9 | uss_MB: 3412.7 | child_rss_MB: 5602.7 | allocated_MB: 370.5 | reserved_MB: 25816.0 | inactive_split_MB: 212.3 +rank7 [2026-10-29 15:15:08,569] INFO : Step 35700/50000 rank7 | Train Loss: 1.0162 | Train Loss Unweighted: 0.5013 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1185 | Classification: 0.0946 | Classification_binary: 0.1537 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1248 | Regression_eta: 0.0015 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1323 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0545 | Regression_eta: 0.0228 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0391 | LR: 2.70e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.5927s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7397 | max_loss: 0.1413 | max_backward: 1.6688 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.5161 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0047 | Memory: rss_MB: 4646.6 | uss_MB: 3411.7 | child_rss_MB: 5604.2 | allocated_MB: 372.2 | reserved_MB: 26084.0 | inactive_split_MB: 249.8 +rank7 [2026-10-29 15:18:49,408] INFO : Step 35800/50000 rank7 | Train Loss: 1.0884 | Train Loss Unweighted: 0.5265 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1296 | Classification: 0.0948 | Classification_binary: 0.1803 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1773 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0751 | Regression_eta: 0.0531 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0938 | LR: 2.68e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2009s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0199 | max_forward: 0.9402 | max_loss: 0.2389 | max_backward: 1.7889 | max_optimizer_step: 0.2466 | max_tensorboard_logging: 0.7071 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0023 | Memory: rss_MB: 4643.4 | uss_MB: 3414.5 | child_rss_MB: 5602.6 | allocated_MB: 371.6 | reserved_MB: 26084.0 | inactive_split_MB: 257.5 +rank7 [2026-10-29 15:22:30,829] INFO : Step 35900/50000 rank7 | Train Loss: 1.0760 | Train Loss Unweighted: 0.5299 | Train Loss Components: Auxiliary: 0.2328 | Cardinality: 0.1277 | Classification: 0.0983 | Classification_binary: 0.1568 | HitGrouping: 0.1993 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1349 | Regression_eta: 0.0016 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0570 | Regression_eta: 0.0415 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0639 | LR: 2.65e-04 | DataLoad Time: 0.0172s | Model Forward Time: 3.2909s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.6268 | max_loss: 0.2373 | max_backward: 1.2570 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.4934 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0016 | Memory: rss_MB: 4639.4 | uss_MB: 3412.6 | child_rss_MB: 5605.5 | allocated_MB: 369.7 | reserved_MB: 26800.0 | inactive_split_MB: 294.3 +rank7 [2026-10-29 15:26:13,159] INFO : Step 36000/50000 rank7 | Train Loss: 1.0831 | Train Loss Unweighted: 0.5411 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1276 | Classification: 0.1031 | Classification_binary: 0.1434 | HitGrouping: 0.2009 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1452 | Regression_eta: 0.0017 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1349 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0974 | Regression_eta: 0.0410 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0436 | LR: 2.63e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.2185s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.6266 | max_loss: 0.1437 | max_backward: 1.3209 | max_optimizer_step: 0.1464 | max_tensorboard_logging: 0.4977 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0033 | Memory: rss_MB: 4620.1 | uss_MB: 3411.5 | child_rss_MB: 5582.3 | allocated_MB: 368.7 | reserved_MB: 26800.0 | inactive_split_MB: 270.9 +rank7 [2026-10-29 15:26:14,080] INFO : Running validation on rank7 +rank7 [2026-10-29 15:26:14,080] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 15:26:14,080] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 15:27:03,342] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:27:03,874] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 15:27:29,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 15:27:30,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 15:27:31,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 15:27:32,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 15:27:42,076] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 15:27:42,775] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:27:43,314] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 15:27:59,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 15:28:00,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 15:28:01,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 15:28:02,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 15:28:12,327] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 15:28:12,651] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:28:13,404] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 15:28:28,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 15:28:29,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 15:28:30,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 15:28:30,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_36000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 15:28:40,939] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 15:32:50,524] INFO : Step 36100/50000 rank7 | Train Loss: 1.0137 | Train Loss Unweighted: 0.5026 | Train Loss Components: Auxiliary: 0.2266 | Cardinality: 0.1079 | Classification: 0.1014 | Classification_binary: 0.1522 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1216 | Regression_eta: 0.0013 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0716 | Regression_eta: 0.0598 | Regression_pt: 0.0754 | Regression_sin_phi: 0.1132 | LR: 2.61e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0454s | Phase Times: max_pre_step_barrier: 0.0114 | max_batch_to_device: 0.0065 | max_forward: 0.6968 | max_loss: 0.2782 | max_backward: 1.4278 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.9130 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 6101.3 | uss_MB: 4907.2 | child_rss_MB: 5609.1 | allocated_MB: 372.5 | reserved_MB: 23174.0 | inactive_split_MB: 246.6 +rank7 [2026-10-29 15:36:33,883] INFO : Step 36200/50000 rank7 | Train Loss: 1.0439 | Train Loss Unweighted: 0.5147 | Train Loss Components: Auxiliary: 0.2338 | Cardinality: 0.1304 | Classification: 0.0909 | Classification_binary: 0.1697 | HitGrouping: 0.1917 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1255 | Regression_eta: 0.0011 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1823 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0569 | Regression_eta: 0.0380 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0406 | LR: 2.59e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.5428s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.7440 | max_loss: 0.1544 | max_backward: 1.4037 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.4830 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0189 | Memory: rss_MB: 6101.4 | uss_MB: 4909.6 | child_rss_MB: 5587.9 | allocated_MB: 367.8 | reserved_MB: 23174.0 | inactive_split_MB: 282.8 +rank7 [2026-10-29 15:40:20,923] INFO : Step 36300/50000 rank7 | Train Loss: 1.0379 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1216 | Classification: 0.0886 | Classification_binary: 0.1447 | HitGrouping: 0.1910 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1266 | Regression_eta: 0.0016 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1536 | Regression_cos_phi: 0.0890 | Regression_energy: 0.0496 | Regression_eta: 0.0461 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0508 | LR: 2.57e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.3299s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.6106 | max_loss: 0.2412 | max_backward: 1.3407 | max_optimizer_step: 0.1535 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1063 | Memory: rss_MB: 5450.1 | uss_MB: 4269.0 | child_rss_MB: 5611.1 | allocated_MB: 370.5 | reserved_MB: 23174.0 | inactive_split_MB: 274.2 +rank7 [2026-10-29 15:44:06,294] INFO : Step 36400/50000 rank7 | Train Loss: 1.0182 | Train Loss Unweighted: 0.4838 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.0976 | Classification: 0.0939 | Classification_binary: 0.1520 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1167 | Regression_eta: 0.0016 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1297 | Regression_cos_phi: 0.1126 | Regression_energy: 0.0402 | Regression_eta: 0.0775 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0966 | LR: 2.55e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5207s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.6772 | max_loss: 0.1711 | max_backward: 1.4058 | max_optimizer_step: 0.1518 | max_tensorboard_logging: 0.5460 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0089 | Memory: rss_MB: 5451.9 | uss_MB: 4272.1 | child_rss_MB: 5615.2 | allocated_MB: 371.8 | reserved_MB: 23382.0 | inactive_split_MB: 230.6 +rank7 [2026-10-29 15:47:54,064] INFO : Step 36500/50000 rank7 | Train Loss: 1.0953 | Train Loss Unweighted: 0.5240 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1370 | Classification: 0.0895 | Classification_binary: 0.1838 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1230 | Regression_eta: 0.0015 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0644 | Regression_eta: 0.0318 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0546 | LR: 2.52e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.9819s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7475 | max_loss: 0.2537 | max_backward: 1.7352 | max_optimizer_step: 0.1520 | max_tensorboard_logging: 0.6267 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0135 | Memory: rss_MB: 5440.5 | uss_MB: 4262.4 | child_rss_MB: 5614.9 | allocated_MB: 372.3 | reserved_MB: 24160.0 | inactive_split_MB: 275.1 +rank7 [2026-10-29 15:51:40,079] INFO : Step 36600/50000 rank7 | Train Loss: 0.9967 | Train Loss Unweighted: 0.4996 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1165 | Classification: 0.0804 | Classification_binary: 0.1438 | HitGrouping: 0.1895 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1366 | Regression_eta: 0.0013 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0583 | Regression_eta: 0.0219 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0463 | LR: 2.50e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.6278s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6592 | max_loss: 0.2501 | max_backward: 1.4213 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.5953 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5450.1 | uss_MB: 4272.8 | child_rss_MB: 5617.4 | allocated_MB: 367.8 | reserved_MB: 27180.0 | inactive_split_MB: 225.4 +rank7 [2026-10-29 15:55:26,196] INFO : Step 36700/50000 rank7 | Train Loss: 0.9770 | Train Loss Unweighted: 0.4817 | Train Loss Components: Auxiliary: 0.2124 | Cardinality: 0.1070 | Classification: 0.0901 | Classification_binary: 0.1347 | HitGrouping: 0.1961 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1267 | Regression_eta: 0.0015 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1121 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0571 | Regression_eta: 0.0415 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0570 | LR: 2.48e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.2737s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0062 | max_forward: 0.5676 | max_loss: 0.2381 | max_backward: 1.1343 | max_optimizer_step: 0.2417 | max_tensorboard_logging: 0.4693 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0131 | Memory: rss_MB: 5448.5 | uss_MB: 4274.1 | child_rss_MB: 5620.1 | allocated_MB: 370.8 | reserved_MB: 27180.0 | inactive_split_MB: 235.2 +rank7 [2026-10-29 15:59:20,982] INFO : Step 36800/50000 rank7 | Train Loss: 1.0234 | Train Loss Unweighted: 0.4967 | Train Loss Components: Auxiliary: 0.2267 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1662 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1197 | Regression_eta: 0.0013 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1632 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0616 | Regression_eta: 0.0616 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0741 | LR: 2.46e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.5122s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0071 | max_forward: 1.1394 | max_loss: 0.1518 | max_backward: 2.3088 | max_optimizer_step: 0.1533 | max_tensorboard_logging: 0.5660 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5450.3 | uss_MB: 4278.9 | child_rss_MB: 5629.5 | allocated_MB: 371.9 | reserved_MB: 27180.0 | inactive_split_MB: 230.7 +rank7 [2026-10-29 16:03:11,411] INFO : Step 36900/50000 rank7 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2268 | Cardinality: 0.1211 | Classification: 0.0855 | Classification_binary: 0.1637 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1219 | Regression_eta: 0.0015 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1726 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0620 | Regression_eta: 0.0318 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0541 | LR: 2.44e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.5001s | Phase Times: max_pre_step_barrier: 0.1086 | max_batch_to_device: 0.0071 | max_forward: 0.7005 | max_loss: 0.1465 | max_backward: 1.2640 | max_optimizer_step: 0.1579 | max_tensorboard_logging: 0.4728 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5451.1 | uss_MB: 4280.1 | child_rss_MB: 5636.6 | allocated_MB: 372.3 | reserved_MB: 27180.0 | inactive_split_MB: 201.7 +rank7 [2026-10-29 16:07:15,146] INFO : Step 37000/50000 rank7 | Train Loss: 1.0150 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2194 | Cardinality: 0.1268 | Classification: 0.0926 | Classification_binary: 0.1402 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1241 | Regression_eta: 0.0014 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0605 | Regression_eta: 0.0287 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0556 | LR: 2.42e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.5239s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.6075 | max_loss: 0.2475 | max_backward: 1.2956 | max_optimizer_step: 0.1585 | max_tensorboard_logging: 0.7613 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0120 | Memory: rss_MB: 5445.1 | uss_MB: 4277.2 | child_rss_MB: 5637.5 | allocated_MB: 368.1 | reserved_MB: 27180.0 | inactive_split_MB: 226.3 +rank7 [2026-10-29 16:07:16,060] INFO : Running validation on rank7 +rank7 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 16:07:16,060] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 16:08:14,061] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:08:14,583] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 16:08:39,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 16:08:40,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 16:08:41,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 16:08:42,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 16:08:52,401] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 16:08:53,058] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:08:53,527] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 16:09:09,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 16:09:10,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 16:09:11,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 16:09:12,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 16:09:22,471] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 16:09:23,588] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:09:24,058] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 16:09:39,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 16:09:39,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 16:09:40,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 16:09:41,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_37000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 16:09:51,316] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 16:13:56,322] INFO : Step 37100/50000 rank7 | Train Loss: 1.1150 | Train Loss Unweighted: 0.5431 | Train Loss Components: Auxiliary: 0.2462 | Cardinality: 0.1226 | Classification: 0.1105 | Classification_binary: 0.1597 | HitGrouping: 0.1946 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1340 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0831 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0872 | Regression_eta: 0.0388 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0948 | LR: 2.40e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8594s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7539 | max_loss: 0.3988 | max_backward: 1.7903 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4747 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0305 | Memory: rss_MB: 6593.7 | uss_MB: 5426.2 | child_rss_MB: 5642.5 | allocated_MB: 372.5 | reserved_MB: 23160.0 | inactive_split_MB: 265.5 +rank7 [2026-10-29 16:17:46,449] INFO : Step 37200/50000 rank7 | Train Loss: 1.0752 | Train Loss Unweighted: 0.5181 | Train Loss Components: Auxiliary: 0.2406 | Cardinality: 0.1258 | Classification: 0.0924 | Classification_binary: 0.1812 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1200 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0584 | Regression_eta: 0.0413 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0690 | LR: 2.38e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1713s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0549 | max_forward: 0.8058 | max_loss: 0.1510 | max_backward: 1.8476 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5280 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6591.2 | uss_MB: 5426.9 | child_rss_MB: 5644.6 | allocated_MB: 371.5 | reserved_MB: 24140.0 | inactive_split_MB: 271.0 +rank7 [2026-10-29 16:21:30,618] INFO : Step 37300/50000 rank7 | Train Loss: 1.0230 | Train Loss Unweighted: 0.4889 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1184 | Classification: 0.0869 | Classification_binary: 0.1563 | HitGrouping: 0.1981 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1213 | Regression_eta: 0.0016 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0562 | Regression_eta: 0.0533 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0444 | LR: 2.36e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.0304s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0073 | max_forward: 0.6853 | max_loss: 0.1502 | max_backward: 1.3245 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 1.0657 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0153 | Memory: rss_MB: 5580.1 | uss_MB: 4415.2 | child_rss_MB: 5668.9 | allocated_MB: 370.9 | reserved_MB: 24140.0 | inactive_split_MB: 261.7 +rank7 [2026-10-29 16:25:14,030] INFO : Step 37400/50000 rank7 | Train Loss: 1.0130 | Train Loss Unweighted: 0.4929 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1180 | Classification: 0.0961 | Classification_binary: 0.1390 | HitGrouping: 0.1948 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1273 | Regression_eta: 0.0015 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0725 | Regression_eta: 0.0298 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0489 | LR: 2.34e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.5040s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.5424 | max_loss: 0.1464 | max_backward: 1.0792 | max_optimizer_step: 0.1675 | max_tensorboard_logging: 0.8668 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5585.6 | uss_MB: 4420.8 | child_rss_MB: 5670.0 | allocated_MB: 368.8 | reserved_MB: 25122.0 | inactive_split_MB: 232.7 +rank7 [2026-10-29 16:30:41,705] INFO : Step 37500/50000 rank7 | Train Loss: 1.0471 | Train Loss Unweighted: 0.5059 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1315 | Classification: 0.0992 | Classification_binary: 0.1679 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1176 | Regression_eta: 0.0013 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0681 | Regression_eta: 0.0414 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0559 | LR: 2.32e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.1481s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0073 | max_forward: 0.7240 | max_loss: 0.1472 | max_backward: 1.5381 | max_optimizer_step: 0.2707 | max_tensorboard_logging: 0.8554 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5585.8 | uss_MB: 4422.3 | child_rss_MB: 5642.8 | allocated_MB: 370.2 | reserved_MB: 26134.0 | inactive_split_MB: 179.7 +rank7 [2026-10-29 16:34:38,362] INFO : Step 37600/50000 rank7 | Train Loss: 1.0243 | Train Loss Unweighted: 0.4867 | Train Loss Components: Auxiliary: 0.2287 | Cardinality: 0.1182 | Classification: 0.0808 | Classification_binary: 0.1619 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1747 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0506 | Regression_eta: 0.0428 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0661 | LR: 2.30e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1670s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0064 | max_forward: 0.7069 | max_loss: 0.1543 | max_backward: 1.5139 | max_optimizer_step: 0.1704 | max_tensorboard_logging: 1.0159 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5587.9 | uss_MB: 4422.7 | child_rss_MB: 5672.0 | allocated_MB: 370.8 | reserved_MB: 26846.0 | inactive_split_MB: 273.3 +rank7 [2026-10-29 16:38:29,294] INFO : Step 37700/50000 rank7 | Train Loss: 0.9610 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2102 | Cardinality: 0.1049 | Classification: 0.0974 | Classification_binary: 0.1317 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0012 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.1440 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0762 | Regression_eta: 0.0368 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0759 | LR: 2.28e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.5155s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6233 | max_loss: 0.2433 | max_backward: 1.3306 | max_optimizer_step: 0.2250 | max_tensorboard_logging: 0.5644 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0170 | Memory: rss_MB: 5591.0 | uss_MB: 4425.5 | child_rss_MB: 5643.2 | allocated_MB: 369.4 | reserved_MB: 28822.0 | inactive_split_MB: 231.5 +rank7 [2026-10-29 16:42:35,307] INFO : Step 37800/50000 rank7 | Train Loss: 0.9994 | Train Loss Unweighted: 0.4769 | Train Loss Components: Auxiliary: 0.2210 | Cardinality: 0.1105 | Classification: 0.0952 | Classification_binary: 0.1458 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1161 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1383 | Regression_cos_phi: 0.1004 | Regression_energy: 0.0590 | Regression_eta: 0.0423 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0443 | LR: 2.26e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8896s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 0.6659 | max_loss: 0.1473 | max_backward: 1.3840 | max_optimizer_step: 0.1670 | max_tensorboard_logging: 0.9558 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5593.5 | uss_MB: 4427.0 | child_rss_MB: 5671.1 | allocated_MB: 371.5 | reserved_MB: 28822.0 | inactive_split_MB: 170.1 +rank7 [2026-10-29 16:46:40,060] INFO : Step 37900/50000 rank7 | Train Loss: 1.0837 | Train Loss Unweighted: 0.5094 | Train Loss Components: Auxiliary: 0.2372 | Cardinality: 0.1419 | Classification: 0.0937 | Classification_binary: 0.1795 | HitGrouping: 0.1848 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1160 | Regression_eta: 0.0015 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1887 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0571 | Regression_eta: 0.0419 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0776 | LR: 2.24e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2197s | Phase Times: max_pre_step_barrier: 0.1076 | max_batch_to_device: 0.0071 | max_forward: 0.7572 | max_loss: 0.1547 | max_backward: 1.4632 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 0.9205 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 5593.0 | uss_MB: 4429.3 | child_rss_MB: 5670.1 | allocated_MB: 370.2 | reserved_MB: 28822.0 | inactive_split_MB: 265.3 +rank7 [2026-10-29 16:50:31,520] INFO : Step 38000/50000 rank7 | Train Loss: 0.9811 | Train Loss Unweighted: 0.4782 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1213 | Classification: 0.0781 | Classification_binary: 0.1413 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1277 | Regression_eta: 0.0012 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1365 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0611 | Regression_eta: 0.0278 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0363 | LR: 2.22e-04 | DataLoad Time: 0.0224s | Model Forward Time: 3.5081s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.6120 | max_loss: 0.2171 | max_backward: 1.2526 | max_optimizer_step: 0.1653 | max_tensorboard_logging: 0.5836 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5590.4 | uss_MB: 4430.0 | child_rss_MB: 5667.2 | allocated_MB: 368.7 | reserved_MB: 28822.0 | inactive_split_MB: 223.4 +rank7 [2026-10-29 16:50:32,415] INFO : Running validation on rank7 +rank7 [2026-10-29 16:50:32,415] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 16:50:32,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 16:51:23,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:51:24,264] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 16:51:51,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 16:51:51,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 16:51:52,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 16:51:53,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 16:52:03,726] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 16:52:04,269] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:52:05,008] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 16:52:21,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 16:52:22,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 16:52:22,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 16:52:24,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 16:52:34,089] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 16:52:34,974] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:52:35,419] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 16:52:51,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 16:52:52,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 16:52:52,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 16:52:53,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_38000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 16:53:03,503] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 16:57:14,095] INFO : Step 38100/50000 rank7 | Train Loss: 1.0158 | Train Loss Unweighted: 0.4912 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1220 | Classification: 0.0982 | Classification_binary: 0.1455 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1224 | Regression_eta: 0.0013 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0301 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0468 | Regression_eta: 0.0198 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0288 | LR: 2.20e-04 | DataLoad Time: 0.0242s | Model Forward Time: 4.9413s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8406 | max_loss: 0.1820 | max_backward: 1.8536 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 1.3354 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0179 | Memory: rss_MB: 5714.6 | uss_MB: 4482.4 | child_rss_MB: 5669.8 | allocated_MB: 370.5 | reserved_MB: 24292.0 | inactive_split_MB: 257.8 +rank7 [2026-10-29 17:01:01,944] INFO : Step 38200/50000 rank7 | Train Loss: 1.0273 | Train Loss Unweighted: 0.5076 | Train Loss Components: Auxiliary: 0.2292 | Cardinality: 0.1060 | Classification: 0.0997 | Classification_binary: 0.1650 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1196 | Regression_eta: 0.0014 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0344 | Regression_energy: 0.0552 | Regression_eta: 0.0454 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0743 | LR: 2.18e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8901s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7758 | max_loss: 0.1794 | max_backward: 1.7969 | max_optimizer_step: 0.1713 | max_tensorboard_logging: 0.4698 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5717.4 | uss_MB: 4558.1 | child_rss_MB: 5664.1 | allocated_MB: 372.9 | reserved_MB: 26212.0 | inactive_split_MB: 221.1 +rank7 [2026-10-29 17:05:07,129] INFO : Step 38300/50000 rank7 | Train Loss: 1.0498 | Train Loss Unweighted: 0.5122 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1249 | Classification: 0.0913 | Classification_binary: 0.1583 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1294 | Regression_eta: 0.0013 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0261 | Regression_energy: 0.0523 | Regression_eta: 0.0278 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0664 | LR: 2.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.7371s | Phase Times: max_pre_step_barrier: 0.1103 | max_batch_to_device: 0.0086 | max_forward: 0.6730 | max_loss: 0.1509 | max_backward: 1.3787 | max_optimizer_step: 0.1705 | max_tensorboard_logging: 0.4782 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5715.4 | uss_MB: 4556.5 | child_rss_MB: 5665.1 | allocated_MB: 369.2 | reserved_MB: 26212.0 | inactive_split_MB: 171.8 +rank7 [2026-10-29 17:09:08,355] INFO : Step 38400/50000 rank7 | Train Loss: 0.9354 | Train Loss Unweighted: 0.4735 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.0968 | Classification: 0.0760 | Classification_binary: 0.1246 | HitGrouping: 0.1889 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1340 | Regression_eta: 0.0013 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0494 | Regression_eta: 0.0350 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0466 | LR: 2.14e-04 | DataLoad Time: 0.0195s | Model Forward Time: 3.6858s | Phase Times: max_pre_step_barrier: 0.1107 | max_batch_to_device: 0.0062 | max_forward: 0.6635 | max_loss: 0.1419 | max_backward: 1.4771 | max_optimizer_step: 0.1685 | max_tensorboard_logging: 0.4619 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0209 | Memory: rss_MB: 5721.1 | uss_MB: 4559.4 | child_rss_MB: 5662.2 | allocated_MB: 370.2 | reserved_MB: 26212.0 | inactive_split_MB: 267.8 +rank7 [2026-10-29 17:13:06,091] INFO : Step 38500/50000 rank7 | Train Loss: 1.0482 | Train Loss Unweighted: 0.5077 | Train Loss Components: Auxiliary: 0.2332 | Cardinality: 0.1228 | Classification: 0.1147 | Classification_binary: 0.1473 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1208 | Regression_eta: 0.0016 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0519 | Regression_eta: 0.0343 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0594 | LR: 2.12e-04 | DataLoad Time: 0.0201s | Model Forward Time: 3.4596s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.6383 | max_loss: 0.1393 | max_backward: 1.3136 | max_optimizer_step: 0.1695 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1208 | Memory: rss_MB: 5717.0 | uss_MB: 4562.8 | child_rss_MB: 5658.1 | allocated_MB: 370.2 | reserved_MB: 27188.0 | inactive_split_MB: 210.0 +rank7 [2026-10-29 17:17:04,837] INFO : Step 38600/50000 rank7 | Train Loss: 1.0321 | Train Loss Unweighted: 0.4950 | Train Loss Components: Auxiliary: 0.2321 | Cardinality: 0.1157 | Classification: 0.0910 | Classification_binary: 0.1735 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1130 | Regression_eta: 0.0014 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.1899 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0586 | Regression_eta: 0.0443 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0674 | LR: 2.11e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.0838s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0127 | max_forward: 0.7516 | max_loss: 0.1551 | max_backward: 1.6059 | max_optimizer_step: 0.2400 | max_tensorboard_logging: 0.7250 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5714.3 | uss_MB: 4563.8 | child_rss_MB: 5662.6 | allocated_MB: 372.6 | reserved_MB: 28172.0 | inactive_split_MB: 283.9 +rank7 [2026-10-29 17:20:59,240] INFO : Step 38700/50000 rank7 | Train Loss: 0.9420 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1147 | Classification: 0.0828 | Classification_binary: 0.1348 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1155 | Regression_eta: 0.0013 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0401 | Regression_energy: 0.0553 | Regression_eta: 0.0328 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0511 | LR: 2.09e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.6827s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0075 | max_forward: 0.8052 | max_loss: 0.2133 | max_backward: 1.3215 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5712.0 | uss_MB: 4410.4 | child_rss_MB: 5659.5 | allocated_MB: 367.4 | reserved_MB: 28172.0 | inactive_split_MB: 206.7 +rank7 [2026-10-29 17:25:08,219] INFO : Step 38800/50000 rank7 | Train Loss: 0.9989 | Train Loss Unweighted: 0.4825 | Train Loss Components: Auxiliary: 0.2201 | Cardinality: 0.1186 | Classification: 0.0965 | Classification_binary: 0.1429 | HitGrouping: 0.1955 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1198 | Regression_eta: 0.0012 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.1274 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0575 | Regression_eta: 0.0290 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0831 | LR: 2.07e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.6551s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.9011 | max_loss: 0.1880 | max_backward: 1.7487 | max_optimizer_step: 0.2166 | max_tensorboard_logging: 0.6271 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 5710.9 | uss_MB: 4431.1 | child_rss_MB: 5666.7 | allocated_MB: 371.5 | reserved_MB: 28172.0 | inactive_split_MB: 220.5 +rank7 [2026-10-29 17:29:07,492] INFO : Step 38900/50000 rank7 | Train Loss: 0.9986 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2211 | Cardinality: 0.1078 | Classification: 0.0988 | Classification_binary: 0.1578 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1155 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.1305 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0510 | Regression_eta: 0.0279 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0428 | LR: 2.05e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.4426s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0073 | max_forward: 0.9800 | max_loss: 0.2159 | max_backward: 2.2158 | max_optimizer_step: 0.1682 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0079 | Memory: rss_MB: 5714.2 | uss_MB: 3754.5 | child_rss_MB: 5665.7 | allocated_MB: 373.3 | reserved_MB: 29164.0 | inactive_split_MB: 189.2 +rank7 [2026-10-29 17:33:02,383] INFO : Step 39000/50000 rank7 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4758 | Train Loss Components: Auxiliary: 0.2186 | Cardinality: 0.1227 | Classification: 0.0847 | Classification_binary: 0.1518 | HitGrouping: 0.1853 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1179 | Regression_eta: 0.0013 | Regression_pt: 0.1189 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1660 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0581 | Regression_eta: 0.0278 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0545 | LR: 2.03e-04 | DataLoad Time: 0.0192s | Model Forward Time: 3.7825s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0515 | max_forward: 0.7410 | max_loss: 0.1790 | max_backward: 1.4655 | max_optimizer_step: 0.1710 | max_tensorboard_logging: 0.4739 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5713.4 | uss_MB: 4569.9 | child_rss_MB: 5663.9 | allocated_MB: 370.5 | reserved_MB: 29164.0 | inactive_split_MB: 192.2 +rank7 [2026-10-29 17:33:03,294] INFO : Running validation on rank7 +rank7 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 17:33:03,294] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 17:33:56,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:33:56,829] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 17:34:24,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 17:34:25,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 17:34:26,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 17:34:27,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 17:34:37,294] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 17:34:38,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:34:38,630] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 17:34:54,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 17:34:55,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 17:34:56,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 17:34:57,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 17:35:07,387] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 17:35:07,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:35:08,263] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 17:35:24,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 17:35:25,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 17:35:25,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 17:35:26,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_39000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 17:35:36,435] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 17:39:50,705] INFO : Step 39100/50000 rank7 | Train Loss: 1.0043 | Train Loss Unweighted: 0.4886 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1149 | Classification: 0.0878 | Classification_binary: 0.1320 | HitGrouping: 0.1942 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1329 | Regression_eta: 0.0012 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0535 | Regression_eta: 0.0276 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0583 | LR: 2.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.6405s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0736 | max_forward: 0.7590 | max_loss: 0.4062 | max_backward: 1.4330 | max_optimizer_step: 0.1902 | max_tensorboard_logging: 0.4941 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0375 | Memory: rss_MB: 6810.5 | uss_MB: 5664.9 | child_rss_MB: 5641.2 | allocated_MB: 371.1 | reserved_MB: 23656.0 | inactive_split_MB: 194.8 +rank7 [2026-10-29 17:43:44,582] INFO : Step 39200/50000 rank7 | Train Loss: 1.0325 | Train Loss Unweighted: 0.5174 | Train Loss Components: Auxiliary: 0.2315 | Cardinality: 0.1117 | Classification: 0.1086 | Classification_binary: 0.1481 | HitGrouping: 0.1907 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1280 | Regression_eta: 0.0015 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.1269 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0891 | Regression_eta: 0.0771 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0636 | LR: 2.00e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.6956s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0102 | max_forward: 0.7546 | max_loss: 0.4268 | max_backward: 1.4370 | max_optimizer_step: 0.1729 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5801.6 | uss_MB: 4660.4 | child_rss_MB: 5666.2 | allocated_MB: 371.2 | reserved_MB: 23934.0 | inactive_split_MB: 196.6 +rank7 [2026-10-29 17:47:41,749] INFO : Step 39300/50000 rank7 | Train Loss: 1.0373 | Train Loss Unweighted: 0.5050 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1241 | Classification: 0.0956 | Classification_binary: 0.1745 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1150 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.1522 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0478 | Regression_eta: 0.0376 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0609 | LR: 1.98e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.2678s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0844 | max_forward: 0.8662 | max_loss: 0.2199 | max_backward: 1.7416 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5805.0 | uss_MB: 4500.0 | child_rss_MB: 5642.3 | allocated_MB: 373.1 | reserved_MB: 23934.0 | inactive_split_MB: 256.8 +rank7 [2026-10-29 17:51:43,103] INFO : Step 39400/50000 rank7 | Train Loss: 0.9741 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2189 | Cardinality: 0.1078 | Classification: 0.0808 | Classification_binary: 0.1461 | HitGrouping: 0.1956 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0012 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0567 | Regression_eta: 0.0580 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0868 | LR: 1.96e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.5596s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.6778 | max_loss: 0.1515 | max_backward: 1.4156 | max_optimizer_step: 0.1740 | max_tensorboard_logging: 0.4653 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5807.5 | uss_MB: 4663.4 | child_rss_MB: 5665.4 | allocated_MB: 368.0 | reserved_MB: 23934.0 | inactive_split_MB: 199.1 +rank7 [2026-10-29 17:55:40,097] INFO : Step 39500/50000 rank7 | Train Loss: 0.9809 | Train Loss Unweighted: 0.4664 | Train Loss Components: Auxiliary: 0.2175 | Cardinality: 0.1105 | Classification: 0.0963 | Classification_binary: 0.1347 | HitGrouping: 0.1979 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1158 | Regression_eta: 0.0014 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0558 | Regression_eta: 0.0342 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0760 | LR: 1.94e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.5551s | Phase Times: max_pre_step_barrier: 0.0259 | max_batch_to_device: 0.1050 | max_forward: 0.6813 | max_loss: 0.1541 | max_backward: 1.3449 | max_optimizer_step: 0.1709 | max_tensorboard_logging: 0.5375 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0111 | Memory: rss_MB: 5806.7 | uss_MB: 4606.4 | child_rss_MB: 5664.0 | allocated_MB: 367.2 | reserved_MB: 24970.0 | inactive_split_MB: 230.9 +rank7 [2026-10-29 17:59:42,116] INFO : Step 39600/50000 rank7 | Train Loss: 1.0814 | Train Loss Unweighted: 0.5099 | Train Loss Components: Auxiliary: 0.2411 | Cardinality: 0.1344 | Classification: 0.0978 | Classification_binary: 0.1731 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0013 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0535 | Regression_eta: 0.0246 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0526 | LR: 1.93e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1490s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.8839 | max_loss: 0.1464 | max_backward: 1.8057 | max_optimizer_step: 0.2568 | max_tensorboard_logging: 0.5748 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 5805.3 | uss_MB: 4666.8 | child_rss_MB: 5646.1 | allocated_MB: 370.6 | reserved_MB: 24970.0 | inactive_split_MB: 243.4 +rank7 [2026-10-29 18:03:55,339] INFO : Step 39700/50000 rank7 | Train Loss: 1.0174 | Train Loss Unweighted: 0.4877 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1185 | Classification: 0.0909 | Classification_binary: 0.1674 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1134 | Regression_eta: 0.0013 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1781 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0563 | Regression_eta: 0.0309 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0540 | LR: 1.91e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.8041s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.6954 | max_loss: 0.2515 | max_backward: 1.5651 | max_optimizer_step: 0.1765 | max_tensorboard_logging: 0.5599 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5806.9 | uss_MB: 4667.8 | child_rss_MB: 5665.2 | allocated_MB: 371.1 | reserved_MB: 24970.0 | inactive_split_MB: 244.9 +rank7 [2026-10-29 18:07:56,651] INFO : Step 39800/50000 rank7 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.2057 | Cardinality: 0.1022 | Classification: 0.0711 | Classification_binary: 0.1322 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1188 | Regression_eta: 0.0012 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0326 | Regression_energy: 0.0732 | Regression_eta: 0.0321 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0302 | LR: 1.89e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.9309s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0063 | max_forward: 1.0012 | max_loss: 0.2859 | max_backward: 2.1063 | max_optimizer_step: 0.1800 | max_tensorboard_logging: 0.4881 | max_tensorboard_flush: 0.0065 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 5810.9 | uss_MB: 3839.1 | child_rss_MB: 5666.9 | allocated_MB: 368.3 | reserved_MB: 24970.0 | inactive_split_MB: 241.7 +rank7 [2026-10-29 18:11:54,162] INFO : Step 39900/50000 rank7 | Train Loss: 0.9667 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.2188 | Cardinality: 0.1055 | Classification: 0.0897 | Classification_binary: 0.1526 | HitGrouping: 0.1883 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1101 | Regression_eta: 0.0012 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0312 | Regression_energy: 0.0641 | Regression_eta: 0.0235 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0439 | LR: 1.88e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.9943s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.3186 | max_loss: 0.4377 | max_backward: 3.6189 | max_optimizer_step: 0.1897 | max_tensorboard_logging: 0.6123 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0551 | Memory: rss_MB: 5815.3 | uss_MB: 4217.6 | child_rss_MB: 5667.9 | allocated_MB: 370.5 | reserved_MB: 24970.0 | inactive_split_MB: 229.4 +rank7 [2026-10-29 18:15:59,749] INFO : Step 40000/50000 rank7 | Train Loss: 1.0232 | Train Loss Unweighted: 0.5131 | Train Loss Components: Auxiliary: 0.2348 | Cardinality: 0.1100 | Classification: 0.0962 | Classification_binary: 0.1712 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1206 | Regression_eta: 0.0013 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0949 | Regression_energy: 0.0675 | Regression_eta: 0.0464 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0744 | LR: 1.86e-04 | DataLoad Time: 0.0197s | Model Forward Time: 7.4609s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0077 | max_forward: 0.8168 | max_loss: 0.2161 | max_backward: 1.5709 | max_optimizer_step: 0.1733 | max_tensorboard_logging: 1.1681 | max_tensorboard_flush: 0.0088 | max_post_step_barrier: 0.1427 | Memory: rss_MB: 5816.6 | uss_MB: 3786.6 | child_rss_MB: 5665.4 | allocated_MB: 375.8 | reserved_MB: 24970.0 | inactive_split_MB: 259.2 +rank7 [2026-10-29 18:16:02,308] INFO : Running validation on rank7 +rank7 [2026-10-29 18:16:02,309] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 18:16:02,309] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 18:16:53,650] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:16:54,181] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 18:17:19,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 18:17:20,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 18:17:21,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 18:17:22,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 18:17:32,543] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:17:33,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:17:33,664] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 18:17:49,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 18:17:50,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 18:17:51,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 18:17:52,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 18:18:02,311] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 18:18:03,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:18:03,746] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 18:18:19,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 18:18:19,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 18:18:20,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 18:18:21,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_40000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 18:18:31,336] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 18:22:46,459] INFO : Step 40100/50000 rank7 | Train Loss: 0.9349 | Train Loss Unweighted: 0.4412 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1220 | Classification: 0.0762 | Classification_binary: 0.1328 | HitGrouping: 0.1873 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1148 | Regression_eta: 0.0012 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0562 | Regression_eta: 0.0312 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0623 | LR: 1.84e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.6889s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0061 | max_forward: 0.7093 | max_loss: 0.3974 | max_backward: 1.3310 | max_optimizer_step: 0.1949 | max_tensorboard_logging: 0.5555 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 7103.1 | uss_MB: 5953.7 | child_rss_MB: 5672.5 | allocated_MB: 367.6 | reserved_MB: 22178.0 | inactive_split_MB: 265.9 +rank7 [2026-10-29 18:26:52,186] INFO : Step 40200/50000 rank7 | Train Loss: 0.9570 | Train Loss Unweighted: 0.4556 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1119 | Classification: 0.0905 | Classification_binary: 0.1293 | HitGrouping: 0.1868 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1165 | Regression_eta: 0.0013 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1051 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0690 | Regression_eta: 0.0239 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0370 | LR: 1.83e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.5895s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.6287 | max_loss: 0.4129 | max_backward: 1.3901 | max_optimizer_step: 0.2281 | max_tensorboard_logging: 0.5609 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5918.3 | uss_MB: 4768.5 | child_rss_MB: 5665.4 | allocated_MB: 370.8 | reserved_MB: 22294.0 | inactive_split_MB: 233.2 +rank7 [2026-10-29 18:30:59,535] INFO : Step 40300/50000 rank7 | Train Loss: 1.0209 | Train Loss Unweighted: 0.4860 | Train Loss Components: Auxiliary: 0.2244 | Cardinality: 0.1232 | Classification: 0.0991 | Classification_binary: 0.1688 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1068 | Regression_eta: 0.0015 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1596 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0437 | Regression_eta: 0.0312 | Regression_pt: 0.0459 | Regression_sin_phi: 0.0555 | LR: 1.81e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9656s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0749 | max_forward: 0.7563 | max_loss: 0.1523 | max_backward: 1.5452 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5632 | max_tensorboard_flush: 0.0039 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 5917.0 | uss_MB: 4772.2 | child_rss_MB: 5652.3 | allocated_MB: 371.8 | reserved_MB: 23296.0 | inactive_split_MB: 234.2 +rank7 [2026-10-29 18:35:20,932] INFO : Step 40400/50000 rank7 | Train Loss: 1.0486 | Train Loss Unweighted: 0.5003 | Train Loss Components: Auxiliary: 0.2403 | Cardinality: 0.1357 | Classification: 0.0888 | Classification_binary: 0.1690 | HitGrouping: 0.1879 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1196 | Regression_eta: 0.0015 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1830 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0507 | Regression_eta: 0.0453 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0396 | LR: 1.79e-04 | DataLoad Time: 0.0144s | Model Forward Time: 5.7370s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0072 | max_forward: 0.7368 | max_loss: 0.2443 | max_backward: 3.1596 | max_optimizer_step: 0.1851 | max_tensorboard_logging: 0.5233 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0202 | Memory: rss_MB: 5913.1 | uss_MB: 3802.2 | child_rss_MB: 5663.8 | allocated_MB: 371.2 | reserved_MB: 23386.0 | inactive_split_MB: 254.8 +rank7 [2026-10-29 18:39:23,871] INFO : Step 40500/50000 rank7 | Train Loss: 0.9309 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.0955 | Classification: 0.0829 | Classification_binary: 0.1355 | HitGrouping: 0.1916 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0014 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1409 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0498 | Regression_eta: 0.0348 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0555 | LR: 1.78e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7230s | Phase Times: max_pre_step_barrier: 0.0186 | max_batch_to_device: 0.0076 | max_forward: 0.6863 | max_loss: 0.2393 | max_backward: 1.3065 | max_optimizer_step: 0.2791 | max_tensorboard_logging: 0.7004 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0234 | Memory: rss_MB: 5916.1 | uss_MB: 4771.6 | child_rss_MB: 5669.9 | allocated_MB: 369.2 | reserved_MB: 24370.0 | inactive_split_MB: 237.3 +rank7 [2026-10-29 18:43:23,923] INFO : Step 40600/50000 rank7 | Train Loss: 1.0104 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2258 | Cardinality: 0.1087 | Classification: 0.0887 | Classification_binary: 0.1521 | HitGrouping: 0.1896 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1140 | Regression_eta: 0.0014 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0527 | Regression_eta: 0.0397 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0618 | LR: 1.76e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.3451s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.1102 | max_forward: 0.7412 | max_loss: 0.1464 | max_backward: 1.4782 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 1.0719 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5917.7 | uss_MB: 4772.7 | child_rss_MB: 5655.3 | allocated_MB: 372.3 | reserved_MB: 24370.0 | inactive_split_MB: 234.4 +rank7 [2026-10-29 18:47:14,843] INFO : Step 40700/50000 rank7 | Train Loss: 1.0611 | Train Loss Unweighted: 0.5075 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1374 | Classification: 0.0904 | Classification_binary: 0.1773 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1190 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1838 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0660 | Regression_eta: 0.0346 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0407 | LR: 1.75e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2480s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 0.7422 | max_loss: 0.1589 | max_backward: 1.5298 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.9328 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5918.6 | uss_MB: 4774.9 | child_rss_MB: 5665.5 | allocated_MB: 372.7 | reserved_MB: 24370.0 | inactive_split_MB: 255.3 +rank7 [2026-10-29 18:51:03,370] INFO : Step 40800/50000 rank7 | Train Loss: 0.9701 | Train Loss Unweighted: 0.4785 | Train Loss Components: Auxiliary: 0.2099 | Cardinality: 0.0983 | Classification: 0.0901 | Classification_binary: 0.1373 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1239 | Regression_eta: 0.0014 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0480 | Regression_eta: 0.0281 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0477 | LR: 1.73e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.7280s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0069 | max_forward: 0.5843 | max_loss: 0.1422 | max_backward: 1.0661 | max_optimizer_step: 0.1736 | max_tensorboard_logging: 1.0389 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5915.8 | uss_MB: 4772.0 | child_rss_MB: 5661.2 | allocated_MB: 367.5 | reserved_MB: 24658.0 | inactive_split_MB: 205.4 +rank7 [2026-10-29 18:54:43,248] INFO : Step 40900/50000 rank7 | Train Loss: 0.9731 | Train Loss Unweighted: 0.4636 | Train Loss Components: Auxiliary: 0.2169 | Cardinality: 0.1073 | Classification: 0.0883 | Classification_binary: 0.1426 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1140 | Regression_eta: 0.0013 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0473 | Regression_eta: 0.0488 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0672 | LR: 1.72e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.8711s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.1047 | max_forward: 0.7107 | max_loss: 0.1679 | max_backward: 1.3343 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.8664 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0251 | Memory: rss_MB: 5919.0 | uss_MB: 4778.3 | child_rss_MB: 5681.4 | allocated_MB: 372.0 | reserved_MB: 25640.0 | inactive_split_MB: 260.9 +rank7 [2026-10-29 18:58:25,754] INFO : Step 41000/50000 rank7 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4913 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1031 | Classification: 0.0930 | Classification_binary: 0.1618 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1164 | Regression_eta: 0.0014 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.1636 | Regression_cos_phi: 0.1091 | Regression_energy: 0.0521 | Regression_eta: 0.0400 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0451 | LR: 1.70e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4152s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.7604 | max_loss: 0.2558 | max_backward: 1.7603 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.8399 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0238 | Memory: rss_MB: 5926.0 | uss_MB: 4782.0 | child_rss_MB: 5654.3 | allocated_MB: 371.9 | reserved_MB: 25640.0 | inactive_split_MB: 195.8 +rank7 [2026-10-29 18:58:26,765] INFO : Running validation on rank7 +rank7 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 18:58:26,765] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 18:59:14,712] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:59:15,274] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 18:59:40,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 18:59:41,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 18:59:41,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 18:59:42,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 18:59:52,807] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 18:59:53,544] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:59:54,025] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 19:00:10,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 19:00:11,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 19:00:12,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 19:00:12,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 19:00:22,940] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 19:00:23,208] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:00:23,685] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 19:00:39,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 19:00:40,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 19:00:40,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 19:00:41,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_41000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 19:00:51,348] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 19:04:58,255] INFO : Step 41100/50000 rank7 | Train Loss: 1.0684 | Train Loss Unweighted: 0.5053 | Train Loss Components: Auxiliary: 0.2374 | Cardinality: 0.1438 | Classification: 0.0856 | Classification_binary: 0.1616 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1271 | Regression_eta: 0.0014 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1903 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0607 | Regression_eta: 0.0571 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0623 | LR: 1.69e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3334s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0066 | max_forward: 0.8006 | max_loss: 0.1726 | max_backward: 1.6699 | max_optimizer_step: 0.2247 | max_tensorboard_logging: 0.6214 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0566 | Memory: rss_MB: 7184.9 | uss_MB: 5901.7 | child_rss_MB: 5647.6 | allocated_MB: 371.5 | reserved_MB: 22898.0 | inactive_split_MB: 258.4 +rank7 [2026-10-29 19:08:41,623] INFO : Step 41200/50000 rank7 | Train Loss: 0.9455 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1027 | Classification: 0.0887 | Classification_binary: 0.1242 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1223 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1351 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0488 | Regression_eta: 0.0414 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0550 | LR: 1.67e-04 | DataLoad Time: 0.0149s | Model Forward Time: 3.4129s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0589 | max_forward: 0.6517 | max_loss: 0.1401 | max_backward: 1.1761 | max_optimizer_step: 0.1806 | max_tensorboard_logging: 0.5685 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0200 | Memory: rss_MB: 5984.6 | uss_MB: 4847.1 | child_rss_MB: 5628.1 | allocated_MB: 370.2 | reserved_MB: 24878.0 | inactive_split_MB: 199.0 +rank7 [2026-10-29 19:12:28,247] INFO : Step 41300/50000 rank7 | Train Loss: 0.9373 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1011 | Classification: 0.0853 | Classification_binary: 0.1448 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0013 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1129 | Regression_cos_phi: 0.0366 | Regression_energy: 0.0515 | Regression_eta: 0.0375 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0334 | LR: 1.66e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8986s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.8726 | max_loss: 0.1617 | max_backward: 1.6974 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.5417 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5987.6 | uss_MB: 4852.4 | child_rss_MB: 5623.6 | allocated_MB: 370.7 | reserved_MB: 24878.0 | inactive_split_MB: 233.6 +rank7 [2026-10-29 19:16:15,464] INFO : Step 41400/50000 rank7 | Train Loss: 1.0355 | Train Loss Unweighted: 0.4965 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1363 | Classification: 0.0957 | Classification_binary: 0.1654 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1162 | Regression_eta: 0.0012 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1574 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0454 | Regression_eta: 0.0310 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0366 | LR: 1.64e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9189s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.7129 | max_loss: 0.2460 | max_backward: 1.6340 | max_optimizer_step: 0.1760 | max_tensorboard_logging: 0.5870 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0204 | Memory: rss_MB: 5967.8 | uss_MB: 4849.9 | child_rss_MB: 5563.5 | allocated_MB: 372.0 | reserved_MB: 24878.0 | inactive_split_MB: 305.4 +rank7 [2026-10-29 19:19:52,654] INFO : Step 41500/50000 rank7 | Train Loss: 0.9689 | Train Loss Unweighted: 0.4689 | Train Loss Components: Auxiliary: 0.2118 | Cardinality: 0.1040 | Classification: 0.0862 | Classification_binary: 0.1340 | HitGrouping: 0.1906 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1227 | Regression_eta: 0.0012 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0570 | Regression_eta: 0.0335 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0629 | LR: 1.63e-04 | DataLoad Time: 0.0194s | Model Forward Time: 3.4466s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0062 | max_forward: 0.6415 | max_loss: 0.1502 | max_backward: 1.1845 | max_optimizer_step: 0.1748 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 5935.8 | uss_MB: 4853.4 | child_rss_MB: 5403.5 | allocated_MB: 368.5 | reserved_MB: 24878.0 | inactive_split_MB: 173.5 +rank7 [2026-10-29 19:23:40,392] INFO : Step 41600/50000 rank7 | Train Loss: 0.9559 | Train Loss Unweighted: 0.4625 | Train Loss Components: Auxiliary: 0.2141 | Cardinality: 0.1097 | Classification: 0.0872 | Classification_binary: 0.1298 | HitGrouping: 0.1967 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1211 | Regression_eta: 0.0012 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1170 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0671 | Regression_eta: 0.0412 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0482 | LR: 1.61e-04 | DataLoad Time: 0.0187s | Model Forward Time: 3.5882s | Phase Times: max_pre_step_barrier: 0.0755 | max_batch_to_device: 0.0062 | max_forward: 0.6849 | max_loss: 0.2047 | max_backward: 1.1646 | max_optimizer_step: 0.2235 | max_tensorboard_logging: 0.5806 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0269 | Memory: rss_MB: 5909.3 | uss_MB: 4827.6 | child_rss_MB: 5308.7 | allocated_MB: 370.4 | reserved_MB: 24878.0 | inactive_split_MB: 287.4 +rank7 [2026-10-29 19:27:23,743] INFO : Step 41700/50000 rank7 | Train Loss: 1.0204 | Train Loss Unweighted: 0.4905 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1165 | Classification: 0.0860 | Classification_binary: 0.1552 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0014 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1275 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0594 | Regression_eta: 0.0259 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0509 | LR: 1.60e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.0312s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0773 | max_forward: 0.7754 | max_loss: 0.1486 | max_backward: 1.5759 | max_optimizer_step: 0.2520 | max_tensorboard_logging: 0.5773 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 5907.3 | uss_MB: 4857.3 | child_rss_MB: 5302.2 | allocated_MB: 370.8 | reserved_MB: 24878.0 | inactive_split_MB: 238.3 +rank7 [2026-10-29 19:31:24,300] INFO : Step 41800/50000 rank7 | Train Loss: 1.0101 | Train Loss Unweighted: 0.4703 | Train Loss Components: Auxiliary: 0.2243 | Cardinality: 0.1282 | Classification: 0.0818 | Classification_binary: 0.1618 | HitGrouping: 0.1890 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1741 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0653 | Regression_eta: 0.0472 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0905 | LR: 1.58e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.9767s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.8147 | max_loss: 0.4259 | max_backward: 1.4886 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5559 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1200 | Memory: rss_MB: 5899.0 | uss_MB: 4855.0 | child_rss_MB: 5276.1 | allocated_MB: 371.9 | reserved_MB: 25802.0 | inactive_split_MB: 266.5 +rank7 [2026-10-29 19:35:32,346] INFO : Step 41900/50000 rank7 | Train Loss: 0.9589 | Train Loss Unweighted: 0.4672 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1185 | Classification: 0.0878 | Classification_binary: 0.1282 | HitGrouping: 0.1927 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1241 | Regression_eta: 0.0011 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0625 | Regression_eta: 0.0410 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0488 | LR: 1.57e-04 | DataLoad Time: 0.0167s | Model Forward Time: 3.4632s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0473 | max_forward: 0.6783 | max_loss: 0.1431 | max_backward: 1.2347 | max_optimizer_step: 0.1795 | max_tensorboard_logging: 0.5634 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0225 | Memory: rss_MB: 5868.0 | uss_MB: 4858.2 | child_rss_MB: 5156.0 | allocated_MB: 370.8 | reserved_MB: 25802.0 | inactive_split_MB: 166.5 +rank7 [2026-10-29 19:39:32,016] INFO : Step 42000/50000 rank7 | Train Loss: 0.9837 | Train Loss Unweighted: 0.4719 | Train Loss Components: Auxiliary: 0.2204 | Cardinality: 0.1093 | Classification: 0.1015 | Classification_binary: 0.1413 | HitGrouping: 0.1899 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1128 | Regression_eta: 0.0014 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1241 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0608 | Regression_eta: 0.0219 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0577 | LR: 1.56e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.6392s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0072 | max_forward: 0.6848 | max_loss: 0.2509 | max_backward: 1.3746 | max_optimizer_step: 0.1770 | max_tensorboard_logging: 0.5786 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5866.0 | uss_MB: 4860.4 | child_rss_MB: 5161.1 | allocated_MB: 370.2 | reserved_MB: 25802.0 | inactive_split_MB: 219.9 +rank7 [2026-10-29 19:39:32,941] INFO : Running validation on rank7 +rank7 [2026-10-29 19:39:32,941] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 19:39:32,941] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 19:40:23,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:40:23,889] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 19:40:52,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 19:40:53,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 19:40:54,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 19:40:55,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 19:41:05,415] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 19:41:06,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:41:06,796] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 19:41:23,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 19:41:25,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 19:41:26,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 19:41:27,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 19:41:37,083] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 19:41:38,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:41:38,675] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 19:41:54,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 19:41:54,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 19:41:55,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 19:41:56,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_42000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 19:42:06,388] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 19:46:29,430] INFO : Step 42100/50000 rank7 | Train Loss: 1.0639 | Train Loss Unweighted: 0.4964 | Train Loss Components: Auxiliary: 0.2447 | Cardinality: 0.1434 | Classification: 0.0999 | Classification_binary: 0.1737 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1097 | Regression_eta: 0.0012 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1921 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0474 | Regression_eta: 0.0519 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0565 | LR: 1.54e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0365s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.7424 | max_loss: 0.1586 | max_backward: 1.4434 | max_optimizer_step: 0.3014 | max_tensorboard_logging: 0.6771 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 7018.7 | uss_MB: 6010.1 | child_rss_MB: 5164.1 | allocated_MB: 370.8 | reserved_MB: 23298.0 | inactive_split_MB: 265.3 +rank7 [2026-10-29 19:50:21,672] INFO : Step 42200/50000 rank7 | Train Loss: 1.0152 | Train Loss Unweighted: 0.4831 | Train Loss Components: Auxiliary: 0.2206 | Cardinality: 0.1169 | Classification: 0.0913 | Classification_binary: 0.1515 | HitGrouping: 0.1921 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1190 | Regression_eta: 0.0015 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0586 | Regression_eta: 0.0368 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0761 | LR: 1.53e-04 | DataLoad Time: 0.0191s | Model Forward Time: 3.5449s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0063 | max_forward: 0.6567 | max_loss: 0.2000 | max_backward: 1.2379 | max_optimizer_step: 0.1803 | max_tensorboard_logging: 0.5782 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5943.7 | uss_MB: 4948.2 | child_rss_MB: 5171.4 | allocated_MB: 370.9 | reserved_MB: 24984.0 | inactive_split_MB: 272.4 +rank7 [2026-10-29 19:51:24,069] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank7 [2026-10-29 19:51:24,069] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-29 19:54:15,428] INFO : Step 42300/50000 rank7 | Train Loss: 1.0224 | Train Loss Unweighted: 0.4869 | Train Loss Components: Auxiliary: 0.2326 | Cardinality: 0.1261 | Classification: 0.0943 | Classification_binary: 0.1704 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1790 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0491 | Regression_eta: 0.0392 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0458 | LR: 1.52e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.7694s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0064 | max_forward: 0.7065 | max_loss: 0.2237 | max_backward: 1.4902 | max_optimizer_step: 0.1814 | max_tensorboard_logging: 0.5353 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0265 | Memory: rss_MB: 6010.9 | uss_MB: 5019.0 | child_rss_MB: 5170.2 | allocated_MB: 371.9 | reserved_MB: 24984.0 | inactive_split_MB: 183.6 +rank7 [2026-10-29 19:58:13,286] INFO : Step 42400/50000 rank7 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4895 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1253 | Classification: 0.0872 | Classification_binary: 0.1616 | HitGrouping: 0.1926 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1184 | Regression_eta: 0.0013 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0566 | Regression_eta: 0.0279 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0666 | LR: 1.50e-04 | DataLoad Time: 0.0147s | Model Forward Time: 3.8008s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7436 | max_loss: 0.2477 | max_backward: 1.5279 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5169 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 5933.8 | uss_MB: 4947.5 | child_rss_MB: 5164.9 | allocated_MB: 373.0 | reserved_MB: 24984.0 | inactive_split_MB: 275.6 +rank7 [2026-10-29 20:02:13,261] INFO : Step 42500/50000 rank7 | Train Loss: 0.9211 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1056 | Classification: 0.0789 | Classification_binary: 0.1296 | HitGrouping: 0.1893 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1162 | Regression_eta: 0.0011 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0477 | Regression_eta: 0.0316 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0662 | LR: 1.49e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.5543s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.6196 | max_loss: 0.2440 | max_backward: 1.3403 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 2.3828 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0244 | Memory: rss_MB: 5926.8 | uss_MB: 4949.7 | child_rss_MB: 5158.5 | allocated_MB: 371.3 | reserved_MB: 25980.0 | inactive_split_MB: 274.9 +rank7 [2026-10-29 20:06:17,860] INFO : Step 42600/50000 rank7 | Train Loss: 0.9688 | Train Loss Unweighted: 0.4619 | Train Loss Components: Auxiliary: 0.2197 | Cardinality: 0.1101 | Classification: 0.0874 | Classification_binary: 0.1472 | HitGrouping: 0.1841 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0012 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1665 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0547 | Regression_eta: 0.0421 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0608 | LR: 1.48e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.9496s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6353 | max_loss: 0.2547 | max_backward: 1.3961 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.8534 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 5918.3 | uss_MB: 4947.3 | child_rss_MB: 5167.3 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 272.5 +rank7 [2026-10-29 20:10:16,915] INFO : Step 42700/50000 rank7 | Train Loss: 1.0476 | Train Loss Unweighted: 0.4808 | Train Loss Components: Auxiliary: 0.2377 | Cardinality: 0.1284 | Classification: 0.0883 | Classification_binary: 0.1739 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1071 | Regression_eta: 0.0014 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1976 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0518 | Regression_eta: 0.0362 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0745 | LR: 1.47e-04 | DataLoad Time: 0.0108s | Model Forward Time: 5.7555s | Phase Times: max_pre_step_barrier: 0.0320 | max_batch_to_device: 0.0114 | max_forward: 2.1726 | max_loss: 0.5101 | max_backward: 2.7494 | max_optimizer_step: 0.2679 | max_tensorboard_logging: 0.6520 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5921.8 | uss_MB: 4952.4 | child_rss_MB: 5151.8 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 482.9 +rank7 [2026-10-29 20:14:17,050] INFO : Step 42800/50000 rank7 | Train Loss: 1.0027 | Train Loss Unweighted: 0.4792 | Train Loss Components: Auxiliary: 0.2262 | Cardinality: 0.1277 | Classification: 0.0841 | Classification_binary: 0.1487 | HitGrouping: 0.1903 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1209 | Regression_eta: 0.0012 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0567 | Regression_eta: 0.0346 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0594 | LR: 1.45e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.2806s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0175 | max_forward: 1.3870 | max_loss: 0.5106 | max_backward: 1.9221 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 2.4017 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5924.3 | uss_MB: 4952.5 | child_rss_MB: 5165.0 | allocated_MB: 372.6 | reserved_MB: 25980.0 | inactive_split_MB: 234.4 +rank7 [2026-10-29 20:18:09,439] INFO : Step 42900/50000 rank7 | Train Loss: 0.8847 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.0993 | Classification: 0.0847 | Classification_binary: 0.1207 | HitGrouping: 0.1828 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1069 | Regression_eta: 0.0011 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1180 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0521 | Regression_eta: 0.0303 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0582 | LR: 1.44e-04 | DataLoad Time: 0.0158s | Model Forward Time: 3.7398s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0082 | max_forward: 0.6133 | max_loss: 0.1478 | max_backward: 1.3445 | max_optimizer_step: 0.1755 | max_tensorboard_logging: 0.7551 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0762 | Memory: rss_MB: 5924.9 | uss_MB: 4957.1 | child_rss_MB: 5179.7 | allocated_MB: 370.8 | reserved_MB: 25980.0 | inactive_split_MB: 276.9 +rank7 [2026-10-29 20:22:02,172] INFO : Step 43000/50000 rank7 | Train Loss: 0.9800 | Train Loss Unweighted: 0.4737 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1096 | Classification: 0.0987 | Classification_binary: 0.1606 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1056 | Regression_eta: 0.0012 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0578 | Regression_eta: 0.0355 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0704 | LR: 1.43e-04 | DataLoad Time: 0.0220s | Model Forward Time: 3.9023s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0071 | max_forward: 0.7477 | max_loss: 0.1477 | max_backward: 1.5663 | max_optimizer_step: 0.1600 | max_tensorboard_logging: 0.5954 | max_tensorboard_flush: 0.0053 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 5929.7 | uss_MB: 4961.8 | child_rss_MB: 5171.4 | allocated_MB: 369.1 | reserved_MB: 25980.0 | inactive_split_MB: 209.4 +rank7 [2026-10-29 20:22:03,144] INFO : Running validation on rank7 +rank7 [2026-10-29 20:22:03,144] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 20:22:03,145] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 20:22:51,801] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:22:52,330] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 20:23:18,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 20:23:19,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 20:23:20,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 20:23:21,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 20:23:31,268] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 20:23:31,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:23:32,375] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 20:23:48,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 20:23:49,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 20:23:50,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 20:23:51,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 20:24:01,340] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 20:24:01,724] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:24:02,214] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 20:24:17,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 20:24:18,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 20:24:19,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 20:24:19,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_43000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 20:24:29,894] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 20:28:29,103] INFO : Step 43100/50000 rank7 | Train Loss: 0.9692 | Train Loss Unweighted: 0.4514 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1151 | Classification: 0.0766 | Classification_binary: 0.1586 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1062 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0372 | Regression_energy: 0.0377 | Regression_eta: 0.0336 | Regression_pt: 0.0400 | Regression_sin_phi: 0.0503 | LR: 1.42e-04 | DataLoad Time: 0.0144s | Model Forward Time: 3.7276s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0065 | max_forward: 0.8051 | max_loss: 0.1438 | max_backward: 1.3224 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 7068.8 | uss_MB: 6101.3 | child_rss_MB: 5191.0 | allocated_MB: 370.5 | reserved_MB: 21096.0 | inactive_split_MB: 247.5 +rank7 [2026-10-29 20:32:30,812] INFO : Step 43200/50000 rank7 | Train Loss: 0.9431 | Train Loss Unweighted: 0.4458 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1175 | Classification: 0.0811 | Classification_binary: 0.1294 | HitGrouping: 0.1909 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0012 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0563 | Regression_eta: 0.0537 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0535 | LR: 1.40e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5216s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0120 | max_forward: 0.6342 | max_loss: 0.1417 | max_backward: 1.2752 | max_optimizer_step: 0.1856 | max_tensorboard_logging: 0.6198 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6068.4 | uss_MB: 5101.7 | child_rss_MB: 5194.2 | allocated_MB: 370.4 | reserved_MB: 22140.0 | inactive_split_MB: 198.0 +rank7 [2026-10-29 20:36:16,429] INFO : Step 43300/50000 rank7 | Train Loss: 1.0382 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1221 | Classification: 0.1031 | Classification_binary: 0.1555 | HitGrouping: 0.1894 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1173 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0749 | Regression_eta: 0.0449 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0712 | LR: 1.39e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2412s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0266 | max_forward: 0.7527 | max_loss: 0.1476 | max_backward: 1.6633 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.7701 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6071.5 | uss_MB: 5107.2 | child_rss_MB: 5175.9 | allocated_MB: 371.1 | reserved_MB: 23082.0 | inactive_split_MB: 236.4 +rank7 [2026-10-29 20:40:05,495] INFO : Step 43400/50000 rank7 | Train Loss: 1.0532 | Train Loss Unweighted: 0.4978 | Train Loss Components: Auxiliary: 0.2364 | Cardinality: 0.1292 | Classification: 0.0981 | Classification_binary: 0.1785 | HitGrouping: 0.1846 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0015 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1729 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0538 | Regression_eta: 0.0330 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0500 | LR: 1.38e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2961s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0691 | max_forward: 0.8057 | max_loss: 0.1503 | max_backward: 1.8319 | max_optimizer_step: 0.1787 | max_tensorboard_logging: 0.6703 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6067.0 | uss_MB: 5102.4 | child_rss_MB: 5192.5 | allocated_MB: 371.5 | reserved_MB: 23082.0 | inactive_split_MB: 776.0 +rank7 [2026-10-29 20:43:57,008] INFO : Step 43500/50000 rank7 | Train Loss: 0.9308 | Train Loss Unweighted: 0.4531 | Train Loss Components: Auxiliary: 0.2084 | Cardinality: 0.0966 | Classification: 0.0846 | Classification_binary: 0.1384 | HitGrouping: 0.1914 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1135 | Regression_eta: 0.0013 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0585 | Regression_eta: 0.0434 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0496 | LR: 1.37e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.6267s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0171 | max_forward: 0.6718 | max_loss: 0.1455 | max_backward: 1.4141 | max_optimizer_step: 0.1861 | max_tensorboard_logging: 0.4733 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 6070.4 | uss_MB: 5108.5 | child_rss_MB: 5191.0 | allocated_MB: 372.0 | reserved_MB: 23082.0 | inactive_split_MB: 228.0 +rank7 [2026-10-29 20:47:52,743] INFO : Step 43600/50000 rank7 | Train Loss: 0.9195 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.0926 | Classification: 0.0847 | Classification_binary: 0.1298 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1175 | Regression_eta: 0.0013 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0551 | Regression_eta: 0.0372 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0749 | LR: 1.36e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5149s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.6453 | max_loss: 0.1399 | max_backward: 1.3514 | max_optimizer_step: 0.1837 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0942 | Memory: rss_MB: 6067.9 | uss_MB: 5106.0 | child_rss_MB: 5190.1 | allocated_MB: 367.6 | reserved_MB: 23082.0 | inactive_split_MB: 270.4 +rank7 [2026-10-29 20:51:54,645] INFO : Step 43700/50000 rank7 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2167 | Cardinality: 0.1181 | Classification: 0.0916 | Classification_binary: 0.1553 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1009 | Regression_eta: 0.0014 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1538 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0469 | Regression_eta: 0.0272 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0451 | LR: 1.35e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.0846s | Phase Times: max_pre_step_barrier: 0.0199 | max_batch_to_device: 0.0121 | max_forward: 0.8005 | max_loss: 0.2526 | max_backward: 1.9159 | max_optimizer_step: 0.1747 | max_tensorboard_logging: 0.5001 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0248 | Memory: rss_MB: 6075.4 | uss_MB: 5110.5 | child_rss_MB: 5168.5 | allocated_MB: 371.6 | reserved_MB: 23674.0 | inactive_split_MB: 256.4 +rank7 [2026-10-29 20:55:51,188] INFO : Step 43800/50000 rank7 | Train Loss: 1.0325 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1341 | Classification: 0.0892 | Classification_binary: 0.1542 | HitGrouping: 0.1886 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0012 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0599 | Regression_eta: 0.0184 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0465 | LR: 1.34e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.6848s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.7618 | max_loss: 0.1715 | max_backward: 1.3818 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.4647 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6073.5 | uss_MB: 5110.6 | child_rss_MB: 5186.9 | allocated_MB: 371.7 | reserved_MB: 23674.0 | inactive_split_MB: 282.3 +rank7 [2026-10-29 20:59:46,232] INFO : Step 43900/50000 rank7 | Train Loss: 0.9104 | Train Loss Unweighted: 0.4325 | Train Loss Components: Auxiliary: 0.2009 | Cardinality: 0.1078 | Classification: 0.0725 | Classification_binary: 0.1272 | HitGrouping: 0.1855 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1146 | Regression_eta: 0.0011 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1296 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0470 | Regression_eta: 0.0252 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0411 | LR: 1.33e-04 | DataLoad Time: 0.0204s | Model Forward Time: 3.3853s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.6145 | max_loss: 0.1412 | max_backward: 1.1415 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.5764 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 6071.8 | uss_MB: 5111.9 | child_rss_MB: 5178.4 | allocated_MB: 368.3 | reserved_MB: 23674.0 | inactive_split_MB: 193.1 +rank7 [2026-10-29 21:03:43,512] INFO : Step 44000/50000 rank7 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4573 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1078 | Classification: 0.0956 | Classification_binary: 0.1421 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1083 | Regression_eta: 0.0011 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1180 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0620 | Regression_eta: 0.0285 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0310 | LR: 1.32e-04 | DataLoad Time: 0.0198s | Model Forward Time: 3.9484s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0061 | max_forward: 1.1697 | max_loss: 0.2455 | max_backward: 1.7366 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 0.4855 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0257 | Memory: rss_MB: 6074.9 | uss_MB: 5118.1 | child_rss_MB: 5179.2 | allocated_MB: 370.7 | reserved_MB: 23674.0 | inactive_split_MB: 207.3 +rank7 [2026-10-29 21:03:44,527] INFO : Running validation on rank7 +rank7 [2026-10-29 21:03:44,527] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 21:03:44,528] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 21:04:32,715] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:04:33,227] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 21:04:58,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 21:04:59,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 21:05:01,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 21:05:01,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 21:05:11,935] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 21:05:12,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:05:13,000] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 21:05:28,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 21:05:29,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 21:05:30,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 21:05:31,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 21:05:41,796] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 21:05:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:05:42,735] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 21:05:58,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 21:05:59,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 21:05:59,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 21:06:00,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_44000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 21:06:10,575] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 21:10:20,129] INFO : Step 44100/50000 rank7 | Train Loss: 1.0009 | Train Loss Unweighted: 0.4605 | Train Loss Components: Auxiliary: 0.2311 | Cardinality: 0.1400 | Classification: 0.0841 | Classification_binary: 0.1644 | HitGrouping: 0.1842 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1044 | Regression_eta: 0.0011 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1707 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0497 | Regression_eta: 0.0297 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0464 | LR: 1.31e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.5837s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0070 | max_forward: 1.2763 | max_loss: 0.5349 | max_backward: 2.2676 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 1.2755 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 7076.5 | uss_MB: 6129.8 | child_rss_MB: 5164.2 | allocated_MB: 372.3 | reserved_MB: 24224.0 | inactive_split_MB: 272.2 +rank7 [2026-10-29 21:14:15,311] INFO : Step 44200/50000 rank7 | Train Loss: 0.9801 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.2190 | Cardinality: 0.1146 | Classification: 0.0882 | Classification_binary: 0.1482 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1167 | Regression_eta: 0.0014 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0501 | Regression_eta: 0.0488 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0556 | LR: 1.30e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.0045s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0132 | max_forward: 0.6993 | max_loss: 0.1574 | max_backward: 1.3862 | max_optimizer_step: 0.2915 | max_tensorboard_logging: 0.7724 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6137.2 | uss_MB: 5191.5 | child_rss_MB: 5174.4 | allocated_MB: 370.2 | reserved_MB: 26182.0 | inactive_split_MB: 176.5 +rank7 [2026-10-29 21:17:58,578] INFO : Step 44300/50000 rank7 | Train Loss: 0.8787 | Train Loss Unweighted: 0.4332 | Train Loss Components: Auxiliary: 0.1952 | Cardinality: 0.0950 | Classification: 0.0869 | Classification_binary: 0.1153 | HitGrouping: 0.1807 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0011 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1120 | Regression_cos_phi: 0.0363 | Regression_energy: 0.0482 | Regression_eta: 0.0257 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0520 | LR: 1.29e-04 | DataLoad Time: 0.0200s | Model Forward Time: 3.7421s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0142 | max_forward: 0.7699 | max_loss: 0.2088 | max_backward: 1.4049 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 0.4839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0121 | Memory: rss_MB: 6137.2 | uss_MB: 4195.8 | child_rss_MB: 5178.8 | allocated_MB: 367.2 | reserved_MB: 27204.0 | inactive_split_MB: 205.8 +rank7 [2026-10-29 21:21:59,794] INFO : Step 44400/50000 rank7 | Train Loss: 0.9558 | Train Loss Unweighted: 0.4680 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.0905 | Classification: 0.0911 | Classification_binary: 0.1587 | HitGrouping: 0.1858 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1697 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0634 | Regression_eta: 0.0371 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0489 | LR: 1.28e-04 | DataLoad Time: 0.0203s | Model Forward Time: 6.3285s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0370 | max_forward: 0.7829 | max_loss: 0.1727 | max_backward: 1.7862 | max_optimizer_step: 0.1845 | max_tensorboard_logging: 2.7361 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6138.4 | uss_MB: 5206.0 | child_rss_MB: 5161.4 | allocated_MB: 373.1 | reserved_MB: 27586.0 | inactive_split_MB: 176.3 +rank7 [2026-10-29 21:26:11,867] INFO : Step 44500/50000 rank7 | Train Loss: 0.9739 | Train Loss Unweighted: 0.4610 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1139 | Classification: 0.0849 | Classification_binary: 0.1578 | HitGrouping: 0.1836 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1077 | Regression_eta: 0.0012 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1649 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0412 | Regression_eta: 0.0496 | Regression_pt: 0.0421 | Regression_sin_phi: 0.0770 | LR: 1.27e-04 | DataLoad Time: 0.0097s | Model Forward Time: 3.9176s | Phase Times: max_pre_step_barrier: 0.0250 | max_batch_to_device: 0.0065 | max_forward: 0.7930 | max_loss: 0.1668 | max_backward: 1.3363 | max_optimizer_step: 0.2871 | max_tensorboard_logging: 0.5852 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 6136.3 | uss_MB: 5200.8 | child_rss_MB: 5177.3 | allocated_MB: 371.5 | reserved_MB: 27586.0 | inactive_split_MB: 195.8 +rank7 [2026-10-29 21:30:04,561] INFO : Step 44600/50000 rank7 | Train Loss: 0.8904 | Train Loss Unweighted: 0.4364 | Train Loss Components: Auxiliary: 0.2007 | Cardinality: 0.1041 | Classification: 0.0775 | Classification_binary: 0.1238 | HitGrouping: 0.1859 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1161 | Regression_eta: 0.0010 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0371 | Regression_energy: 0.0585 | Regression_eta: 0.0226 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0330 | LR: 1.26e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.5135s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0061 | max_forward: 0.6364 | max_loss: 0.1431 | max_backward: 1.3712 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.4900 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 6139.4 | uss_MB: 5205.0 | child_rss_MB: 5167.5 | allocated_MB: 368.4 | reserved_MB: 27586.0 | inactive_split_MB: 170.2 +rank7 [2026-10-29 21:33:51,597] INFO : Step 44700/50000 rank7 | Train Loss: 1.0003 | Train Loss Unweighted: 0.4691 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1252 | Classification: 0.0899 | Classification_binary: 0.1520 | HitGrouping: 0.1900 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1117 | Regression_eta: 0.0013 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0300 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0478 | Regression_eta: 0.0225 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0443 | LR: 1.25e-04 | DataLoad Time: 0.0177s | Model Forward Time: 3.6375s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.6952 | max_loss: 0.1455 | max_backward: 1.5015 | max_optimizer_step: 0.1862 | max_tensorboard_logging: 0.4740 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6145.1 | uss_MB: 5206.5 | child_rss_MB: 5162.4 | allocated_MB: 371.9 | reserved_MB: 28574.0 | inactive_split_MB: 214.1 +rank7 [2026-10-29 21:37:55,987] INFO : Step 44800/50000 rank7 | Train Loss: 0.9665 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1115 | Classification: 0.0889 | Classification_binary: 0.1605 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1042 | Regression_eta: 0.0011 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0655 | Regression_eta: 0.0420 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0472 | LR: 1.24e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.8736s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0066 | max_forward: 0.7593 | max_loss: 0.1924 | max_backward: 2.8086 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.6087 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0443 | Memory: rss_MB: 6148.4 | uss_MB: 5210.8 | child_rss_MB: 5160.0 | allocated_MB: 370.8 | reserved_MB: 28574.0 | inactive_split_MB: 257.7 +rank7 [2026-10-29 21:41:36,916] INFO : Step 44900/50000 rank7 | Train Loss: 0.9730 | Train Loss Unweighted: 0.4621 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.0886 | Classification_binary: 0.1466 | HitGrouping: 0.1845 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1121 | Regression_eta: 0.0013 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1630 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0503 | Regression_eta: 0.0361 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0476 | LR: 1.23e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.5209s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0101 | max_forward: 0.6531 | max_loss: 0.1431 | max_backward: 1.3230 | max_optimizer_step: 0.1849 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6145.9 | uss_MB: 5207.3 | child_rss_MB: 5182.8 | allocated_MB: 368.1 | reserved_MB: 28574.0 | inactive_split_MB: 178.8 +rank7 [2026-10-29 21:45:50,735] INFO : Step 45000/50000 rank7 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4443 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1053 | Classification: 0.0832 | Classification_binary: 0.1311 | HitGrouping: 0.1933 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1136 | Regression_eta: 0.0014 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0656 | Regression_eta: 0.0287 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0585 | LR: 1.22e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.1232s | Phase Times: max_pre_step_barrier: 0.0074 | max_batch_to_device: 0.0159 | max_forward: 1.1897 | max_loss: 0.2457 | max_backward: 1.2951 | max_optimizer_step: 0.1833 | max_tensorboard_logging: 0.8342 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0919 | Memory: rss_MB: 6152.1 | uss_MB: 5213.7 | child_rss_MB: 5170.7 | allocated_MB: 368.8 | reserved_MB: 28576.0 | inactive_split_MB: 266.4 +rank7 [2026-10-29 21:45:51,739] INFO : Running validation on rank7 +rank7 [2026-10-29 21:45:51,740] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 21:45:51,740] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 21:46:40,562] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:46:41,116] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 21:47:08,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 21:47:09,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 21:47:10,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 21:47:11,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 21:47:21,737] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 21:47:23,129] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:47:23,692] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 21:47:39,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 21:47:40,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 21:47:41,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 21:47:42,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 21:47:52,535] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 21:47:53,405] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:47:53,967] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 21:48:09,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 21:48:10,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 21:48:11,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 21:48:12,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_45000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 21:48:22,319] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 21:52:56,451] INFO : Step 45100/50000 rank7 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4848 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1151 | Classification: 0.1010 | Classification_binary: 0.1562 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1118 | Regression_eta: 0.0012 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.1762 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0549 | Regression_eta: 0.0408 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0682 | LR: 1.21e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.3044s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7307 | max_loss: 0.4409 | max_backward: 1.5823 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.8853 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 7066.7 | uss_MB: 6122.4 | child_rss_MB: 5160.9 | allocated_MB: 373.1 | reserved_MB: 23280.0 | inactive_split_MB: 227.2 +rank7 [2026-10-29 21:56:57,436] INFO : Step 45200/50000 rank7 | Train Loss: 0.9978 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1108 | Classification: 0.0815 | Classification_binary: 0.1593 | HitGrouping: 0.1884 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0332 | Regression_energy: 0.0500 | Regression_eta: 0.0294 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0474 | LR: 1.20e-04 | DataLoad Time: 0.0230s | Model Forward Time: 4.2198s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0073 | max_forward: 0.9482 | max_loss: 0.4476 | max_backward: 1.7165 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.4656 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7064.1 | uss_MB: 6120.3 | child_rss_MB: 5157.7 | allocated_MB: 373.0 | reserved_MB: 23338.0 | inactive_split_MB: 277.4 +rank7 [2026-10-29 22:01:28,910] INFO : Step 45300/50000 rank7 | Train Loss: 0.9312 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2092 | Cardinality: 0.1141 | Classification: 0.0882 | Classification_binary: 0.1299 | HitGrouping: 0.1891 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1158 | Regression_eta: 0.0010 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0661 | Regression_eta: 0.0368 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0466 | LR: 1.19e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.4284s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.6106 | max_loss: 0.2394 | max_backward: 1.1660 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4713 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0860 | Memory: rss_MB: 6271.7 | uss_MB: 5329.4 | child_rss_MB: 5157.3 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 223.2 +rank7 [2026-10-29 22:05:37,388] INFO : Step 45400/50000 rank7 | Train Loss: 0.9251 | Train Loss Unweighted: 0.4554 | Train Loss Components: Auxiliary: 0.2112 | Cardinality: 0.1034 | Classification: 0.0967 | Classification_binary: 0.1405 | HitGrouping: 0.1801 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1072 | Regression_eta: 0.0012 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0528 | Regression_eta: 0.0337 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0424 | LR: 1.19e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9540s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.7158 | max_loss: 0.2520 | max_backward: 1.7727 | max_optimizer_step: 0.1894 | max_tensorboard_logging: 0.5080 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 6276.9 | uss_MB: 5335.5 | child_rss_MB: 5162.7 | allocated_MB: 372.6 | reserved_MB: 23338.0 | inactive_split_MB: 212.6 +rank7 [2026-10-29 22:09:49,791] INFO : Step 45500/50000 rank7 | Train Loss: 1.0136 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1238 | Classification: 0.0902 | Classification_binary: 0.1661 | HitGrouping: 0.1832 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1060 | Regression_eta: 0.0013 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1881 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0495 | Regression_eta: 0.0429 | Regression_pt: 0.0522 | Regression_sin_phi: 0.1027 | LR: 1.18e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.1589s | Phase Times: max_pre_step_barrier: 0.0094 | max_batch_to_device: 0.0147 | max_forward: 1.9023 | max_loss: 0.2979 | max_backward: 3.5478 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.5661 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0186 | Memory: rss_MB: 6272.0 | uss_MB: 4281.7 | child_rss_MB: 5159.4 | allocated_MB: 372.4 | reserved_MB: 23338.0 | inactive_split_MB: 256.1 +rank7 [2026-10-29 22:14:16,024] INFO : Step 45600/50000 rank7 | Train Loss: 0.9675 | Train Loss Unweighted: 0.4715 | Train Loss Components: Auxiliary: 0.2172 | Cardinality: 0.1193 | Classification: 0.0849 | Classification_binary: 0.1400 | HitGrouping: 0.1866 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1218 | Regression_eta: 0.0010 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0566 | Regression_eta: 0.0282 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0532 | LR: 1.17e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.7210s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.6781 | max_loss: 0.1576 | max_backward: 1.5089 | max_optimizer_step: 0.1891 | max_tensorboard_logging: 0.5447 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6272.0 | uss_MB: 5337.5 | child_rss_MB: 5161.4 | allocated_MB: 368.8 | reserved_MB: 23338.0 | inactive_split_MB: 216.2 +rank7 [2026-10-29 22:18:15,631] INFO : Step 45700/50000 rank7 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1961 | Cardinality: 0.0908 | Classification: 0.0837 | Classification_binary: 0.1223 | HitGrouping: 0.1794 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1108 | Regression_eta: 0.0011 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0524 | Regression_eta: 0.0451 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0503 | LR: 1.16e-04 | DataLoad Time: 0.0205s | Model Forward Time: 3.4737s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.6216 | max_loss: 0.1658 | max_backward: 1.1797 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6234.4 | uss_MB: 5336.4 | child_rss_MB: 5162.7 | allocated_MB: 370.8 | reserved_MB: 23338.0 | inactive_split_MB: 226.6 +rank7 [2026-10-29 22:22:19,535] INFO : Step 45800/50000 rank7 | Train Loss: 0.9372 | Train Loss Unweighted: 0.4491 | Train Loss Components: Auxiliary: 0.2165 | Cardinality: 0.1073 | Classification: 0.0880 | Classification_binary: 0.1481 | HitGrouping: 0.1781 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1455 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0559 | Regression_eta: 0.0682 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0415 | LR: 1.16e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2356s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8017 | max_loss: 0.2428 | max_backward: 1.9917 | max_optimizer_step: 0.1919 | max_tensorboard_logging: 0.4636 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0978 | Memory: rss_MB: 6240.7 | uss_MB: 5347.0 | child_rss_MB: 5162.1 | allocated_MB: 375.2 | reserved_MB: 23338.0 | inactive_split_MB: 268.8 +rank7 [2026-10-29 22:26:26,931] INFO : Step 45900/50000 rank7 | Train Loss: 0.9600 | Train Loss Unweighted: 0.4592 | Train Loss Components: Auxiliary: 0.2179 | Cardinality: 0.1027 | Classification: 0.0816 | Classification_binary: 0.1590 | HitGrouping: 0.1875 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1072 | Regression_eta: 0.0013 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1762 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0503 | Regression_eta: 0.0366 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0714 | LR: 1.15e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.8437s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 1.3210 | max_loss: 0.4294 | max_backward: 3.2770 | max_optimizer_step: 0.2740 | max_tensorboard_logging: 0.5610 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 6235.6 | uss_MB: 5343.7 | child_rss_MB: 5161.3 | allocated_MB: 370.5 | reserved_MB: 23338.0 | inactive_split_MB: 259.6 +rank7 [2026-10-29 22:30:40,887] INFO : Step 46000/50000 rank7 | Train Loss: 0.8490 | Train Loss Unweighted: 0.3997 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.0938 | Classification: 0.0817 | Classification_binary: 0.1110 | HitGrouping: 0.1819 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0010 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1274 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0565 | Regression_eta: 0.0261 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0538 | LR: 1.14e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.5928s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0063 | max_forward: 0.5955 | max_loss: 0.1430 | max_backward: 1.2483 | max_optimizer_step: 0.1848 | max_tensorboard_logging: 0.6657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0752 | Memory: rss_MB: 6228.9 | uss_MB: 5344.9 | child_rss_MB: 5164.3 | allocated_MB: 365.7 | reserved_MB: 23338.0 | inactive_split_MB: 222.2 +rank7 [2026-10-29 22:30:41,828] INFO : Running validation on rank7 +rank7 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 22:30:41,828] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 22:31:32,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:31:33,519] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 22:31:59,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 22:32:01,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 22:32:02,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 22:32:04,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 22:32:14,301] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 22:32:15,390] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:32:16,223] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 22:32:32,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 22:32:33,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 22:32:34,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 22:32:35,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 22:32:45,100] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 22:32:45,545] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:32:46,016] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 22:33:01,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 22:33:02,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 22:33:02,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 22:33:03,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_46000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 22:33:13,664] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-29 22:37:44,432] INFO : Step 46100/50000 rank7 | Train Loss: 0.9299 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2094 | Cardinality: 0.1082 | Classification: 0.0899 | Classification_binary: 0.1357 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1081 | Regression_eta: 0.0010 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.0906 | Regression_cos_phi: 0.0302 | Regression_energy: 0.0346 | Regression_eta: 0.0166 | Regression_pt: 0.0354 | Regression_sin_phi: 0.0250 | LR: 1.13e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.9690s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 2.5146 | max_loss: 0.4579 | max_backward: 2.4958 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 1.2312 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1014 | Memory: rss_MB: 6980.2 | uss_MB: 6096.7 | child_rss_MB: 5172.0 | allocated_MB: 371.5 | reserved_MB: 22784.0 | inactive_split_MB: 245.0 +rank7 [2026-10-29 22:41:43,512] INFO : Step 46200/50000 rank7 | Train Loss: 0.9944 | Train Loss Unweighted: 0.4707 | Train Loss Components: Auxiliary: 0.2255 | Cardinality: 0.1286 | Classification: 0.0940 | Classification_binary: 0.1623 | HitGrouping: 0.1799 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1052 | Regression_eta: 0.0011 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1263 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0394 | Regression_eta: 0.0329 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0410 | LR: 1.13e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.2050s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.7685 | max_loss: 0.2359 | max_backward: 1.6855 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1253 | Memory: rss_MB: 6976.0 | uss_MB: 6094.5 | child_rss_MB: 5141.0 | allocated_MB: 373.1 | reserved_MB: 22784.0 | inactive_split_MB: 264.3 +rank7 [2026-10-29 22:45:39,133] INFO : Step 46300/50000 rank7 | Train Loss: 0.9564 | Train Loss Unweighted: 0.4515 | Train Loss Components: Auxiliary: 0.2114 | Cardinality: 0.1124 | Classification: 0.0869 | Classification_binary: 0.1435 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1091 | Regression_eta: 0.0012 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1536 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0541 | Regression_eta: 0.0463 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0426 | LR: 1.12e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.8640s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.6709 | max_loss: 0.2410 | max_backward: 1.2189 | max_optimizer_step: 0.2442 | max_tensorboard_logging: 0.8303 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6276.5 | uss_MB: 5413.4 | child_rss_MB: 5109.4 | allocated_MB: 372.3 | reserved_MB: 23784.0 | inactive_split_MB: 228.2 +rank7 [2026-10-29 22:49:34,649] INFO : Step 46400/50000 rank7 | Train Loss: 0.9514 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1147 | Classification: 0.0822 | Classification_binary: 0.1248 | HitGrouping: 0.1968 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0010 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1185 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0428 | Regression_eta: 0.0153 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0471 | LR: 1.11e-04 | DataLoad Time: 0.0199s | Model Forward Time: 3.9551s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.6748 | max_loss: 0.4371 | max_backward: 1.4077 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.8211 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6270.0 | uss_MB: 5413.5 | child_rss_MB: 5075.5 | allocated_MB: 374.2 | reserved_MB: 23988.0 | inactive_split_MB: 241.8 +rank7 [2026-10-29 22:53:41,742] INFO : Step 46500/50000 rank7 | Train Loss: 0.9517 | Train Loss Unweighted: 0.4566 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.0937 | Classification: 0.0928 | Classification_binary: 0.1533 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1031 | Regression_eta: 0.0013 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0530 | Regression_eta: 0.0337 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0589 | LR: 1.11e-04 | DataLoad Time: 0.0094s | Model Forward Time: 3.9604s | Phase Times: max_pre_step_barrier: 0.0138 | max_batch_to_device: 0.0087 | max_forward: 0.7136 | max_loss: 0.1507 | max_backward: 1.5229 | max_optimizer_step: 0.2767 | max_tensorboard_logging: 0.5815 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6272.9 | uss_MB: 5416.3 | child_rss_MB: 5079.6 | allocated_MB: 372.0 | reserved_MB: 23988.0 | inactive_split_MB: 248.1 +rank7 [2026-10-29 22:57:41,614] INFO : Step 46600/50000 rank7 | Train Loss: 0.9541 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.2166 | Cardinality: 0.1078 | Classification: 0.0915 | Classification_binary: 0.1560 | HitGrouping: 0.1800 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1557 | Regression_cos_phi: 0.0879 | Regression_energy: 0.0406 | Regression_eta: 0.0422 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0522 | LR: 1.10e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.5727s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7127 | max_loss: 0.1604 | max_backward: 1.3079 | max_optimizer_step: 0.1911 | max_tensorboard_logging: 0.4722 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0324 | Memory: rss_MB: 6261.6 | uss_MB: 5413.8 | child_rss_MB: 5045.9 | allocated_MB: 370.5 | reserved_MB: 23988.0 | inactive_split_MB: 179.8 +rank7 [2026-10-29 23:02:40,690] INFO : Step 46700/50000 rank7 | Train Loss: 0.9378 | Train Loss Unweighted: 0.4497 | Train Loss Components: Auxiliary: 0.2067 | Cardinality: 0.0977 | Classification: 0.0857 | Classification_binary: 0.1264 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1172 | Regression_eta: 0.0015 | Regression_pt: 0.1176 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0558 | Regression_eta: 0.0405 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0885 | LR: 1.10e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.7492s | Phase Times: max_pre_step_barrier: 0.0197 | max_batch_to_device: 0.0481 | max_forward: 0.6699 | max_loss: 0.2593 | max_backward: 1.4356 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5914 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6261.6 | uss_MB: 5415.0 | child_rss_MB: 5040.6 | allocated_MB: 368.9 | reserved_MB: 24784.0 | inactive_split_MB: 229.2 +rank7 [2026-10-29 23:06:36,578] INFO : Step 46800/50000 rank7 | Train Loss: 0.9152 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.2022 | Cardinality: 0.1011 | Classification: 0.0910 | Classification_binary: 0.1376 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0988 | Regression_eta: 0.0014 | Regression_pt: 0.0999 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0445 | Regression_eta: 0.0490 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0618 | LR: 1.09e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6026s | Phase Times: max_pre_step_barrier: 0.1216 | max_batch_to_device: 0.0144 | max_forward: 1.0361 | max_loss: 0.2737 | max_backward: 1.7951 | max_optimizer_step: 0.2978 | max_tensorboard_logging: 0.5843 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0289 | Memory: rss_MB: 6263.1 | uss_MB: 5417.5 | child_rss_MB: 5044.7 | allocated_MB: 368.9 | reserved_MB: 24994.0 | inactive_split_MB: 204.7 +rank7 [2026-10-29 23:10:45,824] INFO : Step 46900/50000 rank7 | Train Loss: 0.9278 | Train Loss Unweighted: 0.4522 | Train Loss Components: Auxiliary: 0.2149 | Cardinality: 0.1073 | Classification: 0.0827 | Classification_binary: 0.1607 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1027 | Regression_eta: 0.0010 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0561 | Regression_eta: 0.0249 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0906 | LR: 1.09e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0938s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0446 | max_forward: 0.7815 | max_loss: 0.1553 | max_backward: 1.4732 | max_optimizer_step: 0.2929 | max_tensorboard_logging: 0.6668 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6256.7 | uss_MB: 5416.7 | child_rss_MB: 5041.4 | allocated_MB: 369.3 | reserved_MB: 24994.0 | inactive_split_MB: 274.7 +rank7 [2026-10-29 23:14:40,387] INFO : Step 47000/50000 rank7 | Train Loss: 0.9023 | Train Loss Unweighted: 0.4500 | Train Loss Components: Auxiliary: 0.2093 | Cardinality: 0.0953 | Classification: 0.0802 | Classification_binary: 0.1325 | HitGrouping: 0.1876 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1175 | Regression_eta: 0.0010 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1337 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0502 | Regression_eta: 0.0183 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0353 | LR: 1.08e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.5092s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6624 | max_loss: 0.1440 | max_backward: 1.2028 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.5070 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0713 | Memory: rss_MB: 6259.7 | uss_MB: 5421.0 | child_rss_MB: 5043.0 | allocated_MB: 369.4 | reserved_MB: 25784.0 | inactive_split_MB: 224.5 +rank7 [2026-10-29 23:14:41,333] INFO : Running validation on rank7 +rank7 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 23:14:41,333] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 23:15:32,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:15:33,164] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 23:16:02,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 23:16:03,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 23:16:04,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 23:16:05,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 23:16:15,542] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 23:16:16,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:16:16,905] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 23:16:32,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 23:16:33,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 23:16:34,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 23:16:36,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 23:16:46,235] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 23:16:47,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:16:47,863] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 23:17:03,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 23:17:04,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 23:17:05,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 23:17:05,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_47000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 23:17:15,946] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 23:21:36,067] INFO : Step 47100/50000 rank7 | Train Loss: 0.9138 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.2058 | Cardinality: 0.0938 | Classification: 0.0946 | Classification_binary: 0.1243 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1103 | Regression_eta: 0.0011 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1123 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0464 | Regression_eta: 0.0218 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0510 | LR: 1.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1761s | Phase Times: max_pre_step_barrier: 0.0211 | max_batch_to_device: 0.0063 | max_forward: 0.8048 | max_loss: 0.2433 | max_backward: 1.3159 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 1.0353 | max_tensorboard_flush: 0.0042 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 6952.3 | uss_MB: 6113.6 | child_rss_MB: 5037.2 | allocated_MB: 368.1 | reserved_MB: 22106.0 | inactive_split_MB: 269.0 +rank7 [2026-10-29 23:25:45,623] INFO : Step 47200/50000 rank7 | Train Loss: 0.9875 | Train Loss Unweighted: 0.4749 | Train Loss Components: Auxiliary: 0.2263 | Cardinality: 0.1259 | Classification: 0.0903 | Classification_binary: 0.1586 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1110 | Regression_eta: 0.0012 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0498 | Regression_eta: 0.0370 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0732 | LR: 1.07e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2837s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0486 | max_forward: 0.7581 | max_loss: 0.1506 | max_backward: 1.5914 | max_optimizer_step: 0.2300 | max_tensorboard_logging: 0.7953 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 6951.3 | uss_MB: 6117.8 | child_rss_MB: 5032.9 | allocated_MB: 370.5 | reserved_MB: 23104.0 | inactive_split_MB: 195.8 +rank7 [2026-10-29 23:29:53,763] INFO : Step 47300/50000 rank7 | Train Loss: 0.9350 | Train Loss Unweighted: 0.4467 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.0958 | Classification: 0.0857 | Classification_binary: 0.1512 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1033 | Regression_eta: 0.0012 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0444 | Regression_eta: 0.0305 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0849 | LR: 1.06e-04 | DataLoad Time: 0.0220s | Model Forward Time: 6.4004s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0184 | max_forward: 1.9890 | max_loss: 0.4771 | max_backward: 3.1656 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6309.1 | uss_MB: 5475.6 | child_rss_MB: 5028.4 | allocated_MB: 370.8 | reserved_MB: 23104.0 | inactive_split_MB: 271.1 +rank7 [2026-10-29 23:33:58,356] INFO : Step 47400/50000 rank7 | Train Loss: 0.8593 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1905 | Cardinality: 0.0807 | Classification: 0.0812 | Classification_binary: 0.1182 | HitGrouping: 0.1877 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1114 | Regression_eta: 0.0011 | Regression_pt: 0.1124 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0502 | Regression_eta: 0.0480 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0845 | LR: 1.06e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.0148s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0125 | max_forward: 0.7544 | max_loss: 0.1826 | max_backward: 1.4299 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.8027 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6311.1 | uss_MB: 5475.9 | child_rss_MB: 5031.1 | allocated_MB: 369.9 | reserved_MB: 23104.0 | inactive_split_MB: 253.6 +rank7 [2026-10-29 23:37:53,327] INFO : Step 47500/50000 rank7 | Train Loss: 0.9089 | Train Loss Unweighted: 0.4235 | Train Loss Components: Auxiliary: 0.2083 | Cardinality: 0.1000 | Classification: 0.0799 | Classification_binary: 0.1391 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1008 | Regression_eta: 0.0012 | Regression_pt: 0.1013 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1079 | Regression_cos_phi: 0.0326 | Regression_energy: 0.0551 | Regression_eta: 0.0289 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0396 | LR: 1.06e-04 | DataLoad Time: 0.0226s | Model Forward Time: 4.2718s | Phase Times: max_pre_step_barrier: 0.0048 | max_batch_to_device: 0.0068 | max_forward: 0.7329 | max_loss: 0.1827 | max_backward: 1.5814 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.9161 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6306.2 | uss_MB: 5478.3 | child_rss_MB: 5026.1 | allocated_MB: 371.7 | reserved_MB: 23104.0 | inactive_split_MB: 243.6 +rank7 [2026-10-29 23:41:48,171] INFO : Step 47600/50000 rank7 | Train Loss: 0.9781 | Train Loss Unweighted: 0.4635 | Train Loss Components: Auxiliary: 0.2216 | Cardinality: 0.1226 | Classification: 0.0910 | Classification_binary: 0.1607 | HitGrouping: 0.1784 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.1649 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0451 | Regression_eta: 0.0341 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0766 | LR: 1.05e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.4258s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.8435 | max_loss: 0.1581 | max_backward: 1.8989 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.5341 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6310.2 | uss_MB: 5483.0 | child_rss_MB: 5033.7 | allocated_MB: 370.6 | reserved_MB: 24122.0 | inactive_split_MB: 257.4 +rank7 [2026-10-29 23:46:00,212] INFO : Step 47700/50000 rank7 | Train Loss: 0.9311 | Train Loss Unweighted: 0.4523 | Train Loss Components: Auxiliary: 0.2105 | Cardinality: 0.0992 | Classification: 0.0889 | Classification_binary: 0.1321 | HitGrouping: 0.1888 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1138 | Regression_eta: 0.0012 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0503 | Regression_eta: 0.0323 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0443 | LR: 1.05e-04 | DataLoad Time: 0.0208s | Model Forward Time: 3.5052s | Phase Times: max_pre_step_barrier: 0.0044 | max_batch_to_device: 0.0068 | max_forward: 0.6708 | max_loss: 0.1613 | max_backward: 1.2326 | max_optimizer_step: 0.1946 | max_tensorboard_logging: 0.5084 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6306.7 | uss_MB: 5481.0 | child_rss_MB: 5031.8 | allocated_MB: 368.1 | reserved_MB: 24122.0 | inactive_split_MB: 276.0 +rank7 [2026-10-29 23:50:02,911] INFO : Step 47800/50000 rank7 | Train Loss: 0.9063 | Train Loss Unweighted: 0.4379 | Train Loss Components: Auxiliary: 0.2020 | Cardinality: 0.0932 | Classification: 0.0920 | Classification_binary: 0.1284 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0011 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1086 | Regression_cos_phi: 0.0271 | Regression_energy: 0.0396 | Regression_eta: 0.0189 | Regression_pt: 0.0410 | Regression_sin_phi: 0.0219 | LR: 1.04e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8440s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0610 | max_forward: 0.6626 | max_loss: 0.1349 | max_backward: 1.5091 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6302.7 | uss_MB: 5485.7 | child_rss_MB: 5029.8 | allocated_MB: 369.5 | reserved_MB: 24122.0 | inactive_split_MB: 236.5 +rank7 [2026-10-29 23:54:07,644] INFO : Step 47900/50000 rank7 | Train Loss: 0.9591 | Train Loss Unweighted: 0.4478 | Train Loss Components: Auxiliary: 0.2152 | Cardinality: 0.1022 | Classification: 0.0882 | Classification_binary: 0.1545 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1003 | Regression_eta: 0.0014 | Regression_pt: 0.1019 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0523 | Regression_eta: 0.0259 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0416 | LR: 1.04e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.1703s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0068 | max_forward: 0.8744 | max_loss: 0.2241 | max_backward: 1.6370 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.5973 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1378 | Memory: rss_MB: 6302.2 | uss_MB: 5483.9 | child_rss_MB: 5030.3 | allocated_MB: 370.5 | reserved_MB: 24122.0 | inactive_split_MB: 218.9 +rank7 [2026-10-30 00:00:28,988] INFO : Step 48000/50000 rank7 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4530 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1221 | Classification: 0.0823 | Classification_binary: 0.1525 | HitGrouping: 0.1837 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1079 | Regression_eta: 0.0017 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1367 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0460 | Regression_eta: 0.0342 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0467 | LR: 1.04e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1277s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0057 | max_forward: 0.7069 | max_loss: 0.2447 | max_backward: 1.4090 | max_optimizer_step: 0.1933 | max_tensorboard_logging: 0.8132 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1250 | Memory: rss_MB: 6301.2 | uss_MB: 5485.0 | child_rss_MB: 5024.5 | allocated_MB: 368.8 | reserved_MB: 24122.0 | inactive_split_MB: 269.6 +rank7 [2026-10-30 00:00:29,931] INFO : Running validation on rank7 +rank7 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 00:00:29,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 00:01:19,557] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:01:20,065] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 00:01:46,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 00:01:48,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 00:01:49,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 00:01:50,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 00:02:00,111] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-30 00:02:01,322] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:02:01,859] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 00:02:17,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 00:02:18,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 00:02:19,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 00:02:20,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 00:02:30,434] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-30 00:02:32,510] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:02:32,992] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 00:02:48,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 00:02:48,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 00:02:49,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 00:02:50,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_48000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 00:03:00,304] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-30 00:07:25,204] INFO : Step 48100/50000 rank7 | Train Loss: 0.9084 | Train Loss Unweighted: 0.4360 | Train Loss Components: Auxiliary: 0.2014 | Cardinality: 0.1162 | Classification: 0.0769 | Classification_binary: 0.1204 | HitGrouping: 0.1882 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1181 | Regression_eta: 0.0013 | Regression_pt: 0.1183 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0495 | Regression_eta: 0.0331 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0509 | LR: 1.03e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9235s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.6512 | max_loss: 0.1372 | max_backward: 1.2024 | max_optimizer_step: 0.2194 | max_tensorboard_logging: 0.9158 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7374.2 | uss_MB: 6557.5 | child_rss_MB: 5038.7 | allocated_MB: 367.7 | reserved_MB: 24830.0 | inactive_split_MB: 222.3 +rank7 [2026-10-30 00:11:33,471] INFO : Step 48200/50000 rank7 | Train Loss: 0.9585 | Train Loss Unweighted: 0.4453 | Train Loss Components: Auxiliary: 0.2151 | Cardinality: 0.1199 | Classification: 0.0839 | Classification_binary: 0.1503 | HitGrouping: 0.1816 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1039 | Regression_eta: 0.0012 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1258 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0542 | Regression_eta: 0.0351 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0308 | LR: 1.03e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4361s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7766 | max_loss: 0.1498 | max_backward: 1.7414 | max_optimizer_step: 0.3242 | max_tensorboard_logging: 0.7504 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0557 | Memory: rss_MB: 7374.2 | uss_MB: 6560.6 | child_rss_MB: 5031.5 | allocated_MB: 370.4 | reserved_MB: 24830.0 | inactive_split_MB: 179.3 +rank7 [2026-10-30 00:15:41,698] INFO : Step 48300/50000 rank7 | Train Loss: 1.0575 | Train Loss Unweighted: 0.4963 | Train Loss Components: Auxiliary: 0.2408 | Cardinality: 0.1341 | Classification: 0.0958 | Classification_binary: 0.1687 | HitGrouping: 0.1865 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1138 | Regression_eta: 0.0013 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0990 | Regression_energy: 0.0498 | Regression_eta: 0.0335 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0389 | LR: 1.03e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.2043s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7497 | max_loss: 0.2201 | max_backward: 1.7233 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6693 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0313 | Memory: rss_MB: 6378.2 | uss_MB: 5566.5 | child_rss_MB: 5032.3 | allocated_MB: 372.3 | reserved_MB: 24830.0 | inactive_split_MB: 269.6 +rank7 [2026-10-30 00:19:56,564] INFO : Step 48400/50000 rank7 | Train Loss: 0.9108 | Train Loss Unweighted: 0.4349 | Train Loss Components: Auxiliary: 0.2011 | Cardinality: 0.1093 | Classification: 0.0691 | Classification_binary: 0.1311 | HitGrouping: 0.1830 | Regression_cos_phi: 0.0005 | Regression_energy: 0.1153 | Regression_eta: 0.0013 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1512 | Regression_cos_phi: 0.0384 | Regression_energy: 0.0446 | Regression_eta: 0.0308 | Regression_pt: 0.0459 | Regression_sin_phi: 0.0530 | LR: 1.02e-04 | DataLoad Time: 0.0155s | Model Forward Time: 3.8548s | Phase Times: max_pre_step_barrier: 0.1141 | max_batch_to_device: 0.0064 | max_forward: 0.6489 | max_loss: 0.1390 | max_backward: 1.1801 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.6705 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6380.2 | uss_MB: 5566.3 | child_rss_MB: 5030.1 | allocated_MB: 370.1 | reserved_MB: 24830.0 | inactive_split_MB: 261.5 +rank7 [2026-10-30 00:23:53,558] INFO : Step 48500/50000 rank7 | Train Loss: 0.9694 | Train Loss Unweighted: 0.4633 | Train Loss Components: Auxiliary: 0.2162 | Cardinality: 0.1004 | Classification: 0.0844 | Classification_binary: 0.1367 | HitGrouping: 0.1928 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1194 | Regression_eta: 0.0012 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1331 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0580 | Regression_eta: 0.0291 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0530 | LR: 1.02e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7566s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 1.7512 | max_loss: 0.2678 | max_backward: 1.9538 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.4804 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0379 | Memory: rss_MB: 6374.4 | uss_MB: 5565.1 | child_rss_MB: 5023.6 | allocated_MB: 371.5 | reserved_MB: 26842.0 | inactive_split_MB: 176.5 +rank7 [2026-10-30 00:27:51,108] INFO : Step 48600/50000 rank7 | Train Loss: 0.9724 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1148 | Classification: 0.0854 | Classification_binary: 0.1551 | HitGrouping: 0.1844 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0574 | Regression_eta: 0.0403 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0449 | LR: 1.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0072s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0086 | max_forward: 0.6984 | max_loss: 0.2525 | max_backward: 1.6611 | max_optimizer_step: 0.1976 | max_tensorboard_logging: 0.5699 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6375.4 | uss_MB: 5569.8 | child_rss_MB: 5026.2 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 234.8 +rank7 [2026-10-30 00:31:52,432] INFO : Step 48700/50000 rank7 | Train Loss: 1.0253 | Train Loss Unweighted: 0.4863 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1212 | Classification: 0.0968 | Classification_binary: 0.1569 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1143 | Regression_eta: 0.0012 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0347 | Regression_energy: 0.0498 | Regression_eta: 0.0223 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0414 | LR: 1.02e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.9654s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0068 | max_forward: 0.7410 | max_loss: 0.2472 | max_backward: 1.4647 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.6585 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0332 | Memory: rss_MB: 6316.1 | uss_MB: 5565.7 | child_rss_MB: 4823.4 | allocated_MB: 370.8 | reserved_MB: 26842.0 | inactive_split_MB: 275.8 +rank7 [2026-10-30 00:35:40,395] INFO : Step 48800/50000 rank7 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4218 | Train Loss Components: Auxiliary: 0.1981 | Cardinality: 0.0898 | Classification: 0.0814 | Classification_binary: 0.1196 | HitGrouping: 0.1857 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1094 | Regression_eta: 0.0012 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1140 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0547 | Regression_eta: 0.0313 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0325 | LR: 1.01e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.7721s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0061 | max_forward: 0.6122 | max_loss: 0.1775 | max_backward: 1.1932 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.8841 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6316.1 | uss_MB: 5568.8 | child_rss_MB: 4813.9 | allocated_MB: 368.0 | reserved_MB: 26842.0 | inactive_split_MB: 263.7 +rank7 [2026-10-30 00:39:23,570] INFO : Step 48900/50000 rank7 | Train Loss: 0.9391 | Train Loss Unweighted: 0.4419 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1005 | Classification: 0.0873 | Classification_binary: 0.1401 | HitGrouping: 0.1872 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0011 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.0982 | Regression_cos_phi: 0.0350 | Regression_energy: 0.0557 | Regression_eta: 0.0190 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0401 | LR: 1.01e-04 | DataLoad Time: 0.0190s | Model Forward Time: 3.9260s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0809 | max_forward: 0.7397 | max_loss: 0.1455 | max_backward: 1.5027 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6303.5 | uss_MB: 5491.8 | child_rss_MB: 4768.8 | allocated_MB: 373.0 | reserved_MB: 26842.0 | inactive_split_MB: 191.3 +rank7 [2026-10-30 00:43:08,795] INFO : Step 49000/50000 rank7 | Train Loss: 1.0019 | Train Loss Unweighted: 0.4657 | Train Loss Components: Auxiliary: 0.2334 | Cardinality: 0.1354 | Classification: 0.0893 | Classification_binary: 0.1587 | HitGrouping: 0.1838 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0010 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0584 | Regression_eta: 0.0210 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0336 | LR: 1.01e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4538s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1085 | max_forward: 0.8072 | max_loss: 0.1558 | max_backward: 1.7809 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.6082 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 6305.0 | uss_MB: 5577.9 | child_rss_MB: 4749.4 | allocated_MB: 370.9 | reserved_MB: 26842.0 | inactive_split_MB: 271.3 +rank7 [2026-10-30 00:43:09,869] INFO : Running validation on rank7 +rank7 [2026-10-30 00:43:09,869] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 00:43:09,869] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 00:43:59,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:43:59,787] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 00:44:27,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 00:44:28,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 00:44:29,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 00:44:29,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 00:44:39,893] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-30 00:44:40,646] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:44:41,122] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 00:44:57,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 00:44:58,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 00:44:58,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 00:44:59,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 00:45:09,900] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-30 00:45:10,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:45:10,743] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 00:45:26,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 00:45:27,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 00:45:27,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 00:45:28,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_49000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 00:45:38,507] INFO : Time taken to make predictions on device 7 is: 0.22 min +rank7 [2026-10-30 00:49:42,469] INFO : Step 49100/50000 rank7 | Train Loss: 0.9797 | Train Loss Unweighted: 0.4561 | Train Loss Components: Auxiliary: 0.2185 | Cardinality: 0.1125 | Classification: 0.0877 | Classification_binary: 0.1433 | HitGrouping: 0.1905 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0013 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0413 | Regression_eta: 0.0276 | Regression_pt: 0.0429 | Regression_sin_phi: 0.0447 | LR: 1.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.7308s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.6869 | max_loss: 0.1436 | max_backward: 1.2639 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.4741 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0455 | Memory: rss_MB: 7470.2 | uss_MB: 6734.9 | child_rss_MB: 4778.3 | allocated_MB: 370.4 | reserved_MB: 22116.0 | inactive_split_MB: 274.9 +rank7 [2026-10-30 00:53:47,122] INFO : Step 49200/50000 rank7 | Train Loss: 0.8929 | Train Loss Unweighted: 0.4393 | Train Loss Components: Auxiliary: 0.2051 | Cardinality: 0.1020 | Classification: 0.0903 | Classification_binary: 0.1250 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0009 | Regression_pt: 0.1114 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1353 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0481 | Regression_eta: 0.0262 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0527 | LR: 1.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.7529s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.6890 | max_loss: 0.2430 | max_backward: 1.3427 | max_optimizer_step: 0.2258 | max_tensorboard_logging: 0.6347 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 7470.5 | uss_MB: 6738.6 | child_rss_MB: 4771.6 | allocated_MB: 368.3 | reserved_MB: 24092.0 | inactive_split_MB: 187.3 +rank7 [2026-10-30 00:57:33,941] INFO : Step 49300/50000 rank7 | Train Loss: 1.0122 | Train Loss Unweighted: 0.4743 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1210 | Classification: 0.0951 | Classification_binary: 0.1621 | HitGrouping: 0.1839 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1066 | Regression_eta: 0.0015 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0557 | Regression_eta: 0.0581 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0711 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8967s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0070 | max_forward: 0.7291 | max_loss: 0.1495 | max_backward: 1.5906 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6415.4 | uss_MB: 5684.6 | child_rss_MB: 4746.9 | allocated_MB: 370.9 | reserved_MB: 24092.0 | inactive_split_MB: 226.9 +rank7 [2026-10-30 01:01:33,225] INFO : Step 49400/50000 rank7 | Train Loss: 0.9790 | Train Loss Unweighted: 0.4695 | Train Loss Components: Auxiliary: 0.2238 | Cardinality: 0.1137 | Classification: 0.0883 | Classification_binary: 0.1513 | HitGrouping: 0.1861 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1131 | Regression_eta: 0.0013 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0529 | Regression_eta: 0.0234 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0491 | LR: 1.00e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.0361s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.7334 | max_loss: 0.2433 | max_backward: 1.8823 | max_optimizer_step: 0.3081 | max_tensorboard_logging: 0.5951 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 6413.4 | uss_MB: 5608.6 | child_rss_MB: 4765.4 | allocated_MB: 372.2 | reserved_MB: 24092.0 | inactive_split_MB: 278.9 +rank7 [2026-10-30 01:05:37,233] INFO : Step 49500/50000 rank7 | Train Loss: 0.9706 | Train Loss Unweighted: 0.4581 | Train Loss Components: Auxiliary: 0.2120 | Cardinality: 0.1158 | Classification: 0.0879 | Classification_binary: 0.1279 | HitGrouping: 0.1920 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0014 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0610 | Regression_eta: 0.0365 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0557 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9626s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.6307 | max_loss: 0.1505 | max_backward: 1.3337 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.8584 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0342 | Memory: rss_MB: 6409.4 | uss_MB: 5683.7 | child_rss_MB: 4759.9 | allocated_MB: 370.4 | reserved_MB: 24614.0 | inactive_split_MB: 171.1 +rank7 [2026-10-30 01:09:49,675] INFO : Step 49600/50000 rank7 | Train Loss: 0.9965 | Train Loss Unweighted: 0.4752 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1098 | Classification: 0.1016 | Classification_binary: 0.1437 | HitGrouping: 0.1897 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1133 | Regression_eta: 0.0014 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1210 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0481 | Regression_eta: 0.0597 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0667 | LR: 1.00e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.1652s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.6933 | max_loss: 0.1496 | max_backward: 1.5639 | max_optimizer_step: 0.3077 | max_tensorboard_logging: 0.7702 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0330 | Memory: rss_MB: 6412.2 | uss_MB: 5687.4 | child_rss_MB: 4755.4 | allocated_MB: 371.3 | reserved_MB: 24614.0 | inactive_split_MB: 250.0 +rank7 [2026-10-30 01:13:44,417] INFO : Step 49700/50000 rank7 | Train Loss: 1.0063 | Train Loss Unweighted: 0.4775 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1108 | Classification: 0.1000 | Classification_binary: 0.1702 | HitGrouping: 0.1796 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1019 | Regression_eta: 0.0014 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1508 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0511 | Regression_eta: 0.0343 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0524 | LR: 1.00e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.2328s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7591 | max_loss: 0.1933 | max_backward: 1.7410 | max_optimizer_step: 0.1983 | max_tensorboard_logging: 0.5113 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0691 | Memory: rss_MB: 6407.4 | uss_MB: 5675.2 | child_rss_MB: 4703.7 | allocated_MB: 372.4 | reserved_MB: 24614.0 | inactive_split_MB: 259.5 +rank7 [2026-10-30 01:17:46,084] INFO : Step 49800/50000 rank7 | Train Loss: 0.9472 | Train Loss Unweighted: 0.4401 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1188 | Classification: 0.0821 | Classification_binary: 0.1427 | HitGrouping: 0.1862 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0470 | Regression_eta: 0.0397 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0572 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.7733s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7430 | max_loss: 0.2153 | max_backward: 1.0846 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.7647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0388 | Memory: rss_MB: 6388.3 | uss_MB: 5552.5 | child_rss_MB: 4630.2 | allocated_MB: 368.0 | reserved_MB: 24614.0 | inactive_split_MB: 273.7 +rank7 [2026-10-30 01:21:39,661] INFO : Step 49900/50000 rank7 | Train Loss: 0.9411 | Train Loss Unweighted: 0.4586 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.0996 | Classification: 0.0961 | Classification_binary: 0.1284 | HitGrouping: 0.1815 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1145 | Regression_eta: 0.0013 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1124 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0498 | Regression_eta: 0.0296 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0538 | LR: 1.00e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0434s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0074 | max_forward: 0.6992 | max_loss: 0.1527 | max_backward: 1.3836 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.8576 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0403 | Memory: rss_MB: 6363.3 | uss_MB: 5692.1 | child_rss_MB: 4528.3 | allocated_MB: 368.7 | reserved_MB: 24614.0 | inactive_split_MB: 241.3 +rank7 [2026-10-30 01:25:29,378] INFO : Step 50000/50000 rank7 | Train Loss: 1.0036 | Train Loss Unweighted: 0.4712 | Train Loss Components: Auxiliary: 0.2297 | Cardinality: 0.1196 | Classification: 0.0902 | Classification_binary: 0.1583 | HitGrouping: 0.1834 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.5941 | Classification_binary: 1.0000 | Regression_cos_phi: 83.8123 | Regression_energy: 0.4825 | Regression_eta: 27.9991 | Regression_pt: 0.4991 | Regression_sin_phi: 82.3005 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0525 | Regression_eta: 0.0714 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0484 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2531s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0068 | max_forward: 0.7146 | max_loss: 0.1481 | max_backward: 1.6207 | max_optimizer_step: 0.2721 | max_tensorboard_logging: 0.8195 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6364.5 | uss_MB: 5690.8 | child_rss_MB: 4519.4 | allocated_MB: 371.5 | reserved_MB: 25088.0 | inactive_split_MB: 268.9 +rank7 [2026-10-30 01:25:30,314] INFO : Running validation on rank7 +rank7 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 01:25:30,314] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 01:26:21,850] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:26:22,339] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 01:26:50,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 01:26:51,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 01:26:52,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 01:26:53,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 01:27:03,620] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-30 01:27:03,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:27:04,489] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 01:27:20,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 01:27:21,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 01:27:22,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 01:27:23,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 01:27:33,312] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-30 01:27:34,257] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:27:34,756] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 01:27:50,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 01:27:50,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 01:27:51,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 01:27:52,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_attention_seed12345_20261028_111201_979187/preds_step_50000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 01:28:02,277] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 01:28:03,400] INFO : Training completed. Total time on device 7: 638.892min diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-50000.pth b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..7fce8d7f0d437c9a89932ee0ff88ac350953e28d --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f162da4e4e90066cc41fd492f7279c8aa89180040a65358d25eb6c9f45660883 +size 153793740 diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_1000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..3e9ebacca18710190e014978cfcc4e26f50f1455 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 0.0473664266767715, "train": {"Classification_binary": 0.41292887926101685, "Cardinality": 0.25086650252342224, "Classification": 0.32992300391197205, "Regression_pt": 0.482708603143692, "Regression_eta": 0.006314700469374657, "Regression_sin_phi": 0.002930198796093464, "Regression_cos_phi": 0.0026936307549476624, "Regression_energy": 0.48044297099113464, "HitGrouping": 0.2817913889884949, "Auxiliary": 0.5693709254264832, "Total": 2.896371364593506, "Total_unweighted": 1.7179419873282313}, "train_group": {}, "valid": {"Classification_binary": 0.4181739886601766, "Cardinality": 0.3019972840944926, "Classification": 0.33354167143503827, "Regression_pt": 0.48639170328776044, "Regression_eta": 0.006149034947156906, "Regression_sin_phi": 0.0029508542890350022, "Regression_cos_phi": 0.002875694694618384, "Regression_energy": 0.48344679673512775, "HitGrouping": 0.28459763526916504, "Auxiliary": 0.5835183461507162, "Total": 2.9880447387695312, "Total_unweighted": 1.7335297440489132, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.7902812957763672, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.18695303797721863, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3335856795310974, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.18486949801445007, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.17192424833774567, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.5079966187477112, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7600705027580261, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.19558706879615784, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3035450577735901, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.19766755402088165, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.17280781269073486, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5336041450500488, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.80078125, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.17453812062740326, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3094191253185272, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.17281022667884827, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.16416843235492706, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4875723123550415, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.7944396138191223, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.17993344366550446, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3162844181060791, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.17839348316192627, "metrics/hit_grouping/calorimeter/all/object_purity": 0.167188823223114, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.49733370542526245, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0007970688748173416, "metrics/hit_grouping/calorimeter/background_recall": 0.030175330117344856, "metrics/hit_grouping/calorimeter/completeness": 0.2671490013599396, "metrics/hit_grouping/calorimeter/energy_completeness": 0.2869265079498291, "metrics/hit_grouping/calorimeter/energy_purity": 0.2828924059867859, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5650618076324463, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8260650634765625, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.1767905056476593, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.23293127119541168, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.17711003124713898, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.17344725131988525, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.42933544516563416, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7783660292625427, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.18941818177700043, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.34891754388809204, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.18637149035930634, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.1753876954317093, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5365039110183716, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.746148943901062, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.17678669095039368, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4533642828464508, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.1729731559753418, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.1461683064699173, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5965197086334229, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7562255859375, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.2144414633512497, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.40955016016960144, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.21202877163887024, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.20263683795928955, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5813688039779663, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.6967402696609497, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.3469342589378357, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6767884492874146, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.33906176686286926, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.33731403946876526, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6803911328315735, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8450406789779663, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.12485451996326447, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.17824514210224152, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.12495880573987961, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.1107793003320694, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3898460865020752, "metrics/hit_grouping/calorimeter/purity": 0.26263847947120667, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8959653377532959, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.03417758643627167, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.08082706481218338, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.03450590744614601, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.007458142936229706, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04511278122663498, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8744921684265137, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.04203333705663681, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.0859329029917717, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.04241662845015526, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.016122372820973396, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2530900537967682, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7886195182800293, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.18948455154895782, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.33210816979408264, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.18781407177448273, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.17765282094478607, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.5174124836921692, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.638029158115387, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3621613681316376, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.1892785131931305, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.15535472333431244, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7306967973709106, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7108336687088013, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28028932213783264, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.2093343734741211, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.1633560210466385, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5521398186683655, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5553800463676453, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.328464537858963, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.13600540161132812, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.11439788341522217, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.8047201037406921, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.5922644734382629, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.3352626860141754, "metrics/hit_grouping/tracker/all/object_completeness": 0.15779513120651245, "metrics/hit_grouping/tracker/all/object_purity": 0.13060210645198822, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.763312578201294, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.014877170324325562, "metrics/hit_grouping/tracker/background_recall": 0.040456268936395645, "metrics/hit_grouping/tracker/completeness": 0.1639445722103119, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6062051057815552, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.5821704864501953, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.31097668409347534, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.17154735326766968, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.16026891767978668, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7897597551345825, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6030261516571045, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.32437828183174133, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.1422506421804428, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.10692854225635529, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7411508560180664, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6023358702659607, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.396032452583313, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.14495053887367249, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.09309765696525574, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7309287786483765, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6144073009490967, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.37072259187698364, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.16485098004341125, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.12255032360553741, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7483230233192444, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.609647810459137, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5523399114608765, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.255219966173172, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.20633308589458466, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7235221862792969, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5603617429733276, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.25024837255477905, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.13115696609020233, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.12749344110488892, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7899326682090759, "metrics/hit_grouping/tracker/purity": 0.15525877475738525, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.879992663860321, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.014705882407724857, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.013678333722054958, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.011057574301958084, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.052413273602724075, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6759158968925476, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.1703665405511856, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.08665099740028381, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.045768532902002335, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5487867593765259, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5404996871948242, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4007050395011902, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.18689276278018951, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.1575370579957962, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8920597434043884, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4285111725330353, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.23444178700447083, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.2387925535440445, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5273137092590332, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.12064157426357269, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.04716108739376068, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.63982391357422, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.1003028154373169, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8125, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.008679652586579323, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.3764542043209076, "metrics/particle/class_1/pid_accuracy": 0.8279548287391663, "metrics/particle/class_2/efficiency": 0.33537206053733826, "metrics/particle/class_2/pid_accuracy": 0.07396149635314941, "metrics/particle/class_3/efficiency": 0.3479002118110657, "metrics/particle/class_3/pid_accuracy": 0.8525198101997375, "metrics/particle/class_4/efficiency": 0.3877755403518677, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 0.34674328565597534, "metrics/particle/class_5/pid_accuracy": 0.016574585810303688, "metrics/particle/count/bias_mean": -17.68033790588379, "metrics/particle/count/mae": 19.03059959411621, "metrics/particle/count/prediction_mean": 52.974609375, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.1894860714673996, "metrics/particle/event/energy_response_mean": 0.975586473941803, "metrics/particle/event/met_abs_error": 18.8966007232666, "metrics/particle/event/scalar_pt_relative_abs_error": 0.16487963497638702, "metrics/particle/event/scalar_pt_response_mean": 0.9926671385765076, "metrics/particle/event/vector_pt_closure": 0.13952301442623138, "metrics/particle/matched/delta_eta_abs_mean": 0.024471115320920944, "metrics/particle/matched/delta_phi_abs_mean": 0.023022668436169624, "metrics/particle/matched/delta_r_mean": 0.03752816095948219, "metrics/particle/matched/energy_relative_abs_mean": 0.21756771206855774, "metrics/particle/matched/pid_accuracy": 0.7690265774726868, "metrics/particle/matched/pt_relative_abs_mean": 0.21138739585876465, "metrics/particle/matching/duplicate_fraction": 0.12196291238069534, "metrics/particle/matching/efficiency": 0.3610379099845886, "metrics/particle/matching/f1": 0.41267016530036926, "metrics/particle/matching/purity": 0.4815347492694855, "metrics/particle/matching/target_coverage_dr0p05": 0.3050052523612976, "metrics/particle/matching/target_coverage_dr0p10": 0.4462432861328125, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.214215949177742, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.4290400445461273, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.0373246856033802, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.2214231938123703, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.2124708890914917, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.023867163807153702, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2523583173751831, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.02389453537762165, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.6813334226608276, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.08428849279880524, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.4512670040130615, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 3641.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.7344920039176941, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 3.00681734085083, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 4.208763599395752, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 4.01975154876709, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.3962344825267792, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 28.601726531982422, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.04065040498971939, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.3538334369659424, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.29123395681381226, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.855738639831543, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 7011.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.338005930185318, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 4.306542873382568, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 22.094615936279297, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 10.077592849731445, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.4449741840362549, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.9069768190383911, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 18.82103729248047, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 5.277320384979248, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 32.67216110229492, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.034202396869659424, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.22363980114459991, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.2201380729675293, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012841694056987762, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2609255611896515, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.06630824506282806, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 3.079695701599121, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.25406819581985474, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 6.516053199768066, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 558.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.2738487720489502, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 3.1982946395874023, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 11.160323143005371, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 11.0191650390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.5037821531295776, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 95.90486907958984, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09379727393388748, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.294760704040527, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.8144386410713196, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.561891555786133, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1322.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.20232051610946655, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -14.718022346496582, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 19.49215316772461, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 3.013796329498291, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.09536154568195343, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.020801424980163574, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 19.16144561767578, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 2.214998483657837, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 44.450599670410156, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.038134682923555374, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.2170867621898651, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.21101829409599304, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.02311701886355877, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2509683072566986, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.002607278758659959, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.5457565188407898, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.024613186717033386, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.4270975589752197, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 9205.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.7107205390930176, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 4.937417984008789, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 4.1149420738220215, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.930274486541748, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.4267326593399048, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 31.82493782043457, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.022251171991229057, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.4869235754013062, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.249323308467865, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 5.357672214508057, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 19190.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.3955906927585602, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 12.512535095214844, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 24.510831832885742, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 20.213321685791016, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 5.85733699798584, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.5874297618865967, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.451830863952637, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 4.070046901702881, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 22.330480575561523, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.037750959396362305, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.21853750944137573, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.21179251372814178, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.02182384766638279, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.2517683804035187, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.011041480116546154, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.9597055315971375, "metrics/particle/momentum/all/accepted/pz_bias": 0.05037517845630646, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.530150890350342, "metrics/particle/momentum/all/accepted/samples": 13404.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.7438244223594666, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 4.362096786499023, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4.477248668670654, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4.293564796447754, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.42266467213630676, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 36.828731536865234, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.030374595895409584, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.842621088027954, "metrics/particle/momentum/all/assigned/pz_bias": 0.2871432304382324, "metrics/particle/momentum/all/assigned/pz_residual_rms": 5.514673233032227, "metrics/particle/momentum/all/assigned/samples": 27523.0, "metrics/particle/momentum/all/efficiency": 0.36424902081489563, "metrics/particle/momentum/all/event/energy_bias": 2.1010539531707764, "metrics/particle/momentum/all/event/p3_error_mean": 49.57334518432617, "metrics/particle/momentum/all/event/pt_vector_error_mean": 24.69770050048828, "metrics/particle/momentum/all/event/px_bias": 7.397672176361084, "metrics/particle/momentum/all/event/py_bias": -3.4736053943634033, "metrics/particle/momentum/all/event/pz_abs_mean": 36.975135803222656, "metrics/particle/momentum/all/event/pz_bias": 11.562366485595703, "metrics/particle/momentum/all/event/pz_residual_rms": 60.36271667480469, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.045831963419914246, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2307284027338028, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.22158317267894745, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.026290500536561012, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2614295780658722, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.014204545877873898, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 1.145021915435791, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.048380669206380844, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.801405906677246, "metrics/particle/momentum/class_1/accepted/samples": 6688.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.6565898656845093, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 3.427187919616699, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 3.7704038619995117, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 3.557823657989502, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.394627183675766, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 36.30619430541992, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.032966215163469315, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.9974329471588135, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.2698512673377991, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 5.846250057220459, "metrics/particle/momentum/class_1/assigned/samples": 13438.0, "metrics/particle/momentum/class_1/efficiency": 0.37697988748550415, "metrics/particle/momentum/class_1/event/energy_bias": 30.749237060546875, "metrics/particle/momentum/class_1/event/p3_error_mean": 34.56919860839844, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 21.425701141357422, "metrics/particle/momentum/class_1/event/px_bias": 4.899826526641846, "metrics/particle/momentum/class_1/event/py_bias": 2.818225860595703, "metrics/particle/momentum/class_1/event/pz_abs_mean": 22.845420837402344, "metrics/particle/momentum/class_1/event/pz_bias": 6.517934799194336, "metrics/particle/momentum/class_1/event/pz_residual_rms": 31.141807556152344, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.042686186730861664, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24543534219264984, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23778188228607178, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08934983611106873, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2763650417327881, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029182879254221916, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6808687448501587, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.004249082878232002, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6634554862976074, "metrics/particle/momentum/class_2/accepted/samples": 1028.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6310550570487976, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8021941781044006, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1199313402175903, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.772225558757782, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30259525775909424, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.736049175262451, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05742683634161949, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.656126022338867, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.24879176914691925, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.352217197418213, "metrics/particle/momentum/class_2/assigned/samples": 1811.0, "metrics/particle/momentum/class_2/efficiency": 0.34427326917648315, "metrics/particle/momentum/class_2/event/energy_bias": -34.13214874267578, "metrics/particle/momentum/class_2/event/p3_error_mean": 32.769493103027344, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 17.083751678466797, "metrics/particle/momentum/class_2/event/px_bias": -0.6557945609092712, "metrics/particle/momentum/class_2/event/py_bias": -2.2267181873321533, "metrics/particle/momentum/class_2/event/pz_abs_mean": 23.763656616210938, "metrics/particle/momentum/class_2/event/pz_bias": -1.2211425304412842, "metrics/particle/momentum/class_2/event/pz_residual_rms": 37.95138168334961, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.026823531836271286, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.19838270545005798, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.19445374608039856, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.0014455413911491632, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2337675541639328, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.003288872539997101, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.5650278329849243, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.05947147309780121, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.7627248764038086, "metrics/particle/momentum/class_3/accepted/samples": 5473.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.862794816493988, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.818761348724365, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.712068557739258, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.589441299438477, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4733777642250061, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 40.32205581665039, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.022782888263463974, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.5157713890075684, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.31023111939430237, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.965189456939697, "metrics/particle/momentum/class_3/assigned/samples": 11851.0, "metrics/particle/momentum/class_3/efficiency": 0.35234662890434265, "metrics/particle/momentum/class_3/event/energy_bias": 17.375131607055664, "metrics/particle/momentum/class_3/event/p3_error_mean": 30.450563430786133, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 17.528717041015625, "metrics/particle/momentum/class_3/event/px_bias": 2.4182443618774414, "metrics/particle/momentum/class_3/event/py_bias": -2.8509349822998047, "metrics/particle/momentum/class_3/event/pz_abs_mean": 21.367040634155273, "metrics/particle/momentum/class_3/event/pz_bias": 5.638946533203125, "metrics/particle/momentum/class_3/event/pz_residual_rms": 41.43605041503906, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04007343947887421, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.21697577834129333, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.2218026965856552, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.073628269135952, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.26013195514678955, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.027210883796215057, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.875570297241211, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.14408954977989197, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.720475196838379, "metrics/particle/momentum/class_4/accepted/samples": 147.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.7742223143577576, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 11.053868293762207, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 8.624826431274414, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 8.427302360534668, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.4318181872367859, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 31.404895782470703, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03246753290295601, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 2.2403619289398193, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.593217670917511, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.299455165863037, "metrics/particle/momentum/class_4/assigned/samples": 308.0, "metrics/particle/momentum/class_4/efficiency": 0.3919999897480011, "metrics/particle/momentum/class_4/event/energy_bias": -6.539658546447754, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.854221343994141, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.774823188781738, "metrics/particle/momentum/class_4/event/px_bias": 0.6257258057594299, "metrics/particle/momentum/class_4/event/py_bias": -0.5165429711341858, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.601644277572632, "metrics/particle/momentum/class_4/event/pz_bias": -0.0370001345872879, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.958283424377441, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.04282807931303978, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.23843403160572052, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.2298325002193451, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08984620869159698, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.27120107412338257, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.014705882407724857, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.61688232421875, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.13762445747852325, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.6683340072631836, "metrics/particle/momentum/class_5/accepted/samples": 68.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.37166500091552734, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 1.6342756748199463, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.5849835872650146, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 1.105837106704712, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3391304314136505, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 6.760783672332764, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0782608687877655, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.558889627456665, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2873075306415558, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 7.68623685836792, "metrics/particle/momentum/class_5/assigned/samples": 115.0, "metrics/particle/momentum/class_5/efficiency": 0.4146341383457184, "metrics/particle/momentum/class_5/event/energy_bias": -5.351508140563965, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.929119110107422, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.7450597286224365, "metrics/particle/momentum/class_5/event/px_bias": 0.10967011749744415, "metrics/particle/momentum/class_5/event/py_bias": -0.6976351737976074, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.557018995285034, "metrics/particle/momentum/class_5/event/pz_bias": 0.663627028465271, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.915273666381836, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.7458285689353943, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.7475165724754333, "metrics/set_slot/angular_recall_all_dr0p10": 0.820807933807373, "metrics/set_slot/angular_recall_selected_dr0p10": 0.4924810528755188, "metrics/set_slot/soft_count_bias_mean": -2.3204314708709717}, "step_train_time": 5.130382776260376, "step_valid_time": 2790.6498577594757, "step_total_time": 2795.780241250992, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.158702758097112, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.37806054416181434, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2930, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2425, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2091, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7927427263811703, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8276450511945392, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8098179996660544, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17235494880546076, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6835567178816607, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7136518771331058, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6982801803306061, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7927427263811703, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0721929490721231, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3383327705654805, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2131, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1797, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1664, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7864332603938731, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8432660722665415, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.813858695652174, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.15673392773345848, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7282275711159737, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7808540591271703, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7536231884057971, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7864332603938731, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9861804641955143, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3377155356500162, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1156, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 953, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 844, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7208774583963692, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8243944636678201, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7691686844229217, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17560553633217993, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6384266263237519, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7301038062283737, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6811945117029863, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7208774583963692} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_10000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..303999bd6b63d82be1d36a9da94427220e0f3673 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.47366426676771506, "train": {"Classification_binary": 0.29124051332473755, "Cardinality": 0.19688734412193298, "Classification": 0.15896108746528625, "Regression_pt": 0.23562957346439362, "Regression_eta": 0.002913464792072773, "Regression_sin_phi": 0.001585109275765717, "Regression_cos_phi": 0.0016482992796227336, "Regression_energy": 0.2305353581905365, "HitGrouping": 0.19343113899230957, "Auxiliary": 0.32609063386917114, "Total": 1.714699387550354, "Total_unweighted": 0.9225134057924151}, "train_group": {}, "valid": {"Classification_binary": 0.26634512344996136, "Cardinality": 0.1979739268620809, "Classification": 0.14905253052711487, "Regression_pt": 0.22259249289830527, "Regression_eta": 0.0028527562196056047, "Regression_sin_phi": 0.0015185164908568065, "Regression_cos_phi": 0.001517948384086291, "Regression_energy": 0.21904138724009195, "HitGrouping": 0.19352483749389648, "Auxiliary": 0.30302222569783527, "Total": 1.6300438245137532, "Total_unweighted": 0.8629207552100222, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8271056413650513, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.48495644330978394, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3639632761478424, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4911302626132965, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.47023966908454895, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4019116163253784, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7827860713005066, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4659983217716217, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.35228949785232544, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.47106853127479553, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.44701480865478516, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4752584993839264, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8428351283073425, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4508574604988098, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.33594074845314026, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4544827938079834, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4323901832103729, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.37199723720550537, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8334077596664429, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4623633325099945, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3456995487213135, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4668729305267334, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4449923038482666, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.389076828956604, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.00250301044434309, "metrics/hit_grouping/calorimeter/background_recall": 0.111568383872509, "metrics/hit_grouping/calorimeter/completeness": 0.5328526496887207, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5369449853897095, "metrics/hit_grouping/calorimeter/energy_purity": 0.5297927856445312, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4215826988220215, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8599030375480652, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4883756935596466, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2756030857563019, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.49588850140571594, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4687505066394806, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.32823169231414795, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8315937519073486, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.4849766492843628, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3625057637691498, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.48807960748672485, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.47087523341178894, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.39359742403030396, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7812119126319885, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3865433931350708, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4716937243938446, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3863464593887329, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.37047454714775085, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5085846781730652, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8103896975517273, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.49089309573173523, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.41388019919395447, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4983840584754944, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4864206612110138, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.44298774003982544, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.798160195350647, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5714098811149597, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5697374939918518, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5724627375602722, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5280070304870605, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.43283581733703613, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8610752820968628, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.42028409242630005, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24949415028095245, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.42216578125953674, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3928634226322174, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.328898161649704, "metrics/hit_grouping/calorimeter/purity": 0.5247519016265869, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8746576309204102, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.044708043336868286, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1071428582072258, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.044423557817935944, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.013219154439866543, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.839747428894043, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19789938628673553, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11830488592386246, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.1975483000278473, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.17763970792293549, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.273690402507782, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.832405686378479, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.48293113708496094, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3614199459552765, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4877717196941376, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4659403860569, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4009004831314087, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6810750961303711, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.45415037870407104, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.41248470544815063, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.41463324427604675, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.664348840713501, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7477793097496033, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3562386929988861, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4302130937576294, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.3904610872268677, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.48764315247535706, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.589756429195404, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4584112763404846, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3089144825935364, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3194308876991272, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7698230147361755, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6294952034950256, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44955477118492126, "metrics/hit_grouping/tracker/all/object_completeness": 0.34970608353614807, "metrics/hit_grouping/tracker/all/object_purity": 0.35393765568733215, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.716562807559967, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.016709022223949432, "metrics/hit_grouping/tracker/background_recall": 0.12823650240898132, "metrics/hit_grouping/tracker/completeness": 0.35067418217658997, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5547676086425781, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.642244815826416, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4406452775001526, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4128292202949524, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4245530664920807, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7118183374404907, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6138235330581665, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.46124523878097534, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.32139402627944946, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.32996633648872375, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7344345450401306, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6188390254974365, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4562669098377228, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.24799087643623352, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2324981987476349, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7085662484169006, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6376113295555115, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.48382440209388733, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3590759038925171, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3643024265766144, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7175851464271545, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6803516149520874, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5584975481033325, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.39921292662620544, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3259578049182892, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6490147709846497, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6098268628120422, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38547301292419434, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.32864999771118164, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.34547895193099976, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.727342963218689, "metrics/hit_grouping/tracker/purity": 0.33271414041519165, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8799653649330139, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03205128386616707, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.07404371351003647, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.07056901603937149, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07013574987649918, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6496156454086304, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2627774775028229, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.22986914217472076, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2038072794675827, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5684047341346741, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5901453495025635, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5317553281784058, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.40360239148139954, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.41226091980934143, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8285490274429321, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5158429741859436, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.42735618352890015, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.4711368680000305, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6171107888221741, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.03330978378653526, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.042635973542928696, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.408079147338867, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.23716282844543457, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.14553964138031006, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7193424105644226, "metrics/particle/class_1/pid_accuracy": 0.9340898394584656, "metrics/particle/class_2/efficiency": 0.47264695167541504, "metrics/particle/class_2/pid_accuracy": 0.5375030040740967, "metrics/particle/class_3/efficiency": 0.6225652098655701, "metrics/particle/class_3/pid_accuracy": 0.9461338520050049, "metrics/particle/class_4/efficiency": 0.5901803374290466, "metrics/particle/class_4/pid_accuracy": 0.3344652056694031, "metrics/particle/class_5/efficiency": 0.7068965435028076, "metrics/particle/class_5/pid_accuracy": 0.38753387331962585, "metrics/particle/count/bias_mean": -4.610025882720947, "metrics/particle/count/mae": 6.801432132720947, "metrics/particle/count/prediction_mean": 66.044921875, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.10107613354921341, "metrics/particle/event/energy_response_mean": 1.0034981966018677, "metrics/particle/event/met_abs_error": 9.942499160766602, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08320354670286179, "metrics/particle/event/scalar_pt_response_mean": 1.014882206916809, "metrics/particle/event/vector_pt_closure": 0.0861717164516449, "metrics/particle/matched/delta_eta_abs_mean": 0.017313966527581215, "metrics/particle/matched/delta_phi_abs_mean": 0.017628010362386703, "metrics/particle/matched/delta_r_mean": 0.027665946632623672, "metrics/particle/matched/energy_relative_abs_mean": 0.1684974730014801, "metrics/particle/matched/pid_accuracy": 0.9079170823097229, "metrics/particle/matched/pt_relative_abs_mean": 0.16720052063465118, "metrics/particle/matching/duplicate_fraction": 0.1000739336013794, "metrics/particle/matching/efficiency": 0.6572341918945312, "metrics/particle/matching/f1": 0.6793985962867737, "metrics/particle/matching/purity": 0.7031100392341614, "metrics/particle/matching/target_coverage_dr0p05": 0.5978014469146729, "metrics/particle/matching/target_coverage_dr0p10": 0.7242043614387512, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4564620554447174, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7147135138511658, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.026294762268662453, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16239966452121735, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1619764119386673, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.006344207096844912, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20257443189620972, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.011393636465072632, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9260004758834839, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.025205736979842186, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.4244558811187744, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7197.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3721316456794739, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4815951585769653, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.8642112016677856, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.6887181997299194, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.20074968039989471, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.296116828918457, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0236359853297472, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3742278814315796, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05179137364029884, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.506230115890503, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9604.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6681210398674011, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.262331008911133, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.259408950805664, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.162026405334473, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.9372050762176514, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.2871109247207642, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.510833740234375, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.644276738166809, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.336824417114258, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027441632002592087, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17080719769001007, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17025543749332428, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.008309162221848965, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.21002142131328583, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03564484789967537, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.679284930229187, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.213026762008667, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.474904537200928, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1543.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5342010259628296, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2594108581542969, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.1770119667053223, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.9897119998931885, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24954211711883545, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 20.8053035736084, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06913919746875763, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.843681812286377, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.14291657507419586, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.278289794921875, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2184.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.559463381767273, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.43272590637207, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.74250316619873, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4419140815734863, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.036612316966056824, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.10013670474290848, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.560283660888672, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6103712320327759, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.76935386657715, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02838248759508133, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.17058967053890228, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16825278103351593, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.011790571734309196, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2079087346792221, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001503071514889598, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3324790596961975, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.008078520186245441, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.939800500869751, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15302.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.40586721897125244, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.187220811843872, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.7536176443099976, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.5579451322555542, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.23836159706115723, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.364983558654785, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00920245423913002, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7021124362945557, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.029249049723148346, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.417588472366333, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22168.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6576131582260132, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 7.0577473640441895, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 14.76205825805664, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.563727378845215, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.1783363819122314, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -5.138123035430908, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.329158306121826, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.7068455219268799, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.541803359985352, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.027697142213582993, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16815195977687836, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1665024757385254, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.009936761111021042, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20646364986896515, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.006655020173639059, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5965875387191772, "metrics/particle/momentum/all/accepted/pz_bias": -0.026359017938375473, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.899185299873352, "metrics/particle/momentum/all/accepted/samples": 24042.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.4045797884464264, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.9279690980911255, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.876448392868042, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.6870216131210327, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.22844268381595612, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.164691925048828, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.017139827832579613, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.029953956604004, "metrics/particle/momentum/all/assigned/pz_bias": 0.024551408365368843, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.420607328414917, "metrics/particle/momentum/all/assigned/samples": 33956.0, "metrics/particle/momentum/all/efficiency": 0.6533329486846924, "metrics/particle/momentum/all/event/energy_bias": 4.887352466583252, "metrics/particle/momentum/all/event/p3_error_mean": 26.487037658691406, "metrics/particle/momentum/all/event/pt_vector_error_mean": 15.177244186401367, "metrics/particle/momentum/all/event/px_bias": -3.152153968811035, "metrics/particle/momentum/all/event/py_bias": -6.525371074676514, "metrics/particle/momentum/all/event/pz_abs_mean": 17.890138626098633, "metrics/particle/momentum/all/event/pz_bias": 1.7407511472702026, "metrics/particle/momentum/all/event/pz_residual_rms": 28.136545181274414, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03239194303750992, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.16708454489707947, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1670648157596588, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.01259053498506546, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2066856473684311, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005630452185869217, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6294413208961487, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0182444266974926, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7760783433914185, "metrics/particle/momentum/class_1/accepted/samples": 12610.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.29957786202430725, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9226658344268799, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.0009571313858032, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.7977316975593567, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.1787491738796234, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.948792934417725, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01355159841477871, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.9392917156219482, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.038707613945007324, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.9606008529663086, "metrics/particle/momentum/class_1/assigned/samples": 16677.0, "metrics/particle/momentum/class_1/efficiency": 0.7107829451560974, "metrics/particle/momentum/class_1/event/energy_bias": 11.623117446899414, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.928104400634766, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.171833038330078, "metrics/particle/momentum/class_1/event/px_bias": -2.4413177967071533, "metrics/particle/momentum/class_1/event/py_bias": -4.012567043304443, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.30960464477539, "metrics/particle/momentum/class_1/event/pz_bias": 2.4809317588806152, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.898193359375, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.033798813819885254, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20937450230121613, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2100623995065689, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.047724753618240356, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24975456297397614, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03687943145632744, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7780566215515137, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.16094563901424408, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.296090126037598, "metrics/particle/momentum/class_2/accepted/samples": 1410.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5536326766014099, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7706542611122131, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9363572597503662, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5810562372207642, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3007388114929199, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.815058469772339, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.056931767612695694, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.540175676345825, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.05001623556017876, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.061744213104248, "metrics/particle/momentum/class_2/assigned/samples": 2301.0, "metrics/particle/momentum/class_2/efficiency": 0.4722036123275757, "metrics/particle/momentum/class_2/event/energy_bias": -14.337677001953125, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.287778854370117, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.053577423095703, "metrics/particle/momentum/class_2/event/px_bias": -0.9963486790657043, "metrics/particle/momentum/class_2/event/py_bias": -0.3379772901535034, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.894857406616211, "metrics/particle/momentum/class_2/event/pz_bias": -0.8959916234016418, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.608592987060547, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02063993737101555, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16349388659000397, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15950822830200195, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.0018150170799344778, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19910882413387299, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0026834553573280573, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3556467890739441, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.018966276198625565, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3488966226577759, "metrics/particle/momentum/class_3/accepted/samples": 9689.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5004863142967224, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.1524462699890137, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.9406304359436035, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.7922799587249756, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.27224984765052795, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.563580513000488, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.014027087017893791, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.870425820350647, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.026513008400797844, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.3071389198303223, "metrics/particle/momentum/class_3/assigned/samples": 14472.0, "metrics/particle/momentum/class_3/efficiency": 0.6237687468528748, "metrics/particle/momentum/class_3/event/energy_bias": 8.454191207885742, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.994243621826172, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.126550674438477, "metrics/particle/momentum/class_3/event/px_bias": -0.1549350917339325, "metrics/particle/momentum/class_3/event/py_bias": -1.2617764472961426, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.851749420166016, "metrics/particle/momentum/class_3/event/pz_bias": 0.4272012710571289, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.179576873779297, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.029176563024520874, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15951021015644073, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15339796245098114, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.017962059006094933, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19542574882507324, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.018691588193178177, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0129886865615845, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.04650808870792389, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.461235284805298, "metrics/particle/momentum/class_4/accepted/samples": 214.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5798371434211731, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 7.357694149017334, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 6.108602523803711, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.96127462387085, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3237822353839874, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 34.503448486328125, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.022922636941075325, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1686372756958008, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.08791185915470123, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.6052803993225098, "metrics/particle/momentum/class_4/assigned/samples": 349.0, "metrics/particle/momentum/class_4/efficiency": 0.5706666707992554, "metrics/particle/momentum/class_4/event/energy_bias": -0.8226754069328308, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.564326524734497, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.6271190643310547, "metrics/particle/momentum/class_4/event/px_bias": 0.29012808203697205, "metrics/particle/momentum/class_4/event/py_bias": -0.4604107141494751, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.9125620126724243, "metrics/particle/momentum/class_4/event/pz_bias": -0.1130342036485672, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.7874369621276855, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02984732575714588, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18762749433517456, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1838199943304062, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007569524459540844, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22302743792533875, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05882352963089943, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.98489248752594, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.14276562631130219, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.0896735191345215, "metrics/particle/momentum/class_5/accepted/samples": 119.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.1435718834400177, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.7356641888618469, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.149368405342102, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.976852297782898, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.19745223224163055, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 10.295145034790039, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08917197585105896, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.9232122898101807, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.31711331009864807, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.14385461807251, "metrics/particle/momentum/class_5/assigned/samples": 157.0, "metrics/particle/momentum/class_5/efficiency": 0.7256097793579102, "metrics/particle/momentum/class_5/event/energy_bias": -0.02960432693362236, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.5659804344177246, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.9507379531860352, "metrics/particle/momentum/class_5/event/px_bias": 0.15031978487968445, "metrics/particle/momentum/class_5/event/py_bias": -0.45263972878456116, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2691136598587036, "metrics/particle/momentum/class_5/event/pz_bias": -0.15835589170455933, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.096861839294434, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4952593743801117, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.498803973197937, "metrics/set_slot/angular_recall_all_dr0p10": 0.8866354823112488, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7284521460533142, "metrics/set_slot/soft_count_bias_mean": 2.0057249069213867}, "step_train_time": 5.503641843795776, "step_valid_time": 30114.946093559265, "step_total_time": 30120.44973564148, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0300793594209545, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.22270956012901033, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2807, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2731, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9176201372997712, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9273207796498183, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9224449556358857, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07267922035018169, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8927754168028768, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9022134126197555, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8974696023660861, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9176201372997712, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0145319644541175, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21132565079232424, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2244, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2103, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2078, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9203501094091904, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9371657754010695, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9286818282181497, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06283422459893048, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9094091903719912, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9260249554367201, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9176418635460366, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9203501094091904, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0097438426404979, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21409134650017514, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1297, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1173, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1158, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8872919818456884, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9043947571318427, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8957617411225659, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09560524286815729, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.875945537065053, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8928296067848882, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8843069873997709, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8872919818456884} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_11000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..81d37527b178ddbb0806434efbde8833a34315a1 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 0.5210306934444866, "train": {"Classification_binary": 0.28770363330841064, "Cardinality": 0.17365878820419312, "Classification": 0.14765042066574097, "Regression_pt": 0.19651497900485992, "Regression_eta": 0.0029713984113186598, "Regression_sin_phi": 0.0014918794622644782, "Regression_cos_phi": 0.0016633588820695877, "Regression_energy": 0.19425304234027863, "HitGrouping": 0.1891702115535736, "Auxiliary": 0.295910507440567, "Total": 1.599947452545166, "Total_unweighted": 0.8322487120749429}, "train_group": {}, "valid": {"Classification_binary": 0.24962065617243448, "Cardinality": 0.18708956241607666, "Classification": 0.13796624541282654, "Regression_pt": 0.2046041488647461, "Regression_eta": 0.002531562621394793, "Regression_sin_phi": 0.001394470687955618, "Regression_cos_phi": 0.0013461567771931489, "Regression_energy": 0.20288586616516113, "HitGrouping": 0.18866153558095297, "Auxiliary": 0.288485586643219, "Total": 1.5235616366068523, "Total_unweighted": 0.8003491067017118, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8215900659561157, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.49433887004852295, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.35487839579582214, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.502598226070404, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.49745553731918335, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.41847261786460876, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7752481698989868, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4675334393978119, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3260709047317505, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4730687439441681, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4748978316783905, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.47710487246513367, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8408394455909729, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.45658376812934875, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.33695903420448303, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.45992526412010193, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4423684775829315, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3835686147212982, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.829915463924408, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4688718914985657, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.34154242277145386, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.4738735854625702, "metrics/hit_grouping/calorimeter/all/object_purity": 0.461582750082016, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.40140482783317566, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003081700997427106, "metrics/hit_grouping/calorimeter/background_recall": 0.1292704939842224, "metrics/hit_grouping/calorimeter/completeness": 0.5473560094833374, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5511044263839722, "metrics/hit_grouping/calorimeter/energy_purity": 0.5439286231994629, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4186178147792816, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8564392924308777, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5004727840423584, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.27048248052597046, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5086312890052795, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4934837818145752, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3372212052345276, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8276848196983337, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.48615044355392456, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.36411792039871216, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.49039044976234436, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4788958430290222, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.413173645734787, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7780811786651611, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.38703152537345886, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.46368908882141113, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.38636404275894165, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3790956139564514, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5204176306724548, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8096864223480225, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5019629001617432, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.41309839487075806, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5098910331726074, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5016135573387146, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.44737792015075684, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8036004304885864, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5820376873016357, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5692228674888611, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5846487283706665, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5514312386512756, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.43386515974998474, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.853675127029419, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4216514825820923, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24146176874637604, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4239543378353119, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4100647270679474, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.35066527128219604, "metrics/hit_grouping/calorimeter/purity": 0.5393477082252502, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8650264739990234, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05338572338223457, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10902255773544312, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05402345210313797, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.030672919005155563, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.033834587782621384, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8337562084197998, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.19808445870876312, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11948204785585403, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.19777190685272217, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.17771512269973755, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2725132405757904, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8291434645652771, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.48973336815834045, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3568868935108185, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.4950827658176422, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.48337623476982117, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4141015112400055, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6770108938217163, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.44632768630981445, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.42651820182800293, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.45638057589530945, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6739099025726318, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7374686002731323, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3122362792491913, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.43381768465042114, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.43647560477256775, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4990958273410797, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5889478921890259, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.46172112226486206, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.32342156767845154, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.35177120566368103, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.771621823310852, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.626984179019928, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44594836235046387, "metrics/hit_grouping/tracker/all/object_completeness": 0.3632623255252838, "metrics/hit_grouping/tracker/all/object_purity": 0.39017918705940247, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7214604020118713, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.018157031387090683, "metrics/hit_grouping/tracker/background_recall": 0.13628828525543213, "metrics/hit_grouping/tracker/completeness": 0.3652787506580353, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5424084663391113, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6559292078018188, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42951077222824097, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4338940680027008, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4550475478172302, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6999824643135071, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6046534776687622, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.46197131276130676, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3305532932281494, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3715844452381134, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.750408411026001, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.5896304249763489, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4638412892818451, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2504703104496002, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.275219589471817, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7368800640106201, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6253705620765686, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4720217287540436, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3696095049381256, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.41222551465034485, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7288783192634583, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6337429285049438, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5246305465698242, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.36664077639579773, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3851949870586395, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7118226885795593, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6278703808784485, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3979468047618866, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.35440513491630554, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3624117076396942, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7135445475578308, "metrics/hit_grouping/tracker/purity": 0.34708184003829956, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8762256503105164, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.023378582671284676, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.0739399865269661, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.07547865062952042, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06749623268842697, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6470413208007812, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2607124447822571, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.25550585985183716, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.24297405779361725, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5756323933601379, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5878236293792725, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5287337303161621, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.41787633299827576, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.45283499360084534, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8343125581741333, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.525315523147583, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.4241514205932617, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.46739089488983154, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6122715473175049, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.023837242275476456, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0377967432141304, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.724407196044922, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.25107884407043457, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.15945570170879364, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7479879260063171, "metrics/particle/class_1/pid_accuracy": 0.9544875025749207, "metrics/particle/class_2/efficiency": 0.46992865204811096, "metrics/particle/class_2/pid_accuracy": 0.597252368927002, "metrics/particle/class_3/efficiency": 0.618258535861969, "metrics/particle/class_3/pid_accuracy": 0.9328524470329285, "metrics/particle/class_4/efficiency": 0.5901803374290466, "metrics/particle/class_4/pid_accuracy": 0.25806450843811035, "metrics/particle/class_5/efficiency": 0.7662835121154785, "metrics/particle/class_5/pid_accuracy": 0.4625000059604645, "metrics/particle/count/bias_mean": -6.353515625, "metrics/particle/count/mae": 7.572265625, "metrics/particle/count/prediction_mean": 64.30142974853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09524375200271606, "metrics/particle/event/energy_response_mean": 0.9923011660575867, "metrics/particle/event/met_abs_error": 9.12009048461914, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08140965551137924, "metrics/particle/event/scalar_pt_response_mean": 1.0200146436691284, "metrics/particle/event/vector_pt_closure": 0.0800730511546135, "metrics/particle/matched/delta_eta_abs_mean": 0.015875760465860367, "metrics/particle/matched/delta_phi_abs_mean": 0.01693698577582836, "metrics/particle/matched/delta_r_mean": 0.026015089824795723, "metrics/particle/matched/energy_relative_abs_mean": 0.1656813621520996, "metrics/particle/matched/pid_accuracy": 0.9173033833503723, "metrics/particle/matched/pt_relative_abs_mean": 0.1644735187292099, "metrics/particle/matching/duplicate_fraction": 0.09447487443685532, "metrics/particle/matching/efficiency": 0.6693235039710999, "metrics/particle/matching/f1": 0.7008340954780579, "metrics/particle/matching/purity": 0.7354581952095032, "metrics/particle/matching/target_coverage_dr0p05": 0.613429069519043, "metrics/particle/matching/target_coverage_dr0p10": 0.7321010828018188, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4680537283420563, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7194865942001343, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024485258385539055, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16307497024536133, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.16201777756214142, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02206902764737606, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.20242495834827423, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00882432796061039, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8742096424102783, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.04727208614349365, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2562129497528076, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7366.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29725050926208496, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.166772723197937, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.351226568222046, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1969811916351318, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16922584176063538, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.253454208374023, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017594369128346443, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2395693063735962, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.009171581827104092, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.1881914138793945, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9378.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6838098764419556, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.2557907104492188, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.566718101501465, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.9957756996154785, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.8976011872291565, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.9191011190414429, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.87502384185791, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.0191187858581543, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.970162391662598, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025762582197785378, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17287899553775787, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.17328162491321564, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03867831453680992, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.213491290807724, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03010033443570137, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5741034746170044, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1434941440820694, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.7547736167907715, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1495.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.46841979026794434, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4637656211853027, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.137292861938477, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.9909095764160156, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23099273443222046, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 52.025936126708984, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05714285746216774, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.710533618927002, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3849022388458252, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.442803382873535, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2065.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5420594811439514, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.333652496337891, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.634448051452637, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3624584674835205, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.09819664061069489, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.15121078491210938, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.480108261108398, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.0198819637298584, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.40586280822754, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.027216987684369087, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16705738008022308, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16583673655986786, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.006561897695064545, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2064305692911148, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009556575096212327, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3148360848426819, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.001772944233380258, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8797699213027954, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 15696.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.36477237939834595, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.002375364303589, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.6637877225875854, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.4937396049499512, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.21539661288261414, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.711731910705566, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006652291398495436, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6251747012138367, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.06301324814558029, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.988403558731079, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21797.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6745455265045166, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.484769821166992, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 14.310647010803223, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.399823188781738, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -4.51598596572876, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -3.246995210647583, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.759925842285156, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.17908157408237457, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.758782386779785, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026309048756957054, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16621723771095276, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16514445841312408, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01316855102777481, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20567654073238373, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0050901984795928, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5592857003211975, "metrics/particle/momentum/all/accepted/pz_bias": -0.021782048046588898, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6970329284667969, "metrics/particle/momentum/all/accepted/samples": 24557.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.35216137766838074, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.7331660985946655, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.7292687892913818, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.565149188041687, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2033393532037735, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.942303657531738, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012876053340733051, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9280645251274109, "metrics/particle/momentum/all/assigned/pz_bias": 0.01999662257730961, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.144620180130005, "metrics/particle/momentum/all/assigned/samples": 33240.0, "metrics/particle/momentum/all/efficiency": 0.6673279404640198, "metrics/particle/momentum/all/event/energy_bias": 1.4069076776504517, "metrics/particle/momentum/all/event/p3_error_mean": 25.816654205322266, "metrics/particle/momentum/all/event/pt_vector_error_mean": 14.850835800170898, "metrics/particle/momentum/all/event/px_bias": -5.511784076690674, "metrics/particle/momentum/all/event/py_bias": -4.317306995391846, "metrics/particle/momentum/all/event/pz_abs_mean": 17.272220611572266, "metrics/particle/momentum/all/event/pz_bias": 0.178318053483963, "metrics/particle/momentum/all/event/pz_residual_rms": 28.176658630371094, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.030169792473316193, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1662372201681137, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.16309042274951935, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01589202880859375, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.20290333032608032, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004614220932126045, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5879327654838562, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.04030163213610649, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.6087359189987183, "metrics/particle/momentum/class_1/accepted/samples": 13220.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.23329941928386688, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.7853590846061707, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.8715553283691406, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.7088930606842041, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.1486959606409073, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.028981685638428, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009297271259129047, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8296228051185608, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.019184542819857597, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.787972927093506, "metrics/particle/momentum/class_1/assigned/samples": 16564.0, "metrics/particle/momentum/class_1/efficiency": 0.745166540145874, "metrics/particle/momentum/class_1/event/energy_bias": 17.03301239013672, "metrics/particle/momentum/class_1/event/p3_error_mean": 20.342060089111328, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.649073600769043, "metrics/particle/momentum/class_1/event/px_bias": -3.5191681385040283, "metrics/particle/momentum/class_1/event/py_bias": -2.735377311706543, "metrics/particle/momentum/class_1/event/pz_abs_mean": 13.060162544250488, "metrics/particle/momentum/class_1/event/pz_bias": 0.40181946754455566, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.09842872619629, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.032134927809238434, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20176954567432404, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20350457727909088, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.025430038571357727, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24365082383155823, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.025436045601963997, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5752506256103516, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.06457040458917618, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6097543239593506, "metrics/particle/momentum/class_2/accepted/samples": 1376.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.45517659187316895, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6128489375114441, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7324702739715576, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.41931164264678955, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.26737719774246216, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.8387579917907715, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.046802595257759094, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.323072910308838, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.036486733704805374, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.76866340637207, "metrics/particle/momentum/class_2/assigned/samples": 2158.0, "metrics/particle/momentum/class_2/efficiency": 0.4608171582221985, "metrics/particle/momentum/class_2/event/energy_bias": -12.105999946594238, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.26128578186035, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.356825828552246, "metrics/particle/momentum/class_2/event/px_bias": -0.3879414200782776, "metrics/particle/momentum/class_2/event/py_bias": -0.4658372402191162, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.791763305664062, "metrics/particle/momentum/class_2/event/pz_bias": -0.403735876083374, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.70224952697754, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02020559087395668, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16080059111118317, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1624966412782669, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013996871188282967, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20339421927928925, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0020809490233659744, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.34359392523765564, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.008659500628709793, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2469761371612549, "metrics/particle/momentum/class_3/accepted/samples": 9611.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.47667747735977173, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.915827989578247, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.822213649749756, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.6776182651519775, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.25629323720932007, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 23.40298843383789, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011267203837633133, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8053191304206848, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.062079671770334244, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.947659492492676, "metrics/particle/momentum/class_3/assigned/samples": 14023.0, "metrics/particle/momentum/class_3/efficiency": 0.6187471747398376, "metrics/particle/momentum/class_3/event/energy_bias": -1.6241977214813232, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.152365684509277, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.994471549987793, "metrics/particle/momentum/class_3/event/px_bias": -1.4987602233886719, "metrics/particle/momentum/class_3/event/py_bias": -0.6532462239265442, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.756235122680664, "metrics/particle/momentum/class_3/event/pz_bias": 0.37305402755737305, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.070133209228516, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026657434180378914, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16448988020420074, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15420964360237122, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04688253253698349, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19550003111362457, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00917431153357029, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.0197727680206299, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.09632106870412827, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1703732013702393, "metrics/particle/momentum/class_4/accepted/samples": 218.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4864932596683502, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.713423728942871, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.19794225692749, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.101469039916992, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3097345232963562, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.55706787109375, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011799409985542297, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1773113012313843, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.09028330445289612, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.443488359451294, "metrics/particle/momentum/class_4/assigned/samples": 339.0, "metrics/particle/momentum/class_4/efficiency": 0.581333339214325, "metrics/particle/momentum/class_4/event/energy_bias": -1.5593706369400024, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.514333724975586, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.6028318405151367, "metrics/particle/momentum/class_4/event/px_bias": 0.0388333797454834, "metrics/particle/momentum/class_4/event/py_bias": -0.3031236231327057, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.9008369445800781, "metrics/particle/momentum/class_4/event/pz_bias": -0.21161077916622162, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.424028396606445, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02274017594754696, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.19085244834423065, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18183308839797974, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02677944488823414, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22942283749580383, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05303030461072922, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.0437264442443848, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.18565334379673004, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.444274425506592, "metrics/particle/momentum/class_5/accepted/samples": 132.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06302639842033386, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.7353981733322144, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.8062704801559448, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.647059440612793, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.12820513546466827, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.693699836730957, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.07051282376050949, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.5750110149383545, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.016492977738380432, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.232583045959473, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.8048780560493469, "metrics/particle/momentum/class_5/event/energy_bias": -0.3365362286567688, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.3914120197296143, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.8614977598190308, "metrics/particle/momentum/class_5/event/px_bias": -0.1447470486164093, "metrics/particle/momentum/class_5/event/py_bias": -0.15972281992435455, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.1560553312301636, "metrics/particle/momentum/class_5/event/pz_bias": 0.018791303038597107, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.6939995288848877, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4755406081676483, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4785241484642029, "metrics/set_slot/angular_recall_all_dr0p10": 0.8989551067352295, "metrics/set_slot/angular_recall_selected_dr0p10": 0.737288773059845, "metrics/set_slot/soft_count_bias_mean": -1.4797471761703491}, "step_train_time": 5.574024200439453, "step_valid_time": 33186.02903032303, "step_total_time": 33191.603055000305, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0514420933594828, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20144917020955366, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3020, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2823, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2765, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9228506047728016, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9347682119205298, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9287711794703076, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0652317880794702, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9038901601830663, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9155629139072847, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9096890936009212, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9228506047728016, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0273400250843454, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18788468786143697, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2265, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2116, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2101, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.926039387308534, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9342163355408388, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9301098901098901, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06578366445916115, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9194748358862145, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9275938189845475, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9235164835164835, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.926039387308534, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0360995134223585, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18810104117517734, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1299, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1179, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1170, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8918305597579426, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9076212471131639, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8996566196108355, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09237875288683603, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8850226928895613, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9006928406466512, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8927890118275468, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8918305597579426} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_12000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..5d38df1fd16f013acba85396c542a0d812ea5ea2 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 0.5683971201212581, "train": {"Classification_binary": 0.22828291356563568, "Cardinality": 0.1718539595603943, "Classification": 0.1235010102391243, "Regression_pt": 0.19413632154464722, "Regression_eta": 0.0022256257943809032, "Regression_sin_phi": 0.0011769598349928856, "Regression_cos_phi": 0.0012962943874299526, "Regression_energy": 0.19085904955863953, "HitGrouping": 0.18119946122169495, "Auxiliary": 0.2683032155036926, "Total": 1.4073785543441772, "Total_unweighted": 0.7414781749248505}, "train_group": {}, "valid": {"Classification_binary": 0.24410969018936157, "Cardinality": 0.17175225416819254, "Classification": 0.12402657667795818, "Regression_pt": 0.19193055232365927, "Regression_eta": 0.0023777278450628123, "Regression_sin_phi": 0.0012917461184163888, "Regression_cos_phi": 0.0012635382978866498, "Regression_energy": 0.18821010986963907, "HitGrouping": 0.18392125765482584, "Auxiliary": 0.2731391191482544, "Total": 1.4392487208048503, "Total_unweighted": 0.7532099413219839, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8337716460227966, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5215365290641785, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.35487839579582214, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5303568840026855, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5161095857620239, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.38118672370910645, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7780358791351318, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4766460657119751, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3401033878326416, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.48376384377479553, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4646996259689331, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.46639585494995117, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8534132838249207, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.48421263694763184, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3293219208717346, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.4884837567806244, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4686262309551239, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3470030128955841, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8416106104850769, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.4949325621128082, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.33790138363838196, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5008029341697693, "metrics/hit_grouping/calorimeter/all/object_purity": 0.48270660638809204, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3666284382343292, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002702962374314666, "metrics/hit_grouping/calorimeter/background_recall": 0.11626169830560684, "metrics/hit_grouping/calorimeter/completeness": 0.5632839798927307, "metrics/hit_grouping/calorimeter/energy_completeness": 0.568070650100708, "metrics/hit_grouping/calorimeter/energy_purity": 0.560636043548584, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.40448689460754395, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8694475889205933, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5255193710327148, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.26462221145629883, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5347779989242554, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5110278129577637, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3021734058856964, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8420220017433167, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5125760436058044, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.35628741979599, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.517322301864624, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5002875924110413, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3671119213104248, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.784437358379364, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4147922694683075, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.46879351139068604, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.41488656401634216, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.40724873542785645, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.49756380915641785, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8181908130645752, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5235701203346252, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.40696415305137634, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5322568416595459, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.522003173828125, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.41845080256462097, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8015406727790833, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5965476036071777, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5486361384391785, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5944777727127075, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5675148963928223, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.44467318058013916, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8702623844146729, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4536287784576416, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2423815131187439, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4575735926628113, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4325375556945801, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3044944405555725, "metrics/hit_grouping/calorimeter/purity": 0.554831862449646, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8554421067237854, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07510824501514435, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11466165632009506, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07681257277727127, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.03611312061548233, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.016917293891310692, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8510318994522095, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2319553643465042, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11536198109388351, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.23316283524036407, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.22248029708862305, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.25426721572875977, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8408950567245483, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5154582858085632, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.35311955213546753, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5216391682624817, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.50352543592453, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.37817391753196716, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6780079007148743, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4557438790798187, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4342553913593292, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4636355936527252, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6707229018211365, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7369450926780701, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3212778866291046, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4348897933959961, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4350176155567169, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4930680990219116, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5910874605178833, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4688444435596466, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.33320361375808716, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.36234742403030396, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7673046588897705, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6285759210586548, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.45391806960105896, "metrics/hit_grouping/tracker/all/object_completeness": 0.37177255749702454, "metrics/hit_grouping/tracker/all/object_purity": 0.39884573221206665, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7173641920089722, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.010334604419767857, "metrics/hit_grouping/tracker/background_recall": 0.09366119652986526, "metrics/hit_grouping/tracker/completeness": 0.373306006193161, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5316935777664185, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6565818190574646, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4307382106781006, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4402383267879486, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.46058765053749084, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6979659795761108, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.607285737991333, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4735886752605438, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3426283895969391, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3830242156982422, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7449628114700317, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.5921198129653931, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4820559024810791, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.25989413261413574, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.28756213188171387, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.729846715927124, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6278716325759888, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.49231553077697754, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.37995877861976624, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.41643911600112915, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7241232991218567, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.63744056224823, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.508004903793335, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3745526969432831, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4098360538482666, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.693965494632721, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6279022097587585, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3943040072917938, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3606318235397339, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.374003529548645, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7127718329429626, "metrics/hit_grouping/tracker/purity": 0.35190555453300476, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.877927303314209, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.029411764815449715, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.08248920738697052, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.08598282188177109, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07126697152853012, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.646379292011261, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27155396342277527, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.25612151622772217, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.24903155863285065, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5761486887931824, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5896431803703308, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5366795659065247, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4272364377975464, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.46151161193847656, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8285490274429321, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5172433853149414, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7749999761581421, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5168789625167847, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.5792890191078186, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5855782628059387, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.03190935403108597, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.011103513650596142, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.337491989135742, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.24632015824317932, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1546969711780548, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7518594861030579, "metrics/particle/class_1/pid_accuracy": 0.953453540802002, "metrics/particle/class_2/efficiency": 0.46743685007095337, "metrics/particle/class_2/pid_accuracy": 0.5723285675048828, "metrics/particle/class_3/efficiency": 0.6513127684593201, "metrics/particle/class_3/pid_accuracy": 0.9483435750007629, "metrics/particle/class_4/efficiency": 0.6132264733314514, "metrics/particle/class_4/pid_accuracy": 0.3202614486217499, "metrics/particle/class_5/efficiency": 0.7911877632141113, "metrics/particle/class_5/pid_accuracy": 0.4358353614807129, "metrics/particle/count/bias_mean": -4.875, "metrics/particle/count/mae": 6.716145992279053, "metrics/particle/count/prediction_mean": 65.77994537353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09339195489883423, "metrics/particle/event/energy_response_mean": 0.9672541618347168, "metrics/particle/event/met_abs_error": 9.103676795959473, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07276507467031479, "metrics/particle/event/scalar_pt_response_mean": 0.9876266121864319, "metrics/particle/event/vector_pt_closure": 0.07416222244501114, "metrics/particle/matched/delta_eta_abs_mean": 0.015806259587407112, "metrics/particle/matched/delta_phi_abs_mean": 0.01659250818192959, "metrics/particle/matched/delta_r_mean": 0.02567477896809578, "metrics/particle/matched/energy_relative_abs_mean": 0.1659732162952423, "metrics/particle/matched/pid_accuracy": 0.9221708178520203, "metrics/particle/matched/pt_relative_abs_mean": 0.16765376925468445, "metrics/particle/matching/duplicate_fraction": 0.09155961126089096, "metrics/particle/matching/efficiency": 0.6852551698684692, "metrics/particle/matching/f1": 0.7097402215003967, "metrics/particle/matching/purity": 0.7360398769378662, "metrics/particle/matching/target_coverage_dr0p05": 0.6306599378585815, "metrics/particle/matching/target_coverage_dr0p10": 0.7481340765953064, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.47079962491989136, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7360724806785583, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02431001141667366, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16385599970817566, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1694292575120926, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.039789583534002304, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21019327640533447, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009686533361673355, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8940564393997192, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0443795882165432, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.3195652961730957, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7433.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.314995139837265, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3603971004486084, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.6218205690383911, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.480360746383667, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.18319658935070038, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.577245712280273, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017239589244127274, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2311968803405762, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04871271178126335, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3157670497894287, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9629.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6900296807289124, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.7972847819328308, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.32151985168457, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.266179084777832, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3039180040359497, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.040125202387571335, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.894402027130127, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.4606322050094604, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.58177375793457, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025979913771152496, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1722167432308197, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18172071874141693, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.048401590436697006, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22325386106967926, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03257978707551956, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.6292359828948975, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.15556521713733673, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.9766807556152344, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1504.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4907497465610504, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1989303827285767, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.5968892574310303, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.453383445739746, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24834749102592468, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 29.408241271972656, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05618508160114288, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.768893241882324, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.14425475895404816, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.077741622924805, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2118.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5453227162361145, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.137252807617188, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.84821605682373, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.449498176574707, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.171282559633255, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.11374978721141815, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.655333518981934, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.6342074275016785, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.727890014648438, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026427011936903, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1670243740081787, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16608890891075134, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.0015281253727152944, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20597900450229645, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009868015768006444, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3127055764198303, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00298198196105659, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9094525575637817, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16214.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.3475557863712311, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.8295655250549316, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.506064772605896, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.3247590065002441, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.20226646959781647, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.568934440612793, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006411124486476183, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5893332958221436, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.007326316554099321, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.134375810623169, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22149.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6968069076538086, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.1745738983154297, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.780681610107422, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.87441635131836, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.0121796131134033, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.06384118646383286, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.6510210037231445, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.1159988641738892, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.082452774047852, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025774629786610603, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16639851033687592, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.168010875582695, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.013668440282344818, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20830047130584717, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0054470994509756565, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5632420182228088, "metrics/particle/momentum/all/accepted/pz_bias": 0.020495951175689697, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7518434524536133, "metrics/particle/momentum/all/accepted/samples": 25151.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.34725362062454224, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.6568814516067505, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.6695935726165771, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.5019689798355103, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1997285783290863, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.41957950592041, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012597356922924519, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9078609943389893, "metrics/particle/momentum/all/assigned/pz_bias": 0.027639154344797134, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.52789568901062, "metrics/particle/momentum/all/assigned/samples": 33896.0, "metrics/particle/momentum/all/efficiency": 0.6834696531295776, "metrics/particle/momentum/all/event/energy_bias": -6.165392875671387, "metrics/particle/momentum/all/event/p3_error_mean": 23.708730697631836, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.768081665039062, "metrics/particle/momentum/all/event/px_bias": 2.487380027770996, "metrics/particle/momentum/all/event/py_bias": -0.13746581971645355, "metrics/particle/momentum/all/event/pz_abs_mean": 16.675472259521484, "metrics/particle/momentum/all/event/pz_bias": 0.9788410067558289, "metrics/particle/momentum/all/event/pz_residual_rms": 28.732980728149414, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.029085926711559296, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1678091138601303, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.17219293117523193, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02638191170990467, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.21273356676101685, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005352027714252472, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6175198554992676, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.027209855616092682, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7369240522384644, "metrics/particle/momentum/class_1/accepted/samples": 13266.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.24053847789764404, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.7400985956192017, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.8954049348831177, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.7325837016105652, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.15666964650154114, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.1611762046813965, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009012235328555107, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8160937428474426, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.04525536671280861, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8782243728637695, "metrics/particle/momentum/class_1/assigned/samples": 16755.0, "metrics/particle/momentum/class_1/efficiency": 0.7477594017982483, "metrics/particle/momentum/class_1/event/energy_bias": 7.485212326049805, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.797117233276367, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.533899307250977, "metrics/particle/momentum/class_1/event/px_bias": 4.044976234436035, "metrics/particle/momentum/class_1/event/py_bias": 1.6817007064819336, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.253891944885254, "metrics/particle/momentum/class_1/event/pz_bias": 2.395127296447754, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.698272705078125, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03145424649119377, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20075874030590057, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20138350129127502, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.024009963497519493, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24044683575630188, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02951763942837715, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6397274732589722, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.1251787543296814, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.926854133605957, "metrics/particle/momentum/class_2/accepted/samples": 1389.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5054133534431458, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6049421429634094, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8227942585945129, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.511982798576355, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.27330318093299866, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.281461000442505, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04977375641465187, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3348031044006348, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.17127980291843414, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.610225200653076, "metrics/particle/momentum/class_2/assigned/samples": 2210.0, "metrics/particle/momentum/class_2/efficiency": 0.46517080068588257, "metrics/particle/momentum/class_2/event/energy_bias": -13.364435195922852, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.024383544921875, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.832937240600586, "metrics/particle/momentum/class_2/event/px_bias": -1.1224498748779297, "metrics/particle/momentum/class_2/event/py_bias": -1.803634762763977, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.931952476501465, "metrics/particle/momentum/class_2/event/pz_bias": 0.5538978576660156, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.87890625, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02068423479795456, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15951888263225555, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15785689651966095, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0012448015622794628, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1971645951271057, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016743818996474147, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.31912508606910706, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.006302609574049711, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1193933486938477, "metrics/particle/momentum/class_3/accepted/samples": 10153.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4469456970691681, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.7714719772338867, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.6487152576446533, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.496025800704956, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2358444780111313, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.828489303588867, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010049206204712391, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7635958790779114, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.023376336321234703, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.743901252746582, "metrics/particle/momentum/class_3/assigned/samples": 14429.0, "metrics/particle/momentum/class_3/efficiency": 0.653640627861023, "metrics/particle/momentum/class_3/event/energy_bias": 1.0023984909057617, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.85378360748291, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.580753326416016, "metrics/particle/momentum/class_3/event/px_bias": -0.28508296608924866, "metrics/particle/momentum/class_3/event/py_bias": 0.2904777526855469, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.71218490600586, "metrics/particle/momentum/class_3/event/pz_bias": -1.9188110828399658, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.20344352722168, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02694566734135151, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16886818408966064, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16685618460178375, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06363222748041153, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2110695242881775, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004587155766785145, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9631083607673645, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.16594457626342773, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.19899320602417, "metrics/particle/momentum/class_4/accepted/samples": 218.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.46290814876556396, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.929442882537842, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.360203742980957, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.26051664352417, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3119533658027649, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.559459686279297, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.020408162847161293, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3624770641326904, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.47784119844436646, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.089047431945801, "metrics/particle/momentum/class_4/assigned/samples": 343.0, "metrics/particle/momentum/class_4/efficiency": 0.581333339214325, "metrics/particle/momentum/class_4/event/energy_bias": -0.7822800874710083, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.977597236633301, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1348040103912354, "metrics/particle/momentum/class_4/event/px_bias": -0.00024641305208206177, "metrics/particle/momentum/class_4/event/py_bias": -0.03600200265645981, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6636792421340942, "metrics/particle/momentum/class_4/event/pz_bias": 0.02297300100326538, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.467697620391846, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.022660724818706512, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.18936967849731445, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18009959161281586, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0050561693497002125, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22298777103424072, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0560000017285347, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9717251062393188, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.06775142252445221, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.9699981212615967, "metrics/particle/momentum/class_5/accepted/samples": 125.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09790609776973724, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.36480647325515747, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3635239005088806, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.17806445062160492, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1949685513973236, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.8590717911720276, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08805031329393387, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.855553388595581, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.16684003174304962, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.896146774291992, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.7621951103210449, "metrics/particle/momentum/class_5/event/energy_bias": -0.5062891840934753, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.37740159034729, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.786273717880249, "metrics/particle/momentum/class_5/event/px_bias": -0.14981721341609955, "metrics/particle/momentum/class_5/event/py_bias": -0.27000755071640015, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.219072937965393, "metrics/particle/momentum/class_5/event/pz_bias": -0.07434587180614471, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.03021240234375, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4582979083061218, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4637318253517151, "metrics/set_slot/angular_recall_all_dr0p10": 0.9080312848091125, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7496083974838257, "metrics/set_slot/soft_count_bias_mean": 0.3307032883167267}, "step_train_time": 5.0059654712677, "step_valid_time": 36318.66537618637, "step_total_time": 36323.67134189606, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0114425101464044, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18133580172522734, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2840, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2795, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9284079764628964, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9379128137384413, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9331361918843437, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.062087186261558784, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9136972866949984, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9230515191545574, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.91835058321012, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9284079764628964, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9983224305604288, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.179842495015621, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2266, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2114, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2093, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.925164113785558, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9329214474845543, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9290265875631729, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06707855251544571, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9159737417943107, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9236540158870256, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.919797846627115, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.925164113785558, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001949274627501, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1808073895804081, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1293, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1184, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1173, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8956127080181543, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9156999226604795, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9055449330783939, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0843000773395205, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8872919818456884, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9071925754060325, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.897131931166348, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8956127080181543} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_13000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..bf93a97418c0ae43bb82638e100e8f49bd7cae4b --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 0.6157635467980296, "train": {"Classification_binary": 0.19733834266662598, "Cardinality": 0.14061373472213745, "Classification": 0.11470811814069748, "Regression_pt": 0.19624602794647217, "Regression_eta": 0.0022512776777148247, "Regression_sin_phi": 0.0012149992398917675, "Regression_cos_phi": 0.0011156494729220867, "Regression_energy": 0.19439737498760223, "HitGrouping": 0.18196511268615723, "Auxiliary": 0.24844133853912354, "Total": 1.3124325275421143, "Total_unweighted": 0.7072717901319265}, "train_group": {}, "valid": {"Classification_binary": 0.23812214533487955, "Cardinality": 0.174838920434316, "Classification": 0.11644043525060017, "Regression_pt": 0.1870606541633606, "Regression_eta": 0.0023738016995290914, "Regression_sin_phi": 0.0012446383479982615, "Regression_cos_phi": 0.0012116663468380768, "Regression_energy": 0.1865346630414327, "HitGrouping": 0.17980311314264932, "Auxiliary": 0.2655269106229146, "Total": 1.4079264005025227, "Total_unweighted": 0.7329880041846385, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.836821436882019, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5315331816673279, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.34361693263053894, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5410979986190796, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5279249548912048, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3707769513130188, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.779086172580719, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.486042320728302, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34601181745529175, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4927425980567932, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4838772416114807, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.46528804302215576, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.855431318283081, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5025495886802673, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.31876879930496216, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5072510838508606, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.4868605434894562, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3402453064918518, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8438661694526672, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5100486874580383, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.32841169834136963, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5163787007331848, "metrics/hit_grouping/calorimeter/all/object_purity": 0.49906957149505615, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3592029809951782, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002866559661924839, "metrics/hit_grouping/calorimeter/background_recall": 0.12711410224437714, "metrics/hit_grouping/calorimeter/completeness": 0.571333110332489, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5709731578826904, "metrics/hit_grouping/calorimeter/energy_purity": 0.5634813904762268, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.39013490080833435, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8711220026016235, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5419092178344727, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.25540509819984436, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.551148533821106, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5303930044174194, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2950614392757416, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8425784111022949, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5269537568092346, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3557116389274597, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5334528684616089, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5140504837036133, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.37056654691696167, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7895894050598145, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4280552566051483, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.44976797699928284, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.42828285694122314, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.42010942101478577, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.47853827476501465, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8216819763183594, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5389351844787598, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.395357221364975, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.548568844795227, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5375981330871582, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.41153475642204285, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7974560260772705, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5938431620597839, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5316520929336548, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5934153199195862, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5740848183631897, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.44312918186187744, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.872013509273529, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.47061416506767273, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23594334721565247, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.47438105940818787, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.45085030794143677, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.29584890604019165, "metrics/hit_grouping/calorimeter/purity": 0.5628525614738464, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8515686988830566, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09023356437683105, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12406015396118164, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08821608126163483, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.046976517885923386, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.030075188726186752, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8558025360107422, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2680504620075226, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10653325170278549, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.265639066696167, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2527334690093994, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.22836962342262268, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8431195616722107, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5294824838638306, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.34328770637512207, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.536403477191925, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5192551016807556, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.37137430906295776, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6889984011650085, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4445893168449402, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4427642822265625, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4630166292190552, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6502969861030579, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7470133900642395, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.32971668243408203, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4412902593612671, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4288697838783264, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4737793803215027, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.595160722732544, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.46186500787734985, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3422609567642212, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3707861602306366, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7618362307548523, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6352179050445557, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4467943012714386, "metrics/hit_grouping/tracker/all/object_completeness": 0.38046500086784363, "metrics/hit_grouping/tracker/all/object_purity": 0.4034231901168823, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7062778472900391, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011464951559901237, "metrics/hit_grouping/tracker/background_recall": 0.10076571255922318, "metrics/hit_grouping/tracker/completeness": 0.38067498803138733, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5143952965736389, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6604231595993042, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4327546954154968, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.45027661323547363, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.47247132658958435, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6865684986114502, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6172969341278076, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45362135767936707, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3505176901817322, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3860231637954712, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7338899970054626, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6011756658554077, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.468890905380249, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2666163444519043, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2786790728569031, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7193868160247803, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.636638343334198, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4693045914173126, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3860841989517212, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.41906166076660156, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7102827429771423, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.646762490272522, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5277093648910522, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3854467570781708, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3715745806694031, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6976600885391235, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6313018202781677, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.4030246138572693, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.37226682901382446, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3888022303581238, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7026161551475525, "metrics/hit_grouping/tracker/purity": 0.3592624068260193, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.878848135471344, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03921568766236305, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.09900540113449097, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.09632501751184464, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06523378938436508, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6501244902610779, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.277232825756073, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2872869372367859, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2680225968360901, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5529168844223022, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5974482893943787, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5256561040878296, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4323321580886841, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.46367132663726807, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8180292248725891, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5096231698989868, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5469472408294678, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6175801157951355, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6122989058494568, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.039529602974653244, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.03782407566905022, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.259449005126953, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2723427414894104, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18071956932544708, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7916237711906433, "metrics/particle/class_1/pid_accuracy": 0.9570685029029846, "metrics/particle/class_2/efficiency": 0.52202969789505, "metrics/particle/class_2/pid_accuracy": 0.6298546195030212, "metrics/particle/class_3/efficiency": 0.6575651168823242, "metrics/particle/class_3/pid_accuracy": 0.9445127844810486, "metrics/particle/class_4/efficiency": 0.640280544757843, "metrics/particle/class_4/pid_accuracy": 0.3223787248134613, "metrics/particle/class_5/efficiency": 0.8026819825172424, "metrics/particle/class_5/pid_accuracy": 0.448687344789505, "metrics/particle/count/bias_mean": -2.5891926288604736, "metrics/particle/count/mae": 5.383463382720947, "metrics/particle/count/prediction_mean": 68.06575775146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09358096122741699, "metrics/particle/event/energy_response_mean": 0.9363722801208496, "metrics/particle/event/met_abs_error": 8.703404426574707, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0716109350323677, "metrics/particle/event/scalar_pt_response_mean": 0.9603164196014404, "metrics/particle/event/vector_pt_closure": 0.0744006559252739, "metrics/particle/matched/delta_eta_abs_mean": 0.018003802746534348, "metrics/particle/matched/delta_phi_abs_mean": 0.018048973754048347, "metrics/particle/matched/delta_r_mean": 0.02839324064552784, "metrics/particle/matched/energy_relative_abs_mean": 0.16563543677330017, "metrics/particle/matched/pid_accuracy": 0.9246511459350586, "metrics/particle/matched/pt_relative_abs_mean": 0.16170938313007355, "metrics/particle/matching/duplicate_fraction": 0.09171775728464127, "metrics/particle/matching/efficiency": 0.7118478417396545, "metrics/particle/matching/f1": 0.7251343131065369, "metrics/particle/matching/purity": 0.7389262318611145, "metrics/particle/matching/target_coverage_dr0p05": 0.6413301825523376, "metrics/particle/matching/target_coverage_dr0p10": 0.7683780789375305, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5042570233345032, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7654018402099609, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.027188921347260475, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.16088475286960602, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15614449977874756, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.006276988424360752, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19616171717643738, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.013628182001411915, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.9216679334640503, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.06921786814928055, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.600146770477295, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7778.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3284135162830353, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.358318567276001, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.6607404947280884, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.5007476806640625, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.17270445823669434, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.429692268371582, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02267937734723091, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2617472410202026, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.14994056522846222, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.5184524059295654, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9965.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7220571637153625, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.4416191577911377, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.495857238769531, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.695663928985596, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.6152688264846802, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.1169447898864746, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.034770011901855, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.2582194805145264, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.323750495910645, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027866823598742485, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16416800022125244, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16545765101909637, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.00453920429572463, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20534376800060272, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0403422974050045, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.598528504371643, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.03294404596090317, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.082646369934082, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1636.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5474045872688293, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.28703773021698, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.5596044063568115, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.3776230812072754, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22423706948757172, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 32.83323669433594, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06103494018316269, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.640289306640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.02033449336886406, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.322325706481934, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2261.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5931834578514099, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.464344024658203, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.143505096435547, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.425366997718811, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03192652016878128, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.17270934581756592, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.971431732177734, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.1985070705413818, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.334152221679688, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.028761884197592735, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1676749885082245, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.16275745630264282, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.0007655400549992919, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.20238597691059113, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001446393085643649, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.32352909445762634, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0481472946703434, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0544289350509644, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16593.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.34737449884414673, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.7994898557662964, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.4784722328186035, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.298659324645996, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.20189443230628967, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.774850845336914, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005914528388530016, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.6012619137763977, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0713142678141594, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.170928955078125, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22487.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7130946516990662, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -4.5701003074646, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.101183891296387, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.213507652282715, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.9735628366470337, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -3.0574347972869873, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.394502639770508, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.3533318042755127, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.572224617004395, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.028235148638486862, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.16542361676692963, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.16094954311847687, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0016743929591029882, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.20073376595973969, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007536432705819607, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5826218128204346, "metrics/particle/momentum/all/accepted/pz_bias": 0.053492557257413864, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.9441829919815063, "metrics/particle/momentum/all/accepted/samples": 26007.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.35496020317077637, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.6394654512405396, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.6663483381271362, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.49208402633667, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.19497019052505493, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.560464859008789, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014317402616143227, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.923676609992981, "metrics/particle/momentum/all/assigned/pz_bias": 0.08791591972112656, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.334374189376831, "metrics/particle/momentum/all/assigned/samples": 34713.0, "metrics/particle/momentum/all/efficiency": 0.7067311406135559, "metrics/particle/momentum/all/event/energy_bias": -16.476062774658203, "metrics/particle/momentum/all/event/p3_error_mean": 23.828128814697266, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.545215606689453, "metrics/particle/momentum/all/event/px_bias": 2.620758295059204, "metrics/particle/momentum/all/event/py_bias": -4.3470892906188965, "metrics/particle/momentum/all/event/pz_abs_mean": 17.129352569580078, "metrics/particle/momentum/all/event/pz_bias": 5.81005859375, "metrics/particle/momentum/all/event/pz_residual_rms": 27.174604415893555, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03187818080186844, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.16536791622638702, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1592870056629181, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -6.891287921462208e-05, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19829553365707397, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007610022090375423, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6295220255851746, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.08711718767881393, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.9295415878295898, "metrics/particle/momentum/class_1/accepted/samples": 13929.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.23413027822971344, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.677349865436554, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.7828980684280396, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.6051321625709534, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.13407428562641144, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 7.419296741485596, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.011114360764622688, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.8209981918334961, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.108442023396492, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.907062530517578, "metrics/particle/momentum/class_1/assigned/samples": 17095.0, "metrics/particle/momentum/class_1/efficiency": 0.785130500793457, "metrics/particle/momentum/class_1/event/energy_bias": 1.9202053546905518, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.515338897705078, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.021892547607422, "metrics/particle/momentum/class_1/event/px_bias": 2.168715476989746, "metrics/particle/momentum/class_1/event/py_bias": -2.094245433807373, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.641088485717773, "metrics/particle/momentum/class_1/event/pz_bias": 3.8937056064605713, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.580970764160156, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03201330080628395, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2048691064119339, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20596490800380707, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06437371671199799, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24660822749137878, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.033832140266895294, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6841446161270142, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0015953599940985441, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.976512908935547, "metrics/particle/momentum/class_2/accepted/samples": 1537.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5319797396659851, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5836552977561951, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.829069197177887, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4732542335987091, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.27498963475227356, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.33652400970459, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05143094062805176, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4429874420166016, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.060312669724226, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.995665550231934, "metrics/particle/momentum/class_2/assigned/samples": 2411.0, "metrics/particle/momentum/class_2/efficiency": 0.5147354602813721, "metrics/particle/momentum/class_2/event/energy_bias": -16.74612045288086, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.961137771606445, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.295134544372559, "metrics/particle/momentum/class_2/event/px_bias": -0.046049684286117554, "metrics/particle/momentum/class_2/event/py_bias": -0.5065906643867493, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.14736557006836, "metrics/particle/momentum/class_2/event/pz_bias": -2.094334602355957, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.609943389892578, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022680120542645454, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15915615856647491, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15628166496753693, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01408444531261921, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19604843854904175, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0028490028344094753, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3270222246646881, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004072204232215881, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3134424686431885, "metrics/particle/momentum/class_3/accepted/samples": 10179.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.465593546628952, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.7919771671295166, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.7002179622650146, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.5592424869537354, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2505105435848236, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.494047164916992, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011232131160795689, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7691155076026917, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04512711614370346, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.148531198501587, "metrics/particle/momentum/class_3/assigned/samples": 14690.0, "metrics/particle/momentum/class_3/efficiency": 0.6553145051002502, "metrics/particle/momentum/class_3/event/energy_bias": 0.17260229587554932, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.152796745300293, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.0460205078125, "metrics/particle/momentum/class_3/event/px_bias": 0.17134474217891693, "metrics/particle/momentum/class_3/event/py_bias": -1.1576545238494873, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.419025421142578, "metrics/particle/momentum/class_3/event/pz_bias": 3.0147860050201416, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.991018295288086, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.028839558362960815, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17517712712287903, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16217122972011566, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.010169703513383865, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20635513961315155, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.008438818156719208, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8875128626823425, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.2240471988916397, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.0268585681915283, "metrics/particle/momentum/class_4/accepted/samples": 237.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.49574825167655945, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 7.914992332458496, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 7.5517497062683105, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 7.413763046264648, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.28133705258369446, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 32.96341323852539, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.016713092103600502, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.140899419784546, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.3068237602710724, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.7613983154296875, "metrics/particle/momentum/class_4/assigned/samples": 359.0, "metrics/particle/momentum/class_4/efficiency": 0.6320000290870667, "metrics/particle/momentum/class_4/event/energy_bias": -1.2295323610305786, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.645453929901123, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9523699283599854, "metrics/particle/momentum/class_4/event/px_bias": 0.16427963972091675, "metrics/particle/momentum/class_4/event/py_bias": -0.11056941747665405, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.445268154144287, "metrics/particle/momentum/class_4/event/pz_bias": 0.29258254170417786, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.302658557891846, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.027039753273129463, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17848744988441467, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17050234973430634, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.01862066611647606, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2083834409713745, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0560000017285347, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.0480117797851562, "metrics/particle/momentum/class_5/accepted/pz_bias": 1.3482365608215332, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.37045431137085, "metrics/particle/momentum/class_5/accepted/samples": 125.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.12106979638338089, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4344986081123352, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.5325287580490112, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.33005669713020325, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.20253165066242218, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.3710060119628906, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.07594936341047287, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.725870370864868, "metrics/particle/momentum/class_5/assigned/pz_bias": 1.7691638469696045, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.382848262786865, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.7621951103210449, "metrics/particle/momentum/class_5/event/energy_bias": -0.5932179093360901, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.440700054168701, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.8197300434112549, "metrics/particle/momentum/class_5/event/px_bias": 0.16246800124645233, "metrics/particle/momentum/class_5/event/py_bias": -0.47802871465682983, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.3037610054016113, "metrics/particle/momentum/class_5/event/pz_bias": 0.7033183574676514, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.464679718017578, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4604679346084595, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.461899071931839, "metrics/set_slot/angular_recall_all_dr0p10": 0.9160754084587097, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7638169527053833, "metrics/set_slot/soft_count_bias_mean": 1.5945253372192383}, "step_train_time": 5.046962022781372, "step_valid_time": 39415.61067056656, "step_total_time": 39420.65763282776, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9809686159653299, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17978808881072905, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3057, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2848, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2815, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9310232101994116, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9316323192672554, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9313276651406148, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06836768073274452, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9202353710362864, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9208374223094538, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.92053629823414, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9310232101994116, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9613068747390787, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1698226714783908, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2249, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2110, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2100, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9234135667396062, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.938194753223655, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9307454786060874, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.061805246776345045, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9190371991247265, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9337483325922632, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9263343625937362, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9234135667396062, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9622677554251646, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1784221478365321, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1309, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1193, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1181, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9024205748865356, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9113827349121467, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9068795134929685, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08861726508785332, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8933434190620272, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9022154316271963, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8977575066514634, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9024205748865356} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_14000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..0452dd9d46fbc0b93dce67d3e99374fb949f9c6a --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 0.6631299734748011, "train": {"Classification_binary": 0.1852581650018692, "Cardinality": 0.15733274817466736, "Classification": 0.0897558182477951, "Regression_pt": 0.18138761818408966, "Regression_eta": 0.002074127085506916, "Regression_sin_phi": 0.0009549399837851524, "Regression_cos_phi": 0.000960814650170505, "Regression_energy": 0.1780097484588623, "HitGrouping": 0.17301659286022186, "Auxiliary": 0.23619651794433594, "Total": 1.2239187955856323, "Total_unweighted": 0.6384012316120788}, "train_group": {}, "valid": {"Classification_binary": 0.22187515099843344, "Cardinality": 0.17237542072931925, "Classification": 0.11785520116488139, "Regression_pt": 0.17166986068089804, "Regression_eta": 0.002214570219318072, "Regression_sin_phi": 0.0011334504621724288, "Regression_cos_phi": 0.001099404413253069, "Regression_energy": 0.17004017035166422, "HitGrouping": 0.17551881074905396, "Auxiliary": 0.25679608186086017, "Total": 1.3363065719604492, "Total_unweighted": 0.6858878082906207, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.844258189201355, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5386202335357666, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3375603258609772, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5484465956687927, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5332498550415039, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3619759678840637, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7847527265548706, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.4959867000579834, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.32016247510910034, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5010892748832703, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.4954027533531189, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4464549422264099, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8608160614967346, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.510469377040863, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.31700995564460754, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5153839588165283, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.49386000633239746, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3263596296310425, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8498944044113159, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5178734064102173, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3234805166721344, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5242906212806702, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5059130787849426, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3464736342430115, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003026919672265649, "metrics/hit_grouping/calorimeter/background_recall": 0.13128593564033508, "metrics/hit_grouping/calorimeter/completeness": 0.584185779094696, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5851272940635681, "metrics/hit_grouping/calorimeter/energy_purity": 0.5774955749511719, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3873162269592285, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8764417767524719, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5491461157798767, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2532999515533447, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5582245588302612, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5344191193580627, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2814633548259735, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8500340580940247, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5344164967536926, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3410870432853699, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5411770939826965, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5234163403511047, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.35548135638237, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7956244349479675, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.43744319677352905, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4488399028778076, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.43808823823928833, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.430156409740448, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.46995359659194946, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8271915316581726, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5473719239234924, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3897642493247986, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5572044253349304, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5477224588394165, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.40179216861724854, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8094235062599182, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6040502786636353, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5285640954971313, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6047179102897644, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5855687260627747, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4137931168079376, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8778631091117859, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4775310754776001, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23146729171276093, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.48115119338035583, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4537959694862366, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2820528447628021, "metrics/hit_grouping/calorimeter/purity": 0.5756070017814636, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8253334164619446, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08291348814964294, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1184210553765297, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08349892497062683, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05774780362844467, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02255639061331749, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8634178042411804, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2751772999763489, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11889346688985825, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.27356481552124023, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.25157588720321655, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.21306650340557098, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8495908975601196, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5375903844833374, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3374682366847992, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5445204377174377, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.526451051235199, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3586939871311188, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6991678476333618, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4395190477371216, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.45241788029670715, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4716257154941559, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6391423940658569, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7500226497650146, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3230862021446228, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.45124587416648865, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.43917128443717957, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.47920432686805725, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6094847321510315, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45913082361221313, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3507830798625946, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3787764012813568, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.747157871723175, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.647429347038269, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44305431842803955, "metrics/hit_grouping/tracker/all/object_completeness": 0.38944077491760254, "metrics/hit_grouping/tracker/all/object_purity": 0.41177430748939514, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.69416743516922, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009323379956185818, "metrics/hit_grouping/tracker/background_recall": 0.08663561940193176, "metrics/hit_grouping/tracker/completeness": 0.390750914812088, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5215522646903992, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6741610765457153, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4269682765007019, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4603557586669922, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.47900980710983276, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6741188764572144, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6255627274513245, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4557996094226837, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3592681884765625, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3934965133666992, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7202759385108948, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6141670942306519, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4634806215763092, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2735463082790375, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.29163089394569397, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7094680070877075, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6505825519561768, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.46998387575149536, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.39753398299217224, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.42815935611724854, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6967818737030029, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6690481305122375, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5326354503631592, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40479782223701477, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3895567059516907, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6613300442695618, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6394543647766113, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39198586344718933, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.37616634368896484, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3944562077522278, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6966552734375, "metrics/hit_grouping/tracker/purity": 0.3679744005203247, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8856604099273682, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.029411764815449715, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.0899733230471611, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.09575437009334564, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06410256773233414, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6581016182899475, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2705214321613312, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2869248688220978, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2753235995769501, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5513681173324585, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6109198927879333, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5231380462646484, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4449922740459442, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.47346028685569763, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8031447529792786, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5213666558265686, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5500698685646057, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6211308836936951, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6080220937728882, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.02778606116771698, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0335472971200943, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.431726455688477, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2673512101173401, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17572802305221558, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7832704186439514, "metrics/particle/class_1/pid_accuracy": 0.9560993313789368, "metrics/particle/class_2/efficiency": 0.4965454638004303, "metrics/particle/class_2/pid_accuracy": 0.6548813581466675, "metrics/particle/class_3/efficiency": 0.6760159730911255, "metrics/particle/class_3/pid_accuracy": 0.9566988945007324, "metrics/particle/class_4/efficiency": 0.6292585134506226, "metrics/particle/class_4/pid_accuracy": 0.3885350227355957, "metrics/particle/class_5/efficiency": 0.8007662892341614, "metrics/particle/class_5/pid_accuracy": 0.49760764837265015, "metrics/particle/count/bias_mean": -4.892578125, "metrics/particle/count/mae": 6.340494632720947, "metrics/particle/count/prediction_mean": 65.76236724853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08295518904924393, "metrics/particle/event/energy_response_mean": 0.9955136775970459, "metrics/particle/event/met_abs_error": 8.450907707214355, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0674925148487091, "metrics/particle/event/scalar_pt_response_mean": 1.0078359842300415, "metrics/particle/event/vector_pt_closure": 0.07142793387174606, "metrics/particle/matched/delta_eta_abs_mean": 0.017254412174224854, "metrics/particle/matched/delta_phi_abs_mean": 0.01602085866034031, "metrics/particle/matched/delta_r_mean": 0.02630869299173355, "metrics/particle/matched/energy_relative_abs_mean": 0.1530701220035553, "metrics/particle/matched/pid_accuracy": 0.9322035908699036, "metrics/particle/matched/pt_relative_abs_mean": 0.1534322053194046, "metrics/particle/matching/duplicate_fraction": 0.08126837760210037, "metrics/particle/matching/efficiency": 0.7134050726890564, "metrics/particle/matching/f1": 0.7389912009239197, "metrics/particle/matching/purity": 0.7664808630943298, "metrics/particle/matching/target_coverage_dr0p05": 0.6525164246559143, "metrics/particle/matching/target_coverage_dr0p10": 0.76952064037323, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5225199460983276, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7608867883682251, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025274405255913734, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15029475092887878, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15195313096046448, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.023958299309015274, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19241228699684143, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008124838583171368, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7899072766304016, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.10181093961000443, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.006727933883667, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7754.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2848832905292511, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2475415468215942, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4704358577728271, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3280150890350342, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16200944781303406, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.167875289916992, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017053626477718353, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1464473009109497, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06745675951242447, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.1080143451690674, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9734.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7198292016983032, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.5034263134002686, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.555900573730469, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.409524440765381, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2796160876750946, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.626478910446167, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.172836303710938, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.1995058059692383, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.33099365234375, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02597939223051071, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15947741270065308, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16213954985141754, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02690059319138527, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20387467741966248, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.030476190149784088, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.488937258720398, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.3690513074398041, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.590514898300171, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1575.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4691638648509979, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2483811378479004, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.7939627170562744, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.6384615898132324, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23242995142936707, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 54.69024658203125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0629306361079216, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6385061740875244, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.4598316252231598, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.386837005615234, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2177.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5710659623146057, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.3921895027160645, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.852811813354492, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3078763484954834, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.11182454228401184, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.059860341250896454, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.673233985900879, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.070006847381592, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.280746459960938, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026782330125570297, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15418046712875366, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15436868369579315, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.004649686627089977, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19441045820713043, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0010120252845808864, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.28149962425231934, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.03402237966656685, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8709347248077393, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 16798.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.31802672147750854, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.7222908735275269, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.3330997228622437, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.1604864597320557, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.18359516561031342, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.907794952392578, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006681112106889486, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5636798143386841, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.02487373538315296, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.1589415073394775, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22152.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7219046950340271, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.8417792320251465, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.577393531799316, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.646982192993164, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.603508472442627, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.3949732780456543, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.602320194244385, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5962626338005066, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.065535545349121, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026286404579877853, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15334658324718475, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15412025153636932, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011721457354724407, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19440490007400513, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004899146500974894, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5051727294921875, "metrics/particle/momentum/all/accepted/pz_bias": -0.07433711737394333, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5684256553649902, "metrics/particle/momentum/all/accepted/samples": 26127.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3182148039340973, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5563362836837769, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.5296217203140259, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.3667300939559937, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1805478036403656, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.418010711669922, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013240172527730465, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8628183007240295, "metrics/particle/momentum/all/assigned/pz_bias": -0.06484103947877884, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2073404788970947, "metrics/particle/momentum/all/assigned/samples": 34063.0, "metrics/particle/momentum/all/efficiency": 0.7099921107292175, "metrics/particle/momentum/all/event/energy_bias": 1.9530162811279297, "metrics/particle/momentum/all/event/p3_error_mean": 22.9617919921875, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.594856262207031, "metrics/particle/momentum/all/event/px_bias": 1.994949221611023, "metrics/particle/momentum/all/event/py_bias": 3.0813121795654297, "metrics/particle/momentum/all/event/pz_abs_mean": 16.00598907470703, "metrics/particle/momentum/all/event/pz_bias": -4.865775108337402, "metrics/particle/momentum/all/event/pz_residual_rms": 26.836271286010742, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.029208259657025337, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14707732200622559, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14811338484287262, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.010332236997783184, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18813017010688782, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004201071802526712, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5355404019355774, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.06528521329164505, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.475548267364502, "metrics/particle/momentum/class_1/accepted/samples": 13806.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.2155630737543106, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.6747722029685974, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.7353105545043945, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.5738720297813416, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.13000832498073578, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.326643943786621, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00939246267080307, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7643523216247559, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.060497112572193146, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.8656606674194336, "metrics/particle/momentum/class_1/assigned/samples": 16822.0, "metrics/particle/momentum/class_1/efficiency": 0.7781974077224731, "metrics/particle/momentum/class_1/event/energy_bias": 8.298659324645996, "metrics/particle/momentum/class_1/event/p3_error_mean": 19.02895736694336, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.676229476928711, "metrics/particle/momentum/class_1/event/px_bias": 1.6333034038543701, "metrics/particle/momentum/class_1/event/py_bias": 1.5269395112991333, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.56795883178711, "metrics/particle/momentum/class_1/event/pz_bias": -2.235546112060547, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.48193359375, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.031245552003383636, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19519281387329102, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19759105145931244, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.0006432879599742591, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2365449219942093, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02645861543715, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.513283133506775, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.30107739567756653, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.486532211303711, "metrics/particle/momentum/class_2/accepted/samples": 1474.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.48742949962615967, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6173611879348755, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.790416955947876, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5075933933258057, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.26570048928260803, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.103124141693115, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05138339847326279, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3299641609191895, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1514461636543274, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.505368709564209, "metrics/particle/momentum/class_2/assigned/samples": 2277.0, "metrics/particle/momentum/class_2/efficiency": 0.49363696575164795, "metrics/particle/momentum/class_2/event/energy_bias": -8.58048152923584, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.269378662109375, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.502742767333984, "metrics/particle/momentum/class_2/event/px_bias": 0.5206649899482727, "metrics/particle/momentum/class_2/event/py_bias": 0.2613410949707031, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.252253532409668, "metrics/particle/momentum/class_2/event/pz_bias": -2.3832626342773438, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.86798667907715, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021781789138913155, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15524917840957642, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15557223558425903, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.014833810739219189, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19544905424118042, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018121125176548958, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2966504991054535, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.06228777766227722, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0747607946395874, "metrics/particle/momentum/class_3/accepted/samples": 10485.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4078104496002197, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.6232755184173584, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.5286097526550293, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.3820152282714844, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.223775252699852, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 26.645401000976562, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010517575778067112, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7139186263084412, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.05737169459462166, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.026315450668335, "metrics/particle/momentum/class_3/assigned/samples": 14452.0, "metrics/particle/momentum/class_3/efficiency": 0.6750144958496094, "metrics/particle/momentum/class_3/event/energy_bias": 2.4918630123138428, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.206331253051758, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.4484758377075195, "metrics/particle/momentum/class_3/event/px_bias": -0.11180299520492554, "metrics/particle/momentum/class_3/event/py_bias": 1.6644771099090576, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.773168563842773, "metrics/particle/momentum/class_3/event/pz_bias": 0.26648709177970886, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.233534812927246, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026989877223968506, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1627662032842636, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15564864873886108, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04666130244731903, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19914378225803375, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004310344811528921, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.733268678188324, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.09996066987514496, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.756162405014038, "metrics/particle/momentum/class_4/accepted/samples": 232.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5573322772979736, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.492325305938721, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.7673895359039307, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.644585371017456, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2750716209411621, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.978575706481934, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.022922636941075325, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3009182214736938, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07279795408248901, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.8553152084350586, "metrics/particle/momentum/class_4/assigned/samples": 349.0, "metrics/particle/momentum/class_4/efficiency": 0.6186666488647461, "metrics/particle/momentum/class_4/event/energy_bias": -0.2745816111564636, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.5706632137298584, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9334008693695068, "metrics/particle/momentum/class_4/event/px_bias": 0.0073070228099823, "metrics/particle/momentum/class_4/event/py_bias": -0.2110326886177063, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.331648349761963, "metrics/particle/momentum/class_4/event/pz_bias": -0.24071627855300903, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.033268928527832, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.021815424785017967, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1744067668914795, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1793193370103836, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.028511397540569305, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.22502265870571136, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.08461538702249527, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.2607603073120117, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.25235694646835327, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 5.164732456207275, "metrics/particle/momentum/class_5/accepted/samples": 130.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09255418181419373, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4867067039012909, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.4665167033672333, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.29826992750167847, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.17177914083003998, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.922351598739624, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09815950691699982, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.793541193008423, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.26028192043304443, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.915441989898682, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.792682945728302, "metrics/particle/momentum/class_5/event/energy_bias": 0.017557553946971893, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.401735544204712, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.8002219200134277, "metrics/particle/momentum/class_5/event/px_bias": -0.05452325940132141, "metrics/particle/momentum/class_5/event/py_bias": -0.16041254997253418, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2741156816482544, "metrics/particle/momentum/class_5/event/pz_bias": -0.27273690700531006, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.093629360198975, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.43396931886672974, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4363180100917816, "metrics/set_slot/angular_recall_all_dr0p10": 0.9258058071136475, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7675027251243591, "metrics/set_slot/soft_count_bias_mean": -1.3255361318588257}, "step_train_time": 5.2714807987213135, "step_valid_time": 42504.07995009422, "step_total_time": 42509.35143136978, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0149597257833802, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.178275791282958, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3039, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2860, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2823, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9349460608041844, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9410990457387298, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9380124631026566, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05890095426127016, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9228506047728016, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.928923988153998, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9258773368317481, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9349460608041844, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0060218651102795, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17019272747601955, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2257, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2130, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2117, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9321663019693655, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9437306158617634, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9379128137384413, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056269384138236594, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9264770240700219, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9379707576428888, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9321884632320564, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9321663019693655, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0132100299954145, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17870497674657393, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1300, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1201, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1188, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9084720121028744, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9238461538461539, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.916094584286804, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07615384615384616, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8986384266263238, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9138461538461539, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9061784897025171, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9084720121028744} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_15000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..2b1b339a56881762c97f5568abe8aad32a449dbf --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 0.7104964001515726, "train": {"Classification_binary": 0.2121286392211914, "Cardinality": 0.1516219526529312, "Classification": 0.10121574252843857, "Regression_pt": 0.16641715168952942, "Regression_eta": 0.00203088135458529, "Regression_sin_phi": 0.001173718716017902, "Regression_cos_phi": 0.0010226790327578783, "Regression_energy": 0.16547012329101562, "HitGrouping": 0.17549358308315277, "Auxiliary": 0.24503056704998016, "Total": 1.2700707912445068, "Total_unweighted": 0.6494589358335361}, "train_group": {}, "valid": {"Classification_binary": 0.2180169423421224, "Cardinality": 0.15717926621437073, "Classification": 0.10876444975535075, "Regression_pt": 0.17135103543599448, "Regression_eta": 0.002063209656625986, "Regression_sin_phi": 0.0011227827053517103, "Regression_cos_phi": 0.0011221617460250854, "Regression_energy": 0.1700924038887024, "HitGrouping": 0.17671539386113486, "Auxiliary": 0.2511189778645833, "Total": 1.303857962290446, "Total_unweighted": 0.6725329855301728, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.836967408657074, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.534361720085144, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.32213494181632996, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5447155833244324, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5447197556495667, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.38525599241256714, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7835603356361389, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.48595094680786133, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3305022120475769, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.49429428577423096, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.49507859349250793, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.46418020129203796, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8552497029304504, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5170897245407104, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3125665485858917, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5229896306991577, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.512696385383606, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.34140244126319885, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8441451787948608, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5199047923088074, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.31685778498649597, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.52734375, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5210301876068115, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.36422017216682434, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002824726514518261, "metrics/hit_grouping/calorimeter/background_recall": 0.12570470571517944, "metrics/hit_grouping/calorimeter/completeness": 0.5887452363967896, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5928653478622437, "metrics/hit_grouping/calorimeter/energy_purity": 0.5850598216056824, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3713253438472748, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8733639717102051, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5509039163589478, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.244367316365242, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.561204731464386, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.547884464263916, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2903960049152374, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8410367369651794, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5404136776924133, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.33486872911453247, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5476890206336975, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5449334979057312, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.38046982884407043, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7877001762390137, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.43603718280792236, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.44651973247528076, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4378053545951843, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4421941041946411, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.49837586283683777, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8196485042572021, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5491703152656555, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3808034658432007, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5604725480079651, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5695951581001282, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.42747172713279724, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8302009105682373, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6215916872024536, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5404014587402344, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6205617785453796, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5708836913108826, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.37776634097099304, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8707823753356934, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4779522716999054, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2250291258096695, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.482461154460907, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.46557581424713135, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.29811760783195496, "metrics/hit_grouping/calorimeter/purity": 0.5799818634986877, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8322234153747559, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08767659962177277, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07894736528396606, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08836152404546738, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.058465730398893356, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8505879044532776, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2822224795818329, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11241906881332397, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2814548909664154, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.27104315161705017, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.22012948989868164, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8440041542053223, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5393163561820984, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.33137309551239014, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5472922921180725, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5415763854980469, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3772244155406952, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6978787779808044, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4406779706478119, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.45246127247810364, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4751582741737366, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6401564478874207, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7591602802276611, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.314647376537323, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4505442678928375, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4424487054347992, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.46232670545578003, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6120709776878357, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4595625400543213, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3604026734828949, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.39265134930610657, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7437760829925537, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6493083834648132, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44305431842803955, "metrics/hit_grouping/tracker/all/object_completeness": 0.3953548073768616, "metrics/hit_grouping/tracker/all/object_purity": 0.4216878116130829, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.691139817237854, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00852922536432743, "metrics/hit_grouping/tracker/background_recall": 0.08450426161289215, "metrics/hit_grouping/tracker/completeness": 0.39484256505966187, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5138871073722839, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6784244775772095, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4226722717285156, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.46880120038986206, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4899457097053528, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6630720496177673, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6265109181404114, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45961153507232666, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3638497591018677, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4050022065639496, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7233617901802063, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.612066388130188, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.46853020787239075, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2755768597126007, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2978593409061432, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7168620228767395, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6500239372253418, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4724462926387787, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4027087986469269, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.44303837418556213, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6951685547828674, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6667138934135437, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5449507236480713, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40144407749176025, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3695954978466034, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.674876868724823, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6452578902244568, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38657689094543457, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3847026824951172, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.40326988697052, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6888177394866943, "metrics/hit_grouping/tracker/purity": 0.3715297281742096, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8864217400550842, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03242835775017738, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.08669712394475937, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.08714795857667923, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06523378938436508, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6651363372802734, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.27155396342277527, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.2869192361831665, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.2704550325870514, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5622096061706543, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6124058961868286, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5225784778594971, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.45291173458099365, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.487724244594574, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7979967594146729, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.530852735042572, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5491560101509094, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6192964911460876, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6385859251022339, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.018300028517842293, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06411120295524597, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.342695236206055, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2687908113002777, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1771676242351532, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.7952283024787903, "metrics/particle/class_1/pid_accuracy": 0.9589179158210754, "metrics/particle/class_2/efficiency": 0.49575263261795044, "metrics/particle/class_2/pid_accuracy": 0.6545579433441162, "metrics/particle/class_3/efficiency": 0.716044008731842, "metrics/particle/class_3/pid_accuracy": 0.9580509066581726, "metrics/particle/class_4/efficiency": 0.6362725496292114, "metrics/particle/class_4/pid_accuracy": 0.3606299161911011, "metrics/particle/class_5/efficiency": 0.8180076479911804, "metrics/particle/class_5/pid_accuracy": 0.48009368777275085, "metrics/particle/count/bias_mean": -3.869140625, "metrics/particle/count/mae": 5.656900882720947, "metrics/particle/count/prediction_mean": 66.78580474853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.10008666664361954, "metrics/particle/event/energy_response_mean": 0.9179792404174805, "metrics/particle/event/met_abs_error": 7.467702865600586, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07813619822263718, "metrics/particle/event/scalar_pt_response_mean": 0.9429425597190857, "metrics/particle/event/vector_pt_closure": 0.06616737693548203, "metrics/particle/matched/delta_eta_abs_mean": 0.0152963912114501, "metrics/particle/matched/delta_phi_abs_mean": 0.016983482986688614, "metrics/particle/matched/delta_r_mean": 0.025636296719312668, "metrics/particle/matched/energy_relative_abs_mean": 0.15750618278980255, "metrics/particle/matched/pid_accuracy": 0.9345725178718567, "metrics/particle/matched/pt_relative_abs_mean": 0.15649676322937012, "metrics/particle/matching/duplicate_fraction": 0.07257537543773651, "metrics/particle/matching/efficiency": 0.7361369729042053, "metrics/particle/matching/f1": 0.7568601965904236, "metrics/particle/matching/purity": 0.7787840366363525, "metrics/particle/matching/target_coverage_dr0p05": 0.6646149158477783, "metrics/particle/matching/target_coverage_dr0p10": 0.783554196357727, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5309787392616272, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7849639654159546, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02363981306552887, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1511753350496292, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1515757441520691, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.010961330495774746, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19046176970005035, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009509319439530373, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8104093670845032, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.053756728768348694, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.1607632637023926, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7887.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29005157947540283, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9051030874252319, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1096080541610718, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9263091683387756, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14665444195270538, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.772500038146973, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015072817914187908, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0700067281723022, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0641523003578186, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9005095958709717, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9819.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7321760058403015, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -5.30574893951416, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.364609718322754, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.8563055992126465, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.08449442684650421, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.1593594253063202, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.055400848388672, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.4671423137187958, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.301397323608398, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02491290308535099, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1616269201040268, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1615108996629715, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.012685151770710945, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2004241794347763, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.028054863214492798, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4765191078186035, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05141311138868332, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6180262565612793, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1604.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5027998089790344, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9689379334449768, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.0244202613830566, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.831331968307495, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2103559821844101, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 28.730791091918945, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05316689610481262, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.423635721206665, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.237654447555542, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.635708332061768, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2163.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5815808773040771, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -9.56130599975586, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.3641996383667, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3503602743148804, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.029627017676830292, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.1320955902338028, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.179080963134766, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.3984101414680481, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.36690902709961, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.026884617283940315, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.16127492487430573, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15893401205539703, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.01119311898946762, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19835638999938965, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00108540419023484, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.277915358543396, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01794261671602726, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9367981553077698, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17505.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.287302166223526, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1777184009552002, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9496641755104065, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7461221218109131, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.16175419092178345, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.0933332443237305, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004421222023665905, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4742993712425232, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02021779678761959, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6962343454360962, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22392.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7522884607315063, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -7.225187301635742, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.086647033691406, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.317325592041016, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.702330470085144, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.13293419778347015, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.9625444412231445, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.3245741128921509, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.63725471496582, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025819482281804085, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15834519267082214, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1569373905658722, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.011214052326977253, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1962077021598816, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005148910917341709, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5047023296356201, "metrics/particle/momentum/all/accepted/pz_bias": 0.030394556000828743, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6464687585830688, "metrics/particle/momentum/all/accepted/samples": 26996.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3016478419303894, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0867077112197876, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.12590754032135, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9288056492805481, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1604992151260376, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.905134201049805, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010531214997172356, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.767127275466919, "metrics/particle/momentum/all/assigned/pz_bias": 0.016541043296456337, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8189122676849365, "metrics/particle/momentum/all/assigned/samples": 34374.0, "metrics/particle/momentum/all/efficiency": 0.7336068749427795, "metrics/particle/momentum/all/event/energy_bias": -22.092243194580078, "metrics/particle/momentum/all/event/p3_error_mean": 20.335918426513672, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.77927017211914, "metrics/particle/momentum/all/event/px_bias": 1.816451907157898, "metrics/particle/momentum/all/event/py_bias": 0.16019806265830994, "metrics/particle/momentum/all/event/pz_abs_mean": 14.259777069091797, "metrics/particle/momentum/all/event/pz_bias": 0.39330625534057617, "metrics/particle/momentum/all/event/pz_residual_rms": 25.25054168701172, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02775450609624386, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15115338563919067, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14918959140777588, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 5.5001379223540425e-05, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18781183660030365, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004621400497853756, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5406711101531982, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.03481270372867584, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.61417818069458, "metrics/particle/momentum/class_1/accepted/samples": 14065.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.21367190778255463, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5238374471664429, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5864905714988708, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.403860479593277, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.11833934485912323, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.793961048126221, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00827961415052414, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.725012481212616, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0005948369507677853, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.5361709594726562, "metrics/particle/momentum/class_1/assigned/samples": 16909.0, "metrics/particle/momentum/class_1/efficiency": 0.7927963733673096, "metrics/particle/momentum/class_1/event/energy_bias": -3.0409884452819824, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.174312591552734, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.59869384765625, "metrics/particle/momentum/class_1/event/px_bias": 2.7704758644104004, "metrics/particle/momentum/class_1/event/py_bias": 0.4380314350128174, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.082789421081543, "metrics/particle/momentum/class_1/event/pz_bias": -0.1624441146850586, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.22515106201172, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.031013954430818558, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19682806730270386, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20031951367855072, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02337542176246643, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24006158113479614, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03170227259397507, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5740610361099243, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.026989253237843513, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.608607292175293, "metrics/particle/momentum/class_2/accepted/samples": 1451.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5559707283973694, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5257615447044373, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7046589851379395, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3443697392940521, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2736383378505707, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.3566226959228516, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04923747107386589, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.22590708732605, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.04161926358938217, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.426401138305664, "metrics/particle/momentum/class_2/assigned/samples": 2295.0, "metrics/particle/momentum/class_2/efficiency": 0.4859343469142914, "metrics/particle/momentum/class_2/event/energy_bias": -15.557011604309082, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.101699829101562, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.832982063293457, "metrics/particle/momentum/class_2/event/px_bias": -0.7051219940185547, "metrics/particle/momentum/class_2/event/py_bias": -0.2889120876789093, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.486647605895996, "metrics/particle/momentum/class_2/event/pz_bias": -0.6278783082962036, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.364727020263672, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022741490975022316, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16186819970607758, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16077423095703125, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.025015175342559814, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19973115622997284, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016200161771848798, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.29212069511413574, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.02795237861573696, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1162331104278564, "metrics/particle/momentum/class_3/accepted/samples": 11111.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.36199572682380676, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7584164142608643, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7733412981033325, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5832592248916626, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1885402500629425, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.635223388671875, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006480218376964331, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5657110214233398, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.039142727851867676, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.496086359024048, "metrics/particle/momentum/class_3/assigned/samples": 14660.0, "metrics/particle/momentum/class_3/efficiency": 0.7153157591819763, "metrics/particle/momentum/class_3/event/energy_bias": -2.49198317527771, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.990726470947266, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.027425765991211, "metrics/particle/momentum/class_3/event/px_bias": -0.41570714116096497, "metrics/particle/momentum/class_3/event/py_bias": 0.09888669103384018, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.851808547973633, "metrics/particle/momentum/class_3/event/pz_bias": 1.0305484533309937, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.31521224975586, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025912480428814888, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16102685034275055, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15135180950164795, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.035426750779151917, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19446218013763428, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0042553190141916275, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.848029613494873, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1281140297651291, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.9246410131454468, "metrics/particle/momentum/class_4/accepted/samples": 235.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4498131573200226, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.1564741134643555, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.1746773719787598, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.050206184387207, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.28857141733169556, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 12.323713302612305, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0028571428265422583, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8659427165985107, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2027416080236435, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.836988925933838, "metrics/particle/momentum/class_4/assigned/samples": 350.0, "metrics/particle/momentum/class_4/efficiency": 0.6266666650772095, "metrics/particle/momentum/class_4/event/energy_bias": -0.5636171102523804, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.5316882133483887, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9016597270965576, "metrics/particle/momentum/class_4/event/px_bias": 0.0024145767092704773, "metrics/particle/momentum/class_4/event/py_bias": -0.05160408467054367, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2882059812545776, "metrics/particle/momentum/class_4/event/pz_bias": 0.01516950037330389, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.224634647369385, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.021524472162127495, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.19968704879283905, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.19205792248249054, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0002083431463688612, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2350863516330719, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.06716418266296387, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.1746327877044678, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.2191532403230667, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.672464847564697, "metrics/particle/momentum/class_5/accepted/samples": 134.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09761874377727509, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3571949899196625, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.37164998054504395, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.1837967187166214, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.14374999701976776, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.2882930040359497, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08124999701976776, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.5320916175842285, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.1835264265537262, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.175605773925781, "metrics/particle/momentum/class_5/assigned/samples": 160.0, "metrics/particle/momentum/class_5/efficiency": 0.8170731663703918, "metrics/particle/momentum/class_5/event/energy_bias": -0.4386419951915741, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.3300464153289795, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.723800539970398, "metrics/particle/momentum/class_5/event/px_bias": 0.16439077258110046, "metrics/particle/momentum/class_5/event/py_bias": -0.03620382398366928, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2637050151824951, "metrics/particle/momentum/class_5/event/pz_bias": 0.1379106491804123, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.0226216316223145, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.42457467317581177, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.42690983414649963, "metrics/set_slot/angular_recall_all_dr0p10": 0.9317490458488464, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7765880823135376, "metrics/set_slot/soft_count_bias_mean": -0.15128467977046967}, "step_train_time": 4.852492332458496, "step_valid_time": 45431.38040137291, "step_total_time": 45436.23289394379, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.97548700675933, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1623151090322238, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3034, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2854, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2819, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.932984635501798, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.94067237969677, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.936812735926473, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05932762030323006, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9215429879045439, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9291364535266974, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.925324142458559, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.932984635501798, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9460234378484272, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15979207490863934, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2115, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2108, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.925601750547046, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9416740872662511, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9335687486206136, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05832591273374889, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9225382932166302, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9385574354407836, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9304789229750607, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.925601750547046, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9527418515275407, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15485363582515999, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1303, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1206, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1195, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9122541603630863, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9255564082885649, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9188571428571428, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07444359171143515, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9039334341906202, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9171143514965464, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9104761904761904, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9122541603630863} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_16000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..ee24cde413ac3f12fd2264c2f08fccc1d98d075c --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 0.757862826828344, "train": {"Classification_binary": 0.23906996846199036, "Cardinality": 0.1759747564792633, "Classification": 0.10186981409788132, "Regression_pt": 0.15799225866794586, "Regression_eta": 0.0021853779908269644, "Regression_sin_phi": 0.0010241458658128977, "Regression_cos_phi": 0.001078690169379115, "Regression_energy": 0.15475787222385406, "HitGrouping": 0.171359121799469, "Auxiliary": 0.25189974904060364, "Total": 1.3107059001922607, "Total_unweighted": 0.6579781274776906}, "train_group": {}, "valid": {"Classification_binary": 0.21058795849482217, "Cardinality": 0.15702647964159647, "Classification": 0.10643679896990459, "Regression_pt": 0.16199026505152384, "Regression_eta": 0.0019613346084952354, "Regression_sin_phi": 0.0010385864103833835, "Regression_cos_phi": 0.0009714124413828055, "Regression_energy": 0.1613140602906545, "HitGrouping": 0.1737331748008728, "Auxiliary": 0.2439722220102946, "Total": 1.2521398067474365, "Total_unweighted": 0.6443004162671665, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.841320276260376, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5608462691307068, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.33538374304771423, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5714914798736572, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5616304874420166, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.36547744274139404, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7911804914474487, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5039240717887878, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.32570162415504456, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.514880359172821, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5072154402732849, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4335302710533142, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8573282957077026, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5258100628852844, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3150659501552582, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5313835740089417, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5183200240135193, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.32779449224472046, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8473431468009949, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5347251892089844, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.32204702496528625, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5422531366348267, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5305789709091187, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.34741973876953125, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0032876289915293455, "metrics/hit_grouping/calorimeter/background_recall": 0.14177189767360687, "metrics/hit_grouping/calorimeter/completeness": 0.5943636894226074, "metrics/hit_grouping/calorimeter/energy_completeness": 0.595071017742157, "metrics/hit_grouping/calorimeter/energy_purity": 0.5873425006866455, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.37509405612945557, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8721089959144592, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5645135045051575, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2518206536769867, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5747169852256775, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5579111576080322, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2851046919822693, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.847195565700531, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5566215515136719, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.34143251180648804, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5640692114830017, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5516286492347717, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3528327941894531, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.796994686126709, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.45192837715148926, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.44570764899253845, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4540819525718689, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4536430239677429, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.46902552247047424, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8277848362922668, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5658351182937622, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.38627615571022034, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5767447352409363, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.576117217540741, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4000481069087982, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8138148188591003, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.613387942314148, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.520844042301178, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6131061911582947, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5951521992683411, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.40761709213256836, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8712782859802246, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4936351776123047, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23287755250930786, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4986456334590912, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4764569103717804, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28659021854400635, "metrics/hit_grouping/calorimeter/purity": 0.5857886672019958, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8048334717750549, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07715011388063431, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10338345915079117, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07882429659366608, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.053623635321855545, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8405891060829163, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.30010560154914856, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12065920978784561, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.29934921860694885, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2757475972175598, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.20423778891563416, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8483873009681702, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5543904304504395, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.33608993887901306, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5624448657035828, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5516117215156555, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.36010292172431946, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6974676847457886, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4460379481315613, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4629793167114258, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4918030798435211, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.633637547492981, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.751642107963562, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.32790836691856384, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.46387529373168945, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.45367431640625, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.46473780274391174, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6100267171859741, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4596344828605652, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.36482393741607666, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.40928903222084045, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7423370480537415, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6473617553710938, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4457257390022278, "metrics/hit_grouping/tracker/all/object_completeness": 0.40230804681777954, "metrics/hit_grouping/tracker/all/object_purity": 0.4379279315471649, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6884238719940186, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013918890617787838, "metrics/hit_grouping/tracker/background_recall": 0.12152668088674545, "metrics/hit_grouping/tracker/completeness": 0.40006357431411743, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5022901892662048, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6738775372505188, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4316149353981018, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4747055768966675, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5033141374588013, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6652638912200928, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6291138529777527, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4605191648006439, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3748498558998108, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.42440155148506165, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7101107239723206, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6109484434127808, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.46005409955978394, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2806670665740967, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.31686779856681824, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7145175933837891, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.653437077999115, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4731255769729614, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4115425944328308, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4512676000595093, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6886304020881653, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6527318358421326, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5123152732849121, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3928440511226654, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4047733545303345, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6705664992332458, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6385008096694946, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39816758036613464, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.39199936389923096, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4265294373035431, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6913566589355469, "metrics/hit_grouping/tracker/purity": 0.37849998474121094, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8766723275184631, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03770739212632179, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.09392446279525757, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.09198853373527527, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06485671550035477, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.652547299861908, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2968508005142212, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.30871760845184326, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.29947638511657715, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5632421374320984, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6127707362174988, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5224106311798096, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.45821526646614075, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5042707324028015, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7945274710655212, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5464963316917419, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.564259946346283, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6378869414329529, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.619376003742218, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.002656401600688696, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04490125924348831, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.375289916992188, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2655131220817566, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17388993501663208, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8018461465835571, "metrics/particle/class_1/pid_accuracy": 0.960541307926178, "metrics/particle/class_2/efficiency": 0.4984709620475769, "metrics/particle/class_2/pid_accuracy": 0.6609861254692078, "metrics/particle/class_3/efficiency": 0.7093762755393982, "metrics/particle/class_3/pid_accuracy": 0.9589509963989258, "metrics/particle/class_4/efficiency": 0.6322645545005798, "metrics/particle/class_4/pid_accuracy": 0.3946117162704468, "metrics/particle/class_5/efficiency": 0.8601532578468323, "metrics/particle/class_5/pid_accuracy": 0.4699331820011139, "metrics/particle/count/bias_mean": -3.2734375, "metrics/particle/count/mae": 5.221354007720947, "metrics/particle/count/prediction_mean": 67.38150787353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09459484368562698, "metrics/particle/event/energy_response_mean": 1.01902437210083, "metrics/particle/event/met_abs_error": 8.375664710998535, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0736842229962349, "metrics/particle/event/scalar_pt_response_mean": 1.025462031364441, "metrics/particle/event/vector_pt_closure": 0.07005993276834488, "metrics/particle/matched/delta_eta_abs_mean": 0.015446927398443222, "metrics/particle/matched/delta_phi_abs_mean": 0.015924880281090736, "metrics/particle/matched/delta_r_mean": 0.02490842528641224, "metrics/particle/matched/energy_relative_abs_mean": 0.15611480176448822, "metrics/particle/matched/pid_accuracy": 0.936191737651825, "metrics/particle/matched/pt_relative_abs_mean": 0.15480399131774902, "metrics/particle/matching/duplicate_fraction": 0.081421859562397, "metrics/particle/matching/efficiency": 0.7369109988212585, "metrics/particle/matching/f1": 0.754386305809021, "metrics/particle/matching/purity": 0.7727105617523193, "metrics/particle/matching/target_coverage_dr0p05": 0.6775058507919312, "metrics/particle/matching/target_coverage_dr0p10": 0.7946390509605408, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5337246656417847, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.7818310856819153, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02335505746304989, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15787318348884583, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15626269578933716, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03886990621685982, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1970229595899582, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008208107203245163, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7814454436302185, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.11869364976882935, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9168238639831543, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7919.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27667731046676636, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2161914110183716, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3849939107894897, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2498875856399536, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15744294226169586, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.268474578857422, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016461320221424103, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1191842555999756, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.16795964539051056, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.086251974105835, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9902.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.735146701335907, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 6.260981559753418, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.907294273376465, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.573706150054932, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.008649840950965881, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.6380919814109802, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.397315979003906, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.057485818862915, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.36247730255127, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02492639608681202, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1657647043466568, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1655297726392746, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03260958194732666, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2068774402141571, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02913825213909149, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4746873378753662, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.19356046617031097, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.783377170562744, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1613.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4968281686306, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.9630556106567383, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.302905082702637, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6.1510772705078125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21327877044677734, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 88.8181381225586, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05547976493835449, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4790916442871094, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.08994898945093155, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.685120105743408, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2199.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5848441123962402, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.252180099487305, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.028688430786133, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3132264614105225, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.043587297201156616, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.16823618113994598, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.86501693725586, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6788161993026733, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.87179183959961, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.025786984711885452, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15507392585277557, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15401048958301544, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011595215648412704, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.194524884223938, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008567512268200517, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.26635804772377014, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03948832303285599, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7950745224952698, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 17508.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2790277600288391, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.61512291431427, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.3091069459915161, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.1484142541885376, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.16754089295864105, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.008753776550293, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006445590406656265, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5327632427215576, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.044001780450344086, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.1143076419830322, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22496.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7524173855781555, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 7.073322772979736, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.514318466186523, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.450289726257324, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.744225025177002, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.7143099308013916, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.828956604003906, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.7227798700332642, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.8853759765625, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025023426860570908, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.156531423330307, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.155357226729393, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02083650417625904, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.19601571559906006, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004696745425462723, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.48928752541542053, "metrics/particle/momentum/all/accepted/pz_bias": 0.07187535613775253, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5294455289840698, "metrics/particle/momentum/all/accepted/samples": 27040.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2921985387802124, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.523059606552124, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.6482346057891846, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.495428442955017, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16755788028240204, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 24.4388370513916, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012428822927176952, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8243120908737183, "metrics/particle/momentum/all/assigned/pz_bias": 0.07096577435731888, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.063788414001465, "metrics/particle/momentum/all/assigned/samples": 34597.0, "metrics/particle/momentum/all/efficiency": 0.7348026037216187, "metrics/particle/momentum/all/event/energy_bias": 8.082122802734375, "metrics/particle/momentum/all/event/p3_error_mean": 23.281681060791016, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.970799446105957, "metrics/particle/momentum/all/event/px_bias": 1.7964619398117065, "metrics/particle/momentum/all/event/py_bias": -0.09201821684837341, "metrics/particle/momentum/all/event/pz_abs_mean": 17.040611267089844, "metrics/particle/momentum/all/event/pz_bias": 3.101449489593506, "metrics/particle/momentum/all/event/pz_residual_rms": 27.531442642211914, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.027617178857326508, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.15125860273838043, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14953000843524933, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.03328657150268555, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18993520736694336, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.004443190526217222, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5236462950706482, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.08705871552228928, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4542303085327148, "metrics/particle/momentum/class_1/accepted/samples": 14179.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.19984279572963715, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.670381486415863, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.7490159273147583, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.61186283826828, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.11983057856559753, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.428879261016846, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.009353491477668285, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7428648471832275, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.10631348937749863, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.60187029838562, "metrics/particle/momentum/class_1/assigned/samples": 16999.0, "metrics/particle/momentum/class_1/efficiency": 0.7992221117019653, "metrics/particle/momentum/class_1/event/energy_bias": 12.601463317871094, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.783313751220703, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.471552848815918, "metrics/particle/momentum/class_1/event/px_bias": 1.2751885652542114, "metrics/particle/momentum/class_1/event/py_bias": 0.9259094595909119, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.363933563232422, "metrics/particle/momentum/class_1/event/pz_bias": 4.268660545349121, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.5030574798584, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.030139833688735962, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19797447323799133, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19572369754314423, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.025366131216287613, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2382460981607437, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.024965325370430946, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4577609300613403, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.13497208058834076, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.308407783508301, "metrics/particle/momentum/class_2/accepted/samples": 1442.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5092924237251282, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6489759683609009, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.861655056476593, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5928707122802734, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.28826531767845154, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.35369348526001, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04719387739896774, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.2657673358917236, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.11373110115528107, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.449185371398926, "metrics/particle/momentum/class_2/assigned/samples": 2352.0, "metrics/particle/momentum/class_2/efficiency": 0.4829202890396118, "metrics/particle/momentum/class_2/event/energy_bias": -4.7043352127075195, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.580272674560547, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.34264087677002, "metrics/particle/momentum/class_2/event/px_bias": -0.3062174618244171, "metrics/particle/momentum/class_2/event/py_bias": -0.6291450262069702, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.761129379272461, "metrics/particle/momentum/class_2/event/pz_bias": -1.8087151050567627, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.21720314025879, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02111547812819481, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15718965232372284, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15713803470134735, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0035395585000514984, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19706596434116364, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0017219503642991185, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.29190418124198914, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.03754592314362526, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1452367305755615, "metrics/particle/momentum/class_3/accepted/samples": 11034.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3634078800678253, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.5199429988861084, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.739969491958618, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.586102247238159, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20211754739284515, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 36.61335754394531, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009909054264426231, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6687392592430115, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.018742436543107033, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0132393836975098, "metrics/particle/momentum/class_3/assigned/samples": 14734.0, "metrics/particle/momentum/class_3/efficiency": 0.7103586196899414, "metrics/particle/momentum/class_3/event/energy_bias": 1.0548964738845825, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.54544734954834, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.132948875427246, "metrics/particle/momentum/class_3/event/px_bias": 0.634189784526825, "metrics/particle/momentum/class_3/event/py_bias": 0.014187611639499664, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.254033088684082, "metrics/particle/momentum/class_3/event/pz_bias": 0.23501256108283997, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.588985443115234, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.024724895134568214, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1749582141637802, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1637757271528244, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.061298541724681854, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2085038423538208, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004166666883975267, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8363773226737976, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.21050189435482025, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8239446878433228, "metrics/particle/momentum/class_4/accepted/samples": 240.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.40951961278915405, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 7.337135314941406, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.211277961730957, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.125661373138428, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.26912182569503784, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.582735061645508, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014164306223392487, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1189082860946655, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.08093849569559097, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.2130279541015625, "metrics/particle/momentum/class_4/assigned/samples": 353.0, "metrics/particle/momentum/class_4/efficiency": 0.6399999856948853, "metrics/particle/momentum/class_4/event/energy_bias": -0.08612073212862015, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.609731912612915, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.985713243484497, "metrics/particle/momentum/class_4/event/px_bias": -0.054802361875772476, "metrics/particle/momentum/class_4/event/py_bias": -0.22815945744514465, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3128496408462524, "metrics/particle/momentum/class_4/event/pz_bias": 0.14700905978679657, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.9914565086364746, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.018383881077170372, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17940868437290192, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17429077625274658, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0076120770536363125, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21389178931713104, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05517241358757019, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9438642263412476, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.34256675839424133, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.906454086303711, "metrics/particle/momentum/class_5/accepted/samples": 145.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09557900577783585, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3285132944583893, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.34304940700531006, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.18149662017822266, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.056603774428367615, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.1519275903701782, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.056603774428367615, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9717050790786743, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.4764891266822815, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.896409273147583, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.8841463327407837, "metrics/particle/momentum/class_5/event/energy_bias": -0.783780574798584, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.138556480407715, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.6700160503387451, "metrics/particle/momentum/class_5/event/px_bias": 0.24810335040092468, "metrics/particle/momentum/class_5/event/py_bias": -0.17481088638305664, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0369852781295776, "metrics/particle/momentum/class_5/event/pz_bias": 0.2594825327396393, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4635372161865234, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.41356346011161804, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.41488462686538696, "metrics/set_slot/angular_recall_all_dr0p10": 0.9347253441810608, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7868528962135315, "metrics/set_slot/soft_count_bias_mean": -0.8068928718566895}, "step_train_time": 5.530385732650757, "step_valid_time": 48451.29890322685, "step_total_time": 48456.82928919792, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.042301329775765, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17437163409197698, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3061, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2877, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2839, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9405034324942791, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9398889251878471, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9401960784313725, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.060111074812152894, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9280810722458319, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9274746814766416, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9277777777777778, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9405034324942791, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0242471055498372, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16980748693918057, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2259, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2128, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2120, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9312910284463894, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9420097388224878, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9366197183098591, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05799026117751217, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9277899343544858, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9384683488269145, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9330985915492958, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9312910284463894, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0280630606530612, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1798199241606403, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1309, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1211, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1201, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.916036308623298, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9251336898395722, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9205625237552262, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0748663101604278, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9084720121028744, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.917494270435447, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9129608513873052, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.916036308623298} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_17000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..acfba201b3d32637c1e524c3e920b8af2dd8796d --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 0.8052292535051155, "train": {"Classification_binary": 0.22278687357902527, "Cardinality": 0.14666599035263062, "Classification": 0.10180897265672684, "Regression_pt": 0.1461484432220459, "Regression_eta": 0.001809841487556696, "Regression_sin_phi": 0.0009438623674213886, "Regression_cos_phi": 0.0009696810739114881, "Regression_energy": 0.14489969611167908, "HitGrouping": 0.16659337282180786, "Auxiliary": 0.234884113073349, "Total": 1.204854965209961, "Total_unweighted": 0.6193673704983667}, "train_group": {}, "valid": {"Classification_binary": 0.204763134320577, "Cardinality": 0.15412637591362, "Classification": 0.10440231362978618, "Regression_pt": 0.14902718861897787, "Regression_eta": 0.0018221009522676468, "Regression_sin_phi": 0.0009959680028259754, "Regression_cos_phi": 0.0009725150497009357, "Regression_energy": 0.1466415524482727, "HitGrouping": 0.1680704951286316, "Auxiliary": 0.23368372519810995, "Total": 1.2036066055297852, "Total_unweighted": 0.6086247730224084, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8467164039611816, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5711065530776978, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3216617703437805, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.581649899482727, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5742369890213013, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.34948423504829407, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7882680296897888, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5063152313232422, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.31610044836997986, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5162718892097473, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5114588737487793, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.43500739336013794, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8609416484832764, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5438252687454224, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3064568340778351, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5497126579284668, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5369736552238464, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3177505135536194, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8509898781776428, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5491780638694763, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3118119239807129, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5567919015884399, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5462817549705505, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3364678919315338, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003326225094497204, "metrics/hit_grouping/calorimeter/background_recall": 0.1370926797389984, "metrics/hit_grouping/calorimeter/completeness": 0.6087512373924255, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6088070273399353, "metrics/hit_grouping/calorimeter/energy_purity": 0.6009328365325928, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.36122652888298035, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8759901523590088, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5775822401046753, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2373691350221634, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5877629518508911, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5724006295204163, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2717910706996918, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8510259389877319, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5681130290031433, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.33187469840049744, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5767492055892944, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5687673091888428, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3492630124092102, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7999783158302307, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.47218701243400574, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4433874785900116, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.47353702783584595, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4703735113143921, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.45545244216918945, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8305910229682922, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5822575092315674, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.37545105814933777, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5937809944152832, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5950968265533447, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3934929072856903, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8201037645339966, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6169312596321106, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5033453702926636, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6158813834190369, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5965176820755005, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.38651570677757263, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8754674196243286, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5073796510696411, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22410938143730164, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5120394825935364, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4905270040035248, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.27236494421958923, "metrics/hit_grouping/calorimeter/purity": 0.5999918580055237, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8048178553581238, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07802902907133102, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07330827414989471, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07685507833957672, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05812285095453262, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.015037594363093376, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8384715914726257, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.29006868600845337, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10829900205135345, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2876008450984955, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2686004042625427, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.1871689260005951, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.852393627166748, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5703388452529907, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.32628870010375977, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5786204934120178, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5686861872673035, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3494747281074524, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7064267992973328, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4406779706478119, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.48061078786849976, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5082657337188721, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6284224390983582, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7486342191696167, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2977697551250458, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.45595696568489075, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4688975214958191, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4846293032169342, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6211928129196167, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4634479880332947, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.38002151250839233, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.41233372688293457, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.730320930480957, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6568025946617126, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4442119300365448, "metrics/hit_grouping/tracker/all/object_completeness": 0.41654616594314575, "metrics/hit_grouping/tracker/all/object_purity": 0.44599610567092896, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.680854856967926, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011658195406198502, "metrics/hit_grouping/tracker/background_recall": 0.10092359036207199, "metrics/hit_grouping/tracker/completeness": 0.41381973028182983, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.49742767214775085, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6867163181304932, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.42398738861083984, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.48915067315101624, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5139392614364624, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6564965844154358, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6381074786186218, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4587039351463318, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.38746100664138794, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.42794501781463623, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7028498649597168, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6138442158699036, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4714156985282898, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.29609593749046326, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3241717517375946, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7091072797775269, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6591112613677979, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4752483665943146, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.42573046684265137, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4615672528743744, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6826016902923584, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6550359129905701, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5024630427360535, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40339672565460205, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.42438533902168274, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6798029541969299, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6541178822517395, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39342090487480164, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.40696364641189575, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.42962726950645447, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.678772509098053, "metrics/hit_grouping/tracker/purity": 0.3906191289424896, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8756446242332458, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03959276154637337, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11143331229686737, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1088198870420456, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6688109636306763, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.29323697090148926, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3136884570121765, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.30360767245292664, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5358802080154419, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6230255365371704, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5206199884414673, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.47297248244285583, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5114651918411255, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.788260281085968, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5460007786750793, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5929432511329651, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6753820180892944, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6363623142242432, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.00315199070610106, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06188748776912689, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.855566024780273, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2923126220703125, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.200689435005188, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8293664455413818, "metrics/particle/class_1/pid_accuracy": 0.960470974445343, "metrics/particle/class_2/efficiency": 0.5336957573890686, "metrics/particle/class_2/pid_accuracy": 0.6740237474441528, "metrics/particle/class_3/efficiency": 0.7258815765380859, "metrics/particle/class_3/pid_accuracy": 0.9637694358825684, "metrics/particle/class_4/efficiency": 0.6543086171150208, "metrics/particle/class_4/pid_accuracy": 0.3583461046218872, "metrics/particle/class_5/efficiency": 0.8582375645637512, "metrics/particle/class_5/pid_accuracy": 0.5334821343421936, "metrics/particle/count/bias_mean": -2.0455729961395264, "metrics/particle/count/mae": 4.731770992279053, "metrics/particle/count/prediction_mean": 68.609375, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07992983609437943, "metrics/particle/event/energy_response_mean": 0.9867605566978455, "metrics/particle/event/met_abs_error": 6.782884120941162, "metrics/particle/event/scalar_pt_relative_abs_error": 0.061491694301366806, "metrics/particle/event/scalar_pt_response_mean": 0.9943601489067078, "metrics/particle/event/vector_pt_closure": 0.06659918278455734, "metrics/particle/matched/delta_eta_abs_mean": 0.014629476703703403, "metrics/particle/matched/delta_phi_abs_mean": 0.015757285058498383, "metrics/particle/matched/delta_r_mean": 0.02414681389927864, "metrics/particle/matched/energy_relative_abs_mean": 0.15071064233779907, "metrics/particle/matched/pid_accuracy": 0.9383544921875, "metrics/particle/matched/pt_relative_abs_mean": 0.14958594739437103, "metrics/particle/matching/duplicate_fraction": 0.07633986324071884, "metrics/particle/matching/efficiency": 0.7602233290672302, "metrics/particle/matching/f1": 0.7713898420333862, "metrics/particle/matching/purity": 0.7828892469406128, "metrics/particle/matching/target_coverage_dr0p05": 0.697510302066803, "metrics/particle/matching/target_coverage_dr0p10": 0.8108839988708496, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5653115510940552, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8047841191291809, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022856248542666435, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1497463434934616, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14849568903446198, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01384317222982645, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1892944872379303, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.009723076596856117, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7852795720100403, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.035628169775009155, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.178464651107788, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8125.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2761920094490051, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1967101097106934, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3471124172210693, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1948797702789307, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15362577140331268, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.225536346435547, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016301127150654793, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0763871669769287, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.04302956908941269, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.0977885723114014, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10122.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7542703151702881, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.9320895671844482, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.787476539611816, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.221890449523926, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.8272883296012878, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2804350256919861, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.3047380447387695, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7881497740745544, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.113639831542969, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024350376799702644, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15821534395217896, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15610630810260773, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.007837369106709957, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19762030243873596, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.03264094889163971, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.5020296573638916, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.2431873381137848, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.800281286239624, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1685.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5274604558944702, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4885658025741577, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.302097797393799, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.136519432067871, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22025097906589508, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 120.11891174316406, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05538727715611458, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4075064659118652, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2305833250284195, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.393399238586426, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2311.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6109499931335449, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.8816633224487305, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.9453706741333, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2454524040222168, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.016967320814728737, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.05178549140691757, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.797359466552734, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.4265117943286896, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.595972061157227, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.024896901100873947, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15048915147781372, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14909633994102478, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008375625126063824, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18939417600631714, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0012140610488131642, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.26849091053009033, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01625477336347103, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9299175143241882, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18121.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2637813687324524, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.3319779634475708, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0631752014160156, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8970393538475037, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15219494700431824, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.9185991287231445, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004662619903683662, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4764355719089508, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03337797150015831, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7548657655715942, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22734.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7787614464759827, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.956658124923706, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.035839080810547, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.266205787658691, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.9225926399230957, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.5778495073318481, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.987490653991699, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.1554841548204422, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.242618560791016, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02427031472325325, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1507391780614853, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1493445187807083, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.009933638386428356, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1898716241121292, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005585192237049341, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4932383596897125, "metrics/particle/momentum/all/accepted/pz_bias": -0.014489163644611835, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.6771365404129028, "metrics/particle/momentum/all/accepted/samples": 27931.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.284681111574173, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.303334355354309, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.4234607219696045, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.2613632678985596, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1570790857076645, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 31.768102645874023, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01134586427360773, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.776017963886261, "metrics/particle/momentum/all/assigned/pz_bias": -0.005960376001894474, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.888772964477539, "metrics/particle/momentum/all/assigned/samples": 35167.0, "metrics/particle/momentum/all/efficiency": 0.7590152025222778, "metrics/particle/momentum/all/event/energy_bias": -0.9929155111312866, "metrics/particle/momentum/all/event/p3_error_mean": 21.042396545410156, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.188371658325195, "metrics/particle/momentum/all/event/px_bias": -2.766848087310791, "metrics/particle/momentum/all/event/py_bias": 1.9100700616836548, "metrics/particle/momentum/all/event/pz_abs_mean": 15.034636497497559, "metrics/particle/momentum/all/event/pz_bias": -1.0591773986816406, "metrics/particle/momentum/all/event/pz_residual_rms": 25.805030822753906, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02665882743895054, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14156176149845123, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14139540493488312, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008959610015153885, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.18062525987625122, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.005459632724523544, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5272455811500549, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03246993571519852, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5582655668258667, "metrics/particle/momentum/class_1/accepted/samples": 14653.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.18993939459323883, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5434149503707886, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.593402087688446, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4348313510417938, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10652841627597809, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.511254787445068, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00903666764497757, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7037041187286377, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02548259124159813, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.454993486404419, "metrics/particle/momentum/class_1/assigned/samples": 17263.0, "metrics/particle/momentum/class_1/efficiency": 0.8259398937225342, "metrics/particle/momentum/class_1/event/energy_bias": 6.544502258300781, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.042091369628906, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.897592544555664, "metrics/particle/momentum/class_1/event/px_bias": -1.397430181503296, "metrics/particle/momentum/class_1/event/py_bias": 0.39723655581474304, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.396230697631836, "metrics/particle/momentum/class_1/event/pz_bias": -0.2372036874294281, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.648412704467773, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02989341877400875, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18862615525722504, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19080038368701935, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.014368635602295399, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23076844215393066, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02853519283235073, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5250509977340698, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08407167345285416, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7594385147094727, "metrics/particle/momentum/class_2/accepted/samples": 1577.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5404262542724609, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5905893445014954, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8516767621040344, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5313750505447388, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.26866284012794495, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.768258571624756, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.044298604130744934, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.1547658443450928, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.038980867713689804, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.284297943115234, "metrics/particle/momentum/class_2/assigned/samples": 2438.0, "metrics/particle/momentum/class_2/efficiency": 0.5281313061714172, "metrics/particle/momentum/class_2/event/energy_bias": -13.356335639953613, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.14178466796875, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.263315200805664, "metrics/particle/momentum/class_2/event/px_bias": -0.775072455406189, "metrics/particle/momentum/class_2/event/py_bias": 0.14788566529750824, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.828984260559082, "metrics/particle/momentum/class_2/event/pz_bias": -0.7831935882568359, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.219572067260742, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02040824480354786, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15692822635173798, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15370279550552368, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.014085466973483562, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19488780200481415, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0017675651470199227, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2817874252796173, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.012060971930623055, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2151556015014648, "metrics/particle/momentum/class_3/accepted/samples": 11315.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.34991905093193054, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.228198766708374, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.4364593029022217, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.29486346244812, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19459134340286255, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 48.28513717651367, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007898787967860699, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6152817010879517, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.016804449260234833, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7804527282714844, "metrics/particle/momentum/class_3/assigned/samples": 14939.0, "metrics/particle/momentum/class_3/efficiency": 0.7284491062164307, "metrics/particle/momentum/class_3/event/energy_bias": 6.198501110076904, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.383645057678223, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.840891361236572, "metrics/particle/momentum/class_3/event/px_bias": -0.35239988565444946, "metrics/particle/momentum/class_3/event/py_bias": 1.1692137718200684, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.360813617706299, "metrics/particle/momentum/class_3/event/pz_bias": 0.03002220392227173, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.79198169708252, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025846706703305244, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15898539125919342, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1468576341867447, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03607484698295593, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1929243505001068, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.008130080997943878, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.7356393933296204, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17199549078941345, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8921202421188354, "metrics/particle/momentum/class_4/accepted/samples": 246.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4768233299255371, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.53284215927124, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.4910635948181152, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.382808208465576, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.284153014421463, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.88365650177002, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008196720853447914, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8503232002258301, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.12486614286899567, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.0012052059173584, "metrics/particle/momentum/class_4/assigned/samples": 366.0, "metrics/particle/momentum/class_4/efficiency": 0.656000018119812, "metrics/particle/momentum/class_4/event/energy_bias": -0.6392261981964111, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.4304492473602295, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.773034930229187, "metrics/particle/momentum/class_4/event/px_bias": 0.07261356711387634, "metrics/particle/momentum/class_4/event/py_bias": 0.10524293035268784, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3200381994247437, "metrics/particle/momentum/class_4/event/pz_bias": -0.1584208607673645, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.527853965759277, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.020305875688791275, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16982094943523407, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16648782789707184, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004136631265282631, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2068481147289276, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.06428571790456772, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9751005172729492, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.17775295674800873, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.71082067489624, "metrics/particle/momentum/class_5/accepted/samples": 140.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08037576079368591, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4189383387565613, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3885607421398163, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.21928860247135162, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.11801242083311081, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.7865209579467773, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08695652335882187, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.397204637527466, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.17757849395275116, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.221067905426025, "metrics/particle/momentum/class_5/assigned/samples": 161.0, "metrics/particle/momentum/class_5/efficiency": 0.8536585569381714, "metrics/particle/momentum/class_5/event/energy_bias": 0.25964245200157166, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.3808717727661133, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.858814001083374, "metrics/particle/momentum/class_5/event/px_bias": -0.3145594298839569, "metrics/particle/momentum/class_5/event/py_bias": 0.09049102663993835, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.1569820642471313, "metrics/particle/momentum/class_5/event/pz_bias": 0.0896185040473938, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.6803722381591797, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3957225978374481, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3981398940086365, "metrics/set_slot/angular_recall_all_dr0p10": 0.9399406313896179, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8020474314689636, "metrics/set_slot/soft_count_bias_mean": 1.1386810541152954}, "step_train_time": 5.907402276992798, "step_valid_time": 51438.91571187973, "step_total_time": 51444.82311439514, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.008467069589528, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15638443917572842, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2873, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2842, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9391958156260216, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9488110964332893, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9439789715787744, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0511889035667107, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9290617848970252, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9385733157199472, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9337933300476425, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9391958156260216, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9955362539661694, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1523572712070369, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2140, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2127, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.936542669584245, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9528049866429208, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9446038402118737, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04719501335707925, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9308533916849016, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9470169189670525, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9388655925844185, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.936542669584245, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.993618885121735, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1583792420485023, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1299, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1224, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1217, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9258698940998488, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9422632794457275, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9339946585272797, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.057736720554272515, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9205748865355522, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9368745188606621, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9286531858069439, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9258698940998488} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_18000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..61f26776a4af9bf4f20bc029d76dded9d9250704 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 0.852595680181887, "train": {"Classification_binary": 0.2098725140094757, "Cardinality": 0.1449941098690033, "Classification": 0.10882987082004547, "Regression_pt": 0.14657306671142578, "Regression_eta": 0.001998558174818754, "Regression_sin_phi": 0.001148113515228033, "Regression_cos_phi": 0.0011131147621199489, "Regression_energy": 0.14472880959510803, "HitGrouping": 0.16499222815036774, "Auxiliary": 0.23030570149421692, "Total": 1.222737431526184, "Total_unweighted": 0.6142640475882217}, "train_group": {}, "valid": {"Classification_binary": 0.19966437419255575, "Cardinality": 0.16146024068196616, "Classification": 0.10534111658732097, "Regression_pt": 0.14816296100616455, "Regression_eta": 0.001891563180834055, "Regression_sin_phi": 0.0010282387180874746, "Regression_cos_phi": 0.0010542813688516617, "Regression_energy": 0.14652285973230997, "HitGrouping": 0.16923598448435465, "Auxiliary": 0.23660123348236084, "Total": 1.2214457988739014, "Total_unweighted": 0.6036653947861245, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.845646858215332, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5700465440750122, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.317876398563385, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5810739994049072, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5755470395088196, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3489164412021637, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7867247462272644, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5218802690505981, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.32570162415504456, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5292288064956665, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.521151602268219, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.43537667393684387, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8593183159828186, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5439790487289429, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3087711036205292, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5500875115394592, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5394389033317566, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3253876268863678, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8495404720306396, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5501605868339539, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.31284403800964355, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5578555464744568, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5489581823348999, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3410550355911255, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0030206944793462753, "metrics/hit_grouping/calorimeter/background_recall": 0.1272127628326416, "metrics/hit_grouping/calorimeter/completeness": 0.6057816743850708, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6039590239524841, "metrics/hit_grouping/calorimeter/energy_purity": 0.5960360169410706, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3601020276546478, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8772943019866943, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5818407535552979, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.24442420899868011, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5919365286827087, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5718727707862854, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.27304279804229736, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8507208824157715, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5676981210708618, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.32899585366249084, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5763996839523315, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.570014476776123, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3430446684360504, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7917622327804565, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.46789780259132385, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.43607887625694275, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.46968352794647217, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.48102325201034546, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4777262210845947, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8289182186126709, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5781139731407166, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.36973780393600464, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5898320078849792, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5962472558021545, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3923502564430237, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8066454529762268, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.616839587688446, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4966546595096588, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6161563992500305, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6097850203514099, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.43643850088119507, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8756765127182007, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5137165188789368, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23293887078762054, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5183079838752747, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.49349746108055115, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2773928642272949, "metrics/hit_grouping/calorimeter/purity": 0.5968820452690125, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8122113347053528, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08864752948284149, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0827067643404007, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08892319351434708, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06563836336135864, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.024436090141534805, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8535426259040833, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3233301043510437, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12948793172836304, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.321020245552063, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2940598130226135, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2077692747116089, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8499405384063721, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5694846510887146, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3261355757713318, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5778211355209351, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5700981616973877, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3531501591205597, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7049373388290405, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4070693850517273, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.47246024012565613, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5169106125831604, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6255251169204712, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7520908117294312, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3128390610218048, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.46574681997299194, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.47028648853302, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.46895721554756165, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6160226464271545, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44200605154037476, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3697291314601898, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4179111421108246, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7367247343063354, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6534008383750916, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4217275083065033, "metrics/hit_grouping/tracker/all/object_completeness": 0.4083954691886902, "metrics/hit_grouping/tracker/all/object_purity": 0.45220696926116943, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6827693581581116, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013082380406558514, "metrics/hit_grouping/tracker/background_recall": 0.10719924420118332, "metrics/hit_grouping/tracker/completeness": 0.40461206436157227, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.47270023822784424, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6813716888427734, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40952131152153015, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.48466914892196655, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5196362137794495, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6581623554229736, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6344354748725891, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4385550916194916, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.37723419070243835, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4354356527328491, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7113813757896423, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6147077083587646, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4301172196865082, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.28246039152145386, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.33016809821128845, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7049594521522522, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6537268161773682, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4354249835014343, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4096037447452545, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4676039516925812, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.684384822845459, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6570407748222351, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4839901626110077, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40187501907348633, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.43303948640823364, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6699507236480713, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6523247957229614, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3927585780620575, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4079936444759369, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.43562647700309753, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6829671859741211, "metrics/hit_grouping/tracker/purity": 0.38247886300086975, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8788790106773376, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.044871795922517776, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11607029289007187, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11280371993780136, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6635929346084595, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2705214321613312, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3094731867313385, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3108014762401581, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5508518218994141, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.618835985660553, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4940406382083893, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4624975621700287, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5178999304771423, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7890437245368958, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5342587232589722, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5886787176132202, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6680887341499329, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6347097158432007, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.014894029125571251, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06023488938808441, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.44065284729004, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2690272331237793, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17740406095981598, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8396078944206238, "metrics/particle/class_1/pid_accuracy": 0.961679995059967, "metrics/particle/class_2/efficiency": 0.5305244326591492, "metrics/particle/class_2/pid_accuracy": 0.653074324131012, "metrics/particle/class_3/efficiency": 0.7398508787155151, "metrics/particle/class_3/pid_accuracy": 0.9616760015487671, "metrics/particle/class_4/efficiency": 0.657314658164978, "metrics/particle/class_4/pid_accuracy": 0.4085365831851959, "metrics/particle/class_5/efficiency": 0.8390804529190063, "metrics/particle/class_5/pid_accuracy": 0.5045661926269531, "metrics/particle/count/bias_mean": 0.12890625, "metrics/particle/count/mae": 4.259114742279053, "metrics/particle/count/prediction_mean": 70.78385162353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.09808888286352158, "metrics/particle/event/energy_response_mean": 1.0280264616012573, "metrics/particle/event/met_abs_error": 7.642727375030518, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07625026255846024, "metrics/particle/event/scalar_pt_response_mean": 1.0392440557479858, "metrics/particle/event/vector_pt_closure": 0.07107468694448471, "metrics/particle/matched/delta_eta_abs_mean": 0.01526500191539526, "metrics/particle/matched/delta_phi_abs_mean": 0.01668364368379116, "metrics/particle/matched/delta_r_mean": 0.025315025821328163, "metrics/particle/matched/energy_relative_abs_mean": 0.14885422587394714, "metrics/particle/matched/pid_accuracy": 0.9376651048660278, "metrics/particle/matched/pt_relative_abs_mean": 0.1477462351322174, "metrics/particle/matching/duplicate_fraction": 0.08285199105739594, "metrics/particle/matching/efficiency": 0.7707369923591614, "metrics/particle/matching/f1": 0.7700345516204834, "metrics/particle/matching/purity": 0.7693333625793457, "metrics/particle/matching/target_coverage_dr0p05": 0.7041446566581726, "metrics/particle/matching/target_coverage_dr0p10": 0.8234800696372986, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5771980881690979, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8171037435531616, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024183999747037888, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14652974903583527, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1447826325893402, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01572456583380699, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1843658685684204, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00840233825147152, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7494637966156006, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.06084233522415161, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9525965452194214, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8212.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.30371057987213135, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2712104320526123, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.688511610031128, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.5418710708618164, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16315536201000214, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.55986213684082, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017123617231845856, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0959327220916748, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06271857768297195, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.0222129821777344, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10395.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7623468041419983, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 5.890079498291016, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.312457084655762, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.25711727142334, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.9650640487670898, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.10862860083580017, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.890285015106201, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.2252564430236816, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.917689323425293, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02622288465499878, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15846455097198486, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15573036670684814, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012410101480782032, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1975940614938736, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02833031862974167, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4279701709747314, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.10797886550426483, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6614887714385986, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1659.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6203166842460632, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.0266873836517334, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 7.596634387969971, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 7.4201884269714355, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2495867758989334, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 121.3704833984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05991735681891441, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6361372470855713, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.11287734657526016, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.926100730895996, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2420.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6015228629112244, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.182619571685791, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.028087615966797, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2403348684310913, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.08179083466529846, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.13792693614959717, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.886422157287598, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.2083621621131897, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.984663009643555, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.025861088186502457, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14951194822788239, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.148732528090477, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.005616231355816126, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18824230134487152, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007074829773046076, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2570601999759674, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0403713695704937, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8224217295646667, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18375.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.24845711886882782, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.414399266242981, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.18471360206604, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.02121901512146, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15054839849472046, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.68731689453125, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004439414944499731, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4693024754524231, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.04216408357024193, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.739424705505371, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22976.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7896772623062134, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 10.81240463256836, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.177518844604492, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.287385940551758, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.9837123155593872, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.8166191577911377, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.437776565551758, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.226384162902832, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.7999906539917, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025394752621650696, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14917075634002686, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14799517393112183, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.008954072371125221, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18768900632858276, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004567018244415522, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4689895510673523, "metrics/particle/momentum/all/accepted/pz_bias": 0.05029378458857536, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5283573865890503, "metrics/particle/momentum/all/accepted/samples": 28246.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2896479368209839, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4142118692398071, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.7645751237869263, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.6050999164581299, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1609063744544983, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 33.52779006958008, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011874493211507797, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7978085875511169, "metrics/particle/momentum/all/assigned/pz_bias": 0.05291511118412018, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.973688840866089, "metrics/particle/momentum/all/assigned/samples": 35791.0, "metrics/particle/momentum/all/efficiency": 0.767575204372406, "metrics/particle/momentum/all/event/energy_bias": 11.519865989685059, "metrics/particle/momentum/all/event/p3_error_mean": 22.513500213623047, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.230781555175781, "metrics/particle/momentum/all/event/px_bias": -3.030567169189453, "metrics/particle/momentum/all/event/py_bias": 2.787321090698242, "metrics/particle/momentum/all/event/pz_abs_mean": 15.881098747253418, "metrics/particle/momentum/all/event/pz_bias": 3.6600027084350586, "metrics/particle/momentum/all/event/pz_residual_rms": 25.834558486938477, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.027367783710360527, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13964034616947174, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13802717626094818, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.020002538338303566, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.177120640873909, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0031047516968101263, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4786279797554016, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.05468795448541641, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3438963890075684, "metrics/particle/momentum/class_1/accepted/samples": 14816.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.18635962903499603, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5382103323936462, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6697422862052917, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.5272020101547241, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.10772404074668884, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.476731300354004, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00733440276235342, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6904517412185669, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.034958142787218094, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.5114517211914062, "metrics/particle/momentum/class_1/assigned/samples": 17452.0, "metrics/particle/momentum/class_1/efficiency": 0.8351276516914368, "metrics/particle/momentum/class_1/event/energy_bias": 16.673288345336914, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.570770263671875, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.909257888793945, "metrics/particle/momentum/class_1/event/px_bias": -2.500532388687134, "metrics/particle/momentum/class_1/event/py_bias": 1.5748904943466187, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.382190704345703, "metrics/particle/momentum/class_1/event/pz_bias": 1.9363946914672852, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.674991607666016, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.031178578734397888, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19588075578212738, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19415238499641418, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.006335136014968157, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23407936096191406, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0357142873108387, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5510377883911133, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.11648673564195633, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.605125904083252, "metrics/particle/momentum/class_2/accepted/samples": 1568.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6199066042900085, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6804907917976379, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9361621141433716, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6124840974807739, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3005714416503906, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.38811731338501, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05409523844718933, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.4263947010040283, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1569753885269165, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.955402374267578, "metrics/particle/momentum/class_2/assigned/samples": 2625.0, "metrics/particle/momentum/class_2/efficiency": 0.5251172184944153, "metrics/particle/momentum/class_2/event/energy_bias": -11.134445190429688, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.669857025146484, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.979234218597412, "metrics/particle/momentum/class_2/event/px_bias": -1.313525676727295, "metrics/particle/momentum/class_2/event/py_bias": 0.46623510122299194, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.534290313720703, "metrics/particle/momentum/class_2/event/pz_bias": 0.22307416796684265, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.059675216674805, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022094765678048134, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15478835999965668, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1543908417224884, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.00421591941267252, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1934252232313156, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015675346367061138, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2889459431171417, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.024373527616262436, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1813782453536987, "metrics/particle/momentum/class_3/accepted/samples": 11483.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.34957021474838257, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.506188154220581, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.147357702255249, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.9949872493743896, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19485051929950714, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 50.94522476196289, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008955617435276508, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6204379200935364, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04195655509829521, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.6580777168273926, "metrics/particle/momentum/class_3/assigned/samples": 15186.0, "metrics/particle/momentum/class_3/efficiency": 0.7392647862434387, "metrics/particle/momentum/class_3/event/energy_bias": 5.560191631317139, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.355979919433594, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.69496488571167, "metrics/particle/momentum/class_3/event/px_bias": 0.4736798107624054, "metrics/particle/momentum/class_3/event/py_bias": 1.3413728475570679, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.6634039878845215, "metrics/particle/momentum/class_3/event/pz_bias": 1.0388514995574951, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.423820495605469, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02528517320752144, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14810506999492645, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1414901316165924, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04935070127248764, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18768073618412018, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004098360426723957, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5949584245681763, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.23949912190437317, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3860270977020264, "metrics/particle/momentum/class_4/accepted/samples": 244.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4461582601070404, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.5088555812835693, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.0147812366485596, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.932448625564575, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2888283431529999, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.642437934875488, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.00817438680678606, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.766442060470581, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2659892439842224, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7182711362838745, "metrics/particle/momentum/class_4/assigned/samples": 367.0, "metrics/particle/momentum/class_4/efficiency": 0.6506666541099548, "metrics/particle/momentum/class_4/event/energy_bias": -0.06335525959730148, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.6522860527038574, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.947249412536621, "metrics/particle/momentum/class_4/event/px_bias": 0.0719003826379776, "metrics/particle/momentum/class_4/event/py_bias": -0.27340659499168396, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4090588092803955, "metrics/particle/momentum/class_4/event/pz_bias": -0.06793606281280518, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.681583404541016, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02257322147488594, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17668193578720093, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17360298335552216, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02120329812169075, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21248842775821686, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05925925821065903, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.930108904838562, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.6620091795921326, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.169008731842041, "metrics/particle/momentum/class_5/accepted/samples": 135.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0923766940832138, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.5602388381958008, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.6704513430595398, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.5064858198165894, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1552795022726059, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.8486812114715576, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.09937888383865356, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.683623790740967, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.8507117033004761, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.538238525390625, "metrics/particle/momentum/class_5/assigned/samples": 161.0, "metrics/particle/momentum/class_5/efficiency": 0.8231707215309143, "metrics/particle/momentum/class_5/event/energy_bias": 0.48418429493904114, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.556246757507324, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.9072027206420898, "metrics/particle/momentum/class_5/event/px_bias": 0.2379106879234314, "metrics/particle/momentum/class_5/event/py_bias": -0.3217709958553314, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.337372064590454, "metrics/particle/momentum/class_5/event/pz_bias": 0.5296181440353394, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.215909004211426, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.39761558175086975, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.40014541149139404, "metrics/set_slot/angular_recall_all_dr0p10": 0.93436598777771, "metrics/set_slot/angular_recall_selected_dr0p10": 0.812597930431366, "metrics/set_slot/soft_count_bias_mean": 2.076072931289673}, "step_train_time": 5.491407871246338, "step_valid_time": 54510.77325153351, "step_total_time": 54516.26465988159, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0377009416284375, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16856975211403713, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3069, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2872, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2836, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9388689114089572, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9358097100032584, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9373368146214099, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0641902899967416, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9271003595946388, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.924079504724666, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9255874673629243, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9388689114089572, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0300744941786342, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16548154720225083, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2269, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2131, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2114, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9326039387308533, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9391802556192155, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9358805445761967, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06081974438078449, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.925164113785558, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9316879682679594, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9284145805884936, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9326039387308533, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0410021527315045, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1726201042977764, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1313, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1225, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1209, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.926626323751891, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9329779131759329, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9297912713472486, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06702208682406702, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9145234493192134, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9207920792079208, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9176470588235294, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.926626323751891} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_19000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..bdd44402890a1936c37be0240df321bcdd93cd04 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 0.8999621068586586, "train": {"Classification_binary": 0.18703250586986542, "Cardinality": 0.1418130099773407, "Classification": 0.11023388803005219, "Regression_pt": 0.14924876391887665, "Regression_eta": 0.001944804796949029, "Regression_sin_phi": 0.0008897821535356343, "Regression_cos_phi": 0.0009683771640993655, "Regression_energy": 0.14629624783992767, "HitGrouping": 0.17048078775405884, "Auxiliary": 0.23247432708740234, "Total": 1.172633171081543, "Total_unweighted": 0.596614369773306}, "train_group": {}, "valid": {"Classification_binary": 0.1967529058456421, "Cardinality": 0.14575462539990744, "Classification": 0.1056994100411733, "Regression_pt": 0.15587503711382547, "Regression_eta": 0.0019763486149410405, "Regression_sin_phi": 0.0009992088501652081, "Regression_cos_phi": 0.0009685112939526638, "Regression_energy": 0.15420846144358316, "HitGrouping": 0.17144179344177246, "Auxiliary": 0.239496111869812, "Total": 1.20944348971049, "Total_unweighted": 0.616479883203283, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8384092450141907, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5658409595489502, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.33888521790504456, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5764834880828857, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5780307650566101, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3660452365875244, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7861822843551636, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5169410109519958, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.32865583896636963, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.525718092918396, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5219103097915649, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.431314617395401, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8549932241439819, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5338532328605652, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3199259340763092, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5407654643058777, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5278571844100952, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.33408933877944946, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.844626784324646, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.542231023311615, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3263474702835083, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5504181385040283, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5425956845283508, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3513188064098358, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.002841409994289279, "metrics/hit_grouping/calorimeter/background_recall": 0.12635302543640137, "metrics/hit_grouping/calorimeter/completeness": 0.5993826985359192, "metrics/hit_grouping/calorimeter/energy_completeness": 0.5984581708908081, "metrics/hit_grouping/calorimeter/energy_purity": 0.5906221866607666, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.37327104806900024, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8729782104492188, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5733593106269836, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2506258487701416, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5844225287437439, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.567764401435852, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2827150523662567, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8448507189750671, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5646864771842957, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.35122063755989075, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5726900100708008, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5644733309745789, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3569783568382263, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7865931987762451, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4561387598514557, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.45568445324897766, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.45864659547805786, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4692372977733612, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.48549884557724, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8207274079322815, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5709643363952637, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3911474645137787, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5828896164894104, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.5899136662483215, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.41075295209884644, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7960343360900879, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6141663789749146, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5074626803398132, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6139583587646484, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6207449436187744, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4467318654060364, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8747828006744385, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5043655037879944, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.23870255053043365, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5097415447235107, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4850417971611023, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2793549597263336, "metrics/hit_grouping/calorimeter/purity": 0.5904709100723267, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8048328161239624, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08478844165802002, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.09398496150970459, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08454842120409012, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05166016146540642, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.030075188726186752, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.848824143409729, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.30848807096481323, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.13301941752433777, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3072684407234192, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.28024640679359436, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2171865850687027, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8450567722320557, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5618484020233154, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3401941955089569, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5706623792648315, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5642474889755249, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3635333478450775, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7013797760009766, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.43097203969955444, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.47647854685783386, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5226408243179321, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6245110630989075, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7493674755096436, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3152501583099365, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4694494307041168, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.46795007586479187, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4737793803215027, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6164804697036743, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4484098553657532, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.37524694204330444, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.42584189772605896, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7374442219734192, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6523895859718323, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.43321460485458374, "metrics/hit_grouping/tracker/all/object_completeness": 0.413318395614624, "metrics/hit_grouping/tracker/all/object_purity": 0.45870301127433777, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.683259129524231, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009805167093873024, "metrics/hit_grouping/tracker/background_recall": 0.09180612862110138, "metrics/hit_grouping/tracker/completeness": 0.4139871299266815, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.48553287982940674, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6856890916824341, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41320356726646423, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.48862507939338684, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5212018489837646, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6499211192131042, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6326872110366821, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4547104835510254, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.38516589999198914, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.44611087441444397, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7122889757156372, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6034674644470215, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.45302075147628784, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2863830327987671, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.34265393018722534, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7229936718940735, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6529572606086731, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4537658095359802, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.41749751567840576, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.47518137097358704, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.68761146068573, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6416744589805603, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4963054060935974, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.38715919852256775, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4257946312427521, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.689039409160614, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6535725593566895, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.395187109708786, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4125748872756958, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.443180114030838, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6765647530555725, "metrics/hit_grouping/tracker/purity": 0.3900458812713623, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8807705640792847, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03318250551819801, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1095697283744812, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11318748444318771, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06184012070298195, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6627682447433472, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.29839959740638733, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3223024904727936, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.32603228092193604, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5493030548095703, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6173737049102783, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5071904063224792, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.46825873851776123, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5243563055992126, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7899949550628662, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5496598482131958, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7937499284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5900168418884277, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6675039529800415, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6325842142105103, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.0005071079358458519, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.058109451085329056, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.316537857055664, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28748342394828796, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.19586026668548584, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8299004435539246, "metrics/particle/class_1/pid_accuracy": 0.9557164311408997, "metrics/particle/class_2/efficiency": 0.5083248615264893, "metrics/particle/class_2/pid_accuracy": 0.6784759163856506, "metrics/particle/class_3/efficiency": 0.7467809319496155, "metrics/particle/class_3/pid_accuracy": 0.9633196592330933, "metrics/particle/class_4/efficiency": 0.640280544757843, "metrics/particle/class_4/pid_accuracy": 0.35680750012397766, "metrics/particle/class_5/efficiency": 0.8850574493408203, "metrics/particle/class_5/pid_accuracy": 0.49567100405693054, "metrics/particle/count/bias_mean": -0.8157551884651184, "metrics/particle/count/mae": 4.315755367279053, "metrics/particle/count/prediction_mean": 69.83919525146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08644530177116394, "metrics/particle/event/energy_response_mean": 0.9551923871040344, "metrics/particle/event/met_abs_error": 8.327885627746582, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06776398420333862, "metrics/particle/event/scalar_pt_response_mean": 0.9785545468330383, "metrics/particle/event/vector_pt_closure": 0.0725715383887291, "metrics/particle/matched/delta_eta_abs_mean": 0.017848730087280273, "metrics/particle/matched/delta_phi_abs_mean": 0.017401713877916336, "metrics/particle/matched/delta_r_mean": 0.02779083140194416, "metrics/particle/matched/energy_relative_abs_mean": 0.15733622014522552, "metrics/particle/matched/pid_accuracy": 0.9367433786392212, "metrics/particle/matched/pt_relative_abs_mean": 0.1562173068523407, "metrics/particle/matching/duplicate_fraction": 0.07543370872735977, "metrics/particle/matching/efficiency": 0.7672262787818909, "metrics/particle/matching/f1": 0.7716810703277588, "metrics/particle/matching/purity": 0.7761878371238708, "metrics/particle/matching/target_coverage_dr0p05": 0.6938706040382385, "metrics/particle/matching/target_coverage_dr0p10": 0.8190019130706787, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5510661005973816, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8144407868385315, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025882678106427193, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15292608737945557, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15157441794872284, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0036009990144521, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.19077841937541962, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.010376447811722755, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.8210753202438354, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.048244088888168335, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.2772881984710693, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8288.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29868051409721375, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0335197448730469, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2394096851348877, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0618140697479248, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15109170973300934, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.487594604492188, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.018340611830353737, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.125166654586792, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07447212189435959, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.263779878616333, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10305.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7694021463394165, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.870788812637329, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.045238494873047, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.665502548217773, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.6631902456283569, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.9599925875663757, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.496360778808594, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.564660906791687, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.618305206298828, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027718475088477135, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1611703783273697, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16101959347724915, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.019655801355838776, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20133471488952637, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.029481131583452225, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.507961630821228, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.10289409756660461, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.2873382568359375, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1696.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.58087158203125, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3651957511901855, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.789501190185547, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.5869851112365723, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.224978968501091, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 29.701074600219727, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05466778948903084, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5802459716796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.01924278400838375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.074728965759277, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2378.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6149383783340454, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.836297988891602, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.348001480102539, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3639874458312988, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.14639434218406677, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.03720075264573097, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.179378509521484, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.6147247552871704, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.121797561645508, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02902342565357685, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1607619971036911, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15915153920650482, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010016810148954391, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1982559859752655, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0011589403729885817, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.27746596932411194, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.055277734994888306, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8985967636108398, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18120.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2570994794368744, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2698941230773926, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0658259391784668, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8862159252166748, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15676502883434296, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.895599365234375, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0043715848587453365, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4722740650177002, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.05797848477959633, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7925331592559814, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22875.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7787184715270996, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.4181309938430786, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.849818229675293, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.734054565429688, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.478459894657135, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 4.975107192993164, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.8505144119262695, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.3185911178588867, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.257761001586914, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02801845222711563, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15847578644752502, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15702974796295166, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.004210193641483784, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.196269690990448, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.005586393177509308, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.5120360255241394, "metrics/particle/momentum/all/accepted/pz_bias": 0.05607699975371361, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.7774323225021362, "metrics/particle/momentum/all/accepted/samples": 28104.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2908027768135071, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2077642679214478, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.298282265663147, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1177241802215576, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1596827656030655, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.952754020690918, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01178356446325779, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8024617433547974, "metrics/particle/momentum/all/assigned/pz_bias": 0.0575941801071167, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.084580659866333, "metrics/particle/momentum/all/assigned/samples": 35558.0, "metrics/particle/momentum/all/efficiency": 0.7637163996696472, "metrics/particle/momentum/all/event/energy_bias": -9.288956642150879, "metrics/particle/momentum/all/event/p3_error_mean": 23.17711067199707, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.80945873260498, "metrics/particle/momentum/all/event/px_bias": -0.3311246931552887, "metrics/particle/momentum/all/event/py_bias": 5.9723005294799805, "metrics/particle/momentum/all/event/pz_abs_mean": 15.94168758392334, "metrics/particle/momentum/all/event/pz_bias": 3.497976779937744, "metrics/particle/momentum/all/event/pz_residual_rms": 26.045530319213867, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02878899686038494, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1534585803747177, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.15148606896400452, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0284224022179842, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.19087772071361542, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006082974374294281, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.5638189911842346, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06698361784219742, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8702796697616577, "metrics/particle/momentum/class_1/accepted/samples": 14631.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.19203948974609375, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.528785765171051, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6209385991096497, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.4676065146923065, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.11282612383365631, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.358569622039795, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.010021309368312359, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.7377934455871582, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.053260017186403275, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7623603343963623, "metrics/particle/momentum/class_1/assigned/samples": 17363.0, "metrics/particle/momentum/class_1/efficiency": 0.8246998190879822, "metrics/particle/momentum/class_1/event/energy_bias": -0.6636209487915039, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.018274307250977, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.359638214111328, "metrics/particle/momentum/class_1/event/px_bias": 0.6143510937690735, "metrics/particle/momentum/class_1/event/py_bias": 4.579748153686523, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.415874481201172, "metrics/particle/momentum/class_1/event/pz_bias": 0.48313409090042114, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.762638092041016, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.032494306564331055, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18934419751167297, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1906524896621704, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.00970411580055952, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2309771627187729, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029333332553505898, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5551891326904297, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.09964630007743835, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6245288848876953, "metrics/particle/momentum/class_2/accepted/samples": 1500.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6421972513198853, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.789158821105957, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0833379030227661, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7339140176773071, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.31180664896965027, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.897710800170898, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04675118997693062, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.331984281539917, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.2013276070356369, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.485008239746094, "metrics/particle/momentum/class_2/assigned/samples": 2524.0, "metrics/particle/momentum/class_2/efficiency": 0.5023442506790161, "metrics/particle/momentum/class_2/event/energy_bias": -7.820250034332275, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.96571159362793, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.872379302978516, "metrics/particle/momentum/class_2/event/px_bias": -0.7456621527671814, "metrics/particle/momentum/class_2/event/py_bias": 0.30990666151046753, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.309962272644043, "metrics/particle/momentum/class_2/event/pz_bias": 2.1568684577941895, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.838924407958984, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.026493744924664497, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16035990417003632, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1592864841222763, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.02481807768344879, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19754834473133087, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013807386858388782, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.28887948393821716, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.031066328287124634, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.104964256286621, "metrics/particle/momentum/class_3/accepted/samples": 11588.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3438360393047333, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.9126969575881958, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.001715898513794, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8168251514434814, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18563032150268555, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.971822738647461, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007396156433969736, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.596443772315979, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.021304115653038025, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.76693058013916, "metrics/particle/momentum/class_3/assigned/samples": 15143.0, "metrics/particle/momentum/class_3/efficiency": 0.7460246086120605, "metrics/particle/momentum/class_3/event/energy_bias": 1.7570695877075195, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.99573802947998, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.307198524475098, "metrics/particle/momentum/class_3/event/px_bias": -0.28674405813217163, "metrics/particle/momentum/class_3/event/py_bias": 1.525565266609192, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.571010112762451, "metrics/particle/momentum/class_3/event/pz_bias": 0.6504372358322144, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.69821834564209, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03043704479932785, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17100870609283447, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16452929377555847, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.03699860721826553, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20861215889453888, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.008230452425777912, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8861034512519836, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.24260014295578003, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2598111629486084, "metrics/particle/momentum/class_4/accepted/samples": 243.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.45580941438674927, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 7.537819862365723, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 6.243053436279297, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 6.1338043212890625, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2888283431529999, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 53.91983413696289, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01089918240904808, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0432039499282837, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.5049514174461365, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.2479121685028076, "metrics/particle/momentum/class_4/assigned/samples": 367.0, "metrics/particle/momentum/class_4/efficiency": 0.6480000019073486, "metrics/particle/momentum/class_4/event/energy_bias": -1.3040326833724976, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.511197328567505, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.8447626829147339, "metrics/particle/momentum/class_4/event/px_bias": 0.004209673032164574, "metrics/particle/momentum/class_4/event/py_bias": -0.09663790464401245, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.339011788368225, "metrics/particle/momentum/class_4/event/pz_bias": -0.032961029559373856, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.739763259887695, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.021630974486470222, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.17414924502372742, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.17605602741241455, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.030757104977965355, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.21753312647342682, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0422535203397274, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7280422449111938, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.1938919574022293, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.208040714263916, "metrics/particle/momentum/class_5/accepted/samples": 142.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06886444240808487, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2620386779308319, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2822043001651764, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05772699788212776, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.09316769987344742, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.6841289401054382, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.06832297891378403, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.6266863346099854, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.6652415990829468, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 7.9167280197143555, "metrics/particle/momentum/class_5/assigned/samples": 161.0, "metrics/particle/momentum/class_5/efficiency": 0.8658536672592163, "metrics/particle/momentum/class_5/event/energy_bias": -1.2581217288970947, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.4133658409118652, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.8315391540527344, "metrics/particle/momentum/class_5/event/px_bias": 0.08272071927785873, "metrics/particle/momentum/class_5/event/py_bias": -0.34628182649612427, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.222408413887024, "metrics/particle/momentum/class_5/event/pz_bias": 0.24049803614616394, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.436333656311035, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.42267733812332153, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.4254598617553711, "metrics/set_slot/angular_recall_all_dr0p10": 0.9418572783470154, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8019552826881409, "metrics/set_slot/soft_count_bias_mean": 0.6889012455940247}, "step_train_time": 5.690340757369995, "step_valid_time": 57471.77652215958, "step_total_time": 57477.4668636322, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9943456691243288, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17252775869601455, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3043, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2861, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2837, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9352729650212488, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9401906013802169, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9377253359554244, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.059809398619783107, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9274272638117032, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9323036477160697, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9298590626024255, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9352729650212488, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9719538676868325, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15954660938775378, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2251, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2125, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2116, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9299781181619255, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9440248778320747, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9369488536155203, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05597512216792536, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.926039387308534, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.94002665482008, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9329805996472663, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9299781181619255, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9828540358857348, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16614055389844418, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1300, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1201, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1188, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9084720121028744, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9238461538461539, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.916094584286804, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07615384615384616, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8986384266263238, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9138461538461539, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9061784897025171, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9084720121028744} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_2000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..99313e09be617d951353fe01068585fb10e80e10 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 0.094732853353543, "train": {"Classification_binary": 0.4232552945613861, "Cardinality": 0.31492847204208374, "Classification": 0.3055550456047058, "Regression_pt": 0.42044442892074585, "Regression_eta": 0.005754181649535894, "Regression_sin_phi": 0.0026108487509191036, "Regression_cos_phi": 0.002692036097869277, "Regression_energy": 0.41685792803764343, "HitGrouping": 0.26228031516075134, "Auxiliary": 0.5342300534248352, "Total": 2.7893178462982178, "Total_unweighted": 1.5771697636228055}, "train_group": {}, "valid": {"Classification_binary": 0.383864164352417, "Cardinality": 0.2779378096262614, "Classification": 0.29341063896814984, "Regression_pt": 0.4350320100784302, "Regression_eta": 0.006288726503650348, "Regression_sin_phi": 0.0026777073120077453, "Regression_cos_phi": 0.0025823883091410003, "Regression_energy": 0.43059206008911133, "HitGrouping": 0.2642892599105835, "Auxiliary": 0.5237248738606771, "Total": 2.72402286529541, "Total_unweighted": 1.5544476956129074, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8017297983169556, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.22708334028720856, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3409671485424042, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.2265796959400177, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.2191593050956726, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.48774486780166626, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.773363471031189, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.2496088445186615, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3109305799007416, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.24887476861476898, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.22770246863365173, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5192023515701294, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8100206255912781, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.2042100429534912, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3121962547302246, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.20326973497867584, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.20004038512706757, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4741032123565674, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8046630024909973, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.21466422080993652, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.32081422209739685, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.21387220919132233, "metrics/hit_grouping/calorimeter/all/object_purity": 0.20798014104366302, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4817373752593994, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0005916387890465558, "metrics/hit_grouping/calorimeter/background_recall": 0.026722291484475136, "metrics/hit_grouping/calorimeter/completeness": 0.31728723645210266, "metrics/hit_grouping/calorimeter/energy_completeness": 0.34521812200546265, "metrics/hit_grouping/calorimeter/energy_purity": 0.34032803773880005, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.529201090335846, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8291136026382446, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.22044600546360016, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.24971552193164825, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.22117674350738525, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.22050487995147705, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.4238734543323517, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.7923741936683655, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2243110090494156, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.34292951226234436, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.22216010093688965, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.2200244516134262, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5104790329933167, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.767188549041748, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.193156898021698, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4435034692287445, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.1906290501356125, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.1703086942434311, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5707656741142273, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7661421895027161, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.2473018616437912, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.41532355546951294, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.24571959674358368, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.24672013521194458, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5697017312049866, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7448796629905701, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4244222640991211, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6711271405220032, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.42247235774993896, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.37072160840034485, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.6031909584999084, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8510594964027405, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.15639834105968475, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.18272119760513306, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.15654996037483215, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.149093896150589, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.37758293747901917, "metrics/hit_grouping/calorimeter/purity": 0.3118661046028137, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.894862174987793, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.03786621242761612, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.11278195679187775, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.03771929815411568, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.00969741027802229, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.046992480754852295, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8816877603530884, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.04004806652665138, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.07004120200872421, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.03938324376940727, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.026059888303279877, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.25073572993278503, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.7991849780082703, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.2266317903995514, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3372538089752197, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.22582267224788666, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.22067788243293762, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.500842273235321, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6196876168251038, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4150369465351105, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.23711180686950684, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.24346698820590973, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7509778141975403, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7153064012527466, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.30741411447525024, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.24857383966445923, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.213703453540802, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5629897713661194, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5347417593002319, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.40048927068710327, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.17346321046352386, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.18877367675304413, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.8201899528503418, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.5741869211196899, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.39808547496795654, "metrics/hit_grouping/tracker/all/object_completeness": 0.19857339560985565, "metrics/hit_grouping/tracker/all/object_purity": 0.20742489397525787, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7799198627471924, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.010016942396759987, "metrics/hit_grouping/tracker/background_recall": 0.04637669771909714, "metrics/hit_grouping/tracker/completeness": 0.2056613564491272, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6007496118545532, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.575369119644165, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3838330805301666, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.23127418756484985, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.25463590025901794, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.8009819388389587, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.5744303464889526, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.3946269750595093, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.17347881197929382, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.17308129370212555, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7694681286811829, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.571513295173645, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43083858489990234, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.15623994171619415, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.14443297684192657, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7469792366027832, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.5847021341323853, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4282924234867096, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.1999148279428482, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.20313824713230133, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7715038061141968, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.5695414543151855, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.514162540435791, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.23394039273262024, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.2588224709033966, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7567734122276306, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5613495111465454, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3380064070224762, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.1904892921447754, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.20378369092941284, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.795010507106781, "metrics/hit_grouping/tracker/purity": 0.19380922615528107, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8867234587669373, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.013197586871683598, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.012695719487965107, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.010092620737850666, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.05429864302277565, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6412859559059143, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.19566339254379272, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.10819678753614426, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.07644497603178024, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5864739418029785, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5205347537994385, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.47714173793792725, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.23595282435417175, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.25090503692626953, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.908566951751709, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4451823830604553, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.26249101758003235, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.27395597100257874, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5239391326904297, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.10397038608789444, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.05053567513823509, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.083600997924805, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.1474631428718567, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.05583997815847397, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.4940115213394165, "metrics/particle/class_1/pid_accuracy": 0.8685480356216431, "metrics/particle/class_2/efficiency": 0.3580246865749359, "metrics/particle/class_2/pid_accuracy": 0.11610250174999237, "metrics/particle/class_3/efficiency": 0.46759065985679626, "metrics/particle/class_3/pid_accuracy": 0.8700733780860901, "metrics/particle/class_4/efficiency": 0.46893787384033203, "metrics/particle/class_4/pid_accuracy": 0.18803419172763824, "metrics/particle/class_5/efficiency": 0.4214559495449066, "metrics/particle/class_5/pid_accuracy": 0.08636363595724106, "metrics/particle/count/bias_mean": -4.197265625, "metrics/particle/count/mae": 7.990234375, "metrics/particle/count/prediction_mean": 66.45767974853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.16258741915225983, "metrics/particle/event/energy_response_mean": 0.9728820323944092, "metrics/particle/event/met_abs_error": 14.564319610595703, "metrics/particle/event/scalar_pt_relative_abs_error": 0.13734276592731476, "metrics/particle/event/scalar_pt_response_mean": 1.032128095626831, "metrics/particle/event/vector_pt_closure": 0.12790130078792572, "metrics/particle/matched/delta_eta_abs_mean": 0.033512797206640244, "metrics/particle/matched/delta_phi_abs_mean": 0.021976960822939873, "metrics/particle/matched/delta_r_mean": 0.04454226791858673, "metrics/particle/matched/energy_relative_abs_mean": 0.20722749829292297, "metrics/particle/matched/pid_accuracy": 0.8130853772163391, "metrics/particle/matched/pt_relative_abs_mean": 0.2045409232378006, "metrics/particle/matching/duplicate_fraction": 0.11633146554231644, "metrics/particle/matching/efficiency": 0.47123271226882935, "metrics/particle/matching/f1": 0.4856579899787903, "metrics/particle/matching/purity": 0.5009943246841431, "metrics/particle/matching/target_coverage_dr0p05": 0.3531135320663452, "metrics/particle/matching/target_coverage_dr0p10": 0.5652194023132324, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.2895343005657196, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5617547631263733, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.042883966118097305, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20464135706424713, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.20351150631904602, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.02098071202635765, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.24426713585853577, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.02060580998659134, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.3956419229507446, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.006719980854541063, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.1310861110687256, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 4853.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.7580124735832214, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.304180383682251, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.3075501918792725, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.1000852584838867, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.3860529661178589, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.724631309509277, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.034707389771938324, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 2.0466794967651367, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06487395614385605, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.576792240142822, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9364.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.450519859790802, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.4232311248779297, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 15.956372261047363, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.762887954711914, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.29861119389533997, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.0526230335235596, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 13.217147827148438, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.12496523559093475, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 21.745826721191406, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.042670272290706635, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.2003295123577118, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1943143755197525, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.003120855428278446, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.23674441874027252, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0458715595304966, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.0759191513061523, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0974268764257431, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.762266159057617, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 872.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.2187548875808716, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.2491257190704346, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 7.332919597625732, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 7.138949394226074, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.47715988755226135, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 42.863494873046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08689175546169281, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.008126735687256, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2483859360218048, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.44676685333252, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2014.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.31617113947868347, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -15.532737731933594, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 16.426937103271484, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.4575650691986084, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.11015179008245468, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.11701168119907379, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 16.150985717773438, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.17543092370033264, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 36.46147537231445, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.04591291770339012, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20951256155967712, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.20696930587291718, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.031300853937864304, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.24714446067810059, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0024252922739833593, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.46806401014328003, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03762262687087059, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.1558841466903687, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 11545.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.663682758808136, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.3485965728759766, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.8532190322875977, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.629697561264038, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.3755298852920532, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.634303092956543, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.012807792983949184, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0980228185653687, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.05392781272530556, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.999559164047241, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22174.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.49615368247032166, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.202226638793945, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 19.73984146118164, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 16.489404678344727, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.6303539276123047, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -3.4727258682250977, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.869997024536133, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.9305098652839661, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 14.07247543334961, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.044898029416799545, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20768004655838013, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.20535865426063538, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.026662789285182953, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24582301080226898, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00972785148769617, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.8099045157432556, "metrics/particle/momentum/all/accepted/pz_bias": 0.022119808942079544, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.1893370151519775, "metrics/particle/momentum/all/accepted/samples": 17270.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.7233279347419739, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.991114377975464, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.2489173412323, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.031651020050049, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.3845672309398651, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 16.621965408325195, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.023366713896393776, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.5374654531478882, "metrics/particle/momentum/all/assigned/pz_bias": 0.038836006075143814, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.433671951293945, "metrics/particle/momentum/all/assigned/samples": 33552.0, "metrics/particle/momentum/all/efficiency": 0.46930623054504395, "metrics/particle/momentum/all/event/energy_bias": -8.907279968261719, "metrics/particle/momentum/all/event/p3_error_mean": 37.83292770385742, "metrics/particle/momentum/all/event/pt_vector_error_mean": 19.331552505493164, "metrics/particle/momentum/all/event/px_bias": -1.0391169786453247, "metrics/particle/momentum/all/event/py_bias": -4.642360687255859, "metrics/particle/momentum/all/event/pz_abs_mean": 27.520349502563477, "metrics/particle/momentum/all/event/pz_bias": -0.9809755086898804, "metrics/particle/momentum/all/event/pz_residual_rms": 42.39659881591797, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.046442143619060516, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.22248193621635437, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.22119982540607452, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.03983813896775246, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.26059386134147644, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.012224656529724598, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.9768688082695007, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.016586046665906906, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.4351565837860107, "metrics/particle/momentum/class_1/accepted/samples": 8671.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.5993657112121582, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.9079560041427612, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.1636407375335693, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.9136983156204224, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.3492533564567566, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.78537368774414, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.022892756387591362, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.5510427951812744, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.03903988376259804, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.9040772914886475, "metrics/particle/momentum/class_1/assigned/samples": 16206.0, "metrics/particle/momentum/class_1/efficiency": 0.48875486850738525, "metrics/particle/momentum/class_1/event/energy_bias": 21.58373260498047, "metrics/particle/momentum/class_1/event/p3_error_mean": 28.147037506103516, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 17.10465431213379, "metrics/particle/momentum/class_1/event/px_bias": -1.7790571451187134, "metrics/particle/momentum/class_1/event/py_bias": -3.009603977203369, "metrics/particle/momentum/class_1/event/pz_abs_mean": 18.674644470214844, "metrics/particle/momentum/class_1/event/pz_bias": 0.7300556302070618, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.975988388061523, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04817607253789902, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2439958155155182, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23805373907089233, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.1001853495836258, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27516213059425354, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0317460335791111, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7085083723068237, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.0492386668920517, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.784335136413574, "metrics/particle/momentum/class_2/accepted/samples": 1071.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8319859504699707, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8076584339141846, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.2902982234954834, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8598242402076721, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3625498116016388, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.492057800292969, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05887560918927193, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.898542881011963, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.024355486035346985, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.066503524780273, "metrics/particle/momentum/class_2/assigned/samples": 2259.0, "metrics/particle/momentum/class_2/efficiency": 0.3586738109588623, "metrics/particle/momentum/class_2/event/energy_bias": -33.45219039916992, "metrics/particle/momentum/class_2/event/p3_error_mean": 28.20559310913086, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.360834121704102, "metrics/particle/momentum/class_2/event/px_bias": -0.940573513507843, "metrics/particle/momentum/class_2/event/py_bias": -0.6954209208488464, "metrics/particle/momentum/class_2/event/pz_abs_mean": 20.652381896972656, "metrics/particle/momentum/class_2/event/pz_bias": -3.2443532943725586, "metrics/particle/momentum/class_2/event/pz_residual_rms": 33.4193229675293, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.04274167865514755, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.184735968708992, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.18141430616378784, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.00012673655874095857, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.22166676819324493, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.003018246730789542, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4536302387714386, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.026307502761483192, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.375929594039917, "metrics/particle/momentum/class_3/accepted/samples": 7289.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.842221736907959, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.424563884735107, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.691908359527588, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.547431468963623, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.42533066868782043, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.13820457458496, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.017476527020335197, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.2718955278396606, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04644656553864479, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.336838245391846, "metrics/particle/momentum/class_3/assigned/samples": 14591.0, "metrics/particle/momentum/class_3/efficiency": 0.46925899386405945, "metrics/particle/momentum/class_3/event/energy_bias": 8.919266700744629, "metrics/particle/momentum/class_3/event/p3_error_mean": 23.305370330810547, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 13.563915252685547, "metrics/particle/momentum/class_3/event/px_bias": 1.6106876134872437, "metrics/particle/momentum/class_3/event/py_bias": 0.04294419288635254, "metrics/particle/momentum/class_3/event/pz_abs_mean": 16.033288955688477, "metrics/particle/momentum/class_3/event/pz_bias": 1.0504546165466309, "metrics/particle/momentum/class_3/event/pz_residual_rms": 29.67588233947754, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.039545364677906036, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.19493161141872406, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.20380954444408417, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.004075455013662577, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.24440547823905945, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01840490847826004, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.436337947845459, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.11194398999214172, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.6622118949890137, "metrics/particle/momentum/class_4/accepted/samples": 163.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.81259685754776, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 8.15648078918457, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 6.797970771789551, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 6.645762920379639, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.449438214302063, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.429779052734375, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.019662922248244286, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.8065063953399658, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.2570555508136749, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.548532962799072, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.4346666634082794, "metrics/particle/momentum/class_4/event/energy_bias": -1.8457117080688477, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.310580253601074, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.199941635131836, "metrics/particle/momentum/class_4/event/px_bias": 0.2694467306137085, "metrics/particle/momentum/class_4/event/py_bias": -0.368083119392395, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.6128387451171875, "metrics/particle/momentum/class_4/event/pz_bias": 0.26000598073005676, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.394917011260986, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.04082377254962921, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.23498976230621338, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.23704040050506592, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.09831388294696808, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2759595811367035, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03947368264198303, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9233372211456299, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.1572146713733673, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.746816635131836, "metrics/particle/momentum/class_5/accepted/samples": 76.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.7013174891471863, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 1.0751333236694336, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.0659337043762207, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.31970134377479553, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.41428571939468384, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.004534721374512, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.12857143580913544, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.997735977172852, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.9941024780273438, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 11.319465637207031, "metrics/particle/momentum/class_5/assigned/samples": 140.0, "metrics/particle/momentum/class_5/efficiency": 0.46341463923454285, "metrics/particle/momentum/class_5/event/energy_bias": -4.1123785972595215, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.641876220703125, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.55098557472229, "metrics/particle/momentum/class_5/event/px_bias": -0.1996205896139145, "metrics/particle/momentum/class_5/event/py_bias": -0.6121968030929565, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.41143798828125, "metrics/particle/momentum/class_5/event/pz_bias": 0.22286182641983032, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.326637268066406, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.7296294569969177, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.7336015105247498, "metrics/set_slot/angular_recall_all_dr0p10": 0.8226139545440674, "metrics/set_slot/angular_recall_selected_dr0p10": 0.5909367203712463, "metrics/set_slot/soft_count_bias_mean": 1.608831524848938}, "step_train_time": 6.247288942337036, "step_valid_time": 5800.298593044281, "step_total_time": 5806.545882463455, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.115354458361734, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.315609487203514, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2956, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2478, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2241, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8100686498855835, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8382949932341002, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8239401496259352, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.16170500676589986, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7325923504413206, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7581190798376184, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7451371571072319, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8100686498855835, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0543827110465522, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2754276949209651, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2238, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1870, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1780, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8183807439824945, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8355674709562109, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.826884810966173, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1644325290437891, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7789934354485777, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7953529937444147, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7870882157859828, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8183807439824945, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.042706399713773, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28851564666220453, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1243, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 998, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 923, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7549167927382754, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8028962188254224, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7781676413255361, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.19710378117457764, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6981845688350984, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7425583266291231, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7196881091617934, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7549167927382754} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_20000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..828d81ecf5ff512a0957f60be3019a3f455d3073 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.9473285335354301, "train": {"Classification_binary": 0.16036124527454376, "Cardinality": 0.10599350929260254, "Classification": 0.09403322637081146, "Regression_pt": 0.13542801141738892, "Regression_eta": 0.0015280351508408785, "Regression_sin_phi": 0.0008765873499214649, "Regression_cos_phi": 0.0008575548999942839, "Regression_energy": 0.13441456854343414, "HitGrouping": 0.1656055897474289, "Auxiliary": 0.20396199822425842, "Total": 1.0304163694381714, "Total_unweighted": 0.5274992290069349}, "train_group": {}, "valid": {"Classification_binary": 0.18450470765431723, "Cardinality": 0.1376977562904358, "Classification": 0.09897994995117188, "Regression_pt": 0.14031511545181274, "Regression_eta": 0.0017732647247612476, "Regression_sin_phi": 0.000965560165544351, "Regression_cos_phi": 0.000960352054486672, "Regression_energy": 0.13907418648401895, "HitGrouping": 0.16454928119977316, "Auxiliary": 0.2224626143773397, "Total": 1.1349151134490967, "Total_unweighted": 0.5665731364861131, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8426346182823181, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5816848278045654, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.31768715381622314, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5924686193466187, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5921505689620972, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.35279643535614014, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7891101837158203, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5270211100578308, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3002215623855591, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.537770688533783, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5376683473587036, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4283604025840759, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8605031967163086, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5525975227355957, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3065031170845032, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5589686632156372, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5498149394989014, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3207128047943115, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8495469689369202, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5594239234924316, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.309403657913208, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5674718618392944, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5616976022720337, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3387901484966278, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0034066541120409966, "metrics/hit_grouping/calorimeter/background_recall": 0.14722628891468048, "metrics/hit_grouping/calorimeter/completeness": 0.6209818720817566, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6232178807258606, "metrics/hit_grouping/calorimeter/energy_purity": 0.61517333984375, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3550044894218445, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8739246129989624, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5853551030158997, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.23537778854370117, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.596175491809845, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5841080546379089, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2769117057323456, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8524723649024963, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5822356939315796, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.33152925968170166, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5907345414161682, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5828989744186401, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3412021994590759, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7968948483467102, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4835697412490845, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4380510449409485, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.4855101704597473, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.49464449286460876, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4625290036201477, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8306082487106323, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5910249352455139, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3731657564640045, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6024423241615295, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.609453022480011, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3901250958442688, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8129682540893555, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6334298849105835, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4976840019226074, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6314831972122192, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.630108654499054, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.39732372760772705, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8732142448425293, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5183879137039185, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2219633311033249, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5241912007331848, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5048578381538391, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.279477596282959, "metrics/hit_grouping/calorimeter/purity": 0.6120958924293518, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7868219614028931, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.11581093817949295, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.09398496150970459, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.11533074826002121, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0824572965502739, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8381235003471375, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.30923134088516235, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11771630495786667, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.30800649523735046, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2892034947872162, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.21836374700069427, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8511636853218079, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5796719789505005, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.32288891077041626, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5883413553237915, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5836867690086365, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3501485586166382, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7118943929672241, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.42199042439460754, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.47863104939460754, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5144765973091125, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.622627854347229, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7649890184402466, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2875226140022278, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4722234606742859, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4827711880207062, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.44605183601379395, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.630652666091919, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44567564129829407, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.38365453481674194, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.426632285118103, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7187365293502808, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6655447483062744, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.426714152097702, "metrics/hit_grouping/tracker/all/object_completeness": 0.4193873405456543, "metrics/hit_grouping/tracker/all/object_purity": 0.45777758955955505, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6690561175346375, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01124788261950016, "metrics/hit_grouping/tracker/background_recall": 0.10447584092617035, "metrics/hit_grouping/tracker/completeness": 0.4184419810771942, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4847269356250763, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6947261691093445, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.410836398601532, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4916904866695404, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5183786749839783, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6424688696861267, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6487922072410583, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4423670470714569, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.39103832840919495, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.44019344449043274, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6912325024604797, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6221623420715332, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.44382327795028687, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2988257110118866, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3505919575691223, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7017132639884949, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6674615144729614, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.45002973079681396, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4246842563152313, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4709728956222534, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6727519631385803, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6660962104797363, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4981527030467987, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.41024306416511536, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4404546916484833, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6594827771186829, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6629539728164673, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38359642028808594, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.414140522480011, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4437287449836731, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6659675240516663, "metrics/hit_grouping/tracker/purity": 0.39481836557388306, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.88724285364151, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.047888386994600296, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1127079427242279, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.10727592557668686, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.059200603514909744, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6629619002342224, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.29220443964004517, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.32060205936431885, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.322496622800827, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5601445436477661, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.632925271987915, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4975099265575409, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.47560474276542664, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5244537591934204, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7713614106178284, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5632888674736023, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5948388576507568, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6731774210929871, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6292197108268738, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.014136137440800667, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.05474487692117691, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.51249885559082, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2944197952747345, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20279663801193237, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8427165746688843, "metrics/particle/class_1/pid_accuracy": 0.9646049737930298, "metrics/particle/class_2/efficiency": 0.5587269067764282, "metrics/particle/class_2/pid_accuracy": 0.7166024446487427, "metrics/particle/class_3/efficiency": 0.7637889981269836, "metrics/particle/class_3/pid_accuracy": 0.9660541415214539, "metrics/particle/class_4/efficiency": 0.6823647022247314, "metrics/particle/class_4/pid_accuracy": 0.39794421195983887, "metrics/particle/class_5/efficiency": 0.8927202820777893, "metrics/particle/class_5/pid_accuracy": 0.5171673893928528, "metrics/particle/count/bias_mean": -0.9524739384651184, "metrics/particle/count/mae": 4.188150882720947, "metrics/particle/count/prediction_mean": 69.70247650146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07754939794540405, "metrics/particle/event/energy_response_mean": 0.968818187713623, "metrics/particle/event/met_abs_error": 6.455570220947266, "metrics/particle/event/scalar_pt_relative_abs_error": 0.060114964842796326, "metrics/particle/event/scalar_pt_response_mean": 0.9794442057609558, "metrics/particle/event/vector_pt_closure": 0.0619579516351223, "metrics/particle/matched/delta_eta_abs_mean": 0.015096058137714863, "metrics/particle/matched/delta_phi_abs_mean": 0.0157779511064291, "metrics/particle/matched/delta_r_mean": 0.024468829855322838, "metrics/particle/matched/energy_relative_abs_mean": 0.14866089820861816, "metrics/particle/matched/pid_accuracy": 0.9438648223876953, "metrics/particle/matched/pt_relative_abs_mean": 0.1473299115896225, "metrics/particle/matching/duplicate_fraction": 0.0689874216914177, "metrics/particle/matching/efficiency": 0.7851114273071289, "metrics/particle/matching/f1": 0.7904391884803772, "metrics/particle/matching/purity": 0.7958398461341858, "metrics/particle/matching/target_coverage_dr0p05": 0.7195141911506653, "metrics/particle/matching/target_coverage_dr0p10": 0.8318006992340088, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.58790522813797, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8290547728538513, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.0235001128166914, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1437530368566513, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14359939098358154, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015136782079935074, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1824481189250946, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.008127208799123764, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7332722544670105, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.033359345048666, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.037893772125244, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8490.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25127044320106506, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8240363597869873, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.980811357498169, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8316249847412109, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13330093026161194, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.0420241355896, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013903743587434292, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9919479489326477, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.04072018712759018, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.739433765411377, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10285.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7881544828414917, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.0661088153719902, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.28563117980957, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.8456668853759766, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.6494169235229492, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.34948036074638367, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.994038105010986, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.5960279703140259, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.74246597290039, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024814223870635033, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15052685141563416, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15044079720973969, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017710335552692413, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19083957374095917, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02300884947180748, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3239383697509766, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.20271338522434235, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4060966968536377, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1695.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5253156423568726, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2019039392471313, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.6488828659057617, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.477963924407959, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21617020666599274, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 45.42570495605469, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05446808412671089, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4803760051727295, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.47135236859321594, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.708338737487793, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2350.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6145758032798767, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.6681623458862305, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.538826942443848, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.23880136013031, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03838864713907242, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.0851236879825592, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.377206802368164, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.453019380569458, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.566438674926758, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.025112813338637352, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15050968527793884, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14812137186527252, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.001974172191694379, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18706321716308594, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009641135693527758, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.25907638669013977, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.007391743361949921, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8897969722747803, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18670.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23637956380844116, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1470823287963867, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9376966953277588, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.757928729057312, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13633377850055695, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.5179243087768555, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004071625415235758, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.43458980321884155, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.01074266154319048, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5536022186279297, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22841.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8023550510406494, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.36946994066238403, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.755990028381348, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.182221412658691, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.5492451190948486, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.076463222503662, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.4200358390808105, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.7435954809188843, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.363396644592285, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024620769545435905, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14852267503738403, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1469271332025528, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.004216686822474003, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18594278395175934, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0043666609562933445, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.46115103363990784, "metrics/particle/momentum/all/accepted/pz_bias": -0.026505764573812485, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5542559623718262, "metrics/particle/momentum/all/accepted/samples": 28855.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2598363161087036, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0570580959320068, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1297905445098877, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9594749212265015, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14074303209781647, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.691469192504883, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01026045810431242, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7316928505897522, "metrics/particle/momentum/all/assigned/pz_bias": -0.049945272505283356, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7686984539031982, "metrics/particle/momentum/all/assigned/samples": 35476.0, "metrics/particle/momentum/all/efficiency": 0.7841245532035828, "metrics/particle/momentum/all/event/energy_bias": -7.10374116897583, "metrics/particle/momentum/all/event/p3_error_mean": 19.093645095825195, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.02905559539795, "metrics/particle/momentum/all/event/px_bias": -2.237050771713257, "metrics/particle/momentum/all/event/py_bias": -2.5110671520233154, "metrics/particle/momentum/all/event/pz_abs_mean": 13.660882949829102, "metrics/particle/momentum/all/event/pz_bias": -4.792643070220947, "metrics/particle/momentum/all/event/pz_residual_rms": 23.661535263061523, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.026442797854542732, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13863837718963623, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13671208918094635, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01241073478013277, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.175095334649086, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0033527794294059277, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.47827109694480896, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03176450729370117, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4042773246765137, "metrics/particle/momentum/class_1/accepted/samples": 14913.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.18405835330486298, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.46053236722946167, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5241720080375671, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3692653477191925, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09771441668272018, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.817752361297607, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007503174245357513, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6673741936683655, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.05233774334192276, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.457925796508789, "metrics/particle/momentum/class_1/assigned/samples": 17326.0, "metrics/particle/momentum/class_1/efficiency": 0.8405952453613281, "metrics/particle/momentum/class_1/event/energy_bias": 6.510907173156738, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.067909240722656, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.315299034118652, "metrics/particle/momentum/class_1/event/px_bias": -2.7556591033935547, "metrics/particle/momentum/class_1/event/py_bias": -1.2829551696777344, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.464971542358398, "metrics/particle/momentum/class_1/event/pz_bias": -1.5817286968231201, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.955434799194336, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029544560238718987, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18870896100997925, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1921929121017456, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.026498857885599136, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22981663048267365, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029841655865311623, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4904760122299194, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.12231311202049255, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6654562950134277, "metrics/particle/momentum/class_2/accepted/samples": 1642.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.543674886226654, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.586371123790741, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8763261437416077, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5336535573005676, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.25457438826560974, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.756857872009277, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05011933296918869, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.2445473670959473, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.19977335631847382, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.472716331481934, "metrics/particle/momentum/class_2/assigned/samples": 2514.0, "metrics/particle/momentum/class_2/efficiency": 0.5498995184898376, "metrics/particle/momentum/class_2/event/energy_bias": -11.445098876953125, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.02886962890625, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.903953552246094, "metrics/particle/momentum/class_2/event/px_bias": 0.0015939995646476746, "metrics/particle/momentum/class_2/event/py_bias": -0.9124903082847595, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.827214241027832, "metrics/particle/momentum/class_2/event/pz_bias": -2.305736541748047, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.52069854736328, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021717678755521774, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1550287902355194, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1531318575143814, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.002062614541500807, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1919303685426712, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0017647058703005314, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2754640579223633, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.011000561527907848, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1418945789337158, "metrics/particle/momentum/class_3/accepted/samples": 11900.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.29787206649780273, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7561988830566406, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8112365007400513, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6509714126586914, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16905975341796875, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.918333053588867, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006418315228074789, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5335538983345032, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.029455440118908882, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.381232976913452, "metrics/particle/momentum/class_3/assigned/samples": 15113.0, "metrics/particle/momentum/class_3/efficiency": 0.7661108374595642, "metrics/particle/momentum/class_3/event/energy_bias": -0.6073768734931946, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.930949211120605, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.4989094734191895, "metrics/particle/momentum/class_3/event/px_bias": 0.02385101094841957, "metrics/particle/momentum/class_3/event/py_bias": 0.059497296810150146, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.044673442840576, "metrics/particle/momentum/class_3/event/pz_bias": -0.959731936454773, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.564315795898438, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025106817483901978, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16066502034664154, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15866263210773468, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.032341208308935165, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1989249587059021, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007843137718737125, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.8417088985443115, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.05252194032073021, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.8853033781051636, "metrics/particle/momentum/class_4/accepted/samples": 255.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.41408079862594604, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.005412578582764, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.7826755046844482, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.6687796115875244, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2548476457595825, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 23.221128463745117, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008310249075293541, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9553602337837219, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.16607043147087097, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.911522626876831, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.6800000071525574, "metrics/particle/momentum/class_4/event/energy_bias": -0.8794649243354797, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.6641297340393066, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9468492269515991, "metrics/particle/momentum/class_4/event/px_bias": 0.20616009831428528, "metrics/particle/momentum/class_4/event/py_bias": -0.11156859993934631, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4200795888900757, "metrics/particle/momentum/class_4/event/pz_bias": -0.020156435668468475, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.033900737762451, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01886962167918682, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1547277271747589, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15507368743419647, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.024824541062116623, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1906026303768158, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.027586206793785095, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.614036202430725, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.1878059357404709, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.535780906677246, "metrics/particle/momentum/class_5/accepted/samples": 145.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06750212609767914, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.36720120906829834, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3505954444408417, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.14363493025302887, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.08024691045284271, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.8842450380325317, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04938271641731262, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.1193249225616455, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.13817395269870758, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.461030006408691, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.8841463327407837, "metrics/particle/momentum/class_5/event/energy_bias": -0.6827077865600586, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9756559133529663, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5005569458007812, "metrics/particle/momentum/class_5/event/px_bias": 0.2870030701160431, "metrics/particle/momentum/class_5/event/py_bias": -0.2635505199432373, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9887222647666931, "metrics/particle/momentum/class_5/event/pz_bias": 0.0747108981013298, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3242204189300537, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.38431647419929504, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3848026692867279, "metrics/set_slot/angular_recall_all_dr0p10": 0.9421152472496033, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8183661103248596, "metrics/set_slot/soft_count_bias_mean": 0.3956109285354614}, "step_train_time": 5.658618450164795, "step_valid_time": 60412.12830853462, "step_total_time": 60417.786927223206, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9917178774192623, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16035717920908188, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3056, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2889, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2872, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.944426283099052, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9453534031413613, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9448896156991006, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05464659685863874, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9388689114089572, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9397905759162304, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9393295175797219, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.944426283099052, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.982882287515148, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1443747371377898, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2267, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2158, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2152, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9444201312910284, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9519188354653727, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9481546572934973, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04808116453462726, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9417943107221006, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.949272165857962, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.945518453427065, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9444201312910284, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9899698787901589, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1489402477782228, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1312, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1213, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1206, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9175491679273827, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9245426829268293, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9210326499620349, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07545731707317073, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9122541603630863, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9192073170731707, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9157175398633257, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9175491679273827} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_21000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..3066e740dab38107256b879fa22fc3c67ede5c77 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 0.9946949602122016, "train": {"Classification_binary": 0.1572606861591339, "Cardinality": 0.1267452985048294, "Classification": 0.090022012591362, "Regression_pt": 0.1427820324897766, "Regression_eta": 0.0016233989736065269, "Regression_sin_phi": 0.0007835555588826537, "Regression_cos_phi": 0.0008420739904977381, "Regression_energy": 0.1405344158411026, "HitGrouping": 0.16837452352046967, "Auxiliary": 0.20469561219215393, "Total": 1.0503450632095337, "Total_unweighted": 0.533848175604362}, "train_group": {}, "valid": {"Classification_binary": 0.1850876808166504, "Cardinality": 0.13961308201154074, "Classification": 0.09658432006835938, "Regression_pt": 0.13923107584317526, "Regression_eta": 0.0017634305792550247, "Regression_sin_phi": 0.0009893810686965783, "Regression_cos_phi": 0.0009436643061538538, "Regression_energy": 0.13729540506998697, "HitGrouping": 0.16523065169652304, "Auxiliary": 0.22259920835494995, "Total": 1.1354704697926838, "Total_unweighted": 0.5618949577522775, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8444394469261169, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5882200598716736, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.323365181684494, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5980120301246643, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5968149900436401, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.35383743047714233, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7843837738037109, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5185891389846802, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.30871492624282837, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5249930620193481, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5293430685997009, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4420236349105835, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8594093322753906, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5593328475952148, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.29761630296707153, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5654408931732178, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5619827508926392, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.323536217212677, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8490493893623352, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5649210810661316, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.30627867579460144, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5721681118011475, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5700012445449829, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3419151306152344, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0034746327437460423, "metrics/hit_grouping/calorimeter/background_recall": 0.1481001228094101, "metrics/hit_grouping/calorimeter/completeness": 0.6183760762214661, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6163983941078186, "metrics/hit_grouping/calorimeter/energy_purity": 0.6084507703781128, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3466566801071167, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.874454140663147, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.594391942024231, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2337278127670288, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6042648553848267, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5944812297821045, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2794719934463501, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8497656583786011, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5874845385551453, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3278443217277527, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5952319502830505, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5917294025421143, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.34477198123931885, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7965279817581177, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4820995628833771, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.43248260021209717, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.48348861932754517, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.49819737672805786, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4663573205471039, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8294091820716858, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5964481830596924, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3638441264629364, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6071617007255554, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6184219121932983, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3916887044906616, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.804518461227417, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6231780648231506, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4853319525718689, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6241419911384583, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6334922313690186, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4292331337928772, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8743789196014404, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5258367657661438, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22625544667243958, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.530298113822937, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5130693912506104, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28076523542404175, "metrics/hit_grouping/calorimeter/purity": 0.6095689535140991, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7905678749084473, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.10382802039384842, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0827067643404007, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1033610850572586, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.07976875454187393, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02255639061331749, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8500564098358154, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.35252678394317627, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.13831666111946106, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3484073281288147, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.3235567808151245, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.20600353181362152, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8499499559402466, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5834870934486389, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3186621367931366, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5914512872695923, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5908138751983643, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.35419154167175293, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.710269033908844, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4241634011268616, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4854448139667511, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.521205484867096, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6158192157745361, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7562988996505737, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2857142984867096, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.46748945116996765, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.47841116786003113, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4737793803215027, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.62065589427948, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43848034739494324, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3823500871658325, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4353218674659729, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7273708581924438, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6582173705101013, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4227960705757141, "metrics/hit_grouping/tracker/all/object_completeness": 0.4203246831893921, "metrics/hit_grouping/tracker/all/object_purity": 0.4649006724357605, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6743544340133667, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01122141070663929, "metrics/hit_grouping/tracker/background_recall": 0.09946321696043015, "metrics/hit_grouping/tracker/completeness": 0.41906338930130005, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4664504826068878, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6856545209884644, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40469929575920105, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.48986363410949707, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5301368832588196, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6492196917533875, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.640433669090271, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4423670470714569, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3938606083393097, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4464063346385956, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6968596577644348, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6194478869438171, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.44057708978652954, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.30357593297958374, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3490848243236542, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7036970257759094, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6634594798088074, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44875603914260864, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43179360032081604, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4848725199699402, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6722425222396851, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6566119194030762, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4932265877723694, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4110030233860016, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4204292595386505, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6761083602905273, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6516903042793274, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3764212429523468, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.40708571672439575, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.44690898060798645, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6767855286598206, "metrics/hit_grouping/tracker/purity": 0.39539408683776855, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8844905495643616, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04411764815449715, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11867831647396088, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11623896658420563, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07013574987649918, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6695997714996338, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2798141539096832, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3323066234588623, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.33257681131362915, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.533298909664154, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6234054565429688, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.49448826909065247, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.47462812066078186, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5309832692146301, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7793072462081909, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5534745454788208, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6046088933944702, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6877856254577637, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6355755925178528, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.004321803338825703, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06110077723860741, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.414976119995117, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29213374853134155, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20051059126853943, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8509173393249512, "metrics/particle/class_1/pid_accuracy": 0.9679046273231506, "metrics/particle/class_2/efficiency": 0.5290519595146179, "metrics/particle/class_2/pid_accuracy": 0.7092699408531189, "metrics/particle/class_3/efficiency": 0.7273462414741516, "metrics/particle/class_3/pid_accuracy": 0.9630609154701233, "metrics/particle/class_4/efficiency": 0.6593186259269714, "metrics/particle/class_4/pid_accuracy": 0.42553192377090454, "metrics/particle/class_5/efficiency": 0.8544061183929443, "metrics/particle/class_5/pid_accuracy": 0.5269058346748352, "metrics/particle/count/bias_mean": -1.044921875, "metrics/particle/count/mae": 3.9694011211395264, "metrics/particle/count/prediction_mean": 69.61002349853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08097674697637558, "metrics/particle/event/energy_response_mean": 1.0207024812698364, "metrics/particle/event/met_abs_error": 7.856590747833252, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06413675099611282, "metrics/particle/event/scalar_pt_response_mean": 1.0304967164993286, "metrics/particle/event/vector_pt_closure": 0.06549376994371414, "metrics/particle/matched/delta_eta_abs_mean": 0.014828557148575783, "metrics/particle/matched/delta_phi_abs_mean": 0.015595637261867523, "metrics/particle/matched/delta_r_mean": 0.024189261719584465, "metrics/particle/matched/energy_relative_abs_mean": 0.14617057144641876, "metrics/particle/matched/pid_accuracy": 0.9449219107627869, "metrics/particle/matched/pt_relative_abs_mean": 0.14794191718101501, "metrics/particle/matching/duplicate_fraction": 0.06639481335878372, "metrics/particle/matching/efficiency": 0.7709028124809265, "metrics/particle/matching/f1": 0.7766457796096802, "metrics/particle/matching/purity": 0.782474935054779, "metrics/particle/matching/target_coverage_dr0p05": 0.7188692092895508, "metrics/particle/matching/target_coverage_dr0p10": 0.8330630660057068, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5750142931938171, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8127360939979553, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023356452584266663, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14586211740970612, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14608561992645264, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03290028125047684, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18620562553405762, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007242003455758095, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7212017774581909, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.06414008885622025, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9057033061981201, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8285.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2680380046367645, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9463770985603333, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.081551432609558, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9446629285812378, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15407033264636993, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.8397111892700195, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015543035231530666, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0454730987548828, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.03826990723609924, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.842379093170166, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10294.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7691236734390259, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 4.447334289550781, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.196662902832031, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.8712453842163086, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.42885592579841614, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.8221919536590576, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.8676323890686035, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.164929747581482, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.359576225280762, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025740984827280045, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16182389855384827, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.16150055825710297, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04168887063860893, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20435766875743866, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02626865729689598, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4862343072891235, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.12963181734085083, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.9874603748321533, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1675.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6077719926834106, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.965976595878601, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 7.317205429077148, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 7.155939102172852, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24708333611488342, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 144.75640869140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06291666626930237, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7682254314422607, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.26280686259269714, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.212505340576172, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2400.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6073241233825684, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.3704113960266113, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.395187377929688, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2149598598480225, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.13189493119716644, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.13243982195854187, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.23965072631836, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.7423977851867676, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.494020462036133, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02462155744433403, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14436373114585876, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1467398703098297, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.03012731298804283, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18718211352825165, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007124849362298846, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.24611984193325043, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0059508089907467365, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7167561054229736, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18246.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23976394534111023, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0833665132522583, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8862816691398621, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7344902753829956, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.15387991070747375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.975013256072998, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0043451543897390366, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.42705103754997253, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.011263088323175907, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.637900710105896, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22784.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7841333746910095, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.943207263946533, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.673739433288574, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.04494857788086, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.1813654899597168, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.2618494033813477, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.767848491668701, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.1593858152627945, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.961276054382324, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02431643195450306, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14584070444107056, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14742426574230194, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.03162839636206627, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1879611760377884, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0041480534709990025, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4593101441860199, "metrics/particle/momentum/all/accepted/pz_bias": -0.02268863283097744, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.530777931213379, "metrics/particle/momentum/all/accepted/samples": 28206.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2728625535964966, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1033251285552979, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.3779758214950562, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.2298675775527954, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16024014353752136, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 38.4212532043457, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011556457728147507, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7648619413375854, "metrics/particle/momentum/all/assigned/pz_bias": -0.021649155765771866, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9376044273376465, "metrics/particle/momentum/all/assigned/samples": 35478.0, "metrics/particle/momentum/all/efficiency": 0.7664881944656372, "metrics/particle/momentum/all/event/energy_bias": 9.020130157470703, "metrics/particle/momentum/all/event/p3_error_mean": 20.389259338378906, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.888649940490723, "metrics/particle/momentum/all/event/px_bias": 1.7421163320541382, "metrics/particle/momentum/all/event/py_bias": -3.2164812088012695, "metrics/particle/momentum/all/event/pz_abs_mean": 14.415011405944824, "metrics/particle/momentum/all/event/pz_bias": -3.066713333129883, "metrics/particle/momentum/all/event/pz_residual_rms": 25.39330291748047, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.026107246056199074, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13263222575187683, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13416731357574463, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.010620877146720886, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17270447313785553, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0034049940295517445, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.47443023324012756, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03698357194662094, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.4085304737091064, "metrics/particle/momentum/class_1/accepted/samples": 14978.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.18099406361579895, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.45105698704719543, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5182463526725769, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.36549803614616394, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09528753161430359, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.232522964477539, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008190575055778027, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6867955327033997, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.036865103989839554, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.559908151626587, "metrics/particle/momentum/class_1/assigned/samples": 17337.0, "metrics/particle/momentum/class_1/efficiency": 0.8442590832710266, "metrics/particle/momentum/class_1/event/energy_bias": 9.138592720031738, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.82240104675293, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.753711700439453, "metrics/particle/momentum/class_1/event/px_bias": 1.694639801979065, "metrics/particle/momentum/class_1/event/py_bias": -2.151958465576172, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.882863998413086, "metrics/particle/momentum/class_1/event/pz_bias": -2.1767520904541016, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.066659927368164, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.028851572424173355, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18774431943893433, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18972112238407135, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.03318747878074646, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2323581576347351, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02517753466963768, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4526668787002563, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08089368790388107, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4249720573425293, "metrics/particle/momentum/class_2/accepted/samples": 1549.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6165357232093811, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8311218023300171, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1099940538406372, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8045697212219238, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.31279250979423523, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.924169540405273, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.051872074604034424, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3330395221710205, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.09091101586818695, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.384729862213135, "metrics/particle/momentum/class_2/assigned/samples": 2564.0, "metrics/particle/momentum/class_2/efficiency": 0.5187541842460632, "metrics/particle/momentum/class_2/event/energy_bias": -4.277126312255859, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.455802917480469, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.008417129516602, "metrics/particle/momentum/class_2/event/px_bias": -0.06920177489519119, "metrics/particle/momentum/class_2/event/py_bias": -0.4941742420196533, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.267743110656738, "metrics/particle/momentum/class_2/event/pz_bias": -0.6220789551734924, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.481338500976562, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021391842514276505, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15721134841442108, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15898661315441132, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.058626074343919754, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19983235001564026, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0017711654072627425, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2823103070259094, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.004222622606903315, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1676945686340332, "metrics/particle/momentum/class_3/accepted/samples": 11292.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31662341952323914, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8469337224960327, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.380155086517334, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.262795925140381, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20702709257602692, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 58.59103775024414, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008103081956505775, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5681896805763245, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.006068098824471235, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.723562002182007, "metrics/particle/momentum/class_3/assigned/samples": 15056.0, "metrics/particle/momentum/class_3/efficiency": 0.7269684076309204, "metrics/particle/momentum/class_3/event/energy_bias": 4.238077163696289, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.597094535827637, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.9670329093933105, "metrics/particle/momentum/class_3/event/px_bias": -0.2823193669319153, "metrics/particle/momentum/class_3/event/py_bias": 0.15206484496593475, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.387922763824463, "metrics/particle/momentum/class_3/event/pz_bias": -0.1707972288131714, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.061867713928223, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025051888078451157, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15263982117176056, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14740580320358276, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05118696391582489, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19063085317611694, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00401606410741806, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6563987135887146, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.09417077153921127, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5912078619003296, "metrics/particle/momentum/class_4/accepted/samples": 249.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5025809407234192, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.6648974418640137, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.224719285964966, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.125383138656616, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.26243093609809875, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.645505905151367, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005524862091988325, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9001986384391785, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.055340562015771866, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.7732815742492676, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.6639999747276306, "metrics/particle/momentum/class_4/event/energy_bias": -0.031135141849517822, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.975679397583008, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.2367680072784424, "metrics/particle/momentum/class_4/event/px_bias": 0.11314622312784195, "metrics/particle/momentum/class_4/event/py_bias": -0.430878221988678, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.507943034172058, "metrics/particle/momentum/class_4/event/pz_bias": -0.20592470467090607, "metrics/particle/momentum/class_4/event/pz_residual_rms": 7.121338844299316, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.017023876309394836, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16640576720237732, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16544599831104279, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.04980066418647766, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2111400067806244, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.043478261679410934, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7957438230514526, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.10909717530012131, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.9756627082824707, "metrics/particle/momentum/class_5/accepted/samples": 138.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08119608461856842, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3690086305141449, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.3396381437778473, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.21156862378120422, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.11949685215950012, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.2642182111740112, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.069182388484478, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.3040826320648193, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.04551597684621811, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.84739875793457, "metrics/particle/momentum/class_5/assigned/samples": 159.0, "metrics/particle/momentum/class_5/efficiency": 0.8414633870124817, "metrics/particle/momentum/class_5/event/energy_bias": -0.04827757179737091, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.112800121307373, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.6189392805099487, "metrics/particle/momentum/class_5/event/px_bias": 0.285851389169693, "metrics/particle/momentum/class_5/event/py_bias": -0.2915349006652832, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.044830322265625, "metrics/particle/momentum/class_5/event/pz_bias": 0.1088399812579155, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.5322153568267822, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.38310959935188293, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3875977098941803, "metrics/set_slot/angular_recall_all_dr0p10": 0.9471371173858643, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8201260566711426, "metrics/set_slot/soft_count_bias_mean": 0.2324865311384201}, "step_train_time": 5.493429899215698, "step_valid_time": 63377.55170249939, "step_total_time": 63383.045132637024, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0345161333574513, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15403118193469567, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3047, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2895, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2864, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9463877084014384, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9501148670823761, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9482476252866033, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04988513291762389, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9362536776724419, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9399409255004922, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9380936783491648, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9463877084014384, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0305395569591744, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15463726383114318, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2275, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2141, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2129, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.936980306345733, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9410989010989012, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9390350877192982, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0589010989010989, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9317286652078774, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9358241758241759, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9337719298245614, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.936980306345733, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0344709727026495, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1499627972851919, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1313, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1229, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1225, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9296520423600605, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9360243716679361, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9328273244781784, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06397562833206398, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.926626323751891, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9329779131759329, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9297912713472486, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9296520423600605} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_22000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..0031483a7e6ec208ad3d20cb753d964798f8c161 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 1.0420613868889732, "train": {"Classification_binary": 0.15212655067443848, "Cardinality": 0.129447802901268, "Classification": 0.09554048627614975, "Regression_pt": 0.13609014451503754, "Regression_eta": 0.0016040154732763767, "Regression_sin_phi": 0.0008970591006800532, "Regression_cos_phi": 0.0009179333574138582, "Regression_energy": 0.13598981499671936, "HitGrouping": 0.16870912909507751, "Auxiliary": 0.21003679931163788, "Total": 1.0658719539642334, "Total_unweighted": 0.5231660043937154}, "train_group": {}, "valid": {"Classification_binary": 0.19944047927856445, "Cardinality": 0.1827461322148641, "Classification": 0.10878408948580424, "Regression_pt": 0.16192169984181723, "Regression_eta": 0.0017065326683223248, "Regression_sin_phi": 0.0009792563505470753, "Regression_cos_phi": 0.0009732587107767662, "Regression_energy": 0.16085265080134073, "HitGrouping": 0.17794092496236166, "Auxiliary": 0.2584379514058431, "Total": 1.2746466795603435, "Total_unweighted": 0.6346579671371728, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8420680165290833, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5588401556015015, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.31730860471725464, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5672518610954285, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5658782124519348, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3660452365875244, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7857154607772827, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.48713958263397217, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3168390095233917, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4938575327396393, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.48646223545074463, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4579025208950043, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.853900671005249, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5354259014129639, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.30886369943618774, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.540457010269165, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5313901901245117, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.34218931198120117, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8450220823287964, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5387704968452454, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3120412826538086, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5449566841125488, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5383502840995789, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.35840022563934326, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003340169321745634, "metrics/hit_grouping/calorimeter/background_recall": 0.14053162932395935, "metrics/hit_grouping/calorimeter/completeness": 0.5838747620582581, "metrics/hit_grouping/calorimeter/energy_completeness": 0.584999144077301, "metrics/hit_grouping/calorimeter/energy_purity": 0.5773892998695374, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3645257353782654, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8722327351570129, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5725457668304443, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.23839326202869415, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5819278955459595, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.569237232208252, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.28726673126220703, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8458958268165588, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5586662888526917, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.336826354265213, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5647837519645691, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5607643127441406, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3665361702442169, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7886604070663452, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4498599171638489, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.43723899126052856, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.44959893822669983, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.4527921974658966, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4952436089515686, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.823066234588623, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.5679656267166138, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3789992928504944, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.5776529908180237, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.583527147769928, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4158046543598175, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7908268570899963, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.59383225440979, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4935666620731354, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5880442261695862, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6090372204780579, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.48172929883003235, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8738641142845154, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5024443864822388, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22214728593826294, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5064875483512878, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.48386844992637634, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2851799726486206, "metrics/hit_grouping/calorimeter/purity": 0.5754813551902771, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8051580190658569, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0959911048412323, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07518796622753143, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09503670036792755, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06588221341371536, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8364167213439941, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.29702284932136536, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12183637171983719, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.29396721720695496, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2736981511116028, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.221895232796669, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8461195826530457, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5585654973983765, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3257986605167389, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5653490424156189, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5598210692405701, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.37091487646102905, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7095514535903931, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40779370069503784, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4607924520969391, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4910743236541748, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6248008012771606, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7478378415107727, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3025919198989868, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.44971993565559387, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.44478702545166016, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.46654611825942993, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6273953914642334, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4386962056159973, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.36996838450431824, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4040226638317108, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7321916818618774, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6615421175956726, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4191451370716095, "metrics/hit_grouping/tracker/all/object_completeness": 0.40377363562583923, "metrics/hit_grouping/tracker/all/object_purity": 0.4337887167930603, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6795636415481567, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.015353663824498653, "metrics/hit_grouping/tracker/background_recall": 0.12555257976055145, "metrics/hit_grouping/tracker/completeness": 0.40473124384880066, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4946872889995575, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6844952702522278, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.41320356726646423, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.4756511449813843, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5018039345741272, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6607049107551575, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.644068717956543, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43238335847854614, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.3784047067165375, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4188925623893738, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7004901170730591, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6316878199577332, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4182146191596985, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.2811267375946045, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3086817264556885, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6975653767585754, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6657356023788452, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44578415155410767, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.411203533411026, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4457877278327942, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6832809448242188, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6696758270263672, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4501231610774994, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.37717220187187195, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4050739109516144, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6539409160614014, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6546324491500854, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37896016240119934, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3988834023475647, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.42333731055259705, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6793244481086731, "metrics/hit_grouping/tracker/purity": 0.3834740221500397, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8802182674407959, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03921568766236305, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.10209721326828003, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.09671082347631454, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06410256773233414, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6684550642967224, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2829117178916931, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3156319260597229, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.30671226978302, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5420753955841064, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6283419728279114, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4902915358543396, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.458094984292984, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.4975835382938385, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.785798192024231, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5452353954315186, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.5905313491821289, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.6702567934989929, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6176486015319824, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0039173299446702, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.04317382723093033, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.33042335510254, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.26876384019851685, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1771407127380371, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8510127067565918, "metrics/particle/class_1/pid_accuracy": 0.9561427235603333, "metrics/particle/class_2/efficiency": 0.524974524974823, "metrics/particle/class_2/pid_accuracy": 0.677238404750824, "metrics/particle/class_3/efficiency": 0.7438077926635742, "metrics/particle/class_3/pid_accuracy": 0.952533483505249, "metrics/particle/class_4/efficiency": 0.6643286347389221, "metrics/particle/class_4/pid_accuracy": 0.3529411852359772, "metrics/particle/class_5/efficiency": 0.8735632300376892, "metrics/particle/class_5/pid_accuracy": 0.5438596606254578, "metrics/particle/count/bias_mean": 1.3072916269302368, "metrics/particle/count/mae": 4.5625, "metrics/particle/count/prediction_mean": 71.96224212646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08733213692903519, "metrics/particle/event/energy_response_mean": 0.9903063774108887, "metrics/particle/event/met_abs_error": 8.458395004272461, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07032232731580734, "metrics/particle/event/scalar_pt_response_mean": 1.00635826587677, "metrics/particle/event/vector_pt_closure": 0.06967804580926895, "metrics/particle/matched/delta_eta_abs_mean": 0.01504034548997879, "metrics/particle/matched/delta_phi_abs_mean": 0.016823502257466316, "metrics/particle/matched/delta_r_mean": 0.02531781978905201, "metrics/particle/matched/energy_relative_abs_mean": 0.1563483625650406, "metrics/particle/matched/pid_accuracy": 0.9324052333831787, "metrics/particle/matched/pt_relative_abs_mean": 0.15564315021038055, "metrics/particle/matching/duplicate_fraction": 0.08435413241386414, "metrics/particle/matching/efficiency": 0.7776938080787659, "metrics/particle/matching/f1": 0.7705651521682739, "metrics/particle/matching/purity": 0.7635659575462341, "metrics/particle/matching/target_coverage_dr0p05": 0.7101616263389587, "metrics/particle/matching/target_coverage_dr0p10": 0.8319204449653625, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5600132942199707, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8226784467697144, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02488480694591999, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.15511702001094818, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.15245869755744934, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.01650933548808098, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1911894828081131, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00829725805670023, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7967976331710815, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.02027202397584915, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.062685489654541, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8316.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29960161447525024, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1687135696411133, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3801273107528687, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1966824531555176, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1630786806344986, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.66858196258545, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01781291700899601, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1871079206466675, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0009631743305362761, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.411954879760742, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10498.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7720015048980713, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.1430835723876953, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.32260799407959, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.250795841217041, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.20492564141750336, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.5370603203773499, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.908096790313721, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.288316935300827, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.976522445678711, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026252562180161476, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.16449008882045746, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1620786339044571, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.001951610203832388, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.20188160240650177, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.028759736567735672, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4796706438064575, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.17188940942287445, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.719390869140625, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1669.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6671764254570007, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8291923999786377, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.3367180824279785, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.153915882110596, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24403291940689087, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 54.68045425415039, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06913580000400543, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.849534511566162, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3305814266204834, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.250381469726562, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2430.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6051486730575562, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.339082717895508, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.88676643371582, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4255508184432983, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.1991594284772873, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.09025215357542038, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.716858863830566, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.257536768913269, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 26.222023010253906, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.025718126446008682, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.15798743069171906, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.15746180713176727, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.015358432196080685, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.19640417397022247, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009681062656454742, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2780398428440094, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.008661774918437004, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8201513290405273, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18593.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2448354959487915, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4105082750320435, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.2039201259613037, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.0135453939437866, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.14732781052589417, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.618412971496582, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005027957260608673, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.49900856614112854, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0026103691197931767, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.032017946243286, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 23071.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7990459203720093, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.3843846321105957, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.074920654296875, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.162328720092773, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.016265925019979477, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.8586885929107666, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.456404209136963, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6655838489532471, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.838577270507812, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025506844744086266, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.15753191709518433, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.15627555549144745, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.01491035707294941, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1952277272939682, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004723913501948118, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4991719126701355, "metrics/particle/momentum/all/accepted/pz_bias": -0.010302231647074223, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5759528875350952, "metrics/particle/momentum/all/accepted/samples": 28578.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2893151342868805, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3682581186294556, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.5342769622802734, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.346434473991394, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15844884514808655, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 17.765106201171875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013083696365356445, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8583365082740784, "metrics/particle/momentum/all/assigned/pz_bias": -0.02036105841398239, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2612709999084473, "metrics/particle/momentum/all/assigned/samples": 35999.0, "metrics/particle/momentum/all/efficiency": 0.7765972018241882, "metrics/particle/momentum/all/event/energy_bias": 1.1883857250213623, "metrics/particle/momentum/all/event/p3_error_mean": 23.509000778198242, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.260295867919922, "metrics/particle/momentum/all/event/px_bias": 0.4203510284423828, "metrics/particle/momentum/all/event/py_bias": -1.4860011339187622, "metrics/particle/momentum/all/event/pz_abs_mean": 16.78291893005371, "metrics/particle/momentum/all/event/pz_bias": -2.211437702178955, "metrics/particle/momentum/all/event/pz_residual_rms": 29.13825035095215, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.027554143220186234, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.14205710589885712, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14042215049266815, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.002156159607693553, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17889465391635895, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0037793396040797234, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.501543402671814, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.01304940227419138, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.5282952785491943, "metrics/particle/momentum/class_1/accepted/samples": 15082.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1818825751543045, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.6016599535942078, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.7879472374916077, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.6289286613464355, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09986881911754608, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 11.437865257263184, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008897507563233376, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.720755398273468, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.019542083144187927, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.719120979309082, "metrics/particle/momentum/class_1/assigned/samples": 17533.0, "metrics/particle/momentum/class_1/efficiency": 0.8501212000846863, "metrics/particle/momentum/class_1/event/energy_bias": 12.151213645935059, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.321535110473633, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.902088165283203, "metrics/particle/momentum/class_1/event/px_bias": 0.848648190498352, "metrics/particle/momentum/class_1/event/py_bias": -1.483971118927002, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.280248641967773, "metrics/particle/momentum/class_1/event/pz_bias": -2.6696059703826904, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.80419158935547, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03281795606017113, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18615256249904633, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19021105766296387, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.003713049227371812, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23057597875595093, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02604166604578495, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4244662523269653, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0060932510532438755, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.2228569984436035, "metrics/particle/momentum/class_2/accepted/samples": 1536.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6651896238327026, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7036277651786804, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.091060996055603, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7769010066986084, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3209179937839508, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.807499885559082, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.054176073521375656, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.48614764213562, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.027003435418009758, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.088986396789551, "metrics/particle/momentum/class_2/assigned/samples": 2658.0, "metrics/particle/momentum/class_2/efficiency": 0.5144005417823792, "metrics/particle/momentum/class_2/event/energy_bias": -3.4062440395355225, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.903133392333984, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.533522605895996, "metrics/particle/momentum/class_2/event/px_bias": -0.07155356556177139, "metrics/particle/momentum/class_2/event/py_bias": 1.1205415725708008, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.966850280761719, "metrics/particle/momentum/class_2/event/pz_bias": 1.6837165355682373, "metrics/particle/momentum/class_2/event/pz_residual_rms": 24.88513946533203, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02194974757730961, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17338891327381134, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.172150120139122, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.0397949293255806, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20967943966388702, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.00233281496912241, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.34267863631248474, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.005279636476188898, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1322193145751953, "metrics/particle/momentum/class_3/accepted/samples": 11574.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3457001745700836, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.29133939743042, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.4006264209747314, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.201350688934326, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1944480836391449, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.073762893676758, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00995155144482851, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.707115113735199, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03207023814320564, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.08319354057312, "metrics/particle/momentum/class_3/assigned/samples": 15274.0, "metrics/particle/momentum/class_3/efficiency": 0.7451232671737671, "metrics/particle/momentum/class_3/event/energy_bias": -5.190346717834473, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.515121459960938, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.09163761138916, "metrics/particle/momentum/class_3/event/px_bias": -0.3082388639450073, "metrics/particle/momentum/class_3/event/py_bias": -0.7750842571258545, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.70041275024414, "metrics/particle/momentum/class_3/event/pz_bias": -1.0526055097579956, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.626361846923828, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.025230200961232185, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1753844916820526, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16226725280284882, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.01766226813197136, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20344178378582, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.02049180306494236, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1748968362808228, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.13851381838321686, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.157602071762085, "metrics/particle/momentum/class_4/accepted/samples": 244.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4226074516773224, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.619603157043457, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.4406914710998535, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.280444145202637, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2938005328178406, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 18.97113037109375, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.013477088883519173, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.1159064769744873, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.11836648732423782, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.7873291969299316, "metrics/particle/momentum/class_4/assigned/samples": 371.0, "metrics/particle/momentum/class_4/efficiency": 0.6506666541099548, "metrics/particle/momentum/class_4/event/energy_bias": -2.2686779499053955, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.903472423553467, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.2207980155944824, "metrics/particle/momentum/class_4/event/px_bias": 0.126448392868042, "metrics/particle/momentum/class_4/event/py_bias": -0.2491302639245987, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4723073244094849, "metrics/particle/momentum/class_4/event/pz_bias": -0.30677929520606995, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.280782699584961, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01938142627477646, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16840830445289612, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1688261181116104, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.003982154186815023, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.20430466532707214, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0422535203397274, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8326656818389893, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.04688039794564247, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.218017101287842, "metrics/particle/momentum/class_5/accepted/samples": 142.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.12896999716758728, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.7668088674545288, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 1.2433077096939087, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 1.0233960151672363, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.12883435189723969, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 11.229649543762207, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08588957041501999, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.696885585784912, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.43946847319602966, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.374495029449463, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.8658536672592163, "metrics/particle/momentum/class_5/event/energy_bias": -0.09755945205688477, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.3971352577209473, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.7834973335266113, "metrics/particle/momentum/class_5/event/px_bias": -0.17495295405387878, "metrics/particle/momentum/class_5/event/py_bias": -0.09835702925920486, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.273685097694397, "metrics/particle/momentum/class_5/event/pz_bias": 0.13383670151233673, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.372718334197998, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.4079626500606537, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.409046471118927, "metrics/set_slot/angular_recall_all_dr0p10": 0.9411569833755493, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8172327280044556, "metrics/set_slot/soft_count_bias_mean": 2.967323064804077}, "step_train_time": 5.121337652206421, "step_valid_time": 66350.30695414543, "step_total_time": 66355.42829203606, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0006139464383732, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17684805676400672, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2862, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2828, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9355998692383132, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.942998352553542, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9392845421726288, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05700164744645799, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9244851258581236, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9317957166392092, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9281260255989497, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9355998692383132, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0074915520412415, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1728630909009652, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2244, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2130, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2118, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9321663019693655, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9491978609625669, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9406049900640318, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05080213903743316, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9269146608315099, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9438502673796791, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9353058070214175, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9321663019693655, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023877960318976, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18732703373924298, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1313, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1215, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1210, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9190620272314675, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9253617669459253, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9222011385199241, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07463823305407463, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9152798789712556, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9215536938309216, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9184060721062619, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9190620272314675} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_23000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..7fad029bb98b87f5cc2f88eb70bf9de4441f7253 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 1.0894278135657447, "train": {"Classification_binary": 0.1455375850200653, "Cardinality": 0.11835085600614548, "Classification": 0.07809524238109589, "Regression_pt": 0.13790227472782135, "Regression_eta": 0.0016964625101536512, "Regression_sin_phi": 0.0007704154122620821, "Regression_cos_phi": 0.0007172570913098752, "Regression_energy": 0.13668330013751984, "HitGrouping": 0.1653151512145996, "Auxiliary": 0.1950928270816803, "Total": 0.9958593845367432, "Total_unweighted": 0.501402537280228}, "train_group": {}, "valid": {"Classification_binary": 0.17977416515350342, "Cardinality": 0.13267879684766135, "Classification": 0.09733595450719197, "Regression_pt": 0.13022881746292114, "Regression_eta": 0.0018251114524900913, "Regression_sin_phi": 0.0008993034716695547, "Regression_cos_phi": 0.0008703481095532576, "Regression_energy": 0.12885438402493796, "HitGrouping": 0.16305234034856161, "Auxiliary": 0.21671887238820395, "Total": 1.0940728982289631, "Total_unweighted": 0.5397880841822674, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8482142686843872, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5874659419059753, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3071827292442322, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.5973330140113831, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.5921317338943481, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.342765212059021, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7964082956314087, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.523723840713501, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.29652881622314453, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5326268672943115, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.52857905626297, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.41912850737571716, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8647987246513367, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5547175407409668, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2965054512023926, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5613192319869995, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5501294136047363, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31159454584121704, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8544648289680481, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.562232494354248, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2997419834136963, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5700021386146545, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5611810088157654, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32938647270202637, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003931808285415173, "metrics/hit_grouping/calorimeter/background_recall": 0.1610102653503418, "metrics/hit_grouping/calorimeter/completeness": 0.626768946647644, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6277279257774353, "metrics/hit_grouping/calorimeter/energy_purity": 0.6197239756584167, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.349938303232193, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8783469200134277, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5881446003913879, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22564861178398132, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.5980244278907776, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5847811698913574, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.26962903141975403, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8571562767028809, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5827916860580444, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.32508060336112976, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.5908983945846558, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.5775262117385864, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33026254177093506, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8030580878257751, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4886865019798279, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4252900183200836, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.49181368947029114, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.49659422039985657, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.45034801959991455, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8364301323890686, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6028146147727966, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.36053645610809326, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6139673590660095, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.61464923620224, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3753909170627594, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8116129040718079, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6358715295791626, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4966546595096588, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6364193558692932, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6350354552268982, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.41430777311325073, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8779574036598206, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5120834708213806, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21429885923862457, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5172641277313232, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.49786821007728577, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.27236494421958923, "metrics/hit_grouping/calorimeter/purity": 0.618125855922699, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8168747425079346, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1033240482211113, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0958646610379219, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1032872349023819, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.07019302994012833, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.836733341217041, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.28298869729042053, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11653914302587509, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2816392183303833, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2624303996562958, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.21306650340557098, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8559999465942383, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5842416286468506, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3125976324081421, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.5926128625869751, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.5847278833389282, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.34037795662879944, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7164726257324219, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4148920774459839, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4886116087436676, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5209291577339172, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6162537932395935, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7632231712341309, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2736588418483734, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4732746183872223, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.47338977456092834, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4550934433937073, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6328287124633789, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.439847469329834, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39342349767684937, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4411375820636749, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7154986262321472, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6681678891181946, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4199020564556122, "metrics/hit_grouping/tracker/all/object_completeness": 0.4285774230957031, "metrics/hit_grouping/tracker/all/object_purity": 0.4680435061454773, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6657613515853882, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01589633710682392, "metrics/hit_grouping/tracker/background_recall": 0.12736816704273224, "metrics/hit_grouping/tracker/completeness": 0.42480364441871643, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4720522165298462, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6978006362915039, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40347185730934143, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5018624067306519, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5324789881706238, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6404523849487305, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6490697860717773, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4427300691604614, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4017156958580017, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.45276859402656555, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6872390508651733, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6261879801750183, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4310189485549927, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.30451837182044983, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.35067635774612427, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.696483314037323, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6675726771354675, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4391610622406006, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43197786808013916, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.48745062947273254, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6711386442184448, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6737448573112488, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4735221564769745, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4084398150444031, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4296908378601074, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6564039587974548, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6679419875144958, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3852522373199463, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4277667701244354, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4496890902519226, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6604481935501099, "metrics/hit_grouping/tracker/purity": 0.40271416306495667, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8852975368499756, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.048642534762620926, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11931942403316498, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11185123026371002, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.05957767739892006, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6691684126853943, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.290139377117157, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.33723464608192444, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3389848470687866, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.541559100151062, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6358381509780884, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.48906049132347107, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.48437079787254333, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5348897576332092, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.769179105758667, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5635992884635925, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6497870683670044, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7448390126228333, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6360169053077698, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.014446495100855827, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.061542097479104996, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.563066482543945, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.26943379640579224, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1778106391429901, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8538925051689148, "metrics/particle/class_1/pid_accuracy": 0.9650012254714966, "metrics/particle/class_2/efficiency": 0.5561218857765198, "metrics/particle/class_2/pid_accuracy": 0.7193482518196106, "metrics/particle/class_3/efficiency": 0.7559845447540283, "metrics/particle/class_3/pid_accuracy": 0.9664266705513, "metrics/particle/class_4/efficiency": 0.671342670917511, "metrics/particle/class_4/pid_accuracy": 0.3940298557281494, "metrics/particle/class_5/efficiency": 0.8544061183929443, "metrics/particle/class_5/pid_accuracy": 0.49551570415496826, "metrics/particle/count/bias_mean": -1.041015625, "metrics/particle/count/mae": 3.8509113788604736, "metrics/particle/count/prediction_mean": 69.61392974853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08329026401042938, "metrics/particle/event/energy_response_mean": 1.0009866952896118, "metrics/particle/event/met_abs_error": 6.949618816375732, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07222744822502136, "metrics/particle/event/scalar_pt_response_mean": 1.0299047231674194, "metrics/particle/event/vector_pt_closure": 0.06167721748352051, "metrics/particle/matched/delta_eta_abs_mean": 0.016511179506778717, "metrics/particle/matched/delta_phi_abs_mean": 0.015310543589293957, "metrics/particle/matched/delta_r_mean": 0.025263164192438126, "metrics/particle/matched/energy_relative_abs_mean": 0.14765222370624542, "metrics/particle/matched/pid_accuracy": 0.9445186257362366, "metrics/particle/matched/pt_relative_abs_mean": 0.14814406633377075, "metrics/particle/matching/duplicate_fraction": 0.06442712992429733, "metrics/particle/matching/efficiency": 0.7867239117622375, "metrics/particle/matching/f1": 0.7925626635551453, "metrics/particle/matching/purity": 0.7984886765480042, "metrics/particle/matching/target_coverage_dr0p05": 0.7080975770950317, "metrics/particle/matching/target_coverage_dr0p10": 0.8310266733169556, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5873891711235046, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8344175815582275, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02388451062142849, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14753176271915436, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14712077379226685, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.027785489335656166, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1863866001367569, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007390029262751341, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6992689371109009, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.02227078750729561, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8667398691177368, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8525.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25858983397483826, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9604662656784058, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1598763465881348, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0149916410446167, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13076625764369965, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.013725280761719, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01389693096280098, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9882908463478088, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.08933352679014206, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6546618938446045, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10362.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7914036512374878, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.8213562965393066, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.600194931030273, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.97145676612854, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1291227489709854, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2146599441766739, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.268674850463867, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.6068260669708252, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.76980209350586, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025495562702417374, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1490972340106964, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15043418109416962, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.001271542627364397, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19063812494277954, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02857142873108387, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.383470892906189, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.21916364133358002, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.709287405014038, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1750.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5550800561904907, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2577080726623535, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.20784330368042, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.00789737701416, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21182885766029358, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 33.8175048828125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.061661072075366974, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5356404781341553, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.5230811238288879, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.608426094055176, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2384.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6345177888870239, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.823990345001221, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.893291473388672, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.282058596611023, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06660739332437515, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.05505242198705673, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.743375778198242, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.000372052192688, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.637880325317383, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02614361047744751, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14851592481136322, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14898176491260529, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.028579875826835632, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18804310262203217, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006448838976211846, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.24622385203838348, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03553584963083267, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7817175388336182, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18608.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.23258256912231445, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1273950338363647, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8981308341026306, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.740496039390564, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13164812326431274, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.014876365661621, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0035485848784446716, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4264523386955261, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.04164133593440056, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6076974868774414, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22826.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.7996906042098999, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.069416046142578, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.901413917541504, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.314279556274414, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.037252426147461, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9439001083374023, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.35994815826416, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.4298144578933716, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.985300064086914, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02543756365776062, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1482606679201126, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14852048456668854, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02653673104941845, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1877143532037735, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004327805247157812, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4488479495048523, "metrics/particle/momentum/all/accepted/pz_bias": 0.016188526526093483, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5019538402557373, "metrics/particle/momentum/all/accepted/samples": 28883.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2617718577384949, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0875027179718018, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1291708946228027, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9724145531654358, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13676486909389496, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.723075866699219, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010457662865519524, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7314696311950684, "metrics/particle/momentum/all/assigned/pz_bias": 0.017686771228909492, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.755176067352295, "metrics/particle/momentum/all/assigned/samples": 35572.0, "metrics/particle/momentum/all/efficiency": 0.7848854660987854, "metrics/particle/momentum/all/event/energy_bias": 3.066781759262085, "metrics/particle/momentum/all/event/p3_error_mean": 19.931602478027344, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.201200485229492, "metrics/particle/momentum/all/event/px_bias": 1.0997679233551025, "metrics/particle/momentum/all/event/py_bias": -1.21361243724823, "metrics/particle/momentum/all/event/pz_abs_mean": 14.443159103393555, "metrics/particle/momentum/all/event/pz_bias": 2.0362682342529297, "metrics/particle/momentum/all/event/pz_residual_rms": 24.814245223999023, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02664925716817379, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.139322429895401, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.14051595330238342, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.054934896528720856, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1786930412054062, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0032527882140129805, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4655410647392273, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.033938467502593994, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3200398683547974, "metrics/particle/momentum/class_1/accepted/samples": 15064.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.17135296761989594, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.4316265285015106, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5104029178619385, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.39093104004859924, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.09378953278064728, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.553017616271973, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006958021782338619, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6504603624343872, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.038629282265901566, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.360191583633423, "metrics/particle/momentum/class_1/assigned/samples": 17390.0, "metrics/particle/momentum/class_1/efficiency": 0.8491066098213196, "metrics/particle/momentum/class_1/event/energy_bias": 9.70842456817627, "metrics/particle/momentum/class_1/event/p3_error_mean": 16.892776489257812, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.265416145324707, "metrics/particle/momentum/class_1/event/px_bias": 1.3994758129119873, "metrics/particle/momentum/class_1/event/py_bias": 0.25124824047088623, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.254427909851074, "metrics/particle/momentum/class_1/event/pz_bias": 1.423066258430481, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.90372657775879, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.029614632949233055, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18544448912143707, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1883990615606308, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0027478220872581005, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22930775582790375, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.025798525661230087, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4252662658691406, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.15772707760334015, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.448652982711792, "metrics/particle/momentum/class_2/accepted/samples": 1628.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6039143800735474, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5922031402587891, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7552093863487244, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4107818007469177, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.271484375, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.174355983734131, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04804687574505806, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.2210211753845215, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.20919391512870789, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.32746696472168, "metrics/particle/momentum/class_2/assigned/samples": 2560.0, "metrics/particle/momentum/class_2/efficiency": 0.5452109575271606, "metrics/particle/momentum/class_2/event/energy_bias": -9.460390090942383, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.612505912780762, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.0106201171875, "metrics/particle/momentum/class_2/event/px_bias": 0.05865296721458435, "metrics/particle/momentum/class_2/event/py_bias": -1.172306776046753, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.450075149536133, "metrics/particle/momentum/class_2/event/pz_bias": -0.8716433048248291, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.479969024658203, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023378022015094757, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15407393872737885, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15285538136959076, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.007113578729331493, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19205759465694427, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002288329415023327, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.27322515845298767, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.012839339673519135, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1947535276412964, "metrics/particle/momentum/class_3/accepted/samples": 11799.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30695414543151855, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8440154790878296, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8339192867279053, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6641544103622437, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1608373075723648, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.12567138671875, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007684154901653528, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5570228099822998, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.02857455611228943, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5202364921569824, "metrics/particle/momentum/class_3/assigned/samples": 15096.0, "metrics/particle/momentum/class_3/efficiency": 0.7596085667610168, "metrics/particle/momentum/class_3/event/energy_bias": 3.9895784854888916, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.425956726074219, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.7055134773254395, "metrics/particle/momentum/class_3/event/px_bias": -0.6534452438354492, "metrics/particle/momentum/class_3/event/py_bias": -0.3699454069137573, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.521801471710205, "metrics/particle/momentum/class_3/event/pz_bias": 1.5135364532470703, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.711176872253418, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.026148861274123192, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1666809767484665, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1621042937040329, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.09021657705307007, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20537398755550385, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00401606410741806, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6699782609939575, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1341293305158615, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4483835697174072, "metrics/particle/momentum/class_4/accepted/samples": 249.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3671001195907593, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.905874729156494, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.48284912109375, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.403957366943359, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.25619834661483765, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 33.238224029541016, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005509641952812672, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7672436833381653, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2309146225452423, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7876858711242676, "metrics/particle/momentum/class_4/assigned/samples": 363.0, "metrics/particle/momentum/class_4/efficiency": 0.6639999747276306, "metrics/particle/momentum/class_4/event/energy_bias": -0.5151973366737366, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.625317096710205, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9117648601531982, "metrics/particle/momentum/class_4/event/px_bias": 0.07401636242866516, "metrics/particle/momentum/class_4/event/py_bias": 0.10535348206758499, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4441462755203247, "metrics/particle/momentum/class_4/event/pz_bias": -0.0994337871670723, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.579193115234375, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01893511228263378, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15478451550006866, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15641047060489655, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03401202708482742, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19439852237701416, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0419580414891243, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.679870367050171, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.19730255007743835, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.9046719074249268, "metrics/particle/momentum/class_5/accepted/samples": 143.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.11572343856096268, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2731940746307373, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.27917933464050293, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.1234937235713005, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.11042945086956024, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.7264530658721924, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.061349693685770035, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.056389093399048, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.13654281198978424, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.60354471206665, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.8719512224197388, "metrics/particle/momentum/class_5/event/energy_bias": -0.6556356549263, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.156688928604126, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.6624070405960083, "metrics/particle/momentum/class_5/event/px_bias": 0.2210679054260254, "metrics/particle/momentum/class_5/event/py_bias": -0.02796202525496483, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0658644437789917, "metrics/particle/momentum/class_5/event/pz_bias": 0.07074272632598877, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.486213207244873, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.38648155331611633, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.38848596811294556, "metrics/set_slot/angular_recall_all_dr0p10": 0.9425483345985413, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8154911994934082, "metrics/set_slot/soft_count_bias_mean": 0.6536000370979309}, "step_train_time": 5.261852502822876, "step_valid_time": 69318.73286747932, "step_total_time": 69323.99472022057, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.047607476962365, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1369852762552337, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3073, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2896, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2859, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9467146126185028, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9424015619915392, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9445531637312459, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05759843800846079, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.93461915658712, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9303612105434429, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9324853228962818, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9467146126185028, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.032196776831187, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1457985239002606, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2271, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2159, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2149, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9448577680525164, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9506825187142228, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9477611940298507, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04931748128577719, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9404814004376367, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9462791721708499, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9433713784021072, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9448577680525164, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0381359918746291, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15563405449157885, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1310, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1224, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1216, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9258698940998488, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.934351145038168, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9300911854103343, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06564885496183206, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9198184568835098, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9282442748091603, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9240121580547113, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9258698940998488} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_24000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..421a803a31e84984344113e8354e12e7f1f7ed28 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 1.1367942402425162, "train": {"Classification_binary": 0.16742384433746338, "Cardinality": 0.11333027482032776, "Classification": 0.08729765564203262, "Regression_pt": 0.13125737011432648, "Regression_eta": 0.0013761665904894471, "Regression_sin_phi": 0.0008009367156773806, "Regression_cos_phi": 0.0008333484875038266, "Regression_energy": 0.12975376844406128, "HitGrouping": 0.16017645597457886, "Auxiliary": 0.20349419116973877, "Total": 1.0173876285552979, "Total_unweighted": 0.5187430903315544}, "train_group": {}, "valid": {"Classification_binary": 0.17109338442484537, "Cardinality": 0.14080938696861267, "Classification": 0.09475276867548625, "Regression_pt": 0.1338928242524465, "Regression_eta": 0.0016143204023440678, "Regression_sin_phi": 0.0008910290586451689, "Regression_cos_phi": 0.0008800747649123272, "Regression_energy": 0.132175882657369, "HitGrouping": 0.1609892944494883, "Auxiliary": 0.2164133389790853, "Total": 1.0876749356587727, "Total_unweighted": 0.5353002842360487, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8422533869743347, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5992608070373535, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3076559007167816, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6089593172073364, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6125949621200562, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.35449984669685364, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7841827273368835, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5374667644500732, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3024372160434723, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5450899004936218, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5593458414077759, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.44497784972190857, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8635236024856567, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5675343871116638, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.289284884929657, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5742678642272949, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5692143440246582, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3176116645336151, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8509199023246765, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.574811577796936, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2958715558052063, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5825124979019165, "metrics/hit_grouping/calorimeter/all/object_purity": 0.581590473651886, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.33867546916007996, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003304810496047139, "metrics/hit_grouping/calorimeter/background_recall": 0.14702898263931274, "metrics/hit_grouping/calorimeter/completeness": 0.6350639462471008, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6350950002670288, "metrics/hit_grouping/calorimeter/energy_purity": 0.6268473267555237, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3369014263153076, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8747239112854004, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.601453423500061, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.23014338314533234, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6117329001426697, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6035863161087036, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2770823836326599, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8533625602722168, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5947954654693604, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.306886225938797, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6025246977806091, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6014241576194763, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33901429176330566, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7999231815338135, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5003572702407837, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4187934994697571, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5027716755867004, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5167605876922607, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.46392109990119934, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8328871726989746, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6144260168075562, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3504931330680847, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6265509128570557, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6386550664901733, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3861558735370636, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8084886074066162, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6389647722244263, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.46989190578460693, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6375504732131958, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.648142397403717, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.42717447876930237, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8743605017662048, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.526779294013977, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21944938600063324, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5310555100440979, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5154808759689331, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.27972283959388733, "metrics/hit_grouping/calorimeter/purity": 0.6259124875068665, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7972297072410583, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.12018010020256042, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.09210526198148727, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.11963390558958054, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.09244268387556076, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8450011014938354, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3237674832344055, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1265450268983841, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3229711651802063, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.30326783657073975, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.21012359857559204, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8521028161048889, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5952835083007812, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.308003306388855, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6035609245300293, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6040443778038025, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.35036295652389526, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.708280622959137, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40909749269485474, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4896465241909027, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5383153557777405, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6248008012771606, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7614235281944275, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.29294756054878235, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4833980202674866, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.48718351125717163, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.45449066162109375, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6236833333969116, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.45071232318878174, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39631304144859314, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.45318788290023804, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7244927287101746, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6598581671714783, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42626893520355225, "metrics/hit_grouping/tracker/all/object_completeness": 0.43143123388290405, "metrics/hit_grouping/tracker/all/object_purity": 0.48186254501342773, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6739091873168945, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011814379133284092, "metrics/hit_grouping/tracker/background_recall": 0.11422482132911682, "metrics/hit_grouping/tracker/completeness": 0.42998206615448, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4618500769138336, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6865254044532776, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40496230125427246, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5015662312507629, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5481815934181213, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6464141607284546, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6407132148742676, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45017245411872864, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4045642912387848, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.46500205993652344, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7030314207077026, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6240245699882507, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.44634807109832764, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3138570189476013, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3621959686279297, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7015329003334045, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6608893871307373, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4489258825778961, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4366862177848816, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.49773094058036804, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6787806749343872, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6589375734329224, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4802955687046051, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40910667181015015, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.44181516766548157, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6754925847053528, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6586825847625732, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3871288299560547, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4286016821861267, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4684124290943146, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6672921776771545, "metrics/hit_grouping/tracker/purity": 0.405939519405365, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8876097798347473, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04185520485043526, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.12018612772226334, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11922132968902588, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06447964161634445, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6699956655502319, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2978833317756653, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3479291796684265, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.352679580450058, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.541559100151062, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6249617338180542, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4972301423549652, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.486669659614563, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5496792793273926, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7786917090415955, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5703905820846558, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6384057998657227, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7309396266937256, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6393979787826538, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.021237879991531372, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06492319703102112, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.63613510131836, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2812204957008362, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18959732353687286, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8537971377372742, "metrics/particle/class_1/pid_accuracy": 0.9685713052749634, "metrics/particle/class_2/efficiency": 0.5298448204994202, "metrics/particle/class_2/pid_accuracy": 0.7099187970161438, "metrics/particle/class_3/efficiency": 0.7649257779121399, "metrics/particle/class_3/pid_accuracy": 0.9708774089813232, "metrics/particle/class_4/efficiency": 0.6583166122436523, "metrics/particle/class_4/pid_accuracy": 0.3942161202430725, "metrics/particle/class_5/efficiency": 0.8735632300376892, "metrics/particle/class_5/pid_accuracy": 0.5570175647735596, "metrics/particle/count/bias_mean": -1.9713541269302368, "metrics/particle/count/mae": 4.046875, "metrics/particle/count/prediction_mean": 68.68359375, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.08080784231424332, "metrics/particle/event/energy_response_mean": 0.9902077317237854, "metrics/particle/event/met_abs_error": 7.660243988037109, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06582751125097275, "metrics/particle/event/scalar_pt_response_mean": 1.0163379907608032, "metrics/particle/event/vector_pt_closure": 0.06345433741807938, "metrics/particle/matched/delta_eta_abs_mean": 0.014590704813599586, "metrics/particle/matched/delta_phi_abs_mean": 0.015102816745638847, "metrics/particle/matched/delta_r_mean": 0.02356860041618347, "metrics/particle/matched/energy_relative_abs_mean": 0.14532436430454254, "metrics/particle/matched/pid_accuracy": 0.9487662315368652, "metrics/particle/matched/pt_relative_abs_mean": 0.14684323966503143, "metrics/particle/matching/duplicate_fraction": 0.05669301748275757, "metrics/particle/matching/efficiency": 0.7882811427116394, "metrics/particle/matching/f1": 0.799433708190918, "metrics/particle/matching/purity": 0.8109063506126404, "metrics/particle/matching/target_coverage_dr0p05": 0.733897864818573, "metrics/particle/matching/target_coverage_dr0p10": 0.8387483358383179, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5892320871353149, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8278477191925049, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022615326568484306, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14462368190288544, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14647029340267181, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.043482694774866104, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.18674293160438538, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006231628358364105, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6807188987731934, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.08894941955804825, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8197659254074097, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8505.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.23092547059059143, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7746966481208801, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8923972845077515, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7546350359916687, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13105468451976776, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.316009521484375, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014941406436264515, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0151822566986084, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.17891520261764526, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9616146087646484, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10240.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7895469665527344, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.6524163484573364, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.46231460571289, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.003134250640869, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.41069647669792175, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2274603247642517, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.254207611083984, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.736466407775879, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.344161033630371, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024410782381892204, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15150631964206696, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15428704023361206, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.047670941799879074, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19572895765304565, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022261276841163635, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3168878555297852, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.10023236274719238, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.417567491531372, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1707.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.47609585523605347, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1426522731781006, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.237377643585205, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.081081390380859, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21007685363292694, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 121.37158203125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05337318405508995, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5904014110565186, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0022413090337067842, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.678230285644531, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2342.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6189267635345459, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.383708953857422, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.875832557678223, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2494349479675293, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.08887599408626556, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09781551361083984, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.73281192779541, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.9182224869728088, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.685020446777344, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.024026500061154366, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14621096849441528, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1474941223859787, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.03490043058991432, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18778611719608307, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006389436312019825, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.24126163125038147, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.029685640707612038, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.8259762525558472, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18781.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.21900881826877594, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0554134845733643, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8791765570640564, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7248495817184448, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.13045203685760498, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.578607559204102, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003572216024622321, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.40217772126197815, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03831658884882927, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4773622751235962, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22675.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8071253895759583, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.690391540527344, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.238459587097168, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.653632164001465, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6667764782905579, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.8375797271728516, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.5121588706970215, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.2825409173965454, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.74122428894043, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02363516017794609, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14605712890625, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14759373664855957, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.03816988691687584, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18795841932296753, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003552581649273634, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.43350380659103394, "metrics/particle/momentum/all/accepted/pz_bias": 0.0512240007519722, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.449493169784546, "metrics/particle/momentum/all/accepted/samples": 28993.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23954728245735168, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9796773791313171, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1060899496078491, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9564432501792908, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13591627776622772, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 32.936912536621094, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010182375088334084, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7255741357803345, "metrics/particle/momentum/all/assigned/pz_bias": 0.07675552368164062, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9923040866851807, "metrics/particle/momentum/all/assigned/samples": 35257.0, "metrics/particle/momentum/all/efficiency": 0.787874698638916, "metrics/particle/momentum/all/event/energy_bias": 0.9591000080108643, "metrics/particle/momentum/all/event/p3_error_mean": 21.625667572021484, "metrics/particle/momentum/all/event/pt_vector_error_mean": 10.760763168334961, "metrics/particle/momentum/all/event/px_bias": 1.1663490533828735, "metrics/particle/momentum/all/event/py_bias": -0.9672245979309082, "metrics/particle/momentum/all/event/pz_abs_mean": 15.973769187927246, "metrics/particle/momentum/all/event/pz_bias": 5.937229633331299, "metrics/particle/momentum/all/event/pz_residual_rms": 27.55692481994629, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02546357735991478, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13605807721614838, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13781149685382843, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.045474354177713394, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17696161568164825, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0033763654064387083, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.46473824977874756, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.10844969004392624, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3549062013626099, "metrics/particle/momentum/class_1/accepted/samples": 15105.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1574658453464508, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3838614523410797, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4557983875274658, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3240785598754883, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08880778402090073, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.6370158195495605, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00828409194946289, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6688787937164307, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.11273317039012909, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.490567445755005, "metrics/particle/momentum/class_1/assigned/samples": 17262.0, "metrics/particle/momentum/class_1/efficiency": 0.851417601108551, "metrics/particle/momentum/class_1/event/energy_bias": 7.98558235168457, "metrics/particle/momentum/class_1/event/p3_error_mean": 17.854888916015625, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.199702262878418, "metrics/particle/momentum/class_1/event/px_bias": 2.092921733856201, "metrics/particle/momentum/class_1/event/py_bias": 0.2614503502845764, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.533767700195312, "metrics/particle/momentum/class_1/event/pz_bias": 5.956620216369629, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.191804885864258, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.028299856930971146, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1882544606924057, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18786373734474182, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.03860365226864815, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23057037591934204, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.022493572905659676, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3693441152572632, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.039688825607299805, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.256251335144043, "metrics/particle/momentum/class_2/accepted/samples": 1556.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.535776674747467, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5827425718307495, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7309420108795166, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4198397397994995, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.28016358613967896, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.261784553527832, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0453987717628479, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.2562015056610107, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.16715441644191742, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.457595348358154, "metrics/particle/momentum/class_2/assigned/samples": 2445.0, "metrics/particle/momentum/class_2/efficiency": 0.5210984349250793, "metrics/particle/momentum/class_2/event/energy_bias": -9.684774398803711, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.600605010986328, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.606118202209473, "metrics/particle/momentum/class_2/event/px_bias": -0.11902280151844025, "metrics/particle/momentum/class_2/event/py_bias": -1.3745877742767334, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.557489395141602, "metrics/particle/momentum/class_2/event/pz_bias": -0.295805424451828, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.692049026489258, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02075282111763954, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15272259712219238, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15437474846839905, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.028351228684186935, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1946832686662674, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010051934514194727, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2541100084781647, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.013464830815792084, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0763200521469116, "metrics/particle/momentum/class_3/accepted/samples": 11938.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.28435733914375305, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.617418646812439, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8086565732955933, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6638435125350952, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16434985399246216, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 49.22898483276367, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006385101471096277, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5295367240905762, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.013840634375810623, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.954936981201172, "metrics/particle/momentum/class_3/assigned/samples": 15035.0, "metrics/particle/momentum/class_3/efficiency": 0.7685572504997253, "metrics/particle/momentum/class_3/event/energy_bias": 3.961414098739624, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.373652458190918, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.737010955810547, "metrics/particle/momentum/class_3/event/px_bias": -0.6388229131698608, "metrics/particle/momentum/class_3/event/py_bias": 0.5811128616333008, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.388950347900391, "metrics/particle/momentum/class_3/event/pz_bias": 0.12298783659934998, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.74271297454834, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.024729380384087563, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15830303728580475, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15430457890033722, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06478695571422577, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19743244349956512, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5545957088470459, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08824411779642105, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.0342720746994019, "metrics/particle/momentum/class_4/accepted/samples": 249.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3650006651878357, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.984590530395508, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.914918899536133, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.810590744018555, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2549019753932953, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 53.57130813598633, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.002801120514050126, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6599465012550354, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.10381963849067688, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3278898000717163, "metrics/particle/momentum/class_4/assigned/samples": 357.0, "metrics/particle/momentum/class_4/efficiency": 0.6639999747276306, "metrics/particle/momentum/class_4/event/energy_bias": -0.7354281544685364, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.565551280975342, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9349231719970703, "metrics/particle/momentum/class_4/event/px_bias": -0.0018089786171913147, "metrics/particle/momentum/class_4/event/py_bias": -0.16311326622962952, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2738980054855347, "metrics/particle/momentum/class_4/event/pz_bias": -0.08533753454685211, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.702106952667236, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.018534716218709946, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16505707800388336, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16468437016010284, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03526359796524048, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2094307243824005, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03448275849223137, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6989424228668213, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.32780033349990845, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.227332592010498, "metrics/particle/momentum/class_5/accepted/samples": 145.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07563996315002441, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.22201381623744965, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.23728343844413757, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.06521595269441605, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07594936341047287, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.4654698669910431, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.050632912665605545, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.036593198776245, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.6729089021682739, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.953634738922119, "metrics/particle/momentum/class_5/assigned/samples": 158.0, "metrics/particle/momentum/class_5/efficiency": 0.8841463327407837, "metrics/particle/momentum/class_5/event/energy_bias": -0.5676946640014648, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8994930982589722, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4447542428970337, "metrics/particle/momentum/class_5/event/px_bias": -0.1669178307056427, "metrics/particle/momentum/class_5/event/py_bias": -0.2720867991447449, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9752345681190491, "metrics/particle/momentum/class_5/event/pz_bias": 0.23876437544822693, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4789657592773438, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.36949512362480164, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3720064163208008, "metrics/set_slot/angular_recall_all_dr0p10": 0.951348066329956, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8239868879318237, "metrics/set_slot/soft_count_bias_mean": -1.0498284101486206}, "step_train_time": 5.314521312713623, "step_valid_time": 72346.90248441696, "step_total_time": 72352.21700620651, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0286644541644794, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1457402617559418, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3053, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2890, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2868, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9447531873161163, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9466098919095971, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9456806282722513, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053390108090402885, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9375612945406996, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9394038650507698, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9384816753926701, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9447531873161163, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.020024214394147, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1422970526018178, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2270, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2149, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2140, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9404814004376367, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9466960352422907, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9435784851811196, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05330396475770925, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.936542669584245, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9427312775330396, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9396267837541163, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9404814004376367, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0211887908909894, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15892272586058565, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1229, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1219, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9296520423600605, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9346007604562737, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9321198331437239, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06539923954372624, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9220877458396369, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9269961977186312, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9245354569586651, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9296520423600605} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_25000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..0a5f8a60afdf1227caaa5ce7e7ee5b04691c7b63 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 1.1841606669192877, "train": {"Classification_binary": 0.18988323211669922, "Cardinality": 0.16712415218353271, "Classification": 0.09146229177713394, "Regression_pt": 0.12116250395774841, "Regression_eta": 0.0020364983938634396, "Regression_sin_phi": 0.0009029625216498971, "Regression_cos_phi": 0.0008598871063441038, "Regression_energy": 0.12010994553565979, "HitGrouping": 0.15538737177848816, "Auxiliary": 0.22540442645549774, "Total": 1.1315391063690186, "Total_unweighted": 0.5264173214090988}, "train_group": {}, "valid": {"Classification_binary": 0.17628935972849527, "Cardinality": 0.132829616467158, "Classification": 0.09424030780792236, "Regression_pt": 0.12708447376887003, "Regression_eta": 0.0018594499366978805, "Regression_sin_phi": 0.0008855179573098818, "Regression_cos_phi": 0.0008681920201828083, "Regression_energy": 0.1261551777521769, "HitGrouping": 0.1609711547692617, "Auxiliary": 0.21539386113484701, "Total": 1.0818430582682292, "Total_unweighted": 0.5273824789716551, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8488948941230774, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5969874262809753, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.310305655002594, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6063575148582458, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6063629984855652, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.33954763412475586, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7909073233604431, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5371391177177429, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.315361887216568, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5447949171066284, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5416019558906555, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.41691285371780396, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8651341199874878, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5718082189559937, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.29266372323036194, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5785054564476013, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5734068751335144, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3148345351219177, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8544516563415527, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5767447352409363, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.29977065324783325, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5843260884284973, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5809217691421509, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.33024656772613525, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0035192049108445644, "metrics/hit_grouping/calorimeter/background_recall": 0.15262430906295776, "metrics/hit_grouping/calorimeter/completeness": 0.6318132877349854, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6319491267204285, "metrics/hit_grouping/calorimeter/energy_purity": 0.6237857341766357, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3384142518043518, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.880094051361084, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6043636202812195, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.23583295941352844, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.614740788936615, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6000038981437683, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.26547566056251526, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8541549444198608, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5947643518447876, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.31241363286972046, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6031015515327454, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6044570803642273, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3361354172229767, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8024662733078003, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5022769570350647, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.41740140318870544, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5033963322639465, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5183034539222717, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4563805162906647, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8339595198631287, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6090909838676453, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.35241761803627014, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6203410029411316, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.634040117263794, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3837502896785736, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8231659531593323, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6429916620254517, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4678332507610321, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6413871049880981, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6324406266212463, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.38857436180114746, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8790718913078308, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.535873293876648, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22607149183750153, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5408087372779846, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5206266045570374, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.26874732971191406, "metrics/hit_grouping/calorimeter/purity": 0.6228426694869995, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7819660902023315, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0998724102973938, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.08834586292505264, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.10250178724527359, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.07787095010280609, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.015037594363093376, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8627374768257141, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.3354421854019165, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12889935076236725, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3343066871166229, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.31297796964645386, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.21071217954158783, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8552016019821167, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5970720648765564, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.31210756301879883, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6051878333091736, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6030620336532593, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3416031002998352, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7126864194869995, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4132985770702362, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.49144989252090454, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5337041616439819, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6053889393806458, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7676559686660767, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.30078360438346863, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.48009297251701355, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.47595444321632385, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.43339359760284424, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6251876950263977, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43639373779296875, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39255592226982117, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.44884994626045227, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7244927287101746, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6626034379005432, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41927871108055115, "metrics/hit_grouping/tracker/all/object_completeness": 0.4294165372848511, "metrics/hit_grouping/tracker/all/object_purity": 0.476931631565094, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6663846969604492, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01606840267777443, "metrics/hit_grouping/tracker/background_recall": 0.12843385338783264, "metrics/hit_grouping/tracker/completeness": 0.42893218994140625, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.46896496415138245, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6933743357658386, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4029458165168762, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5034497380256653, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5405882000923157, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6356303691864014, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.642758309841156, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4356507658958435, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4010065495967865, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4648660719394684, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6935923099517822, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6190244555473328, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43660956621170044, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3053567409515381, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.35797813534736633, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.702614963054657, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6605879664421082, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43788740038871765, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.43138039112091064, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4971548914909363, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.675214409828186, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6695337295532227, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.48645320534706116, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4208047091960907, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4393267333507538, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6490147709846497, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6639813780784607, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3830444812774658, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.42840731143951416, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4573822021484375, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6580196619033813, "metrics/hit_grouping/tracker/purity": 0.40669912099838257, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8781911134719849, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04072398319840431, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11945977061986923, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1222112700343132, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06334841996431351, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6662912368774414, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2968508005142212, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3461553156375885, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3514980971813202, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5425916314125061, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6302112340927124, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4887247383594513, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4844376742839813, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5431665182113647, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7692910432815552, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5772441625595093, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6176637411117554, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7039101719856262, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6384319067001343, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.028091348707675934, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.06395711749792099, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.604476928710938, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29444295167922974, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20281977951526642, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8610062003135681, "metrics/particle/class_1/pid_accuracy": 0.9675054550170898, "metrics/particle/class_2/efficiency": 0.5566881895065308, "metrics/particle/class_2/pid_accuracy": 0.6988809704780579, "metrics/particle/class_3/efficiency": 0.771068811416626, "metrics/particle/class_3/pid_accuracy": 0.969351589679718, "metrics/particle/class_4/efficiency": 0.679358720779419, "metrics/particle/class_4/pid_accuracy": 0.3879055976867676, "metrics/particle/class_5/efficiency": 0.8754789233207703, "metrics/particle/class_5/pid_accuracy": 0.54923415184021, "metrics/particle/count/bias_mean": -1.3287760019302368, "metrics/particle/count/mae": 3.9576823711395264, "metrics/particle/count/prediction_mean": 69.326171875, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07537069916725159, "metrics/particle/event/energy_response_mean": 0.97083979845047, "metrics/particle/event/met_abs_error": 6.595005035400391, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05795447155833244, "metrics/particle/event/scalar_pt_response_mean": 0.9966792464256287, "metrics/particle/event/vector_pt_closure": 0.058354590088129044, "metrics/particle/matched/delta_eta_abs_mean": 0.018201908096671104, "metrics/particle/matched/delta_phi_abs_mean": 0.015466084703803062, "metrics/particle/matched/delta_r_mean": 0.026694513857364655, "metrics/particle/matched/energy_relative_abs_mean": 0.14595913887023926, "metrics/particle/matched/pid_accuracy": 0.9462338089942932, "metrics/particle/matched/pt_relative_abs_mean": 0.14531612396240234, "metrics/particle/matching/duplicate_fraction": 0.0637366771697998, "metrics/particle/matching/efficiency": 0.796739935874939, "metrics/particle/matching/f1": 0.8043030500411987, "metrics/particle/matching/purity": 0.8120110630989075, "metrics/particle/matching/target_coverage_dr0p05": 0.7210161685943604, "metrics/particle/matching/target_coverage_dr0p10": 0.8421300053596497, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.598833441734314, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8349059224128723, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025228174403309822, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.14341291785240173, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.14271602034568787, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015098700299859047, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1824982762336731, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007347795646637678, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.7166665196418762, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.013812786899507046, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9564273357391357, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8574.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25624358654022217, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.067525863647461, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2665129899978638, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1220777034759521, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1357714980840683, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.562165260314941, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013413104228675365, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9807804822921753, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.012170976027846336, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6666157245635986, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10363.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7959524989128113, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.7481088638305664, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.149645805358887, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.340157508850098, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.28533247113227844, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2092338651418686, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.693758010864258, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.17334219813346863, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.837936401367188, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026989609003067017, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1560274213552475, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15719668567180634, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.020577993243932724, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1976993829011917, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02918855845928192, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.4296340942382812, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.14521047472953796, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5032455921173096, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1713.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5233566164970398, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.468047022819519, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.830606698989868, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.669440507888794, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.21237812936306, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 33.16499710083008, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05849936231970787, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.528719663619995, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.32323870062828064, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.213177680969238, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2359.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6211022734642029, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.26465368270874, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.560535430908203, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3360036611557007, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0026137754321098328, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08335288614034653, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.411744117736816, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.0941515564918518, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 25.633644104003906, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02762814797461033, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14646418392658234, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1451183706521988, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02171807736158371, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18418139219284058, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007910975255072117, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2447686642408371, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.05275982245802879, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.746757984161377, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18961.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2210230529308319, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.196343183517456, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9569240808486938, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8073506355285645, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12776584923267365, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.8845343589782715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004512991290539503, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4239932596683502, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.062208402901887894, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5348080396652222, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22823.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.814860999584198, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.8131088018417358, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.64139175415039, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.855284214019775, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.8372697234153748, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.06831584870815277, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.61613655090332, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.1149075031280518, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.748076438903809, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0268872007727623, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1461298018693924, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1451215296983719, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01971084624528885, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.184510737657547, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004376367665827274, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.45249998569488525, "metrics/particle/momentum/all/accepted/pz_bias": 0.0216494332998991, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4840353727340698, "metrics/particle/momentum/all/accepted/samples": 29248.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.251356303691864, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.176819086074829, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2379000186920166, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0890552997589111, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1357152909040451, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.405618667602539, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010690674185752869, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7260057926177979, "metrics/particle/momentum/all/assigned/pz_bias": 0.022039389237761497, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8394548892974854, "metrics/particle/momentum/all/assigned/samples": 35545.0, "metrics/particle/momentum/all/efficiency": 0.7948042154312134, "metrics/particle/momentum/all/event/energy_bias": -5.329653263092041, "metrics/particle/momentum/all/event/p3_error_mean": 19.43736457824707, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.404411315917969, "metrics/particle/momentum/all/event/px_bias": 1.1199883222579956, "metrics/particle/momentum/all/event/py_bias": 0.0575651079416275, "metrics/particle/momentum/all/event/pz_abs_mean": 14.581912994384766, "metrics/particle/momentum/all/event/pz_bias": 1.3824012279510498, "metrics/particle/momentum/all/event/pz_residual_rms": 26.213441848754883, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02847450040280819, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13436958193778992, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.13386154174804688, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.021804865449666977, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.17146889865398407, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003028706880286336, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.46500006318092346, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.01713051274418831, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3543710708618164, "metrics/particle/momentum/class_1/accepted/samples": 15188.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.16882464289665222, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5094065070152283, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.6236941814422607, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.48556312918663025, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08690393716096878, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.8679728507995605, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006622897926717997, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6356498599052429, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.012616516090929508, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.336146831512451, "metrics/particle/momentum/class_1/assigned/samples": 17364.0, "metrics/particle/momentum/class_1/efficiency": 0.8560960292816162, "metrics/particle/momentum/class_1/event/energy_bias": 5.149550437927246, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.761306762695312, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.527796745300293, "metrics/particle/momentum/class_1/event/px_bias": 0.6222192645072937, "metrics/particle/momentum/class_1/event/py_bias": 0.25715744495391846, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.615455627441406, "metrics/particle/momentum/class_1/event/pz_bias": 0.46169838309288025, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.003742218017578, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03236902505159378, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1903505176305771, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19098474085330963, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.023347467184066772, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23030224442481995, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030704395845532417, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5426017045974731, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.020086178556084633, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6035289764404297, "metrics/particle/momentum/class_2/accepted/samples": 1661.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5521350502967834, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6860269904136658, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8578500747680664, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5390459299087524, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2461046725511551, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.230896472930908, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05313623696565628, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.2695138454437256, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.04111391305923462, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.323441982269287, "metrics/particle/momentum/class_2/assigned/samples": 2503.0, "metrics/particle/momentum/class_2/efficiency": 0.5562625527381897, "metrics/particle/momentum/class_2/event/energy_bias": -15.242715835571289, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.881383895874023, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.559899806976318, "metrics/particle/momentum/class_2/event/px_bias": 0.09713421761989594, "metrics/particle/momentum/class_2/event/py_bias": -0.09197578579187393, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.994054794311523, "metrics/particle/momentum/class_2/event/pz_bias": -0.003458082675933838, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.773786544799805, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02416502684354782, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1542617678642273, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15261287987232208, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.021686330437660217, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19265565276145935, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0020008336286991835, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.26529979705810547, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01902640424668789, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9963423013687134, "metrics/particle/momentum/class_3/accepted/samples": 11995.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2939940094947815, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.9352622032165527, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.9479433298110962, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8133854866027832, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17092324793338776, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.506869316101074, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007985217496752739, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.558972954750061, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.032057516276836395, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.771657705307007, "metrics/particle/momentum/class_3/assigned/samples": 15153.0, "metrics/particle/momentum/class_3/efficiency": 0.772226870059967, "metrics/particle/momentum/class_3/event/energy_bias": 6.033524990081787, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.099839210510254, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.9070892333984375, "metrics/particle/momentum/class_3/event/px_bias": -0.02363864704966545, "metrics/particle/momentum/class_3/event/py_bias": 0.1399334967136383, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.116220474243164, "metrics/particle/momentum/class_3/event/pz_bias": 0.7621071338653564, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.255901336669922, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.027507511898875237, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16559140384197235, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15367506444454193, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07035358250141144, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19565370678901672, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.007633587811142206, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6913468241691589, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.12922099232673645, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6659656763076782, "metrics/particle/momentum/class_4/accepted/samples": 262.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.40650051832199097, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.158339977264404, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.011195659637451, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.9242053031921387, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.25824177265167236, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.868804931640625, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8721358776092529, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.3112407922744751, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.159539222717285, "metrics/particle/momentum/class_4/assigned/samples": 364.0, "metrics/particle/momentum/class_4/efficiency": 0.6986666917800903, "metrics/particle/momentum/class_4/event/energy_bias": -0.7248645424842834, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.416224718093872, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.8765723705291748, "metrics/particle/momentum/class_4/event/px_bias": 0.17184370756149292, "metrics/particle/momentum/class_4/event/py_bias": -0.003134649246931076, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.199904203414917, "metrics/particle/momentum/class_4/event/pz_bias": -0.06480706483125687, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.9142367839813232, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.021794194355607033, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16389040648937225, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16440372169017792, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03908812254667282, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.20275388658046722, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.035211268812417984, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7368615865707397, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.546363890171051, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.069486141204834, "metrics/particle/momentum/class_5/accepted/samples": 142.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.11265455931425095, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4031847417354584, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.291106641292572, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.14456887543201447, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.09316769987344742, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.0446720123291016, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.043478261679410934, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.865113615989685, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.42339029908180237, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.154929161071777, "metrics/particle/momentum/class_5/assigned/samples": 161.0, "metrics/particle/momentum/class_5/efficiency": 0.8658536672592163, "metrics/particle/momentum/class_5/event/energy_bias": -0.5451482534408569, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9048558473587036, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4707379341125488, "metrics/particle/momentum/class_5/event/px_bias": 0.25242990255355835, "metrics/particle/momentum/class_5/event/py_bias": -0.24441540241241455, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9413723349571228, "metrics/particle/momentum/class_5/event/pz_bias": 0.22686083614826202, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.1403086185455322, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.39375364780426025, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.39574289321899414, "metrics/set_slot/angular_recall_all_dr0p10": 0.9485284686088562, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8265669345855713, "metrics/set_slot/soft_count_bias_mean": -0.21668554842472076}, "step_train_time": 5.40084981918335, "step_valid_time": 75364.16010427475, "step_total_time": 75369.56095433235, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0027178686336522, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1383422691400119, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3047, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2895, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2879, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9463877084014384, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9501148670823761, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9482476252866033, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04988513291762389, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.941157240928408, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9448638004594684, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9430068784801834, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9463877084014384, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038187544302937, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13695695006383302, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2269, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2136, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2132, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9347921225382932, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9413838695460556, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9380764163372859, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05861613045394447, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9330415754923413, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9396209784045835, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9363197189284146, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9347921225382932, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0080674558653295, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14707773218371423, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1303, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1220, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1217, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9228441754916793, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9363008442056792, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9295238095238095, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0636991557943208, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9205748865355522, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9339984650805833, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9272380952380952, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9228441754916793} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_26000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..2d3b453def501c33f70f325cd3fef1d956a3c29b --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 1.2315270935960592, "train": {"Classification_binary": 0.19377604126930237, "Cardinality": 0.13466517627239227, "Classification": 0.08963124454021454, "Regression_pt": 0.1153777539730072, "Regression_eta": 0.001533656264655292, "Regression_sin_phi": 0.0008791105938144028, "Regression_cos_phi": 0.0007571486057713628, "Regression_energy": 0.11492956429719925, "HitGrouping": 0.1562909334897995, "Auxiliary": 0.21101506054401398, "Total": 1.0575052499771118, "Total_unweighted": 0.5168845195439644}, "train_group": {}, "valid": {"Classification_binary": 0.17459323008855185, "Cardinality": 0.13217843572298685, "Classification": 0.08928038676579793, "Regression_pt": 0.124062180519104, "Regression_eta": 0.0016050090392430623, "Regression_sin_phi": 0.0008441242389380932, "Regression_cos_phi": 0.0008100427221506834, "Regression_energy": 0.12283868590990703, "HitGrouping": 0.16031942764918009, "Auxiliary": 0.2096890409787496, "Total": 1.0511380036671956, "Total_unweighted": 0.5140336592836926, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8440213203430176, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5995476841926575, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.29790857434272766, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6104158163070679, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6148617267608643, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.35118764638900757, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7868039011955261, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5408080816268921, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.30612999200820923, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5485346913337708, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5495786070823669, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.43168389797210693, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8592231273651123, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5662357807159424, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.27993521094322205, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5741398334503174, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5723863840103149, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3236750662326813, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8489952087402344, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5743535161018372, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.28741398453712463, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5831418633460999, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5834836959838867, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.34039562940597534, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.004020703490823507, "metrics/hit_grouping/calorimeter/background_recall": 0.16062971949577332, "metrics/hit_grouping/calorimeter/completeness": 0.6362535953521729, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6358243823051453, "metrics/hit_grouping/calorimeter/energy_purity": 0.6277452707290649, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.32892945408821106, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.872207522392273, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6012420058250427, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2204710990190506, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6122121214866638, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6053372621536255, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.27987027168273926, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8529508113861084, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.59405916929245, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2996315062046051, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6033232808113098, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6043500304222107, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3397051990032196, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7976810336112976, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.4996766149997711, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.41160091757774353, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5035369992256165, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5179034471511841, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.46450117230415344, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.831471860408783, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6137010455131531, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.34381765127182007, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6267732381820679, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6422260403633118, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3911474645137787, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8216909170150757, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6388950347900391, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4781266152858734, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.640617311000824, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6410143375396729, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.400926411151886, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8701143860816956, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5265471935272217, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20718622207641602, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5318096280097961, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5167384147644043, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2814396917819977, "metrics/hit_grouping/calorimeter/purity": 0.6275357007980347, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7647131681442261, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09123539924621582, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.06578947603702545, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09028374403715134, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06600790470838547, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02255639061331749, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8270200490951538, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.30759498476982117, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10535608977079391, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.3061629831790924, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.29363664984703064, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.20953501760959625, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8515121936798096, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5961074829101562, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3004992604255676, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6055862903594971, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.606998860836029, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.35238444805145264, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.715319812297821, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41025641560554504, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.49483898282051086, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5391367673873901, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6133565306663513, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7646718621253967, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2778782546520233, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.47535786032676697, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.486847460269928, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4617239236831665, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6323662400245667, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4394876956939697, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.39877772331237793, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.45034247636795044, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7135559320449829, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6676344871520996, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41856634616851807, "metrics/hit_grouping/tracker/all/object_completeness": 0.43395838141441345, "metrics/hit_grouping/tracker/all/object_purity": 0.4803295135498047, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6641585230827332, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011869970709085464, "metrics/hit_grouping/tracker/background_recall": 0.10779128223657608, "metrics/hit_grouping/tracker/completeness": 0.43395739793777466, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4724043607711792, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6989814043045044, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4013676941394806, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5096054077148438, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5479779839515686, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6316850781440735, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6471285820007324, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4345616400241852, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.40489983558654785, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.46529191732406616, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6934107542037964, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6235267519950867, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43805229663848877, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3072230815887451, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3561173677444458, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7018935680389404, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6659802198410034, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44434067606925964, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4392593502998352, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.498672753572464, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6699498891830444, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6620940566062927, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4624384343624115, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40519753098487854, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4409179985523224, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6662561297416687, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.670778214931488, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37719395756721497, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.43222281336784363, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.46354788541793823, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.656253457069397, "metrics/hit_grouping/tracker/purity": 0.40971559286117554, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8807063698768616, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03506787493824959, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.10719079524278641, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11143828183412552, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.07126697152853012, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6742892861366272, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2741352617740631, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.346981406211853, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3506496548652649, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5291688442230225, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6352939605712891, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.49113088846206665, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.49187690019607544, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5491275191307068, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.766772985458374, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5664651393890381, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6469565629959106, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7406691312789917, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6463406085968018, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.017312364652752876, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07186584919691086, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.553539276123047, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2887330651283264, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1971098929643631, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8744898438453674, "metrics/particle/class_1/pid_accuracy": 0.9694894552230835, "metrics/particle/class_2/efficiency": 0.5811530351638794, "metrics/particle/class_2/pid_accuracy": 0.7427402138710022, "metrics/particle/class_3/efficiency": 0.7865465879440308, "metrics/particle/class_3/pid_accuracy": 0.9705661535263062, "metrics/particle/class_4/efficiency": 0.6613226532936096, "metrics/particle/class_4/pid_accuracy": 0.3848484754562378, "metrics/particle/class_5/efficiency": 0.9233716726303101, "metrics/particle/class_5/pid_accuracy": 0.54356849193573, "metrics/particle/count/bias_mean": 0.2916666567325592, "metrics/particle/count/mae": 3.9427082538604736, "metrics/particle/count/prediction_mean": 70.94661712646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07917287200689316, "metrics/particle/event/energy_response_mean": 0.990224301815033, "metrics/particle/event/met_abs_error": 7.307781219482422, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06113943085074425, "metrics/particle/event/scalar_pt_response_mean": 1.0025463104248047, "metrics/particle/event/vector_pt_closure": 0.06180928647518158, "metrics/particle/matched/delta_eta_abs_mean": 0.01556221954524517, "metrics/particle/matched/delta_phi_abs_mean": 0.014788246713578701, "metrics/particle/matched/delta_r_mean": 0.024139083921909332, "metrics/particle/matched/energy_relative_abs_mean": 0.1415613889694214, "metrics/particle/matched/pid_accuracy": 0.9500141739845276, "metrics/particle/matched/pt_relative_abs_mean": 0.1414540857076645, "metrics/particle/matching/duplicate_fraction": 0.0704847052693367, "metrics/particle/matching/efficiency": 0.8118330836296082, "metrics/particle/matching/f1": 0.8101609349250793, "metrics/particle/matching/purity": 0.8084955811500549, "metrics/particle/matching/target_coverage_dr0p05": 0.7429187297821045, "metrics/particle/matching/target_coverage_dr0p10": 0.8543574810028076, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6217219829559326, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8492619395256042, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022941308096051216, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13844376802444458, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1384764164686203, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.004761858843266964, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17779360711574554, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006652901880443096, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6843007802963257, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.007623822428286076, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9169899225234985, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8718.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24517004191875458, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0836504697799683, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2538998126983643, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.104028344154358, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12761686742305756, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.3248929977417, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014816938899457455, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9858413934707642, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0756082832813263, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.797917366027832, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10461.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8093204498291016, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.7555301189422607, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.45719051361084, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.754617214202881, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1145019680261612, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.6687605977058411, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.2300519943237305, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.8177738189697266, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.930315971374512, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024007122963666916, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1498613804578781, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15110647678375244, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0086929090321064, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18977956473827362, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.027272727340459824, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3982110023498535, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.15772242844104767, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6799354553222656, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1760.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5730621814727783, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4740002155303955, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.121503829956055, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.9281694889068604, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2168377786874771, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 55.43820571899414, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06160164251923561, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5974366664886475, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.19988484680652618, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.675417900085449, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2435.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6381435990333557, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.638726234436035, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.588179588317871, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2260786294937134, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.10100030899047852, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.06593790650367737, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.415297508239746, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.1621171534061432, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.65409278869629, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.024847377091646194, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14414389431476593, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14367321133613586, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.00603527994826436, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18241699039936066, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000931435963138938, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.23977608978748322, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.011375319212675095, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7788040637969971, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19325.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20828856527805328, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1851754188537598, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9613324403762817, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7987650632858276, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12231778353452682, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.439267635345459, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004256797954440117, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4076005220413208, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.011484754271805286, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4440447092056274, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 23022.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8305041193962097, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.1865735054016113, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.555851936340332, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.085925102233887, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.773535966873169, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.0375945568084717, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.194714069366455, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.09347901493310928, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.606451988220215, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.024240192025899887, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14281411468982697, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14259201288223267, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.004793012514710426, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18152308464050293, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0041606551967561245, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4382195770740509, "metrics/particle/momentum/all/accepted/pz_bias": -0.018920376896858215, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5059795379638672, "metrics/particle/momentum/all/accepted/samples": 29803.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2437593787908554, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1751869916915894, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2607802152633667, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0998246669769287, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13026894629001617, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 16.579383850097656, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011220000684261322, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7244675755500793, "metrics/particle/momentum/all/assigned/pz_bias": 0.015831075608730316, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7586939334869385, "metrics/particle/momentum/all/assigned/samples": 35918.0, "metrics/particle/momentum/all/efficiency": 0.8098861575126648, "metrics/particle/momentum/all/event/energy_bias": -0.6966230273246765, "metrics/particle/momentum/all/event/p3_error_mean": 19.547029495239258, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.918787002563477, "metrics/particle/momentum/all/event/px_bias": 1.9890382289886475, "metrics/particle/momentum/all/event/py_bias": -2.7722933292388916, "metrics/particle/momentum/all/event/pz_abs_mean": 14.08520793914795, "metrics/particle/momentum/all/event/pz_bias": 1.5621777772903442, "metrics/particle/momentum/all/event/pz_residual_rms": 24.96451187133789, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.025790754705667496, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.13009735941886902, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.130034402012825, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004989128094166517, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.167046919465065, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0031047866214066744, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4546988606452942, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.015126634389162064, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3492512702941895, "metrics/particle/momentum/class_1/accepted/samples": 15460.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.16089068353176117, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.4552547037601471, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5320934653282166, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.38226255774497986, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.08195222169160843, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.3407392501831055, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007143673487007618, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6192643046379089, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.012538101524114609, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.279453992843628, "metrics/particle/momentum/class_1/assigned/samples": 17498.0, "metrics/particle/momentum/class_1/efficiency": 0.8714277744293213, "metrics/particle/momentum/class_1/event/energy_bias": 5.832049369812012, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.480512619018555, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.358526229858398, "metrics/particle/momentum/class_1/event/px_bias": 1.0737800598144531, "metrics/particle/momentum/class_1/event/py_bias": -1.6267716884613037, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.316425323486328, "metrics/particle/momentum/class_1/event/pz_bias": 0.3057915270328522, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.062925338745117, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02808641642332077, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18365256488323212, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18462315201759338, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0005695486324839294, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22435802221298218, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.025160912424325943, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.413135051727295, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.014515833929181099, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5412559509277344, "metrics/particle/momentum/class_2/accepted/samples": 1709.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.602973222732544, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6463141441345215, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9856722950935364, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.679553747177124, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2629587650299072, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.605718612670898, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.053348466753959656, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.334336757659912, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.13747265934944153, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.620828151702881, "metrics/particle/momentum/class_2/assigned/samples": 2643.0, "metrics/particle/momentum/class_2/efficiency": 0.5723375678062439, "metrics/particle/momentum/class_2/event/energy_bias": -7.049149036407471, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.424371719360352, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.670934200286865, "metrics/particle/momentum/class_2/event/px_bias": -0.4073958992958069, "metrics/particle/momentum/class_2/event/py_bias": -0.7510947585105896, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.56112289428711, "metrics/particle/momentum/class_2/event/pz_bias": 1.049591064453125, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.447673797607422, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021844740957021713, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15268580615520477, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15226881206035614, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.004839437082409859, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19165880978107452, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0020428174175322056, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2609759271144867, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.03412133455276489, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0972402095794678, "metrics/particle/momentum/class_3/accepted/samples": 12238.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2733845114707947, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.998555302619934, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.062060832977295, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9125401973724365, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15985837578773499, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.591171264648438, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008261753246188164, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5496933460235596, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.016480525955557823, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4845614433288574, "metrics/particle/momentum/class_3/assigned/samples": 15251.0, "metrics/particle/momentum/class_3/efficiency": 0.7878710031509399, "metrics/particle/momentum/class_3/event/energy_bias": 2.971358299255371, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.16712760925293, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.441648006439209, "metrics/particle/momentum/class_3/event/px_bias": 1.2330293655395508, "metrics/particle/momentum/class_3/event/py_bias": 0.16010499000549316, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.386030673980713, "metrics/particle/momentum/class_3/event/pz_bias": 0.13470670580863953, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.711028099060059, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02398783154785633, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16114257276058197, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14881005883216858, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.05232781171798706, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19604286551475525, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5689302682876587, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.11484666168689728, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.2600176334381104, "metrics/particle/momentum/class_4/accepted/samples": 245.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4616113305091858, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.4800262451171875, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.004491329193115, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.899544715881348, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2857142984867096, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 25.312442779541016, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008241758681833744, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.9071515202522278, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2477431297302246, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.205638885498047, "metrics/particle/momentum/class_4/assigned/samples": 364.0, "metrics/particle/momentum/class_4/efficiency": 0.653333306312561, "metrics/particle/momentum/class_4/event/energy_bias": -1.3541321754455566, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.5962862968444824, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9116066694259644, "metrics/particle/momentum/class_4/event/px_bias": -0.11135169118642807, "metrics/particle/momentum/class_4/event/py_bias": -0.2933860123157501, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3817297220230103, "metrics/particle/momentum/class_4/event/pz_bias": -0.20826829969882965, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.591104030609131, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.016508035361766815, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15280479192733765, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15823113918304443, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.03548181429505348, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1932724118232727, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05298013240098953, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8699029684066772, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.5577536225318909, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.377387046813965, "metrics/particle/momentum/class_5/accepted/samples": 151.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05563289299607277, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.3791118562221527, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.6103386878967285, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4137670695781708, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04938271641731262, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 5.181305408477783, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04938271641731262, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.8661184310913086, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.5132662057876587, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.313311576843262, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.9207317233085632, "metrics/particle/momentum/class_5/event/energy_bias": -1.0967490673065186, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.0139107704162598, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5834665298461914, "metrics/particle/momentum/class_5/event/px_bias": 0.20097637176513672, "metrics/particle/momentum/class_5/event/py_bias": -0.2611457109451294, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9504424929618835, "metrics/particle/momentum/class_5/event/pz_bias": 0.2803567945957184, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2070376873016357, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.36167508363723755, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3640148341655731, "metrics/set_slot/angular_recall_all_dr0p10": 0.9488694071769714, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8365829586982727, "metrics/set_slot/soft_count_bias_mean": 1.0931371450424194}, "step_train_time": 6.158954858779907, "step_valid_time": 78555.03436565399, "step_total_time": 78561.19332122803, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.010985285578879, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15016764132813232, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3047, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2896, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2886, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9467146126185028, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9504430587463079, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9485751719620046, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04955694125369216, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9434455704478588, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9471611421069904, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9452997052079921, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9467146126185028, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028614509626999, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14236370073285964, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2277, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2168, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2162, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9487964989059081, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9521299956082565, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9504603244191144, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04787000439174352, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9461706783369803, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9494949494949495, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.947829899167032, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9487964989059081, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0128301805780484, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14905943298276625, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1310, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1226, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1215, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9273827534039334, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9358778625954198, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9316109422492401, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06412213740458016, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9190620272314675, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9274809160305344, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9232522796352584, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9273827534039334} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_27000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..4ae66dadb64853e45e3c1b1119371e0c5bfb82f5 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 1.2788935202728307, "train": {"Classification_binary": 0.18880479037761688, "Cardinality": 0.12434390187263489, "Classification": 0.08617230504751205, "Regression_pt": 0.10898944735527039, "Regression_eta": 0.0016188425943255424, "Regression_sin_phi": 0.0008015710045583546, "Regression_cos_phi": 0.0008953941287472844, "Regression_energy": 0.10750052332878113, "HitGrouping": 0.15385101735591888, "Auxiliary": 0.20180359482765198, "Total": 1.0284993648529053, "Total_unweighted": 0.49478287383681163}, "train_group": {}, "valid": {"Classification_binary": 0.1665788491566976, "Cardinality": 0.1323947012424469, "Classification": 0.08861727515856425, "Regression_pt": 0.1194801926612854, "Regression_eta": 0.001513399959852298, "Regression_sin_phi": 0.0008025865536183119, "Regression_cos_phi": 0.0007902403982977072, "Regression_energy": 0.11814886331558228, "HitGrouping": 0.15851834416389465, "Auxiliary": 0.20597511529922485, "Total": 1.0246174335479736, "Total_unweighted": 0.49593140720389783, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.847672164440155, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6012666821479797, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3010315001010895, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6117561459541321, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6129835247993469, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3472130298614502, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7945910692214966, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5374355912208557, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.30428361892700195, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5468943119049072, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5441297292709351, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4139586389064789, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8648470044136047, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5821034908294678, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2774357795715332, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5895984172821045, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5876365900039673, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31298309564590454, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8541893362998962, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5844411253929138, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2866685688495636, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5929957628250122, "metrics/hit_grouping/calorimeter/all/object_purity": 0.591937780380249, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.33119267225265503, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0038152732886373997, "metrics/hit_grouping/calorimeter/background_recall": 0.1629975140094757, "metrics/hit_grouping/calorimeter/completeness": 0.6436487436294556, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6413924098014832, "metrics/hit_grouping/calorimeter/energy_purity": 0.6331825256347656, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.322623610496521, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8790183067321777, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6090880632400513, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22570550441741943, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6198336482048035, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6082662343978882, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2687187194824219, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8543868660926819, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6067342162132263, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2996315062046051, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6158663630485535, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6136849522590637, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3353293538093567, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.803364634513855, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.511728048324585, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.39791184663772583, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5152331590652466, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5367357134819031, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.45440834760665894, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8360558748245239, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6258728504180908, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.33882609009742737, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.638637900352478, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6537547707557678, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3797209560871124, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8209534287452698, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.647252082824707, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.44570252299308777, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.648212194442749, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.652523398399353, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.40710240602493286, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8766370415687561, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5347159504890442, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2145441174507141, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.539882481098175, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5216936469078064, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.27267152070999146, "metrics/hit_grouping/calorimeter/purity": 0.6346986293792725, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7778486609458923, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.11572764068841934, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07518796622753143, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.1136009618639946, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.08644382655620575, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8431861400604248, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.32339802384376526, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11477339267730713, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.32297778129577637, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.3035644292831421, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.22012948989868164, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.856005847454071, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6056628227233887, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2990596890449524, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6148585081100464, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.615181028842926, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.34184813499450684, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7162722945213318, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.39634940028190613, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.4938911199569702, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5341013073921204, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.601622462272644, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7702488899230957, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.27245327830314636, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.47641393542289734, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.48720499873161316, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4352019429206848, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6284450888633728, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4455317258834839, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.3964301347732544, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4429721236228943, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7195280194282532, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6659126877784729, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4176313579082489, "metrics/hit_grouping/tracker/all/object_completeness": 0.4322923421859741, "metrics/hit_grouping/tracker/all/object_purity": 0.4742475748062134, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.66228848695755, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.012494705617427826, "metrics/hit_grouping/tracker/background_recall": 0.1160009503364563, "metrics/hit_grouping/tracker/completeness": 0.4321584105491638, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4679563641548157, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.69720059633255, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.40022796392440796, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5026382207870483, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5332272052764893, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6297562718391418, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6483425498008728, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44018879532814026, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.40896958112716675, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.45884352922439575, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6872390508651733, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6190100312232971, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4310189485549927, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3107631206512451, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.36823129653930664, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.704418420791626, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.668117344379425, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4474823772907257, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.44190743565559387, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.48975348472595215, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6618833541870117, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6643595695495605, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4661329984664917, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4103551506996155, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4525487720966339, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6637930870056152, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6633251309394836, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37012916803359985, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4237251877784729, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4579794108867645, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6625455617904663, "metrics/hit_grouping/tracker/purity": 0.4082752466201782, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8856164216995239, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03431372717022896, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11373550444841385, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11693553626537323, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6759457588195801, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2839442491531372, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.35484376549720764, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.36102285981178284, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5327826738357544, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6322219967842102, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.489004522562027, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4879596531391144, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5395437479019165, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7652621269226074, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5756838321685791, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6729304790496826, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7746382355690002, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6476228833198547, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.026531139388680458, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07314810156822205, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.635066986083984, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.28401947021484375, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.19239631295204163, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8854369521141052, "metrics/particle/class_1/pid_accuracy": 0.9729252457618713, "metrics/particle/class_2/efficiency": 0.5717521905899048, "metrics/particle/class_2/pid_accuracy": 0.7299920916557312, "metrics/particle/class_3/efficiency": 0.7826771140098572, "metrics/particle/class_3/pid_accuracy": 0.9696106314659119, "metrics/particle/class_4/efficiency": 0.6843687295913696, "metrics/particle/class_4/pid_accuracy": 0.4450951814651489, "metrics/particle/class_5/efficiency": 0.9252873659133911, "metrics/particle/class_5/pid_accuracy": 0.5879917144775391, "metrics/particle/count/bias_mean": 0.1204427108168602, "metrics/particle/count/mae": 3.5813801288604736, "metrics/particle/count/prediction_mean": 70.775390625, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07263556867837906, "metrics/particle/event/energy_response_mean": 0.9705418944358826, "metrics/particle/event/met_abs_error": 6.271721363067627, "metrics/particle/event/scalar_pt_relative_abs_error": 0.055800069123506546, "metrics/particle/event/scalar_pt_response_mean": 0.9825723171234131, "metrics/particle/event/vector_pt_closure": 0.05646556615829468, "metrics/particle/matched/delta_eta_abs_mean": 0.01485522836446762, "metrics/particle/matched/delta_phi_abs_mean": 0.014485693536698818, "metrics/particle/matched/delta_r_mean": 0.023316625505685806, "metrics/particle/matched/energy_relative_abs_mean": 0.1398751139640808, "metrics/particle/matched/pid_accuracy": 0.9515393972396851, "metrics/particle/matched/pt_relative_abs_mean": 0.13909918069839478, "metrics/particle/matching/duplicate_fraction": 0.057482682168483734, "metrics/particle/matching/efficiency": 0.8149475455284119, "metrics/particle/matching/f1": 0.814253568649292, "metrics/particle/matching/purity": 0.8135607242584229, "metrics/particle/matching/target_coverage_dr0p05": 0.7608407139778137, "metrics/particle/matching/target_coverage_dr0p10": 0.8622449636459351, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6292501091957092, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8525053858757019, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022263143211603165, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.138840451836586, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1387607455253601, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.006556818727403879, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1778429001569748, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006356413010507822, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6867185235023499, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.019241489470005035, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.876800537109375, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8810.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.22198998928070068, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.787653923034668, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9012019634246826, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7467808127403259, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12051526457071304, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.579218864440918, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012404580600559711, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9369434118270874, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.026411615312099457, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8368263244628906, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10480.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8178611397743225, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.016397371888160706, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.5372796058654785, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.5683493614196777, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2515298128128052, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.16415314376354218, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.30429220199585, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.0440075397491455, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.365439414978027, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02482420578598976, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14863720536231995, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14969541132450104, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0039946213364601135, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18986962735652924, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02551303431391716, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2967184782028198, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.14452917873859406, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2556025981903076, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1803.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5301476120948792, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6643290519714355, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.565092086791992, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.363295555114746, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.20860128104686737, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 98.6903076171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06149517744779587, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.621361494064331, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.22099855542182922, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.7983551025390625, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2488.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.653734564781189, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.273677349090576, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.149507522583008, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1429437398910522, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.12724466621875763, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.06841006875038147, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.01848316192627, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.13407999277114868, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.730937957763672, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.023690873757004738, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1405920535326004, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13907472789287567, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01631964184343815, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17885567247867584, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007282942533493042, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.23381277918815613, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.021062364801764488, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7710290551185608, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19223.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.201165571808815, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9812416434288025, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7520381212234497, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5987170338630676, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12398063391447067, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.761804103851318, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0033142906613647938, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3890989422798157, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.033357203006744385, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4930099248886108, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22931.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8261206150054932, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.6663166284561157, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.017572402954102, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.636924743652344, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.2480432987213135, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.3745826482772827, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.9333269596099854, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.24230360984802246, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.021573066711426, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02333778142929077, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14056099951267242, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13962382078170776, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012209268286824226, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17924311757087708, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0038879206404089928, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4317789375782013, "metrics/particle/momentum/all/accepted/pz_bias": 0.010517937131226063, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.436527132987976, "metrics/particle/momentum/all/accepted/samples": 29836.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2300451397895813, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9720692038536072, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.0598493814468384, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9028475284576416, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12883366644382477, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 26.939064025878906, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010000278241932392, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7037394642829895, "metrics/particle/momentum/all/assigned/pz_bias": 0.013701312243938446, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8262979984283447, "metrics/particle/momentum/all/assigned/samples": 35899.0, "metrics/particle/momentum/all/efficiency": 0.8107829093933105, "metrics/particle/momentum/all/event/energy_bias": -5.956390857696533, "metrics/particle/momentum/all/event/p3_error_mean": 18.21670150756836, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.209169387817383, "metrics/particle/momentum/all/event/px_bias": 1.6268178224563599, "metrics/particle/momentum/all/event/py_bias": 0.470325767993927, "metrics/particle/momentum/all/event/pz_abs_mean": 13.206037521362305, "metrics/particle/momentum/all/event/pz_bias": 1.152231216430664, "metrics/particle/momentum/all/event/pz_residual_rms": 23.762807846069336, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.024749135598540306, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1245584636926651, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12399236857891083, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0029110244940966368, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.16112181544303894, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.003513030242174864, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.44957080483436584, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.02230989746749401, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3778910636901855, "metrics/particle/momentum/class_1/accepted/samples": 15656.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14714910089969635, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.40949276089668274, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.4484313726425171, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2941969633102417, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06975813210010529, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.627218246459961, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0072617074474692345, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6179981827735901, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.014891091734170914, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4286458492279053, "metrics/particle/momentum/class_1/assigned/samples": 17489.0, "metrics/particle/momentum/class_1/efficiency": 0.8824756145477295, "metrics/particle/momentum/class_1/event/energy_bias": 4.159781455993652, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.820487022399902, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.435384750366211, "metrics/particle/momentum/class_1/event/px_bias": 2.039875030517578, "metrics/particle/momentum/class_1/event/py_bias": 1.6434434652328491, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.63289737701416, "metrics/particle/momentum/class_1/event/pz_bias": 0.8949168920516968, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.872108459472656, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02894134446978569, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18608446419239044, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1862240880727768, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.00658068573102355, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22596044838428497, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.019725045189261436, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3219329118728638, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.03737211599946022, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.0491788387298584, "metrics/particle/momentum/class_2/accepted/samples": 1673.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5818672180175781, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5813069939613342, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8916935324668884, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5745353698730469, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.27237948775291443, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.645911693572998, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.047819435596466064, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.3522393703460693, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.008421818725764751, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.984697341918945, "metrics/particle/momentum/class_2/assigned/samples": 2614.0, "metrics/particle/momentum/class_2/efficiency": 0.5602813363075256, "metrics/particle/momentum/class_2/event/energy_bias": -9.250467300415039, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.806394577026367, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.9789018630981445, "metrics/particle/momentum/class_2/event/px_bias": -0.08699245005846024, "metrics/particle/momentum/class_2/event/py_bias": 0.14270764589309692, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.201225280761719, "metrics/particle/momentum/class_2/event/pz_bias": 0.49264490604400635, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.114238739013672, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020847417414188385, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1546071320772171, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1532147228717804, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03136567398905754, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19331060349941254, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018998843152076006, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2673426866531372, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0011219640728086233, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0746428966522217, "metrics/particle/momentum/class_3/accepted/samples": 12106.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.26162123680114746, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5932724475860596, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.697627305984497, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5631415843963623, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1696469485759735, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 40.6227912902832, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006419073790311813, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5050246119499207, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.010868324898183346, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.374284029006958, "metrics/particle/momentum/class_3/assigned/samples": 15267.0, "metrics/particle/momentum/class_3/efficiency": 0.7793729305267334, "metrics/particle/momentum/class_3/event/energy_bias": 0.18225941061973572, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.991548538208008, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.093341827392578, "metrics/particle/momentum/class_3/event/px_bias": -0.6798041462898254, "metrics/particle/momentum/class_3/event/py_bias": -1.008497714996338, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.248778343200684, "metrics/particle/momentum/class_3/event/pz_bias": -0.3542039096355438, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.17663860321045, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022880686447024345, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15731440484523773, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14777244627475739, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06689392030239105, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19656433165073395, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004000000189989805, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6472371220588684, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1977877914905548, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6893391609191895, "metrics/particle/momentum/class_4/accepted/samples": 250.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4270017445087433, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.047929763793945, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.2295732498168945, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.13919734954834, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.25885558128356934, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 34.1344108581543, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.01089918240904808, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8719834089279175, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0819617435336113, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.123147487640381, "metrics/particle/momentum/class_4/assigned/samples": 367.0, "metrics/particle/momentum/class_4/efficiency": 0.6666666865348816, "metrics/particle/momentum/class_4/event/energy_bias": -0.6244581937789917, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.281144142150879, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6982758045196533, "metrics/particle/momentum/class_4/event/px_bias": 0.11151339113712311, "metrics/particle/momentum/class_4/event/py_bias": -0.012473389506340027, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.190800428390503, "metrics/particle/momentum/class_4/event/pz_bias": -0.07801367342472076, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.39056396484375, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.015335171483457088, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1415134221315384, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14091826975345612, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.015926770865917206, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17589247226715088, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.026490066200494766, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5511435270309448, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.058354347944259644, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.251871347427368, "metrics/particle/momentum/class_5/accepted/samples": 151.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08035283535718918, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.23505888879299164, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.22901268303394318, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08459687232971191, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04938271641731262, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.8249635100364685, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03086419776082039, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7061163187026978, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.08278944343328476, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.736337900161743, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.9207317233085632, "metrics/particle/momentum/class_5/event/energy_bias": -0.4235060214996338, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.048180341720581, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5501506328582764, "metrics/particle/momentum/class_5/event/px_bias": 0.24222619831562042, "metrics/particle/momentum/class_5/event/py_bias": -0.29485437273979187, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0134012699127197, "metrics/particle/momentum/class_5/event/pz_bias": 0.1968870759010315, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.195962429046631, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3498890995979309, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3522941768169403, "metrics/set_slot/angular_recall_all_dr0p10": 0.9548126459121704, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8463778495788574, "metrics/set_slot/soft_count_bias_mean": 1.1902118921279907}, "step_train_time": 5.878077268600464, "step_valid_time": 81965.66207170486, "step_total_time": 81971.54014968872, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9910152621664081, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12909676583165086, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3057, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2892, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2877, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9454069957502452, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9460255152109912, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9457161543492478, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053974484789008834, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9405034324942791, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9411187438665358, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9408109875735775, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9454069957502452, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9846479142336915, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1257601793093177, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2277, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2156, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2151, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9435448577680525, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9468599033816425, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9451994739149496, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05314009661835749, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9413566739606127, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9446640316205533, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9430074528715475, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9435448577680525, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9887018382659292, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13991329573549116, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1231, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1224, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9311649016641452, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9339908952959028, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9325757575757576, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06600910470409711, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9258698940998488, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.928679817905918, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9272727272727272, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9311649016641452} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_28000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..4d9b9ced5562412d0162d40510404784ac0a0cf6 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 1.3262599469496021, "train": {"Classification_binary": 0.15640603005886078, "Cardinality": 0.1113729178905487, "Classification": 0.09273421764373779, "Regression_pt": 0.11132800579071045, "Regression_eta": 0.0015173912979662418, "Regression_sin_phi": 0.0008805067627690732, "Regression_cos_phi": 0.0007588641019538045, "Regression_energy": 0.1099361777305603, "HitGrouping": 0.1628461331129074, "Auxiliary": 0.19486787915229797, "Total": 0.9839516878128052, "Total_unweighted": 0.47356119338655844}, "train_group": {}, "valid": {"Classification_binary": 0.16217897335688272, "Cardinality": 0.12078661719957988, "Classification": 0.08864536881446838, "Regression_pt": 0.1182968517144521, "Regression_eta": 0.0014629724125067394, "Regression_sin_phi": 0.0008199899457395077, "Regression_cos_phi": 0.0007814334239810705, "Regression_energy": 0.11709501345952351, "HitGrouping": 0.15844797094662985, "Auxiliary": 0.20352017879486084, "Total": 1.0041279792785645, "Total_unweighted": 0.48928060312755406, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8457450866699219, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5959843993186951, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.311062753200531, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6052572131156921, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6085616946220398, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.34532034397125244, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.795185923576355, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5426750183105469, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.30096012353897095, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5509689450263977, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5527081489562988, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4261447489261627, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.858676016330719, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5636802315711975, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.28859061002731323, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5711178183555603, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5696454048156738, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.32325851917266846, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8498293161392212, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5718360543251038, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.29635894298553467, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5798961520195007, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5801202058792114, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3379300534725189, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0035321530885994434, "metrics/hit_grouping/calorimeter/background_recall": 0.15287800133228302, "metrics/hit_grouping/calorimeter/completeness": 0.6381661295890808, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6384984254837036, "metrics/hit_grouping/calorimeter/energy_purity": 0.6302508115768433, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.33943647146224976, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8741854429244995, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5936102867126465, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2261606752872467, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.604070782661438, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5936835408210754, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.27474966645240784, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8518556356430054, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5958703756332397, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.31345000863075256, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6035431027412415, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6046914458274841, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3425840735435486, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7981262803077698, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.503226101398468, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.42227378487586975, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5067819356918335, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5277113914489746, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4620649516582489, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8312689065933228, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6162171363830566, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3595140874385834, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6282029747962952, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6427402496337891, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3904859125614166, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.811257541179657, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.640835702419281, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4693772494792938, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6423924565315247, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.655479371547699, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4107050895690918, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.87334805727005, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5183665752410889, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21135568618774414, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5231989622116089, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5072972774505615, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.27567601203918457, "metrics/hit_grouping/calorimeter/purity": 0.6291134357452393, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7774279713630676, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09854581207036972, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07706766575574875, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.0998433530330658, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.07607805728912354, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.033834587782621384, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.827923059463501, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.28693312406539917, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10771042108535767, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.28463131189346313, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2701832056045532, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.21130076050758362, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8521490097045898, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5943740010261536, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.30974915623664856, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6030834913253784, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6044620275497437, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3494747281074524, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7199519276618958, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.41402289271354675, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5015274286270142, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5374594926834106, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6023467779159546, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7748414874076843, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.30560579895973206, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4901677966117859, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4834303855895996, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4309825301170349, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6372466087341309, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4453878402709961, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4025194048881531, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.44846269488334656, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7024032473564148, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6728291511535645, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4254229664802551, "metrics/hit_grouping/tracker/all/object_completeness": 0.4394232928752899, "metrics/hit_grouping/tracker/all/object_purity": 0.4783984124660492, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6516028642654419, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011020224541425705, "metrics/hit_grouping/tracker/background_recall": 0.10960688441991806, "metrics/hit_grouping/tracker/completeness": 0.44143417477607727, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4790118336677551, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7039945721626282, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4103103578090668, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5136376023292542, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5418930649757385, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6208136081695557, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.655422031879425, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.44617897272109985, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.41321122646331787, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4617956876754761, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6739880442619324, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6260165572166443, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4358882009983063, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.31280720233917236, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3642854690551758, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6926960945129395, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6772729754447937, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4577566385269165, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4515651762485504, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4939480721950531, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6482975482940674, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.653685986995697, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4575123190879822, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3922007083892822, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4558134377002716, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6644088625907898, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6704840064048767, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37763550877571106, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4321039915084839, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4622320830821991, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6536041498184204, "metrics/hit_grouping/tracker/purity": 0.41641661524772644, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8829021453857422, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03808446601033211, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11099376529455185, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11015890538692474, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06523378938436508, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6772229075431824, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.281362920999527, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3498503267765045, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3508346974849701, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5296850800514221, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6411787867546082, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.49851715564727783, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.4978698194026947, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5468702912330627, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7518326044082642, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5710103511810303, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6555736064910889, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7495756149291992, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.659603476524353, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.021857617422938347, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.08512870967388153, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.220970153808594, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2760251760482788, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1844020038843155, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8774840831756592, "metrics/particle/class_1/pid_accuracy": 0.9729406833648682, "metrics/particle/class_2/efficiency": 0.5737909078598022, "metrics/particle/class_2/pid_accuracy": 0.739834189414978, "metrics/particle/class_3/efficiency": 0.7837046384811401, "metrics/particle/class_3/pid_accuracy": 0.9718820452690125, "metrics/particle/class_4/efficiency": 0.6753507256507874, "metrics/particle/class_4/pid_accuracy": 0.43323442339897156, "metrics/particle/class_5/efficiency": 0.9003831148147583, "metrics/particle/class_5/pid_accuracy": 0.5914893746376038, "metrics/particle/count/bias_mean": -0.9388020634651184, "metrics/particle/count/mae": 3.546875, "metrics/particle/count/prediction_mean": 69.71614837646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.072247713804245, "metrics/particle/event/energy_response_mean": 0.9725618958473206, "metrics/particle/event/met_abs_error": 5.8785834312438965, "metrics/particle/event/scalar_pt_relative_abs_error": 0.054785799235105515, "metrics/particle/event/scalar_pt_response_mean": 0.9981023669242859, "metrics/particle/event/vector_pt_closure": 0.054979562759399414, "metrics/particle/matched/delta_eta_abs_mean": 0.013861829414963722, "metrics/particle/matched/delta_phi_abs_mean": 0.013945653103291988, "metrics/particle/matched/delta_r_mean": 0.02208082191646099, "metrics/particle/matched/energy_relative_abs_mean": 0.13910622894763947, "metrics/particle/matched/pid_accuracy": 0.9529346227645874, "metrics/particle/matched/pt_relative_abs_mean": 0.13940440118312836, "metrics/particle/matching/duplicate_fraction": 0.05649770423769951, "metrics/particle/matching/efficiency": 0.8115013837814331, "metrics/particle/matching/f1": 0.8169286847114563, "metrics/particle/matching/purity": 0.8224291205406189, "metrics/particle/matching/target_coverage_dr0p05": 0.7628862857818604, "metrics/particle/matching/target_coverage_dr0p10": 0.8585039377212524, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6262186169624329, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.846856951713562, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02148263156414032, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13916754722595215, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1399158239364624, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018988806754350662, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1794632524251938, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.007199177052825689, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6819666028022766, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.12080863863229752, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8967704772949219, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8751.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.21818611025810242, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8613668084144592, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9419585466384888, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7936041355133057, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11836262047290802, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.620323657989502, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013516122475266457, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9511258602142334, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.1068919450044632, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7681031227111816, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10358.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8123839497566223, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.277632236480713, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.7957658767700195, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.5743608474731445, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.3443361222743988, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.24791209399700165, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.606076717376709, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.3737213611602783, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.682950019836426, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02410157024860382, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14995433390140533, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15090300142765045, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.024089079350233078, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19261284172534943, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.026463963091373444, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3535579442977905, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.20259340107440948, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.7558717727661133, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1776.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5363403558731079, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1431230306625366, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.599883794784546, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.4197158813476562, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2044517695903778, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 25.253122329711914, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.056471556425094604, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.468276023864746, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.41323190927505493, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.5695929527282715, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2426.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6439448595046997, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.730812072753906, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.458876609802246, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.18588387966156, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.06587644666433334, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.045198120176792145, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.310516357421875, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5483942031860352, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.295969009399414, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02243347279727459, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14032113552093506, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14033854007720947, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.023450642824172974, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1803349107503891, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005704506766051054, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.23012371361255646, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.02599608153104782, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7417528629302979, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19283.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.19629527628421783, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9723200798034668, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7767550945281982, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6322568655014038, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12323020398616791, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.548251628875732, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003058905713260174, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.37959638237953186, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.019052335992455482, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4864422082901, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22884.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8286991119384766, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.1745797395706177, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.744648933410645, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.313371658325195, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.02598128467798233, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.6601788997650146, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.015969276428223, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.413971185684204, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.1122727394104, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022253723815083504, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1405564397573471, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14084386825561523, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02217886596918106, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18083535134792328, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.004059040453284979, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.42969754338264465, "metrics/particle/momentum/all/accepted/pz_bias": -0.06435037404298782, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.5008275508880615, "metrics/particle/momentum/all/accepted/samples": 29810.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2257809340953827, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9517164826393127, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9487323760986328, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8006882071495056, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12734103202819824, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.367467880249023, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009728608652949333, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.68763267993927, "metrics/particle/momentum/all/assigned/pz_bias": -0.07137157022953033, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.745903491973877, "metrics/particle/momentum/all/assigned/samples": 35668.0, "metrics/particle/momentum/all/efficiency": 0.8100763559341431, "metrics/particle/momentum/all/event/energy_bias": -5.833864212036133, "metrics/particle/momentum/all/event/p3_error_mean": 17.474472045898438, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.65323257446289, "metrics/particle/momentum/all/event/px_bias": -0.25247836112976074, "metrics/particle/momentum/all/event/py_bias": 0.9532890319824219, "metrics/particle/momentum/all/event/pz_abs_mean": 12.894147872924805, "metrics/particle/momentum/all/event/pz_bias": -5.336086273193359, "metrics/particle/momentum/all/event/pz_residual_rms": 23.857479095458984, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.024219289422035217, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.12713782489299774, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12755022943019867, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01175636239349842, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1653774231672287, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0034735624212771654, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.44731608033180237, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.07994154840707779, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3338044881820679, "metrics/particle/momentum/class_1/accepted/samples": 15546.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14772136509418488, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.37998905777931213, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.43917226791381836, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2955184280872345, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07647261768579483, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.124633312225342, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00723389582708478, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6189452409744263, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.08742313086986542, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4109158515930176, "metrics/particle/momentum/class_1/assigned/samples": 17418.0, "metrics/particle/momentum/class_1/efficiency": 0.8762753009796143, "metrics/particle/momentum/class_1/event/energy_bias": 5.939980506896973, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.290332794189453, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.805948257446289, "metrics/particle/momentum/class_1/event/px_bias": 0.27625060081481934, "metrics/particle/momentum/class_1/event/py_bias": 1.3734699487686157, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.479326248168945, "metrics/particle/momentum/class_1/event/pz_bias": -2.0314273834228516, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.76860237121582, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027723591774702072, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1807744950056076, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18317851424217224, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0012821960262954235, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22267957031726837, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.025897586718201637, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3858143091201782, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.15533843636512756, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3726322650909424, "metrics/particle/momentum/class_2/accepted/samples": 1699.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5366529226303101, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5850357413291931, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7322136163711548, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.41835421323776245, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.25242340564727783, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.9685325622558594, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04536642134189606, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.1683857440948486, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.2407260239124298, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.246540546417236, "metrics/particle/momentum/class_2/assigned/samples": 2579.0, "metrics/particle/momentum/class_2/efficiency": 0.5689886212348938, "metrics/particle/momentum/class_2/event/energy_bias": -11.72145938873291, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.791033744812012, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.796039581298828, "metrics/particle/momentum/class_2/event/px_bias": -0.5312038660049438, "metrics/particle/momentum/class_2/event/py_bias": 0.03667840361595154, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.404439926147461, "metrics/particle/momentum/class_2/event/pz_bias": -1.932704210281372, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.70050811767578, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01898469589650631, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15130729973316193, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15131986141204834, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03731633722782135, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.192082479596138, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001398026361130178, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2538031041622162, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.030327852815389633, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1970852613449097, "metrics/particle/momentum/class_3/accepted/samples": 12160.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.25896453857421875, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6007511615753174, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4914027452468872, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3648486137390137, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16239768266677856, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.619065284729004, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0063374703750014305, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5002927780151367, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.022139480337500572, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4743587970733643, "metrics/particle/momentum/class_3/assigned/samples": 15148.0, "metrics/particle/momentum/class_3/efficiency": 0.7828494310379028, "metrics/particle/momentum/class_3/event/energy_bias": 0.7278618812561035, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.934499740600586, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.966445446014404, "metrics/particle/momentum/class_3/event/px_bias": -0.06040625646710396, "metrics/particle/momentum/class_3/event/py_bias": -0.1784387081861496, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.3419013023376465, "metrics/particle/momentum/class_3/event/pz_bias": -1.1882829666137695, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.83371639251709, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.024254314601421356, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16764980554580688, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1575247198343277, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0713803768157959, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20442955195903778, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.537062406539917, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.07180905342102051, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1075215339660645, "metrics/particle/momentum/class_4/accepted/samples": 254.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4417329728603363, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.035799503326416, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.297112464904785, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.200364589691162, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24653740227222443, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 32.97038650512695, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.002770083025097847, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6953101754188538, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.025099316611886024, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6700713634490967, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.6773333549499512, "metrics/particle/momentum/class_4/event/energy_bias": -0.5273551344871521, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2914626598358154, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7099758386611938, "metrics/particle/momentum/class_4/event/px_bias": 0.06990209221839905, "metrics/particle/momentum/class_4/event/py_bias": -0.0261918343603611, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1879780292510986, "metrics/particle/momentum/class_4/event/pz_bias": -0.01525881141424179, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.591622829437256, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.018235795199871063, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15818819403648376, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16144618391990662, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.05740887671709061, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2016109675168991, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0397351011633873, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.8420531749725342, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.16269686818122864, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.009214401245117, "metrics/particle/momentum/class_5/accepted/samples": 151.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08552824705839157, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.6991623044013977, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9782665967941284, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.8741482496261597, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.06172839552164078, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 6.182492733001709, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.043209876865148544, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.9999175071716309, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.35606828331947327, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.270252227783203, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.9207317233085632, "metrics/particle/momentum/class_5/event/energy_bias": -0.25289061665534973, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.056899309158325, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.5924510955810547, "metrics/particle/momentum/class_5/event/px_bias": -0.007021002471446991, "metrics/particle/momentum/class_5/event/py_bias": -0.2522287368774414, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0159242153167725, "metrics/particle/momentum/class_5/event/pz_bias": -0.168412983417511, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.314988136291504, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.34090039134025574, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.34276726841926575, "metrics/set_slot/angular_recall_all_dr0p10": 0.953670084476471, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8436319231987, "metrics/set_slot/soft_count_bias_mean": 0.2460464984178543}, "step_train_time": 5.579974174499512, "step_valid_time": 84984.62450194359, "step_total_time": 84990.20447659492, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.007475450085254, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13509782892038813, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3038, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2900, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2885, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9480222294867604, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9545753785385122, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9512875184516976, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04542462146148782, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9431186662307943, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9496379196840027, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9463670657700508, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9480222294867604, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9929063881431407, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.127064347821082, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2260, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2160, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2154, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9452954048140044, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9557522123893806, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9504950495049505, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04424778761061947, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9426695842450766, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9530973451327434, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9478547854785478, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9452954048140044, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0051186667804406, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13373743725800935, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1313, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1243, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1231, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9402420574886535, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9466869763899467, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9434535104364327, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053313023610053314, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9311649016641452, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9375476009139375, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9343453510436432, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9402420574886535} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_29000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..c278c9701deaebcc8d316d7b94bd2a23c1185b25 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 1.3736263736263736, "train": {"Classification_binary": 0.1307935118675232, "Cardinality": 0.09387918561697006, "Classification": 0.08553790301084518, "Regression_pt": 0.11708732694387436, "Regression_eta": 0.0015228213742375374, "Regression_sin_phi": 0.0008109998889267445, "Regression_cos_phi": 0.0008303424692712724, "Regression_energy": 0.11687019467353821, "HitGrouping": 0.1542995423078537, "Auxiliary": 0.18262219429016113, "Total": 0.923147976398468, "Total_unweighted": 0.4534531002282165}, "train_group": {}, "valid": {"Classification_binary": 0.16129561265309653, "Cardinality": 0.12315954764684041, "Classification": 0.08789012829462688, "Regression_pt": 0.11912051836649577, "Regression_eta": 0.0015321900136768818, "Regression_sin_phi": 0.000813561026006937, "Regression_cos_phi": 0.0007947180420160294, "Regression_energy": 0.11855689684549968, "HitGrouping": 0.15794164935747781, "Auxiliary": 0.2059772809346517, "Total": 1.0108998616536458, "Total_unweighted": 0.4900036252414187, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8449172973632812, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5987515449523926, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3045329749584198, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6113450527191162, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6154689192771912, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.34768620133399963, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7947290539741516, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5410309433937073, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2961595356464386, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5508707761764526, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5574042201042175, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.41802066564559937, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8576226830482483, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5591301918029785, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2779912054538727, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5682219862937927, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5704110860824585, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.326128214597702, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8488907217979431, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.569728434085846, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2874426543712616, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5799391865730286, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5830517411231995, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3397935926914215, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0031342413276433945, "metrics/hit_grouping/calorimeter/background_recall": 0.13962960243225098, "metrics/hit_grouping/calorimeter/completeness": 0.6415017247200012, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6391944885253906, "metrics/hit_grouping/calorimeter/energy_purity": 0.630850613117218, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.32822611927986145, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8735690712928772, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5925387740135193, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21882112324237823, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6054390668869019, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5990405678749084, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.27611514925956726, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.851527214050293, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5940802097320557, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3056195378303528, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6048187017440796, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6054852604866028, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3428143858909607, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7959160208702087, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.49868595600128174, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.40904873609542847, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5028810501098633, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5278505086898804, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4665893316268921, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8310782313346863, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6166854500770569, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.34826797246932983, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.631751537322998, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6490451693534851, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.39078661799430847, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8180673122406006, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.641289472579956, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.48018527030944824, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6435155272483826, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6489881873130798, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3921770453453064, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8707236647605896, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5133273601531982, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20246489346027374, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5195390582084656, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5079119205474854, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28156232833862305, "metrics/hit_grouping/calorimeter/purity": 0.6321492791175842, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7944817543029785, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09752772748470306, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.08646616339683533, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09812477231025696, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.08209910988807678, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8234730958938599, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2714056670665741, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1059446707367897, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.26901721954345703, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2500159740447998, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24543848633766174, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8510999083518982, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5929469466209412, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.30016234517097473, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6039699912071228, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6085450053215027, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.34971973299980164, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7284975647926331, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4105461537837982, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5069040656089783, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5307233333587646, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5867014527320862, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7782405018806458, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2917420268058777, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.48451322317123413, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4862280786037445, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4364074766635895, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6421384215354919, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4417182207107544, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.40537482500076294, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4534427225589752, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7065764665603638, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6787337064743042, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.42105966806411743, "metrics/hit_grouping/tracker/all/object_completeness": 0.4424249827861786, "metrics/hit_grouping/tracker/all/object_purity": 0.47961628437042236, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.649777352809906, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.010443138889968395, "metrics/hit_grouping/tracker/background_recall": 0.10443637520074844, "metrics/hit_grouping/tracker/completeness": 0.4446377158164978, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4786636531352997, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7076423764228821, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4057513475418091, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5133721828460693, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5405383110046387, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6194984912872314, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6642522811889648, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4449083209037781, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.421682208776474, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4617307782173157, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6756216883659363, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6336564421653748, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4288548231124878, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3170955181121826, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.372069776058197, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6863841414451599, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6822659373283386, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.44654834270477295, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4514119029045105, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.4974225163459778, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6487221121788025, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6744281649589539, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4587438404560089, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4180276393890381, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.45704248547554016, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6379310488700867, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6749135851860046, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3811678886413574, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.43511536717414856, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4605143368244171, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6532729864120483, "metrics/hit_grouping/tracker/purity": 0.41919398307800293, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8817984461784363, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03318250551819801, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11253850907087326, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11650381982326508, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06900452822446823, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6818901896476746, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.28755807876586914, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3548027276992798, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3608125150203705, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5327826738357544, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6482574343681335, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.49308934807777405, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5008763074874878, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.546377956867218, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7486430406570435, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5812795758247375, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6701648831367493, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.76932692527771, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6479505896568298, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.03212680295109749, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07347579300403595, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.500043869018555, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2816963195800781, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1900731474161148, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8704466819763184, "metrics/particle/class_1/pid_accuracy": 0.9716482758522034, "metrics/particle/class_2/efficiency": 0.5665420889854431, "metrics/particle/class_2/pid_accuracy": 0.733706533908844, "metrics/particle/class_3/efficiency": 0.7850162982940674, "metrics/particle/class_3/pid_accuracy": 0.9743239879608154, "metrics/particle/class_4/efficiency": 0.6623246669769287, "metrics/particle/class_4/pid_accuracy": 0.4387291967868805, "metrics/particle/class_5/efficiency": 0.8793103694915771, "metrics/particle/class_5/pid_accuracy": 0.6034858226776123, "metrics/particle/count/bias_mean": -1.6360677480697632, "metrics/particle/count/mae": 3.8001301288604736, "metrics/particle/count/prediction_mean": 69.01888275146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06576376408338547, "metrics/particle/event/energy_response_mean": 0.9820422530174255, "metrics/particle/event/met_abs_error": 5.73272705078125, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05079637095332146, "metrics/particle/event/scalar_pt_response_mean": 1.0050781965255737, "metrics/particle/event/vector_pt_closure": 0.05436396598815918, "metrics/particle/matched/delta_eta_abs_mean": 0.014207178726792336, "metrics/particle/matched/delta_phi_abs_mean": 0.014136351644992828, "metrics/particle/matched/delta_r_mean": 0.022489197552204132, "metrics/particle/matched/energy_relative_abs_mean": 0.14132006466388702, "metrics/particle/matched/pid_accuracy": 0.9532234072685242, "metrics/particle/matched/pt_relative_abs_mean": 0.14120519161224365, "metrics/particle/matching/duplicate_fraction": 0.05289917439222336, "metrics/particle/matching/efficiency": 0.8078432679176331, "metrics/particle/matching/f1": 0.8173059225082397, "metrics/particle/matching/purity": 0.826992928981781, "metrics/particle/matching/target_coverage_dr0p05": 0.7568693161010742, "metrics/particle/matching/target_coverage_dr0p10": 0.853141188621521, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.617547869682312, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8441939949989319, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02163870818912983, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13734593987464905, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1385187953710556, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02729019522666931, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17846879363059998, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006085658445954323, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6702631115913391, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.01304396241903305, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.9158743619918823, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8709.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.21038222312927246, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7628379464149475, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8919360041618347, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7545796036720276, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11702954769134521, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.748449325561523, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012344478629529476, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9388908743858337, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02493739314377308, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.815692901611328, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10288.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8084849715232849, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.06182285398244858, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.126708507537842, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.2047996520996094, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.03942084312438965, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.24161580204963684, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.117158889770508, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.5564219355583191, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.497464179992676, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024292629212141037, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.15279032289981842, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15421755611896515, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.014798673801124096, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19325891137123108, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021799888461828232, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3046989440917969, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06687255948781967, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.190753936767578, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1789.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4638320505619049, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.7369663119316101, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.066831588745117, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.8918347358703613, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18715904653072357, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 18.67568588256836, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05161561071872711, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4198338985443115, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.21059562265872955, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.596045017242432, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2383.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6486584544181824, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.8674349784851074, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.7497334480285645, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0863971710205078, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.16927123069763184, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06536675989627838, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.618260383605957, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.05032682418823242, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.398887634277344, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02284066379070282, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1425129473209381, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1416388750076294, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.027493052184581757, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1819888800382614, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008355091558769345, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.22885195910930634, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.026758067309856415, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7445244789123535, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19150.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20439156889915466, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0255639553070068, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7839992046356201, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6365343332290649, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11870076507329941, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.914465427398682, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004269178491085768, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.40358060598373413, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.028231538832187653, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6242979764938354, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22721.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8229833841323853, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.9972354173660278, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.870292663574219, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.362366199493408, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.03451502323150635, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.06386399269104, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.213131904602051, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2600528299808502, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.987977504730225, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02257520519196987, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.14161531627178192, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.14148136973381042, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02666747011244297, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.18166568875312805, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0036427415907382965, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4234330356121063, "metrics/particle/momentum/all/accepted/pz_bias": 0.017079796642065048, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4319876432418823, "metrics/particle/momentum/all/accepted/samples": 29648.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22360152006149292, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9297611117362976, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9017501473426819, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7553699612617493, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12282436341047287, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.742131233215332, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.00980447605252266, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6949460506439209, "metrics/particle/momentum/all/assigned/pz_bias": 0.011193357408046722, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.807739019393921, "metrics/particle/momentum/all/assigned/samples": 35392.0, "metrics/particle/momentum/all/efficiency": 0.8056740760803223, "metrics/particle/momentum/all/event/energy_bias": -2.9320223331451416, "metrics/particle/momentum/all/event/p3_error_mean": 17.490360260009766, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.846847534179688, "metrics/particle/momentum/all/event/px_bias": 0.24320706725120544, "metrics/particle/momentum/all/event/py_bias": -1.2401130199432373, "metrics/particle/momentum/all/event/pz_abs_mean": 12.899343490600586, "metrics/particle/momentum/all/event/pz_bias": 0.766148030757904, "metrics/particle/momentum/all/event/pz_residual_rms": 22.957763671875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0243680477142334, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.12685780227184296, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12731324136257172, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.021860621869564056, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.16549275815486908, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0029209398198872805, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.43720880150794983, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.006920375861227512, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.262378454208374, "metrics/particle/momentum/class_1/accepted/samples": 15406.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.14957402646541595, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3635684847831726, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.378358393907547, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24388688802719116, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.07707373052835464, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.9716084003448486, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00697004608809948, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.6170005798339844, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0010355371050536633, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4827158451080322, "metrics/particle/momentum/class_1/assigned/samples": 17360.0, "metrics/particle/momentum/class_1/efficiency": 0.8683839440345764, "metrics/particle/momentum/class_1/event/energy_bias": 4.209816932678223, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.349650382995605, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.225505828857422, "metrics/particle/momentum/class_1/event/px_bias": 0.40978163480758667, "metrics/particle/momentum/class_1/event/py_bias": -0.0862971693277359, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.319500923156738, "metrics/particle/momentum/class_1/event/pz_bias": -0.5255380272865295, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.946199417114258, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027830885723233223, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1801086664199829, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.184936985373497, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.007416751701384783, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22614967823028564, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02522522583603859, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4663511514663696, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.059140369296073914, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.770839214324951, "metrics/particle/momentum/class_2/accepted/samples": 1665.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5130254626274109, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5803403854370117, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7703492641448975, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4639643430709839, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2476000040769577, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.371439456939697, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.047600001096725464, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.277086019515991, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.026176510378718376, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.668081760406494, "metrics/particle/momentum/class_2/assigned/samples": 2500.0, "metrics/particle/momentum/class_2/efficiency": 0.5576021671295166, "metrics/particle/momentum/class_2/event/energy_bias": -11.529047012329102, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.512788772583008, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.913643836975098, "metrics/particle/momentum/class_2/event/px_bias": -0.18694809079170227, "metrics/particle/momentum/class_2/event/py_bias": -0.8398001790046692, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.003332138061523, "metrics/particle/momentum/class_2/event/pz_bias": 0.6912907958030701, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.634614944458008, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019663995131850243, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15406470000743866, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1526612937450409, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03355250135064125, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19292810559272766, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009842519648373127, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24254350364208221, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.016030116006731987, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8824994564056396, "metrics/particle/momentum/class_3/accepted/samples": 12192.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2574175298213959, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5456690788269043, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4820032119750977, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3473155498504639, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15165844559669495, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.667841911315918, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006460636854171753, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5061275959014893, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.008905568160116673, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4090585708618164, "metrics/particle/momentum/class_3/assigned/samples": 15014.0, "metrics/particle/momentum/class_3/efficiency": 0.7849095463752747, "metrics/particle/momentum/class_3/event/energy_bias": 4.295172691345215, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.262320518493652, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.987844944000244, "metrics/particle/momentum/class_3/event/px_bias": 0.08580075204372406, "metrics/particle/momentum/class_3/event/py_bias": -0.12373816967010498, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.694001197814941, "metrics/particle/momentum/class_3/event/pz_bias": 0.34284865856170654, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.946885108947754, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022526565939188004, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18254955112934113, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17415371537208557, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.10521893203258514, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22085236012935638, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.004098360426723957, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6041441559791565, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1667783111333847, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.350575566291809, "metrics/particle/momentum/class_4/accepted/samples": 244.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.414412260055542, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.328763961791992, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.153458833694458, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.051150321960449, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.266853928565979, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.013797760009766, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.00561797758564353, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7743350863456726, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.26942986249923706, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.1918087005615234, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.6506666541099548, "metrics/particle/momentum/class_4/event/energy_bias": -0.19591282308101654, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.2880449295043945, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7423360347747803, "metrics/particle/momentum/class_4/event/px_bias": 0.1250491589307785, "metrics/particle/momentum/class_4/event/py_bias": 0.03390486538410187, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1601316928863525, "metrics/particle/momentum/class_4/event/pz_bias": -0.04796252399682999, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.099593639373779, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.016434552147984505, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15219418704509735, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15313486754894257, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.04792916774749756, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.20038224756717682, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.05673758685588837, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.9314017295837402, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.46216118335723877, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.318508625030518, "metrics/particle/momentum/class_5/accepted/samples": 141.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.13665953278541565, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2468327134847641, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2909855544567108, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.1571618616580963, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1111111119389534, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.8871845006942749, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04938271641731262, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.956920862197876, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.5130364894866943, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.188089847564697, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.8597561120986938, "metrics/particle/momentum/class_5/event/energy_bias": 0.2879469096660614, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.9357341527938843, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4972604513168335, "metrics/particle/momentum/class_5/event/px_bias": -0.19047635793685913, "metrics/particle/momentum/class_5/event/py_bias": -0.2241823673248291, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9547924399375916, "metrics/particle/momentum/class_5/event/pz_bias": 0.30550915002822876, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.937709093093872, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3513958156108856, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3530404567718506, "metrics/set_slot/angular_recall_all_dr0p10": 0.9539833664894104, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8398264050483704, "metrics/set_slot/soft_count_bias_mean": -0.17419572174549103}, "step_train_time": 5.5438432693481445, "step_valid_time": 88099.87840485573, "step_total_time": 88105.42224860191, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0158204076218709, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12378848320530678, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2911, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2895, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9516181758744687, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9516181758744687, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9516181758744687, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04838182412553122, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9463877084014384, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9463877084014384, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9463877084014384, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9516181758744687, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0081393808121923, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12060449519448468, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2166, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2158, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9479212253829322, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9525065963060686, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9502083790304892, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047493403693931395, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9444201312910284, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9489885664028144, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9466988374643562, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9479212253829322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0153927747959557, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12524529423651565, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1321, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1241, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1235, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9387291981845688, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.939439818319455, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9390843738176314, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06056018168054504, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9341906202723147, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9348978046934141, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9345440786984487, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9387291981845688} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_3000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..0b9ed99afed8b6b614645de98fc8f59b489620c2 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 0.14209928003031452, "train": {"Classification_binary": 0.39073410630226135, "Cardinality": 0.2599808871746063, "Classification": 0.31695735454559326, "Regression_pt": 0.3818801939487457, "Regression_eta": 0.005448197014629841, "Regression_sin_phi": 0.002857508137822151, "Regression_cos_phi": 0.002832359168678522, "Regression_energy": 0.374339759349823, "HitGrouping": 0.2509213984012604, "Auxiliary": 0.47599369287490845, "Total": 2.617856502532959, "Total_unweighted": 1.4750494784675539}, "train_group": {}, "valid": {"Classification_binary": 0.35427359739939374, "Cardinality": 0.25174721082051593, "Classification": 0.26119448741277057, "Regression_pt": 0.39636313915252686, "Regression_eta": 0.005036612351735433, "Regression_sin_phi": 0.0024490783301492534, "Regression_cos_phi": 0.002394587267190218, "Regression_energy": 0.3881613413492839, "HitGrouping": 0.2486902674039205, "Auxiliary": 0.45980040232340497, "Total": 2.4543093045552573, "Total_unweighted": 1.4098728432630498, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8059875965118408, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.2828194499015808, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3783476948738098, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.28343722224235535, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.2707792818546295, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.46853411197662354, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7664993405342102, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3097488582134247, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.362629234790802, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3099299669265747, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.2773003876209259, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5062776803970337, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8135837316513062, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.24737152457237244, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3444110155105591, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.24737198650836945, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.23941972851753235, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.4599398374557495, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8076269626617432, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.26295343041419983, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.35610663890838623, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.26315492391586304, "metrics/hit_grouping/calorimeter/all/object_purity": 0.2518611550331116, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4661410450935364, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0009634050657041371, "metrics/hit_grouping/calorimeter/background_recall": 0.043494194746017456, "metrics/hit_grouping/calorimeter/completeness": 0.36307188868522644, "metrics/hit_grouping/calorimeter/energy_completeness": 0.3809284567832947, "metrics/hit_grouping/calorimeter/energy_purity": 0.37560611963272095, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5325214862823486, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8385183811187744, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.27447009086608887, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2883477509021759, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.2767336070537567, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.26200249791145325, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.39724624156951904, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.792111337184906, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2725430727005005, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.37551820278167725, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.2725818157196045, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.27058714628219604, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.5034546256065369, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.760270893573761, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.22981026768684387, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.47470998764038086, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.22597138583660126, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.21231809258460999, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5690255165100098, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7667884230613708, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.29774191975593567, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4480394423007965, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.2987520694732666, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.2986842393875122, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5564108490943909, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7408804893493652, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.44009658694267273, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6587750911712646, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.43569105863571167, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.41609716415405273, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5867215394973755, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8572162389755249, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2063801884651184, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.22631676495075226, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.2063060849905014, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.18455572426319122, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3597400188446045, "metrics/hit_grouping/calorimeter/purity": 0.35700127482414246, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9040927886962891, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.044773343950510025, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.15225563943386078, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.045537952333688736, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.012782224453985691, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.030075188726186752, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8927298188209534, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.05494948476552963, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.08652148395776749, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.053769707679748535, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.03670264035463333, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.21836374700069427, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.801626443862915, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.27733275294303894, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.37345707416534424, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.27759692072868347, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.26695331931114197, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4861404597759247, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6203993558883667, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4545849561691284, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.2811664938926697, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.2949220538139343, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7503983974456787, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7016235589981079, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.35684147477149963, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3075627088546753, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.2720261514186859, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5780590772628784, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5458542704582214, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43833646178245544, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.2052333652973175, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.2080351710319519, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.814433753490448, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.580271303653717, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4373107850551605, "metrics/hit_grouping/tracker/all/object_completeness": 0.23612964153289795, "metrics/hit_grouping/tracker/all/object_purity": 0.23946620523929596, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7772929668426514, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.021725434809923172, "metrics/hit_grouping/tracker/background_recall": 0.11110672354698181, "metrics/hit_grouping/tracker/completeness": 0.24435055255889893, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6099851727485657, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.5889164805412292, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4373137056827545, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.2835389971733093, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.2866027355194092, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7866911888122559, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.566734790802002, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4318388104438782, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.20798075199127197, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.21904772520065308, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7836267948150635, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.5759369134902954, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4427412152290344, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.1665753275156021, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.16279290616512299, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7516681551933289, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.5868256688117981, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4553791284561157, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.2319667786359787, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.2436506301164627, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7740511298179626, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.5863463878631592, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.517241358757019, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.2572745084762573, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.28730788826942444, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7555418610572815, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5706613659858704, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.39949220418930054, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.23775090277194977, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2254498153924942, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.785406768321991, "metrics/hit_grouping/tracker/purity": 0.23302476108074188, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8689064979553223, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.015837104991078377, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.020083526149392128, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.01810065098106861, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.061463046818971634, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6318048238754272, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2173464149236679, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.11906754970550537, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.09725074470043182, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.6122870445251465, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5318531394004822, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5236976146697998, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.28087836503982544, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.2877305746078491, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.9014045000076294, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.46383118629455566, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.2951373755931854, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.31274861097335815, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.544799268245697, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.0853215754032135, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.029675528407096863, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.51772117614746, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.15507875382900238, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.06345559656620026, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.537818968296051, "metrics/particle/class_1/pid_accuracy": 0.897624135017395, "metrics/particle/class_2/efficiency": 0.38588741421699524, "metrics/particle/class_2/pid_accuracy": 0.24390959739685059, "metrics/particle/class_3/efficiency": 0.4902389347553253, "metrics/particle/class_3/pid_accuracy": 0.8800891637802124, "metrics/particle/class_4/efficiency": 0.4759519100189209, "metrics/particle/class_4/pid_accuracy": 0.19578947126865387, "metrics/particle/class_5/efficiency": 0.469348669052124, "metrics/particle/class_5/pid_accuracy": 0.12244898080825806, "metrics/particle/count/bias_mean": -6.466145992279053, "metrics/particle/count/mae": 8.631510734558105, "metrics/particle/count/prediction_mean": 64.18880462646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.14311520755290985, "metrics/particle/event/energy_response_mean": 1.0189951658248901, "metrics/particle/event/met_abs_error": 13.71438980102539, "metrics/particle/event/scalar_pt_relative_abs_error": 0.1186031922698021, "metrics/particle/event/scalar_pt_response_mean": 1.0263930559158325, "metrics/particle/event/vector_pt_closure": 0.11545244604349136, "metrics/particle/matched/delta_eta_abs_mean": 0.02311687543988228, "metrics/particle/matched/delta_phi_abs_mean": 0.0213151928037405, "metrics/particle/matched/delta_r_mean": 0.035142816603183746, "metrics/particle/matched/energy_relative_abs_mean": 0.20470552146434784, "metrics/particle/matched/pid_accuracy": 0.8402067422866821, "metrics/particle/matched/pt_relative_abs_mean": 0.20054598152637482, "metrics/particle/matching/duplicate_fraction": 0.12105198949575424, "metrics/particle/matching/efficiency": 0.5045058131217957, "metrics/particle/matching/f1": 0.5286983251571655, "metrics/particle/matching/purity": 0.5553278923034668, "metrics/particle/matching/target_coverage_dr0p05": 0.43395131826400757, "metrics/particle/matching/target_coverage_dr0p10": 0.5991836190223694, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3048301935195923, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5760831236839294, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03503808751702309, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.20159080624580383, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1997201293706894, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.027476079761981964, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2402799427509308, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.018993351608514786, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.3320649862289429, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.07969464361667633, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 3.176553249359131, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5265.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.6291959285736084, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.468242645263672, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.2062008380889893, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 2.9976718425750732, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.34550905227661133, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 23.34886932373047, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03375152498483658, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.9637686014175415, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.23600690066814423, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.417087078094482, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9007.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.48876717686653137, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 4.175522327423096, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 15.019852638244629, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.132045269012451, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.7305648922920227, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.2546314001083374, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 12.684039115905762, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 3.7084178924560547, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 20.457111358642578, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03308837488293648, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.20137155055999756, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.19906705617904663, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.006020127795636654, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2423594444990158, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04840271174907684, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.1673460006713867, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.08550939708948135, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 5.17003870010376, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1033.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.9145326614379883, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.4891059398651123, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.945044994354248, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6.772479057312012, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.4069000482559204, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 52.69737243652344, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08828005939722061, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.8190369606018066, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.009325223974883556, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.282113075256348, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1971.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.37454676628112793, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.742147445678711, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.773247718811035, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.9216121435165405, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.11425835639238358, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.004737772047519684, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.54769515991211, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.5922622680664062, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 30.100425720214844, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03553316369652748, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.20635735988616943, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.20181410014629364, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.0311806108802557, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.24131959676742554, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.002548503689467907, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.44556576013565063, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.03129833936691284, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.1345022916793823, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 12164.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.5980429649353027, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.3956563472747803, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.7601571083068848, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.548818349838257, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.35859188437461853, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.957304000854492, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.01298880111426115, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.049870491027832, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.01988096348941326, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.0584805011749268, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21788.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5227556228637695, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 10.81988525390625, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 18.345399856567383, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 15.766341209411621, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.41827654838562, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 4.0751752853393555, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.459896564483643, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.1497703641653061, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.836044311523438, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03525518998503685, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.20471908152103424, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.20106324553489685, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.028042666614055634, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.24108192324638367, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.009803921915590763, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7947162985801697, "metrics/particle/momentum/all/accepted/pz_bias": 0.006890396121889353, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.2849974632263184, "metrics/particle/momentum/all/accepted/samples": 18462.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.6256446838378906, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.086188316345215, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.1345067024230957, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.9262726306915283, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.3579014837741852, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 20.701444625854492, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.023225294426083565, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.4676663875579834, "metrics/particle/momentum/all/assigned/pz_bias": 0.07865655422210693, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.386366844177246, "metrics/particle/momentum/all/assigned/samples": 32766.0, "metrics/particle/momentum/all/efficiency": 0.5016984343528748, "metrics/particle/momentum/all/event/energy_bias": 6.253260135650635, "metrics/particle/momentum/all/event/p3_error_mean": 33.866477966308594, "metrics/particle/momentum/all/event/pt_vector_error_mean": 19.179237365722656, "metrics/particle/momentum/all/event/px_bias": 4.2631001472473145, "metrics/particle/momentum/all/event/py_bias": 5.325069427490234, "metrics/particle/momentum/all/event/pz_abs_mean": 23.166744232177734, "metrics/particle/momentum/all/event/pz_bias": 2.9663851261138916, "metrics/particle/momentum/all/event/pz_residual_rms": 36.49476623535156, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.04262397065758705, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.2179129421710968, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21438725292682648, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.028263656422495842, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.25415441393852234, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.009932375513017178, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.9009757041931152, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.007686711847782135, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.3346755504608154, "metrics/particle/momentum/class_1/accepted/samples": 9464.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.5007867813110352, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.8982229232788086, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.024158000946045, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.7986844778060913, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.3266068696975708, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 11.673874855041504, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.020428501069545746, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.4161293506622314, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.11398980021476746, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.666233539581299, "metrics/particle/momentum/class_1/assigned/samples": 16056.0, "metrics/particle/momentum/class_1/efficiency": 0.5334535837173462, "metrics/particle/momentum/class_1/event/energy_bias": 38.556549072265625, "metrics/particle/momentum/class_1/event/p3_error_mean": 29.611616134643555, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 17.544597625732422, "metrics/particle/momentum/class_1/event/px_bias": 3.1505157947540283, "metrics/particle/momentum/class_1/event/py_bias": 4.4930338859558105, "metrics/particle/momentum/class_1/event/pz_abs_mean": 19.76892852783203, "metrics/particle/momentum/class_1/event/pz_bias": 3.953493118286133, "metrics/particle/momentum/class_1/event/pz_residual_rms": 27.1011962890625, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04284973442554474, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23668281733989716, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23669636249542236, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08375691622495651, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27359718084335327, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04283216968178749, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9675348997116089, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08438918739557266, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.7844109535217285, "metrics/particle/momentum/class_2/accepted/samples": 1144.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6685218811035156, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.933786153793335, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.151718020439148, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7543318867683411, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3580246865749359, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.002256870269775, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06447187811136246, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.89046573638916, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.17056985199451447, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.95326566696167, "metrics/particle/momentum/class_2/assigned/samples": 2187.0, "metrics/particle/momentum/class_2/efficiency": 0.38312122225761414, "metrics/particle/momentum/class_2/event/energy_bias": -24.302921295166016, "metrics/particle/momentum/class_2/event/p3_error_mean": 24.658763885498047, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 13.123320579528809, "metrics/particle/momentum/class_2/event/px_bias": 0.44894886016845703, "metrics/particle/momentum/class_2/event/py_bias": 0.1251162588596344, "metrics/particle/momentum/class_2/event/pz_abs_mean": 17.66607666015625, "metrics/particle/momentum/class_2/event/pz_bias": -1.5518383979797363, "metrics/particle/momentum/class_2/event/pz_residual_rms": 27.526317596435547, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.024800298735499382, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1831328570842743, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.17879444360733032, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.01914091967046261, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.21754500269889832, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0039468491449952126, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.45996323227882385, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.011525651440024376, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.4352742433547974, "metrics/particle/momentum/class_3/accepted/samples": 7601.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.7617619633674622, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 4.645985126495361, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 4.638332366943359, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 4.481373310089111, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3928012549877167, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 28.71875762939453, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.018921609967947006, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.2679526805877686, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.06198003143072128, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.519144535064697, "metrics/particle/momentum/class_3/assigned/samples": 14058.0, "metrics/particle/momentum/class_3/efficiency": 0.4893452525138855, "metrics/particle/momentum/class_3/event/energy_bias": -1.948061466217041, "metrics/particle/momentum/class_3/event/p3_error_mean": 19.477508544921875, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 11.402425765991211, "metrics/particle/momentum/class_3/event/px_bias": 0.1858321875333786, "metrics/particle/momentum/class_3/event/py_bias": 1.0245081186294556, "metrics/particle/momentum/class_3/event/pz_abs_mean": 13.316333770751953, "metrics/particle/momentum/class_3/event/pz_bias": 0.02782389521598816, "metrics/particle/momentum/class_3/event/pz_residual_rms": 25.153261184692383, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.04017136991024017, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.19602227210998535, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.19838644564151764, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.01693037338554859, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.24238057434558868, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.034090910106897354, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.4760684967041016, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.10196716338396072, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 3.130790948867798, "metrics/particle/momentum/class_4/accepted/samples": 176.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.6676115393638611, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 9.283370971679688, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 6.965639114379883, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 6.799837112426758, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3994082808494568, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 24.444461822509766, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.03550295904278755, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.8030034303665161, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.16373296082019806, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 4.417771816253662, "metrics/particle/momentum/class_4/assigned/samples": 338.0, "metrics/particle/momentum/class_4/efficiency": 0.46933332085609436, "metrics/particle/momentum/class_4/event/energy_bias": -1.8281021118164062, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.307283878326416, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.3883142471313477, "metrics/particle/momentum/class_4/event/px_bias": 0.22798778116703033, "metrics/particle/momentum/class_4/event/py_bias": -0.03381405770778656, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.1434483528137207, "metrics/particle/momentum/class_4/event/pz_bias": 0.017628192901611328, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.406479835510254, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0375414714217186, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2589297592639923, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.23837785422801971, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.07725558429956436, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2791610062122345, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025974025949835777, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7972590923309326, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.5902857780456543, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.010382652282715, "metrics/particle/momentum/class_5/accepted/samples": 77.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.4935447573661804, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 1.1886310577392578, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.9960305094718933, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4359091520309448, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.3385826647281647, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.432369232177734, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.11023622006177902, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 4.6964216232299805, "metrics/particle/momentum/class_5/assigned/pz_bias": 1.5229887962341309, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 10.931602478027344, "metrics/particle/momentum/class_5/assigned/samples": 127.0, "metrics/particle/momentum/class_5/efficiency": 0.4695121943950653, "metrics/particle/momentum/class_5/event/energy_bias": -4.224202632904053, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.528429985046387, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.442176580429077, "metrics/particle/momentum/class_5/event/px_bias": 0.24981549382209778, "metrics/particle/momentum/class_5/event/py_bias": -0.28377509117126465, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.3234829902648926, "metrics/particle/momentum/class_5/event/pz_bias": 0.519278347492218, "metrics/particle/momentum/class_5/event/pz_residual_rms": 7.991452217102051, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6734198331832886, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.680116593837738, "metrics/set_slot/angular_recall_all_dr0p10": 0.8371818661689758, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6284577250480652, "metrics/set_slot/soft_count_bias_mean": 1.352185845375061}, "step_train_time": 5.465908765792847, "step_valid_time": 8809.08369755745, "step_total_time": 8814.54960656166, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1025708799336735, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2921182325450453, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2955, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2543, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2330, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8313174239947695, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.860575296108291, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8456933821084137, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13942470389170897, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7616868257600523, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7884940778341794, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7748586631193881, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8313174239947695, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0370026897975777, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2619351324983279, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2262, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1963, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1895, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8590809628008753, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.867816091954023, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8634264350120959, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13218390804597702, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8293216630196937, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8377541998231653, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8335166043545195, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8590809628008753, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0265846835360244, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27083652009449277, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1264, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1076, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1022, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8139183055975794, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8512658227848101, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8321732405259087, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.14873417721518986, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.773071104387292, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8085443037974683, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7904098994586234, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8139183055975794} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_31000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..5a69329821f5400b523a3c3f0076c2c8f5cf62b6 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 1.4683592269799166, "train": {"Classification_binary": 0.159181147813797, "Cardinality": 0.12174397706985474, "Classification": 0.0820540189743042, "Regression_pt": 0.10987113416194916, "Regression_eta": 0.0017688744701445103, "Regression_sin_phi": 0.0008562179282307625, "Regression_cos_phi": 0.0007581430836580694, "Regression_energy": 0.10848668217658997, "HitGrouping": 0.15537212789058685, "Auxiliary": 0.20055744051933289, "Total": 0.9921956062316895, "Total_unweighted": 0.46297621860867366}, "train_group": {}, "valid": {"Classification_binary": 0.15898075699806213, "Cardinality": 0.12422723571459453, "Classification": 0.08585893114407857, "Regression_pt": 0.11435846487681071, "Regression_eta": 0.001453785536189874, "Regression_sin_phi": 0.0007720771245658398, "Regression_cos_phi": 0.0007378372053305308, "Regression_energy": 0.11391877134641011, "HitGrouping": 0.15654242038726807, "Auxiliary": 0.2025899887084961, "Total": 0.9879931608835856, "Total_unweighted": 0.47608062423144776, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8499550223350525, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5980547070503235, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3025456666946411, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6099677681922913, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6109716892242432, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.33954763412475586, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7947244644165039, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5460313558578491, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2884047329425812, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5564372539520264, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5539854764938354, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.41691285371780396, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8635251522064209, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5626895427703857, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2886368930339813, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5705432295799255, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5633266568183899, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31525111198425293, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8540724515914917, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5721101760864258, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2928325831890106, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5813918113708496, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5770356059074402, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3305045962333679, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0034123812802135944, "metrics/hit_grouping/calorimeter/background_recall": 0.15469613671302795, "metrics/hit_grouping/calorimeter/completeness": 0.6462271809577942, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6455214023590088, "metrics/hit_grouping/calorimeter/energy_purity": 0.6371618509292603, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3330064117908478, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8767135739326477, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5914037227630615, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.22434000670909882, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6032154560089111, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5905084609985352, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2714497148990631, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8542776703834534, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.5967751145362854, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.31057116389274597, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6068413853645325, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6033293008804321, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33694151043891907, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8077009916305542, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5079230666160583, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4146171808242798, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5112553834915161, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5230759978294373, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4444315433502197, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8372713327407837, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6214099526405334, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.35337984561920166, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6346949338912964, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6462862491607666, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3768342435359955, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8201704025268555, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6545684337615967, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4678332507610321, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6548467874526978, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6511898040771484, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.40452906489372253, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8752413988113403, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5120222568511963, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21025200188159943, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5182949900627136, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.49759599566459656, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2744496762752533, "metrics/hit_grouping/calorimeter/purity": 0.6369836330413818, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7890552282333374, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.11622656136751175, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10902255773544312, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.11793322116136551, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.08679500967264175, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.827624499797821, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.26827794313430786, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1059446707367897, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.26697230339050293, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.25081926584243774, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2472042441368103, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.856508195400238, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.5953495502471924, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.30555298924446106, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.605305552482605, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6019997000694275, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3397960066795349, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7194342017173767, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40011587738990784, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.506241500377655, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5605317950248718, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6035057306289673, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7690777778625488, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28270041942596436, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.48828428983688354, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.496684193611145, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.47136828303337097, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6391651630401611, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43272411823272705, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.40725192427635193, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4656420052051544, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7053532600402832, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6734315156936646, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4116206467151642, "metrics/hit_grouping/tracker/all/object_completeness": 0.4436614215373993, "metrics/hit_grouping/tracker/all/object_purity": 0.49709901213645935, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6567676067352295, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.010816391557455063, "metrics/hit_grouping/tracker/background_recall": 0.11008051782846451, "metrics/hit_grouping/tracker/completeness": 0.44519102573394775, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4639337658882141, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7079248428344727, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.38506048917770386, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5179546475410461, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.557371199131012, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6230054497718811, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6506696939468384, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43474313616752625, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.41684985160827637, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4832609295845032, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6877835988998413, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6250931620597839, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.44328224658966064, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3174784779548645, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3868679106235504, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6954012513160706, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6699561476707458, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4253205358982086, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.44682547450065613, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5220680236816406, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.662307858467102, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6519420146942139, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4532019793987274, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4021649658679962, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4796992838382721, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.671798050403595, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6818020939826965, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3863561153411865, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.44698700308799744, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.46775752305984497, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.646870493888855, "metrics/hit_grouping/tracker/purity": 0.41987401247024536, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8883957266807556, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.040346909314394, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.11808323115110397, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.12195322662591934, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06674208492040634, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6816487312316895, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.28755807876586914, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.35390517115592957, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3641529977321625, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5167785286903381, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.640640914440155, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4801633954048157, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5017046928405762, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5671791434288025, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.759498655796051, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6004661321640015, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6643441915512085, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.761916995048523, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.647993803024292, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.0513133779168129, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07351894676685333, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.25043487548828, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30271127820014954, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.21108810603618622, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8819277286529541, "metrics/particle/class_1/pid_accuracy": 0.9734446406364441, "metrics/particle/class_2/efficiency": 0.5702797770500183, "metrics/particle/class_2/pid_accuracy": 0.7449851036071777, "metrics/particle/class_3/efficiency": 0.7811905741691589, "metrics/particle/class_3/pid_accuracy": 0.9747859239578247, "metrics/particle/class_4/efficiency": 0.6863727569580078, "metrics/particle/class_4/pid_accuracy": 0.45255473256111145, "metrics/particle/class_5/efficiency": 0.9042145609855652, "metrics/particle/class_5/pid_accuracy": 0.5529661178588867, "metrics/particle/count/bias_mean": -2.033203125, "metrics/particle/count/mae": 3.7962238788604736, "metrics/particle/count/prediction_mean": 68.62174224853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07677426934242249, "metrics/particle/event/energy_response_mean": 1.0019594430923462, "metrics/particle/event/met_abs_error": 5.694910526275635, "metrics/particle/event/scalar_pt_relative_abs_error": 0.060313042253255844, "metrics/particle/event/scalar_pt_response_mean": 1.0208110809326172, "metrics/particle/event/vector_pt_closure": 0.054498378187417984, "metrics/particle/matched/delta_eta_abs_mean": 0.013527795672416687, "metrics/particle/matched/delta_phi_abs_mean": 0.013781702145934105, "metrics/particle/matched/delta_r_mean": 0.021700289100408554, "metrics/particle/matched/energy_relative_abs_mean": 0.1394157111644745, "metrics/particle/matched/pid_accuracy": 0.9546439051628113, "metrics/particle/matched/pt_relative_abs_mean": 0.138947993516922, "metrics/particle/matching/duplicate_fraction": 0.05330967903137207, "metrics/particle/matching/efficiency": 0.8124228119850159, "metrics/particle/matching/f1": 0.8242828249931335, "metrics/particle/matching/purity": 0.8364942073822021, "metrics/particle/matching/target_coverage_dr0p05": 0.7647660374641418, "metrics/particle/matching/target_coverage_dr0p10": 0.8552236557006836, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6267991065979004, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8461382389068604, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020984284579753876, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1337808072566986, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13389652967453003, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02678663842380047, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17395271360874176, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00533605832606554, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6251104474067688, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.03289507329463959, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6944547891616821, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8808.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.204556405544281, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9031183123588562, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9950206875801086, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8645543456077576, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11142413318157196, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.910104751586914, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013308723457157612, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9216908812522888, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.053846269845962524, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7189524173736572, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10294.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8176754713058472, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.2489407062530518, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.1349639892578125, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.207617998123169, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.537174642086029, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.22412914037704468, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.111886501312256, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.6587592363357544, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.245668411254883, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023825377225875854, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1491713970899582, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14704830944538116, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.011023040860891342, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1871148943901062, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022765129804611206, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2366169691085815, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.06526288390159607, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.987384796142578, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1801.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4424130618572235, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1633309125900269, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.1672685146331787, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.99894380569458, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17826825380325317, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 17.01494789123535, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05348047614097595, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.472933053970337, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.09218788892030716, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.492030143737793, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2356.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6530094146728516, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.898459434509277, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.462924003601074, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1078064441680908, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06461036950349808, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0512307807803154, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.33885669708252, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6470469236373901, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.47745704650879, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021958038210868835, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.14153043925762177, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.14113320410251617, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.03818044438958168, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.18067847192287445, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006749039748683572, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.22616688907146454, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.019283678382635117, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6866365075111389, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19262.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.19695809483528137, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0036115646362305, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7697424292564392, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.637230396270752, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11744006723165512, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.424647331237793, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003562788711860776, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3853844106197357, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02828351967036724, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4647599458694458, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22735.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8277966380119324, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.165406703948975, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.74005126953125, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.350764274597168, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.9437202215194702, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.24784791469573975, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.9405910968780518, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.044712781906128, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.47926139831543, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021783497184515, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13970601558685303, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13935597240924835, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.03318338841199875, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17912043631076813, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0033812059555202723, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.40472519397735596, "metrics/particle/momentum/all/accepted/pz_bias": 0.02606944739818573, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2995073795318604, "metrics/particle/momentum/all/accepted/samples": 29871.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21551141142845154, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.985011100769043, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9283291101455688, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7940276265144348, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11974000185728073, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.967926025390625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009721633046865463, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6803964376449585, "metrics/particle/momentum/all/assigned/pz_bias": 0.02769887074828148, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8862903118133545, "metrics/particle/momentum/all/assigned/samples": 35385.0, "metrics/particle/momentum/all/efficiency": 0.8117340207099915, "metrics/particle/momentum/all/event/energy_bias": 3.515888214111328, "metrics/particle/momentum/all/event/p3_error_mean": 18.615070343017578, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.698465347290039, "metrics/particle/momentum/all/event/px_bias": -1.5455050468444824, "metrics/particle/momentum/all/event/py_bias": 0.07494955509901047, "metrics/particle/momentum/all/event/pz_abs_mean": 14.114816665649414, "metrics/particle/momentum/all/event/pz_bias": 2.056424856185913, "metrics/particle/momentum/all/event/pz_residual_rms": 25.726667404174805, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02353675104677677, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.12734359502792358, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12780220806598663, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.03912070766091347, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.16587314009666443, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002302231965586543, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.42391690611839294, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.03971193730831146, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2224273681640625, "metrics/particle/momentum/class_1/accepted/samples": 15637.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13146114349365234, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.37975913286209106, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.417064905166626, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.30149731040000916, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06911247968673706, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.207374095916748, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006162529345601797, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5902738571166992, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.04079793393611908, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.324939250946045, "metrics/particle/momentum/class_1/assigned/samples": 17363.0, "metrics/particle/momentum/class_1/efficiency": 0.8814046382904053, "metrics/particle/momentum/class_1/event/energy_bias": 9.16495132446289, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.010685920715332, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.49368953704834, "metrics/particle/momentum/class_1/event/px_bias": -0.46041035652160645, "metrics/particle/momentum/class_1/event/py_bias": 0.7948529720306396, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.594232559204102, "metrics/particle/momentum/class_1/event/pz_bias": 2.330482006072998, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.036340713500977, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027411989867687225, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18159887194633484, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.18153657019138336, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.022263260558247566, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22211803495883942, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02548903413116932, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.335472822189331, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.00957005750387907, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.1835520267486572, "metrics/particle/momentum/class_2/accepted/samples": 1687.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5105870962142944, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5339842438697815, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7451605796813965, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4645681083202362, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2395666092634201, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.965546607971191, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.046147674322128296, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.154365062713623, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.002739757765084505, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.5122151374816895, "metrics/particle/momentum/class_2/assigned/samples": 2492.0, "metrics/particle/momentum/class_2/efficiency": 0.5649698376655579, "metrics/particle/momentum/class_2/event/energy_bias": -7.13441276550293, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.340063095092773, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.451862335205078, "metrics/particle/momentum/class_2/event/px_bias": -0.4916898012161255, "metrics/particle/momentum/class_2/event/py_bias": -0.6439875364303589, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.04541301727295, "metrics/particle/momentum/class_2/event/pz_bias": -1.1440496444702148, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.27700424194336, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018811024725437164, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14917083084583282, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1478845328092575, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.025828182697296143, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1879757195711136, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013170891907066107, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23324887454509735, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0025783309247344732, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8044722080230713, "metrics/particle/momentum/class_3/accepted/samples": 12148.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2613890469074249, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6788012981414795, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5165964365005493, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.383721947669983, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15570656955242157, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.308144569396973, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007395562715828419, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5231412053108215, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.001146695576608181, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.824937343597412, "metrics/particle/momentum/class_3/assigned/samples": 15009.0, "metrics/particle/momentum/class_3/efficiency": 0.782076895236969, "metrics/particle/momentum/class_3/event/energy_bias": 2.493990898132324, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.277996063232422, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.059528350830078, "metrics/particle/momentum/class_3/event/px_bias": -0.27644509077072144, "metrics/particle/momentum/class_3/event/py_bias": -0.20843404531478882, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.5541510581970215, "metrics/particle/momentum/class_3/event/pz_bias": 0.5870878100395203, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.670143127441406, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022363198921084404, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1646909862756729, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15949805080890656, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0877600759267807, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20349368453025818, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0078125, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6260875463485718, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.23428170382976532, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5940064191818237, "metrics/particle/momentum/class_4/accepted/samples": 256.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3716672956943512, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.660243511199951, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.502182722091675, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.415792465209961, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23676879703998566, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.159394264221191, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011142061091959476, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8047149777412415, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2675292491912842, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.114262580871582, "metrics/particle/momentum/class_4/assigned/samples": 359.0, "metrics/particle/momentum/class_4/efficiency": 0.6826666593551636, "metrics/particle/momentum/class_4/event/energy_bias": -0.6139686107635498, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0197913646698, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4565852880477905, "metrics/particle/momentum/class_4/event/px_bias": -0.20746590197086334, "metrics/particle/momentum/class_4/event/py_bias": 0.10563568025827408, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1099064350128174, "metrics/particle/momentum/class_4/event/pz_bias": -0.1071813628077507, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.8530991077423096, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0151417450979352, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14853805303573608, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14457595348358154, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03989475965499878, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1883089244365692, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0279720276594162, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.496708631515503, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.35176581144332886, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.2027370929718018, "metrics/particle/momentum/class_5/accepted/samples": 143.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08834303170442581, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.37046685814857483, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.5531248450279236, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.42291390895843506, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.1111111119389534, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.1173996925354, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.043209876865148544, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.95992910861969, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.9362279772758484, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.861680030822754, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.8719512224197388, "metrics/particle/momentum/class_5/event/energy_bias": -0.3946721851825714, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8524672985076904, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4007090330123901, "metrics/particle/momentum/class_5/event/px_bias": -0.10949404537677765, "metrics/particle/momentum/class_5/event/py_bias": 0.026882503181695938, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.972213864326477, "metrics/particle/momentum/class_5/event/pz_bias": 0.39008599519729614, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4887495040893555, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3360816538333893, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.33765938878059387, "metrics/set_slot/angular_recall_all_dr0p10": 0.9559644460678101, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8413099050521851, "metrics/set_slot/soft_count_bias_mean": -0.8521141409873962}, "step_train_time": 5.14007568359375, "step_valid_time": 2787.0267164707184, "step_total_time": 2792.166792869568, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0296498991964866, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13086728040551343, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3049, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2922, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2900, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552141222621772, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9583469990160708, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9567779960707269, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041653000983929156, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9480222294867604, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9511315185306658, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9495743287491814, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552141222621772, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0243976979777363, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12631792290523136, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2272, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2163, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2152, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9466083150984683, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952024647887324, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9493087557603687, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047975352112676055, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9417943107221006, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9471830985915493, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9444810182137371, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9466083150984683, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0297342580857762, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13176536912770576, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1230, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1222, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9304084720121029, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9360730593607306, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9332321699544764, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0639269406392694, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.924357034795764, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9299847792998478, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9271623672230652, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9304084720121029} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_32000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..c0ac49f7e5e52d4b067e037cf7ecfb8c35febc56 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 1.515725653656688, "train": {"Classification_binary": 0.1650419384241104, "Cardinality": 0.11724472045898438, "Classification": 0.08606227487325668, "Regression_pt": 0.10178910940885544, "Regression_eta": 0.001506940694525838, "Regression_sin_phi": 0.0007595614879392087, "Regression_cos_phi": 0.0008167830528691411, "Regression_energy": 0.10100107640028, "HitGrouping": 0.1508377641439438, "Auxiliary": 0.1943080574274063, "Total": 0.9662113189697266, "Total_unweighted": 0.4569776843418367}, "train_group": {}, "valid": {"Classification_binary": 0.15329618255297342, "Cardinality": 0.1225743293762207, "Classification": 0.0862102210521698, "Regression_pt": 0.1096363365650177, "Regression_eta": 0.001342292254169782, "Regression_sin_phi": 0.0007331996069600185, "Regression_cos_phi": 0.0007310928000758091, "Regression_energy": 0.10829809308052063, "HitGrouping": 0.15363802512486777, "Auxiliary": 0.1973492701848348, "Total": 0.9601074059804281, "Total_unweighted": 0.46024741791188717, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8488351106643677, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6081238389015198, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.299328088760376, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6191744208335876, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6198134422302246, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.32828617095947266, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7936392426490784, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5514208078384399, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2994830012321472, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5605432987213135, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5619874596595764, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4087887704372406, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8625764846801758, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5783187747001648, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2715112268924713, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.586360514163971, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5902618765830994, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3099745512008667, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8530613780021667, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5852600932121277, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2821100950241089, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5942972302436829, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5970195531845093, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32319381833076477, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.00389371020719409, "metrics/hit_grouping/calorimeter/background_recall": 0.16505524516105652, "metrics/hit_grouping/calorimeter/completeness": 0.6527219414710999, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6501287817955017, "metrics/hit_grouping/calorimeter/energy_purity": 0.6418557167053223, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3184339106082916, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8748192191123962, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6048219203948975, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21626080572605133, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6162137985229492, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6103734374046326, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.26234638690948486, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8557787537574768, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6069314479827881, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.29272225499153137, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6163008213043213, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6219049692153931, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3270382285118103, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8059601187705994, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5235413908958435, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4056844413280487, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.527442991733551, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5447207689285278, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4433874785900116, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.839096188545227, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6357211470603943, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3377435505390167, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6483012437820435, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.664444625377655, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.36991819739341736, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.818746030330658, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6539819836616516, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4580545425415039, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6556969285011292, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6667550802230835, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.38600102066993713, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8713880777359009, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5256245136260986, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.20442700386047363, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5319219827651978, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5199676156044006, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.26807284355163574, "metrics/hit_grouping/calorimeter/purity": 0.6436963677406311, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7516917586326599, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08443086594343185, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.08646616339683533, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08335123211145401, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06638920307159424, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03947368264198303, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8058559894561768, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2902242839336395, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11300764977931976, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.28804048895835876, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2739863991737366, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.201883465051651, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8571696281433105, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6087738871574402, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2940978407859802, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6185599565505981, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6224759817123413, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3341296911239624, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7212532758712769, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3879472613334656, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5049340724945068, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5587962865829468, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6050992608070374, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7725690007209778, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28631705045700073, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4909886121749878, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.4971805214881897, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4412296712398529, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6369794607162476, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.42977407574653625, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4179835021495819, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.48962298035621643, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7096704840660095, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6728960275650024, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4063223600387573, "metrics/hit_grouping/tracker/all/object_completeness": 0.45009997487068176, "metrics/hit_grouping/tracker/all/object_purity": 0.5114414095878601, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6577025651931763, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.012417937628924847, "metrics/hit_grouping/tracker/background_recall": 0.1199873685836792, "metrics/hit_grouping/tracker/completeness": 0.44788095355033875, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4412405490875244, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7012912631034851, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.38067683577537537, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5195156931877136, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5734485983848572, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6294055581092834, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.655680239200592, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4340170621871948, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.42679938673973083, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.49530288577079773, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6796151995658875, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6315910220146179, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4315599501132965, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3304618000984192, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.39992690086364746, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6941388845443726, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6740812659263611, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43160396814346313, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4585050046443939, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5283557772636414, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6585717797279358, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.667896568775177, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45258620381355286, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4167934060096741, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4822985529899597, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6644088625907898, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.672251284122467, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3651617169380188, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.44514384865760803, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4946765601634979, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6553703546524048, "metrics/hit_grouping/tracker/purity": 0.4230960011482239, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8842048645019531, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03883861377835274, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.12713484466075897, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1254238784313202, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06033182516694069, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6842010021209717, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2978833317756653, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3862296938896179, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3962230980396271, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5209085941314697, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6403130888938904, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4726092517375946, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5049497485160828, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5812133550643921, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7611773014068604, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5819317102432251, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6524588465690613, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7487903237342834, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6543475985527039, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.03277895227074623, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07987286150455475, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.251073837280273, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.26499077677726746, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.17336760461330414, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8870007991790771, "metrics/particle/class_1/pid_accuracy": 0.9747575521469116, "metrics/particle/class_2/efficiency": 0.5732246041297913, "metrics/particle/class_2/pid_accuracy": 0.7575578093528748, "metrics/particle/class_3/efficiency": 0.806833803653717, "metrics/particle/class_3/pid_accuracy": 0.9753976464271545, "metrics/particle/class_4/efficiency": 0.680360734462738, "metrics/particle/class_4/pid_accuracy": 0.4241531789302826, "metrics/particle/class_5/efficiency": 0.9214559197425842, "metrics/particle/class_5/pid_accuracy": 0.5467775464057922, "metrics/particle/count/bias_mean": -1.9283853769302368, "metrics/particle/count/mae": 3.6875, "metrics/particle/count/prediction_mean": 68.7265625, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06725665926933289, "metrics/particle/event/energy_response_mean": 0.9838301539421082, "metrics/particle/event/met_abs_error": 5.6730122566223145, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05307556316256523, "metrics/particle/event/scalar_pt_response_mean": 1.0060677528381348, "metrics/particle/event/vector_pt_closure": 0.052387386560440063, "metrics/particle/matched/delta_eta_abs_mean": 0.012899162247776985, "metrics/particle/matched/delta_phi_abs_mean": 0.013659410178661346, "metrics/particle/matched/delta_r_mean": 0.021090026944875717, "metrics/particle/matched/energy_relative_abs_mean": 0.1346115916967392, "metrics/particle/matched/pid_accuracy": 0.9562903642654419, "metrics/particle/matched/pt_relative_abs_mean": 0.13476544618606567, "metrics/particle/matching/duplicate_fraction": 0.04770565778017044, "metrics/particle/matching/efficiency": 0.8259495496749878, "metrics/particle/matching/f1": 0.8373768329620361, "metrics/particle/matching/purity": 0.8491247296333313, "metrics/particle/matching/target_coverage_dr0p05": 0.7787442803382874, "metrics/particle/matching/target_coverage_dr0p10": 0.863240122795105, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6455135345458984, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8573060631752014, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020902050659060478, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13247695565223694, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1334785670042038, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01579851284623146, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1728566288948059, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004455335438251495, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6171706318855286, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0046495553106069565, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6781384944915771, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8978.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18407021462917328, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.6394082903862, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.6541323661804199, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.5115365386009216, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09636098891496658, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.910939693450928, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.009995147585868835, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8610636591911316, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.041373006999492645, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.582869052886963, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10305.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8334571123123169, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.2243475764989853, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.76546573638916, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.1558618545532227, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.047922831028699875, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.1358795166015625, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.693455219268799, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.751710057258606, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.034552574157715, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023548375815153122, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14419592916965485, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14588892459869385, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.003807468106970191, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18677353858947754, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.026864035055041313, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3261873722076416, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.13079650700092316, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.3756613731384277, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1824.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.421474814414978, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1901239156723022, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.024894714355469, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.835021018981934, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17123576998710632, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 138.4514617919922, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05820329114794731, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4386355876922607, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.30414333939552307, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.652834415435791, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2371.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.661348819732666, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.8059797286987305, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.989248275756836, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.086726188659668, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.0711786225438118, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.008893669582903385, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.851960182189941, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.2499128580093384, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.17080307006836, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021059855818748474, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1355459988117218, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13540221750736237, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02061126008629799, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17521391808986664, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003577817406039685, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2163037806749344, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.016508441418409348, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.648718535900116, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19565.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18269042670726776, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8875247240066528, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7039690017700195, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.557620108127594, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10527940839529037, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.1074347496032715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002822740701958537, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3627362549304962, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.01560839544981718, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4106580018997192, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22673.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8408182263374329, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.342541456222534, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.49117374420166, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.154000759124756, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.15298393368721008, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.2970037460327148, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.840466022491455, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.20854412019252777, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.757618427276611, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021162673830986023, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1351582109928131, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13546337187290192, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01772165484726429, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17523890733718872, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003161326516419649, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4014854431152344, "metrics/particle/momentum/all/accepted/pz_bias": 0.001405180897563696, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3372286558151245, "metrics/particle/momentum/all/accepted/samples": 30367.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19910889863967896, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8354899883270264, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9792623519897461, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8310882449150085, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10710345208644867, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 36.32959747314453, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.00862825009971857, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6472483277320862, "metrics/particle/momentum/all/assigned/pz_bias": 0.0016722992295399308, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.673839807510376, "metrics/particle/momentum/all/assigned/samples": 35349.0, "metrics/particle/momentum/all/efficiency": 0.8252126574516296, "metrics/particle/momentum/all/event/energy_bias": -2.239091157913208, "metrics/particle/momentum/all/event/p3_error_mean": 16.90993309020996, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.494863510131836, "metrics/particle/momentum/all/event/px_bias": 0.27208542823791504, "metrics/particle/momentum/all/event/py_bias": 1.4239894151687622, "metrics/particle/momentum/all/event/pz_abs_mean": 12.403828620910645, "metrics/particle/momentum/all/event/pz_bias": -0.289658784866333, "metrics/particle/momentum/all/event/pz_residual_rms": 23.556535720825195, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.022642921656370163, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11752692610025406, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1179170235991478, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01630607806146145, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1552046835422516, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0024182258639484644, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4173642694950104, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.016206758096814156, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.253353238105774, "metrics/particle/momentum/class_1/accepted/samples": 15714.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13075178861618042, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3046415448188782, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.32518666982650757, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.19493241608142853, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06538660824298859, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.6907589435577393, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005074093118309975, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5481725931167603, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.018819177523255348, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1573500633239746, "metrics/particle/momentum/class_1/assigned/samples": 17343.0, "metrics/particle/momentum/class_1/efficiency": 0.8857448697090149, "metrics/particle/momentum/class_1/event/energy_bias": 3.404266595840454, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.587423324584961, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.655516624450684, "metrics/particle/momentum/class_1/event/px_bias": 0.44983577728271484, "metrics/particle/momentum/class_1/event/py_bias": 1.8691190481185913, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.782510757446289, "metrics/particle/momentum/class_1/event/pz_bias": -0.7643031477928162, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.246612548828125, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.026662681251764297, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17541347444057465, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.177461639046669, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.008606687188148499, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2185022234916687, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.021264027804136276, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3125429153442383, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.10267133265733719, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.1377692222595215, "metrics/particle/momentum/class_2/accepted/samples": 1693.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.4893735945224762, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6325015425682068, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8478380441665649, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5525058507919312, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2345132678747177, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.062617301940918, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04505229368805885, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.2245888710021973, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.157881498336792, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.686953544616699, "metrics/particle/momentum/class_2/assigned/samples": 2486.0, "metrics/particle/momentum/class_2/efficiency": 0.5669792294502258, "metrics/particle/momentum/class_2/event/energy_bias": -6.752832412719727, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.548798561096191, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.888430595397949, "metrics/particle/momentum/class_2/event/px_bias": 0.04803396761417389, "metrics/particle/momentum/class_2/event/py_bias": -0.4586286246776581, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.928199768066406, "metrics/particle/momentum/class_2/event/pz_bias": -0.5241791009902954, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.51438331604004, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018614986911416054, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1511218249797821, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1511191427707672, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.019084010273218155, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19092032313346863, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001513220719061792, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24152196943759918, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.029674407094717026, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9399768114089966, "metrics/particle/momentum/class_3/accepted/samples": 12556.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.22719231247901917, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.432647466659546, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7313858270645142, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.585084080696106, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13091903924942017, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 55.628475189208984, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006535947788506746, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.48687922954559326, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0440741702914238, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.428274154663086, "metrics/particle/momentum/class_3/assigned/samples": 14994.0, "metrics/particle/momentum/class_3/efficiency": 0.8083435297012329, "metrics/particle/momentum/class_3/event/energy_bias": 2.089700222015381, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.290512084960938, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.08128023147583, "metrics/particle/momentum/class_3/event/px_bias": -0.4214380979537964, "metrics/particle/momentum/class_3/event/py_bias": 0.01676367223262787, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.628825664520264, "metrics/particle/momentum/class_3/event/pz_bias": 0.8429404497146606, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.845564842224121, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02282492071390152, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16695603728294373, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16672535240650177, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.09402169287204742, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20802541077136993, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6539251208305359, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.13939239084720612, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3083022832870483, "metrics/particle/momentum/class_4/accepted/samples": 254.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3803485035896301, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.2005579471588135, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.4124832153320312, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.3350539207458496, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2637362778186798, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.312225341796875, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7663821578025818, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.17483699321746826, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6918752193450928, "metrics/particle/momentum/class_4/assigned/samples": 364.0, "metrics/particle/momentum/class_4/efficiency": 0.6773333549499512, "metrics/particle/momentum/class_4/event/energy_bias": -0.3820451498031616, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.336733341217041, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7589446306228638, "metrics/particle/momentum/class_4/event/px_bias": 0.04523005336523056, "metrics/particle/momentum/class_4/event/py_bias": 0.2034667730331421, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2079575061798096, "metrics/particle/momentum/class_4/event/pz_bias": 0.047506652772426605, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.557685375213623, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014458701014518738, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1377565711736679, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13616937398910522, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02565564587712288, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1733875572681427, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.019999999552965164, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4177583456039429, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.4211277663707733, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.076099395751953, "metrics/particle/momentum/class_5/accepted/samples": 150.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.056305285543203354, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.19641941785812378, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18499571084976196, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.04437396675348282, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.06172839552164078, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.30418097972869873, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03086419776082039, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6238832473754883, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.3302459120750427, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6452653408050537, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.9146341681480408, "metrics/particle/momentum/class_5/event/energy_bias": -0.5981804728507996, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6811575889587402, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.306418538093567, "metrics/particle/momentum/class_5/event/px_bias": 0.1504238098859787, "metrics/particle/momentum/class_5/event/py_bias": -0.20673136413097382, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7931724786758423, "metrics/particle/momentum/class_5/event/pz_bias": 0.10837629437446594, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.7373523712158203, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.32601410150527954, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.32826587557792664, "metrics/set_slot/angular_recall_all_dr0p10": 0.9583141207695007, "metrics/set_slot/angular_recall_selected_dr0p10": 0.849317193031311, "metrics/set_slot/soft_count_bias_mean": -0.7145116329193115}, "step_train_time": 4.952741384506226, "step_valid_time": 5783.6769869327545, "step_total_time": 5788.629728794098, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0155045642612122, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11899072013812106, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3041, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2906, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2896, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9499836547891468, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9556067083196317, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9527868852459016, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0443932916803683, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9467146126185028, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9523183163433081, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9495081967213115, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9499836547891468, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0063404105587586, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11561693240864601, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2273, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2161, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2155, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9457330415754923, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9507259128904532, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9482229047827995, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04927408710954685, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9431072210065645, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9480862296524417, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9455901711276876, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9457330415754923, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0180613644494736, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13003210816375954, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1310, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1237, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1233, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9357034795763994, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9442748091603054, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9399696048632219, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.055725190839694655, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.93267776096823, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9412213740458015, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9369300911854104, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9357034795763994} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_33000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..6f9b675d639e1f723c795a4af7feeb8ed94a3815 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 1.5630920803334596, "train": {"Classification_binary": 0.17569458484649658, "Cardinality": 0.11089136451482773, "Classification": 0.09022916108369827, "Regression_pt": 0.11109031736850739, "Regression_eta": 0.0014687007060274482, "Regression_sin_phi": 0.0007393760606646538, "Regression_cos_phi": 0.0008122549625113606, "Regression_energy": 0.1096058338880539, "HitGrouping": 0.15049880743026733, "Auxiliary": 0.20473046600818634, "Total": 0.9899652004241943, "Total_unweighted": 0.4896402289159596}, "train_group": {}, "valid": {"Classification_binary": 0.15293320020039877, "Cardinality": 0.11455049117406209, "Classification": 0.08436252673467, "Regression_pt": 0.10935827096303304, "Regression_eta": 0.0013430975377559662, "Regression_sin_phi": 0.0007301206545283397, "Regression_cos_phi": 0.0007013105787336826, "Regression_energy": 0.10812864700953166, "HitGrouping": 0.15319724877675375, "Auxiliary": 0.19590246677398682, "Total": 0.9457102616628011, "Total_unweighted": 0.45755717367865145, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8448234796524048, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6036853194236755, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2912841737270355, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6147347688674927, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6202828884124756, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3445632755756378, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7882445454597473, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5399840474128723, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.27400293946266174, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5512263774871826, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5693722367286682, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.42872968316078186, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8600354790687561, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5692842602729797, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.27174264192581177, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5773159861564636, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5761016607284546, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3090488314628601, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8498533368110657, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.577431321144104, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2778382897377014, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5866266489028931, "metrics/hit_grouping/calorimeter/all/object_purity": 0.58896404504776, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32909977436065674, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.004230615682899952, "metrics/hit_grouping/calorimeter/background_recall": 0.17919155955314636, "metrics/hit_grouping/calorimeter/completeness": 0.6538448929786682, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6511437892913818, "metrics/hit_grouping/calorimeter/energy_purity": 0.6429225206375122, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3212207853794098, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8701165318489075, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5897598266601562, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21074192225933075, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6015249490737915, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5954314470291138, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2690031826496124, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8556023836135864, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6064469814300537, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2885766923427582, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6159052848815918, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6174980998039246, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3270382285118103, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8027452230453491, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5230628252029419, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.40382829308509827, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5267532467842102, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5470314621925354, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4537122845649719, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8365628123283386, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6390437483787537, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3397281765937805, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6530364155769348, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6693857908248901, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.37725523114204407, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8122963309288025, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6527026295661926, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4369531571865082, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6530264019966125, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6771532297134399, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4091610908508301, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.867878258228302, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.505646288394928, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.19578146934509277, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5110071301460266, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.49646270275115967, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.270464152097702, "metrics/hit_grouping/calorimeter/purity": 0.6450219750404358, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7346476316452026, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08034958690404892, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.06954887509346008, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07994987070560455, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05694578215479851, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.033834587782621384, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7889432311058044, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2299780398607254, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09770452976226807, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2268427014350891, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2068389505147934, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2071806937456131, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8549001812934875, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.603611946105957, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.29060614109039307, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.613605260848999, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6175181865692139, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3402554392814636, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7265641093254089, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40243372321128845, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5132411122322083, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.553765594959259, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5923511385917664, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.767792820930481, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.26401445269584656, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5026962757110596, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5191853642463684, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.44605183601379395, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6479726433753967, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43545833230018616, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4191168546676636, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4632410705089569, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6920419931411743, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6809778809547424, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41264471411705017, "metrics/hit_grouping/tracker/all/object_completeness": 0.45421916246414185, "metrics/hit_grouping/tracker/all/object_purity": 0.49519580602645874, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6432324051856995, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013161796145141125, "metrics/hit_grouping/tracker/background_recall": 0.12373697757720947, "metrics/hit_grouping/tracker/completeness": 0.4516434669494629, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.45972535014152527, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.712394118309021, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3929510712623596, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.52516770362854, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5530416965484619, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6105558276176453, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6610040068626404, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4405518174171448, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.42800718545913696, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.48573583364486694, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.670176088809967, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6361993551254272, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4254283010959625, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3343205749988556, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3856058716773987, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6836789846420288, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6821499466896057, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43763267993927, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4645598232746124, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5182068943977356, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6463445425033569, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6786515712738037, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4519704580307007, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4268597662448883, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4689081609249115, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6459359526634216, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6798713803291321, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3731096088886261, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4456806480884552, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.46999305486679077, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.638701856136322, "metrics/hit_grouping/tracker/purity": 0.4269719123840332, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8776060342788696, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04072398319840431, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.12394500523805618, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.12593525648117065, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06410256773233414, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6853581070899963, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2968508005142212, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3736792206764221, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3766382038593292, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5116158723831177, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6513241529464722, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.48038721084594727, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5119603872299194, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5628430843353271, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7434390783309937, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5920421481132507, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6726512908935547, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7741162180900574, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6641151905059814, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.04288940504193306, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.08964038640260696, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.221601486206055, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2854824662208557, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1938592940568924, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8938856720924377, "metrics/particle/class_1/pid_accuracy": 0.9743759632110596, "metrics/particle/class_2/efficiency": 0.5939517617225647, "metrics/particle/class_2/pid_accuracy": 0.7576277852058411, "metrics/particle/class_3/efficiency": 0.8119930624961853, "metrics/particle/class_3/pid_accuracy": 0.974342405796051, "metrics/particle/class_4/efficiency": 0.7064128518104553, "metrics/particle/class_4/pid_accuracy": 0.48510637879371643, "metrics/particle/class_5/efficiency": 0.9195402264595032, "metrics/particle/class_5/pid_accuracy": 0.5708333253860474, "metrics/particle/count/bias_mean": -0.912109375, "metrics/particle/count/mae": 3.494140625, "metrics/particle/count/prediction_mean": 69.74283599853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07421644777059555, "metrics/particle/event/energy_response_mean": 0.9439045786857605, "metrics/particle/event/met_abs_error": 5.01019811630249, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05849944427609444, "metrics/particle/event/scalar_pt_response_mean": 0.957530677318573, "metrics/particle/event/vector_pt_closure": 0.04966091737151146, "metrics/particle/matched/delta_eta_abs_mean": 0.013745814561843872, "metrics/particle/matched/delta_phi_abs_mean": 0.013433406129479408, "metrics/particle/matched/delta_r_mean": 0.02156801149249077, "metrics/particle/matched/energy_relative_abs_mean": 0.1334417313337326, "metrics/particle/matched/pid_accuracy": 0.9558390974998474, "metrics/particle/matched/pt_relative_abs_mean": 0.13400812447071075, "metrics/particle/matching/duplicate_fraction": 0.047458577901124954, "metrics/particle/matching/efficiency": 0.833367109298706, "metrics/particle/matching/f1": 0.8387811779975891, "metrics/particle/matching/purity": 0.8442660570144653, "metrics/particle/matching/target_coverage_dr0p05": 0.7798407673835754, "metrics/particle/matching/target_coverage_dr0p10": 0.8682343363761902, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6572526693344116, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8663177490234375, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021053310483694077, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13100102543830872, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13257566094398499, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.002499770373106003, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1712636500597, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006878952495753765, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6404095888137817, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.09203899651765823, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8792587518692017, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9013.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18995149433612823, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.698985755443573, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.774675726890564, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6176876425743103, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10197020322084427, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.733564376831055, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012974531389772892, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8758291006088257, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.09813296794891357, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6904687881469727, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10405.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.836706280708313, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.833233118057251, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.070003986358643, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.172179698944092, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.41697990894317627, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.00721258670091629, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.067537784576416, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.6465821266174316, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.033783912658691, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02372516505420208, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14398665726184845, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14589166641235352, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.002360420534387231, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18628577888011932, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025233132764697075, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3003573417663574, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.13656042516231537, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5279006958007812, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1823.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.48846814036369324, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.8561336398124695, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.2043237686157227, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.0101561546325684, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18530884385108948, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 22.530500411987305, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05592654272913933, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.435072898864746, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.35659411549568176, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.831343650817871, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2396.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6609862446784973, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -7.018986701965332, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.455341339111328, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1652048826217651, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.054005496203899384, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.011876959353685379, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.315958976745605, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3430945873260498, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.114423751831055, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02180144190788269, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13373233377933502, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13377222418785095, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.010854647494852543, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1716996729373932, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008076321100816131, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.22197069227695465, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.027213700115680695, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7107921242713928, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19811.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18505409359931946, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7653535604476929, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6097834706306458, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4381541311740875, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10180137306451797, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.736509323120117, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029659352730959654, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3425801694393158, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.017791572958230972, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.267003059387207, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22927.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8513902425765991, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -3.589714288711548, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.834428310394287, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.54130220413208, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.950128436088562, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.24586619436740875, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.6062018871307373, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.020205020904541, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.906344413757324, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021695852279663086, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13353905081748962, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1341412365436554, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.007611472625285387, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17247435450553894, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00404607318341732, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4091762900352478, "metrics/particle/momentum/all/accepted/pz_bias": -0.052782583981752396, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.4510499238967896, "metrics/particle/momentum/all/accepted/samples": 30647.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20682799816131592, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7521132826805115, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.764738142490387, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5958613157272339, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10745073854923248, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 7.851328372955322, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009432378225028515, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6382045149803162, "metrics/particle/momentum/all/assigned/pz_bias": -0.0639101192355156, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.692795753479004, "metrics/particle/momentum/all/assigned/samples": 35728.0, "metrics/particle/momentum/all/efficiency": 0.8328215479850769, "metrics/particle/momentum/all/event/energy_bias": -13.441934585571289, "metrics/particle/momentum/all/event/p3_error_mean": 16.75082015991211, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.807862281799316, "metrics/particle/momentum/all/event/px_bias": -1.4211138486862183, "metrics/particle/momentum/all/event/py_bias": 0.2649557292461395, "metrics/particle/momentum/all/event/pz_abs_mean": 12.671146392822266, "metrics/particle/momentum/all/event/pz_bias": -4.009881973266602, "metrics/particle/momentum/all/event/pz_residual_rms": 23.465721130371094, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.023292049765586853, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11595050990581512, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11751001328229904, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.016309548169374466, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15283294022083282, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0032826210372149944, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4224545955657959, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.05733064189553261, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.355941653251648, "metrics/particle/momentum/class_1/accepted/samples": 15841.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.13252225518226624, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.30259594321250916, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3241020441055298, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1629316210746765, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06257521361112595, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.6137375831604004, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006589880213141441, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5637165904045105, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.06353230029344559, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.306774139404297, "metrics/particle/momentum/class_1/assigned/samples": 17451.0, "metrics/particle/momentum/class_1/efficiency": 0.8929034471511841, "metrics/particle/momentum/class_1/event/energy_bias": -0.7472100257873535, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.475785255432129, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.645662307739258, "metrics/particle/momentum/class_1/event/px_bias": -1.1027042865753174, "metrics/particle/momentum/class_1/event/py_bias": 1.1818835735321045, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.689640045166016, "metrics/particle/momentum/class_1/event/pz_bias": -0.6398354172706604, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.914154052734375, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02700391598045826, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17853088676929474, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17879512906074524, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.022378673776984215, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.217665895819664, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.023782558739185333, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3632932901382446, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.13471627235412598, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.395552396774292, "metrics/particle/momentum/class_2/accepted/samples": 1766.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.546323299407959, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.49869561195373535, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7147604823112488, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3868331015110016, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.22512680292129517, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.6041064262390137, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.046039797365665436, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.169496536254883, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.24475233256816864, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.5802321434021, "metrics/particle/momentum/class_2/assigned/samples": 2563.0, "metrics/particle/momentum/class_2/efficiency": 0.5914266705513, "metrics/particle/momentum/class_2/event/energy_bias": -9.726154327392578, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.199464797973633, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.551656723022461, "metrics/particle/momentum/class_2/event/px_bias": -0.007906325161457062, "metrics/particle/momentum/class_2/event/py_bias": -0.8460718989372253, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.883864402770996, "metrics/particle/momentum/class_2/event/pz_bias": -2.0427372455596924, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.84449577331543, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019023975357413292, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1486411988735199, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14825786650180817, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.004474951885640621, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1872793734073639, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018206285312771797, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23803828656673431, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0390104316174984, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.963797926902771, "metrics/particle/momentum/class_3/accepted/samples": 12633.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2319028228521347, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2609065771102905, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2385940551757812, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0865223407745361, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1363576501607895, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.182873725891113, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006254937965422869, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4510952830314636, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0378144346177578, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3286592960357666, "metrics/particle/momentum/class_3/assigned/samples": 15188.0, "metrics/particle/momentum/class_3/efficiency": 0.8133007287979126, "metrics/particle/momentum/class_3/event/energy_bias": -1.8530656099319458, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.617986679077148, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.661770343780518, "metrics/particle/momentum/class_3/event/px_bias": -0.28618207573890686, "metrics/particle/momentum/class_3/event/py_bias": 0.2579614222049713, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.1972126960754395, "metrics/particle/momentum/class_3/event/pz_bias": -1.3262232542037964, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.54504108428955, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02218400314450264, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1599060595035553, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15073484182357788, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04483868181705475, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19538907706737518, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0038759689778089523, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6599985361099243, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.015713950619101524, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.7131530046463013, "metrics/particle/momentum/class_4/accepted/samples": 258.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.38366758823394775, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.0794646739959717, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.692399024963379, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.572591543197632, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23966942727565765, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.354141235351562, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.002754820976406336, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6665269136428833, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.00464250473305583, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.5934308767318726, "metrics/particle/momentum/class_4/assigned/samples": 363.0, "metrics/particle/momentum/class_4/efficiency": 0.6880000233650208, "metrics/particle/momentum/class_4/event/energy_bias": -0.5269458293914795, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.280385971069336, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6868605613708496, "metrics/particle/momentum/class_4/event/px_bias": -0.01739659160375595, "metrics/particle/momentum/class_4/event/py_bias": -0.1193864494562149, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.205162525177002, "metrics/particle/momentum/class_4/event/pz_bias": -0.17261949181556702, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.467870235443115, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01477316115051508, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1441183090209961, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14743146300315857, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.02341599203646183, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1841297745704651, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.04026845470070839, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7646191120147705, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.11557308584451675, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.136991024017334, "metrics/particle/momentum/class_5/accepted/samples": 149.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09366202354431152, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.27147260308265686, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.27989286184310913, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.11170700937509537, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07361963391304016, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.9099686741828918, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04907975345849991, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.906446933746338, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.15498435497283936, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.3740153312683105, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9085366129875183, "metrics/particle/momentum/class_5/event/energy_bias": -0.5885584950447083, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8625741004943848, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4290801286697388, "metrics/particle/momentum/class_5/event/px_bias": -0.006924495100975037, "metrics/particle/momentum/class_5/event/py_bias": -0.2094310224056244, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9217348694801331, "metrics/particle/momentum/class_5/event/pz_bias": 0.17153380811214447, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2072575092315674, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3273222744464874, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3297725021839142, "metrics/set_slot/angular_recall_all_dr0p10": 0.9591434597969055, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8529937267303467, "metrics/set_slot/soft_count_bias_mean": 0.5978749394416809}, "step_train_time": 4.792045831680298, "step_valid_time": 8697.185607910156, "step_total_time": 8701.977653980255, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9750801640424994, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11534278259738073, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2915, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2905, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9529257927427264, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9604612850082372, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9566787003610109, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.039538714991762765, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9496567505720824, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.957166392092257, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9533967837216935, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9529257927427264, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9647388074291052, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11494793907542356, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2259, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2158, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2155, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9444201312910284, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9552899513058876, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9498239436619719, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044710048694112436, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9431072210065645, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9539619300575476, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9485035211267606, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9444201312910284, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9707911053277678, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11699825946793352, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1304, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1231, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1228, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9311649016641452, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9440184049079755, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9375476009139375, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05598159509202454, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9288956127080181, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.941717791411043, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9352627570449352, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9311649016641452} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_34000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..71b0c8a94b56a9d7818794a9c8aad82d33830b77 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 1.610458507010231, "train": {"Classification_binary": 0.16061699390411377, "Cardinality": 0.09297239035367966, "Classification": 0.08752875030040741, "Regression_pt": 0.09727980941534042, "Regression_eta": 0.0012781590921804309, "Regression_sin_phi": 0.0006651940639130771, "Regression_cos_phi": 0.0007135752239264548, "Regression_energy": 0.09560117870569229, "HitGrouping": 0.1473015397787094, "Auxiliary": 0.19261416792869568, "Total": 0.9047976136207581, "Total_unweighted": 0.44368366070557386}, "train_group": {}, "valid": {"Classification_binary": 0.15143543481826782, "Cardinality": 0.11441948016484578, "Classification": 0.08483454585075378, "Regression_pt": 0.10628836353619893, "Regression_eta": 0.0013449668573836486, "Regression_sin_phi": 0.0007213192681471506, "Regression_cos_phi": 0.0006916613783687353, "Regression_energy": 0.1054848035176595, "HitGrouping": 0.15266128381093344, "Auxiliary": 0.19516366720199585, "Total": 0.9384307066599528, "Total_unweighted": 0.4508010952267796, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8504900932312012, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6000957489013672, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.28891831636428833, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.611205518245697, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6148024201393127, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3328286111354828, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7946774363517761, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.543795108795166, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2865583598613739, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5532575845718384, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5610389709472656, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.42245200276374817, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8661303520202637, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5732396245002747, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2670678198337555, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5813742280006409, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5791240930557251, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.30469799041748047, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8558446168899536, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5790897607803345, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2752006947994232, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5882287621498108, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5885288715362549, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32236239314079285, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003540868405252695, "metrics/hit_grouping/calorimeter/background_recall": 0.1555558741092682, "metrics/hit_grouping/calorimeter/completeness": 0.6549181342124939, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6517784595489502, "metrics/hit_grouping/calorimeter/energy_purity": 0.6433904767036438, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.31473785638809204, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8775796294212341, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.589701771736145, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.20892125368118286, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.601148247718811, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5919771790504456, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.26115158200263977, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8587459921836853, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6055974960327148, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2880009114742279, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6153245568275452, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6180706024169922, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3270382285118103, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8086044192314148, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5307474732398987, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.397447794675827, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5345892906188965, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5517367124557495, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4424594044685364, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.839557409286499, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.637395441532135, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3325715661048889, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6508335471153259, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6676592826843262, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3707601726055145, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8280160427093506, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6577139496803284, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.44518786668777466, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6592251062393188, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6564371585845947, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3808543384075165, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8757659792900085, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5102766156196594, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1964559406042099, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5159412026405334, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.499760240316391, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.26604941487312317, "metrics/hit_grouping/calorimeter/purity": 0.6456335186958313, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7798894643783569, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0852556824684143, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07706766575574875, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08302900195121765, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06498131901025772, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8100438714027405, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2411767691373825, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10005886107683182, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.23790837824344635, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.21707913279533386, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2319011241197586, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8594657182693481, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6047208905220032, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.28754326701164246, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.614690899848938, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6163894534111023, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3318018913269043, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7279671430587769, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40388238430023193, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5103068351745605, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5484105944633484, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5900332927703857, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7812032103538513, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.294153094291687, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5054646134376526, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.49507448077201843, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4327908456325531, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6423869729042053, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43258023262023926, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4160022735595703, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4671682119369507, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7052093744277954, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.678943395614624, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4135351777076721, "metrics/hit_grouping/tracker/all/object_completeness": 0.4515945315361023, "metrics/hit_grouping/tracker/all/object_purity": 0.49419909715652466, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6496883630752563, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009082485921680927, "metrics/hit_grouping/tracker/background_recall": 0.09946321696043015, "metrics/hit_grouping/tracker/completeness": 0.45055103302001953, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.46439167857170105, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7096882462501526, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3979484438896179, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5230926275253296, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5520980954170227, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.615465521812439, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6604980826377869, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4340170621871948, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.42690885066986084, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4819123148918152, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6825194954872131, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6340269446372986, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4252479672431946, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3290494680404663, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.38730841875076294, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6874662041664124, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6777870059013367, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43245309591293335, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.46098592877388, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5193025469779968, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6570433974266052, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6749140620231628, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45628079771995544, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4251183867454529, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.46764233708381653, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6477832794189453, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6811689734458923, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38127827644348145, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.44413185119628906, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.46632450819015503, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6404680609703064, "metrics/hit_grouping/tracker/purity": 0.4241856634616852, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8862923383712769, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04110105708241463, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.131537526845932, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13055579364299774, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0656108632683754, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6913476586341858, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2803304195404053, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.37547042965888977, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3780023455619812, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5054207444190979, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6468288898468018, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4832409918308258, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5073409676551819, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5607568025588989, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7520004510879517, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6064740419387817, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6724482178688049, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7724644541740417, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6610890626907349, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.05732134357094765, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.08661430329084396, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.537809371948242, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2972852885723114, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20566213130950928, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8914254307746887, "metrics/particle/class_1/pid_accuracy": 0.9755247235298157, "metrics/particle/class_2/efficiency": 0.5693736672401428, "metrics/particle/class_2/pid_accuracy": 0.7513427734375, "metrics/particle/class_3/efficiency": 0.8153597116470337, "metrics/particle/class_3/pid_accuracy": 0.9775049090385437, "metrics/particle/class_4/efficiency": 0.7054108381271362, "metrics/particle/class_4/pid_accuracy": 0.4488636255264282, "metrics/particle/class_5/efficiency": 0.9233716726303101, "metrics/particle/class_5/pid_accuracy": 0.591286301612854, "metrics/particle/count/bias_mean": -1.240234375, "metrics/particle/count/mae": 3.552734375, "metrics/particle/count/prediction_mean": 69.41471099853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06882520020008087, "metrics/particle/event/energy_response_mean": 0.9682767987251282, "metrics/particle/event/met_abs_error": 5.293715953826904, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05243637040257454, "metrics/particle/event/scalar_pt_response_mean": 0.9774388670921326, "metrics/particle/event/vector_pt_closure": 0.05096076801419258, "metrics/particle/matched/delta_eta_abs_mean": 0.013160424306988716, "metrics/particle/matched/delta_phi_abs_mean": 0.013324141502380371, "metrics/particle/matched/delta_r_mean": 0.021046344190835953, "metrics/particle/matched/energy_relative_abs_mean": 0.1322568655014038, "metrics/particle/matched/pid_accuracy": 0.9576957821846008, "metrics/particle/matched/pt_relative_abs_mean": 0.1315741389989853, "metrics/particle/matching/duplicate_fraction": 0.04611662030220032, "metrics/particle/matching/efficiency": 0.8316071629524231, "metrics/particle/matching/f1": 0.8389705419540405, "metrics/particle/matching/purity": 0.846465528011322, "metrics/particle/matching/target_coverage_dr0p05": 0.7847059965133667, "metrics/particle/matching/target_coverage_dr0p10": 0.8682896494865417, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6597313284873962, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8640786409378052, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02080758661031723, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13011585175991058, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1301206648349762, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.006801288109272718, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16836987435817719, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004929419606924057, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6036921739578247, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.004614783450961113, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6833655834197998, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8926.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.19935449957847595, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.854794442653656, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8987860679626465, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7512131333351135, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10293833166360855, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 15.442480087280273, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011212063021957874, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8610515594482422, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.020058000460267067, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6793150901794434, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10346.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8286297917366028, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.24727006256580353, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.556730270385742, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.092456340789795, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.24303552508354187, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.07959651947021484, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.517855644226074, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.4967123866081238, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.79326343536377, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02368900366127491, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14362582564353943, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14422915875911713, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.007550824899226427, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18505817651748657, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021786492317914963, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2150522470474243, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.13270631432533264, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0786430835723877, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1836.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.46029800176620483, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9511552453041077, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.380420446395874, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.195676565170288, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1746431589126587, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 29.93058204650879, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0512174628674984, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3954012393951416, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.4628848433494568, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.321090698242188, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2382.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6656997799873352, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.373106002807617, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.852889537811279, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0266566276550293, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03455875441431999, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0814499706029892, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.728252410888672, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.1897324323654175, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.08036231994629, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021108362823724747, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13345487415790558, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1325710564851761, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0008087760070338845, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17172393202781677, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005039560492150486, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20981472730636597, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005797224119305611, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6841545701026917, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19843.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17569518089294434, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7902233600616455, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6137414574623108, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4494074583053589, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0985637977719307, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.426469802856445, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028461336623877287, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.33908867835998535, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.008839583024382591, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3093947172164917, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22838.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8527655005455017, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.521243691444397, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.905562400817871, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.646124839782715, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.5187238454818726, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.09779918938875198, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.5562713146209717, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.1734430342912674, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.773484706878662, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021175455302000046, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13309119641780853, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1325557678937912, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0029609589837491512, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17158618569374084, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003071393584832549, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.38499414920806885, "metrics/particle/momentum/all/accepted/pz_bias": -0.01037384569644928, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.303274393081665, "metrics/particle/momentum/all/accepted/samples": 30605.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2016385793685913, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8197851181030273, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8149814009666443, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6541562080383301, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10493167489767075, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.176885604858398, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008519371971487999, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.628645122051239, "metrics/particle/momentum/all/assigned/pz_bias": -0.019490333274006844, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.797592878341675, "metrics/particle/momentum/all/assigned/samples": 35566.0, "metrics/particle/momentum/all/efficiency": 0.8316802382469177, "metrics/particle/momentum/all/event/energy_bias": -6.141619682312012, "metrics/particle/momentum/all/event/p3_error_mean": 16.283000946044922, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.9632134437561035, "metrics/particle/momentum/all/event/px_bias": 0.2411295473575592, "metrics/particle/momentum/all/event/py_bias": -0.09594576805830002, "metrics/particle/momentum/all/event/pz_abs_mean": 11.981316566467285, "metrics/particle/momentum/all/event/pz_bias": -1.8598875999450684, "metrics/particle/momentum/all/event/pz_residual_rms": 24.092782974243164, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.022677311673760414, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11638285964727402, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11663762480020523, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0003708452277351171, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15314707159996033, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0023379249032586813, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4046786427497864, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.006598781794309616, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2064285278320312, "metrics/particle/momentum/class_1/accepted/samples": 15826.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12595050036907196, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.33116239309310913, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3389386236667633, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1952054500579834, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06271897256374359, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.841423273086548, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005111711099743843, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5382334589958191, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0031623179093003273, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1412575244903564, "metrics/particle/momentum/class_1/assigned/samples": 17411.0, "metrics/particle/momentum/class_1/efficiency": 0.8920579552650452, "metrics/particle/momentum/class_1/event/energy_bias": 4.663247585296631, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.258685111999512, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.368050575256348, "metrics/particle/momentum/class_1/event/px_bias": 0.7986030578613281, "metrics/particle/momentum/class_1/event/py_bias": -0.13538844883441925, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.702920913696289, "metrics/particle/momentum/class_1/event/pz_bias": -0.6149654984474182, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.589370727539062, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02606266923248768, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17688016593456268, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17820580303668976, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.0052436115220189095, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.218384250998497, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.022458629682660103, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.2894484996795654, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.009556511417031288, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.0840718746185303, "metrics/particle/momentum/class_2/accepted/samples": 1692.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5599492788314819, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5541120171546936, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6685881614685059, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.33542415499687195, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.23906125128269196, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.7885890007019043, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.044152744114398956, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.1868247985839844, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.18757683038711548, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.57644510269165, "metrics/particle/momentum/class_2/assigned/samples": 2514.0, "metrics/particle/momentum/class_2/efficiency": 0.5666443109512329, "metrics/particle/momentum/class_2/event/energy_bias": -10.180273056030273, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.283036231994629, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.751082897186279, "metrics/particle/momentum/class_2/event/px_bias": -0.40335118770599365, "metrics/particle/momentum/class_2/event/py_bias": -0.20669613778591156, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.766207695007324, "metrics/particle/momentum/class_2/event/pz_bias": -0.8343778848648071, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.708921432495117, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018713729456067085, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14741437137126923, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14588384330272675, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0042096516117453575, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1851382851600647, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011836186749860644, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22329767048358917, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0165557824075222, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9330334663391113, "metrics/particle/momentum/class_3/accepted/samples": 12673.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2265404760837555, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.39035964012146, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3596432209014893, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2063566446304321, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12898531556129456, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.31037712097168, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00641619274392724, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.46404582262039185, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.009752804413437843, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.791890859603882, "metrics/particle/momentum/class_3/assigned/samples": 15118.0, "metrics/particle/momentum/class_3/efficiency": 0.8158758878707886, "metrics/particle/momentum/class_3/event/energy_bias": 0.47469431161880493, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.592260360717773, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.9830474853515625, "metrics/particle/momentum/class_3/event/px_bias": -0.1825276017189026, "metrics/particle/momentum/class_3/event/py_bias": 0.5368701815605164, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.003990173339844, "metrics/particle/momentum/class_3/event/pz_bias": -0.33228904008865356, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.178468704223633, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02176825702190399, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1617116928100586, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1488422006368637, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06772983074188232, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19634316861629486, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5664302706718445, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.025248650461435318, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.2090251445770264, "metrics/particle/momentum/class_4/accepted/samples": 264.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3659855127334595, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.644155740737915, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.2816646099090576, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.1749441623687744, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2200557142496109, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.178815841674805, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.002785515272989869, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6141417622566223, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.048212964087724686, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.310611367225647, "metrics/particle/momentum/class_4/assigned/samples": 359.0, "metrics/particle/momentum/class_4/efficiency": 0.7039999961853027, "metrics/particle/momentum/class_4/event/energy_bias": -0.7200629711151123, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.246795177459717, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.658296823501587, "metrics/particle/momentum/class_4/event/px_bias": 0.049238547682762146, "metrics/particle/momentum/class_4/event/py_bias": -0.13675186038017273, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1895394325256348, "metrics/particle/momentum/class_4/event/pz_bias": -0.20216912031173706, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.626288414001465, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014531086198985577, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14150327444076538, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14238256216049194, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.030995845794677734, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17568407952785492, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02666666731238365, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4477771520614624, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.1305830180644989, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.095384359359741, "metrics/particle/momentum/class_5/accepted/samples": 150.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08911067992448807, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.17597439885139465, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.1788705289363861, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.031939249485731125, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.060975611209869385, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.24651393294334412, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.030487805604934692, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.546403169631958, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.011063655838370323, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.3998160362243652, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9146341681480408, "metrics/particle/momentum/class_5/event/energy_bias": -0.3792259097099304, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7059946060180664, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3180171251296997, "metrics/particle/momentum/class_5/event/px_bias": -0.020833201706409454, "metrics/particle/momentum/class_5/event/py_bias": -0.15397949516773224, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8190484642982483, "metrics/particle/momentum/class_5/event/pz_bias": 0.12391394376754761, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.7780508995056152, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3224846124649048, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3243059515953064, "metrics/set_slot/angular_recall_all_dr0p10": 0.9592816233634949, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8526343703269958, "metrics/set_slot/soft_count_bias_mean": 0.017154449597001076}, "step_train_time": 5.0411224365234375, "step_valid_time": 11738.037446022034, "step_total_time": 11743.078568696976, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9866457449721774, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11393495326227054, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3048, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2913, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2899, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9522719843085976, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9557086614173228, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9539872277714099, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04429133858267716, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.947695325269696, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9511154855643045, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9494023252005895, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9522719843085976, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9774002692624824, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1077036063078004, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2249, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2155, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2150, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9431072210065645, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9582036460649177, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9505955006616674, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041796353935082256, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9409190371991247, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9559804357492219, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9483899426554918, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9431072210065645, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9868684504381059, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1144211656437708, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1301, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1230, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1223, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9304084720121029, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9454265949269792, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9378574151734655, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054573405073020755, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9251134644478064, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9400461183704842, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9325200152497141, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9304084720121029} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_35000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..461f82dbdd29d94db35cfb5946cd59c0e8b303e3 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 1.6578249336870026, "train": {"Classification_binary": 0.1455221176147461, "Cardinality": 0.1229759082198143, "Classification": 0.07708808034658432, "Regression_pt": 0.09857019782066345, "Regression_eta": 0.0014071079203858972, "Regression_sin_phi": 0.0006993836723268032, "Regression_cos_phi": 0.000693376234266907, "Regression_energy": 0.0979824885725975, "HitGrouping": 0.14588876068592072, "Auxiliary": 0.1896921694278717, "Total": 0.9157446622848511, "Total_unweighted": 0.421962752181571}, "train_group": {}, "valid": {"Classification_binary": 0.15308642387390137, "Cardinality": 0.11893657843271892, "Classification": 0.08504478136698405, "Regression_pt": 0.1098928451538086, "Regression_eta": 0.0013855785752336185, "Regression_sin_phi": 0.0007208528307576975, "Regression_cos_phi": 0.0006984227802604437, "Regression_energy": 0.10918975869814555, "HitGrouping": 0.15471844871838888, "Auxiliary": 0.19994630416234335, "Total": 0.9572927157084147, "Total_unweighted": 0.46001866327909136, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8484857678413391, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6066884398460388, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3011261522769928, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6173027753829956, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6175603866577148, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.337276428937912, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7964411973953247, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5466530323028564, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3024372160434723, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5563998818397522, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5559868812561035, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4194977879524231, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8607895970344543, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5678338408470154, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.27317750453948975, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5750263929367065, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5747266411781311, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31205740571022034, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8520662188529968, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5779605507850647, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.28391629457473755, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5863880515098572, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5862483382225037, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3280389904975891, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0038351938128471375, "metrics/hit_grouping/calorimeter/background_recall": 0.16781768202781677, "metrics/hit_grouping/calorimeter/completeness": 0.6495321393013, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6480836272239685, "metrics/hit_grouping/calorimeter/energy_purity": 0.6398096084594727, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.32159510254859924, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8735839128494263, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.596840500831604, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.21700045466423035, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6076128482818604, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5965689420700073, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2713358998298645, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8563539981842041, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6043780446052551, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3014739751815796, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6122390031814575, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6127007007598877, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3231229782104492, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8038725852966309, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5128511190414429, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.40266820788383484, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5170683264732361, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5385561585426331, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4486078917980194, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8394215703010559, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6363931894302368, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3425547182559967, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6491840481758118, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6633179783821106, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3646259307861328, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8145584464073181, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6469283103942871, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4673185646533966, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6471901535987854, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6601718068122864, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.40607309341430664, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8694268465042114, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5101683735847473, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.2022809535264969, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5151199698448181, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4988642930984497, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2814396917819977, "metrics/hit_grouping/calorimeter/purity": 0.640513002872467, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7671970725059509, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.10655387490987778, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.08458646386861801, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.10341478884220123, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.07514753192663193, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.024436090141534805, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8091500997543335, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.24871058762073517, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1006474420428276, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.24656619131565094, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.22900789976119995, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2277810424566269, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8556824922561646, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6027755737304688, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2967012822628021, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6119416356086731, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6131666898727417, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3382033109664917, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7271118760108948, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3941764533519745, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5194532871246338, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.565447986125946, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5826452374458313, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.771152675151825, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.26582279801368713, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.49530452489852905, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5095040798187256, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4472573697566986, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6452836394309998, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4408547878265381, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4238397479057312, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.48087066411972046, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6936249732971191, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6797304749488831, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41357970237731934, "metrics/hit_grouping/tracker/all/object_completeness": 0.45850491523742676, "metrics/hit_grouping/tracker/all/object_purity": 0.5089802145957947, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6413179039955139, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013016200624406338, "metrics/hit_grouping/tracker/background_recall": 0.12263182550668716, "metrics/hit_grouping/tracker/completeness": 0.4553322196006775, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.45545145869255066, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7134881615638733, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39487987756729126, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5331323146820068, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5677129030227661, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6074872612953186, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6586944460868835, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43673989176750183, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.43217194080352783, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5004979372024536, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6674532294273376, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.631191074848175, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42903515696525574, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.33115923404693604, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.3965950310230255, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6849414110183716, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6782780885696411, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4337267577648163, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4670240879058838, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5328434705734253, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6444765329360962, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6595784425735474, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4236453175544739, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.40821632742881775, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4887720048427582, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6582512259483337, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6852313876152039, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38558340072631836, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.45644497871398926, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.48157989978790283, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6341759562492371, "metrics/hit_grouping/tracker/purity": 0.43039563298225403, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8795451521873474, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04336350038647652, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.12237827479839325, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1235133707523346, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.05882352963089943, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.684425413608551, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.29736706614494324, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.38496366143226624, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.3849050998687744, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5141972303390503, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6495698094367981, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4811146557331085, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5163560509681702, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5796305537223816, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.741536557674408, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6025334000587463, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6748420596122742, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.775571882724762, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6474535465240479, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.053380679339170456, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07297873497009277, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.493762969970703, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.27777183055877686, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18614867329597473, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9038600921630859, "metrics/particle/class_1/pid_accuracy": 0.9766421318054199, "metrics/particle/class_2/efficiency": 0.5671083927154541, "metrics/particle/class_2/pid_accuracy": 0.7321749329566956, "metrics/particle/class_3/efficiency": 0.7991167902946472, "metrics/particle/class_3/pid_accuracy": 0.9771570563316345, "metrics/particle/class_4/efficiency": 0.7014027833938599, "metrics/particle/class_4/pid_accuracy": 0.44857141375541687, "metrics/particle/class_5/efficiency": 0.938697338104248, "metrics/particle/class_5/pid_accuracy": 0.581632673740387, "metrics/particle/count/bias_mean": -0.5364583134651184, "metrics/particle/count/mae": 3.5013020038604736, "metrics/particle/count/prediction_mean": 70.11849212646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06684412807226181, "metrics/particle/event/energy_response_mean": 0.9778136610984802, "metrics/particle/event/met_abs_error": 5.620124816894531, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05203702673316002, "metrics/particle/event/scalar_pt_response_mean": 0.9886195063591003, "metrics/particle/event/vector_pt_closure": 0.05263875797390938, "metrics/particle/matched/delta_eta_abs_mean": 0.014542174525558949, "metrics/particle/matched/delta_phi_abs_mean": 0.01348190102726221, "metrics/particle/matched/delta_r_mean": 0.02223886363208294, "metrics/particle/matched/energy_relative_abs_mean": 0.1350044459104538, "metrics/particle/matched/pid_accuracy": 0.957024335861206, "metrics/particle/matched/pt_relative_abs_mean": 0.13379482924938202, "metrics/particle/matching/duplicate_fraction": 0.052060313522815704, "metrics/particle/matching/efficiency": 0.8306212425231934, "metrics/particle/matching/f1": 0.8337865471839905, "metrics/particle/matching/purity": 0.8369761109352112, "metrics/particle/matching/target_coverage_dr0p05": 0.7839227318763733, "metrics/particle/matching/target_coverage_dr0p10": 0.8708972930908203, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6517977118492126, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8623831868171692, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022681238129734993, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13768325746059418, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1356775462627411, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02714819274842739, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17550073564052582, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.006593652069568634, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6536058187484741, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.023142503574490547, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.8479056358337402, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8948.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.20525962114334106, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7208045125007629, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7909727096557617, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6560747027397156, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10897435992956161, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.1237263679504395, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011194029822945595, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8574385046958923, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0381484217941761, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4489755630493164, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10452.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8306720852851868, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.9367150068283081, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.2401885986328125, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.423743486404419, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.12930169701576233, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.06097283214330673, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.113523960113525, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.225160837173462, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.979866981506348, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026110336184501648, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1517125368118286, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14744725823402405, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02090083435177803, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18865178525447845, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02498611807823181, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2779909372329712, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.06016077846288681, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.244696855545044, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1801.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4931884706020355, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9335236549377441, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.3303816318511963, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.163160562515259, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.19628098607063293, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 20.54146385192871, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.058264464139938354, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4248664379119873, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.29553133249282837, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.602846622467041, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2420.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6530094146728516, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.653375625610352, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.556682109832764, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0321046113967896, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.01990518718957901, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.018013831228017807, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.418348789215088, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.3016773462295532, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.21196937561035, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021980809047818184, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13353560864925385, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13306210935115814, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01683359593153, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17243190109729767, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006067347712814808, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.21393613517284393, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.023128939792513847, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6708731055259705, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19778.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17892558872699738, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.958712100982666, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7102377414703369, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5670449137687683, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10677877813577652, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.8133864402771, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026574889197945595, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3416215777397156, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.032462622970342636, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3316540718078613, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22954.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8499720692634583, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.03288796544075012, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.053239822387695, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.577772617340088, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.4308169186115265, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.3988118171691895, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.152584552764893, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.11772754788398743, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.806139945983887, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022429747506976128, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13582374155521393, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13467742502689362, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02009693905711174, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1743309199810028, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003799914848059416, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.4055870473384857, "metrics/particle/momentum/all/accepted/pz_bias": 0.025317681953310966, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3833446502685547, "metrics/particle/momentum/all/accepted/samples": 30527.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20783643424510956, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.887602686882019, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8432303667068481, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.700834333896637, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1134650781750679, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 7.816373825073242, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008904147893190384, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6328282952308655, "metrics/particle/momentum/all/assigned/pz_bias": 0.011965853162109852, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6058456897735596, "metrics/particle/momentum/all/assigned/samples": 35826.0, "metrics/particle/momentum/all/efficiency": 0.8295605778694153, "metrics/particle/momentum/all/event/energy_bias": -3.7495481967926025, "metrics/particle/momentum/all/event/p3_error_mean": 16.74888038635254, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.788457870483398, "metrics/particle/momentum/all/event/px_bias": 0.3214203715324402, "metrics/particle/momentum/all/event/py_bias": 2.4777984619140625, "metrics/particle/momentum/all/event/pz_abs_mean": 11.91270637512207, "metrics/particle/momentum/all/event/pz_bias": 0.8057559728622437, "metrics/particle/momentum/all/event/pz_residual_rms": 22.045419692993164, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.023688040673732758, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11907004565000534, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11851070076227188, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01799987629055977, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.15537045896053314, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0031825273763388395, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.426339328289032, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.03803001344203949, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.3486732244491577, "metrics/particle/momentum/class_1/accepted/samples": 16025.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11951153725385666, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.33007413148880005, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3709675371646881, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24666419625282288, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05888736620545387, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.336226463317871, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005654558073729277, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5515097379684448, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02850164845585823, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2196755409240723, "metrics/particle/momentum/class_1/assigned/samples": 17508.0, "metrics/particle/momentum/class_1/efficiency": 0.9032748937606812, "metrics/particle/momentum/class_1/event/energy_bias": 4.110932350158691, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.147918701171875, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 9.23625373840332, "metrics/particle/momentum/class_1/event/px_bias": 0.04136466234922409, "metrics/particle/momentum/class_1/event/py_bias": 2.4354825019836426, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.99278450012207, "metrics/particle/momentum/class_1/event/pz_bias": 0.7410544157028198, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.836219787597656, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.028004387393593788, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17922015488147736, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17985795438289642, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.01450152974575758, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22177284955978394, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0256563238799572, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.297776222229004, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.006223754957318306, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.1035802364349365, "metrics/particle/momentum/class_2/accepted/samples": 1676.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5640807747840881, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.535909116268158, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6978060603141785, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.37768709659576416, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.26232051849365234, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.0564053058624268, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.048894062638282776, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.1419215202331543, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.10516923666000366, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.306763172149658, "metrics/particle/momentum/class_2/assigned/samples": 2577.0, "metrics/particle/momentum/class_2/efficiency": 0.5612859725952148, "metrics/particle/momentum/class_2/event/energy_bias": -11.07614803314209, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.375409126281738, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.908007621765137, "metrics/particle/momentum/class_2/event/px_bias": -0.3126786947250366, "metrics/particle/momentum/class_2/event/py_bias": -0.5176851153373718, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.742741584777832, "metrics/particle/momentum/class_2/event/pz_bias": -0.7665902972221375, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.45903205871582, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020115548744797707, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.150440976023674, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14849507808685303, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.021835563704371452, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18850694596767426, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001208021305501461, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2369992733001709, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0034192344173789024, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8990897536277771, "metrics/particle/momentum/class_3/accepted/samples": 12417.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.24640390276908875, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5205789804458618, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3490896224975586, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2141834497451782, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14815302193164825, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.61882495880127, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005586959421634674, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4556269943714142, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0044461749494075775, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.300333261489868, "metrics/particle/momentum/class_3/assigned/samples": 15214.0, "metrics/particle/momentum/class_3/efficiency": 0.7993948459625244, "metrics/particle/momentum/class_3/event/energy_bias": 4.085107803344727, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.101449966430664, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.917088031768799, "metrics/particle/momentum/class_3/event/px_bias": 0.3007045388221741, "metrics/particle/momentum/class_3/event/py_bias": 0.8369849920272827, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.565125942230225, "metrics/particle/momentum/class_3/event/pz_bias": 0.6316996216773987, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.367901802062988, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022824082523584366, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1728673279285431, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16488544642925262, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0991617888212204, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21118219196796417, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00390625, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.6486846208572388, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.22453702986240387, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6738252639770508, "metrics/particle/momentum/class_4/accepted/samples": 256.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.39165276288986206, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.025971412658691, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.699892044067383, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.629920721054077, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2657534182071686, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 18.12619400024414, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005479452200233936, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7319105863571167, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.205641508102417, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7960926294326782, "metrics/particle/momentum/class_4/assigned/samples": 365.0, "metrics/particle/momentum/class_4/efficiency": 0.6826666593551636, "metrics/particle/momentum/class_4/event/energy_bias": -0.3412107527256012, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.354743719100952, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.760854721069336, "metrics/particle/momentum/class_4/event/px_bias": 0.13488414883613586, "metrics/particle/momentum/class_4/event/py_bias": -0.12320244312286377, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2473928928375244, "metrics/particle/momentum/class_4/event/pz_bias": -0.018583275377750397, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.581137180328369, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0167253315448761, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.16693615913391113, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.16109566390514374, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.027640890330076218, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1951438933610916, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03921568766236305, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.7340624332427979, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.483233779668808, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.936856746673584, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05037820339202881, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2204231172800064, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.25255197286605835, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.11534254252910614, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.043209876865148544, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.7641707062721252, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.043209876865148544, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.8338581323623657, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.3580225706100464, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.125467777252197, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.9329268336296082, "metrics/particle/momentum/class_5/event/energy_bias": -0.528231680393219, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8240820169448853, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.382722020149231, "metrics/particle/momentum/class_5/event/px_bias": 0.15714570879936218, "metrics/particle/momentum/class_5/event/py_bias": -0.15378135442733765, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9103683233261108, "metrics/particle/momentum/class_5/event/pz_bias": 0.21817554533481598, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.0996828079223633, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.33445942401885986, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3353936970233917, "metrics/set_slot/angular_recall_all_dr0p10": 0.9577059745788574, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8554908633232117, "metrics/set_slot/soft_count_bias_mean": 0.6372238993644714}, "step_train_time": 5.219681978225708, "step_valid_time": 14778.836749315262, "step_total_time": 14784.056432008743, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0051091245770207, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12313649133491689, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3052, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2911, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2901, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9516181758744687, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9538007863695938, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9527082310587465, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04619921363040629, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9483491337038248, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.950524246395806, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9494354442808051, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9516181758744687, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9915622179024431, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12000691872310887, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2273, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2161, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2156, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9457330415754923, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9507259128904532, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9482229047827995, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04927408710954685, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9435448577680525, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9485261768587769, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9460289600702062, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9457330415754923, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9983206789309834, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12328566992104983, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1243, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1237, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9402420574886535, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9430955993930197, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9416666666666667, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05690440060698027, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9357034795763994, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9385432473444613, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9371212121212121, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9402420574886535} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_36000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_36000.json new file mode 100644 index 0000000000000000000000000000000000000000..688a8711928bcb5062d09795621fe52a026a82c3 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_36000.json @@ -0,0 +1 @@ +{"epoch": 1.705191360363774, "train": {"Classification_binary": 0.13136182725429535, "Cardinality": 0.09120474755764008, "Classification": 0.0938466489315033, "Regression_pt": 0.11614204198122025, "Regression_eta": 0.0014759146142750978, "Regression_sin_phi": 0.0006749841850250959, "Regression_cos_phi": 0.0007998938672244549, "Regression_energy": 0.11349919438362122, "HitGrouping": 0.15868090093135834, "Auxiliary": 0.1925990730524063, "Total": 0.9230538010597229, "Total_unweighted": 0.45780050521716475}, "train_group": {}, "valid": {"Classification_binary": 0.1516130566596985, "Cardinality": 0.11681002378463745, "Classification": 0.0829888383547465, "Regression_pt": 0.10495386521021526, "Regression_eta": 0.0013215563570459683, "Regression_sin_phi": 0.0007004653258870045, "Regression_cos_phi": 0.000680550467222929, "Regression_energy": 0.10353098313013713, "HitGrouping": 0.15293152133623758, "Auxiliary": 0.19402764240900675, "Total": 0.933885415395101, "Total_unweighted": 0.44578931550495327, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8450923562049866, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.5975698232650757, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2819153964519501, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6094990968704224, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6166326999664307, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.34162959456443787, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7911771535873413, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5453318357467651, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2887740135192871, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5556508898735046, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.562036395072937, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4143279194831848, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8632838726043701, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5670344233512878, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2580884099006653, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5761078000068665, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5767516493797302, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3077991306781769, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8521744608879089, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5746002793312073, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.26768922805786133, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5846355557441711, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5876913070678711, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32631880044937134, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.004082954954355955, "metrics/hit_grouping/calorimeter/background_recall": 0.17331434786319733, "metrics/hit_grouping/calorimeter/completeness": 0.6580684781074524, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6576437950134277, "metrics/hit_grouping/calorimeter/energy_purity": 0.6493026614189148, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30855318903923035, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8737791180610657, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5888352394104004, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.202947199344635, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6007637977600098, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5940911769866943, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2655894458293915, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8572705388069153, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6008516550064087, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.279940128326416, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6114470958709717, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6138710975646973, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3280746340751648, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.80298912525177, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.51912921667099, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3873549997806549, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5247399210929871, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5482680201530457, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4483758807182312, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8393506407737732, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6403519511222839, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.32337021827697754, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6547130942344666, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6738168001174927, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3684147298336029, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8249960541725159, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6675308346748352, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.44415852427482605, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6709607839584351, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6759724617004395, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.38651570677757263, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8684871196746826, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.49649113416671753, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1898951530456543, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5029029250144958, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4893636107444763, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2762278616428375, "metrics/hit_grouping/calorimeter/purity": 0.6490922570228577, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7766761183738708, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09004640579223633, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07330827414989471, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08771257847547531, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.060881271958351135, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03195488825440407, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7960103750228882, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.22816798090934753, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09770452976226807, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.22752302885055542, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.20804567635059357, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23602119088172913, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8563274145126343, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6005236506462097, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2797022759914398, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6113162636756897, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6160315275192261, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3358142673969269, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7253564596176147, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4093872308731079, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5198274254798889, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5675131678581238, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5943792462348938, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7738198041915894, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.28028932213783264, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5000099539756775, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5142488479614258, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.437010258436203, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6437520384788513, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43279609084129333, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4260937571525574, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.49152424931526184, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7001726627349854, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6784403324127197, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41433659195899963, "metrics/hit_grouping/tracker/all/object_completeness": 0.4603622555732727, "metrics/hit_grouping/tracker/all/object_purity": 0.5165577530860901, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6482190489768982, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.012113511562347412, "metrics/hit_grouping/tracker/background_recall": 0.11773760616779327, "metrics/hit_grouping/tracker/completeness": 0.45514777302742004, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.44969677925109863, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.712942361831665, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39356479048728943, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5335832834243774, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5718179941177368, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6109065413475037, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6607497930526733, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4380105137825012, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.43313273787498474, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5069496631622314, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6738064885139465, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6250457763671875, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43354374170303345, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.33680030703544617, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4124336540699005, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.69954913854599, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6776708364486694, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43024539947509766, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.46874162554740906, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5382436513900757, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6517788767814636, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6577275991439819, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.44581282138824463, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4202968180179596, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5021221041679382, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6804187297821045, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6831538081169128, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3880119323730469, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.45665135979652405, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.49095308780670166, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6378187537193298, "metrics/hit_grouping/tracker/purity": 0.4298281669616699, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8884478807449341, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.051659125834703445, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.12918753921985626, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.12446554750204086, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.060708899050951004, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6850379705429077, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2968508005142212, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.39503219723701477, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4030033349990845, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5172947645187378, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.646560788154602, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.48089084029197693, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.516588568687439, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5870509147644043, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7495943307876587, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6028998494148254, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7004623413085938, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8068112730979919, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6748515963554382, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.05374710634350777, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.10037680715322495, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.37042236328125, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2956150472164154, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20399188995361328, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8982148766517639, "metrics/particle/class_1/pid_accuracy": 0.9764103889465332, "metrics/particle/class_2/efficiency": 0.5865896344184875, "metrics/particle/class_2/pid_accuracy": 0.7657849192619324, "metrics/particle/class_3/efficiency": 0.8127363920211792, "metrics/particle/class_3/pid_accuracy": 0.9757376909255981, "metrics/particle/class_4/efficiency": 0.7064128518104553, "metrics/particle/class_4/pid_accuracy": 0.44255319237709045, "metrics/particle/class_5/efficiency": 0.9118773937225342, "metrics/particle/class_5/pid_accuracy": 0.5945377945899963, "metrics/particle/count/bias_mean": -0.6165364384651184, "metrics/particle/count/mae": 3.5149738788604736, "metrics/particle/count/prediction_mean": 70.03841400146484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06844913214445114, "metrics/particle/event/energy_response_mean": 0.9694251418113708, "metrics/particle/event/met_abs_error": 5.061615467071533, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05058863386511803, "metrics/particle/event/scalar_pt_response_mean": 0.9931633472442627, "metrics/particle/event/vector_pt_closure": 0.0507146455347538, "metrics/particle/matched/delta_eta_abs_mean": 0.01355327945202589, "metrics/particle/matched/delta_phi_abs_mean": 0.013036862947046757, "metrics/particle/matched/delta_r_mean": 0.021141760051250458, "metrics/particle/matched/energy_relative_abs_mean": 0.1359494924545288, "metrics/particle/matched/pid_accuracy": 0.9579407572746277, "metrics/particle/matched/pt_relative_abs_mean": 0.13741973042488098, "metrics/particle/matching/duplicate_fraction": 0.05058608204126358, "metrics/particle/matching/efficiency": 0.8351362943649292, "metrics/particle/matching/f1": 0.8387959599494934, "metrics/particle/matching/purity": 0.8424878716468811, "metrics/particle/matching/target_coverage_dr0p05": 0.7886404991149902, "metrics/particle/matching/target_coverage_dr0p10": 0.8729889392852783, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6472365856170654, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8662900924682617, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02059505134820938, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.13480006158351898, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.138743057847023, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.025651726871728897, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17810091376304626, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004818466957658529, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6227344274520874, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.09183192998170853, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.683096170425415, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8924.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.20455391705036163, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.6915112137794495, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7965789437294006, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.65910804271698, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1099768877029419, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.882007598876953, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011074730195105076, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8754583597183228, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0951879546046257, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.631716251373291, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10384.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8284441232681274, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.15417969226837158, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.765764236450195, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.0208160877227783, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.3254478871822357, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.13976642489433289, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.803983211517334, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.0847573280334473, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.771509170532227, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0226930920034647, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1443079560995102, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.15073642134666443, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017901698127388954, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19200614094734192, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025428414344787598, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.3808352947235107, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.17996695637702942, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.991849422454834, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1809.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4997977614402771, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0752755403518677, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.9718382358551025, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.795755386352539, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18726591765880585, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 18.613203048706055, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05285060405731201, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4178967475891113, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.013621380552649498, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.631620407104492, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2403.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.655910074710846, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.9330058097839355, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.015003204345703, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0708560943603516, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.02384348027408123, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07423385977745056, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.885271072387695, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.2884700894355774, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.10175323486328, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021317685022950172, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.136819526553154, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13695678114891052, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.020783687010407448, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.175564706325531, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000551185046788305, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.21138206124305725, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.016186509281396866, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6647545695304871, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19957.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17204856872558594, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7619503140449524, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.630705714225769, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4851372539997101, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10207100957632065, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.494838714599609, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002610511612147093, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.34151601791381836, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.030668573454022408, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3715217113494873, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22984.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8576647043228149, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.5675320625305176, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.884613037109375, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.699603080749512, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.39161017537117004, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.03963252902030945, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.4217312335968018, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.32474106550216675, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.733819961547852, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021188629791140556, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.136673703789711, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1382884383201599, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0220293328166008, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17731331288814545, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0032583903521299362, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.39992716908454895, "metrics/particle/momentum/all/accepted/pz_bias": 0.047836512327194214, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.43190336227417, "metrics/particle/momentum/all/accepted/samples": 30690.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2035018801689148, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7625508904457092, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.768950879573822, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6236832737922668, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11008917540311813, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 7.791561126708984, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008442592807114124, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.636000394821167, "metrics/particle/momentum/all/assigned/pz_bias": 0.048252783715724945, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6705243587493896, "metrics/particle/momentum/all/assigned/samples": 35771.0, "metrics/particle/momentum/all/efficiency": 0.8339900374412537, "metrics/particle/momentum/all/event/energy_bias": -6.654717445373535, "metrics/particle/momentum/all/event/p3_error_mean": 16.844947814941406, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.11087417602539, "metrics/particle/momentum/all/event/px_bias": -0.740901529788971, "metrics/particle/momentum/all/event/py_bias": -0.10516516864299774, "metrics/particle/momentum/all/event/pz_abs_mean": 12.585297584533691, "metrics/particle/momentum/all/event/pz_bias": 2.6979684829711914, "metrics/particle/momentum/all/event/pz_residual_rms": 23.756885528564453, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02254597656428814, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.12226039916276932, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1264004409313202, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.03605087473988533, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.16343671083450317, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0027657300233840942, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.41272932291030884, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.06690853089094162, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2384965419769287, "metrics/particle/momentum/class_1/accepted/samples": 15909.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.12039899826049805, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3368818461894989, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.36961066722869873, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.25223618745803833, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06333676725625992, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.6865644454956055, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0058359079994261265, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5530809164047241, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.063962921500206, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.203293800354004, "metrics/particle/momentum/class_1/assigned/samples": 17478.0, "metrics/particle/momentum/class_1/efficiency": 0.8967363834381104, "metrics/particle/momentum/class_1/event/energy_bias": 5.6994547843933105, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.235623359680176, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.446605682373047, "metrics/particle/momentum/class_1/event/px_bias": -0.31530535221099854, "metrics/particle/momentum/class_1/event/py_bias": 0.07086067646741867, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.605149269104004, "metrics/particle/momentum/class_1/event/pz_bias": 3.0570261478424072, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.373226165771484, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02588213048875332, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17699852585792542, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17923426628112793, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.011350121349096298, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22056227922439575, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.017301037907600403, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.2993254661560059, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0004937140038236976, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.282263994216919, "metrics/particle/momentum/class_2/accepted/samples": 1734.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5448039770126343, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.562825620174408, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7113306522369385, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.402619332075119, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2564696669578552, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.795537233352661, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04094244912266731, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.173621654510498, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.09780743718147278, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.4715800285339355, "metrics/particle/momentum/class_2/assigned/samples": 2589.0, "metrics/particle/momentum/class_2/efficiency": 0.5807099938392639, "metrics/particle/momentum/class_2/event/energy_bias": -8.806206703186035, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.000171661376953, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.507285118103027, "metrics/particle/momentum/class_2/event/px_bias": -0.10087385773658752, "metrics/particle/momentum/class_2/event/py_bias": -0.7483106255531311, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.566795349121094, "metrics/particle/momentum/class_2/event/pz_bias": -0.9481706619262695, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.92312240600586, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018840961158275604, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14836330711841583, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14691942930221558, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.00446073804050684, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18617255985736847, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015832805074751377, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2410401999950409, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.026205427944660187, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1655707359313965, "metrics/particle/momentum/class_3/accepted/samples": 12632.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2386244833469391, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.180507779121399, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1436198949813843, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9923796057701111, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13639958202838898, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.861316680908203, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005403268150985241, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.45193275809288025, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.019521255046129227, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.407991647720337, "metrics/particle/momentum/class_3/assigned/samples": 15176.0, "metrics/particle/momentum/class_3/efficiency": 0.8132363557815552, "metrics/particle/momentum/class_3/event/energy_bias": -1.9690749645233154, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.427474021911621, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.560527801513672, "metrics/particle/momentum/class_3/event/px_bias": -0.14427801966667175, "metrics/particle/momentum/class_3/event/py_bias": 0.6033526659011841, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.0066447257995605, "metrics/particle/momentum/class_3/event/pz_bias": 0.6183202266693115, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.243847846984863, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.023659680038690567, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17109133303165436, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16299264132976532, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07906743139028549, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21244952082633972, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003759398590773344, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.617316484451294, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.17611567676067352, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.6055034399032593, "metrics/particle/momentum/class_4/accepted/samples": 266.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.356070339679718, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 5.447953701019287, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.979194641113281, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.899008274078369, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23076923191547394, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 28.306434631347656, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.8064810037612915, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.21089333295822144, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.312671184539795, "metrics/particle/momentum/class_4/assigned/samples": 364.0, "metrics/particle/momentum/class_4/efficiency": 0.7093333601951599, "metrics/particle/momentum/class_4/event/energy_bias": -1.2272886037826538, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.768286943435669, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3414095640182495, "metrics/particle/momentum/class_4/event/px_bias": -0.16456608474254608, "metrics/particle/momentum/class_4/event/py_bias": 0.010315172374248505, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.921801745891571, "metrics/particle/momentum/class_4/event/pz_bias": -0.06682287156581879, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.0047554969787598, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01626216620206833, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1538550853729248, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15525361895561218, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03681683540344238, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19645851850509644, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.033557046204805374, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6483070850372314, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.17877064645290375, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.6591403484344482, "metrics/particle/momentum/class_5/accepted/samples": 149.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08329228311777115, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.20479144155979156, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2222520112991333, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0927746444940567, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.07926829159259796, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.41212859749794006, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.04878048598766327, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.8538161516189575, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.11059049516916275, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.1374192237854, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9085366129875183, "metrics/particle/momentum/class_5/event/energy_bias": -0.35160166025161743, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7415553331375122, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3008348941802979, "metrics/particle/momentum/class_5/event/px_bias": -0.015878237783908844, "metrics/particle/momentum/class_5/event/py_bias": -0.04138299822807312, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9066916704177856, "metrics/particle/momentum/class_5/event/pz_bias": 0.037615373730659485, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.066272735595703, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3304954469203949, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.33389341831207275, "metrics/set_slot/angular_recall_all_dr0p10": 0.9595028162002563, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8574074506759644, "metrics/set_slot/soft_count_bias_mean": 0.5886064171791077}, "step_train_time": 4.964316129684448, "step_valid_time": 17778.085774421692, "step_total_time": 17783.050090789795, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031188877794546, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12192407311204778, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3057, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2920, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2908, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9545603138280484, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9551848217206411, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9548724656638325, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044815178279358846, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9506374632232756, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9512594046450769, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9509483322432962, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9545603138280484, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9960307830390327, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11186126263325136, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2282, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2162, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2155, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9461706783369803, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9474145486415425, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9467922049485439, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05258545135845749, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9431072210065645, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9443470639789658, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9437267352747974, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9461706783369803, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048219785545651, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12345074156368518, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1231, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1229, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9311649016641452, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9361216730038023, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9336367083807356, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06387832699619772, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9296520423600605, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9346007604562737, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9321198331437239, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9311649016641452} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_37000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_37000.json new file mode 100644 index 0000000000000000000000000000000000000000..e95bcced9e1e4b45f5aa068795fa01928427d721 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_37000.json @@ -0,0 +1 @@ +{"epoch": 1.7525577870405458, "train": {"Classification_binary": 0.13177841901779175, "Cardinality": 0.11922861635684967, "Classification": 0.08069109171628952, "Regression_pt": 0.10235950350761414, "Regression_eta": 0.0013518192572519183, "Regression_sin_phi": 0.0005863639526069164, "Regression_cos_phi": 0.0007527527050115168, "Regression_energy": 0.10070452839136124, "HitGrouping": 0.15712976455688477, "Auxiliary": 0.18422390520572662, "Total": 0.9040235280990601, "Total_unweighted": 0.418224478547927}, "train_group": {}, "valid": {"Classification_binary": 0.148520827293396, "Cardinality": 0.11740245421727498, "Classification": 0.08517746130625407, "Regression_pt": 0.10335787137349446, "Regression_eta": 0.0012958416094382603, "Regression_sin_phi": 0.0007079370940725008, "Regression_cos_phi": 0.0007033436559140682, "Regression_energy": 0.10230209430058797, "HitGrouping": 0.1516843835512797, "Auxiliary": 0.1959033409754435, "Total": 0.9335909684499105, "Total_unweighted": 0.4420653766331573, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8476505279541016, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.606558620929718, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2857953906059265, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.619268000125885, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6238919496536255, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3409671485424042, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7953569889068604, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5444616079330444, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.27511078119277954, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5554118752479553, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5653689503669739, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.42023634910583496, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8629740476608276, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5691862106323242, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2612821161746979, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5779323577880859, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5781111717224121, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31298309564590454, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8530820608139038, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5785887837409973, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.26978209614753723, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5887067317962646, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5909913182258606, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32978785037994385, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003728121053427458, "metrics/hit_grouping/calorimeter/background_recall": 0.16282838582992554, "metrics/hit_grouping/calorimeter/completeness": 0.6591103076934814, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6586437821388245, "metrics/hit_grouping/calorimeter/energy_purity": 0.6502010226249695, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3075048625469208, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8731165528297424, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5897107720375061, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.20055757462978363, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6019303202629089, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5956576466560364, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.27224624156951904, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8576156497001648, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6063573360443115, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.28247350454330444, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6179094314575195, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.622452974319458, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.33049285411834717, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8076649308204651, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5279363989830017, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.39814385771751404, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5323244333267212, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5497814416885376, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.44640371203422546, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8407816886901855, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6431873440742493, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.32571566104888916, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6577572226524353, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6760810017585754, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.373466432094574, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8259214162826538, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6692456007003784, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.45136386156082153, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6670613288879395, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6694052815437317, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.38908904790878296, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8688589334487915, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.501926064491272, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.19112147390842438, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5089706182479858, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.49489521980285645, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2781899571418762, "metrics/hit_grouping/calorimeter/purity": 0.6498883366584778, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7808808088302612, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09423545747995377, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.06203007698059082, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08833691477775574, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06576784700155258, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.033834587782621384, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7999676465988159, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.23048590123653412, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10241318494081497, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.22778290510177612, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.20870649814605713, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2271924614906311, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8570225834846497, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6045957803726196, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.28187692165374756, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6156418919563293, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6194429993629456, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3399491608142853, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7304771542549133, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40648993849754333, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5170443058013916, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5560198426246643, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5855425000190735, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7767820954322815, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2778782546520233, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.4965095818042755, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5095682740211487, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.437010258436203, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6488862633705139, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.42214706540107727, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.42244085669517517, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4823226034641266, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6934091448783875, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6834099888801575, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40667852759361267, "metrics/hit_grouping/tracker/all/object_completeness": 0.4569879472255707, "metrics/hit_grouping/tracker/all/object_purity": 0.5069856643676758, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6413179039955139, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.010985811240971088, "metrics/hit_grouping/tracker/background_recall": 0.10277865827083588, "metrics/hit_grouping/tracker/completeness": 0.45547693967819214, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.448217511177063, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7147355079650879, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39058390259742737, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5292057991027832, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5651431083679199, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6070489287376404, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6658521294593811, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4296605587005615, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.43188440799713135, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.4939659833908081, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6601924300193787, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6364173293113708, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4169522225856781, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.33337724208831787, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4002917408943176, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6930568218231201, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6834515333175659, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4241317808628082, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4658748209476471, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5272393226623535, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6480427980422974, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.672498881816864, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4378078877925873, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4104640483856201, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4725535809993744, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6551724076271057, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6853118538856506, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3784082233905792, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4537750482559204, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4868278205394745, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6300916075706482, "metrics/hit_grouping/tracker/purity": 0.42964857816696167, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8881475329399109, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.050150830298662186, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.12337838858366013, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.11725421994924545, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06259427219629288, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6852791905403137, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2978833317756653, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.38901379704475403, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4013805389404297, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5080020427703857, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.652824878692627, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4713782072067261, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.513862133026123, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5762668251991272, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7416484951972961, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.598292887210846, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6931470036506653, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7978144884109497, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6690264344215393, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.04914017394185066, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09455156326293945, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.53550148010254, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2799868881702423, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.18836373090744019, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.8951062560081482, "metrics/particle/class_1/pid_accuracy": 0.9767972230911255, "metrics/particle/class_2/efficiency": 0.572431743144989, "metrics/particle/class_2/pid_accuracy": 0.7506925463676453, "metrics/particle/class_3/efficiency": 0.8067245483398438, "metrics/particle/class_3/pid_accuracy": 0.9787274599075317, "metrics/particle/class_4/efficiency": 0.6913827657699585, "metrics/particle/class_4/pid_accuracy": 0.5188405513763428, "metrics/particle/class_5/efficiency": 0.9214559197425842, "metrics/particle/class_5/pid_accuracy": 0.6237006187438965, "metrics/particle/count/bias_mean": -1.9153646230697632, "metrics/particle/count/mae": 3.5885417461395264, "metrics/particle/count/prediction_mean": 68.73958587646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06600891798734665, "metrics/particle/event/energy_response_mean": 0.9914519786834717, "metrics/particle/event/met_abs_error": 5.636712551116943, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05313460901379585, "metrics/particle/event/scalar_pt_response_mean": 1.0169566869735718, "metrics/particle/event/vector_pt_closure": 0.051571521908044815, "metrics/particle/matched/delta_eta_abs_mean": 0.012663744390010834, "metrics/particle/matched/delta_phi_abs_mean": 0.01297706551849842, "metrics/particle/matched/delta_r_mean": 0.02036389335989952, "metrics/particle/matched/energy_relative_abs_mean": 0.13379208743572235, "metrics/particle/matched/pid_accuracy": 0.9595052003860474, "metrics/particle/matched/pt_relative_abs_mean": 0.13573455810546875, "metrics/particle/matching/duplicate_fraction": 0.04605811461806297, "metrics/particle/matching/efficiency": 0.8298564553260803, "metrics/particle/matching/f1": 0.8412591814994812, "metrics/particle/matching/purity": 0.8529796004295349, "metrics/particle/matching/target_coverage_dr0p05": 0.7897370457649231, "metrics/particle/matching/target_coverage_dr0p10": 0.8674418926239014, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6447487473487854, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8603283762931824, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020545288920402527, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1335897594690323, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.13673393428325653, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.04201773926615715, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.176163911819458, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004479283466935158, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6213359832763672, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0009226970723830163, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.7230548858642578, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8930.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1889621615409851, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.825745165348053, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8167639970779419, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6973483562469482, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10137039422988892, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.0067138671875, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.009621926583349705, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8582337498664856, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.014647073112428188, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.65335750579834, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10289.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8290011286735535, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.6141774654388428, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.855916976928711, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.2044122219085693, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.06966105103492737, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.17004147171974182, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.7891693115234375, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7990077137947083, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.757756233215332, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023655852302908897, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14439772069454193, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1488182544708252, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04088559001684189, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.19019228219985962, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02252747304737568, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2971304655075073, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.09101460129022598, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.564004898071289, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1820.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.445477694272995, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3074358701705933, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.8325955867767334, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.6762912273406982, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1774873584508896, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 32.6719970703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.057757165282964706, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4504446983337402, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2058069258928299, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.980914115905762, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2372.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6598984599113464, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.215305805206299, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.058393478393555, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.063997745513916, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.027596071362495422, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08445543795824051, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.939701080322266, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.9722801446914673, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.766916275024414, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020243480801582336, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.13316190242767334, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13475042581558228, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.030537445098161697, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1746245175600052, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005067139863967896, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.21113182604312897, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.009921734221279621, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6641639471054077, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19735.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17351515591144562, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9344903230667114, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6972084045410156, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5646580457687378, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10077894479036331, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.638798713684082, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003212603973224759, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.35093942284584045, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02598859742283821, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4085828065872192, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22723.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.848124086856842, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.411945343017578, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.323516845703125, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.9488959312438965, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 1.4339215755462646, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.4204047918319702, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.8856120109558105, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.6563030481338501, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.345508575439453, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02053561434149742, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13395804166793823, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1361713409423828, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.03451818227767944, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17604252696037292, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0029850746504962444, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3961290419101715, "metrics/particle/momentum/all/accepted/pz_bias": 0.0007190151955001056, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3833246231079102, "metrics/particle/momentum/all/accepted/samples": 30485.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19623811542987823, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9278699159622192, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8751205801963806, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7447971701622009, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1060931459069252, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.356878280639648, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008732760325074196, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6391933560371399, "metrics/particle/momentum/all/assigned/pz_bias": 0.007152064237743616, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7552132606506348, "metrics/particle/momentum/all/assigned/samples": 35384.0, "metrics/particle/momentum/all/efficiency": 0.8284192681312561, "metrics/particle/momentum/all/event/energy_bias": 0.8108171820640564, "metrics/particle/momentum/all/event/p3_error_mean": 16.783422470092773, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.035554885864258, "metrics/particle/momentum/all/event/px_bias": 1.4759864807128906, "metrics/particle/momentum/all/event/py_bias": 1.6749014854431152, "metrics/particle/momentum/all/event/pz_abs_mean": 12.54548454284668, "metrics/particle/momentum/all/event/pz_bias": 0.48303064703941345, "metrics/particle/momentum/all/event/pz_residual_rms": 23.274385452270508, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021882755681872368, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11753010749816895, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.12069271504878998, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.04644203931093216, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1585298776626587, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0022730142809450626, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.4074162542819977, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.012791772373020649, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.225316047668457, "metrics/particle/momentum/class_1/accepted/samples": 15838.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11704698950052261, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.3220861852169037, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.37575072050094604, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.2774188816547394, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.06323063373565674, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.526883125305176, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005528361536562443, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5493553876876831, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01603781059384346, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2869205474853516, "metrics/particle/momentum/class_1/assigned/samples": 17365.0, "metrics/particle/momentum/class_1/efficiency": 0.8927343487739563, "metrics/particle/momentum/class_1/event/energy_bias": 9.342302322387695, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.899530410766602, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.939919471740723, "metrics/particle/momentum/class_1/event/px_bias": 0.6422547101974487, "metrics/particle/momentum/class_1/event/py_bias": 1.4021714925765991, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.067086219787598, "metrics/particle/momentum/class_1/event/pz_bias": 0.07928359508514404, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.457260131835938, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027223803102970123, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17326892912387848, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1765676587820053, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.033335380256175995, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21856729686260223, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.018834607675671577, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.2977137565612793, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.009755993261933327, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.2964401245117188, "metrics/particle/momentum/class_2/accepted/samples": 1699.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.4851323664188385, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.48408547043800354, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.592469334602356, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3187239170074463, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2284201979637146, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.5175840854644775, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04153094440698624, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.095149278640747, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.04028286784887314, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.222337245941162, "metrics/particle/momentum/class_2/assigned/samples": 2456.0, "metrics/particle/momentum/class_2/efficiency": 0.5689886212348938, "metrics/particle/momentum/class_2/event/energy_bias": -10.622215270996094, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.796087265014648, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.255889892578125, "metrics/particle/momentum/class_2/event/px_bias": -0.006142191588878632, "metrics/particle/momentum/class_2/event/py_bias": -0.011062175035476685, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.694278717041016, "metrics/particle/momentum/class_2/event/pz_bias": -0.9646468162536621, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.6817569732666, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01798754744231701, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14840780198574066, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14936362206935883, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.017726585268974304, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18896162509918213, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001434948993846774, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2426767796278, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.016143513843417168, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.056316614151001, "metrics/particle/momentum/class_3/accepted/samples": 12544.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.23754730820655823, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5699135065078735, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3596315383911133, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.21383535861969, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13279691338539124, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.116072654724121, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006782816722989082, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4906967580318451, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.026531873270869255, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.673992395401001, "metrics/particle/momentum/class_3/assigned/samples": 15038.0, "metrics/particle/momentum/class_3/efficiency": 0.8075709939002991, "metrics/particle/momentum/class_3/event/energy_bias": 1.583949089050293, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.049968719482422, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.800718307495117, "metrics/particle/momentum/class_3/event/px_bias": 0.16693395376205444, "metrics/particle/momentum/class_3/event/py_bias": 0.466895192861557, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.611945152282715, "metrics/particle/momentum/class_3/event/pz_bias": 1.3581414222717285, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.326337814331055, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02119777724146843, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16991400718688965, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16672976315021515, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.10247917473316193, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21527810394763947, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5785862803459167, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.07505659759044647, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1873334646224976, "metrics/particle/momentum/class_4/accepted/samples": 254.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.38399288058280945, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.641783714294434, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 6.904191493988037, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 6.83823823928833, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24242424964904785, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 45.6428108215332, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005509641952812672, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7569780349731445, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1057024747133255, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6314170360565186, "metrics/particle/momentum/class_4/assigned/samples": 363.0, "metrics/particle/momentum/class_4/efficiency": 0.6773333549499512, "metrics/particle/momentum/class_4/event/energy_bias": 0.4636591076850891, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.287708044052124, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7591853141784668, "metrics/particle/momentum/class_4/event/px_bias": 0.36916303634643555, "metrics/particle/momentum/class_4/event/py_bias": -0.005314141511917114, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1832462549209595, "metrics/particle/momentum/class_4/event/pz_bias": -0.07802507281303406, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.20022439956665, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014505458064377308, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15399779379367828, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.15797747671604156, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.07805902510881424, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19808664917945862, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03333333507180214, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5161452293395996, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.13014696538448334, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.40547251701355, "metrics/particle/momentum/class_5/accepted/samples": 150.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.049740590155124664, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.18818117678165436, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2030375897884369, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.10989456623792648, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.06172839552164078, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3218182325363159, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.043209876865148544, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7166216373443604, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.02917204424738884, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.779906988143921, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.9146341681480408, "metrics/particle/momentum/class_5/event/energy_bias": 0.0431230366230011, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8540050983428955, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.4294458627700806, "metrics/particle/momentum/class_5/event/px_bias": 0.3037770390510559, "metrics/particle/momentum/class_5/event/py_bias": -0.17778879404067993, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9354408979415894, "metrics/particle/momentum/class_5/event/pz_bias": 0.0882774367928505, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.8822271823883057, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.32217857241630554, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3246972858905792, "metrics/set_slot/angular_recall_all_dr0p10": 0.960682213306427, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8540257811546326, "metrics/set_slot/soft_count_bias_mean": -0.8138137459754944}, "step_train_time": 4.594891309738159, "step_valid_time": 20953.805755615234, "step_total_time": 20958.40064716339, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0296183614418002, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11693088221151404, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2926, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2915, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9565217391304348, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9565217391304348, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9565217391304348, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043478260869565216, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9529257927427264, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9529257927427264, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9529257927427264, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9565217391304348, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0220388368775848, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10935984245900865, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2276, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2170, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2163, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.949671772428884, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.953427065026362, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9515457136592852, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04657293497363796, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9466083150984683, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9503514938488576, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9484762113571585, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.949671772428884, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.030580622433157, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12190168768739218, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1323, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1237, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1233, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9357034795763994, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9349962207105065, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9353497164461247, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06500377928949358, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.93267776096823, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9319727891156463, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9323251417769376, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9357034795763994} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_38000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_38000.json new file mode 100644 index 0000000000000000000000000000000000000000..afcd41b11549b8a89ddce1231ea03338a6303dc1 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_38000.json @@ -0,0 +1 @@ +{"epoch": 1.7999242137173173, "train": {"Classification_binary": 0.12934282422065735, "Cardinality": 0.10902243107557297, "Classification": 0.07299648970365524, "Regression_pt": 0.09840776026248932, "Regression_eta": 0.001055089058354497, "Regression_sin_phi": 0.0005586999468505383, "Regression_cos_phi": 0.0006590168341062963, "Regression_energy": 0.09719989448785782, "HitGrouping": 0.14963816106319427, "Auxiliary": 0.18333883583545685, "Total": 0.8556353449821472, "Total_unweighted": 0.40021977451397106}, "train_group": {}, "valid": {"Classification_binary": 0.14543549219767252, "Cardinality": 0.11593518654505412, "Classification": 0.0831167201201121, "Regression_pt": 0.10100153088569641, "Regression_eta": 0.001269472607721885, "Regression_sin_phi": 0.0006881385731200377, "Regression_cos_phi": 0.0006794421933591366, "Regression_energy": 0.10024278362592061, "HitGrouping": 0.15023119250933328, "Auxiliary": 0.19374134143193564, "Total": 0.917323112487793, "Total_unweighted": 0.4324335802036027, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8521336317062378, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6049234867095947, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.28721490502357483, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6157846450805664, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6152040362358093, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3274344801902771, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7969376444816589, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5485888719558716, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2854505181312561, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5571132898330688, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5560705661773682, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.41986706852912903, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8673572540283203, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5797671675682068, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.26276326179504395, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5872420072555542, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5861256122589111, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3040037155151367, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8572779893875122, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5849677324295044, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.27193233370780945, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5935499668121338, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5926015973091125, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3200974762439728, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0036008788738399744, "metrics/hit_grouping/calorimeter/background_recall": 0.16119349002838135, "metrics/hit_grouping/calorimeter/completeness": 0.6639721393585205, "metrics/hit_grouping/calorimeter/energy_completeness": 0.664997398853302, "metrics/hit_grouping/calorimeter/energy_purity": 0.6564240455627441, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3099413812160492, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8787137866020203, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5972933173179626, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.20966090261936188, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6078323125839233, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5949391722679138, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2638256847858429, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8601224422454834, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6145669221878052, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2810916602611542, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6243273615837097, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6277961134910583, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.32128050923347473, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8107054829597473, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5300171971321106, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.38967517018318176, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5334227681159973, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5523794293403625, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4336426854133606, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.845680296421051, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6543697714805603, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3254750967025757, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6668227910995483, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.680249810218811, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3553043007850647, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8261144757270813, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6693187355995178, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4482758641242981, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6687588691711426, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6758209466934204, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3818836808204651, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8728153705596924, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5041589140892029, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.19633331894874573, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5098837614059448, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4933246374130249, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.27684101462364197, "metrics/hit_grouping/calorimeter/purity": 0.6545985341072083, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7790189981460571, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08751095086336136, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07330827414989471, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08756265789270401, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.057790983468294144, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03947368264198303, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8187209367752075, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.23898063600063324, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1059446707367897, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.23490434885025024, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.21758009493350983, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2377869337797165, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8605596423149109, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6110782027244568, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.28380656242370605, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6204581260681152, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6208316087722778, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.32895341515541077, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7354837656021118, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3953353762626648, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5234852433204651, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5656780004501343, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.575402021408081, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7842134833335876, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2694394290447235, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5033655166625977, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5088675618171692, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4285714328289032, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.653245210647583, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4236580729484558, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.42918097972869873, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.49034473299980164, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6923298239707947, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6881948113441467, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40356189012527466, "metrics/hit_grouping/tracker/all/object_completeness": 0.46364471316337585, "metrics/hit_grouping/tracker/all/object_purity": 0.5148663520812988, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.636910080909729, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.008505400270223618, "metrics/hit_grouping/tracker/background_recall": 0.09472686797380447, "metrics/hit_grouping/tracker/completeness": 0.4605276584625244, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.44753938913345337, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7228112816810608, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.38576188683509827, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5375369191169739, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5736435651779175, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.596528172492981, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6675570607185364, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42403340339660645, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4409351944923401, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5074282288551331, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.664730429649353, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6374931931495667, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.41983768343925476, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3342112898826599, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.40135207772254944, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6923354268074036, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6854706406593323, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.41793325543403625, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.47105035185813904, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.540899932384491, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6427782773971558, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6727991700172424, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.45258620381355286, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.41709163784980774, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.46998926997184753, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6582512259483337, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6944963932037354, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37609007954597473, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4623625576496124, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.4890668988227844, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6254553198814392, "metrics/hit_grouping/tracker/purity": 0.4333261251449585, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8876671195030212, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04223227873444557, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.12218614667654037, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.12059947103261948, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6889212727546692, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.31078988313674927, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3996220529079437, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.40577638149261475, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5141972303390503, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6585150361061096, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4672374129295349, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.521255373954773, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5851983428001404, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7353813648223877, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5748618245124817, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8124999403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.6959342956542969, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8002490997314453, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.66199791431427, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.025709057226777077, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0875231996178627, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.552833557128906, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29793813824653625, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20631498098373413, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9039936065673828, "metrics/particle/class_1/pid_accuracy": 0.9778692126274109, "metrics/particle/class_2/efficiency": 0.5946313142776489, "metrics/particle/class_2/pid_accuracy": 0.7607619166374207, "metrics/particle/class_3/efficiency": 0.8090199828147888, "metrics/particle/class_3/pid_accuracy": 0.9785715937614441, "metrics/particle/class_4/efficiency": 0.7094188332557678, "metrics/particle/class_4/pid_accuracy": 0.44915252923965454, "metrics/particle/class_5/efficiency": 0.9272030591964722, "metrics/particle/class_5/pid_accuracy": 0.5743801593780518, "metrics/particle/count/bias_mean": -1.291015625, "metrics/particle/count/mae": 3.4772136211395264, "metrics/particle/count/prediction_mean": 69.36392974853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06570512801408768, "metrics/particle/event/energy_response_mean": 0.9731672406196594, "metrics/particle/event/met_abs_error": 5.06174373626709, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04890300706028938, "metrics/particle/event/scalar_pt_response_mean": 0.9923157691955566, "metrics/particle/event/vector_pt_closure": 0.04914301633834839, "metrics/particle/matched/delta_eta_abs_mean": 0.013510392047464848, "metrics/particle/matched/delta_phi_abs_mean": 0.012980456463992596, "metrics/particle/matched/delta_r_mean": 0.021045377478003502, "metrics/particle/matched/energy_relative_abs_mean": 0.12899108231067657, "metrics/particle/matched/pid_accuracy": 0.9593391418457031, "metrics/particle/matched/pt_relative_abs_mean": 0.12887629866600037, "metrics/particle/matching/duplicate_fraction": 0.04326891526579857, "metrics/particle/matching/efficiency": 0.8371173739433289, "metrics/particle/matching/f1": 0.8448358178138733, "metrics/particle/matching/purity": 0.8526979684829712, "metrics/particle/matching/target_coverage_dr0p05": 0.7935241460800171, "metrics/particle/matching/target_coverage_dr0p10": 0.8744816780090332, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6715257167816162, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8671009540557861, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021063432097434998, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12892161309719086, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12887747585773468, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.007397443056106567, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16783232986927032, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004081632476300001, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.6033139824867249, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.009896286763250828, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.711065649986267, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9065.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1773025393486023, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7290835380554199, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7508501410484314, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6069336533546448, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09315147995948792, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.8918609619140625, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.009866511449217796, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8241025805473328, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.004439825192093849, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5586109161376953, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10338.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8415336012840271, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.14924770593643188, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.78664493560791, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.0892117023468018, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.23006625473499298, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.06812885403633118, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.831660747528076, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.9321304559707642, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.68149471282959, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02406321093440056, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14136050641536713, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14134691655635834, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.002056638477370143, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18235546350479126, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02083333395421505, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2242546081542969, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.16321931779384613, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.225555896759033, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1872.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.43818289041519165, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.8788548707962036, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.1907858848571777, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9982153177261353, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17498968541622162, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 23.136911392211914, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05530334264039993, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3833343982696533, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.43736371397972107, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.534509658813477, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2423.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6787527203559875, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.938879013061523, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.116743087768555, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1030893325805664, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.026260465383529663, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0953902155160904, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.98439359664917, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.9853189587593079, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.964950561523438, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020972471684217453, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1285688430070877, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12811434268951416, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013149112462997437, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1672312319278717, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005038544768467546, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20801398158073425, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0054491315968334675, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6666273474693298, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19847.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1682736575603485, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8102469444274902, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6279510259628296, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.48379483819007874, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09863975644111633, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.621884822845459, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028082493226975203, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3339994549751282, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02183820679783821, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3030154705047607, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22790.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8529374003410339, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.706508219242096, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.440152168273926, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.217907905578613, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.08756084740161896, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.663395345211029, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.4201712608337402, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.043638914823532104, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.408163547515869, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021187208592891693, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12945058941841125, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12914375960826874, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010530737228691578, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16836601495742798, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027936589904129505, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3862168490886688, "metrics/particle/momentum/all/accepted/pz_bias": -0.009326516650617123, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3346657752990723, "metrics/particle/momentum/all/accepted/samples": 30784.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18929500877857208, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7913212180137634, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7702053189277649, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6228190660476685, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10224747657775879, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.85474681854248, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008438581600785255, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6161919832229614, "metrics/particle/momentum/all/assigned/pz_bias": -0.014518316835165024, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.619389057159424, "metrics/particle/momentum/all/assigned/samples": 35551.0, "metrics/particle/momentum/all/efficiency": 0.8365444540977478, "metrics/particle/momentum/all/event/energy_bias": -5.4961395263671875, "metrics/particle/momentum/all/event/p3_error_mean": 16.79493522644043, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.795376777648926, "metrics/particle/momentum/all/event/px_bias": -0.2913666367530823, "metrics/particle/momentum/all/event/py_bias": 0.8269144296646118, "metrics/particle/momentum/all/event/pz_abs_mean": 12.837291717529297, "metrics/particle/momentum/all/event/pz_bias": -0.00954948179423809, "metrics/particle/momentum/all/event/pz_residual_rms": 24.479595184326172, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0224246084690094, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11032857745885849, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11075594276189804, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006296167150139809, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14704705774784088, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0019981267396360636, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.39576736092567444, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.02218576706945896, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2175757884979248, "metrics/particle/momentum/class_1/accepted/samples": 16015.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11720538139343262, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.24087513983249664, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2627308666706085, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.13308307528495789, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05272528901696205, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.900588035583496, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004652231466025114, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5195376873016357, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.024812906980514526, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.147536516189575, "metrics/particle/momentum/class_1/assigned/samples": 17411.0, "metrics/particle/momentum/class_1/efficiency": 0.9027112126350403, "metrics/particle/momentum/class_1/event/energy_bias": 5.698271751403809, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.28369426727295, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.28030776977539, "metrics/particle/momentum/class_1/event/px_bias": 0.6076732277870178, "metrics/particle/momentum/class_1/event/py_bias": 1.0879943370819092, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.826488494873047, "metrics/particle/momentum/class_1/event/pz_bias": -0.28705206513404846, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.658727645874023, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.027204688638448715, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17448867857456207, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17734086513519287, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.010827350430190563, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21700017154216766, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.019662922248244286, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3096643686294556, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.021702663972973824, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.36578631401062, "metrics/particle/momentum/class_2/accepted/samples": 1780.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.48223140835762024, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5461841821670532, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7968940734863281, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5029336214065552, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.21980197727680206, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.2541351318359375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04633663222193718, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.1715829372406006, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.018067868426442146, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.474878311157227, "metrics/particle/momentum/class_2/assigned/samples": 2525.0, "metrics/particle/momentum/class_2/efficiency": 0.596115231513977, "metrics/particle/momentum/class_2/event/energy_bias": -9.668281555175781, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.31500244140625, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.673959255218506, "metrics/particle/momentum/class_2/event/px_bias": -0.33707568049430847, "metrics/particle/momentum/class_2/event/py_bias": -0.5623536705970764, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.827845573425293, "metrics/particle/momentum/class_2/event/pz_bias": 0.04136422276496887, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.4970645904541, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018820885568857193, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14665962755680084, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1451670080423355, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01416624803096056, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18434292078018188, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011135857785120606, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2256534993648529, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0004667041648644954, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8201273083686829, "metrics/particle/momentum/class_3/accepted/samples": 12572.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.21962833404541016, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3897638320922852, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2789677381515503, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1346575021743774, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.13730037212371826, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.274558067321777, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006295142695307732, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4548822343349457, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.008468332700431347, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3694634437561035, "metrics/particle/momentum/class_3/assigned/samples": 15091.0, "metrics/particle/momentum/class_3/efficiency": 0.8093736171722412, "metrics/particle/momentum/class_3/event/energy_bias": -0.18072517216205597, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.301779747009277, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.594812393188477, "metrics/particle/momentum/class_3/event/px_bias": -0.5849000215530396, "metrics/particle/momentum/class_3/event/py_bias": 0.3553510904312134, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.878106117248535, "metrics/particle/momentum/class_3/event/pz_bias": 0.038440853357315063, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.204972267150879, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022163625806570053, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15627583861351013, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14723573625087738, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06736120581626892, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1936948597431183, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0037878789007663727, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5278508067131042, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.07030428946018219, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.44234299659729, "metrics/particle/momentum/class_4/accepted/samples": 264.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.40741583704948425, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.2768683433532715, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.025492191314697, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.9057016372680664, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22714681923389435, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 33.49458694458008, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008310249075293541, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6842790842056274, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.1431313306093216, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7867841720581055, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.7039999961853027, "metrics/particle/momentum/class_4/event/energy_bias": -1.0774723291397095, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6779723167419434, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.243109941482544, "metrics/particle/momentum/class_4/event/px_bias": -0.05627971887588501, "metrics/particle/momentum/class_4/event/py_bias": 0.10876508057117462, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8892930150032043, "metrics/particle/momentum/class_4/event/pz_bias": -0.08992263674736023, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.009010076522827, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014413099735975266, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14669083058834076, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14528322219848633, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.053535815328359604, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18784961104393005, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.026143791154026985, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5922555923461914, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.11028695851564407, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.5465505123138428, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.060377687215805054, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2600800096988678, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.25091803073883057, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.133425772190094, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04907975345849991, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.9851850867271423, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.024539876729249954, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.6298643350601196, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.2308187484741211, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.5555012226104736, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9329268336296082, "metrics/particle/momentum/class_5/event/energy_bias": -0.2679322361946106, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8448911905288696, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.38382887840271, "metrics/particle/momentum/class_5/event/px_bias": 0.07921555638313293, "metrics/particle/momentum/class_5/event/py_bias": -0.16284233331680298, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9507021307945251, "metrics/particle/momentum/class_5/event/pz_bias": 0.28762006759643555, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.1104812622070312, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3132023811340332, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3150460720062256, "metrics/set_slot/angular_recall_all_dr0p10": 0.9608573317527771, "metrics/set_slot/angular_recall_selected_dr0p10": 0.858522355556488, "metrics/set_slot/soft_count_bias_mean": -0.40215909481048584}, "step_train_time": 4.838921070098877, "step_valid_time": 24049.231190919876, "step_total_time": 24054.070112466812, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0001613261908524, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10720429072452575, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3047, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2922, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2911, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552141222621772, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.958976042008533, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.957091385522437, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04102395799146702, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9516181758744687, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9553659337052839, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534883720930233, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552141222621772, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9953454428896297, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10864450392126546, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2259, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2165, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2160, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9474835886214442, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9583886675520141, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9529049295774648, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04161133244798584, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9452954048140044, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9561752988047809, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9507042253521126, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9474835886214442, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9983049852819948, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11653321077099654, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1303, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1236, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1232, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9349470499243571, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9485801995395242, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9417142857142857, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.051419800460475826, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9319213313161876, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.945510360706063, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9386666666666666, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9349470499243571} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_39000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_39000.json new file mode 100644 index 0000000000000000000000000000000000000000..5d58034fe85d312bb7137108ad236439d8adddb9 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_39000.json @@ -0,0 +1 @@ +{"epoch": 1.8472906403940887, "train": {"Classification_binary": 0.13677485287189484, "Cardinality": 0.0941254273056984, "Classification": 0.08078167587518692, "Regression_pt": 0.10012387484312057, "Regression_eta": 0.001516931806690991, "Regression_sin_phi": 0.0005715910228900611, "Regression_cos_phi": 0.0005435752682387829, "Regression_energy": 0.09927007555961609, "HitGrouping": 0.14283035695552826, "Auxiliary": 0.18133127689361572, "Total": 0.8504943251609802, "Total_unweighted": 0.41958257724763826}, "train_group": {}, "valid": {"Classification_binary": 0.1411932905515035, "Cardinality": 0.10937270522117615, "Classification": 0.08232614398002625, "Regression_pt": 0.09829397002855937, "Regression_eta": 0.0012612488741676013, "Regression_sin_phi": 0.0006809063876668612, "Regression_cos_phi": 0.0006587516982108355, "Regression_energy": 0.09764886895815532, "HitGrouping": 0.148668110370636, "Auxiliary": 0.18716665108998617, "Total": 0.8923829396565756, "Total_unweighted": 0.4220631804782897, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8480977416038513, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6102381944656372, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2819153964519501, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6222758889198303, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6293073296546936, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.33888521790504456, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7998340725898743, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.549446702003479, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.28618907928466797, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5594960451126099, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5654434561729431, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4161742925643921, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8614262342453003, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.582103431224823, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2616060972213745, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5904720425605774, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5927460193634033, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.31043741106987, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8526064157485962, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5880916118621826, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2696674168109894, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5977022051811218, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6017026901245117, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3272649049758911, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.00367682590149343, "metrics/hit_grouping/calorimeter/background_recall": 0.16098207235336304, "metrics/hit_grouping/calorimeter/completeness": 0.6666423082351685, "metrics/hit_grouping/calorimeter/energy_completeness": 0.665820837020874, "metrics/hit_grouping/calorimeter/energy_purity": 0.6572719812393188, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3064276874065399, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8736317753791809, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5964584350585938, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2078971266746521, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6083106994628906, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6007408499717712, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2726445198059082, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8556544184684753, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6203575134277344, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.27809765934944153, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6303439140319824, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6369385719299316, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.326347291469574, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8066656589508057, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5385262370109558, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3871229588985443, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5431875586509705, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5681663751602173, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4395591616630554, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8418151140213013, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6561554670333862, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.32577580213546753, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6700531840324402, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6886610388755798, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3632427155971527, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8204895853996277, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6729438900947571, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4297478199005127, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6745946407318115, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6865655183792114, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3844570219516754, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8674351572990417, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5085873603820801, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1933901458978653, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5147753357887268, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5029332637786865, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2837696969509125, "metrics/hit_grouping/calorimeter/purity": 0.6572811603546143, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7711958289146423, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08328980952501297, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07894736528396606, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07991183549165726, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.055538393557071686, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.020676691085100174, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.804986298084259, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.23349188268184662, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09417304396629333, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2321532815694809, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.21414217352867126, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24602706730365753, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8564110994338989, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6147698760032654, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2819075584411621, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.62516188621521, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6307702660560608, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.33648809790611267, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7293426394462585, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4019991159439087, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.523655891418457, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5707235932350159, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5884398221969604, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7753615379333496, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.26883664727211, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5051712393760681, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5144127011299133, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4448463022708893, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6496189832687378, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43437904119491577, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4325319826602936, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.49054431915283203, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6908907890319824, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6834096908569336, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.412199467420578, "metrics/hit_grouping/tracker/all/object_completeness": 0.4659040570259094, "metrics/hit_grouping/tracker/all/object_purity": 0.5169500708580017, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6412288546562195, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.008349216543138027, "metrics/hit_grouping/tracker/background_recall": 0.09046415984630585, "metrics/hit_grouping/tracker/completeness": 0.4609844982624054, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4518192708492279, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7191892266273499, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39926353096961975, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.539501965045929, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5738294124603271, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.6008241176605225, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6646384000778198, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4269377291202545, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4416027367115021, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5100002884864807, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6709021329879761, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6284610033035278, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42416590452194214, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3386576771736145, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4068549573421478, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6948602199554443, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6842447519302368, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4350004196166992, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4756299555301666, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5385956168174744, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6412498950958252, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6577848196029663, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.44211822748184204, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4178200662136078, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.48258429765701294, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6785714030265808, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6869179010391235, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37719395756721497, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.46187999844551086, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.49497100710868835, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6345071196556091, "metrics/hit_grouping/tracker/purity": 0.43368765711784363, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8890572190284729, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04449472203850746, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13478828966617584, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13599693775177002, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.059954751282930374, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6852565407752991, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.29839959740638733, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3957376182079315, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.409748911857605, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5105834007263184, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6526920795440674, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.47910022735595703, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5226457715034485, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5851017236709595, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7416484951972961, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5994797945022583, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7072073221206665, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8151057362556458, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6601452827453613, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.050327006727457047, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.08567044883966446, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.443803787231445, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2923615276813507, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.2007383406162262, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9026967287063599, "metrics/particle/class_1/pid_accuracy": 0.9753655195236206, "metrics/particle/class_2/efficiency": 0.6018801927566528, "metrics/particle/class_2/pid_accuracy": 0.772299587726593, "metrics/particle/class_3/efficiency": 0.8068994283676147, "metrics/particle/class_3/pid_accuracy": 0.9789758920669556, "metrics/particle/class_4/efficiency": 0.6853707432746887, "metrics/particle/class_4/pid_accuracy": 0.48099416494369507, "metrics/particle/class_5/efficiency": 0.9291187524795532, "metrics/particle/class_5/pid_accuracy": 0.6123711466789246, "metrics/particle/count/bias_mean": -0.7584635615348816, "metrics/particle/count/mae": 3.283203125, "metrics/particle/count/prediction_mean": 69.896484375, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.07017461210489273, "metrics/particle/event/energy_response_mean": 0.9996142983436584, "metrics/particle/event/met_abs_error": 5.332352161407471, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05573461949825287, "metrics/particle/event/scalar_pt_response_mean": 1.0198792219161987, "metrics/particle/event/vector_pt_closure": 0.04986036941409111, "metrics/particle/matched/delta_eta_abs_mean": 0.01337526272982359, "metrics/particle/matched/delta_phi_abs_mean": 0.013030895963311195, "metrics/particle/matched/delta_r_mean": 0.020993566140532494, "metrics/particle/matched/energy_relative_abs_mean": 0.13174320757389069, "metrics/particle/matched/pid_accuracy": 0.9592725038528442, "metrics/particle/matched/pt_relative_abs_mean": 0.1318536102771759, "metrics/particle/matching/duplicate_fraction": 0.04693510755896568, "metrics/particle/matching/efficiency": 0.8359748125076294, "metrics/particle/matching/f1": 0.8404859900474548, "metrics/particle/matching/purity": 0.8450461626052856, "metrics/particle/matching/target_coverage_dr0p05": 0.7989698052406311, "metrics/particle/matching/target_coverage_dr0p10": 0.8795680403709412, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6610028743743896, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8655253052711487, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020822936668992043, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1304895132780075, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1302815079689026, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03949829563498497, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.17041172087192535, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004120726138353348, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5957032442092896, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.03209421783685684, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6806398630142212, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8979.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18471552431583405, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8390941023826599, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9494602084159851, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.833062469959259, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11037527769804001, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.741066932678223, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.008830022066831589, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8059588670730591, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05173634737730026, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.335066318511963, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10419.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8335499167442322, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.917947769165039, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.288519859313965, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.289724349975586, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.04073382169008255, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.45383790135383606, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.242569923400879, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.7751847505569458, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.17126750946045, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024010811001062393, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14282897114753723, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14075100421905518, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.016591308638453484, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18074244260787964, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023382272571325302, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2307260036468506, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.04997595399618149, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0087742805480957, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1839.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.48164287209510803, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1689302921295166, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.8993420600891113, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.7300643920898438, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.19074907898902893, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 30.676151275634766, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05525992810726166, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.470273017883301, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1733836978673935, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.713912010192871, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2443.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6667875051498413, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.558244228363037, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.821196556091309, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0774049758911133, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.004949834197759628, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.05869390070438385, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.696414947509766, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.3117474913597107, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.50981903076172, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020984066650271416, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1310051530599594, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.13191695511341095, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.039190053939819336, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.17168517410755157, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00045228403178043664, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.2037312537431717, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.007115448825061321, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.643967866897583, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19899.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16037090122699738, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9309490323066711, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7221359014511108, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6012886762619019, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10234466940164566, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.815659046173096, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003100030589848757, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3407471477985382, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02025074139237404, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.41231369972229, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22903.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8551720976829529, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.872590065002441, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.071142196655273, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.7520575523376465, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6318702697753906, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.1837329864501953, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.714839220046997, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2395089566707611, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.0714898109436035, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021118175238370895, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.13156232237815857, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.13196778297424316, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.037927187979221344, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.17187075316905975, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002897418336942792, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.37979528307914734, "metrics/particle/momentum/all/accepted/pz_bias": 0.01698310486972332, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2791714668273926, "metrics/particle/momentum/all/accepted/samples": 30717.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.189408078789711, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9204457402229309, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9370779991149902, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8142188191413879, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11072277277708054, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.29759693145752, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.00833216868340969, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6217333674430847, "metrics/particle/momentum/all/assigned/pz_bias": 0.016196485608816147, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.632546901702881, "metrics/particle/momentum/all/assigned/samples": 35765.0, "metrics/particle/momentum/all/efficiency": 0.8347237706184387, "metrics/particle/momentum/all/event/energy_bias": 3.2322933673858643, "metrics/particle/momentum/all/event/p3_error_mean": 16.72008514404297, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.122102737426758, "metrics/particle/momentum/all/event/px_bias": 0.5960862636566162, "metrics/particle/momentum/all/event/py_bias": -1.5788769721984863, "metrics/particle/momentum/all/event/pz_abs_mean": 12.443955421447754, "metrics/particle/momentum/all/event/pz_bias": 1.7029461860656738, "metrics/particle/momentum/all/event/pz_residual_rms": 22.856475830078125, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.022832293063402176, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11290881782770157, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11500145494937897, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.03412776440382004, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1518852412700653, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0022512662690132856, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.38917797803878784, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0009169875993393362, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1720545291900635, "metrics/particle/momentum/class_1/accepted/samples": 15991.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.1105021983385086, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.33872830867767334, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.43236300349235535, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.3266344666481018, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05808819457888603, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.8642497062683105, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004759447183459997, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5237654447555542, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.005989463068544865, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1916298866271973, "metrics/particle/momentum/class_1/assigned/samples": 17439.0, "metrics/particle/momentum/class_1/efficiency": 0.901358425617218, "metrics/particle/momentum/class_1/event/energy_bias": 7.663453102111816, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.347175598144531, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.352009773254395, "metrics/particle/momentum/class_1/event/px_bias": 1.4965451955795288, "metrics/particle/momentum/class_1/event/py_bias": -0.5505163669586182, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.857928276062012, "metrics/particle/momentum/class_1/event/pz_bias": 0.3852185606956482, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.502613067626953, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.026128221303224564, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1745765209197998, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1771148294210434, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.020147385075688362, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21797505021095276, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.020304568111896515, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.2867399454116821, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.09158150851726532, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.1689162254333496, "metrics/particle/momentum/class_2/accepted/samples": 1773.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5215418934822083, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6062445044517517, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7631121873855591, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.47751882672309875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.24384614825248718, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.7997076511383057, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04500000178813934, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.178978204727173, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.0856679156422615, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.419132709503174, "metrics/particle/momentum/class_2/assigned/samples": 2600.0, "metrics/particle/momentum/class_2/efficiency": 0.5937709212303162, "metrics/particle/momentum/class_2/event/energy_bias": -6.870915412902832, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.952062606811523, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.4666900634765625, "metrics/particle/momentum/class_2/event/px_bias": -0.3232196569442749, "metrics/particle/momentum/class_2/event/py_bias": -0.5245584845542908, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.612889289855957, "metrics/particle/momentum/class_2/event/pz_bias": 0.6880024075508118, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.484819412231445, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01825782284140587, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14864450693130493, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1466921716928482, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.044172823429107666, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1871492713689804, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009564801584929228, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22334209084510803, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.022014759480953217, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8221111297607422, "metrics/particle/momentum/class_3/accepted/samples": 12546.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2200554609298706, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5659027099609375, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4964592456817627, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3802367448806763, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.14561957120895386, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.387721061706543, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005853722803294659, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4547373950481415, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0290928203612566, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3574187755584717, "metrics/particle/momentum/class_3/assigned/samples": 15204.0, "metrics/particle/momentum/class_3/efficiency": 0.8076997399330139, "metrics/particle/momentum/class_3/event/energy_bias": 3.4187591075897217, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.367964744567871, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.518482685089111, "metrics/particle/momentum/class_3/event/px_bias": -0.5368292331695557, "metrics/particle/momentum/class_3/event/py_bias": -0.27322426438331604, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.057013511657715, "metrics/particle/momentum/class_3/event/pz_bias": 0.4841535687446594, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.340130805969238, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022586068138480186, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15185202658176422, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14940406382083893, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08327217400074005, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19449536502361298, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003968254197388887, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.589765727519989, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.13863596320152283, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.4008111953735352, "metrics/particle/momentum/class_4/accepted/samples": 252.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.374970406293869, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.457061290740967, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.363161325454712, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.2907707691192627, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.25626739859580994, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.151561737060547, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011142061091959476, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.778337299823761, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.022798484191298485, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9096271991729736, "metrics/particle/momentum/class_4/assigned/samples": 359.0, "metrics/particle/momentum/class_4/efficiency": 0.671999990940094, "metrics/particle/momentum/class_4/event/energy_bias": -0.8207266330718994, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7151339054107666, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.31509268283844, "metrics/particle/momentum/class_4/event/px_bias": -0.10345388948917389, "metrics/particle/momentum/class_4/event/py_bias": -0.045039691030979156, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8548781871795654, "metrics/particle/momentum/class_4/event/pz_bias": -0.027733029797673225, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.8072903156280518, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01610412821173668, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14832152426242828, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.145752415060997, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.05402801185846329, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18443982303142548, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025806451216340065, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3597816228866577, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.21612288057804108, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.12754487991333, "metrics/particle/momentum/class_5/accepted/samples": 155.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06616734713315964, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.17395643889904022, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.19011135399341583, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.10002820938825607, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.042944785207509995, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3577374815940857, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.030674846842885017, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.4954346418380737, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.16465316712856293, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.450042963027954, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9451219439506531, "metrics/particle/momentum/class_5/event/energy_bias": -0.15827655792236328, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.561671257019043, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.1967872381210327, "metrics/particle/momentum/class_5/event/px_bias": 0.063043974339962, "metrics/particle/momentum/class_5/event/py_bias": -0.18553820252418518, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7675748467445374, "metrics/particle/momentum/class_5/event/pz_bias": 0.17330460250377655, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.7086105346679688, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.30908408761024475, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.310773640871048, "metrics/set_slot/angular_recall_all_dr0p10": 0.9616036415100098, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8637285232543945, "metrics/set_slot/soft_count_bias_mean": 0.19642142951488495}, "step_train_time": 4.9668965339660645, "step_valid_time": 27098.726259708405, "step_total_time": 27103.69315648079, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0269187880876949, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11333394715201961, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3052, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2924, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2908, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9558679306963059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9580602883355177, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9569628538700704, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04193971166448231, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9506374632232756, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9528178243774574, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.951726395025364, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9558679306963059, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.01685205489934, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1125863599702106, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2275, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2168, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2163, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9487964989059081, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952967032967033, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9508771929824561, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047032967032967034, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9466083150984683, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9507692307692308, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9486842105263158, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9487964989059081, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0303239655755494, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12402727641168165, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1317, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1246, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1239, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9425113464447806, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9460895975702354, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9442970822281167, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053910402429764616, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9372163388804842, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9407744874715261, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9389920424403183, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9425113464447806} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_4000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..fe17a7fe2600ef0209db65f022d2a614754e9079 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 0.189465706707086, "train": {"Classification_binary": 0.3523973524570465, "Cardinality": 0.22543060779571533, "Classification": 0.2720145583152771, "Regression_pt": 0.36918145418167114, "Regression_eta": 0.00480747502297163, "Regression_sin_phi": 0.0025423425249755383, "Regression_cos_phi": 0.0026054061017930508, "Regression_energy": 0.36470335721969604, "HitGrouping": 0.2404739409685135, "Auxiliary": 0.43658551573753357, "Total": 2.392148494720459, "Total_unweighted": 1.368251945823431}, "train_group": {}, "valid": {"Classification_binary": 0.33964014053344727, "Cardinality": 0.24380211035410562, "Classification": 0.2520398696263631, "Regression_pt": 0.3587688207626343, "Regression_eta": 0.004629669400552909, "Regression_sin_phi": 0.0023209049055973687, "Regression_cos_phi": 0.0022807465866208076, "Regression_energy": 0.35379568735758465, "HitGrouping": 0.23873277505238852, "Auxiliary": 0.42779942353566486, "Total": 2.312616507212321, "Total_unweighted": 1.3134758391728003, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8200887441635132, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.29308274388313293, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3598940074443817, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.2936743497848511, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.2763294279575348, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.43919751048088074, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7781461477279663, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3213444948196411, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.34711965918540955, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.32198452949523926, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.280710905790329, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.48781388998031616, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8298709392547607, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.26706600189208984, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3371904790401459, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.2673846483230591, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.25050604343414307, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.42138394713401794, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8228915929794312, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.27916190028190613, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3448394536972046, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.2795881927013397, "metrics/hit_grouping/calorimeter/all/object_purity": 0.260674387216568, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4319380819797516, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0014517062809318304, "metrics/hit_grouping/calorimeter/background_recall": 0.05672849342226982, "metrics/hit_grouping/calorimeter/completeness": 0.3863121271133423, "metrics/hit_grouping/calorimeter/energy_completeness": 0.40712088346481323, "metrics/hit_grouping/calorimeter/energy_purity": 0.4015466868877411, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.53245609998703, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.850403368473053, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.2886102795600891, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2806668281555176, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.29122820496559143, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.2710592746734619, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3700500726699829, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8166967034339905, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.2959924042224884, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.36262091994285583, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.29567599296569824, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.2778756320476532, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4464532434940338, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7730364799499512, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.24294142425060272, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4577726125717163, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.23964732885360718, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.22217079997062683, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.54350346326828, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7876097559928894, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3224455714225769, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.43438777327537537, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.32343968749046326, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3107455372810364, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5131705403327942, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7702555060386658, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4724649488925934, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6670097708702087, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.47044628858566284, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.42110949754714966, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5167267322540283, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8651344776153564, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.21200208365917206, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.21515727043151855, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.21214082837104797, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.1905101090669632, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.33901527523994446, "metrics/hit_grouping/calorimeter/purity": 0.38003867864608765, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9010897874832153, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.040532536804676056, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.129699245095253, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.04088345915079117, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.005128219723701477, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.033834587782621384, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.9016680717468262, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.056568391621112823, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.08122424781322479, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.05613662675023079, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.03447747603058815, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.19894056022167206, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8175179958343506, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.2946336567401886, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3620631694793701, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.29510578513145447, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.27660924196243286, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.45054978132247925, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6333684325218201, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4485006630420685, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.29188981652259827, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.2989714443683624, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7469215989112854, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.716614842414856, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3586497902870178, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3163647949695587, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.2808571755886078, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5617842078208923, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.555962324142456, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.44790616631507874, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.22059501707553864, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.2219952940940857, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.810404360294342, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.5916193723678589, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44149598479270935, "metrics/hit_grouping/tracker/all/object_completeness": 0.2495812028646469, "metrics/hit_grouping/tracker/all/object_purity": 0.250001460313797, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7725289463996887, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.007563002873212099, "metrics/hit_grouping/tracker/background_recall": 0.05529681220650673, "metrics/hit_grouping/tracker/completeness": 0.2576924264431, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.6212452054023743, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.5993256568908691, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.43871647119522095, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.29992127418518066, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3056912124156952, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7821322083473206, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.5840794444084167, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4345616400241852, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.21831722557544708, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.22104409337043762, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7738246321678162, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.5832586288452148, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4541027843952179, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.1770932525396347, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.16421759128570557, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7514878511428833, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.5986244082450867, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4630211293697357, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.24729754030704498, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.2529034912586212, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7711641192436218, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.5842597484588623, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5338670015335083, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.27406054735183716, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.2975570857524872, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.756157636642456, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5838319659233093, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3969533145427704, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.24816162884235382, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.23770341277122498, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.777238130569458, "metrics/hit_grouping/tracker/purity": 0.24224132299423218, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.886101245880127, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.012820512987673283, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.021925220265984535, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.02243269607424736, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06372549384832382, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6515041589736938, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.21528136730194092, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.1291375756263733, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.0988037958741188, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5792462825775146, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5414283871650696, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5296289920806885, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.29641926288604736, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.30015993118286133, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8986626267433167, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4731084704399109, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.29516834020614624, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.31359025835990906, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5806139707565308, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.07604431360960007, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.006139172241091728, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.524131774902344, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.16050243377685547, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.06887926161289215, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5587214231491089, "metrics/particle/class_1/pid_accuracy": 0.9013517498970032, "metrics/particle/class_2/efficiency": 0.40967267751693726, "metrics/particle/class_2/pid_accuracy": 0.2985900044441223, "metrics/particle/class_3/efficiency": 0.469295859336853, "metrics/particle/class_3/pid_accuracy": 0.8931382894515991, "metrics/particle/class_4/efficiency": 0.495991975069046, "metrics/particle/class_4/pid_accuracy": 0.21010100841522217, "metrics/particle/class_5/efficiency": 0.5996168851852417, "metrics/particle/class_5/pid_accuracy": 0.207667738199234, "metrics/particle/count/bias_mean": -7.637369632720947, "metrics/particle/count/mae": 9.145182609558105, "metrics/particle/count/prediction_mean": 63.017578125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.16498537361621857, "metrics/particle/event/energy_response_mean": 1.0713187456130981, "metrics/particle/event/met_abs_error": 13.73414134979248, "metrics/particle/event/scalar_pt_relative_abs_error": 0.16955041885375977, "metrics/particle/event/scalar_pt_response_mean": 1.1259697675704956, "metrics/particle/event/vector_pt_closure": 0.11389917135238647, "metrics/particle/matched/delta_eta_abs_mean": 0.022135527804493904, "metrics/particle/matched/delta_phi_abs_mean": 0.020623665302991867, "metrics/particle/matched/delta_r_mean": 0.03381820395588875, "metrics/particle/matched/energy_relative_abs_mean": 0.19557929039001465, "metrics/particle/matched/pid_accuracy": 0.8485177755355835, "metrics/particle/matched/pt_relative_abs_mean": 0.19545890390872955, "metrics/particle/matching/duplicate_fraction": 0.12931452691555023, "metrics/particle/matching/efficiency": 0.5085232853889465, "metrics/particle/matching/f1": 0.537577748298645, "metrics/particle/matching/purity": 0.5701534152030945, "metrics/particle/matching/target_coverage_dr0p05": 0.46075594425201416, "metrics/particle/matching/target_coverage_dr0p10": 0.609107494354248, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3147909343242645, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.574442982673645, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03208201751112938, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.19434180855751038, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.19672825932502747, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013339856639504433, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.23828300833702087, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.015858208760619164, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.248816728591919, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.06299948692321777, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.8149335384368896, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5360.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.5582600235939026, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.754038095474243, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.707024097442627, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.5317165851593018, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.3244626522064209, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 53.00736999511719, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.033094506710767746, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.9223978519439697, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.22436228394508362, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.1794915199279785, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8793.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.49758633971214294, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 10.311625480651855, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 17.054901123046875, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.99104118347168, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.9081628322601318, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -1.2369208335876465, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 14.390128135681152, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 4.48616886138916, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 22.670188903808594, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030839601531624794, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.19769012928009033, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.19975808262825012, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01512034609913826, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.24303331971168518, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04541154205799103, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.146191120147705, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0922638326883316, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.95330810546875, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1057.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8395956158638, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.433126926422119, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 8.22925090789795, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 8.06368350982666, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3788037896156311, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 98.8018569946289, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08132214099168777, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.648103713989258, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.39572247862815857, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.2161226272583, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1906.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.3832487165927887, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.420380592346191, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 13.37282657623291, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.9348914623260498, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.2241145670413971, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.20680183172225952, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 13.156255722045898, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 3.3163158893585205, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 31.801408767700195, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.034940022975206375, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19491912424564362, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1939818561077118, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.0023189943749457598, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23544959723949432, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.001300073112361133, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.42260101437568665, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.003713548183441162, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0582239627838135, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 12307.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.5691943764686584, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.8723762035369873, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 3.3511226177215576, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 3.1722006797790527, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.3552286922931671, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 16.129283905029297, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.015812456607818604, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 1.0968997478485107, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.09277290105819702, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.304884672164917, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21755.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5289011001586914, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 24.9208984375, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 20.37453842163086, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 16.893844604492188, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -4.542544364929199, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -4.051701545715332, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 9.537344932556152, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.182003378868103, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 15.058289527893066, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03389040753245354, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.19491025805473328, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.19509412348270416, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.003148043993860483, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23669728636741638, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007957701571285725, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7564159035682678, "metrics/particle/momentum/all/accepted/pz_bias": 0.01526687853038311, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.0950937271118164, "metrics/particle/momentum/all/accepted/samples": 18724.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.5821123123168945, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.484851121902466, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.7340383529663086, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.556880235671997, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.34827756881713867, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 38.84543991088867, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.024342145770788193, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.470388412475586, "metrics/particle/momentum/all/assigned/pz_bias": 0.14621737599372864, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.264253616333008, "metrics/particle/momentum/all/assigned/samples": 32454.0, "metrics/particle/momentum/all/efficiency": 0.5088181495666504, "metrics/particle/momentum/all/event/energy_bias": 26.812145233154297, "metrics/particle/momentum/all/event/p3_error_mean": 37.04398727416992, "metrics/particle/momentum/all/event/pt_vector_error_mean": 19.290739059448242, "metrics/particle/momentum/all/event/px_bias": -5.674821853637695, "metrics/particle/momentum/all/event/py_bias": -5.495424270629883, "metrics/particle/momentum/all/event/pz_abs_mean": 26.87957763671875, "metrics/particle/momentum/all/event/pz_bias": 8.984488487243652, "metrics/particle/momentum/all/event/pz_residual_rms": 40.84318542480469, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03920646756887436, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.21150004863739014, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.21135424077510834, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.009747792966663837, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.2515457272529602, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007756623439490795, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.8439891338348389, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.04858589917421341, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.0787065029144287, "metrics/particle/momentum/class_1/accepted/samples": 9927.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.42630067467689514, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.8385165929794312, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.1620335578918457, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.9538289308547974, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.3032812476158142, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 15.220221519470215, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.018678786233067513, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.3401647806167603, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.1370055377483368, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.5373313426971436, "metrics/particle/momentum/class_1/assigned/samples": 16061.0, "metrics/particle/momentum/class_1/efficiency": 0.5595512986183167, "metrics/particle/momentum/class_1/event/energy_bias": 43.28075408935547, "metrics/particle/momentum/class_1/event/p3_error_mean": 29.57070541381836, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.946096420288086, "metrics/particle/momentum/class_1/event/px_bias": -4.287410259246826, "metrics/particle/momentum/class_1/event/py_bias": -3.0478858947753906, "metrics/particle/momentum/class_1/event/pz_abs_mean": 20.496198654174805, "metrics/particle/momentum/class_1/event/pz_bias": 6.3330607414245605, "metrics/particle/momentum/class_1/event/pz_residual_rms": 28.678932189941406, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0393986813724041, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23297035694122314, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22782482206821442, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06114362180233002, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2659570574760437, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03562752902507782, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.912392020225525, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.2013559192419052, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.415615558624268, "metrics/particle/momentum/class_2/accepted/samples": 1235.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6012711524963379, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7613911032676697, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.026645541191101, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6957454085350037, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.30304449796676636, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.151370525360107, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.051053863018751144, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.648822546005249, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.01122494786977768, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.403064250946045, "metrics/particle/momentum/class_2/assigned/samples": 2135.0, "metrics/particle/momentum/class_2/efficiency": 0.4135967791080475, "metrics/particle/momentum/class_2/event/energy_bias": -13.54189682006836, "metrics/particle/momentum/class_2/event/p3_error_mean": 26.200969696044922, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.18912124633789, "metrics/particle/momentum/class_2/event/px_bias": -0.3934849500656128, "metrics/particle/momentum/class_2/event/py_bias": -1.2051113843917847, "metrics/particle/momentum/class_2/event/pz_abs_mean": 18.519046783447266, "metrics/particle/momentum/class_2/event/pz_bias": 2.94600248336792, "metrics/particle/momentum/class_2/event/pz_residual_rms": 29.180757522583008, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02564002014696598, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.165428027510643, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1673623025417328, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.032484009861946106, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20879998803138733, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0031576056499034166, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.416171133518219, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0024325235281139612, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3475953340530396, "metrics/particle/momentum/class_3/accepted/samples": 7284.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.7633103728294373, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.750946521759033, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.953760147094727, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.837754726409912, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.4067648947238922, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 57.199073791503906, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.026348261162638664, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.4165934324264526, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.18768899142742157, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.575078010559082, "metrics/particle/momentum/class_3/assigned/samples": 13777.0, "metrics/particle/momentum/class_3/efficiency": 0.46893709897994995, "metrics/particle/momentum/class_3/event/energy_bias": 0.5840330719947815, "metrics/particle/momentum/class_3/event/p3_error_mean": 18.139972686767578, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 10.421399116516113, "metrics/particle/momentum/class_3/event/px_bias": -0.8947166800498962, "metrics/particle/momentum/class_3/event/py_bias": 0.17975646257400513, "metrics/particle/momentum/class_3/event/pz_abs_mean": 12.682144165039062, "metrics/particle/momentum/class_3/event/pz_bias": 0.31702345609664917, "metrics/particle/momentum/class_3/event/pz_residual_rms": 22.970439910888672, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03678351640701294, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1870415210723877, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17850373685359955, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0018709091236814857, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2242690771818161, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.011428571306169033, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.2517297267913818, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.10781319439411163, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.337491273880005, "metrics/particle/momentum/class_4/accepted/samples": 175.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.6307867765426636, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 7.697389602661133, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 6.302427291870117, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 6.168076038360596, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.42729970812797546, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.247793197631836, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.020771512761712074, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.57217276096344, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2525347173213959, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.4725770950317383, "metrics/particle/momentum/class_4/assigned/samples": 337.0, "metrics/particle/momentum/class_4/efficiency": 0.46666666865348816, "metrics/particle/momentum/class_4/event/energy_bias": -2.0733866691589355, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.8770556449890137, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.1929311752319336, "metrics/particle/momentum/class_4/event/px_bias": -0.19280177354812622, "metrics/particle/momentum/class_4/event/py_bias": -0.05687473714351654, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4851107597351074, "metrics/particle/momentum/class_4/event/pz_bias": -0.3819773197174072, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.526191711425781, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.034028440713882446, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.2379678338766098, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.22485597431659698, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.05551706999540329, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.26381897926330566, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.029126213863492012, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6757701635360718, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.5181377530097961, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.1608312129974365, "metrics/particle/momentum/class_5/accepted/samples": 103.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.2266865074634552, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.8233234882354736, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.8284389972686768, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4426047205924988, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.2569444477558136, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.546154260635376, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0763888880610466, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.431448459625244, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.7085925340652466, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 7.994813919067383, "metrics/particle/momentum/class_5/assigned/samples": 144.0, "metrics/particle/momentum/class_5/efficiency": 0.6280487775802612, "metrics/particle/momentum/class_5/event/energy_bias": -1.4373598098754883, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.972020149230957, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.9645538330078125, "metrics/particle/momentum/class_5/event/px_bias": 0.0935918539762497, "metrics/particle/momentum/class_5/event/py_bias": -1.3653085231781006, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.027784824371338, "metrics/particle/momentum/class_5/event/pz_bias": -0.22962085902690887, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.854328632354736, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6522539854049683, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6554368138313293, "metrics/set_slot/angular_recall_all_dr0p10": 0.839347243309021, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6380037665367126, "metrics/set_slot/soft_count_bias_mean": -1.1076076030731201}, "step_train_time": 5.6465840339660645, "step_valid_time": 11779.227527618408, "step_total_time": 11784.874111890793, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1776272673150718, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3055177818164583, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3061, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2676, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2354, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8747956848643348, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8742241097680497, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8745098039215686, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12577589023195035, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.769532526969598, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7690297288467821, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.769281045751634, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8747956848643348, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.136805938229125, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26447765748330854, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2294, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2027, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1902, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.887089715536105, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8836094158674804, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8853461454466041, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11639058413251961, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8323851203501094, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8291194420226679, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8307490718497489, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.887089715536105, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1279833789631108, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.29807758602890155, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1293, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1111, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1026, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.840393343419062, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8592420726991493, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8497131931166348, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.14075792730085074, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7760968229954615, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7935034802784223, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7847036328871893, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.840393343419062} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_40000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..a42548d1d97f56df4319d479b86cbedf098f2f2a --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 1.8946570670708602, "train": {"Classification_binary": 0.155012845993042, "Cardinality": 0.11697222292423248, "Classification": 0.08706113696098328, "Regression_pt": 0.10778424888849258, "Regression_eta": 0.0012925261398777366, "Regression_sin_phi": 0.0006347417365759611, "Regression_cos_phi": 0.0006492348620668054, "Regression_energy": 0.10607496649026871, "HitGrouping": 0.14470794796943665, "Auxiliary": 0.20470578968524933, "Total": 0.9365091323852539, "Total_unweighted": 0.45850970107130706}, "train_group": {}, "valid": {"Classification_binary": 0.14068013429641724, "Cardinality": 0.11565076311429341, "Classification": 0.08143722514311473, "Regression_pt": 0.0980895459651947, "Regression_eta": 0.0012503471225500107, "Regression_sin_phi": 0.0007016895494113365, "Regression_cos_phi": 0.0006553845790525278, "Regression_energy": 0.09720337390899658, "HitGrouping": 0.14892664551734924, "Auxiliary": 0.19005942344665527, "Total": 0.901519775390625, "Total_unweighted": 0.42001770056473714, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8515616655349731, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6089891791343689, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2779407501220703, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6211650371551514, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6255077123641968, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3315037488937378, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8022534847259521, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5468454360961914, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.26883310079574585, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5566958785057068, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5664821863174438, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.413220077753067, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8641817569732666, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5753867626190186, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2576718330383301, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.583577036857605, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5867614150047302, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3113168179988861, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8555505871772766, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5833508372306824, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2646788954734802, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5928774476051331, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5969252586364746, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3253440260887146, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0035983887501060963, "metrics/hit_grouping/calorimeter/background_recall": 0.15774044394493103, "metrics/hit_grouping/calorimeter/completeness": 0.6649959683418274, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6646243929862976, "metrics/hit_grouping/calorimeter/energy_purity": 0.656075656414032, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3021460175514221, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8772792816162109, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5915481448173523, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2015817016363144, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6035032272338867, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.596847414970398, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2682066559791565, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8596802949905396, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6127663850784302, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.27752187848091125, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6233494877815247, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6281933784484863, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.32128050923347473, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8070856928825378, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5370028614997864, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.38039442896842957, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5405135154724121, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5655837655067444, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.44593968987464905, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8424044847488403, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6533992886543274, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3208443522453308, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6672037243843079, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6861814260482788, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3634832799434662, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8295350670814514, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6688476800918579, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4297478199005127, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6686883568763733, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6798316240310669, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.37828099727630615, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8720529079437256, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5017464756965637, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.18774910271167755, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5080655217170715, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.49604612588882446, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2801520526409149, "metrics/hit_grouping/calorimeter/purity": 0.6556062698364258, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7896996736526489, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09635713696479797, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0714285746216774, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09422887861728668, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.07465734332799911, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0357142873108387, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8067452907562256, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2243882119655609, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09005296975374222, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2229907065629959, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.20432180166244507, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2477928251028061, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8591632843017578, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.609965980052948, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2769150733947754, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6202509999275208, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6258658170700073, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.33409905433654785, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.735377311706543, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3966391384601593, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5286717414855957, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5740649104118347, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.586266815662384, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7803770303726196, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2652200162410736, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5109641551971436, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5193052291870117, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4279686510562897, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6575612425804138, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4339473247528076, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4353119134902954, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4900679886341095, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6851345300674438, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6905494928359985, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41001781821250916, "metrics/hit_grouping/tracker/all/object_completeness": 0.46959373354911804, "metrics/hit_grouping/tracker/all/object_purity": 0.5180436968803406, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6357524394989014, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.01074756495654583, "metrics/hit_grouping/tracker/background_recall": 0.10388380289077759, "metrics/hit_grouping/tracker/completeness": 0.4667190611362457, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.45207837224006653, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.722763180732727, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39426618814468384, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5436740517616272, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5784383416175842, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5973172187805176, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6722093224525452, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4320203363895416, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4439103305339813, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5042397975921631, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6681793332099915, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6425073742866516, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4205590486526489, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3427281677722931, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4075269401073456, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6825969219207764, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6912360191345215, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4301604926586151, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.47892656922340393, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5395796895027161, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6375986933708191, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6864439249038696, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4538177251815796, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4323151409626007, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4726789891719818, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6366994976997375, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6903929114341736, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3759796917438507, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.46414363384246826, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.49817878007888794, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6331824660301208, "metrics/hit_grouping/tracker/purity": 0.44014719128608704, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.886556088924408, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03996983543038368, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13276362419128418, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13879160583019257, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06598793715238571, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.693659245967865, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.28755807876586914, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.3930560052394867, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4100627601146698, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5183272957801819, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.661125659942627, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4782049059867859, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5278740525245667, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5860273838043213, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.733031153678894, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6007730960845947, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.692906379699707, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.7990213632583618, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6537836790084839, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.05162036418914795, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.07930883765220642, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.70123863220215, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2968335747718811, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20521043241024017, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9034596085548401, "metrics/particle/class_1/pid_accuracy": 0.9772017002105713, "metrics/particle/class_2/efficiency": 0.573677659034729, "metrics/particle/class_2/pid_accuracy": 0.7662388682365417, "metrics/particle/class_3/efficiency": 0.8211748003959656, "metrics/particle/class_3/pid_accuracy": 0.9807257056236267, "metrics/particle/class_4/efficiency": 0.7124248743057251, "metrics/particle/class_4/pid_accuracy": 0.5091420412063599, "metrics/particle/class_5/efficiency": 0.9291187524795532, "metrics/particle/class_5/pid_accuracy": 0.5896907448768616, "metrics/particle/count/bias_mean": -1.6061197519302368, "metrics/particle/count/mae": 3.4524738788604736, "metrics/particle/count/prediction_mean": 69.048828125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.0656130239367485, "metrics/particle/event/energy_response_mean": 0.9848013520240784, "metrics/particle/event/met_abs_error": 5.2708306312561035, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0507291741669178, "metrics/particle/event/scalar_pt_response_mean": 1.0065597295761108, "metrics/particle/event/vector_pt_closure": 0.049709171056747437, "metrics/particle/matched/delta_eta_abs_mean": 0.01248271856456995, "metrics/particle/matched/delta_phi_abs_mean": 0.012377233244478703, "metrics/particle/matched/delta_r_mean": 0.019769612699747086, "metrics/particle/matched/energy_relative_abs_mean": 0.12705989181995392, "metrics/particle/matched/pid_accuracy": 0.9612263441085815, "metrics/particle/matched/pt_relative_abs_mean": 0.12768885493278503, "metrics/particle/matching/duplicate_fraction": 0.04348523169755936, "metrics/particle/matching/efficiency": 0.8403147459030151, "metrics/particle/matching/f1": 0.8499755263328552, "metrics/particle/matching/purity": 0.8598610162734985, "metrics/particle/matching/target_coverage_dr0p05": 0.8021488189697266, "metrics/particle/matching/target_coverage_dr0p10": 0.8749331831932068, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6741702556610107, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8693032264709473, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01957986131310463, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12498197704553604, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12633326649665833, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016376279294490814, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16520743072032928, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0054396092891693115, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5915619730949402, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.009301756508648396, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6772979497909546, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9008.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18048802018165588, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.6905651092529297, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7730602622032166, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6397591829299927, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09829721599817276, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.3540678024292, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01151315774768591, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8309183716773987, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0001234489755006507, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5811214447021484, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10336.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8362420797348022, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.8144367337226868, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.63937520980835, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.9747159481048584, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.020857911556959152, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.06422896683216095, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.67608642578125, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.42634549736976624, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.05252742767334, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023206504061818123, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13644909858703613, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13958244025707245, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018320584669709206, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17869551479816437, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02348443493247032, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2275757789611816, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.08931644260883331, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2017064094543457, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1831.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.42622730135917664, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.186855435371399, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.3521170616149902, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.1796796321868896, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17971380054950714, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 29.19554328918457, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0597643107175827, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4685535430908203, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.29556289315223694, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.072415351867676, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2376.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6638868451118469, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.25528621673584, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.063492774963379, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.10713529586792, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.0373329222202301, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06598164886236191, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.931880950927734, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.2696565389633179, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.332693099975586, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01966431923210621, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12741079926490784, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12744754552841187, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011099090799689293, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1667732447385788, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00030010504997335374, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20146234333515167, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004120559897273779, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6282932162284851, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19993.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16574864089488983, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8755390048027039, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6307805776596069, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.48653918504714966, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09367599338293076, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.87835168838501, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002368213376030326, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.32875460386276245, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.006773407571017742, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3568013906478882, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22802.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.859211802482605, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.016000747680664, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.889848709106445, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.6933698654174805, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6723242998123169, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.6713612675666809, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.5086870193481445, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.24338498711585999, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.887497901916504, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019850002601742744, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.127237930893898, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12784264981746674, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.013069752603769302, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1670510321855545, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003178515937179327, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3763725459575653, "metrics/particle/momentum/all/accepted/pz_bias": -0.0053498405031859875, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.298728346824646, "metrics/particle/momentum/all/accepted/samples": 30832.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18746526539325714, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.842532217502594, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7873526215553284, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6444088220596313, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10077715665102005, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.97622299194336, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.00886974111199379, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6180638074874878, "metrics/particle/momentum/all/assigned/pz_bias": -0.015461118891835213, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.735069513320923, "metrics/particle/momentum/all/assigned/samples": 35514.0, "metrics/particle/momentum/all/efficiency": 0.8378488421440125, "metrics/particle/momentum/all/event/energy_bias": -1.4248489141464233, "metrics/particle/momentum/all/event/p3_error_mean": 16.34009552001953, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.969069480895996, "metrics/particle/momentum/all/event/px_bias": 0.73051518201828, "metrics/particle/momentum/all/event/py_bias": 0.8015719056129456, "metrics/particle/momentum/all/event/pz_abs_mean": 12.022385597229004, "metrics/particle/momentum/all/event/pz_bias": -0.5999261140823364, "metrics/particle/momentum/all/event/pz_residual_rms": 23.241376876831055, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02124875597655773, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10874190926551819, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10952934622764587, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.015633130446076393, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14641322195529938, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0026923799887299538, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.39107435941696167, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0007821686449460685, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2132279872894287, "metrics/particle/momentum/class_1/accepted/samples": 15971.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11629556864500046, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.33274951577186584, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3517475724220276, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23196028172969818, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05825967341661453, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.91346549987793, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005510274320840836, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5276868343353271, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.006410809233784676, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.228689193725586, "metrics/particle/momentum/class_1/assigned/samples": 17422.0, "metrics/particle/momentum/class_1/efficiency": 0.900231122970581, "metrics/particle/momentum/class_1/event/energy_bias": 7.026257038116455, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.398704528808594, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.47684097290039, "metrics/particle/momentum/class_1/event/px_bias": 0.740703821182251, "metrics/particle/momentum/class_1/event/py_bias": 0.7372150421142578, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.763906478881836, "metrics/particle/momentum/class_1/event/pz_bias": 0.8163779973983765, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.458877563476562, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.026123810559511185, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17089101672172546, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17376750707626343, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.02487776055932045, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2146407961845398, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.021893491968512535, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.3334922790527344, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.027097243815660477, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3266444206237793, "metrics/particle/momentum/class_2/accepted/samples": 1690.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.472927063703537, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.48745352029800415, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6930003762245178, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.41411691904067993, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2365981489419937, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.542323112487793, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04554615169763565, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.136538028717041, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.027482284232974052, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.287715911865234, "metrics/particle/momentum/class_2/assigned/samples": 2481.0, "metrics/particle/momentum/class_2/efficiency": 0.5659745335578918, "metrics/particle/momentum/class_2/event/energy_bias": -8.4410400390625, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.779393196105957, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.489815711975098, "metrics/particle/momentum/class_2/event/px_bias": -0.11757224798202515, "metrics/particle/momentum/class_2/event/py_bias": -0.10031050443649292, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.397937774658203, "metrics/particle/momentum/class_2/event/pz_bias": -0.906808614730835, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.455726623535156, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0172896571457386, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14370892941951752, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14401330053806305, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0065100365318357944, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1824108362197876, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010192880872637033, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21406324207782745, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01028510183095932, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7853859663009644, "metrics/particle/momentum/class_3/accepted/samples": 12754.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.21882012486457825, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4623385667800903, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2628424167633057, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.113553762435913, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12501657009124756, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.659832954406738, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00656237592920661, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4609416723251343, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03226607292890549, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.66265869140625, "metrics/particle/momentum/class_3/assigned/samples": 15086.0, "metrics/particle/momentum/class_3/efficiency": 0.821090579032898, "metrics/particle/momentum/class_3/event/energy_bias": 0.5400190949440002, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.261222839355469, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.5356764793396, "metrics/particle/momentum/class_3/event/px_bias": 0.07285100966691971, "metrics/particle/momentum/class_3/event/py_bias": -0.04167658090591431, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.8855299949646, "metrics/particle/momentum/class_3/event/pz_bias": -0.7956069707870483, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.06937026977539, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02214931510388851, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16018378734588623, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15120869874954224, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0667518749833107, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1963701993227005, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.00374531839042902, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.597923219203949, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08206330239772797, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.5176855325698853, "metrics/particle/momentum/class_4/accepted/samples": 267.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.41629141569137573, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.2526705265045166, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.8211164474487305, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.7282257080078125, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22589531540870667, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 26.121070861816406, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.002754820976406336, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6493968963623047, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07260330766439438, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.5318635702133179, "metrics/particle/momentum/class_4/assigned/samples": 363.0, "metrics/particle/momentum/class_4/efficiency": 0.7120000123977661, "metrics/particle/momentum/class_4/event/energy_bias": -0.8539038896560669, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7917790412902832, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.28168523311615, "metrics/particle/momentum/class_4/event/px_bias": -0.11994000524282455, "metrics/particle/momentum/class_4/event/py_bias": 0.0870971828699112, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0171375274658203, "metrics/particle/momentum/class_4/event/pz_bias": 0.06950998306274414, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.8778717517852783, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0138397803530097, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14562709629535675, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14377270638942719, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.06929812580347061, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19154159724712372, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02666666731238365, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4337328672409058, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.1491909772157669, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.956990957260132, "metrics/particle/momentum/class_5/accepted/samples": 150.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03686998039484024, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2259083241224289, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.24231743812561035, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.1696367859840393, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.0555555559694767, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.9511183500289917, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03703703731298447, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.643884539604187, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.27882513403892517, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.4727866649627686, "metrics/particle/momentum/class_5/assigned/samples": 162.0, "metrics/particle/momentum/class_5/efficiency": 0.9146341681480408, "metrics/particle/momentum/class_5/event/energy_bias": 0.30381929874420166, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8093273639678955, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3045462369918823, "metrics/particle/momentum/class_5/event/px_bias": 0.15447258949279785, "metrics/particle/momentum/class_5/event/py_bias": 0.11924684047698975, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9765083193778992, "metrics/particle/momentum/class_5/event/pz_bias": 0.21660137176513672, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.744781255722046, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.3042842149734497, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3063710331916809, "metrics/set_slot/angular_recall_all_dr0p10": 0.9605808854103088, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8604574203491211, "metrics/set_slot/soft_count_bias_mean": -0.7438536286354065}, "step_train_time": 6.156254768371582, "step_valid_time": 30338.109176397324, "step_total_time": 30344.265431404114, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0107898764902647, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11046940889579715, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3044, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2926, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2910, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9565217391304348, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9612352168199737, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9588726855644765, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03876478318002628, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9512912716574043, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9559789750328516, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9536293626085531, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9565217391304348, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0092323877582907, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10382511535439376, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2267, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2169, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2163, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.949234135667396, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.956771063078959, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.95298769771529, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043228936921041024, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9466083150984683, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9541243934715483, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9503514938488576, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.949234135667396, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.017249400349999, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11552412776951315, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1310, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1233, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1229, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.93267776096823, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9412213740458015, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9369300911854104, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05877862595419847, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9296520423600605, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9381679389312977, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9338905775075987, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.93267776096823} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_41000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_41000.json new file mode 100644 index 0000000000000000000000000000000000000000..15a975a39b5666e5c10a0051150090fdb7efa093 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_41000.json @@ -0,0 +1 @@ +{"epoch": 1.9420234937476317, "train": {"Classification_binary": 0.14973866939544678, "Cardinality": 0.09562607854604721, "Classification": 0.08931286633014679, "Regression_pt": 0.09741561859846115, "Regression_eta": 0.0013573836768046021, "Regression_sin_phi": 0.0006735136266797781, "Regression_cos_phi": 0.0006941041792742908, "Regression_energy": 0.09607192873954773, "HitGrouping": 0.14460060000419617, "Auxiliary": 0.1993022859096527, "Total": 0.9032452702522278, "Total_unweighted": 0.4352640845463611}, "train_group": {}, "valid": {"Classification_binary": 0.13952417174975076, "Cardinality": 0.10587618748346965, "Classification": 0.08497776587804158, "Regression_pt": 0.09901274243990581, "Regression_eta": 0.0012480419439574082, "Regression_sin_phi": 0.0007002657124151787, "Regression_cos_phi": 0.0006717077145973841, "Regression_energy": 0.09810419877370198, "HitGrouping": 0.14792638023694357, "Auxiliary": 0.1934273640314738, "Total": 0.8973255157470703, "Total_unweighted": 0.4242388942123701, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8550570607185364, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6071022748947144, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.28182077407836914, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6190938353538513, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6167410612106323, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.32516324520111084, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8008728623390198, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5457383394241333, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.28581979870796204, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5540227890014648, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5553088784217834, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4121122658252716, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8657424449920654, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5744322538375854, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2575792670249939, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5828405022621155, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.584597647190094, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3059939742088318, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8574689030647278, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5821020007133484, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.26711583137512207, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5915862321853638, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5920616984367371, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32004013657569885, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003833450609818101, "metrics/hit_grouping/calorimeter/background_recall": 0.1638854444026947, "metrics/hit_grouping/calorimeter/completeness": 0.6703632473945618, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6718051433563232, "metrics/hit_grouping/calorimeter/energy_purity": 0.6631834506988525, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30567696690559387, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8798914551734924, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.589670717716217, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.20266272127628326, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6009662747383118, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5886683464050293, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2581360936164856, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8587920069694519, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6090947985649109, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.27257025241851807, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6195645928382874, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6256071329116821, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3231229782104492, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.810417115688324, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5394762754440308, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.39303943514823914, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5442743301391602, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.565186083316803, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.443155437707901, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8441246747970581, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.659593939781189, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.32192686200141907, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6732136011123657, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6919748783111572, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3616189658641815, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8322952389717102, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.677833080291748, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4374678432941437, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6804020404815674, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6870453357696533, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3746783435344696, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8740734457969666, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4916892349720001, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.19093751907348633, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4977809488773346, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4788781404495239, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.27113863825798035, "metrics/hit_grouping/calorimeter/purity": 0.6610537171363831, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7857993841171265, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.0889054387807846, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07706766575574875, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08784909546375275, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05885472148656845, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.020676691085100174, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8039495944976807, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.2151785045862198, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09417304396629333, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.21331758797168732, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.1904665231704712, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23660977184772491, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8614218235015869, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.609232485294342, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.27921223640441895, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6194788813591003, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.621648371219635, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3292596936225891, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7369626760482788, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4033029079437256, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5265189409255981, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5629674196243286, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5745328068733215, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7762027978897095, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.26883664727211, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5005910396575928, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5048948526382446, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4496684670448303, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6482869386672974, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4357461631298065, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.43764767050743103, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.4957115352153778, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6907468438148499, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6849895715713501, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4134461283683777, "metrics/hit_grouping/tracker/all/object_completeness": 0.46961116790771484, "metrics/hit_grouping/tracker/all/object_purity": 0.5170606970787048, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6372217535972595, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011536424979567528, "metrics/hit_grouping/tracker/background_recall": 0.10937006771564484, "metrics/hit_grouping/tracker/completeness": 0.4650421142578125, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.44987449049949646, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7191774249076843, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3922497034072876, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5429635047912598, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5766010880470276, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5960021018981934, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6659882664680481, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4352877140045166, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4440730810165405, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5101810693740845, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6685423851013184, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6335436105728149, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43534716963768005, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3440987467765808, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.40142178535461426, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6908926963806152, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6870147585868835, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.43890634179115295, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.479536771774292, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5330822467803955, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6360703110694885, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6648433208465576, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.44396552443504333, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4172157943248749, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.47974398732185364, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6570196747779846, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6859683394432068, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3748758137226105, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4661006033420563, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5029219388961792, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6351694464683533, "metrics/hit_grouping/tracker/purity": 0.4389156997203827, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8871966004371643, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04223227873444557, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.12720800936222076, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13199718296527863, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06598793715238571, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6927534341812134, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.29426950216293335, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.40257203578948975, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4067136347293854, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5172947645187378, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6541411280632019, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4814504086971283, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5276889801025391, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5861631631851196, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7349896430969238, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6110823154449463, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7074683308601379, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8141002058982849, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6656945943832397, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.061929523944854736, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09121982753276825, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.53384780883789, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2783273160457611, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.1867041289806366, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9028111696243286, "metrics/particle/class_1/pid_accuracy": 0.9767839908599854, "metrics/particle/class_2/efficiency": 0.5936119556427002, "metrics/particle/class_2/pid_accuracy": 0.7782865762710571, "metrics/particle/class_3/efficiency": 0.8236669898033142, "metrics/particle/class_3/pid_accuracy": 0.980518639087677, "metrics/particle/class_4/efficiency": 0.6983968019485474, "metrics/particle/class_4/pid_accuracy": 0.5136298537254333, "metrics/particle/class_5/efficiency": 0.9406130313873291, "metrics/particle/class_5/pid_accuracy": 0.5885946750640869, "metrics/particle/count/bias_mean": -0.7350260615348816, "metrics/particle/count/mae": 3.2141926288604736, "metrics/particle/count/prediction_mean": 69.919921875, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06598085165023804, "metrics/particle/event/energy_response_mean": 0.9825213551521301, "metrics/particle/event/met_abs_error": 5.159819602966309, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04924936965107918, "metrics/particle/event/scalar_pt_response_mean": 1.0030488967895508, "metrics/particle/event/vector_pt_closure": 0.048593729734420776, "metrics/particle/matched/delta_eta_abs_mean": 0.012412519194185734, "metrics/particle/matched/delta_phi_abs_mean": 0.01253577508032322, "metrics/particle/matched/delta_r_mean": 0.019833451136946678, "metrics/particle/matched/energy_relative_abs_mean": 0.12811212241649628, "metrics/particle/matched/pid_accuracy": 0.9613315463066101, "metrics/particle/matched/pt_relative_abs_mean": 0.12873750925064087, "metrics/particle/matching/duplicate_fraction": 0.04511299356818199, "metrics/particle/matching/efficiency": 0.8425999283790588, "metrics/particle/matching/f1": 0.8470056653022766, "metrics/particle/matching/purity": 0.8514576554298401, "metrics/particle/matching/target_coverage_dr0p05": 0.8040561676025391, "metrics/particle/matching/target_coverage_dr0p10": 0.8791441917419434, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6744190454483032, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8725743293762207, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020136220380663872, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1278097927570343, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.128636434674263, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.021684449166059494, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1676132082939148, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0050649638287723064, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5889689922332764, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.06886892765760422, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6617714166641235, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9082.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1808137446641922, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.6623361110687256, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7084211707115173, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.5765364766120911, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10194500535726547, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.705101490020752, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.010539427399635315, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8196674585342407, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.06009334325790405, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4753923416137695, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10437.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8431117534637451, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.5561964511871338, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.786743640899658, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.0571255683898926, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2556625306606293, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.14323434233665466, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.784523010253906, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.4035041332244873, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.958800315856934, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023179227486252785, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.14211367070674896, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1426958292722702, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.021187255159020424, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1842702478170395, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019967619329690933, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1869938373565674, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.11340387910604477, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0558595657348633, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1853.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.45998334884643555, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.6347686648368835, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.3784351348876953, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.20400869846344, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18144330382347107, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 10.48684310913086, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05154639109969139, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.351598024368286, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3873150050640106, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.661374568939209, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2425.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6718636751174927, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.890584945678711, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.001420021057129, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.055147409439087, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.048366978764534, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07320001721382141, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.873801231384277, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3541491031646729, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.26339340209961, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019570767879486084, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12852314114570618, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12876948714256287, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02316252700984478, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16787268221378326, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00039890300831757486, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1998058557510376, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.02383456751704216, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6698760986328125, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20055.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16411057114601135, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8190906643867493, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6578899025917053, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5210549831390381, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09640707820653915, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.409336090087891, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0025725995656102896, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3232940435409546, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.02420613169670105, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2843916416168213, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22934.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8618763089179993, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.7698326706886292, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.475697994232178, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.166286468505859, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.5247581005096436, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.7816126346588135, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.556044578552246, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.1162662506103516, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.025028228759766, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01995224505662918, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12912671267986298, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12956321239471436, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02261125110089779, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16882236301898956, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0029364312067627907, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.37288230657577515, "metrics/particle/momentum/all/accepted/pz_bias": -0.04238810017704964, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2876530885696411, "metrics/particle/momentum/all/accepted/samples": 30990.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18902459740638733, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7608988881111145, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7214365005493164, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5834984183311462, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1037825420498848, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 7.218917369842529, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008213208056986332, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6054285168647766, "metrics/particle/momentum/all/assigned/pz_bias": -0.0592685341835022, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.611499071121216, "metrics/particle/momentum/all/assigned/samples": 35796.0, "metrics/particle/momentum/all/efficiency": 0.8421424627304077, "metrics/particle/momentum/all/event/energy_bias": -2.5645558834075928, "metrics/particle/momentum/all/event/p3_error_mean": 16.11351776123047, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.499691486358643, "metrics/particle/momentum/all/event/px_bias": 0.8287876844406128, "metrics/particle/momentum/all/event/py_bias": 0.9980469942092896, "metrics/particle/momentum/all/event/pz_abs_mean": 12.168050765991211, "metrics/particle/momentum/all/event/pz_bias": -3.8739194869995117, "metrics/particle/momentum/all/event/pz_residual_rms": 23.772533416748047, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021208548918366432, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.11120249330997467, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.11312190443277359, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.031049629673361778, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1503346711397171, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0023739614989608526, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.38807034492492676, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.04076819866895676, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1973274946212769, "metrics/particle/momentum/class_1/accepted/samples": 16007.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11603334546089172, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2865132987499237, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.30385470390319824, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.19386497139930725, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0586889386177063, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.3599610328674316, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005090950522571802, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5132097005844116, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.04780413210391998, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.0896389484405518, "metrics/particle/momentum/class_1/assigned/samples": 17482.0, "metrics/particle/momentum/class_1/efficiency": 0.9022603034973145, "metrics/particle/momentum/class_1/event/energy_bias": 5.54661750793457, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.442752838134766, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.882404804229736, "metrics/particle/momentum/class_1/event/px_bias": 0.8751423358917236, "metrics/particle/momentum/class_1/event/py_bias": 0.4871153235435486, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.074469566345215, "metrics/particle/momentum/class_1/event/pz_bias": -1.2501736879348755, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.973836898803711, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.025688162073493004, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17569629848003387, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1755586415529251, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.022824998944997787, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21734090149402618, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.021227767691016197, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.2768243551254272, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.06472839415073395, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.2411515712738037, "metrics/particle/momentum/class_2/accepted/samples": 1743.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5232954621315002, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5674480199813843, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7024184465408325, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4007812440395355, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.24824629724025726, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.6906979084014893, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04364769905805588, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.133206605911255, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.11256157606840134, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.499304294586182, "metrics/particle/momentum/class_2/assigned/samples": 2566.0, "metrics/particle/momentum/class_2/efficiency": 0.5837240219116211, "metrics/particle/momentum/class_2/event/energy_bias": -7.934820652008057, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.670708656311035, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.264613628387451, "metrics/particle/momentum/class_2/event/px_bias": 0.2080734223127365, "metrics/particle/momentum/class_2/event/py_bias": 0.08436320722103119, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.526839256286621, "metrics/particle/momentum/class_2/event/pz_bias": -1.0681650638580322, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.4789981842041, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017618337646126747, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14453931152820587, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14334888756275177, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.010730940848588943, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1818547546863556, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008581012370996177, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2162034511566162, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.037895068526268005, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8119943737983704, "metrics/particle/momentum/class_3/accepted/samples": 12819.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.21366702020168304, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2781859636306763, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1568578481674194, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0127410888671875, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1289326697587967, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.421100616455078, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005517241545021534, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4414120316505432, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.062411386519670486, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3879663944244385, "metrics/particle/momentum/class_3/assigned/samples": 15225.0, "metrics/particle/momentum/class_3/efficiency": 0.8252752423286438, "metrics/particle/momentum/class_3/event/energy_bias": 0.6148334741592407, "metrics/particle/momentum/class_3/event/p3_error_mean": 7.990739822387695, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.358798027038574, "metrics/particle/momentum/class_3/event/px_bias": -0.1484432965517044, "metrics/particle/momentum/class_3/event/py_bias": 0.5444134473800659, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.787628650665283, "metrics/particle/momentum/class_3/event/pz_bias": -1.520444393157959, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.710355758666992, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02269441820681095, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16008225083351135, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1531878262758255, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.06856193393468857, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19874295592308044, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003759398590773344, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5588623285293579, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.09085194766521454, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.380391240119934, "metrics/particle/momentum/class_4/accepted/samples": 266.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.367093950510025, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.575860023498535, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.9746270179748535, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.8815839290618896, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22777777910232544, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 16.334362030029297, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0055555556900799274, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6922944784164429, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.02877550572156906, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6925195455551147, "metrics/particle/momentum/class_4/assigned/samples": 360.0, "metrics/particle/momentum/class_4/efficiency": 0.7093333601951599, "metrics/particle/momentum/class_4/event/energy_bias": -0.7608712911605835, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9151941537857056, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4070701599121094, "metrics/particle/momentum/class_4/event/px_bias": -0.15739232301712036, "metrics/particle/momentum/class_4/event/py_bias": -0.0020011253654956818, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.052152395248413, "metrics/particle/momentum/class_4/event/pz_bias": -0.051543958485126495, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.96523380279541, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014026564545929432, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1286989450454712, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.12958303093910217, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.05245053023099899, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1741194874048233, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.025806451216340065, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2781039476394653, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.24687433242797852, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.743042230606079, "metrics/particle/momentum/class_5/accepted/samples": 155.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06024039909243584, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.15062077343463898, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.1602713167667389, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0797748789191246, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.042944785207509995, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2429354190826416, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.042944785207509995, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.5733569860458374, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.35078293085098267, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.786877155303955, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9451219439506531, "metrics/particle/momentum/class_5/event/energy_bias": -0.030314385890960693, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7771008014678955, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2984172105789185, "metrics/particle/momentum/class_5/event/px_bias": 0.05140747129917145, "metrics/particle/momentum/class_5/event/py_bias": -0.11584386974573135, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9398267865180969, "metrics/particle/momentum/class_5/event/pz_bias": 0.016407767310738564, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2404003143310547, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.30159375071525574, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.3039475381374359, "metrics/set_slot/angular_recall_all_dr0p10": 0.9608296751976013, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8642536997795105, "metrics/set_slot/soft_count_bias_mean": 0.3755010664463043}, "step_train_time": 5.791576862335205, "step_valid_time": 33725.906257390976, "step_total_time": 33731.69783449173, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0091044805085323, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11222277437280681, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3060, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2938, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2924, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9604445897352076, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9601307189542484, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9602876286974996, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03986928104575163, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9558679306963059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9555555555555556, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9557117176009152, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9604445897352076, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0071551603858637, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1100207720491273, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2281, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2179, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2174, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9536105032822757, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9552827707145989, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9544459045116075, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04471722928540114, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9514223194748359, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9530907496711969, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9522558037669733, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9536105032822757, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0178488568539998, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11083118127502456, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1320, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1237, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1232, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9357034795763994, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9371212121212121, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9364118092354277, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06287878787878788, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9319213313161876, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9333333333333333, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9326267978803936, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9357034795763994} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_42000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_42000.json new file mode 100644 index 0000000000000000000000000000000000000000..0eea41abfe28726ccf7ea420309feb4fda5e9410 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_42000.json @@ -0,0 +1 @@ +{"epoch": 1.9893899204244032, "train": {"Classification_binary": 0.13773873448371887, "Cardinality": 0.08929336816072464, "Classification": 0.08737461268901825, "Regression_pt": 0.09726949781179428, "Regression_eta": 0.0012455238029360771, "Regression_sin_phi": 0.0007320315926335752, "Regression_cos_phi": 0.0006637400365434587, "Regression_energy": 0.09580355137586594, "HitGrouping": 0.1489727646112442, "Auxiliary": 0.18468934297561646, "Total": 0.8723663091659546, "Total_unweighted": 0.42082769179251045}, "train_group": {}, "valid": {"Classification_binary": 0.13843934734662375, "Cardinality": 0.10600767532984416, "Classification": 0.08129751185576121, "Regression_pt": 0.09552968541781108, "Regression_eta": 0.0011830040408919256, "Regression_sin_phi": 0.0006391608233874043, "Regression_cos_phi": 0.0006381093602006634, "Regression_energy": 0.09484806656837463, "HitGrouping": 0.1478476126988729, "Auxiliary": 0.18664666016896567, "Total": 0.8738349278767904, "Total_unweighted": 0.41257488541305065, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8504773378372192, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6110115647315979, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2826724648475647, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6230418682098389, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6249954700469971, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3295164108276367, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.795321524143219, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5453780889511108, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2854505181312561, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.555473268032074, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5597788095474243, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.416543573141098, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8599939942359924, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5820956826210022, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.25420042872428894, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5907356142997742, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.599362850189209, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.30835455656051636, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8520898818969727, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5880050659179688, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.2652522921562195, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.597785234451294, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6040551066398621, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3231651484966278, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003928820136934519, "metrics/hit_grouping/calorimeter/background_recall": 0.1643364578485489, "metrics/hit_grouping/calorimeter/completeness": 0.6708922386169434, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6713374257087708, "metrics/hit_grouping/calorimeter/energy_purity": 0.6628216505050659, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3007794916629791, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8726780414581299, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.597388744354248, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.20209376513957977, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6092396378517151, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.603168249130249, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.26615840196609497, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.857725203037262, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6152860522270203, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2771764099597931, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6258063316345215, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6335180401802063, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.32277750968933105, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8044341206550598, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5413888692855835, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3820185661315918, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5462007522583008, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5761814713478088, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.43979117274284363, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.842527449131012, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6610190272331238, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.31759682297706604, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6753870844841003, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6989926099777222, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3576497435569763, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.817435085773468, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6796738505363464, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.43386515974998474, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6815692186355591, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7015923261642456, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3911477029323578, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8659681081771851, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.502642035484314, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1917959451675415, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5086837410926819, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.49564027786254883, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2799068093299866, "metrics/hit_grouping/calorimeter/purity": 0.6616387367248535, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7660222053527832, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08390339463949203, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.06766917556524277, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08198621869087219, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06305712461471558, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.028195489197969437, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7870592474937439, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.22651053965091705, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09417304396629333, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2239554226398468, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.21530896425247192, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2177751660346985, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8568764925003052, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.615030825138092, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2773745059967041, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6256434321403503, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6331000924110413, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3334558606147766, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7353333234786987, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.40388238430023193, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5315172076225281, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5730328559875488, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5764160752296448, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7799559831619263, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2730560600757599, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5029946565628052, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5134807229042053, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4352019429206848, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6489383578300476, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4322924017906189, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.44009652733802795, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.500350296497345, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6857101917266846, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6851691007614136, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.41179874539375305, "metrics/hit_grouping/tracker/all/object_completeness": 0.4728403389453888, "metrics/hit_grouping/tracker/all/object_purity": 0.5236589908599854, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6336153149604797, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011102287098765373, "metrics/hit_grouping/tracker/background_recall": 0.10123934596776962, "metrics/hit_grouping/tracker/completeness": 0.46926429867744446, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.44243934750556946, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7204995155334473, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.38970717787742615, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.546661913394928, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5790905356407166, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5907416939735413, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6669893264770508, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43438011407852173, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.44725045561790466, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5171384811401367, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6665456295013428, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6305566430091858, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43480613827705383, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3464137613773346, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.41611477732658386, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.689089298248291, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6878499984741211, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4383968710899353, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.48398464918136597, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5432853698730469, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6351363062858582, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6479333639144897, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4341132938861847, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4175368547439575, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5047962665557861, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6773399114608765, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6883591413497925, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3732199966907501, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4682665169239044, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5015252828598022, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6237995624542236, "metrics/hit_grouping/tracker/purity": 0.44270622730255127, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8872394561767578, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04901960864663124, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13342542946338654, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13314077258110046, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06297134608030319, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6903886198997498, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.3154362440109253, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.4059150516986847, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4091731011867523, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5054207444190979, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6546167731285095, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4760785698890686, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5304622650146484, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5940194129943848, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.732191801071167, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.5957516431808472, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7250060439109802, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8360394239425659, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6657732129096985, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.046598874032497406, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09129839390516281, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.21107292175293, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3111695647239685, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.2195463925600052, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9096006155014038, "metrics/particle/class_1/pid_accuracy": 0.9764540791511536, "metrics/particle/class_2/efficiency": 0.5921395421028137, "metrics/particle/class_2/pid_accuracy": 0.7823259234428406, "metrics/particle/class_3/efficiency": 0.8287606835365295, "metrics/particle/class_3/pid_accuracy": 0.9820891618728638, "metrics/particle/class_4/efficiency": 0.7064128518104553, "metrics/particle/class_4/pid_accuracy": 0.4921985864639282, "metrics/particle/class_5/efficiency": 0.9291187524795532, "metrics/particle/class_5/pid_accuracy": 0.6432989835739136, "metrics/particle/count/bias_mean": -1.0475260019302368, "metrics/particle/count/mae": 3.236328125, "metrics/particle/count/prediction_mean": 69.607421875, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06985598802566528, "metrics/particle/event/energy_response_mean": 0.9598560333251953, "metrics/particle/event/met_abs_error": 4.893466949462891, "metrics/particle/event/scalar_pt_relative_abs_error": 0.050162360072135925, "metrics/particle/event/scalar_pt_response_mean": 0.9781889915466309, "metrics/particle/event/vector_pt_closure": 0.04856244847178459, "metrics/particle/matched/delta_eta_abs_mean": 0.01216951385140419, "metrics/particle/matched/delta_phi_abs_mean": 0.012371398508548737, "metrics/particle/matched/delta_r_mean": 0.01947769708931446, "metrics/particle/matched/energy_relative_abs_mean": 0.1257391721010208, "metrics/particle/matched/pid_accuracy": 0.9622807502746582, "metrics/particle/matched/pt_relative_abs_mean": 0.12578341364860535, "metrics/particle/matching/duplicate_fraction": 0.04197648540139198, "metrics/particle/matching/efficiency": 0.8479258418083191, "metrics/particle/matching/f1": 0.8542584180831909, "metrics/particle/matching/purity": 0.8606863021850586, "metrics/particle/matching/target_coverage_dr0p05": 0.8106167912483215, "metrics/particle/matching/target_coverage_dr0p10": 0.8816505074501038, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6855776309967041, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8761310577392578, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019620133563876152, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12466773390769958, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12505339086055756, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.007288326974958181, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16408134996891022, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004160280339419842, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5722498893737793, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0025593251921236515, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.667726993560791, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9134.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18038545548915863, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.5314306020736694, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.6257490515708923, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.4824121594429016, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09369594603776932, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.539440155029297, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.009417643770575523, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7708002328872681, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.014069187454879284, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.294074296951294, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10406.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8479390740394592, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.9196125268936157, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.32079553604126, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.9097378253936768, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.16824094951152802, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.32434090971946716, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.357794284820557, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7223250269889832, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.240312576293945, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02258867584168911, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13687197864055634, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13835103809833527, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.011121312156319618, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17819660902023315, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0200324859470129, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1546988487243652, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0343424454331398, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.938154697418213, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1847.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.46652278304100037, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.978081464767456, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.953075647354126, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.766270875930786, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1739310920238495, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 46.57648849487305, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.056039851158857346, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3899905681610107, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.27907758951187134, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.821516990661621, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2409.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6696881651878357, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.879810333251953, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.983035087585449, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0248926877975464, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.025388332083821297, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0912247970700264, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.862234592437744, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.1368844509124756, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.292930603027344, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.019282182678580284, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12594446539878845, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12547992169857025, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008985216729342937, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16431158781051636, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00044625147711485624, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19508294761180878, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.01145370863378048, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6191210150718689, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20168.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15867933630943298, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8010707497596741, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5724363923072815, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.42432093620300293, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09178554266691208, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.538862705230713, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002268464071676135, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3130486309528351, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.011217520572245121, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4418905973434448, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22923.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8667325377464294, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.10190533101558685, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.296952247619629, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.1592607498168945, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.2203565537929535, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.888708770275116, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.2325997352600098, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.403444766998291, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.5829691886901855, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01957734301686287, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1262180507183075, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12611804902553558, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.008614289574325085, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1651003509759903, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002696715760976076, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.36258283257484436, "metrics/particle/momentum/all/accepted/pz_bias": -0.0046290745958685875, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2552475929260254, "metrics/particle/momentum/all/accepted/samples": 31149.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18575048446655273, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.734490156173706, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7484320402145386, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5991000533103943, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09787900745868683, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.355191230773926, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.00797470472753048, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5863351821899414, "metrics/particle/momentum/all/assigned/pz_bias": -0.007520139217376709, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6437928676605225, "metrics/particle/momentum/all/assigned/samples": 35738.0, "metrics/particle/momentum/all/efficiency": 0.8464632034301758, "metrics/particle/momentum/all/event/energy_bias": -7.901327133178711, "metrics/particle/momentum/all/event/p3_error_mean": 15.45284652709961, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.544188976287842, "metrics/particle/momentum/all/event/px_bias": -0.41398581862449646, "metrics/particle/momentum/all/event/py_bias": 1.3042744398117065, "metrics/particle/momentum/all/event/pz_abs_mean": 11.344039916992188, "metrics/particle/momentum/all/event/pz_bias": -0.818004310131073, "metrics/particle/momentum/all/event/pz_residual_rms": 22.458633422851562, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.020717168226838112, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10621292144060135, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10681182891130447, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0057427287101745605, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14280211925506592, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002610966097563505, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3831292688846588, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.01049573253840208, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2379943132400513, "metrics/particle/momentum/class_1/accepted/samples": 16086.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.11480741947889328, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.26217177510261536, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.283376008272171, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.15652304887771606, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05410821735858917, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.352940320968628, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00538219278678298, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5122262239456177, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01444999873638153, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.210658550262451, "metrics/particle/momentum/class_1/assigned/samples": 17465.0, "metrics/particle/momentum/class_1/efficiency": 0.9067132472991943, "metrics/particle/momentum/class_1/event/energy_bias": -0.5420712232589722, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.924521446228027, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.672191143035889, "metrics/particle/momentum/class_1/event/px_bias": -0.116449736058712, "metrics/particle/momentum/class_1/event/py_bias": 1.1716142892837524, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.665915489196777, "metrics/particle/momentum/class_1/event/pz_bias": -0.6579688191413879, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.022122383117676, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02455810084939003, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.171479731798172, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1713259518146515, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.008815987035632133, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21291273832321167, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.015410958789288998, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.2001867294311523, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.01800462417304516, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 2.9291813373565674, "metrics/particle/momentum/class_2/accepted/samples": 1752.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5125986337661743, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.4583677053451538, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6445708274841309, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.31897154450416565, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.24405458569526672, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.948591470718384, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.03820662945508957, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.042707681655884, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.001977458829060197, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.209254741668701, "metrics/particle/momentum/class_2/assigned/samples": 2565.0, "metrics/particle/momentum/class_2/efficiency": 0.586738109588623, "metrics/particle/momentum/class_2/event/energy_bias": -7.559085369110107, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.560555458068848, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.300390720367432, "metrics/particle/momentum/class_2/event/px_bias": 0.34367600083351135, "metrics/particle/momentum/class_2/event/py_bias": -0.2534278631210327, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.354069709777832, "metrics/particle/momentum/class_2/event/pz_bias": -0.3355256915092468, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.179521560668945, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01751359924674034, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1443224847316742, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1435556858778, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.010243899188935757, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18204404413700104, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010076737962663174, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21029454469680786, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.000186872566700913, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.79283207654953, "metrics/particle/momentum/class_3/accepted/samples": 12901.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.20836150646209717, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2883023023605347, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2725465297698975, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1251636743545532, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12046367675065994, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.196043014526367, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005861819256097078, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.41850578784942627, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0008417609496973455, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.474597930908203, "metrics/particle/momentum/class_3/assigned/samples": 15183.0, "metrics/particle/momentum/class_3/efficiency": 0.8305543065071106, "metrics/particle/momentum/class_3/event/energy_bias": 0.7366729974746704, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.287751197814941, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.475364685058594, "metrics/particle/momentum/class_3/event/px_bias": -0.6761903166770935, "metrics/particle/momentum/class_3/event/py_bias": 0.5470936298370361, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.897294044494629, "metrics/particle/momentum/class_3/event/pz_bias": 0.20380359888076782, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.015690803527832, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.021784963086247444, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.15776512026786804, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1510707437992096, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07465450465679169, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.19729334115982056, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5090377330780029, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.05943522974848747, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9848353266716003, "metrics/particle/momentum/class_4/accepted/samples": 259.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3916740417480469, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.4942193031311035, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.1825976371765137, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.086803674697876, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.24450549483299255, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.094280242919922, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5437224507331848, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.005013998597860336, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0368483066558838, "metrics/particle/momentum/class_4/assigned/samples": 364.0, "metrics/particle/momentum/class_4/efficiency": 0.690666675567627, "metrics/particle/momentum/class_4/event/energy_bias": -0.7996788620948792, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.617403268814087, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.225440263748169, "metrics/particle/momentum/class_4/event/px_bias": 0.007359410636126995, "metrics/particle/momentum/class_4/event/py_bias": 0.010227911174297333, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8374729156494141, "metrics/particle/momentum/class_4/event/pz_bias": -0.053181275725364685, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.721299648284912, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01289529912173748, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1313011199235916, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1256546974182129, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.05967746675014496, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.162510946393013, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.013245033100247383, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2152096033096313, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.04027893766760826, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.6155948638916016, "metrics/particle/momentum/class_5/accepted/samples": 151.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0764080286026001, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.16446323692798615, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18284328281879425, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0984182357788086, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05590062215924263, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3660309910774231, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.02484472095966339, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.346476435661316, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.1383153349161148, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.0247042179107666, "metrics/particle/momentum/class_5/assigned/samples": 161.0, "metrics/particle/momentum/class_5/efficiency": 0.9207317233085632, "metrics/particle/momentum/class_5/event/energy_bias": 0.2628343999385834, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5442265272140503, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.1764204502105713, "metrics/particle/momentum/class_5/event/px_bias": 0.02761881798505783, "metrics/particle/momentum/class_5/event/py_bias": -0.1712334305047989, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7790010571479797, "metrics/particle/momentum/class_5/event/pz_bias": 0.02486797794699669, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.612096071243286, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.297950804233551, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2998676300048828, "metrics/set_slot/angular_recall_all_dr0p10": 0.9625527262687683, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8658385872840881, "metrics/set_slot/soft_count_bias_mean": 0.20321404933929443}, "step_train_time": 5.511737108230591, "step_valid_time": 36893.067073345184, "step_total_time": 36898.57881069183, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9911691081574843, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10472035536403534, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3040, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2933, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2920, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9588100686498856, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9648026315789474, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9617970159042466, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03519736842105263, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9545603138280484, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9605263157894737, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9575340219708149, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9588100686498856, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9822988759713812, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10222781647110601, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2267, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2162, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2159, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9461706783369803, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9536832818703131, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9499121265377856, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04631671812968681, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9448577680525164, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9523599470666079, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9485940246045694, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9461706783369803, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9897696792131088, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1114358779194552, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1314, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1242, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1240, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9394856278366112, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9452054794520548, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9423368740515933, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0547945205479452, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9379727685325264, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.943683409436834, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9408194233687405, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9394856278366112} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_43000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_43000.json new file mode 100644 index 0000000000000000000000000000000000000000..c35749a4d3de07bf87f85a3c011857ad0579d722 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_43000.json @@ -0,0 +1 @@ +{"epoch": 2.0367563471011745, "train": {"Classification_binary": 0.1529199182987213, "Cardinality": 0.10352778434753418, "Classification": 0.08465775847434998, "Regression_pt": 0.09088625013828278, "Regression_eta": 0.0012538637965917587, "Regression_sin_phi": 0.0007013584254309535, "Regression_cos_phi": 0.000588325085118413, "Regression_energy": 0.08969976752996445, "HitGrouping": 0.14795169234275818, "Auxiliary": 0.19105347990989685, "Total": 0.8898809552192688, "Total_unweighted": 0.42070724174845964}, "train_group": {}, "valid": {"Classification_binary": 0.13953299323717752, "Cardinality": 0.1091078519821167, "Classification": 0.08132530748844147, "Regression_pt": 0.09795314073562622, "Regression_eta": 0.001224836412196358, "Regression_sin_phi": 0.0006477312029649814, "Regression_cos_phi": 0.0006190861652915677, "Regression_energy": 0.09761669238408406, "HitGrouping": 0.14882922172546387, "Auxiliary": 0.19022007783253989, "Total": 0.8857476711273193, "Total_unweighted": 0.4189197876257822, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8460060358047485, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6027221083641052, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2804012596607208, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6141476035118103, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6213738322257996, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3359515368938446, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7933564186096191, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5398418307304382, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2865583598613739, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5488426089286804, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5577672719955444, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.41912850737571716, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8609925508499146, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5796069502830505, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2620226740837097, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5882073044776917, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5930759310722351, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.30969682335853577, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8512012958526611, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5835224390029907, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.26949542760849, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5930097699165344, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5989076495170593, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32614678144454956, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.004342419560998678, "metrics/hit_grouping/calorimeter/background_recall": 0.1833634078502655, "metrics/hit_grouping/calorimeter/completeness": 0.6673898696899414, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6665098071098328, "metrics/hit_grouping/calorimeter/energy_purity": 0.6581490635871887, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.3065135180950165, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8711752891540527, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5912818312644958, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.20493854582309723, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6028978228569031, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5975350737571716, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2675238847732544, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8566132187843323, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.613118052482605, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2810916602611542, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6226847767829895, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.630364179611206, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.32738369703292847, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8050225973129272, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5378860235214233, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.38944315910339355, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5429531335830688, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5700160264968872, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4444315433502197, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8411345481872559, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6567001342773438, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3269785940647125, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6706149578094482, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6910455226898193, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.36372384428977966, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8197522163391113, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6708585619926453, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.43386515974998474, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6726637482643127, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6894305348396301, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.385486364364624, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8652116060256958, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.49850860238075256, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.19130541384220123, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5043970346450806, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.494182825088501, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.28076523542404175, "metrics/hit_grouping/calorimeter/purity": 0.6584580540657043, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7622597813606262, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.07645783573389053, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.06390977650880814, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.07603159546852112, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05611509457230568, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.028195489197969437, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7784772515296936, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.23316709697246552, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10535608977079391, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.23165351152420044, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.21985197067260742, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23425544798374176, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8564350605010986, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6100168228149414, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2813868820667267, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6202381253242493, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6274775862693787, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.33578363060951233, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7326040267944336, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3925829231739044, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5291544795036316, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5816341042518616, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5780095458030701, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7735323309898376, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.25557565689086914, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5022055506706238, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5186209082603455, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4472573697566986, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6536784172058105, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4312850832939148, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4420293867588043, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5044288039207458, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6864296793937683, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.68678879737854, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40641140937805176, "metrics/hit_grouping/tracker/all/object_completeness": 0.4732518494129181, "metrics/hit_grouping/tracker/all/object_purity": 0.5292057991027832, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6354407668113708, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.010853451676666737, "metrics/hit_grouping/tracker/background_recall": 0.10960688441991806, "metrics/hit_grouping/tracker/completeness": 0.471230685710907, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4356543719768524, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7219443917274475, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.39032089710235596, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5463994145393372, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5810317397117615, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5938102602958679, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6702689528465271, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4278453588485718, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4507371783256531, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5245271325111389, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6614630818367004, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6308867931365967, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4182146191596985, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.34515658020973206, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.427249014377594, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.695220947265625, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.689357340335846, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4273584187030792, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.48497676849365234, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.552091121673584, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6351363062858582, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6530264019966125, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4181034564971924, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.41454795002937317, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5097193717956543, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6884236335754395, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6895021200180054, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37708356976509094, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.46853289008140564, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5029475688934326, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6263384222984314, "metrics/hit_grouping/tracker/purity": 0.4444495141506195, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8891337513923645, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.03921568766236305, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.12725086510181427, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1280336081981659, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06485671550035477, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6912721395492554, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.29065564274787903, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.4110466539859772, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.422986775636673, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5121321678161621, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6562755107879639, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.47344860434532166, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5313395857810974, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6002514362335205, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.733478844165802, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6174321174621582, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7100175619125366, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8182440400123596, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6689014434814453, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.06827934831380844, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09442663192749023, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.282583236694336, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3080817461013794, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.21645857393741608, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9085135459899902, "metrics/particle/class_1/pid_accuracy": 0.9764468669891357, "metrics/particle/class_2/efficiency": 0.5882886052131653, "metrics/particle/class_2/pid_accuracy": 0.7720446586608887, "metrics/particle/class_3/efficiency": 0.8147257566452026, "metrics/particle/class_3/pid_accuracy": 0.980251133441925, "metrics/particle/class_4/efficiency": 0.7144288420677185, "metrics/particle/class_4/pid_accuracy": 0.538569450378418, "metrics/particle/class_5/efficiency": 0.9291187524795532, "metrics/particle/class_5/pid_accuracy": 0.5855669975280762, "metrics/particle/count/bias_mean": -1.7291666269302368, "metrics/particle/count/mae": 3.4674479961395264, "metrics/particle/count/prediction_mean": 68.92578125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06795389205217361, "metrics/particle/event/energy_response_mean": 0.9727866649627686, "metrics/particle/event/met_abs_error": 4.9838738441467285, "metrics/particle/event/scalar_pt_relative_abs_error": 0.049638163298368454, "metrics/particle/event/scalar_pt_response_mean": 0.9938902854919434, "metrics/particle/event/vector_pt_closure": 0.04826981946825981, "metrics/particle/matched/delta_eta_abs_mean": 0.013289052061736584, "metrics/particle/matched/delta_phi_abs_mean": 0.012108990922570229, "metrics/particle/matched/delta_r_mean": 0.02013031765818596, "metrics/particle/matched/energy_relative_abs_mean": 0.1271689534187317, "metrics/particle/matched/pid_accuracy": 0.9608749747276306, "metrics/particle/matched/pt_relative_abs_mean": 0.12709321081638336, "metrics/particle/matching/duplicate_fraction": 0.04368565231561661, "metrics/particle/matching/efficiency": 0.8412454128265381, "metrics/particle/matching/f1": 0.8516669869422913, "metrics/particle/matching/purity": 0.8623500466346741, "metrics/particle/matching/target_coverage_dr0p05": 0.8047288060188293, "metrics/particle/matching/target_coverage_dr0p10": 0.8754123449325562, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6749995350837708, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8691465854644775, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020357146859169006, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12457539141178131, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12532219290733337, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01370953768491745, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16425350308418274, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0046552871353924274, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5858340263366699, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.021947214379906654, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6863919496536255, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9022.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.19126513600349426, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7458072304725647, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7628746628761292, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6283737421035767, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09598062932491302, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.672713279724121, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.00978208240121603, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8085652589797974, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05881277844309807, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5176963806152344, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10325.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8375417590141296, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.20304659008979797, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.965453624725342, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.054427146911621, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.08104804158210754, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.16949746012687683, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.037827014923096, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.3528811931610107, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.790482521057129, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0238749161362648, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1434450000524521, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.14337526261806488, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.003580857068300247, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18351148068904877, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025233132764697075, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2804126739501953, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05118345096707344, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.280102014541626, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1823.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.44426387548446655, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.9651637673377991, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.3596882820129395, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.184114933013916, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17367303371429443, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 23.35017967224121, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.052653927356004715, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.2942073345184326, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1672096848487854, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.3103203773498535, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2355.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6609862446784973, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.457458972930908, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.089239120483398, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0566210746765137, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.020420951768755913, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.031673479825258255, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.968684196472168, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.09749603271484375, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.78213119506836, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01975514367222786, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12733928859233856, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12718644738197327, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.019725464284420013, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16615703701972961, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003989427932538092, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.20140787959098816, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0280342698097229, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6221454739570618, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20053.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16431866586208344, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8442609906196594, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.625523567199707, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.48956021666526794, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09438753128051758, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.4805779457092285, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002495403168722987, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.33178040385246277, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.04975520074367523, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4816521406173706, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22842.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8617903590202332, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.8393405079841614, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.700855731964111, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.397149562835693, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.07728365063667297, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.3462754487991333, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.5331296920776367, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.162216067314148, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.709131717681885, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.020173992961645126, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12748248875141144, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12759725749492645, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01701631397008896, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1666809767484665, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0031069971155375242, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3773195743560791, "metrics/particle/momentum/all/accepted/pz_bias": 0.027622709050774574, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3101158142089844, "metrics/particle/momentum/all/accepted/samples": 30898.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.19071060419082642, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8236594796180725, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7804165482521057, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6422522664070129, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10010697692632675, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.270659446716309, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.007938742637634277, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6004679799079895, "metrics/particle/momentum/all/assigned/pz_bias": 0.03800380975008011, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.607121467590332, "metrics/particle/momentum/all/assigned/samples": 35522.0, "metrics/particle/momentum/all/efficiency": 0.8396424055099487, "metrics/particle/momentum/all/event/energy_bias": -4.415071964263916, "metrics/particle/momentum/all/event/p3_error_mean": 16.464153289794922, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.515347480773926, "metrics/particle/momentum/all/event/px_bias": 0.016656696796417236, "metrics/particle/momentum/all/event/py_bias": 0.20845142006874084, "metrics/particle/momentum/all/event/pz_abs_mean": 12.562498092651367, "metrics/particle/momentum/all/event/pz_bias": 3.612593412399292, "metrics/particle/momentum/all/event/pz_residual_rms": 23.713367462158203, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.021649498492479324, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10942039638757706, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10972967743873596, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.018438082188367844, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14608773589134216, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002610966097563505, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.388044536113739, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.030769873410463333, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2240862846374512, "metrics/particle/momentum/class_1/accepted/samples": 16086.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10668251663446426, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.32023364305496216, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3576454818248749, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24326105415821075, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.054483868181705475, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.345407485961914, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005052244756370783, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5145301222801208, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.03694899007678032, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1746835708618164, "metrics/particle/momentum/class_1/assigned/samples": 17418.0, "metrics/particle/momentum/class_1/efficiency": 0.9067132472991943, "metrics/particle/momentum/class_1/event/energy_bias": 3.233436346054077, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.789776802062988, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.984936237335205, "metrics/particle/momentum/class_1/event/px_bias": 0.4023157060146332, "metrics/particle/momentum/class_1/event/py_bias": 0.7708359956741333, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.444387435913086, "metrics/particle/momentum/class_1/event/pz_bias": 1.5760960578918457, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.95451831817627, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.026458846405148506, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17161598801612854, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17078611254692078, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.017528092488646507, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21048226952552795, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.020325202494859695, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.32774817943573, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.07691370695829391, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3172225952148438, "metrics/particle/momentum/class_2/accepted/samples": 1722.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5193395018577576, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.48524966835975647, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6079226136207581, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3108956515789032, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.23055332899093628, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.4036364555358887, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.035685647279024124, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.004272699356079, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.022008569911122322, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 4.874281406402588, "metrics/particle/momentum/class_2/assigned/samples": 2494.0, "metrics/particle/momentum/class_2/efficiency": 0.576691210269928, "metrics/particle/momentum/class_2/event/energy_bias": -9.511451721191406, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.91696548461914, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.227392196655273, "metrics/particle/momentum/class_2/event/px_bias": -0.4085323214530945, "metrics/particle/momentum/class_2/event/py_bias": -0.16551479697227478, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.770208358764648, "metrics/particle/momentum/class_2/event/pz_bias": 1.0181171894073486, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.640071868896484, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017456624656915665, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14342164993286133, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14352576434612274, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013227425515651703, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1824609488248825, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001104623661376536, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21590106189250946, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01279862504452467, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7691357135772705, "metrics/particle/momentum/class_3/accepted/samples": 12674.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.22952671349048615, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3770748376846313, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2296817302703857, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.088658332824707, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1286623328924179, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.293550491333008, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.006496089044958353, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4510672986507416, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03523155301809311, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.522249221801758, "metrics/particle/momentum/class_3/assigned/samples": 15086.0, "metrics/particle/momentum/class_3/efficiency": 0.815940260887146, "metrics/particle/momentum/class_3/event/energy_bias": 2.4548873901367188, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.378898620605469, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.533908843994141, "metrics/particle/momentum/class_3/event/px_bias": 0.14973662793636322, "metrics/particle/momentum/class_3/event/py_bias": -0.2816879153251648, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.935131072998047, "metrics/particle/momentum/class_3/event/pz_bias": 0.6768451929092407, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.159590721130371, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.021936718374490738, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1650351881980896, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1606954038143158, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08915869891643524, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2092643678188324, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0037878789007663727, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5806543827056885, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.12543386220932007, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.549701452255249, "metrics/particle/momentum/class_4/accepted/samples": 264.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.41579049825668335, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.614157199859619, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.8669676780700684, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.779461145401001, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22714681923389435, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.209257125854492, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005540166050195694, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7684267163276672, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.2675814628601074, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.951216459274292, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.7039999961853027, "metrics/particle/momentum/class_4/event/energy_bias": -0.511055588722229, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8652364015579224, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.337460994720459, "metrics/particle/momentum/class_4/event/px_bias": -0.17256835103034973, "metrics/particle/momentum/class_4/event/py_bias": -0.05114638805389404, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0433573722839355, "metrics/particle/momentum/class_4/event/pz_bias": 0.13326773047447205, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.964080333709717, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.016339164227247238, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1447315514087677, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.14359056949615479, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.05137773975729942, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1872696727514267, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02631578966975212, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.5811169147491455, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.20231974124908447, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.488673448562622, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05062474310398102, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.18237145245075226, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18021900951862335, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08407938480377197, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05521472543478012, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.26806455850601196, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.030674846842885017, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.7599848508834839, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.14358526468276978, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.892972469329834, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9268292784690857, "metrics/particle/momentum/class_5/event/energy_bias": -0.08088824152946472, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7760238647460938, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3518874645233154, "metrics/particle/momentum/class_5/event/px_bias": 0.04570496827363968, "metrics/particle/momentum/class_5/event/py_bias": -0.06403548270463943, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9025139808654785, "metrics/particle/momentum/class_5/event/pz_bias": 0.20826704800128937, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.0658626556396484, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.31123822927474976, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.312715083360672, "metrics/set_slot/angular_recall_all_dr0p10": 0.9620736241340637, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8610563278198242, "metrics/set_slot/soft_count_bias_mean": -0.539405882358551}, "step_train_time": 5.534571409225464, "step_valid_time": 40060.09875249863, "step_total_time": 40065.63332414627, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0086097072710405, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10535845778544661, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3052, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2935, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2920, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9594638770840144, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9616644823066841, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.960562919325806, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038335517693315856, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9545603138280484, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9567496723460026, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9556537391588938, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9594638770840144, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0008050716878445, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0997501567366954, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2268, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2166, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2162, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9479212253829322, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.955026455026455, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9514605754447617, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04497354497354497, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9461706783369803, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.95326278659612, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9497034922029431, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9479212253829322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0107443033922532, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11285840424367577, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1312, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1241, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1237, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9387291981845688, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9458841463414634, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9422930903568717, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054115853658536585, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9357034795763994, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9428353658536586, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9392558845861807, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9387291981845688} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_44000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_44000.json new file mode 100644 index 0000000000000000000000000000000000000000..f7f123d2459d3df8b41f823573ac324881ffe52c --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_44000.json @@ -0,0 +1 @@ +{"epoch": 2.0841227737779464, "train": {"Classification_binary": 0.13176068663597107, "Cardinality": 0.10270900279283524, "Classification": 0.08403386175632477, "Regression_pt": 0.08776634186506271, "Regression_eta": 0.0011342620709910989, "Regression_sin_phi": 0.0006320016109384596, "Regression_cos_phi": 0.0005186706548556685, "Regression_energy": 0.08679870516061783, "HitGrouping": 0.139747753739357, "Auxiliary": 0.17631743848323822, "Total": 0.8262921571731567, "Total_unweighted": 0.3926445297547616}, "train_group": {}, "valid": {"Classification_binary": 0.13477168480555216, "Cardinality": 0.10520043969154358, "Classification": 0.08007091283798218, "Regression_pt": 0.0931036372979482, "Regression_eta": 0.001178247466062506, "Regression_sin_phi": 0.0006550056083748738, "Regression_cos_phi": 0.0006037169368937612, "Regression_energy": 0.09255996346473694, "HitGrouping": 0.14749629298845926, "Auxiliary": 0.18339176972707114, "Total": 0.8606537183125814, "Total_unweighted": 0.4029431684175506, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8511551022529602, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6122657060623169, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.27708905935287476, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6236793994903564, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6270961165428162, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3266773819923401, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7983257174491882, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5483046174049377, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2791728079319, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5577284097671509, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5648799538612366, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4076809585094452, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8664566874504089, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5809457302093506, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2558204233646393, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5896288156509399, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5918999314308167, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3044665455818176, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8565315008163452, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5879000425338745, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.264076828956604, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5974678993225098, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6004649996757507, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.31920871138572693, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0033700501080602407, "metrics/hit_grouping/calorimeter/background_recall": 0.15344175696372986, "metrics/hit_grouping/calorimeter/completeness": 0.6692659258842468, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6705836057662964, "metrics/hit_grouping/calorimeter/energy_purity": 0.6618949174880981, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.2995226979255676, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8789113759994507, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5996246933937073, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.20260582864284515, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6110092997550964, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6026341319084167, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.26046881079673767, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.858988344669342, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6160464286804199, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2748733162879944, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6259141564369202, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6325911283493042, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.32427453994750977, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8084244132041931, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5356382727622986, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3785382807254791, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5411996245384216, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5636775493621826, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4338746964931488, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8428941369056702, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6596101522445679, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.32000240683555603, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6730232834815979, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6931464672088623, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3584916889667511, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8265296816825867, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6765962839126587, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4287184774875641, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6785489320755005, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.688049852848053, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.37828099727630615, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8740099668502808, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.504220187664032, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.18744251132011414, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5107749700546265, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.49553608894348145, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2721197009086609, "metrics/hit_grouping/calorimeter/purity": 0.6596647500991821, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7857858538627625, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.1048927828669548, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.09774436056613922, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.10234066843986511, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.0665300115942955, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.028195489197969437, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8198977708816528, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.23003335297107697, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.1006474420428276, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.22743497788906097, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.19985899329185486, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.22542671859264374, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8595905900001526, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6143931150436401, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2752917408943176, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6247916221618652, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6300120949745178, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.32883089780807495, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7361250519752502, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.3956250846385956, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5333307981491089, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5743579268455505, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5743879675865173, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7795435786247253, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2628089189529419, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5002629160881042, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5190203189849854, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.44243520498275757, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6540127396583557, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43186068534851074, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4411526322364807, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5004211664199829, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6826161742210388, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6885219812393188, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40823686122894287, "metrics/hit_grouping/tracker/all/object_completeness": 0.4738493859767914, "metrics/hit_grouping/tracker/all/object_purity": 0.5245192050933838, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6316117644309998, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.008770118467509747, "metrics/hit_grouping/tracker/background_recall": 0.09354278445243835, "metrics/hit_grouping/tracker/completeness": 0.4705922305583954, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4433072507381439, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7238938808441162, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.38997018337249756, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5480026602745056, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5807893872261047, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5883745551109314, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.670931339263916, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.43038663268089294, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4473324418067932, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5137087106704712, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6634597778320312, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.633238673210144, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.42380523681640625, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.34766194224357605, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.41951239109039307, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6889089345932007, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6867631077766418, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.42294302582740784, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4804476499557495, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5463337898254395, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6383628845214844, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6659706234931946, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4575123190879822, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4300520122051239, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4865463972091675, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6551724076271057, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.694851279258728, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.38028478622436523, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4731230139732361, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5029667615890503, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6186113357543945, "metrics/hit_grouping/tracker/purity": 0.4429144859313965, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.887216329574585, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04185520485043526, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.12963473796844482, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1319665163755417, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06372549384832382, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6925380825996399, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.3004646301269531, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.40887323021888733, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.425550639629364, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5069695115089417, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6586010456085205, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4742879569530487, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5319725275039673, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5934997200965881, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7293940186500549, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.608055055141449, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7205365896224976, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8315901160240173, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.664772629737854, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.05890225991606712, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09029785543680191, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.181194305419922, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.2996150255203247, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20799186825752258, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9101346731185913, "metrics/particle/class_1/pid_accuracy": 0.9788566827774048, "metrics/particle/class_2/efficiency": 0.6005210280418396, "metrics/particle/class_2/pid_accuracy": 0.7834779620170593, "metrics/particle/class_3/efficiency": 0.8316682577133179, "metrics/particle/class_3/pid_accuracy": 0.98110032081604, "metrics/particle/class_4/efficiency": 0.7174348831176758, "metrics/particle/class_4/pid_accuracy": 0.49162012338638306, "metrics/particle/class_5/efficiency": 0.9521072506904602, "metrics/particle/class_5/pid_accuracy": 0.6076458692550659, "metrics/particle/count/bias_mean": -1.0625, "metrics/particle/count/mae": 3.1770832538604736, "metrics/particle/count/prediction_mean": 69.59244537353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06699497997760773, "metrics/particle/event/energy_response_mean": 0.9658432006835938, "metrics/particle/event/met_abs_error": 4.828368186950684, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04887118935585022, "metrics/particle/event/scalar_pt_response_mean": 0.9808730483055115, "metrics/particle/event/vector_pt_closure": 0.04797830805182457, "metrics/particle/matched/delta_eta_abs_mean": 0.011939393356442451, "metrics/particle/matched/delta_phi_abs_mean": 0.012120213359594345, "metrics/particle/matched/delta_r_mean": 0.019132336601614952, "metrics/particle/matched/energy_relative_abs_mean": 0.12513698637485504, "metrics/particle/matched/pid_accuracy": 0.9627763032913208, "metrics/particle/matched/pt_relative_abs_mean": 0.12500107288360596, "metrics/particle/matching/duplicate_fraction": 0.03873931244015694, "metrics/particle/matching/efficiency": 0.8503031730651855, "metrics/particle/matching/f1": 0.8567449450492859, "metrics/particle/matching/purity": 0.8632851243019104, "metrics/particle/matching/target_coverage_dr0p05": 0.814625084400177, "metrics/particle/matching/target_coverage_dr0p10": 0.883714497089386, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6880931854248047, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8784070014953613, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019588978961110115, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12431850284337997, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12480556219816208, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014106404967606068, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16441437602043152, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003597514471039176, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5625821948051453, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.023263275623321533, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6003565788269043, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9173.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.16813752055168152, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.5273840427398682, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.6131576299667358, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.4733593463897705, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09087412059307098, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.139856338500977, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.009231656789779663, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7796387076377869, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07729598879814148, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.418430805206299, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10399.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.851559579372406, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.9864130020141602, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.596851348876953, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.935877799987793, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.005328778177499771, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.30075541138648987, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.654716491699219, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.7369340658187866, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.847834587097168, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02306758239865303, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13710057735443115, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13803023099899292, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.005801189225167036, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17817263305187225, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021356113255023956, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.191656231880188, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.030949581414461136, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0262720584869385, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1873.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4613032042980194, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.792594850063324, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.8341972827911377, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.6435229778289795, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17418032884597778, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 19.87288475036621, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.055327869951725006, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4151389598846436, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.269310861825943, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.231517791748047, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2440.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6791152954101562, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.265841007232666, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.575434684753418, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1200605630874634, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.05662991851568222, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.04096418619155884, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.445455551147461, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6622816324234009, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.695972442626953, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018735410645604134, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12523752450942993, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12455278635025024, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013387354090809822, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16387079656124115, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0002967652690131217, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19400350749492645, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01731344498693943, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6112040281295776, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20218.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15334147214889526, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7027300000190735, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5342889428138733, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3885694742202759, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08835886418819427, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.799739360809326, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002100656507536769, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.301716685295105, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.029419835656881332, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1784894466400146, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22850.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.868881344795227, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.8102297782897949, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.1453633308410645, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.011592388153076, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.5597552061080933, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.8940144181251526, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.194352626800537, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.6201344728469849, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.237401485443115, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019245386123657227, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12567859888076782, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12543436884880066, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01314384676516056, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16492138803005219, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002526867901906371, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.36191466450691223, "metrics/particle/momentum/all/accepted/pz_bias": 0.01616775617003441, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2416523694992065, "metrics/particle/momentum/all/accepted/samples": 31264.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.178707554936409, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6577818989753723, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.646142303943634, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.4990745186805725, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09495922923088074, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 7.523591041564941, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.007817534729838371, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5854641199111938, "metrics/particle/momentum/all/assigned/pz_bias": 0.02294616587460041, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.688107490539551, "metrics/particle/momentum/all/assigned/samples": 35689.0, "metrics/particle/momentum/all/efficiency": 0.8495882749557495, "metrics/particle/momentum/all/event/energy_bias": -7.062483787536621, "metrics/particle/momentum/all/event/p3_error_mean": 16.24839210510254, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.3709540367126465, "metrics/particle/momentum/all/event/px_bias": 0.6110563278198242, "metrics/particle/momentum/all/event/py_bias": -1.2357341051101685, "metrics/particle/momentum/all/event/pz_abs_mean": 12.432555198669434, "metrics/particle/momentum/all/event/pz_bias": 1.694786787033081, "metrics/particle/momentum/all/event/pz_residual_rms": 24.191068649291992, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.020348066464066505, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10618112981319427, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10656363517045975, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.015816202387213707, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1434229165315628, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001797892153263092, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.37331974506378174, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.016070997342467308, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1559758186340332, "metrics/particle/momentum/class_1/accepted/samples": 16130.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10821301490068436, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.23088690638542175, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2593992352485657, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1411658078432083, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05063654109835625, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.4876456260681152, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0045876819640398026, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5056312680244446, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0003154777514282614, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4484262466430664, "metrics/particle/momentum/class_1/assigned/samples": 17438.0, "metrics/particle/momentum/class_1/efficiency": 0.9091933965682983, "metrics/particle/momentum/class_1/event/energy_bias": 2.787024736404419, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.152828216552734, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.788602828979492, "metrics/particle/momentum/class_1/event/px_bias": 0.4433559775352478, "metrics/particle/momentum/class_1/event/py_bias": 0.08304408192634583, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.979511260986328, "metrics/particle/momentum/class_1/event/pz_bias": 0.7553862929344177, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.565126419067383, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02539120800793171, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17025423049926758, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17101840674877167, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.014687594026327133, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21258588135242462, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.019187359139323235, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.244205117225647, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.00367944547906518, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.1268324851989746, "metrics/particle/momentum/class_2/accepted/samples": 1772.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5104825496673584, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5101354122161865, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7571550011634827, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.43904027342796326, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.230979323387146, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.426097869873047, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.046039797365665436, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.144484281539917, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.08656301349401474, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.591798305511475, "metrics/particle/momentum/class_2/assigned/samples": 2563.0, "metrics/particle/momentum/class_2/efficiency": 0.593436062335968, "metrics/particle/momentum/class_2/event/energy_bias": -8.620394706726074, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.930683135986328, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.224130630493164, "metrics/particle/momentum/class_2/event/px_bias": -0.0024740397930145264, "metrics/particle/momentum/class_2/event/py_bias": -0.9563729763031006, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.755030632019043, "metrics/particle/momentum/class_2/event/pz_bias": 0.5606641173362732, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.476228713989258, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017034616321325302, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1431427001953125, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14222699403762817, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.008263781666755676, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.181073859333992, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009274287149310112, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21246790885925293, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01623285934329033, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7736169099807739, "metrics/particle/momentum/class_3/accepted/samples": 12939.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.20075055956840515, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.0960112810134888, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.0316531658172607, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.8787875771522522, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12006860971450806, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.866962432861328, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.004881910514086485, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.40271803736686707, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.035244714468717575, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1527106761932373, "metrics/particle/momentum/class_3/assigned/samples": 15158.0, "metrics/particle/momentum/class_3/efficiency": 0.8330007195472717, "metrics/particle/momentum/class_3/event/energy_bias": 0.0071784257888793945, "metrics/particle/momentum/class_3/event/p3_error_mean": 7.880523681640625, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.171906471252441, "metrics/particle/momentum/class_3/event/px_bias": 0.02162085473537445, "metrics/particle/momentum/class_3/event/py_bias": -0.17503494024276733, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.712733745574951, "metrics/particle/momentum/class_3/event/pz_bias": 0.2783437967300415, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.57248306274414, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022473646327853203, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1570310741662979, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.14494676887989044, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07385803759098053, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.18997979164123535, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.48784613609313965, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.03672071918845177, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9985108375549316, "metrics/particle/momentum/class_4/accepted/samples": 270.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.34410765767097473, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.015251636505127, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.373350143432617, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.2828457355499268, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23433242738246918, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.824719429016113, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.00544959120452404, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6429075002670288, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.15746091306209564, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7403277158737183, "metrics/particle/momentum/class_4/assigned/samples": 367.0, "metrics/particle/momentum/class_4/efficiency": 0.7200000286102295, "metrics/particle/momentum/class_4/event/energy_bias": -0.9758837819099426, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6779237985610962, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2415075302124023, "metrics/particle/momentum/class_4/event/px_bias": -0.03216385841369629, "metrics/particle/momentum/class_4/event/py_bias": -0.07323162257671356, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8985569477081299, "metrics/particle/momentum/class_4/event/pz_bias": -0.015109866857528687, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.0617918968200684, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013081307522952557, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13269011676311493, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13237231969833374, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.019089577719569206, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17127537727355957, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.026143791154026985, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.3573966026306152, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.21445702016353607, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.9233930110931396, "metrics/particle/momentum/class_5/accepted/samples": 153.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08125491440296173, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.33716097474098206, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.5359441041946411, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4054948687553406, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.04907975345849991, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.800638198852539, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.030674846842885017, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.4771236181259155, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.06465327739715576, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.3708841800689697, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9329268336296082, "metrics/particle/momentum/class_5/event/energy_bias": -0.2604081332683563, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5559885501861572, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.1831713914871216, "metrics/particle/momentum/class_5/event/px_bias": 0.1807173788547516, "metrics/particle/momentum/class_5/event/py_bias": -0.11413869261741638, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7782358527183533, "metrics/particle/momentum/class_5/event/pz_bias": 0.1155027374625206, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.7898786067962646, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.29562264680862427, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2971644699573517, "metrics/set_slot/angular_recall_all_dr0p10": 0.9629489779472351, "metrics/set_slot/angular_recall_selected_dr0p10": 0.868160605430603, "metrics/set_slot/soft_count_bias_mean": 0.013445901684463024}, "step_train_time": 5.245321989059448, "step_valid_time": 43115.36123609543, "step_total_time": 43120.60655832291, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9913338041401036, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10374423493727625, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3049, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2938, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2928, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9604445897352076, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9635946211872746, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9620170268500328, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.036405378812725486, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9571755475645636, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9603148573302722, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9587426326129665, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9604445897352076, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9869851909170761, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09923258838133697, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2266, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2163, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2158, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9466083150984683, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9545454545454546, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.950560316413975, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045454545454545456, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9444201312910284, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9523389232127096, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9483629971434849, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9466083150984683, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9945799540637541, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11214085832459308, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1313, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1249, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1247, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9447806354009077, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9512566641279513, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9480075901328273, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048743335872048744, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.943267776096823, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9497334348819497, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9464895635673625, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9447806354009077} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_45000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_45000.json new file mode 100644 index 0000000000000000000000000000000000000000..87f59a1a696c90754a9ce2bfe64d8b09efc70ac9 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_45000.json @@ -0,0 +1 @@ +{"epoch": 2.131489200454718, "train": {"Classification_binary": 0.12372233718633652, "Cardinality": 0.10362009704113007, "Classification": 0.08011221885681152, "Regression_pt": 0.09497380256652832, "Regression_eta": 0.0013238778337836266, "Regression_sin_phi": 0.0007512806914746761, "Regression_cos_phi": 0.000669300789013505, "Regression_energy": 0.09411635249853134, "HitGrouping": 0.1521444469690323, "Auxiliary": 0.1790265142917633, "Total": 0.8677870631217957, "Total_unweighted": 0.3956691704224795}, "train_group": {}, "valid": {"Classification_binary": 0.1351796785990397, "Cardinality": 0.10448434948921204, "Classification": 0.08062583704789479, "Regression_pt": 0.09237598379453023, "Regression_eta": 0.001135944544027249, "Regression_sin_phi": 0.000623005093075335, "Regression_cos_phi": 0.000593818297299246, "Regression_energy": 0.09160655736923218, "HitGrouping": 0.1470325986544291, "Auxiliary": 0.18387818336486816, "Total": 0.8551230430603027, "Total_unweighted": 0.4021408247450987, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8491730093955994, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6082308888435364, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2766158878803253, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6202110648155212, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6259048581123352, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3343427777290344, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7956581115722656, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.545424222946167, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.27732643485069275, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5562615394592285, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5696489810943604, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4117429852485657, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8612990379333496, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5839739441871643, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.25595927238464355, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5928769707679749, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5999206900596619, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3112242519855499, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8525292277336121, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5883297324180603, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.26387614011764526, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5983151793479919, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6054424047470093, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3260321021080017, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003960941918194294, "metrics/hit_grouping/calorimeter/background_recall": 0.1705942004919052, "metrics/hit_grouping/calorimeter/completeness": 0.6701307892799377, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6692169308662415, "metrics/hit_grouping/calorimeter/energy_purity": 0.6607093811035156, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.2998269200325012, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8723673224449158, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5965787768363953, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.19862312078475952, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6085345149040222, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6047192215919495, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2700842022895813, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8575789928436279, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6152812838554382, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2755642533302307, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6261380910873413, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6345898509025574, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.32485029101371765, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8069923520088196, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5443586707115173, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.38515082001686096, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5494487285614014, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5775531530380249, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.44129928946495056, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8427698612213135, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.66046541929245, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3186793327331543, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6747868657112122, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6967235803604126, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3602357506752014, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8223205804824829, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.673888623714447, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4302624762058258, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6739622950553894, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6890922784805298, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.38908904790878296, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8660783171653748, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5045899748802185, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.18817830085754395, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5113353133201599, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.5024099946022034, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2836470603942871, "metrics/hit_grouping/calorimeter/purity": 0.6609090566635132, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7643148303031921, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09282974153757095, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0958646610379219, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09159953147172928, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05399053543806076, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.020676691085100174, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7892038226127625, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.23800647258758545, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10359034687280655, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.2362087368965149, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.21988125145435333, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24426132440567017, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8572618961334229, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6146339178085327, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.27495482563972473, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6254152655601501, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.634492039680481, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3352629542350769, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.734699547290802, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.38591915369033813, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5316628217697144, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5793954133987427, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5743879675865173, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7808324098587036, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.26220613718032837, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5094574093818665, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5254677534103394, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4327908456325531, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6532777547836304, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4227946400642395, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.44451552629470825, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5120431780815125, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6854943037033081, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6877241730690002, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.3995992839336395, "metrics/hit_grouping/tracker/all/object_completeness": 0.47609683871269226, "metrics/hit_grouping/tracker/all/object_purity": 0.5337352156639099, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6326802968978882, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009265141561627388, "metrics/hit_grouping/tracker/background_recall": 0.09587148576974869, "metrics/hit_grouping/tracker/completeness": 0.4719003140926361, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4323732256889343, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7210986614227295, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3771699070930481, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5475373268127441, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5925517678260803, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5910047292709351, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6696826219558716, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.423307329416275, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45288223028182983, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5266903638839722, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6665456295013428, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6369975805282593, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4221821427345276, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.35220858454704285, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.41974955797195435, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6847610473632812, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6879140138626099, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4169992506504059, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4837782680988312, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5521565675735474, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6359004974365234, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6625097393989563, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4464285671710968, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.42621204257011414, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4891715347766876, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6705664992332458, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6919976472854614, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3685837388038635, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.47505366802215576, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5177759528160095, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6217021942138672, "metrics/hit_grouping/tracker/purity": 0.44436755776405334, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8834260106086731, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.044871795922517776, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.1370914727449417, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.14001049101352692, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06447964161634445, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6906465291976929, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2978833317756653, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.41634246706962585, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.43144601583480835, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5033556818962097, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6583659052848816, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.46326449513435364, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5328808426856995, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.603249728679657, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7310167551040649, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6065827012062073, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.709733247756958, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8172740936279297, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6709602475166321, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.05742993950843811, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09648549556732178, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.32842254638672, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29985353350639343, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.2082303762435913, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9093717932701111, "metrics/particle/class_1/pid_accuracy": 0.9764271378517151, "metrics/particle/class_2/efficiency": 0.5964435338973999, "metrics/particle/class_2/pid_accuracy": 0.787884533405304, "metrics/particle/class_3/efficiency": 0.8324770927429199, "metrics/particle/class_3/pid_accuracy": 0.9816176295280457, "metrics/particle/class_4/efficiency": 0.7224448919296265, "metrics/particle/class_4/pid_accuracy": 0.5062413215637207, "metrics/particle/class_5/efficiency": 0.9425287246704102, "metrics/particle/class_5/pid_accuracy": 0.6361788511276245, "metrics/particle/count/bias_mean": -0.9114583134651184, "metrics/particle/count/mae": 3.2018229961395264, "metrics/particle/count/prediction_mean": 69.74349212646484, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.0675705075263977, "metrics/particle/event/energy_response_mean": 0.9929592609405518, "metrics/particle/event/met_abs_error": 4.908129692077637, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0506792776286602, "metrics/particle/event/scalar_pt_response_mean": 1.0090144872665405, "metrics/particle/event/vector_pt_closure": 0.04780559614300728, "metrics/particle/matched/delta_eta_abs_mean": 0.012020591646432877, "metrics/particle/matched/delta_phi_abs_mean": 0.012006030417978764, "metrics/particle/matched/delta_r_mean": 0.019103102385997772, "metrics/particle/matched/energy_relative_abs_mean": 0.12585291266441345, "metrics/particle/matched/pid_accuracy": 0.96231609582901, "metrics/particle/matched/pt_relative_abs_mean": 0.1262267529964447, "metrics/particle/matching/duplicate_fraction": 0.0430147685110569, "metrics/particle/matching/efficiency": 0.8499438166618347, "metrics/particle/matching/f1": 0.855461597442627, "metrics/particle/matching/purity": 0.8610514998435974, "metrics/particle/matching/target_coverage_dr0p05": 0.8166706562042236, "metrics/particle/matching/target_coverage_dr0p10": 0.8848847150802612, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6840664744377136, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8775684833526611, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019628968089818954, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12590062618255615, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12721869349479675, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.030769702047109604, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16696834564208984, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004710779991000891, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.576153576374054, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.055756643414497375, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6389175653457642, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9128.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.18338869512081146, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7547270059585571, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8620899319648743, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7405527830123901, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09824930876493454, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.586709022521973, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011575624346733093, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8108669519424438, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.026171425357460976, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4967403411865234, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10453.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.847382128238678, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.761234164237976, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.287971496582031, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8766682147979736, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.12828503549098969, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.048585448414087296, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.3347554206848145, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.2778191864490509, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.23517894744873, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02270367555320263, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1383388489484787, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13960029184818268, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03662047162652016, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18006354570388794, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019469983875751495, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1446905136108398, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06916528195142746, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8391082286834717, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1849.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4546578526496887, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.201674461364746, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.3669774532318115, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.2059121131896973, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.18204283714294434, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 41.19993591308594, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05642504245042801, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.340456962585449, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1831917017698288, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.324524879455566, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2428.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6704133152961731, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.4969332218170166, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.9059906005859375, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0460379123687744, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.024242395535111427, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06292388588190079, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.7793779373168945, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.4163281917572021, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.321706771850586, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018623365089297295, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1259552389383316, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.1257692128419876, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02044900879263878, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16595052182674408, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004441812343429774, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19653235375881195, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.002649083500728011, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.621290385723114, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20262.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1538790613412857, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8119137287139893, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5998116135597229, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4656016230583191, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08747705817222595, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.914707660675049, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002097352175042033, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.30654633045196533, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.006387612782418728, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2184693813323975, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22886.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8707722425460815, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.7859745025634766, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.205747127532959, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.074448108673096, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.05140677094459534, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.4845818281173706, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.24626088142395, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.2370491623878479, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.501483917236328, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019158709794282913, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12667225301265717, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12701140344142914, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.024421874433755875, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16711561381816864, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028169916477054358, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.36357760429382324, "metrics/particle/momentum/all/accepted/pz_bias": -0.018667610362172127, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2297642230987549, "metrics/particle/momentum/all/accepted/samples": 31239.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.18292135000228882, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8216591477394104, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8643089532852173, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7319795489311218, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09704475849866867, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.134933471679688, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.00855537224560976, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5920049548149109, "metrics/particle/momentum/all/assigned/pz_bias": -0.01599721796810627, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.532524347305298, "metrics/particle/momentum/all/assigned/samples": 35767.0, "metrics/particle/momentum/all/efficiency": 0.8489089608192444, "metrics/particle/momentum/all/event/energy_bias": 1.0502758026123047, "metrics/particle/momentum/all/event/p3_error_mean": 15.95046615600586, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.3664045333862305, "metrics/particle/momentum/all/event/px_bias": -0.10112066566944122, "metrics/particle/momentum/all/event/py_bias": 0.5960911512374878, "metrics/particle/momentum/all/event/pz_abs_mean": 12.087936401367188, "metrics/particle/momentum/all/event/pz_bias": -1.3755582571029663, "metrics/particle/momentum/all/event/pz_residual_rms": 23.808420181274414, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02013198472559452, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10615526139736176, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10773448646068573, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.028120307251811028, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14516843855381012, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002357320161536336, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3777324855327606, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.02638600766658783, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1520094871520996, "metrics/particle/momentum/class_1/accepted/samples": 16120.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10806183516979218, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.31265756487846375, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.32064661383628845, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.21614815294742584, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05323106795549393, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.3194003105163574, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005494819954037666, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5100538730621338, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02827627956867218, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.179795742034912, "metrics/particle/momentum/class_1/assigned/samples": 17471.0, "metrics/particle/momentum/class_1/efficiency": 0.9086297154426575, "metrics/particle/momentum/class_1/event/energy_bias": 5.514686584472656, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.299223899841309, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.710465431213379, "metrics/particle/momentum/class_1/event/px_bias": 0.4072508215904236, "metrics/particle/momentum/class_1/event/py_bias": 0.8959766626358032, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.102494239807129, "metrics/particle/momentum/class_1/event/pz_bias": -0.39561253786087036, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.990792274475098, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02497546188533306, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17226988077163696, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17249388992786407, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.030747275799512863, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21379345655441284, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02054794505238533, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.256844401359558, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.10535638779401779, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.0559213161468506, "metrics/particle/momentum/class_2/accepted/samples": 1752.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.5288322567939758, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5322405099868774, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8400734663009644, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5531193614006042, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.23833592236042023, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.353204727172852, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04432348534464836, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.093608856201172, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.010651826858520508, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.179468154907227, "metrics/particle/momentum/class_2/assigned/samples": 2572.0, "metrics/particle/momentum/class_2/efficiency": 0.586738109588623, "metrics/particle/momentum/class_2/event/energy_bias": -6.300804138183594, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.658924102783203, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.397286415100098, "metrics/particle/momentum/class_2/event/px_bias": -0.03781871497631073, "metrics/particle/momentum/class_2/event/py_bias": -0.13387590646743774, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.321187019348145, "metrics/particle/momentum/class_2/event/pz_bias": -0.3522148132324219, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.319730758666992, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017161879688501358, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14513741433620453, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14411942660808563, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.017348386347293854, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18367889523506165, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007723200251348317, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21046200394630432, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.005344906821846962, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7860678434371948, "metrics/particle/momentum/class_3/accepted/samples": 12948.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2064380943775177, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3864126205444336, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4552301168441772, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3154079914093018, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12068965286016464, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.17850685119629, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005790997762233019, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4190082550048828, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0028167327400296926, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.20805025100708, "metrics/particle/momentum/class_3/assigned/samples": 15196.0, "metrics/particle/momentum/class_3/efficiency": 0.8335801362991333, "metrics/particle/momentum/class_3/event/energy_bias": 2.25457501411438, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.453848838806152, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.494779586791992, "metrics/particle/momentum/class_3/event/px_bias": -0.5909501314163208, "metrics/particle/momentum/class_3/event/py_bias": 0.0298159122467041, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.174537181854248, "metrics/particle/momentum/class_3/event/pz_bias": -0.3933413624763489, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.233217239379883, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.021847277879714966, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16569310426712036, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15959782898426056, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08035745471715927, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20406268537044525, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5594163537025452, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.06269649416208267, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1831289529800415, "metrics/particle/momentum/class_4/accepted/samples": 267.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4020175337791443, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.875847816467285, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.6295883655548096, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.54373836517334, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.232876718044281, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 13.07105827331543, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005479452200233936, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7051296234130859, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.011803501285612583, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7396624088287354, "metrics/particle/momentum/class_4/assigned/samples": 365.0, "metrics/particle/momentum/class_4/efficiency": 0.7120000123977661, "metrics/particle/momentum/class_4/event/energy_bias": -0.6382362246513367, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6312700510025024, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2460954189300537, "metrics/particle/momentum/class_4/event/px_bias": -0.07287516444921494, "metrics/particle/momentum/class_4/event/py_bias": -0.019354723393917084, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8291959762573242, "metrics/particle/momentum/class_4/event/pz_bias": -0.19868862628936768, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.857316493988037, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01427072286605835, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13550017774105072, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1325589269399643, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.06357929110527039, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.1755317598581314, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02631578966975212, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.265364646911621, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.16905613243579865, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.707719087600708, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.06547916680574417, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.45589059591293335, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.4758889675140381, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.3948448896408081, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05521472543478012, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.2276949882507324, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03680981695652008, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.5564794540405273, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.41140639781951904, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.668095827102661, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9268292784690857, "metrics/particle/momentum/class_5/event/energy_bias": 0.22005483508110046, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6903905868530273, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2667895555496216, "metrics/particle/momentum/class_5/event/px_bias": 0.19327248632907867, "metrics/particle/momentum/class_5/event/py_bias": -0.17647093534469604, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8794195652008057, "metrics/particle/momentum/class_5/event/pz_bias": -0.03570099547505379, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.0754282474517822, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.2912307381629944, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2931271493434906, "metrics/set_slot/angular_recall_all_dr0p10": 0.9652986526489258, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8710170984268188, "metrics/set_slot/soft_count_bias_mean": 0.011582176201045513}, "step_train_time": 5.35199761390686, "step_valid_time": 46133.39199447632, "step_total_time": 46138.743992328644, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0162120362635303, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10766121833599518, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3047, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2932, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2921, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9584831644328212, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9622579586478504, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9603668522764494, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.037742041352149654, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9548872180451128, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9586478503446012, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9567638388470358, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9584831644328212, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.011419473897149, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10405539222517823, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2279, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2171, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2164, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.950109409190372, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952610794207986, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9513584574934268, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04738920579201404, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9470459518599562, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9495392716103555, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9482909728308502, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.950109409190372, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0171784688496994, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11371707111147211, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1319, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1247, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1242, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.943267776096823, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9454131918119788, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9443392654297614, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05458680818802123, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9394856278366112, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9416224412433661, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9405528209011738, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.943267776096823} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_46000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_46000.json new file mode 100644 index 0000000000000000000000000000000000000000..75bc77967b7846422c48b45e516b6dfca3d50ee1 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_46000.json @@ -0,0 +1 @@ +{"epoch": 2.1788556271314894, "train": {"Classification_binary": 0.1025916188955307, "Cardinality": 0.08696316182613373, "Classification": 0.068959079682827, "Regression_pt": 0.09148244559764862, "Regression_eta": 0.0011050839675590396, "Regression_sin_phi": 0.0005562707665376365, "Regression_cos_phi": 0.0005402031820267439, "Regression_energy": 0.09032014012336731, "HitGrouping": 0.14470285177230835, "Auxiliary": 0.15345004200935364, "Total": 0.7530547380447388, "Total_unweighted": 0.35555484221549705}, "train_group": {}, "valid": {"Classification_binary": 0.13425800204277039, "Cardinality": 0.10561635096867879, "Classification": 0.0784548819065094, "Regression_pt": 0.09307714303334554, "Regression_eta": 0.0011332947760820389, "Regression_sin_phi": 0.0006257482649137577, "Regression_cos_phi": 0.0005954314644138018, "Regression_energy": 0.0923032561937968, "HitGrouping": 0.14751645922660828, "Auxiliary": 0.1837180256843567, "Total": 0.8553949991861979, "Total_unweighted": 0.4004477576818317, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8500690460205078, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6107845902442932, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2761427164077759, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.622301459312439, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6283876299858093, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3346266746520996, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7991503477096558, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5495221614837646, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2872968912124634, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5599664449691772, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5637785792350769, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.40620383620262146, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8648228645324707, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5759260058403015, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.25420042872428894, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5858293771743774, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5864601731300354, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.30483683943748474, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8552545309066772, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5844365954399109, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.26341742277145386, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5948707461357117, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5974013209342957, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.32173165678977966, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0036486880853772163, "metrics/hit_grouping/calorimeter/background_recall": 0.16003777086734772, "metrics/hit_grouping/calorimeter/completeness": 0.670438289642334, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6697980761528015, "metrics/hit_grouping/calorimeter/energy_purity": 0.6612104773521423, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30473077297210693, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8766757249832153, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5943148732185364, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.19896449148654938, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6064717769622803, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5982162356376648, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2618912160396576, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8585605025291443, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6098312139511108, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2715338468551636, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6220991015434265, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6290397047996521, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3232381343841553, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8082465529441833, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5387116074562073, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3866589367389679, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5437861680984497, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5638662576675415, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4422273635864258, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8444871306419373, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6589582562446594, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.31777724623680115, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.673615038394928, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6899653673171997, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.35915324091911316, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.821611225605011, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6751945614814758, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.43592381477355957, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.676590085029602, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6878217458724976, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.38857436180114746, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8702405691146851, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.4976446330547333, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.18744251132011414, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5048505663871765, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4922542870044708, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2756146788597107, "metrics/hit_grouping/calorimeter/purity": 0.6610050797462463, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7771482467651367, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.10084328055381775, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.08458646386861801, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09800617396831512, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06099120154976845, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.030075188726186752, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7952849268913269, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.22029925882816315, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09299588203430176, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.220220685005188, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.2054348737001419, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24014125764369965, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8596479296684265, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6112655997276306, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.27519986033439636, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.622463047504425, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6265391111373901, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3307298719882965, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7400091290473938, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.38186296820640564, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5359516143798828, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5833038687705994, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5697522759437561, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7882353663444519, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.23869800567626953, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5099852085113525, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5285741090774536, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4279686510562897, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6603796482086182, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4188372492790222, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.44408804178237915, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.49997976422309875, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6767880320549011, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6942974925041199, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.394167423248291, "metrics/hit_grouping/tracker/all/object_completeness": 0.47718948125839233, "metrics/hit_grouping/tracker/all/object_purity": 0.5277012586593628, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6255120038986206, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009408089332282543, "metrics/hit_grouping/tracker/background_recall": 0.09610830247402191, "metrics/hit_grouping/tracker/completeness": 0.47507548332214355, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4314839541912079, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.728865385055542, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3735752999782562, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5501700043678284, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5858635902404785, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5825004577636719, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6753901243209839, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.41205301880836487, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.44960281252861023, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5172675848007202, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6563804745674133, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6419764161109924, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4187556505203247, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3544768989086151, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.41842788457870483, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6833183169364929, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6950258612632751, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.41691431403160095, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.48702943325042725, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5483981370925903, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6288528442382812, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6635517477989197, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4162561595439911, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.42756015062332153, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.5001915097236633, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6625615954399109, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6988624334335327, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.36063581705093384, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4732941687107086, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5057262778282166, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6145269870758057, "metrics/hit_grouping/tracker/purity": 0.44742199778556824, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8889631628990173, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04147813096642494, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.14058414101600647, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13844022154808044, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06334841996431351, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6962217092514038, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2870418131351471, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.40550386905670166, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4127846360206604, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5069695115089417, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6652011871337891, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4581165015697479, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5349104404449463, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5979219079017639, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7217838764190674, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6043031811714172, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7219184637069702, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8336424231529236, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6694660782814026, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.05515044182538986, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0949912816286087, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.35425567626953, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3134431540966034, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.22182001173496246, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9111263751983643, "metrics/particle/class_1/pid_accuracy": 0.9775400757789612, "metrics/particle/class_2/efficiency": 0.6058443784713745, "metrics/particle/class_2/pid_accuracy": 0.7887455821037292, "metrics/particle/class_3/efficiency": 0.828323483467102, "metrics/particle/class_3/pid_accuracy": 0.981050431728363, "metrics/particle/class_4/efficiency": 0.7214428782463074, "metrics/particle/class_4/pid_accuracy": 0.47361111640930176, "metrics/particle/class_5/efficiency": 0.9348658919334412, "metrics/particle/class_5/pid_accuracy": 0.631147563457489, "metrics/particle/count/bias_mean": -1.2076822519302368, "metrics/particle/count/mae": 3.2233073711395264, "metrics/particle/count/prediction_mean": 69.447265625, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06492803245782852, "metrics/particle/event/energy_response_mean": 0.977823793888092, "metrics/particle/event/met_abs_error": 4.951838970184326, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04727727547287941, "metrics/particle/event/scalar_pt_response_mean": 0.9963598251342773, "metrics/particle/event/vector_pt_closure": 0.04775324463844299, "metrics/particle/matched/delta_eta_abs_mean": 0.011846671812236309, "metrics/particle/matched/delta_phi_abs_mean": 0.01183186937123537, "metrics/particle/matched/delta_r_mean": 0.018824147060513496, "metrics/particle/matched/energy_relative_abs_mean": 0.12561599910259247, "metrics/particle/matched/pid_accuracy": 0.9622645378112793, "metrics/particle/matched/pt_relative_abs_mean": 0.12614955008029938, "metrics/particle/matching/duplicate_fraction": 0.04132332280278206, "metrics/particle/matching/efficiency": 0.8497595191001892, "metrics/particle/matching/f1": 0.8570844531059265, "metrics/particle/matching/purity": 0.8645367622375488, "metrics/particle/matching/target_coverage_dr0p05": 0.8185688257217407, "metrics/particle/matching/target_coverage_dr0p10": 0.8847833871841431, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6833754181861877, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8765918016433716, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018916917964816093, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12368103861808777, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1252044290304184, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.023938633501529694, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16421326994895935, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004710264038294554, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5725477933883667, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.015279672108590603, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.634084939956665, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9129.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.17006926238536835, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8150179386138916, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7388266921043396, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6162564754486084, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09506398439407349, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.039913177490234, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.010391609743237495, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8052864074707031, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.015817973762750626, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.0916748046875, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10393.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8474749326705933, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.005172513425350189, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.093775272369385, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.7751989364624023, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.05961545556783676, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.07467743754386902, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.1807050704956055, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.425381064414978, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.865933418273926, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022700179368257523, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1410607248544693, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1430450826883316, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.025137055665254593, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1838930994272232, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023579850792884827, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.2253888845443726, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0047116950154304504, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.5082406997680664, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1866.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.42768365144729614, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.7878649830818176, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.95758855342865, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.7978851795196533, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17067109048366547, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 15.835247039794922, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.053438276052474976, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.329725980758667, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3484875559806824, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.074902534484863, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2414.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6765772104263306, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.383233070373535, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.862558841705322, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0709328651428223, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.037431467324495316, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07961445301771164, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.736458778381348, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.0724916458129883, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.878986358642578, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018458472564816475, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12568289041519165, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12577220797538757, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.024851057678461075, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16589321196079254, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004944131360389292, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19589875638484955, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.006359708961099386, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6227285861968994, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20226.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15005990862846375, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7220794558525085, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5718767046928406, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4425199329853058, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08913271874189377, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.237748146057129, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026228362694382668, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3099537789821625, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0056822472251951694, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2829158306121826, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22876.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8692251443862915, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.0874507427215576, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.286250591278076, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.050903797149658, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.027779079973697662, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.6007452011108398, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.3835458755493164, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.08349847793579102, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.791329383850098, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018846038728952408, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12601663172245026, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12663854658603668, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02460135705769062, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16653740406036377, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0031068832613527775, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3675606846809387, "metrics/particle/momentum/all/accepted/pz_bias": -0.0006293418700806797, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3295117616653442, "metrics/particle/momentum/all/accepted/samples": 31221.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.17466939985752106, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7535990476608276, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7142475843429565, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5848144292831421, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09637642651796341, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 7.306066036224365, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008323291316628456, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5908640623092651, "metrics/particle/momentum/all/assigned/pz_bias": -0.015325667336583138, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8723292350769043, "metrics/particle/momentum/all/assigned/samples": 35683.0, "metrics/particle/momentum/all/efficiency": 0.8484197854995728, "metrics/particle/momentum/all/event/energy_bias": -3.300955295562744, "metrics/particle/momentum/all/event/p3_error_mean": 15.419597625732422, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.372214317321777, "metrics/particle/momentum/all/event/px_bias": -0.049963004887104034, "metrics/particle/momentum/all/event/py_bias": 0.6056821942329407, "metrics/particle/momentum/all/event/pz_abs_mean": 11.525749206542969, "metrics/particle/momentum/all/event/pz_bias": -0.7306092977523804, "metrics/particle/momentum/all/event/pz_residual_rms": 22.73429298400879, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01993144489824772, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10627444833517075, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10817952454090118, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02852439135313034, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14501242339611053, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002727159997448325, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.37888962030410767, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0036051063798367977, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1727503538131714, "metrics/particle/momentum/class_1/accepted/samples": 16134.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.104617640376091, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.34175044298171997, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3453315496444702, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.24250803887844086, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.052785418927669525, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.502460479736328, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005272810813039541, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5080520510673523, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.008114570751786232, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.733686923980713, "metrics/particle/momentum/class_1/assigned/samples": 17448.0, "metrics/particle/momentum/class_1/efficiency": 0.9094188809394836, "metrics/particle/momentum/class_1/event/energy_bias": 4.002198219299316, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.257781028747559, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.629432201385498, "metrics/particle/momentum/class_1/event/px_bias": 0.13707341253757477, "metrics/particle/momentum/class_1/event/py_bias": 0.7191162705421448, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.033323287963867, "metrics/particle/momentum/class_1/event/pz_bias": 0.48861202597618103, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.25664710998535, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.024760760366916656, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17446458339691162, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1742316484451294, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.02489575929939747, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21662963926792145, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.019036954268813133, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.220133900642395, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.06465033441781998, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.006075382232666, "metrics/particle/momentum/class_2/accepted/samples": 1786.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.47699210047721863, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5507256388664246, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7030907273292542, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4166180491447449, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.22452978789806366, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.363060474395752, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04192790016531944, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.009809732437134, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1452948898077011, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.072378635406494, "metrics/particle/momentum/class_2/assigned/samples": 2552.0, "metrics/particle/momentum/class_2/efficiency": 0.5981245636940002, "metrics/particle/momentum/class_2/event/energy_bias": -6.836282253265381, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.517912864685059, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.273189544677734, "metrics/particle/momentum/class_2/event/px_bias": -0.1414175033569336, "metrics/particle/momentum/class_2/event/py_bias": -0.45800337195396423, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.285640716552734, "metrics/particle/momentum/class_2/event/pz_bias": -0.5203911066055298, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.21331024169922, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01665296033024788, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14293913543224335, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14223602414131165, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.018197305500507355, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18177494406700134, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010093951132148504, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2179955095052719, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.003860201220959425, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0642825365066528, "metrics/particle/momentum/class_3/accepted/samples": 12879.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.20109497010707855, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2305055856704712, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1197377443313599, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.9847497344017029, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.12235201150178909, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.819073677062988, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005939417984336615, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4352498948574066, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.023925160989165306, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.50727915763855, "metrics/particle/momentum/class_3/assigned/samples": 15153.0, "metrics/particle/momentum/class_3/efficiency": 0.8291379809379578, "metrics/particle/momentum/class_3/event/energy_bias": 0.5954370498657227, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.593550682067871, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.370931148529053, "metrics/particle/momentum/class_3/event/px_bias": -0.12090213596820831, "metrics/particle/momentum/class_3/event/py_bias": 0.35708871483802795, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.375658988952637, "metrics/particle/momentum/class_3/event/pz_bias": -0.6757199764251709, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.943235397338867, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.022440360859036446, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17343662679195404, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16843761503696442, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08879028260707855, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.217971071600914, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.003703703638166189, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5984390377998352, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06730547547340393, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.401375651359558, "metrics/particle/momentum/class_4/accepted/samples": 270.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3490552008152008, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.277677536010742, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.7943792343139648, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7146626710891724, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22131147980690002, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 10.616002082824707, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005464480724185705, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6674988865852356, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.027300715446472168, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.5451301336288452, "metrics/particle/momentum/class_4/assigned/samples": 366.0, "metrics/particle/momentum/class_4/efficiency": 0.7200000286102295, "metrics/particle/momentum/class_4/event/energy_bias": -0.9073875546455383, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6596949100494385, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2139126062393188, "metrics/particle/momentum/class_4/event/px_bias": 0.03458773344755173, "metrics/particle/momentum/class_4/event/py_bias": 0.052133481949567795, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9252747297286987, "metrics/particle/momentum/class_4/event/pz_bias": -0.26048460602760315, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.268878221511841, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013573006726801395, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13420143723487854, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.13092251121997833, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.03333029896020889, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17087343335151672, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.032894738018512726, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4098984003067017, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.455230712890625, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.3824985027313232, "metrics/particle/momentum/class_5/accepted/samples": 152.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.09225311130285263, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.2614500820636749, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.2605770528316498, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.1461227685213089, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.060975611209869385, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.1233960390090942, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03658536449074745, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.5282350778579712, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.3345969021320343, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6726620197296143, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9268292784690857, "metrics/particle/momentum/class_5/event/energy_bias": -0.15492035448551178, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6409283876419067, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2544275522232056, "metrics/particle/momentum/class_5/event/px_bias": 0.04069555923342705, "metrics/particle/momentum/class_5/event/py_bias": -0.06465278565883636, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8074478507041931, "metrics/particle/momentum/class_5/event/pz_bias": 0.23737463355064392, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.8785922527313232, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.29071667790412903, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2925269603729248, "metrics/set_slot/angular_recall_all_dr0p10": 0.9655197858810425, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8710262775421143, "metrics/set_slot/soft_count_bias_mean": -0.23279206454753876}, "step_train_time": 5.146845102310181, "step_valid_time": 49251.65327548981, "step_total_time": 49256.800120830536, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0081076138814555, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10450819498198105, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3042, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2939, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2928, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.960771493952272, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9661406969099277, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9634486149811506, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03385930309007232, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9571755475645636, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9625246548323472, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9598426487461071, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.960771493952272, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007997005433868, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1020036738049448, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2278, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2167, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2163, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9483588621444201, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9512730465320457, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9498137190444883, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048726953467954345, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9466083150984683, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9495171202809481, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.948060486522025, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9483588621444201, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0114082672247684, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10999389646277613, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1323, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1246, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1242, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9425113464447806, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9417989417989417, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9421550094517959, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0582010582010582, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9394856278366112, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9387755102040817, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9391304347826087, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9425113464447806} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_47000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_47000.json new file mode 100644 index 0000000000000000000000000000000000000000..e58aafbb6b60a7f5c03f5d9964665dc3cf748618 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_47000.json @@ -0,0 +1 @@ +{"epoch": 2.226222053808261, "train": {"Classification_binary": 0.11962471902370453, "Cardinality": 0.09816861897706985, "Classification": 0.07735079526901245, "Regression_pt": 0.09437121450901031, "Regression_eta": 0.0009820382110774517, "Regression_sin_phi": 0.00046886593918316066, "Regression_cos_phi": 0.0005481631378643215, "Regression_energy": 0.09320713579654694, "HitGrouping": 0.14443230628967285, "Auxiliary": 0.1720600128173828, "Total": 0.797981858253479, "Total_unweighted": 0.38655293188639916}, "train_group": {}, "valid": {"Classification_binary": 0.13381598393122354, "Cardinality": 0.10613463322321574, "Classification": 0.07950642208258311, "Regression_pt": 0.09065951903661092, "Regression_eta": 0.00111917561541001, "Regression_sin_phi": 0.000605895339200894, "Regression_cos_phi": 0.0005984227561081449, "Regression_energy": 0.09019477168718974, "HitGrouping": 0.14630322655042013, "Auxiliary": 0.1839601198832194, "Total": 0.8508480389912924, "Total_unweighted": 0.39650019044832635, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8515655398368835, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6142268180847168, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2757641673088074, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6270729899406433, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6314575672149658, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3295164108276367, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.798509955406189, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.554418683052063, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.28471195697784424, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5653237700462341, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5681239366531372, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4098966121673584, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8644344806671143, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5851768851280212, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2547095715999603, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.594420850276947, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5966701507568359, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.30756768584251404, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8554176688194275, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.591589629650116, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.26341742277145386, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.6020539402961731, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6049928665161133, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3221617043018341, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.003546595573425293, "metrics/hit_grouping/calorimeter/background_recall": 0.15659882128238678, "metrics/hit_grouping/calorimeter/completeness": 0.6772280931472778, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6763708591461182, "metrics/hit_grouping/calorimeter/energy_purity": 0.6676756739616394, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.2986626923084259, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8766298890113831, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.6008187532424927, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.20061446726322174, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6131965517997742, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6043271422386169, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.2657032310962677, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8601160049438477, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6210159659385681, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.27671578526496887, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6324499249458313, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6374760270118713, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.31506219506263733, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8074331283569336, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5431267619132996, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3780742585659027, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5487123727798462, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.573625922203064, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4444315433502197, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8449403047561646, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6699128746986389, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3171156942844391, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.684871256351471, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.7032025456428528, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3554847240447998, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8224944472312927, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6802611947059631, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4117344319820404, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6813946962356567, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.7005249857902527, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3942357301712036, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8700222373008728, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5011703968048096, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.190998837351799, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5081642866134644, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.4934808909893036, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.27960023283958435, "metrics/hit_grouping/calorimeter/purity": 0.6676310300827026, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7804041504859924, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.10365281999111176, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0827067643404007, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.10355576872825623, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06855445355176926, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.0357142873108387, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8011162281036377, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.23742307722568512, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09711594879627228, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.23473218083381653, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.22023944556713104, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.24543848633766174, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8594657778739929, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6179705858230591, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.27501606941223145, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6292915344238281, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6337557435035706, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.33082178235054016, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.738814115524292, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.39750832319259644, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5389296412467957, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5836628079414368, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5630885362625122, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7828052639961243, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2501506805419922, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5182409286499023, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5357825756072998, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.42736586928367615, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6560437679290771, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4248093366622925, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4462391436100006, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.505488932132721, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6780831813812256, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6908460259437561, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40351736545562744, "metrics/hit_grouping/tracker/all/object_completeness": 0.4800456166267395, "metrics/hit_grouping/tracker/all/object_purity": 0.5317530632019043, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6242208480834961, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.008640406653285027, "metrics/hit_grouping/tracker/background_recall": 0.09338490664958954, "metrics/hit_grouping/tracker/completeness": 0.476034551858902, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4381139576435089, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7260646224021912, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3868139684200287, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5536164045333862, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5851275324821472, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5817990303039551, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6709004640579224, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.41604647040367126, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45438072085380554, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5293542146682739, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6558359265327454, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6382180452346802, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4254283010959625, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.35420966148376465, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.42434564232826233, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6800721287727356, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6906232833862305, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4224335551261902, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4901546835899353, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5569010972976685, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6292774081230164, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6697162985801697, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4433497488498688, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.433359295129776, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4956737160682678, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6453201770782471, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6949235796928406, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3717849552631378, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4752729535102844, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.505527675151825, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6138646602630615, "metrics/hit_grouping/tracker/purity": 0.44797807931900024, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8875992894172668, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.04260935261845589, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13576017320156097, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1367446631193161, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06334841996431351, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6969528198242188, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.290139377117157, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.4155013859272003, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.43768104910850525, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.501806914806366, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.660986602306366, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4693637788295746, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5381322503089905, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.6005673408508301, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7207207083702087, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6185250282287598, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7164944410324097, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8269615173339844, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6783087253570557, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.0693722814321518, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.10383396595716476, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.425735473632812, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.29948464035987854, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.20786148309707642, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.911660373210907, "metrics/particle/class_1/pid_accuracy": 0.9779088497161865, "metrics/particle/class_2/efficiency": 0.606184184551239, "metrics/particle/class_2/pid_accuracy": 0.7847533822059631, "metrics/particle/class_3/efficiency": 0.8333515524864197, "metrics/particle/class_3/pid_accuracy": 0.981322169303894, "metrics/particle/class_4/efficiency": 0.703406810760498, "metrics/particle/class_4/pid_accuracy": 0.5327635407447815, "metrics/particle/class_5/efficiency": 0.9501915574073792, "metrics/particle/class_5/pid_accuracy": 0.600806474685669, "metrics/particle/count/bias_mean": -1.013671875, "metrics/particle/count/mae": 3.1686198711395264, "metrics/particle/count/prediction_mean": 69.64127349853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06539735943078995, "metrics/particle/event/energy_response_mean": 0.986137866973877, "metrics/particle/event/met_abs_error": 4.899781703948975, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0482037253677845, "metrics/particle/event/scalar_pt_response_mean": 1.0031601190567017, "metrics/particle/event/vector_pt_closure": 0.047317828983068466, "metrics/particle/matched/delta_eta_abs_mean": 0.01215285249054432, "metrics/particle/matched/delta_phi_abs_mean": 0.011961265467107296, "metrics/particle/matched/delta_r_mean": 0.01913781464099884, "metrics/particle/matched/energy_relative_abs_mean": 0.12402930855751038, "metrics/particle/matched/pid_accuracy": 0.9627346396446228, "metrics/particle/matched/pt_relative_abs_mean": 0.12458155304193497, "metrics/particle/matching/duplicate_fraction": 0.04166627675294876, "metrics/particle/matching/efficiency": 0.8520722985267639, "metrics/particle/matching/f1": 0.8582287430763245, "metrics/particle/matching/purity": 0.8644747734069824, "metrics/particle/matching/target_coverage_dr0p05": 0.8181173205375671, "metrics/particle/matching/target_coverage_dr0p10": 0.8855665922164917, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6899452805519104, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8794482350349426, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019337603822350502, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12353140115737915, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12420763075351715, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.022212635725736618, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16384592652320862, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00477845361456275, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5784672498703003, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0013773178216069937, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6910736560821533, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9208.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.16984793543815613, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7565363049507141, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7519533634185791, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6265514492988586, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.08937475830316544, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.874109745025635, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.010356731712818146, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8064520359039307, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.048643700778484344, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.280900716781616, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10428.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8548087477684021, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.0271793603897095, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.340177536010742, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.912034511566162, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.0025749392807483673, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.09523499011993408, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.390663146972656, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7007633447647095, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.569496154785156, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02251465991139412, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13731499016284943, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13857989013195038, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.024034863337874413, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.18043674528598785, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023391813039779663, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1856580972671509, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.02244962565600872, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9902865886688232, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1881.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.44569727778434753, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0166597366333008, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.440342426300049, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.284013271331787, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1648760288953781, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 26.774864196777344, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.057024791836738586, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3211236000061035, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.20551609992980957, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.370829105377197, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2420.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6820159554481506, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.7242836952209473, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.358156204223633, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0170732736587524, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.07297977805137634, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.03198716789484024, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.23684024810791, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.47189515829086304, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.848875045776367, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018954770639538765, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12408819049596786, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12447517365217209, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.019825803115963936, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16425034403800964, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00029600394191220403, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1928802728652954, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.004417184740304947, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5976840853691101, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20270.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14843213558197021, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7594163417816162, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6118592023849487, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.47861236333847046, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08650987595319748, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.975444793701172, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0023605525493621826, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.30768266320228577, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0007507034461013973, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2778213024139404, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22876.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8711161017417908, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.057818651199341, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.061508655548096, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.868771076202393, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.35001978278160095, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.15828104317188263, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.262739419937134, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.17841121554374695, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.482983589172363, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01928071118891239, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1247180923819542, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12524265050888062, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.020779123529791832, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1651480346918106, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0029975445941090584, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3656504154205322, "metrics/particle/momentum/all/accepted/pz_bias": -0.0037973695434629917, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.267662525177002, "metrics/particle/momentum/all/accepted/samples": 31359.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.17482072114944458, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7760018706321716, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7766176462173462, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6440971493721008, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09265479445457458, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.670292854309082, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.008397716097533703, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.589669406414032, "metrics/particle/momentum/all/assigned/pz_bias": 0.0007580525707453489, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.805009603500366, "metrics/particle/momentum/all/assigned/samples": 35724.0, "metrics/particle/momentum/all/efficiency": 0.8521698713302612, "metrics/particle/momentum/all/event/energy_bias": -0.6392858028411865, "metrics/particle/momentum/all/event/p3_error_mean": 15.319573402404785, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.1944169998168945, "metrics/particle/momentum/all/event/px_bias": 0.42042458057403564, "metrics/particle/momentum/all/event/py_bias": 0.28550317883491516, "metrics/particle/momentum/all/event/pz_abs_mean": 11.562970161437988, "metrics/particle/momentum/all/event/pz_bias": 0.40727946162223816, "metrics/particle/momentum/all/event/pz_residual_rms": 22.46646499633789, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02050994336605072, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1036543920636177, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10474175959825516, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.020816246047616005, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.1414288878440857, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002410830231383443, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.37627479434013367, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.02314414083957672, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.191814661026001, "metrics/particle/momentum/class_1/accepted/samples": 16177.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10822856426239014, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.351708322763443, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3405444622039795, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.23347200453281403, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05112661421298981, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.179101467132568, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0053185406140983105, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5178101062774658, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01701791025698185, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7586135864257812, "metrics/particle/momentum/class_1/assigned/samples": 17486.0, "metrics/particle/momentum/class_1/efficiency": 0.9118426442146301, "metrics/particle/momentum/class_1/event/energy_bias": 5.241040229797363, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.522859573364258, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.657683849334717, "metrics/particle/momentum/class_1/event/px_bias": 0.893843948841095, "metrics/particle/momentum/class_1/event/py_bias": 0.6337689161300659, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.250588417053223, "metrics/particle/momentum/class_1/event/pz_bias": -0.3741219639778137, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.09945297241211, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.025130117312073708, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1678880751132965, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1690104901790619, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.01575074903666973, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2109082043170929, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.021288515999913216, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.2787452936172485, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.026667440310120583, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.160264253616333, "metrics/particle/momentum/class_2/accepted/samples": 1785.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.49507632851600647, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5786826014518738, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9326870441436768, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6372700333595276, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2240031212568283, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 9.974082946777344, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04612978920340538, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.13753604888916, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.06325672566890717, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.441049098968506, "metrics/particle/momentum/class_2/assigned/samples": 2558.0, "metrics/particle/momentum/class_2/efficiency": 0.5977897047996521, "metrics/particle/momentum/class_2/event/energy_bias": -7.579580783843994, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.53456974029541, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.104135990142822, "metrics/particle/momentum/class_2/event/px_bias": 6.623566150665283e-05, "metrics/particle/momentum/class_2/event/py_bias": -0.26149094104766846, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.450102806091309, "metrics/particle/momentum/class_2/event/pz_bias": -0.02969089150428772, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.241336822509766, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.016927240416407585, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14404238760471344, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14394992589950562, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.019702978432178497, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18333974480628967, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010017723543569446, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21142709255218506, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.021958308294415474, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7848938703536987, "metrics/particle/momentum/class_3/accepted/samples": 12977.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.19355979561805725, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2315772771835327, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.1929494142532349, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0568897724151611, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.11581864953041077, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.912823677062988, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005477463360875845, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4012697637081146, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.029339918866753578, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.147097587585449, "metrics/particle/momentum/class_3/assigned/samples": 15153.0, "metrics/particle/momentum/class_3/efficiency": 0.8354471325874329, "metrics/particle/momentum/class_3/event/energy_bias": 2.487947702407837, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.158543586730957, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.380835056304932, "metrics/particle/momentum/class_3/event/px_bias": -0.370410680770874, "metrics/particle/momentum/class_3/event/py_bias": -0.01952780783176422, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.936488151550293, "metrics/particle/momentum/class_3/event/pz_bias": 0.7135893106460571, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.21452522277832, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02261301502585411, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16391484439373016, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.15858419239521027, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.09338057041168213, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.207312673330307, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0038022813387215137, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5457589626312256, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0321052223443985, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.237388014793396, "metrics/particle/momentum/class_4/accepted/samples": 263.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.40228888392448425, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.867025136947632, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 3.581602096557617, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.501931667327881, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.22865013778209686, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 26.463268280029297, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005509641952812672, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.627321183681488, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.14189930260181427, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.437758445739746, "metrics/particle/momentum/class_4/assigned/samples": 363.0, "metrics/particle/momentum/class_4/efficiency": 0.7013333439826965, "metrics/particle/momentum/class_4/event/energy_bias": -0.591869056224823, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.681415319442749, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2761449813842773, "metrics/particle/momentum/class_4/event/px_bias": -0.1720910370349884, "metrics/particle/momentum/class_4/event/py_bias": 0.0797727108001709, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8904597163200378, "metrics/particle/momentum/class_4/event/pz_bias": -0.04041241109371185, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.984459400177002, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01506523136049509, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.14133380353450775, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1378832906484604, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.041454944759607315, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.174866184592247, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.019108280539512634, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.335356593132019, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.060673754662275314, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.753340482711792, "metrics/particle/momentum/class_5/accepted/samples": 157.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.044898077845573425, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.15747448801994324, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.16117040812969208, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.06614407151937485, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.030487805604934692, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.24677614867687225, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.024390242993831635, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.432603359222412, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.058723147958517075, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.1967124938964844, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.957317054271698, "metrics/particle/momentum/class_5/event/energy_bias": -0.19682344794273376, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5283695459365845, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.1710792779922485, "metrics/particle/momentum/class_5/event/px_bias": 0.06901620328426361, "metrics/particle/momentum/class_5/event/py_bias": -0.1470196545124054, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7797243595123291, "metrics/particle/momentum/class_5/event/pz_bias": 0.13791553676128387, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.707578659057617, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.28961381316185, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.29143255949020386, "metrics/set_slot/angular_recall_all_dr0p10": 0.964773416519165, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8710170984268188, "metrics/set_slot/soft_count_bias_mean": -0.05027272179722786}, "step_train_time": 5.571587324142456, "step_valid_time": 52188.66818737984, "step_total_time": 52194.2397749424, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0071260157066366, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10408896789297484, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3041, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2938, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2925, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9604445897352076, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9661295626438672, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9632786885245902, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03387043735613285, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9561948349133704, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9618546530746465, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9590163934426229, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9604445897352076, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038164707173873, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09942704354528686, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2272, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2171, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2168, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.950109409190372, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9555457746478874, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9528198376124644, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04445422535211268, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9487964989059081, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.954225352112676, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9515031819179285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.950109409190372, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0157088068782731, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10413438470593817, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1324, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1243, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1243, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9402420574886535, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9388217522658611, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9395313681027967, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06117824773413897, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9402420574886535, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9388217522658611, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9395313681027967, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9402420574886535} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_48000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_48000.json new file mode 100644 index 0000000000000000000000000000000000000000..8378ff4119d68115b807262beb934c4d7df4405b --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_48000.json @@ -0,0 +1 @@ +{"epoch": 2.2735884804850324, "train": {"Classification_binary": 0.13961933553218842, "Cardinality": 0.09718575328588486, "Classification": 0.07675634324550629, "Regression_pt": 0.0887027233839035, "Regression_eta": 0.0014775285962969065, "Regression_sin_phi": 0.0007077091722749174, "Regression_cos_phi": 0.0006046167109161615, "Regression_energy": 0.08897852152585983, "HitGrouping": 0.14254258573055267, "Auxiliary": 0.1885657012462616, "Total": 0.8640626668930054, "Total_unweighted": 0.396846778166946}, "train_group": {}, "valid": {"Classification_binary": 0.1347977121671041, "Cardinality": 0.10547163089116414, "Classification": 0.07910092671712239, "Regression_pt": 0.09154204527537028, "Regression_eta": 0.00115995683396856, "Regression_sin_phi": 0.0006468475718672076, "Regression_cos_phi": 0.0006312758196145296, "Regression_energy": 0.09102043509483337, "HitGrouping": 0.1461255153020223, "Auxiliary": 0.18783974647521973, "Total": 0.8628574212392172, "Total_unweighted": 0.39889919947988045, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8518436551094055, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6085809469223022, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2779407501220703, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6212329268455505, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6234981417655945, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.32894861698150635, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7969302535057068, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.55064857006073, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.2776957154273987, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5601146817207336, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5663414597511292, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.413220077753067, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8632886409759521, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5786534547805786, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.2498495727777481, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5875672101974487, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5927537083625793, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3060402572154999, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.854669451713562, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5855458378791809, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.260521799325943, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5956349968910217, "metrics/hit_grouping/calorimeter/all/object_purity": 0.600017249584198, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3213016092777252, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0037692070472985506, "metrics/hit_grouping/calorimeter/background_recall": 0.16542169451713562, "metrics/hit_grouping/calorimeter/completeness": 0.673052966594696, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6733381748199463, "metrics/hit_grouping/calorimeter/energy_purity": 0.6647228598594666, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.2968285381793976, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.876175582408905, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5960767865180969, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.1998179405927658, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6079826354980469, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.5984912514686584, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.26035502552986145, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8605330586433411, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6152021288871765, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2688853144645691, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6259523630142212, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6302324533462524, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.31655919551849365, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8049118518829346, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5341967940330505, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.37587007880210876, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.539915919303894, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.57268887758255, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.45034801959991455, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8443032503128052, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6661415100097656, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.3093577027320862, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6804344058036804, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.703729510307312, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3554847240447998, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.822565495967865, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6773757934570312, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.4225424528121948, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6798896193504333, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6967390775680542, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.39886772632598877, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.869063138961792, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.49243348836898804, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1914280503988266, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.4991390109062195, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.48275312781333923, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2772088944911957, "metrics/hit_grouping/calorimeter/purity": 0.6636632680892944, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7743884325027466, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.09361937642097473, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.08646616339683533, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.09108709543943405, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.06206469610333443, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.030075188726186752, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8012698292732239, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.22755400836467743, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10829900205135345, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.22671514749526978, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.20967997610569, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.22483813762664795, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.858756422996521, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.612190842628479, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.27127936482429504, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6230543255805969, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6290954351425171, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.3310667872428894, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7420167922973633, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4018542766571045, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5420420169830322, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5795868039131165, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5632333755493164, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7818669676780701, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2561784088611603, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5068286657333374, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5213063359260559, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4291742146015167, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6578625440597534, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.42790329456329346, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.44597581028938293, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.503364622592926, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6785868406295776, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6928865909576416, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4072128236293793, "metrics/hit_grouping/tracker/all/object_completeness": 0.47999635338783264, "metrics/hit_grouping/tracker/all/object_purity": 0.5281164646148682, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6247106194496155, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009731045924127102, "metrics/hit_grouping/tracker/background_recall": 0.10309441387653351, "metrics/hit_grouping/tracker/completeness": 0.47588983178138733, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4439201056957245, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7262859344482422, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3826933205127716, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5523253679275513, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.585729718208313, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5835525393486023, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.675215482711792, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4312942326068878, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45665332674980164, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5227124094963074, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6543837189674377, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6417409777641296, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4337240755558014, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3544076383113861, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4149756133556366, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6798917651176453, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6926649212837219, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.42481106519699097, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4882555902004242, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.551546037197113, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6286830306053162, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6852626204490662, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.48091134428977966, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4442571997642517, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.47039109468460083, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6354680061340332, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6945415735244751, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3711226284503937, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.475665807723999, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5080055594444275, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6176178455352783, "metrics/hit_grouping/tracker/purity": 0.4483351409435272, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8895933628082275, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.049396682530641556, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13858595490455627, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1349751353263855, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.059954751282930374, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6960816979408264, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.3076923191547394, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.4175227880477905, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.43055737018585205, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5038719773292542, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6633496284484863, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4710984230041504, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5374318957328796, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5970316529273987, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7216160297393799, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.61943519115448, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.728491485118866, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8416303396224976, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.672039270401001, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.07028240710496902, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09756442904472351, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.45610237121582, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3189583420753479, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.22733518481254578, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9135103225708008, "metrics/particle/class_1/pid_accuracy": 0.9778701066970825, "metrics/particle/class_2/efficiency": 0.6064106822013855, "metrics/particle/class_2/pid_accuracy": 0.7796040177345276, "metrics/particle/class_3/efficiency": 0.8304439783096313, "metrics/particle/class_3/pid_accuracy": 0.9816253185272217, "metrics/particle/class_4/efficiency": 0.7004008293151855, "metrics/particle/class_4/pid_accuracy": 0.5379112958908081, "metrics/particle/class_5/efficiency": 0.9501915574073792, "metrics/particle/class_5/pid_accuracy": 0.6330645084381104, "metrics/particle/count/bias_mean": -0.787109375, "metrics/particle/count/mae": 3.2233073711395264, "metrics/particle/count/prediction_mean": 69.86783599853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06570615619421005, "metrics/particle/event/energy_response_mean": 0.981910228729248, "metrics/particle/event/met_abs_error": 4.942559242248535, "metrics/particle/event/scalar_pt_relative_abs_error": 0.047715455293655396, "metrics/particle/event/scalar_pt_response_mean": 0.9978587627410889, "metrics/particle/event/vector_pt_closure": 0.0484868586063385, "metrics/particle/matched/delta_eta_abs_mean": 0.01200063619762659, "metrics/particle/matched/delta_phi_abs_mean": 0.012091170065104961, "metrics/particle/matched/delta_r_mean": 0.01913675293326378, "metrics/particle/matched/energy_relative_abs_mean": 0.12395986169576645, "metrics/particle/matched/pid_accuracy": 0.9627522230148315, "metrics/particle/matched/pt_relative_abs_mean": 0.12408053129911423, "metrics/particle/matching/duplicate_fraction": 0.04195048287510872, "metrics/particle/matching/efficiency": 0.8517313599586487, "metrics/particle/matching/f1": 0.8565021753311157, "metrics/particle/matching/purity": 0.8613266944885254, "metrics/particle/matching/target_coverage_dr0p05": 0.814754068851471, "metrics/particle/matching/target_coverage_dr0p10": 0.8846175074577332, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6911707520484924, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8795772194862366, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019496316090226173, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12254053354263306, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12340305000543594, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.021865393966436386, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1634887009859085, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004593175835907459, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5609691143035889, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.021755235269665718, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6181195974349976, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9144.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.17889326810836792, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.5733844637870789, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.658930242061615, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.5325242877006531, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.09326674044132233, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.6948676109313965, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.00854865089058876, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7470142841339111, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.059354592114686966, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.2144603729248047, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10411.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8488674163818359, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.2942441701889038, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.212101936340332, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8502724170684814, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.42319393157958984, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.4443456530570984, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.285473823547363, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.4693560600280762, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.34081745147705, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02282854914665222, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13901527225971222, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1374359428882599, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02462855726480484, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17859363555908203, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01816239394247532, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1746454238891602, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05031174421310425, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.114240884780884, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1872.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4616921842098236, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1348347663879395, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.7700858116149902, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.6057140827178955, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.17287996411323547, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 35.66554260253906, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05161818861961365, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3421220779418945, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.19687554240226746, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.538623332977295, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2441.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6787527203559875, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.087377548217773, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.885205268859863, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0162760019302368, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.05659469962120056, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.0003960803151130676, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.766148567199707, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.52033931016922, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.252365112304688, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018892481923103333, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12392129749059677, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12425245344638824, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01585792750120163, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1634122133255005, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00044256492401473224, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1946716010570526, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01261829398572445, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6215168237686157, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20336.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14829809963703156, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7552968263626099, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5663147568702698, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4312763512134552, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0864553302526474, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.627175807952881, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0025761942379176617, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3037322163581848, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03743850812315941, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2073585987091064, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22902.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8739524483680725, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.5932925939559937, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.688088417053223, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.610353946685791, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.032151833176612854, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.6559404134750366, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.0278196334838867, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.8217974901199341, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.74666166305542, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019303614273667336, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12441983819007874, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12479190528392792, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018133727833628654, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1643802374601364, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027111507952213287, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.3600178360939026, "metrics/particle/momentum/all/accepted/pz_bias": 0.01753377914428711, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2622566223144531, "metrics/particle/momentum/all/accepted/samples": 31352.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.17860299348831177, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7282386422157288, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7437388896942139, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6092115044593811, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09433909505605698, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.962587356567383, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.007663478143513203, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5719740986824036, "metrics/particle/momentum/all/assigned/pz_bias": 0.027823012322187424, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.498321294784546, "metrics/particle/momentum/all/assigned/samples": 35754.0, "metrics/particle/momentum/all/efficiency": 0.851979672908783, "metrics/particle/momentum/all/event/energy_bias": -2.199841260910034, "metrics/particle/momentum/all/event/p3_error_mean": 15.277934074401855, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.076173782348633, "metrics/particle/momentum/all/event/px_bias": 0.44763678312301636, "metrics/particle/momentum/all/event/py_bias": -1.100682020187378, "metrics/particle/momentum/all/event/pz_abs_mean": 11.61712646484375, "metrics/particle/momentum/all/event/pz_bias": 1.7708139419555664, "metrics/particle/momentum/all/event/pz_residual_rms": 22.90936279296875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02015276998281479, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10412038117647171, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10463063418865204, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02005317062139511, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14085423946380615, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0022864912170916796, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.36760151386260986, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.014380826614797115, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1530112028121948, "metrics/particle/momentum/class_1/accepted/samples": 16182.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10591690987348557, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.30695703625679016, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.3069995939731598, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.19845302402973175, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05305282399058342, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.155003547668457, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00491653336212039, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4797018766403198, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02464093081653118, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9855968952178955, "metrics/particle/momentum/class_1/assigned/samples": 17492.0, "metrics/particle/momentum/class_1/efficiency": 0.9121244549751282, "metrics/particle/momentum/class_1/event/energy_bias": 3.9333739280700684, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.943758964538574, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.709606647491455, "metrics/particle/momentum/class_1/event/px_bias": 0.27503883838653564, "metrics/particle/momentum/class_1/event/py_bias": -0.24540738761425018, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.701498985290527, "metrics/particle/momentum/class_1/event/pz_bias": 0.8241432905197144, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.244929313659668, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.025332458317279816, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17197532951831818, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17258858680725098, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.02110929600894451, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21475332975387573, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.01883656531572342, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.2538920640945435, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.004492004867643118, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.1980345249176025, "metrics/particle/momentum/class_2/accepted/samples": 1805.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.4939255118370056, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5659928917884827, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9613718390464783, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.670310378074646, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2300885021686554, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 15.882125854492188, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04001539200544357, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.096339225769043, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.005930628627538681, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.411440849304199, "metrics/particle/momentum/class_2/assigned/samples": 2599.0, "metrics/particle/momentum/class_2/efficiency": 0.6044875979423523, "metrics/particle/momentum/class_2/event/energy_bias": -8.492476463317871, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.316618919372559, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.144469738006592, "metrics/particle/momentum/class_2/event/px_bias": 0.24461230635643005, "metrics/particle/momentum/class_2/event/py_bias": -0.5049474835395813, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.15093994140625, "metrics/particle/momentum/class_2/event/pz_bias": 0.29628822207450867, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.46921157836914, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017462823539972305, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1419607549905777, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14227958023548126, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013550118543207645, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18130017817020416, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006951953982934356, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2086375653743744, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.021697983145713806, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.795291543006897, "metrics/particle/momentum/class_3/accepted/samples": 12946.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.20473548769950867, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.1719310283660889, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.132387638092041, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 0.993595540523529, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.11667986959218979, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.162676811218262, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005018488969653845, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4028681516647339, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03209299221634865, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.2219185829162598, "metrics/particle/momentum/class_3/assigned/samples": 15144.0, "metrics/particle/momentum/class_3/efficiency": 0.8334513902664185, "metrics/particle/momentum/class_3/event/energy_bias": 2.914250373840332, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.23491096496582, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.318063735961914, "metrics/particle/momentum/class_3/event/px_bias": -0.3491933345794678, "metrics/particle/momentum/class_3/event/py_bias": -0.1690923571586609, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.019381999969482, "metrics/particle/momentum/class_3/event/pz_bias": 0.35960352420806885, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.76772403717041, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02026325650513172, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1684315800666809, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16301588714122772, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.09231884777545929, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20981380343437195, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5566692352294922, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06953239440917969, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1467409133911133, "metrics/particle/momentum/class_4/accepted/samples": 262.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.4024761915206909, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 3.996309518814087, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.34820032119751, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 4.247369766235352, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2162921279668808, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 29.110546112060547, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.00561797758564353, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.7188500165939331, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.15657052397727966, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9602630138397217, "metrics/particle/momentum/class_4/assigned/samples": 356.0, "metrics/particle/momentum/class_4/efficiency": 0.6986666917800903, "metrics/particle/momentum/class_4/event/energy_bias": -0.5093540549278259, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9762914180755615, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5280205011367798, "metrics/particle/momentum/class_4/event/px_bias": 0.10454431921243668, "metrics/particle/momentum/class_4/event/py_bias": -0.025905631482601166, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9560256004333496, "metrics/particle/momentum/class_4/event/pz_bias": 0.15585824847221375, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.791696071624756, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012655789963901043, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.15010344982147217, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1475103795528412, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.04024561494588852, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.19217444956302643, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.03184713423252106, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.4560906887054443, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.16558565199375153, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.0355923175811768, "metrics/particle/momentum/class_5/accepted/samples": 157.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03413916751742363, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.16398774087429047, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.16050584614276886, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05642377957701683, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.01840490847826004, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.23022966086864471, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.03680981695652008, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.5587923526763916, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.04046507552266121, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.465496301651001, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.957317054271698, "metrics/particle/momentum/class_5/event/energy_bias": -0.045635201036930084, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.6384586095809937, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2592614889144897, "metrics/particle/momentum/class_5/event/px_bias": 0.172634556889534, "metrics/particle/momentum/class_5/event/py_bias": -0.15532924234867096, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8194088935852051, "metrics/particle/momentum/class_5/event/pz_bias": 0.13492077589035034, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.7518136501312256, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.29101642966270447, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.29242977499961853, "metrics/set_slot/angular_recall_all_dr0p10": 0.9615023136138916, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8689253926277161, "metrics/set_slot/soft_count_bias_mean": 0.34057295322418213}, "step_train_time": 5.166590929031372, "step_valid_time": 55076.33906316757, "step_total_time": 55081.50565457344, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0070237914403677, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10316043326463276, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3045, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2930, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2921, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9578293559986923, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9622331691297209, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9600262123197904, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03776683087027915, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9548872180451128, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9592775041050903, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9570773263433814, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9578293559986923, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019700613701223, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09878763692619263, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2278, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2176, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2173, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9522975929978118, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9552238805970149, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9537584922200307, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04477611940298507, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9509846827133479, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9539069359086918, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9524435678281832, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9522975929978118, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0088134778660116, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1068996299520486, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1315, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1247, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1244, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.943267776096823, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9482889733840304, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9457717102768297, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05171102661596958, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9409984871406959, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9460076045627377, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9434963974213121, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.943267776096823} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_49000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_49000.json new file mode 100644 index 0000000000000000000000000000000000000000..a1a9d63b60c77828431d14215bb57a067ee5e029 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_49000.json @@ -0,0 +1 @@ +{"epoch": 2.320954907161804, "train": {"Classification_binary": 0.14985863864421844, "Cardinality": 0.12483762204647064, "Classification": 0.08002623915672302, "Regression_pt": 0.09479352086782455, "Regression_eta": 0.0010967354755848646, "Regression_sin_phi": 0.0006150118424557149, "Regression_cos_phi": 0.000553245423361659, "Regression_energy": 0.09385164827108383, "HitGrouping": 0.14314766228199005, "Auxiliary": 0.20102466642856598, "Total": 0.9004557728767395, "Total_unweighted": 0.4207950396812521}, "train_group": {}, "valid": {"Classification_binary": 0.1339059273401896, "Cardinality": 0.10556210080782573, "Classification": 0.08035214742024739, "Regression_pt": 0.09114701549212138, "Regression_eta": 0.0011261894057194393, "Regression_sin_phi": 0.0006222883239388466, "Regression_cos_phi": 0.0006082981902485093, "Regression_energy": 0.0905902087688446, "HitGrouping": 0.14652377367019653, "Auxiliary": 0.18812974294026694, "Total": 0.8581422170003256, "Total_unweighted": 0.3983520749413098, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8517434597015381, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6072902679443359, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.27680516242980957, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6202034950256348, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6245487332344055, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3263934850692749, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.8020568490028381, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5484220385551453, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.27511078119277954, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.558681845664978, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5621301531791687, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.40731167793273926, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8656045794487, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5799079537391663, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.25915297865867615, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5890988707542419, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5888798832893372, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.30381855368614197, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.856471598148346, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5857590436935425, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.26573967933654785, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.5961605310440063, "metrics/hit_grouping/calorimeter/all/object_purity": 0.5976091027259827, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.3186926543712616, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0034564551897346973, "metrics/hit_grouping/calorimeter/background_recall": 0.15610553324222565, "metrics/hit_grouping/calorimeter/completeness": 0.6739817261695862, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6750980019569397, "metrics/hit_grouping/calorimeter/energy_purity": 0.6663651466369629, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.30055761337280273, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8789823055267334, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5971224904060364, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.19805416464805603, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.6092653870582581, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6013020277023315, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.25984296202659607, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8593892455101013, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.6132042407989502, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.2779825031757355, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6242615580558777, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6254159212112427, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.32047444581985474, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8076331615447998, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5349403023719788, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3914153277873993, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5411306023597717, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5620658993721008, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.43689095973968506, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8457070589065552, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6611037850379944, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.32595622539520264, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.676042914390564, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.6902425289154053, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.35085397958755493, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8231632113456726, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6858685612678528, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.44467318058013916, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6875031590461731, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6944254636764526, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.3916623890399933, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.871414840221405, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.49701380729675293, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.18302777409553528, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.503833532333374, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.49162936210632324, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2772088944911957, "metrics/hit_grouping/calorimeter/purity": 0.6643705368041992, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7764872908592224, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08500295132398605, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.07518796622753143, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08414339274168015, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.05678476393222809, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.033834587782621384, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8135831952095032, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.24096959829330444, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10005886107683182, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.23992832005023956, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.22075031697750092, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23131254315376282, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8600066900253296, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6118608713150024, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.27746638655662537, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6230414509773254, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6260326504707336, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.327881395816803, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7400388121604919, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.39634940028190613, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5364595651626587, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5793340802192688, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5620744824409485, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7805546522140503, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.258589506149292, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5068519711494446, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5184274911880493, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.4327908456325531, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6609219908714294, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4329400062561035, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.44806385040283203, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5040494799613953, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6736940741539001, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6940749287605286, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4088156819343567, "metrics/hit_grouping/tracker/all/object_completeness": 0.4795743227005005, "metrics/hit_grouping/tracker/all/object_purity": 0.5282499194145203, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6215939521789551, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009725751355290413, "metrics/hit_grouping/tracker/background_recall": 0.09863435477018356, "metrics/hit_grouping/tracker/completeness": 0.47455906867980957, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.4448871612548828, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7264100313186646, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3908469080924988, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.5518392324447632, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5876111388206482, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5825004577636719, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6769386529922485, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42675620317459106, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.45247721672058105, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5187558531761169, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.645852267742157, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6445870399475098, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.4279531240463257, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.3578474223613739, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4155769944190979, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6779080033302307, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6967496871948242, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4301604926586151, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.49032190442085266, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5454130172729492, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6218051910400391, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6664085984230042, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.4267241358757019, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.42658188939094543, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.49419519305229187, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6508620977401733, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6955574154853821, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3778562843799591, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.4751019775867462, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5120422840118408, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.616072416305542, "metrics/hit_grouping/tracker/purity": 0.44707900285720825, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8895246982574463, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.047888386994600296, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13311447203159332, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.13017573952674866, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.062217194586992264, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6919226050376892, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.30201342701911926, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.4146234095096588, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4317438006401062, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.515745997428894, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6653041243553162, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.47395220398902893, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5380277633666992, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5977829694747925, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7160763144493103, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6115689873695374, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8312499523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7217218279838562, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8333526849746704, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6711080074310303, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.062416188418865204, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09663327783346176, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.484222412109375, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.30198562145233154, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.2103624790906906, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9142922759056091, "metrics/particle/class_1/pid_accuracy": 0.9770546555519104, "metrics/particle/class_2/efficiency": 0.6161513328552246, "metrics/particle/class_2/pid_accuracy": 0.7880514860153198, "metrics/particle/class_3/efficiency": 0.8280174136161804, "metrics/particle/class_3/pid_accuracy": 0.9817562699317932, "metrics/particle/class_4/efficiency": 0.7154308557510376, "metrics/particle/class_4/pid_accuracy": 0.5532212853431702, "metrics/particle/class_5/efficiency": 0.9501915574073792, "metrics/particle/class_5/pid_accuracy": 0.6290322542190552, "metrics/particle/count/bias_mean": -0.8736979365348816, "metrics/particle/count/mae": 3.25390625, "metrics/particle/count/prediction_mean": 69.78125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06646118313074112, "metrics/particle/event/energy_response_mean": 0.9803052544593811, "metrics/particle/event/met_abs_error": 4.876638889312744, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04828421398997307, "metrics/particle/event/scalar_pt_response_mean": 0.9979140758514404, "metrics/particle/event/vector_pt_closure": 0.04855474457144737, "metrics/particle/matched/delta_eta_abs_mean": 0.012002273462712765, "metrics/particle/matched/delta_phi_abs_mean": 0.01196278352290392, "metrics/particle/matched/delta_r_mean": 0.019058965146541595, "metrics/particle/matched/energy_relative_abs_mean": 0.12488540261983871, "metrics/particle/matched/pid_accuracy": 0.9627214074134827, "metrics/particle/matched/pt_relative_abs_mean": 0.1254097819328308, "metrics/particle/matching/duplicate_fraction": 0.04139610379934311, "metrics/particle/matching/efficiency": 0.8520170450210571, "metrics/particle/matching/f1": 0.8573176860809326, "metrics/particle/matching/purity": 0.8626847267150879, "metrics/particle/matching/target_coverage_dr0p05": 0.816845715045929, "metrics/particle/matching/target_coverage_dr0p10": 0.8850229382514954, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6897056698799133, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8800656199455261, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019113922491669655, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12393828481435776, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.12438500672578812, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.025200381875038147, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16391804814338684, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00491534685716033, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5694460272789001, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.015735147520899773, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.6536779403686523, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9155.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.16775210201740265, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.6296170353889465, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7148191928863525, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.5937286615371704, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.0925481915473938, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 6.674936294555664, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.008919152431190014, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7550724148750305, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0047828624956309795, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.1746346950531006, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10427.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8498886227607727, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.5508973002433777, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.847092628479004, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.746701240539551, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.11005069315433502, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.04348006844520569, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.899807929992676, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.9508668780326843, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.510517120361328, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022886235266923904, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13748271763324738, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13884882628917694, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.024334900081157684, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.17945928871631622, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019849784672260284, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1702747344970703, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.02971414104104042, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.889816999435425, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1864.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4668421149253845, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6966787576675415, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.214217185974121, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6.059423446655273, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.1755186766386032, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 212.32000732421875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05186722055077553, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.341104745864868, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2665046751499176, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.887037754058838, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2410.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6758520603179932, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.636985778808594, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.697763919830322, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9672658443450928, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.028824346140027046, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.059779487550258636, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.580219745635986, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.6238572597503662, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.7847900390625, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018959002569317818, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12526097893714905, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12603890895843506, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01839533820748329, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.16499337553977966, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004425866645760834, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.19569525122642517, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004371238872408867, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6381685733795166, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20335.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15505282580852509, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7700892090797424, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6074305772781372, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.47191306948661804, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08753929287195206, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.682205677032471, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00275061116553843, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.31675899028778076, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.014177277684211731, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3419734239578247, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22904.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8739094734191895, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.4209861755371094, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.892323970794678, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.64481782913208, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.1008090227842331, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.5551794767379761, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.538796901702881, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.25962501764297485, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.816774368286133, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019237710162997246, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12560135126113892, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12631753087043762, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02073543891310692, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16557708382606506, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0029023410752415657, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.36276495456695557, "metrics/particle/momentum/all/accepted/pz_bias": -0.003525971435010433, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.248634934425354, "metrics/particle/momentum/all/accepted/samples": 31354.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.1797814965248108, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7915877103805542, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.0168230533599854, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8842147588729858, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.09493298828601837, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 55.43824768066406, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.007862119004130363, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5811322927474976, "metrics/particle/momentum/all/assigned/pz_bias": -0.010280369780957699, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.5938754081726074, "metrics/particle/momentum/all/assigned/samples": 35741.0, "metrics/particle/momentum/all/efficiency": 0.8520340323448181, "metrics/particle/momentum/all/event/energy_bias": -2.665102005004883, "metrics/particle/momentum/all/event/p3_error_mean": 16.0268497467041, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.929880142211914, "metrics/particle/momentum/all/event/px_bias": -0.1820354014635086, "metrics/particle/momentum/all/event/py_bias": 0.6584389805793762, "metrics/particle/momentum/all/event/pz_abs_mean": 11.708077430725098, "metrics/particle/momentum/all/event/pz_bias": -0.41336557269096375, "metrics/particle/momentum/all/event/pz_residual_rms": 23.485563278198242, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.019976265728473663, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10631438344717026, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10774538666009903, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.028365550562739372, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14430280029773712, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.002030394272878766, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.3718351721763611, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.003665980650112033, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.1222363710403442, "metrics/particle/momentum/class_1/accepted/samples": 16253.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10390686243772507, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.26907458901405334, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2831512987613678, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.18034586310386658, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.05147857964038849, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.3660635948181152, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004518675152212381, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5009000897407532, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.026845695450901985, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3098978996276855, "metrics/particle/momentum/class_1/assigned/samples": 17483.0, "metrics/particle/momentum/class_1/efficiency": 0.9161264896392822, "metrics/particle/momentum/class_1/event/energy_bias": 4.217362403869629, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.269465446472168, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.944455146789551, "metrics/particle/momentum/class_1/event/px_bias": -0.1059931144118309, "metrics/particle/momentum/class_1/event/py_bias": 0.5533804893493652, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.880731582641602, "metrics/particle/momentum/class_1/event/pz_bias": 0.604041337966919, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.68251895904541, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.02517939731478691, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.17483694851398468, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.17455455660820007, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.013573815114796162, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21581174433231354, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.022676991298794746, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.253461480140686, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.06172946095466614, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.183089017868042, "metrics/particle/momentum/class_2/accepted/samples": 1808.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.49302613735198975, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.47660231590270996, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6360971927642822, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.349032998085022, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2212320864200592, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.8946712017059326, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.04378148168325424, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.017092227935791, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.006395983509719372, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.060385704040527, "metrics/particle/momentum/class_2/assigned/samples": 2581.0, "metrics/particle/momentum/class_2/efficiency": 0.6054922938346863, "metrics/particle/momentum/class_2/event/energy_bias": -7.842075347900391, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.719076156616211, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.155341148376465, "metrics/particle/momentum/class_2/event/px_bias": 0.2612515389919281, "metrics/particle/momentum/class_2/event/py_bias": -0.18916933238506317, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.502317428588867, "metrics/particle/momentum/class_2/event/pz_bias": -0.6439883708953857, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.24123191833496, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01751813478767872, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14197391271591187, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14201949536800385, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.010283173061907291, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1804898977279663, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010093951132148504, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21074649691581726, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.002693384187296033, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8111048340797424, "metrics/particle/momentum/class_3/accepted/samples": 12879.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.2101905643939972, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4096969366073608, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.909392237663269, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.767457127571106, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1208501085639, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 85.07328796386719, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005478186067193747, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4195624589920044, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.00671965628862381, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.2806220054626465, "metrics/particle/momentum/class_3/assigned/samples": 15151.0, "metrics/particle/momentum/class_3/efficiency": 0.8291379809379578, "metrics/particle/momentum/class_3/event/energy_bias": 1.5062198638916016, "metrics/particle/momentum/class_3/event/p3_error_mean": 7.934869766235352, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.12630558013916, "metrics/particle/momentum/class_3/event/px_bias": -0.40380632877349854, "metrics/particle/momentum/class_3/event/py_bias": 0.3358609676361084, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.841998100280762, "metrics/particle/momentum/class_3/event/pz_bias": -0.18342901766300201, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.595206260681152, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.020962925627827644, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16723118722438812, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1650991290807724, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0874214619398117, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2151959091424942, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0038610037881881, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5779129266738892, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.017337951809167862, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3215140104293823, "metrics/particle/momentum/class_4/accepted/samples": 259.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3953525424003601, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.680030107498169, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.1881449222564697, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.094726324081421, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.23415978252887726, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.086783409118652, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005509641952812672, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6051483750343323, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.014699462801218033, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3765227794647217, "metrics/particle/momentum/class_4/assigned/samples": 363.0, "metrics/particle/momentum/class_4/efficiency": 0.690666675567627, "metrics/particle/momentum/class_4/event/energy_bias": -0.5196682214736938, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.544859528541565, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1594769954681396, "metrics/particle/momentum/class_4/event/px_bias": -0.04375235736370087, "metrics/particle/momentum/class_4/event/py_bias": -0.007958058267831802, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8197170495986938, "metrics/particle/momentum/class_4/event/pz_bias": -0.256083607673645, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.81135630607605, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012484628707170486, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.143722802400589, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1416134089231491, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.06124591827392578, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.18661275506019592, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.019354838877916336, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2938990592956543, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.19638186693191528, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 2.7478644847869873, "metrics/particle/momentum/class_5/accepted/samples": 155.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05126754194498062, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1633264720439911, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.16363759338855743, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05983396992087364, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.03680981695652008, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.2305336594581604, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.024539876729249954, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.4137290716171265, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.06917325407266617, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.1376421451568604, "metrics/particle/momentum/class_5/assigned/samples": 163.0, "metrics/particle/momentum/class_5/efficiency": 0.9451219439506531, "metrics/particle/momentum/class_5/event/energy_bias": -0.026940228417515755, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5812642574310303, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2028963565826416, "metrics/particle/momentum/class_5/event/px_bias": 0.11026490479707718, "metrics/particle/momentum/class_5/event/py_bias": -0.033675096929073334, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7897716760635376, "metrics/particle/momentum/class_5/event/pz_bias": 0.06609407067298889, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.6510798931121826, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.2909826338291168, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.29271194338798523, "metrics/set_slot/angular_recall_all_dr0p10": 0.9635847806930542, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8698099851608276, "metrics/set_slot/soft_count_bias_mean": 0.15481002628803253}, "step_train_time": 5.7561469078063965, "step_valid_time": 57980.04713153839, "step_total_time": 57985.803278923035, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0086002563142618, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1000296886586245, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2940, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2929, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9610983981693364, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9610983981693364, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9610983981693364, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038901601830663615, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.957502451781628, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.957502451781628, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.957502451781628, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9610983981693364, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001477864799938, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0993773674063777, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2280, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2172, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2167, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9505470459518599, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526315789473684, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9515881708652792, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04736842105263158, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9483588621444201, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9504385964912281, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9493975903614458, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9505470459518599, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0068855317287455, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10304790398652064, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1311, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1240, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1239, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9379727685325264, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9458428680396643, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.941891378655526, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05415713196033562, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9372163388804842, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9450800915331807, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9411317888340296, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9379727685325264} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_5000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..a3e0af4f92c55b28eb0d9de05b4817e3e2b1a14f --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 0.23683213338385753, "train": {"Classification_binary": 0.2940808832645416, "Cardinality": 0.21541538834571838, "Classification": 0.24616467952728271, "Regression_pt": 0.3218519389629364, "Regression_eta": 0.004180764313787222, "Regression_sin_phi": 0.0019490965642035007, "Regression_cos_phi": 0.0020758728496730328, "Regression_energy": 0.3168503940105438, "HitGrouping": 0.22405393421649933, "Auxiliary": 0.3915833532810211, "Total": 2.082736015319824, "Total_unweighted": 1.1871536294929683}, "train_group": {}, "valid": {"Classification_binary": 0.33489028612772626, "Cardinality": 0.27481192350387573, "Classification": 0.23477381467819214, "Regression_pt": 0.3355843226114909, "Regression_eta": 0.004316498525440693, "Regression_sin_phi": 0.002247108922650417, "Regression_cos_phi": 0.002199360945572456, "Regression_energy": 0.32957526048024494, "HitGrouping": 0.2319511373837789, "Auxiliary": 0.4163423379262288, "Total": 2.2620910008748374, "Total_unweighted": 1.2435866522913177, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8187731504440308, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3431074619293213, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.37872621417045593, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.34546926617622375, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3227728307247162, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.4311535954475403, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7771860957145691, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.36242416501045227, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.37813884019851685, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.3655109107494354, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3246733546257019, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4826440215110779, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8294519186019897, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.30792856216430664, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3515852689743042, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3090375065803528, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.28927329182624817, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.414811372756958, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.822158932685852, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.3228170573711395, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.36186927556991577, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3244590759277344, "metrics/hit_grouping/calorimeter/all/object_purity": 0.30217045545578003, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4250286817550659, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0006875061662867665, "metrics/hit_grouping/calorimeter/background_recall": 0.029090089723467827, "metrics/hit_grouping/calorimeter/completeness": 0.41830191016197205, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4312748610973358, "metrics/hit_grouping/calorimeter/energy_purity": 0.42521169781684875, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4988205134868622, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8534747958183289, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.34170791506767273, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2909080684185028, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.3451102674007416, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.31731870770454407, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3566226661205292, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8144455552101135, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.33637410402297974, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.38254261016845703, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.3380511999130249, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.32314035296440125, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4480654001235962, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7660771012306213, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.2706412076950073, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4857308566570282, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2686586380004883, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2501579821109772, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5412992835044861, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7844735980033875, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.3557763397693634, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4452129006385803, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.359157532453537, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.3487074077129364, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5054125785827637, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7640271186828613, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.48859044909477234, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6525990962982178, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4890574514865875, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.45519527792930603, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5156973600387573, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8675071001052856, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2694633901119232, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24225887656211853, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.26947224140167236, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.2364923655986786, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3322705328464508, "metrics/hit_grouping/calorimeter/purity": 0.4111942648887634, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.9048491716384888, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.04164239764213562, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.10338345915079117, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.04151002690196037, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.008863111957907677, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.033834587782621384, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8915200233459473, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.07856951653957367, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.08946438878774643, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.07754630595445633, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.05169219523668289, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.21895232796669006, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.817202091217041, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.34010887145996094, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.38025668263435364, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.3419185280799866, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3199842572212219, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.4421268701553345, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6427654027938843, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4637114405632019, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.33049362897872925, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.3337048888206482, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7243227362632751, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7277907729148865, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.3701024651527405, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.357033371925354, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.3093460500240326, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.517179012298584, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5483254790306091, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4510001540184021, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.23588927090168, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.25115033984184265, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.8054396510124207, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.5906073451042175, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44893142580986023, "metrics/hit_grouping/tracker/all/object_completeness": 0.2739138603210449, "metrics/hit_grouping/tracker/all/object_purity": 0.2808218002319336, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7592163681983948, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.015589263290166855, "metrics/hit_grouping/tracker/background_recall": 0.07850489765405655, "metrics/hit_grouping/tracker/completeness": 0.2800234854221344, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5745678544044495, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6035525798797607, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.43976855278015137, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.3272680342197418, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3381476402282715, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7619673609733582, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.5815812945365906, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.45489200949668884, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.24695566296577454, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.2560897469520569, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7638409733772278, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.5729467868804932, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.46185752749443054, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.19094811379909515, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.18747468292713165, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7489630579948425, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.595533013343811, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.48289036750793457, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.2775533199310303, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.2876836359500885, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7611445784568787, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.5902612805366516, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.546798050403595, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3063444197177887, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3306920826435089, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7426108121871948, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5842658877372742, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3872392177581787, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.2633685767650604, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2629609704017639, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7596864700317383, "metrics/hit_grouping/tracker/purity": 0.2653796672821045, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8739244937896729, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01847662217915058, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03667445480823517, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.03855026513338089, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06674208492040634, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6333552598953247, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.22973670065402985, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.15338850021362305, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.13255174458026886, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5937016010284424, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5439306497573853, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5365676283836365, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.32218292355537415, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.3328447639942169, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8799172043800354, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.47715651988983154, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.34050118923187256, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.36438509821891785, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5733339190483093, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.07199627161026001, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": -0.0011408464051783085, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.413408279418945, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.1676901876926422, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.07606703042984009, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.5936987400054932, "metrics/particle/class_1/pid_accuracy": 0.9028910994529724, "metrics/particle/class_2/efficiency": 0.43175897002220154, "metrics/particle/class_2/pid_accuracy": 0.31558236479759216, "metrics/particle/class_3/efficiency": 0.5144393444061279, "metrics/particle/class_3/pid_accuracy": 0.903110682964325, "metrics/particle/class_4/efficiency": 0.5020040273666382, "metrics/particle/class_4/pid_accuracy": 0.2894211709499359, "metrics/particle/class_5/efficiency": 0.6436781883239746, "metrics/particle/class_5/pid_accuracy": 0.2410714328289032, "metrics/particle/count/bias_mean": -2.93359375, "metrics/particle/count/mae": 7.145833492279053, "metrics/particle/count/prediction_mean": 67.72135162353516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.16413964331150055, "metrics/particle/event/energy_response_mean": 1.0866234302520752, "metrics/particle/event/met_abs_error": 14.93955135345459, "metrics/particle/event/scalar_pt_relative_abs_error": 0.16631609201431274, "metrics/particle/event/scalar_pt_response_mean": 1.1271506547927856, "metrics/particle/event/vector_pt_closure": 0.11029653996229172, "metrics/particle/matched/delta_eta_abs_mean": 0.02065923437476158, "metrics/particle/matched/delta_phi_abs_mean": 0.02001408487558365, "metrics/particle/matched/delta_r_mean": 0.03216836228966713, "metrics/particle/matched/energy_relative_abs_mean": 0.19197258353233337, "metrics/particle/matched/pid_accuracy": 0.8562998175621033, "metrics/particle/matched/pt_relative_abs_mean": 0.1899608075618744, "metrics/particle/matching/duplicate_fraction": 0.13444529473781586, "metrics/particle/matching/efficiency": 0.5465142130851746, "metrics/particle/matching/f1": 0.5581003427505493, "metrics/particle/matching/purity": 0.5701884031295776, "metrics/particle/matching/target_coverage_dr0p05": 0.49966827034950256, "metrics/particle/matching/target_coverage_dr0p10": 0.6460018754005432, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.3457973301410675, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6131618022918701, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.030365534126758575, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.18919537961483002, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1895253211259842, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01567418873310089, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.23186077177524567, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.013038942590355873, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.1450968980789185, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.20045624673366547, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.638692855834961, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5752.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.6080878973007202, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.546130895614624, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 3.474118709564209, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 3.308807134628296, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.3384374976158142, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.189453125, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.03187499940395355, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.8589529991149902, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.22863779962062836, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.431529998779297, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9600.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5339769721031189, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 12.068314552307129, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 17.26609992980957, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.9502458572387695, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.9443032145500183, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.10699462890625, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 14.686552047729492, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -7.220643997192383, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 23.832841873168945, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02873057685792446, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.196332186460495, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1962331384420395, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03172798454761505, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.23753298819065094, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.039232783019542694, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.9168657064437866, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.38557857275009155, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.115809917449951, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1147.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8640434145927429, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.665356397628784, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 9.006914138793945, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 8.854585647583008, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.39140698313713074, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 90.24524688720703, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08687441051006317, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.6973769664764404, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.6208714246749878, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.479254722595215, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2118.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.41588106751441956, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.582063674926758, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 12.126646041870117, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.8679255247116089, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.011667490005493164, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.02100794017314911, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.913054466247559, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.5352911949157715, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 30.353878021240234, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.033277131617069244, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19455668330192566, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.19081541895866394, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0012644121889024973, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2315424531698227, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0015202189097180963, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.4093674123287201, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.057915009558200836, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0621618032455444, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 13156.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.5528688430786133, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 3.466094970703125, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.953584671020508, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 2.77958345413208, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.3385113775730133, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 14.457197189331055, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.011850339360535145, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.9715309143066406, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.09211492538452148, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.1030027866363525, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22531.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5653874278068542, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 23.570140838623047, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 19.56292152404785, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 16.386577606201172, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 5.586606502532959, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 3.5663108825683594, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.842667579650879, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -2.7395288944244385, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 13.568676948547363, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03218202292919159, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.1931205689907074, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.19075524806976318, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.007139592431485653, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.23198044300079346, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0069808028638362885, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7066009044647217, "metrics/particle/momentum/all/accepted/pz_bias": -0.11753740161657333, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.9250729084014893, "metrics/particle/momentum/all/accepted/samples": 20055.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.5875901579856873, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 3.158710241317749, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3.4738357067108154, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3.303610324859619, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.3417617976665497, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 27.090328216552734, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.022102834656834602, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.3888450860977173, "metrics/particle/momentum/all/assigned/pz_bias": -0.16308125853538513, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.316250801086426, "metrics/particle/momentum/all/assigned/samples": 34249.0, "metrics/particle/momentum/all/efficiency": 0.5449876189231873, "metrics/particle/momentum/all/event/energy_bias": 31.0563907623291, "metrics/particle/momentum/all/event/p3_error_mean": 36.06673049926758, "metrics/particle/momentum/all/event/pt_vector_error_mean": 19.416500091552734, "metrics/particle/momentum/all/event/px_bias": 6.519242286682129, "metrics/particle/momentum/all/event/py_bias": 4.652297496795654, "metrics/particle/momentum/all/event/pz_abs_mean": 25.69801139831543, "metrics/particle/momentum/all/event/pz_bias": -12.495463371276855, "metrics/particle/momentum/all/event/pz_residual_rms": 39.249263763427734, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.037267159670591354, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.20341691374778748, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.20407386124134064, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 6.369209586409852e-05, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.24437768757343292, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.006580202374607325, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7931451201438904, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.12818531692028046, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.9400017261505127, "metrics/particle/momentum/class_1/accepted/samples": 10486.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.45305129885673523, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.6974749565124512, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 2.051548719406128, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.8538827896118164, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.2954274117946625, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 12.75670337677002, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.018803725019097328, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.2942521572113037, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.16020475327968597, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.5559518337249756, "metrics/particle/momentum/class_1/assigned/samples": 16752.0, "metrics/particle/momentum/class_1/efficiency": 0.5910602807998657, "metrics/particle/momentum/class_1/event/energy_bias": 42.30716323852539, "metrics/particle/momentum/class_1/event/p3_error_mean": 31.420671463012695, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 18.855640411376953, "metrics/particle/momentum/class_1/event/px_bias": 6.791880130767822, "metrics/particle/momentum/class_1/event/py_bias": 5.417478084564209, "metrics/particle/momentum/class_1/event/pz_abs_mean": 20.969755172729492, "metrics/particle/momentum/class_1/event/pz_bias": -9.821365356445312, "metrics/particle/momentum/class_1/event/pz_residual_rms": 30.09855842590332, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03907284513115883, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22663292288780212, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2247881442308426, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06758935004472733, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.261811763048172, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.034029390662908554, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7292673587799072, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.32259297370910645, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.848144292831421, "metrics/particle/momentum/class_2/accepted/samples": 1293.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6380054950714111, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8292567729949951, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.6277084350585938, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 1.2924385070800781, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.323817640542984, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 18.75685691833496, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05794631317257881, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.657165050506592, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.4678695499897003, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.628505229949951, "metrics/particle/momentum/class_2/assigned/samples": 2347.0, "metrics/particle/momentum/class_2/efficiency": 0.43302077054977417, "metrics/particle/momentum/class_2/event/energy_bias": -17.575990676879883, "metrics/particle/momentum/class_2/event/p3_error_mean": 23.487369537353516, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 12.961238861083984, "metrics/particle/momentum/class_2/event/px_bias": -0.6173151135444641, "metrics/particle/momentum/class_2/event/py_bias": -1.929636001586914, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.40146255493164, "metrics/particle/momentum/class_2/event/pz_bias": -0.5094336867332458, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.225605010986328, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.024360356852412224, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17415721714496613, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16790346801280975, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.030010854825377464, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2089913934469223, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0028818442951887846, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.4035842716693878, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0725107192993164, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3037031888961792, "metrics/particle/momentum/class_3/accepted/samples": 7981.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.7354332208633423, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 5.112792491912842, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 5.336933612823486, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 5.225602149963379, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3963797688484192, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 38.24146270751953, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01974043808877468, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.2701526880264282, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.11110089719295502, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.594385147094727, "metrics/particle/momentum/class_3/assigned/samples": 14640.0, "metrics/particle/momentum/class_3/efficiency": 0.513809323310852, "metrics/particle/momentum/class_3/event/energy_bias": 8.34130573272705, "metrics/particle/momentum/class_3/event/p3_error_mean": 18.15180206298828, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 10.593011856079102, "metrics/particle/momentum/class_3/event/px_bias": -0.06661508977413177, "metrics/particle/momentum/class_3/event/py_bias": 1.1509487628936768, "metrics/particle/momentum/class_3/event/pz_abs_mean": 12.419286727905273, "metrics/particle/momentum/class_3/event/pz_bias": -1.7554696798324585, "metrics/particle/momentum/class_3/event/pz_residual_rms": 23.277910232543945, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.03316683694720268, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18148861825466156, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17700952291488647, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.0014067006995901465, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22235560417175293, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.145933985710144, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.3479911983013153, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.079026699066162, "metrics/particle/momentum/class_4/accepted/samples": 182.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.6713728904724121, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 8.004096984863281, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 7.134549617767334, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 6.996637344360352, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.4378530979156494, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.98554229736328, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011299435049295425, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.4300259351730347, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.1612023115158081, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.0728039741516113, "metrics/particle/momentum/class_4/assigned/samples": 354.0, "metrics/particle/momentum/class_4/efficiency": 0.48533332347869873, "metrics/particle/momentum/class_4/event/energy_bias": -0.4240210950374603, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.4845290184020996, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.574411392211914, "metrics/particle/momentum/class_4/event/px_bias": 0.2501910924911499, "metrics/particle/momentum/class_4/event/py_bias": 0.2037705034017563, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8877406120300293, "metrics/particle/momentum/class_4/event/pz_bias": -0.11653870344161987, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.56151819229126, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.03229622170329094, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.21227174997329712, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.20153813064098358, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08275149017572403, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.24553915858268738, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.02654867246747017, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.6677148342132568, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.407911092042923, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.466939926147461, "metrics/particle/momentum/class_5/accepted/samples": 113.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.21187283098697662, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.7408949732780457, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.8287307620048523, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4880668818950653, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.24358974397182465, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 3.793473243713379, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.0833333358168602, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.510343074798584, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.7688952088356018, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 8.299067497253418, "metrics/particle/momentum/class_5/assigned/samples": 156.0, "metrics/particle/momentum/class_5/efficiency": 0.6890243887901306, "metrics/particle/momentum/class_5/event/energy_bias": -1.5920641422271729, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.44889235496521, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.551762819290161, "metrics/particle/momentum/class_5/event/px_bias": 0.16110186278820038, "metrics/particle/momentum/class_5/event/py_bias": -0.1902635097503662, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.7956634759902954, "metrics/particle/momentum/class_5/event/pz_bias": -0.29265686869621277, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.147158145904541, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6301315426826477, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6341782808303833, "metrics/set_slot/angular_recall_all_dr0p10": 0.8435858488082886, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6656838059425354, "metrics/set_slot/soft_count_bias_mean": 3.797346830368042}, "step_train_time": 4.9668848514556885, "step_valid_time": 14849.771719455719, "step_total_time": 14854.73860502243, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1798419185753553, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.3054524968274648, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3109, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2725, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2461, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8908139915004903, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8764876165969765, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8835927367055771, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12351238340302348, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8045112781954887, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7915728530073979, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7979896238651103, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8908139915004903, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.136708420033735, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26015993109268587, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2313, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2044, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1913, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8945295404814004, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8837008214440121, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8890822096563723, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1162991785559879, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8371991247264771, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8270644185041072, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8321009134406263, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8945295404814004, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.1436935514976545, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2903988763020049, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1318, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1130, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1060, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8547655068078669, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8573596358118362, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8560606060606061, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1426403641881639, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8018154311649016, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8042488619119879, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.803030303030303, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8547655068078669} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_50000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..2981b6819a4ae1a1f67bc030a2139bc6dc9a20a3 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 2.3683213338385753, "train": {"Classification_binary": 0.1435994654893875, "Cardinality": 0.11128018796443939, "Classification": 0.08175532519817352, "Regression_pt": 0.08876355737447739, "Regression_eta": 0.0013643899001181126, "Regression_sin_phi": 0.0006177339819259942, "Regression_cos_phi": 0.000707208935637027, "Regression_energy": 0.08721043914556503, "HitGrouping": 0.14237156510353088, "Auxiliary": 0.19206266105175018, "Total": 0.8856877088546753, "Total_unweighted": 0.4040181200252846}, "train_group": {}, "valid": {"Classification_binary": 0.13145244121551514, "Cardinality": 0.10550551613171895, "Classification": 0.07977477709452312, "Regression_pt": 0.09056104222933452, "Regression_eta": 0.001147797719265024, "Regression_sin_phi": 0.0006417882783959309, "Regression_cos_phi": 0.0006231583732490739, "Regression_energy": 0.08982298771540324, "HitGrouping": 0.1455767552057902, "Auxiliary": 0.18457706769307455, "Total": 0.8534990946451823, "Total_unweighted": 0.39402399262568605, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8500478267669678, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.6153002381324768, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.2751017212867737, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.6278479695320129, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.6326109170913696, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3253525197505951, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7957455515861511, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.5528420209884644, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.27732643485069275, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.5630117058753967, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.5751407742500305, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4143279194831848, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8651796579360962, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.5830251574516296, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.25197869539260864, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.5927042961120605, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.5963061451911926, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.2991437315940857, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.855204701423645, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.5904596447944641, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.26095184683799744, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.6010459065437317, "metrics/hit_grouping/calorimeter/all/object_purity": 0.6056615710258484, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.31602638959884644, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.004120803903788328, "metrics/hit_grouping/calorimeter/background_recall": 0.17821907997131348, "metrics/hit_grouping/calorimeter/completeness": 0.67548006772995, "metrics/hit_grouping/calorimeter/energy_completeness": 0.6747704744338989, "metrics/hit_grouping/calorimeter/energy_purity": 0.6662212014198303, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.29522445797920227, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8749905824661255, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.5984744429588318, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.19896449148654938, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.610781192779541, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.6037306785583496, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.25921711325645447, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8611997365951538, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.621256947517395, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.27107325196266174, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.6327037215232849, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.6382431387901306, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.3096499443054199, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.8088222146034241, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.5430917739868164, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.3771461844444275, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.5493030548095703, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.5767751336097717, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.4382830560207367, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8452455401420593, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.6685317158699036, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.31278565526008606, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.6836185455322266, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.7050227522850037, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.3518763482570648, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.8227930665016174, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.6762802004814148, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.42717447876930237, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.6780797839164734, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.6937330365180969, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.39732372760772705, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8692201375961304, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.5006361603736877, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.1883009374141693, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.5076806545257568, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.49386441707611084, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.2697896957397461, "metrics/hit_grouping/calorimeter/purity": 0.6662917137145996, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.7664808630943298, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.08826953172683716, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.0714285746216774, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.08750894665718079, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.07030624896287918, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.02631578966975212, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.7939803600311279, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.23160964250564575, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.09535020589828491, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.22933624684810638, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.22194181382656097, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.23131254315376282, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8598363995552063, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.6173166036605835, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.2726576626300812, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.6287568807601929, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.6343531012535095, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.32515543699264526, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.7413046360015869, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.39620456099510193, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.5415322184562683, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.5818454623222351, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.5632333755493164, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7829362750053406, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.2567811906337738, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.5082246661186218, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.5213147401809692, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.43158528208732605, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.6573231816291809, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4284069538116455, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.4468746483325958, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.5054700374603271, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.6798100471496582, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6924129724502563, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.40583258867263794, "metrics/hit_grouping/tracker/all/object_completeness": 0.48049893975257874, "metrics/hit_grouping/tracker/all/object_purity": 0.5301141142845154, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.6256455779075623, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.009876641444861889, "metrics/hit_grouping/tracker/background_recall": 0.10688348859548569, "metrics/hit_grouping/tracker/completeness": 0.4762161374092102, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.44368627667427063, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.7281585931777954, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.3838330805301666, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.553240180015564, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.5850445032119751, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.5823251008987427, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6739640235900879, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.42657470703125, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.4550275504589081, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.5253058671951294, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.6547467708587646, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.6372137069702148, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.43047791719436646, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.35617712140083313, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.4218999743461609, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.6858431100845337, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6928059458732605, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.42718857526779175, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.4923897385597229, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.5530667304992676, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.6264753341674805, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6693791747093201, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.44519704580307007, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.4311869144439697, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.4863920211791992, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.6545566320419312, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.696031391620636, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.37101224064826965, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.47388070821762085, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.5081129670143127, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.6193840503692627, "metrics/hit_grouping/tracker/purity": 0.44870853424072266, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8890920281410217, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.044871795922517776, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.13048844039440155, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.1314883828163147, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.059200603514909744, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6962935924530029, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2989158630371094, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.42232298851013184, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.4352002739906311, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.4992256164550781, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.6628057956695557, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.4709865152835846, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.5387449264526367, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.5995564460754395, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.7234066128730774, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.6141595840454102, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.7291848659515381, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.8421127200126648, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.6652998328208923, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": 0.0650068074464798, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.09082505106925964, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 22.19678497314453, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.3110201954841614, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.21939703822135925, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.9153984189033508, "metrics/particle/class_1/pid_accuracy": 0.9782074093818665, "metrics/particle/class_2/efficiency": 0.6009740829467773, "metrics/particle/class_2/pid_accuracy": 0.7840180993080139, "metrics/particle/class_3/efficiency": 0.8358218669891357, "metrics/particle/class_3/pid_accuracy": 0.9830513000488281, "metrics/particle/class_4/efficiency": 0.711422860622406, "metrics/particle/class_4/pid_accuracy": 0.5464788675308228, "metrics/particle/class_5/efficiency": 0.938697338104248, "metrics/particle/class_5/pid_accuracy": 0.6081632375717163, "metrics/particle/count/bias_mean": -1.2936197519302368, "metrics/particle/count/mae": 3.2845051288604736, "metrics/particle/count/prediction_mean": 69.361328125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.06432946026325226, "metrics/particle/event/energy_response_mean": 0.9760835766792297, "metrics/particle/event/met_abs_error": 4.790363788604736, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046838611364364624, "metrics/particle/event/scalar_pt_response_mean": 0.9934223294258118, "metrics/particle/event/vector_pt_closure": 0.046584803611040115, "metrics/particle/matched/delta_eta_abs_mean": 0.011888275854289532, "metrics/particle/matched/delta_phi_abs_mean": 0.011838835664093494, "metrics/particle/matched/delta_r_mean": 0.0188346765935421, "metrics/particle/matched/energy_relative_abs_mean": 0.12227761745452881, "metrics/particle/matched/pid_accuracy": 0.963833212852478, "metrics/particle/matched/pt_relative_abs_mean": 0.12283195555210114, "metrics/particle/matching/duplicate_fraction": 0.03913120925426483, "metrics/particle/matching/efficiency": 0.8545141220092773, "metrics/particle/matching/f1": 0.8624090552330017, "metrics/particle/matching/purity": 0.8704512119293213, "metrics/particle/matching/target_coverage_dr0p05": 0.820632815361023, "metrics/particle/matching/target_coverage_dr0p10": 0.8860457539558411, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.696312427520752, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8805539608001709, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019043680280447006, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.12089261412620544, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1216587945818901, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013709912076592445, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.16102184355258942, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004878048785030842, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.5686051845550537, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.024513600394129753, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.705468773841858, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 9225.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.1638886034488678, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.5695551633834839, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.6086714863777161, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.4785631597042084, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.08479475975036621, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.488346576690674, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0092503372579813, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7691444158554077, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.01728319190442562, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4662532806396484, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10378.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8563868999481201, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.047479450702667236, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.921010494232178, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.65971302986145, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.03801668435335159, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.026961971074342728, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.079813003540039, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.1581388711929321, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.02735710144043, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023262569680809975, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.13798268139362335, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.13797540962696075, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.004051737952977419, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1782909482717514, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023609653115272522, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1957893371582031, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0981164500117302, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.223755121231079, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1906.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.42126503586769104, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 0.929811954498291, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.116304874420166, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.938408613204956, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.15499378740787506, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 35.45561981201172, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.05511811003088951, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.2765488624572754, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.2743796706199646, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.3970746994018555, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2413.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6910805106163025, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.8828325271606445, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.810495376586914, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0012205839157104, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.028380826115608215, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04277532920241356, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.689719200134277, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.406324863433838, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.608430862426758, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01856318861246109, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.12222365289926529, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.12318237870931625, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.018901748582720757, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.162155881524086, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00034467477235011756, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1908985823392868, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.010576624423265457, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5919722318649292, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 20309.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14776673913002014, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8188784122467041, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.63027423620224, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5007113814353943, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08368439227342606, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.864027976989746, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002369668334722519, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.30292579531669617, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.01241071242839098, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1833809614181519, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22788.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.87279212474823, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.358428955078125, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.30302095413208, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.1845221519470215, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.4047703742980957, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.6606481075286865, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.1844406127929688, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.18676140904426575, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.519839763641357, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018989063799381256, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.12278848141431808, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.12363214045763016, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016478123143315315, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.16284963488578796, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0030852416530251503, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.36264368891716003, "metrics/particle/momentum/all/accepted/pz_bias": 0.008076612837612629, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.3076070547103882, "metrics/particle/momentum/all/accepted/samples": 31440.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.17101821303367615, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7536771893501282, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7247569561004639, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5917569398880005, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.08884454518556595, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.773503303527832, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.007954129949212074, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5727695226669312, "metrics/particle/momentum/all/assigned/pz_bias": -0.0056184218265116215, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.526270627975464, "metrics/particle/momentum/all/assigned/samples": 35579.0, "metrics/particle/momentum/all/efficiency": 0.8543710708618164, "metrics/particle/momentum/all/event/energy_bias": -3.4769248962402344, "metrics/particle/momentum/all/event/p3_error_mean": 15.765058517456055, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.543038368225098, "metrics/particle/momentum/all/event/px_bias": 0.3383728861808777, "metrics/particle/momentum/all/event/py_bias": -0.6448346376419067, "metrics/particle/momentum/all/event/pz_abs_mean": 11.62942886352539, "metrics/particle/momentum/all/event/pz_bias": -0.061424389481544495, "metrics/particle/momentum/all/event/pz_residual_rms": 23.296525955200195, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.02015914022922516, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.10317730158567429, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.10422690957784653, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.019408604130148888, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.14027471840381622, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0024020695127546787, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.37601420283317566, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.011421584524214268, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.2311547994613647, "metrics/particle/momentum/class_1/accepted/samples": 16236.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.10117712616920471, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.24223287403583527, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.27087074518203735, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.16393756866455078, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.04697447642683983, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.655748128890991, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004645827226340771, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.48482242226600647, "metrics/particle/momentum/class_1/assigned/pz_bias": 4.658788020606153e-05, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.0933749675750732, "metrics/particle/momentum/class_1/assigned/samples": 17435.0, "metrics/particle/momentum/class_1/efficiency": 0.9151682257652283, "metrics/particle/momentum/class_1/event/energy_bias": 3.0608370304107666, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.770005226135254, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.542867660522461, "metrics/particle/momentum/class_1/event/px_bias": 0.5379383563995361, "metrics/particle/momentum/class_1/event/py_bias": 0.3253701329231262, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.632577896118164, "metrics/particle/momentum/class_1/event/pz_bias": -0.14600470662117004, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.443170547485352, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.025264348834753036, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.16833920776844025, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.16959787905216217, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": 0.02238314412534237, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.21232999861240387, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.024089636281132698, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.2682093381881714, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.02000637911260128, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.302757978439331, "metrics/particle/momentum/class_2/accepted/samples": 1785.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.477618545293808, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.5077731609344482, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6755485534667969, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4020373821258545, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.21289294958114624, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6.262665271759033, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0437723845243454, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.0302369594573975, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.03631645068526268, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.099668979644775, "metrics/particle/momentum/class_2/assigned/samples": 2513.0, "metrics/particle/momentum/class_2/efficiency": 0.5977897047996521, "metrics/particle/momentum/class_2/event/energy_bias": -7.701416969299316, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.995059967041016, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 5.971457004547119, "metrics/particle/momentum/class_2/event/px_bias": -0.20233580470085144, "metrics/particle/momentum/class_2/event/py_bias": -0.6266233921051025, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.850593566894531, "metrics/particle/momentum/class_2/event/pz_bias": -0.16054563224315643, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.014148712158203, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01666971854865551, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1399289220571518, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1406591236591339, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.010260856710374355, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1792193502187729, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009230059222318232, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20740455389022827, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.001547429128549993, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7820029258728027, "metrics/particle/momentum/class_3/accepted/samples": 13001.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.1977025270462036, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3456133604049683, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2247470617294312, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.0836998224258423, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.11350242048501968, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.985329627990723, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.005694987252354622, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.42142218351364136, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.011028082109987736, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.339247941970825, "metrics/particle/momentum/class_3/assigned/samples": 15101.0, "metrics/particle/momentum/class_3/efficiency": 0.8369922041893005, "metrics/particle/momentum/class_3/event/energy_bias": 1.682909369468689, "metrics/particle/momentum/class_3/event/p3_error_mean": 7.964319229125977, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.357497692108154, "metrics/particle/momentum/class_3/event/px_bias": -0.010345784947276115, "metrics/particle/momentum/class_3/event/py_bias": -0.10207834839820862, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.6571526527404785, "metrics/particle/momentum/class_3/event/pz_bias": -0.10346533358097076, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.492066383361816, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.021292347460985184, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.16730447113513947, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.16081355512142181, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.08496110141277313, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.20840749144554138, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.5349216461181641, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0026947043370455503, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1145050525665283, "metrics/particle/momentum/class_4/accepted/samples": 264.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.3497913181781769, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.641746997833252, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.2961480617523193, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.202982187271118, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.2295081913471222, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 11.602675437927246, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027322403620928526, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.6018365621566772, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.023216748610138893, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3268382549285889, "metrics/particle/momentum/class_4/assigned/samples": 366.0, "metrics/particle/momentum/class_4/efficiency": 0.7039999961853027, "metrics/particle/momentum/class_4/event/energy_bias": -0.4709398150444031, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7336698770523071, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2419795989990234, "metrics/particle/momentum/class_4/event/px_bias": -0.05724754557013512, "metrics/particle/momentum/class_4/event/py_bias": -0.0657646581530571, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9762009382247925, "metrics/particle/momentum/class_4/event/pz_bias": 0.17027275264263153, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.8727569580078125, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014748790301382542, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.13905015587806702, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.1355193704366684, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.04655199125409126, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.17627303302288055, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.019480518996715546, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 1.2669501304626465, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.3388521671295166, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 3.0130624771118164, "metrics/particle/momentum/class_5/accepted/samples": 154.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04176289588212967, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.1751762181520462, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.18624871969223022, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.08722849935293198, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.05487804859876633, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.3812553882598877, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.030487805604934692, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 1.460570216178894, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.2962859869003296, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.5181291103363037, "metrics/particle/momentum/class_5/assigned/samples": 164.0, "metrics/particle/momentum/class_5/efficiency": 0.9390243887901306, "metrics/particle/momentum/class_5/event/energy_bias": -0.048314910382032394, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5040487051010132, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.1570472717285156, "metrics/particle/momentum/class_5/event/px_bias": 0.07036367803812027, "metrics/particle/momentum/class_5/event/py_bias": -0.1757383644580841, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7566331624984741, "metrics/particle/momentum/class_5/event/pz_bias": 0.17831853032112122, "metrics/particle/momentum/class_5/event/pz_residual_rms": 2.724687099456787, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.2871219217777252, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.2887839078903198, "metrics/set_slot/angular_recall_all_dr0p10": 0.9644969701766968, "metrics/set_slot/angular_recall_selected_dr0p10": 0.8720859289169312, "metrics/set_slot/soft_count_bias_mean": -0.41902589797973633}, "step_train_time": 6.438153266906738, "step_valid_time": 60944.537479162216, "step_total_time": 60950.97563290596, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023630252358553, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10084218217621777, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3051, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2946, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2937, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9630598234717228, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9655850540806293, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9643207855973813, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0344149459193707, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9601176855181431, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9626352015732547, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9613747954173486, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9630598234717228, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9924064735661093, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10190346942342943, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2274, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2176, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2173, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9522975929978118, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9569041336851363, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9545953059881553, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04309586631486367, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9509846827133479, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.955584872471416, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9532792279008554, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9522975929978118, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0043667961622518, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1091157891697625, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1320, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1247, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1247, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.943267776096823, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9446969696969697, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9439818319454958, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.055303030303030305, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.943267776096823, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9446969696969697, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9439818319454958, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.943267776096823} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_6000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..7fc2f0aecc4e6e2fe55b0ce74e71e49987d1b9d8 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 0.28419856006062905, "train": {"Classification_binary": 0.2710229754447937, "Cardinality": 0.17959105968475342, "Classification": 0.18193015456199646, "Regression_pt": 0.28999462723731995, "Regression_eta": 0.003246933687478304, "Regression_sin_phi": 0.001989056821912527, "Regression_cos_phi": 0.001701543340459466, "Regression_energy": 0.28646403551101685, "HitGrouping": 0.22613264620304108, "Auxiliary": 0.3397940993309021, "Total": 1.8458296060562134, "Total_unweighted": 1.0363493266049773}, "train_group": {}, "valid": {"Classification_binary": 0.3171665867169698, "Cardinality": 0.24901429812113443, "Classification": 0.2138811151186625, "Regression_pt": 0.30836792786916095, "Regression_eta": 0.0042038122192025185, "Regression_sin_phi": 0.002152077853679657, "Regression_cos_phi": 0.0021612239070236683, "Regression_energy": 0.3041638135910034, "HitGrouping": 0.22275408109029135, "Auxiliary": 0.3819309075673421, "Total": 2.118783791859945, "Total_unweighted": 1.1520965572757025, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8229705095291138, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.3807240426540375, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.3961389362812042, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.3841235637664795, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.3540855646133423, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.41601210832595825, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7731117606163025, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.3874417841434479, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.3951255679130554, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.39128348231315613, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3502846956253052, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.48744460940361023, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8343701362609863, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.33367279171943665, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3551492691040039, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.33667969703674316, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.30825212597846985, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.399120569229126, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8261605501174927, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.352101594209671, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.370670884847641, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.35529229044914246, "metrics/hit_grouping/calorimeter/all/object_purity": 0.32540082931518555, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.41109517216682434, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0012888561468571424, "metrics/hit_grouping/calorimeter/background_recall": 0.054290223866701126, "metrics/hit_grouping/calorimeter/completeness": 0.4476739168167114, "metrics/hit_grouping/calorimeter/energy_completeness": 0.4555734694004059, "metrics/hit_grouping/calorimeter/energy_purity": 0.44925498962402344, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.5070137977600098, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8574406504631042, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.37453827261924744, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3039940893650055, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.37947309017181396, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3439418077468872, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3418866693973541, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8187376260757446, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.3650869131088257, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.39371258020401, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.36836251616477966, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.34498244524002075, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4322892725467682, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7698594331741333, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.29327192902565, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.48341068625450134, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.2928207814693451, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.2678690552711487, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5308584570884705, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7917032837867737, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.39437076449394226, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4551960527896881, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4005957543849945, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.37940317392349243, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.48833292722702026, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7681728601455688, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.49694353342056274, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6217190027236938, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.4977176785469055, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.46892449259757996, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4956253170967102, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.868200421333313, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.2917497158050537, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.25458335876464844, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.29213452339172363, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.25324323773384094, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3222760558128357, "metrics/hit_grouping/calorimeter/purity": 0.44033217430114746, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8997650146484375, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.04437430948019028, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.12030075490474701, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.044799499213695526, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.015317436307668686, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.028195489197969437, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8830482363700867, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.0806058719754219, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.08475574105978012, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.07988876849412918, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.05650635063648224, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2224838137626648, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8220009207725525, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.37124407291412354, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.3896290957927704, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.374683141708374, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.3444463014602661, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.42714938521385193, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6505722999572754, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4583514332771301, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.3432058095932007, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.352582186460495, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7202665209770203, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7191906571388245, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.38155514001846313, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3750673532485962, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.345474511384964, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5533453822135925, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5746763348579407, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.43941575288772583, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.25075381994247437, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.26136264204978943, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7877392172813416, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6086771488189697, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44096171855926514, "metrics/hit_grouping/tracker/all/object_completeness": 0.2883509695529938, "metrics/hit_grouping/tracker/all/object_purity": 0.29561156034469604, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7496883273124695, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.011618487536907196, "metrics/hit_grouping/tracker/background_recall": 0.08560940623283386, "metrics/hit_grouping/tracker/completeness": 0.295493483543396, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5827478170394897, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6172117590904236, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.43880414962768555, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.342284619808197, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.35431569814682007, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.756005585193634, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.5993607044219971, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4403702914714813, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.2602189779281616, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.27219176292419434, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7573062181472778, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.60037761926651, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.44598737359046936, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.20535939931869507, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.19812548160552979, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7291253209114075, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6132918000221252, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.47134244441986084, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.29468581080436707, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3083323538303375, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.752993106842041, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6325011849403381, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5227832794189453, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3207637667655945, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.33201178908348083, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7081280946731567, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5984072089195251, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3867976665496826, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.2743048667907715, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2725486159324646, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7528424859046936, "metrics/hit_grouping/tracker/purity": 0.2789156138896942, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8820695877075195, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.018853696063160896, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.030672337859869003, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.029533231630921364, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06334841996431351, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6520678400993347, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.21373257040977478, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.16379769146442413, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.14296311140060425, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.576664924621582, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5634036064147949, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5282300710678101, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.34008973836898804, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.3516419231891632, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8702926635742188, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4909258186817169, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.33383235335350037, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.3562133312225342, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5765743255615234, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.05822695419192314, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.0020995293743908405, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.479278564453125, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.17592188715934753, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.08429872989654541, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.602929413318634, "metrics/particle/class_1/pid_accuracy": 0.9265198707580566, "metrics/particle/class_2/efficiency": 0.4060482382774353, "metrics/particle/class_2/pid_accuracy": 0.35536959767341614, "metrics/particle/class_3/efficiency": 0.5127341747283936, "metrics/particle/class_3/pid_accuracy": 0.9133197069168091, "metrics/particle/class_4/efficiency": 0.5340681076049805, "metrics/particle/class_4/pid_accuracy": 0.2870544195175171, "metrics/particle/class_5/efficiency": 0.6570881009101868, "metrics/particle/class_5/pid_accuracy": 0.323615163564682, "metrics/particle/count/bias_mean": -10.635416984558105, "metrics/particle/count/mae": 11.401041984558105, "metrics/particle/count/prediction_mean": 60.01953125, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.10094887763261795, "metrics/particle/event/energy_response_mean": 0.9671924710273743, "metrics/particle/event/met_abs_error": 13.6764497756958, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08846381306648254, "metrics/particle/event/scalar_pt_response_mean": 0.9935492873191833, "metrics/particle/event/vector_pt_closure": 0.0984005555510521, "metrics/particle/matched/delta_eta_abs_mean": 0.02063540369272232, "metrics/particle/matched/delta_phi_abs_mean": 0.018930453807115555, "metrics/particle/matched/delta_r_mean": 0.031388819217681885, "metrics/particle/matched/energy_relative_abs_mean": 0.18935327231884003, "metrics/particle/matched/pid_accuracy": 0.8777234554290771, "metrics/particle/matched/pt_relative_abs_mean": 0.18807417154312134, "metrics/particle/matching/duplicate_fraction": 0.11138952523469925, "metrics/particle/matching/efficiency": 0.54852294921875, "metrics/particle/matching/f1": 0.5931664705276489, "metrics/particle/matching/purity": 0.6457207798957825, "metrics/particle/matching/target_coverage_dr0p05": 0.49464643001556396, "metrics/particle/matching/target_coverage_dr0p10": 0.6302913427352905, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.34715184569358826, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6066104173660278, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03014172799885273, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.18853944540023804, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.1867729127407074, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.009232552722096443, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2274562567472458, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.014920311979949474, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.1659119129180908, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.10365494340658188, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.736452102661133, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 5898.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3724379241466522, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5636528730392456, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.8175904750823975, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.622849702835083, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.23826192319393158, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.774361610412598, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.025467712432146072, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.6218416690826416, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.15706443786621094, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.8136184215545654, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8285.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5475306510925293, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.6381117105484009, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.985180854797363, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.809779167175293, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 1.1381722688674927, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.6139569282531738, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.145774841308594, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -5.3937811851501465, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.84994125366211, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030636508017778397, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.19312269985675812, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.1910688877105713, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.003077780595049262, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.2322341650724411, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.040387723594903946, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 2.042757034301758, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.17149308323860168, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.971199035644531, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1238.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.49446308612823486, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3756178617477417, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.8357346057891846, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.6582443714141846, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2750949561595917, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 31.11835289001465, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06565382331609726, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.106952667236328, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.4173012971878052, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.879619598388672, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1843.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4488759934902191, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -6.4975504875183105, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.868607521057129, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.62855064868927, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.12122082710266113, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.044935621321201324, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.674036979675293, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.557032823562622, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 27.10008430480957, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03207937628030777, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19048911333084106, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.18972937762737274, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.005363115109503269, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.23086996376514435, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0013194660423323512, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3996956944465637, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.04177963361144066, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0122274160385132, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 12884.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.4954673945903778, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.6283605098724365, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 2.1755850315093994, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.9872124195098877, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.30382052063941956, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.760238647460938, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.010162504389882088, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.855353593826294, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.10244060307741165, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.6789498329162598, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20861.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5536980628967285, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.560516834259033, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 17.190235137939453, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.54231071472168, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 6.347986221313477, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.9036145210266113, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.722334384918213, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -3.6478543281555176, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 12.276680946350098, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0314193069934845, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.19007758796215057, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.18894121050834656, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.0059811025857925415, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.22995449602603912, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007742257788777351, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7270309925079346, "metrics/particle/momentum/all/accepted/pz_bias": -0.06802967190742493, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.0961012840270996, "metrics/particle/momentum/all/accepted/samples": 20020.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.46251532435417175, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.2692039012908936, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.178607702255249, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.9891797304153442, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2845848500728607, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.385076522827148, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.017554616555571556, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1941851377487183, "metrics/particle/momentum/all/assigned/pz_bias": -0.1357700675725937, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.8110973834991455, "metrics/particle/momentum/all/assigned/samples": 30989.0, "metrics/particle/momentum/all/efficiency": 0.5440365076065063, "metrics/particle/momentum/all/event/energy_bias": -5.575145721435547, "metrics/particle/momentum/all/event/p3_error_mean": 31.669429779052734, "metrics/particle/momentum/all/event/pt_vector_error_mean": 17.630313873291016, "metrics/particle/momentum/all/event/px_bias": 7.60737943649292, "metrics/particle/momentum/all/event/py_bias": 3.4726357460021973, "metrics/particle/momentum/all/event/pz_abs_mean": 22.35745620727539, "metrics/particle/momentum/all/event/pz_bias": -11.598669052124023, "metrics/particle/momentum/all/event/pz_residual_rms": 33.21603775024414, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03643226623535156, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.20342722535133362, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.20268477499485016, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.01434862520545721, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.24237172305583954, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007785795722156763, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.8199558854103088, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.061308376491069794, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.1873631477355957, "metrics/particle/momentum/class_1/accepted/samples": 10532.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.35193049907684326, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.2019503116607666, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.3817627429962158, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 1.1716804504394531, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.2517559230327606, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 9.560242652893066, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.014047296717762947, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.1221665143966675, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.12096308171749115, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.290431022644043, "metrics/particle/momentum/class_1/assigned/samples": 15519.0, "metrics/particle/momentum/class_1/efficiency": 0.59365314245224, "metrics/particle/momentum/class_1/event/energy_bias": 18.55921173095703, "metrics/particle/momentum/class_1/event/p3_error_mean": 24.895845413208008, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 16.077104568481445, "metrics/particle/momentum/class_1/event/px_bias": 6.447541236877441, "metrics/particle/momentum/class_1/event/py_bias": 3.006441116333008, "metrics/particle/momentum/class_1/event/pz_abs_mean": 15.692098617553711, "metrics/particle/momentum/class_1/event/pz_bias": -7.069315433502197, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.831748962402344, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03688988462090492, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23132012784481049, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2261228859424591, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.062053512781858444, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2667238414287567, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03921568766236305, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8882170915603638, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1750795990228653, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.044911861419678, "metrics/particle/momentum/class_2/accepted/samples": 1224.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.4299984872341156, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6645823121070862, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7873536348342896, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4336116313934326, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.2609367072582245, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.083929538726807, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.056613482534885406, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.6448147296905518, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.443279504776001, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.5707106590271, "metrics/particle/momentum/class_2/assigned/samples": 1943.0, "metrics/particle/momentum/class_2/efficiency": 0.40991291403770447, "metrics/particle/momentum/class_2/event/energy_bias": -20.849058151245117, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.37362289428711, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.88813591003418, "metrics/particle/momentum/class_2/event/px_bias": 0.614863395690918, "metrics/particle/momentum/class_2/event/py_bias": -0.08726371824741364, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.31814956665039, "metrics/particle/momentum/class_2/event/pz_bias": -3.362919807434082, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.110078811645508, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02396918274462223, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16586339473724365, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16491374373435974, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.014647928066551685, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20532819628715515, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001881821546703577, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.39176851511001587, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.06744223088026047, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2690057754516602, "metrics/particle/momentum/class_3/accepted/samples": 7971.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5997795462608337, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.6040070056915283, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.205977439880371, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.0657529830932617, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.32619503140449524, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.48412322998047, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01529636699706316, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.0421286821365356, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.1133306697010994, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.5385892391204834, "metrics/particle/momentum/class_3/assigned/samples": 13075.0, "metrics/particle/momentum/class_3/efficiency": 0.5131655335426331, "metrics/particle/momentum/class_3/event/energy_bias": -1.052262544631958, "metrics/particle/momentum/class_3/event/p3_error_mean": 16.070648193359375, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.331320762634277, "metrics/particle/momentum/class_3/event/px_bias": -0.2434556782245636, "metrics/particle/momentum/class_3/event/py_bias": 0.6826454997062683, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.118484497070312, "metrics/particle/momentum/class_3/event/pz_bias": -1.335237979888916, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.52610969543457, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.033479928970336914, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.17899219691753387, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17655938863754272, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.004891532473266125, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22078025341033936, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.010582010261714458, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1235393285751343, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.23939332365989685, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.1123175621032715, "metrics/particle/momentum/class_4/accepted/samples": 189.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.5457600355148315, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 10.218226432800293, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 8.178844451904297, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 8.046022415161133, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.34405145049095154, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 41.979469299316406, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.012861736118793488, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3453311920166016, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.15630295872688293, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.465379476547241, "metrics/particle/momentum/class_4/assigned/samples": 311.0, "metrics/particle/momentum/class_4/efficiency": 0.5040000081062317, "metrics/particle/momentum/class_4/event/energy_bias": -0.11071696877479553, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.4253900051116943, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.5576722621917725, "metrics/particle/momentum/class_4/event/px_bias": 0.3111313581466675, "metrics/particle/momentum/class_4/event/py_bias": 0.10530970990657806, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8175698518753052, "metrics/particle/momentum/class_4/event/pz_bias": 0.1736326664686203, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.414707660675049, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02664167806506157, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.22879846394062042, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.22361144423484802, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.08175576478242874, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.27036356925964355, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.07692307978868484, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.6256840229034424, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.7776006460189819, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 6.095306873321533, "metrics/particle/momentum/class_5/accepted/samples": 104.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.16981586813926697, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.5372293591499329, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.551021933555603, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.21169203519821167, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.23404255509376526, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 1.442783236503601, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08510638028383255, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.897843837738037, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.43651315569877625, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.0145416259765625, "metrics/particle/momentum/class_5/assigned/samples": 141.0, "metrics/particle/momentum/class_5/efficiency": 0.6341463327407837, "metrics/particle/momentum/class_5/event/energy_bias": -2.122319221496582, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.0401148796081543, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.3639726638793945, "metrics/particle/momentum/class_5/event/px_bias": 0.47729918360710144, "metrics/particle/momentum/class_5/event/py_bias": -0.23449701070785522, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4727486371994019, "metrics/particle/momentum/class_5/event/pz_bias": -0.004828333854675293, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.568075656890869, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6142915487289429, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6163086891174316, "metrics/set_slot/angular_recall_all_dr0p10": 0.8493448495864868, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6550043225288391, "metrics/set_slot/soft_count_bias_mean": -2.7518482208251953}, "step_train_time": 5.133610486984253, "step_valid_time": 17940.029988527298, "step_total_time": 17945.1635992527, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0348650931134893, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2699195520908303, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3019, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2737, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2615, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8947368421052632, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9065915866180855, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9006252056597565, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09340841338191454, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8548545276234063, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8661808545876117, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8604804211911813, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8947368421052632, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9921748386586838, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25076583308795797, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2254, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2041, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2003, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8932166301969365, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9055013309671694, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8993170301828597, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09449866903283052, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8765864332603939, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.888642413487134, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8825732540207094, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8932166301969365, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9944247142622749, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2365026680725275, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1299, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1122, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1091, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.848714069591528, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8637413394919169, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8561617703166731, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13625866050808313, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8252647503782148, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8398768283294842, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8325066768409004, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.848714069591528} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_7000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac518dea3691a499888b047368baf8d37feec2d --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 0.33156498673740054, "train": {"Classification_binary": 0.27379268407821655, "Cardinality": 0.20356249809265137, "Classification": 0.19464552402496338, "Regression_pt": 0.30444082617759705, "Regression_eta": 0.003938889130949974, "Regression_sin_phi": 0.0018239959608763456, "Regression_cos_phi": 0.0018044277094304562, "Regression_energy": 0.29983797669410706, "HitGrouping": 0.22045385837554932, "Auxiliary": 0.3607715666294098, "Total": 1.925643801689148, "Total_unweighted": 1.0802843237761408}, "train_group": {}, "valid": {"Classification_binary": 0.30638506015141803, "Cardinality": 0.25403908888498944, "Classification": 0.2140026092529297, "Regression_pt": 0.308510700861613, "Regression_eta": 0.004225158753494422, "Regression_sin_phi": 0.002075655541072289, "Regression_cos_phi": 0.0020485434991618, "Regression_energy": 0.30291299025217694, "HitGrouping": 0.22013856967290243, "Auxiliary": 0.3884708086649577, "Total": 2.1012436548868814, "Total_unweighted": 1.140160718311866, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.811806321144104, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4114302098751068, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.41128039360046387, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4156959652900696, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.39181244373321533, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.43370872735977173, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.763800323009491, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.41619399189949036, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.39771050214767456, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4190841019153595, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.38707584142684937, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.49076810479164124, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8117371201515198, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3562552332878113, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3805137574672699, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.35952430963516235, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.34651440382003784, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.43573248386383057, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8080362677574158, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.37762415409088135, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.3911697268486023, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3811657428741455, "metrics/hit_grouping/calorimeter/all/object_purity": 0.36338669061660767, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.43939220905303955, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0016314886743202806, "metrics/hit_grouping/calorimeter/background_recall": 0.06295805424451828, "metrics/hit_grouping/calorimeter/completeness": 0.45272359251976013, "metrics/hit_grouping/calorimeter/energy_completeness": 0.45466941595077515, "metrics/hit_grouping/calorimeter/energy_purity": 0.44842976331710815, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4726107716560364, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8423917293548584, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.40833780169487, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.3116750121116638, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.41380074620246887, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.3883053958415985, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.36595356464385986, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8000001311302185, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.39073094725608826, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.4111008644104004, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.39393550157546997, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.3840903043746948, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.45796868205070496, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7460821866989136, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3017956018447876, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.5331786274909973, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3017590343952179, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.29172056913375854, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5704176425933838, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7754490971565247, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.40015947818756104, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4649988114833832, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4079338610172272, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.40521737933158875, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.5089607834815979, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.723335862159729, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.4869076609611511, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.6176016330718994, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.48647236824035645, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5021602511405945, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.5707668662071228, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8513518571853638, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.34162837266921997, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.28892022371292114, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3413281738758087, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3042047619819641, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.352811336517334, "metrics/hit_grouping/calorimeter/purity": 0.44545185565948486, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8523741960525513, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.05538332089781761, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.129699245095253, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.05554511770606041, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.017927316948771477, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04135338217020035, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8518722653388977, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.12410340458154678, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.11065332591533661, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.12294748425483704, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.09396661072969437, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2560329735279083, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8050327301025391, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.39606770873069763, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.4100278615951538, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.39990878105163574, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.38303595781326294, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.45541977882385254, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6685759425163269, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.46458062529563904, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.3642646074295044, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.36118435859680176, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6954947113990784, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7341888546943665, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.34840264916419983, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3830689787864685, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.35791927576065063, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5394816398620605, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5738922953605652, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4622247815132141, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.26342201232910156, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.2764326333999634, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7943589091300964, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6148332357406616, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.45454141497612, "metrics/hit_grouping/tracker/all/object_completeness": 0.30325332283973694, "metrics/hit_grouping/tracker/all/object_purity": 0.3084997534751892, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7451469302177429, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.005823803599923849, "metrics/hit_grouping/tracker/background_recall": 0.0560072623193264, "metrics/hit_grouping/tracker/completeness": 0.3099306523799896, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5763148069381714, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6286712884902954, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4363493025302887, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.36260029673576355, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3745321035385132, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7437313795089722, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6015039086341858, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4701397716999054, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.271678626537323, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.27984878420829773, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.762570321559906, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.5996112823486328, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.47646528482437134, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.21254701912403107, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.20113696157932281, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7307484149932861, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.617652416229248, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.49290990829467773, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3065457344055176, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.31490081548690796, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7487475872039795, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6216595768928528, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5369458198547363, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.32553765177726746, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3433258831501007, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7333743572235107, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6099443435668945, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3898884952068329, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.29497817158699036, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.2939348518848419, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7425764203071594, "metrics/hit_grouping/tracker/purity": 0.29083767533302307, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8826217651367188, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01998491771519184, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.03945367410778999, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0415102019906044, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06636501103639603, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6524807810783386, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.229220449924469, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.16978931427001953, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.1521584540605545, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5875064730644226, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5710136890411377, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5434502959251404, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.3568662106990814, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.36506563425064087, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8629623651504517, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.495173841714859, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.35472729802131653, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.38040071725845337, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5802760124206543, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.05397893115878105, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.005801224149763584, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.367759704589844, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.19933705031871796, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.10771386325359344, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6383072137832642, "metrics/particle/class_1/pid_accuracy": 0.9110819101333618, "metrics/particle/class_2/efficiency": 0.43243855237960815, "metrics/particle/class_2/pid_accuracy": 0.3622315227985382, "metrics/particle/class_3/efficiency": 0.5633648633956909, "metrics/particle/class_3/pid_accuracy": 0.9214590787887573, "metrics/particle/class_4/efficiency": 0.5430861711502075, "metrics/particle/class_4/pid_accuracy": 0.3099631071090698, "metrics/particle/class_5/efficiency": 0.6743295192718506, "metrics/particle/class_5/pid_accuracy": 0.34090909361839294, "metrics/particle/count/bias_mean": -4.985025882720947, "metrics/particle/count/mae": 7.612630367279053, "metrics/particle/count/prediction_mean": 65.669921875, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.10153243690729141, "metrics/particle/event/energy_response_mean": 0.9414107203483582, "metrics/particle/event/met_abs_error": 9.222831726074219, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08537673205137253, "metrics/particle/event/scalar_pt_response_mean": 0.949255645275116, "metrics/particle/event/vector_pt_closure": 0.07831176370382309, "metrics/particle/matched/delta_eta_abs_mean": 0.023781077936291695, "metrics/particle/matched/delta_phi_abs_mean": 0.021900389343500137, "metrics/particle/matched/delta_r_mean": 0.03575088828802109, "metrics/particle/matched/energy_relative_abs_mean": 0.18915799260139465, "metrics/particle/matched/pid_accuracy": 0.8742631077766418, "metrics/particle/matched/pt_relative_abs_mean": 0.1865517646074295, "metrics/particle/matching/duplicate_fraction": 0.10648465156555176, "metrics/particle/matching/efficiency": 0.5892689228057861, "metrics/particle/matching/f1": 0.6108168959617615, "metrics/particle/matching/purity": 0.6340005397796631, "metrics/particle/matching/target_coverage_dr0p05": 0.5127158761024475, "metrics/particle/matching/target_coverage_dr0p10": 0.6672133803367615, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.37731972336769104, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6589019894599915, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03365161642432213, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1844043880701065, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.18185016512870789, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.01977143995463848, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.22225305438041687, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.013840830884873867, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.0986570119857788, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.23151256144046783, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.720231294631958, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6358.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.47919824719429016, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4029319286346436, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.832722783088684, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.6227426528930664, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.26207485795021057, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.159634590148926, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02335003763437271, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.5118639469146729, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.1541929543018341, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.6030898094177246, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9379.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.5902339220046997, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.8476924896240234, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.375886917114258, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.273897171020508, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.14993271231651306, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.6132252812385559, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.787492752075195, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.9088096618652344, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.051454544067383, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0332636758685112, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.19294089078903198, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.19100259244441986, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.00830933265388012, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.23186001181602478, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0443764328956604, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.978655219078064, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.18253278732299805, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.998974323272705, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1307.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6950123906135559, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4362431764602661, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.1129069328308105, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.9216229915618896, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.31857556104660034, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 37.48177719116211, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07074109464883804, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1378345489501953, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.049401674419641495, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.566203117370605, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2078.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4738941192626953, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -8.175689697265625, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.054258346557617, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.6146767139434814, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06280223280191422, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.16233068704605103, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.845102310180664, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.1012160778045654, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 27.56916618347168, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03683387488126755, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.1920522004365921, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.18941226601600647, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": -0.03375191241502762, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.2293611466884613, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0018064889591187239, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.4059985876083374, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.09110220521688461, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0767381191253662, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 13839.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.4964805543422699, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.223695993423462, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.8256255388259888, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.6012415885925293, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.29077789187431335, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.360244750976562, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.007756482344120741, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7731518745422363, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.05207626149058342, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.3791022300720215, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 22175.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5947397947311401, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.199749708175659, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 14.744490623474121, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 12.388766288757324, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.3870590627193451, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 2.972594738006592, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.795624732971191, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -2.926252603530884, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.076613426208496, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03567599505186081, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.189845010638237, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1872730851173401, "metrics/particle/momentum/all/accepted/pt_relative_bias": -0.027061911299824715, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.22743672132492065, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.007952009327709675, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.7063791155815125, "metrics/particle/momentum/all/accepted/pz_bias": -0.13817384839057922, "metrics/particle/momentum/all/accepted/pz_residual_rms": 2.110166549682617, "metrics/particle/momentum/all/accepted/samples": 21504.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.5039275288581848, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.9461548328399658, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.9689279794692993, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.750605583190918, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2844909727573395, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 12.501341819763184, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.015996670350432396, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.12526273727417, "metrics/particle/momentum/all/assigned/pz_bias": -0.08038842678070068, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.606459617614746, "metrics/particle/momentum/all/assigned/samples": 33632.0, "metrics/particle/momentum/all/efficiency": 0.5843636989593506, "metrics/particle/momentum/all/event/energy_bias": -13.22313117980957, "metrics/particle/momentum/all/event/p3_error_mean": 27.60140037536621, "metrics/particle/momentum/all/event/pt_vector_error_mean": 14.049419403076172, "metrics/particle/momentum/all/event/px_bias": 0.17432409524917603, "metrics/particle/momentum/all/event/py_bias": 3.423489570617676, "metrics/particle/momentum/all/event/pz_abs_mean": 20.325767517089844, "metrics/particle/momentum/all/event/pz_bias": -5.733846664428711, "metrics/particle/momentum/all/event/pz_residual_rms": 31.685726165771484, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03810206428170204, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.19920171797275543, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.19798006117343903, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.029923196882009506, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.23750369250774384, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.00877428613603115, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.8090596199035645, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.1424439251422882, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 2.2438528537750244, "metrics/particle/momentum/class_1/accepted/samples": 11169.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.381303608417511, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9801837801933289, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.123274564743042, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.8820444345474243, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.2425924837589264, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.844504356384277, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.013215859420597553, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.066886305809021, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.11779609322547913, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.0279014110565186, "metrics/particle/momentum/class_1/assigned/samples": 16571.0, "metrics/particle/momentum/class_1/efficiency": 0.6295586228370667, "metrics/particle/momentum/class_1/event/energy_bias": -1.5828886032104492, "metrics/particle/momentum/class_1/event/p3_error_mean": 21.99002456665039, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 13.163801193237305, "metrics/particle/momentum/class_1/event/px_bias": 1.3190245628356934, "metrics/particle/momentum/class_1/event/py_bias": 2.197042465209961, "metrics/particle/momentum/class_1/event/pz_abs_mean": 14.9405517578125, "metrics/particle/momentum/class_1/event/pz_bias": -8.246745109558105, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.780933380126953, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.042170971632003784, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22874411940574646, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22816866636276245, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.087629534304142, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2660118341445923, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03596559911966324, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8646206855773926, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.3386383652687073, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.284870624542236, "metrics/particle/momentum/class_2/accepted/samples": 1279.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.6198614239692688, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6878613829612732, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8957405686378479, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4768235981464386, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.31227436661720276, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.1093945503234863, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05731046944856644, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.648759126663208, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.13577069342136383, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.612308502197266, "metrics/particle/momentum/class_2/assigned/samples": 2216.0, "metrics/particle/momentum/class_2/efficiency": 0.42833220958709717, "metrics/particle/momentum/class_2/event/energy_bias": -20.005916595458984, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.624622344970703, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.84915828704834, "metrics/particle/momentum/class_2/event/px_bias": -1.5075671672821045, "metrics/particle/momentum/class_2/event/py_bias": -0.17190204560756683, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.441200256347656, "metrics/particle/momentum/class_2/event/pz_bias": 4.184663772583008, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.954254150390625, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.031641412526369095, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17224429547786713, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16752472519874573, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.014183728024363518, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20713016390800476, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0020580836571753025, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.37937769293785095, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.10486825555562973, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2143216133117676, "metrics/particle/momentum/class_3/accepted/samples": 8746.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.6282696723937988, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.20253849029541, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.076347589492798, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.9158051013946533, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3269672989845276, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.875104904174805, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011988569051027298, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.933377206325531, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.01783641427755356, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.534884214401245, "metrics/particle/momentum/class_3/assigned/samples": 14347.0, "metrics/particle/momentum/class_3/efficiency": 0.5630592703819275, "metrics/particle/momentum/class_3/event/energy_bias": 9.637308120727539, "metrics/particle/momentum/class_3/event/p3_error_mean": 16.199907302856445, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 9.574319839477539, "metrics/particle/momentum/class_3/event/px_bias": 0.19634021818637848, "metrics/particle/momentum/class_3/event/py_bias": 1.3393211364746094, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.004886627197266, "metrics/particle/momentum/class_3/event/pz_bias": -1.489444613456726, "metrics/particle/momentum/class_3/event/pz_residual_rms": 21.351627349853516, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.036078259348869324, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18064649403095245, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.17931868135929108, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.025146735832095146, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.22288165986537933, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.005076142027974129, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.9795661568641663, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.33338528871536255, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.9456480741500854, "metrics/particle/momentum/class_4/accepted/samples": 197.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.6181709170341492, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 4.837489128112793, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.037097930908203, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.860746145248413, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.36627906560897827, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 15.603069305419922, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.011627906933426857, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.0748505592346191, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.053484030067920685, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.3669559955596924, "metrics/particle/momentum/class_4/assigned/samples": 344.0, "metrics/particle/momentum/class_4/efficiency": 0.5253333449363708, "metrics/particle/momentum/class_4/event/energy_bias": -0.07696657627820969, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.3703699111938477, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.4583840370178223, "metrics/particle/momentum/class_4/event/px_bias": 0.13616004586219788, "metrics/particle/momentum/class_4/event/py_bias": -0.13219034671783447, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.8513736724853516, "metrics/particle/momentum/class_4/event/pz_bias": -0.05121438950300217, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.679540634155273, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.033936090767383575, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.20303845405578613, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.20846357941627502, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.058797456324100494, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.24542298913002014, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.07079645991325378, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.2807793617248535, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.3153857886791229, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.60238790512085, "metrics/particle/momentum/class_5/accepted/samples": 113.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.19131216406822205, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.4885319769382477, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.6175865530967712, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.2741081416606903, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.2532467544078827, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.241872787475586, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.10389610379934311, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.473385810852051, "metrics/particle/momentum/class_5/assigned/pz_bias": -1.1458381414413452, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.9423065185546875, "metrics/particle/momentum/class_5/assigned/samples": 154.0, "metrics/particle/momentum/class_5/efficiency": 0.6890243887901306, "metrics/particle/momentum/class_5/event/energy_bias": -1.1946666240692139, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.158263921737671, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.3925774097442627, "metrics/particle/momentum/class_5/event/px_bias": 0.030366437509655952, "metrics/particle/momentum/class_5/event/py_bias": 0.19121821224689484, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6361089944839478, "metrics/particle/momentum/class_5/event/pz_bias": -0.13110661506652832, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.87498664855957, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.6228352785110474, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.6280986666679382, "metrics/set_slot/angular_recall_all_dr0p10": 0.8529753088951111, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6760868430137634, "metrics/set_slot/soft_count_bias_mean": 3.184584379196167}, "step_train_time": 5.402956008911133, "step_valid_time": 20925.310448408127, "step_total_time": 20930.713404655457, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9960817731773786, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21835867066816494, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3019, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2742, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2692, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8963713631905852, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9082477641603179, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9022704837117473, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09175223583968202, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8800261523373651, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8916859887379928, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8858177031918394, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8963713631905852, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9607272296881876, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19186954941795353, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2242, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2041, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2021, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8932166301969365, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9103479036574487, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9017009056770489, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0896520963425513, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8844638949671773, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9014272970561998, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.892865032030042, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8932166301969365, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9601040173062628, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20357243149733562, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1153, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1130, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8721633888048411, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9014855355746677, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8865820838139177, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0985144644253323, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8547655068078669, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8835027365129007, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8688965782391388, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8721633888048411} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_8000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..90b59fbe22e5968db702c92166d8827a830653b4 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 0.378931413414172, "train": {"Classification_binary": 0.28962013125419617, "Cardinality": 0.21534961462020874, "Classification": 0.16253545880317688, "Regression_pt": 0.26159846782684326, "Regression_eta": 0.003467972856014967, "Regression_sin_phi": 0.0019108369015157223, "Regression_cos_phi": 0.0017434158362448215, "Regression_energy": 0.2604523301124573, "HitGrouping": 0.20810598134994507, "Auxiliary": 0.33552345633506775, "Total": 1.8391435146331787, "Total_unweighted": 0.9813286135904491}, "train_group": {}, "valid": {"Classification_binary": 0.29599769910176593, "Cardinality": 0.27863876024882, "Classification": 0.18151644865671793, "Regression_pt": 0.27021880944569904, "Regression_eta": 0.0036428626626729965, "Regression_sin_phi": 0.0019332855008542538, "Regression_cos_phi": 0.0019175194514294465, "Regression_energy": 0.27082473039627075, "HitGrouping": 0.2104483445485433, "Auxiliary": 0.36258093516031903, "Total": 1.9817482630411785, "Total_unweighted": 1.0260513552154102, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8057000041007996, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.41737446188926697, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.37919938564300537, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.41922301054000854, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.41430315375328064, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.457177996635437, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7700896859169006, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.39336878061294556, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.333456426858902, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.39510151743888855, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.3887943625450134, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.5044313073158264, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8223087191581726, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.3788083493709564, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.3591761291027069, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.3795413672924042, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.367592453956604, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.42513307929039, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8132228851318359, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.39162251353263855, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.36324542760849, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.3927711248397827, "metrics/hit_grouping/calorimeter/all/object_purity": 0.3833896517753601, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.4409977197647095, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.0023115247022360563, "metrics/hit_grouping/calorimeter/background_recall": 0.09348573535680771, "metrics/hit_grouping/calorimeter/completeness": 0.48397114872932434, "metrics/hit_grouping/calorimeter/energy_completeness": 0.49726802110671997, "metrics/hit_grouping/calorimeter/energy_purity": 0.49059051275253296, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.4681270122528076, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.8378037810325623, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.4110487997531891, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2902821898460388, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.41471874713897705, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4058309495449066, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.3801775276660919, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.8092735409736633, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.41477903723716736, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.389912486076355, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.414409339427948, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.4040362536907196, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.46027177572250366, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7670819163322449, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.3286842107772827, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.4851508140563965, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.32622137665748596, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.31683242321014404, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5455916523933411, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.7977674007415771, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4367899000644684, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4522492289543152, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.44070249795913696, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4258451759815216, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4763050377368927, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7894483208656311, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.543400228023529, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5949562788009644, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5429518222808838, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5053592324256897, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4884199798107147, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8318131566047668, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.32748937606811523, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.24489545822143555, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.326010137796402, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.325572669506073, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.39935004711151123, "metrics/hit_grouping/calorimeter/purity": 0.4765220880508423, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8658580183982849, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.04591206833720207, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.14473684132099152, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.04538130760192871, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.01664835400879383, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.03759398311376572, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8226995468139648, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.10813948512077332, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.10359034687280655, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.10749033093452454, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.10617406666278839, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.2978222370147705, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8118721842765808, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.4120076894760132, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.38031792640686035, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.41327720880508423, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.4037913680076599, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.45502158999443054, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6568840742111206, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.46602925658226013, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.3722423017024994, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.38393545150756836, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.7078081965446472, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7366746664047241, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.35081374645233154, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3942038118839264, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.3579666018486023, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5015069246292114, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5607544779777527, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4604259729385376, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.2706874907016754, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.2957027554512024, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7945747375488281, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.6032938361167908, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.4540516436100006, "metrics/hit_grouping/tracker/all/object_completeness": 0.311023473739624, "metrics/hit_grouping/tracker/all/object_purity": 0.32741984724998474, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7462600469589233, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.013627700507640839, "metrics/hit_grouping/tracker/background_recall": 0.10214713960886002, "metrics/hit_grouping/tracker/completeness": 0.3167491555213928, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5607497096061707, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6188699007034302, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.43880414962768555, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.3705480098724365, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.3937284052371979, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.74311763048172, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.5883144736289978, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.46832457184791565, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.2821190655231476, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.29851385951042175, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7651116251945496, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.5861362218856812, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.47123533487319946, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.21729889512062073, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.2197422832250595, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7339946031570435, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6084597110748291, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.4920608103275299, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.3193175792694092, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3415307104587555, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7469643950462341, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6284675598144531, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.556034505367279, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3410976529121399, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.3240084648132324, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7136699557304382, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.5920652151107788, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3863561153411865, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.29484954476356506, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3096868395805359, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7511866688728333, "metrics/hit_grouping/tracker/purity": 0.29958778619766235, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8752642869949341, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.01923076994717121, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.05333980172872543, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.05397116392850876, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.0708898976445198, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6345842480659485, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.2612287104129791, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.20783257484436035, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.1866777390241623, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5967991948127747, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5595428347587585, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5394773483276367, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.36044758558273315, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.3832534849643707, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.862682580947876, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.4914412200450897, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7374999523162842, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.39389288425445557, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.42820826172828674, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5796071290969849, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.05771152302622795, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.005132339429110289, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.47319221496582, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.19427570700645447, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.10265253484249115, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6459358334541321, "metrics/particle/class_1/pid_accuracy": 0.939620316028595, "metrics/particle/class_2/efficiency": 0.4144297242164612, "metrics/particle/class_2/pid_accuracy": 0.4695271849632263, "metrics/particle/class_3/efficiency": 0.5446297526359558, "metrics/particle/class_3/pid_accuracy": 0.9145426154136658, "metrics/particle/class_4/efficiency": 0.5511022210121155, "metrics/particle/class_4/pid_accuracy": 0.2545454502105713, "metrics/particle/class_5/efficiency": 0.6724137663841248, "metrics/particle/class_5/pid_accuracy": 0.3390313386917114, "metrics/particle/count/bias_mean": -9.873046875, "metrics/particle/count/mae": 10.749348640441895, "metrics/particle/count/prediction_mean": 60.78190231323242, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.12131598591804504, "metrics/particle/event/energy_response_mean": 1.0401664972305298, "metrics/particle/event/met_abs_error": 12.372836112976074, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08585236221551895, "metrics/particle/event/scalar_pt_response_mean": 1.0243092775344849, "metrics/particle/event/vector_pt_closure": 0.09323054552078247, "metrics/particle/matched/delta_eta_abs_mean": 0.019285131245851517, "metrics/particle/matched/delta_phi_abs_mean": 0.020842494443058968, "metrics/particle/matched/delta_r_mean": 0.03156227245926857, "metrics/particle/matched/energy_relative_abs_mean": 0.18841485679149628, "metrics/particle/matched/pid_accuracy": 0.89332515001297, "metrics/particle/matched/pt_relative_abs_mean": 0.18152715265750885, "metrics/particle/matching/duplicate_fraction": 0.10102719813585281, "metrics/particle/matching/efficiency": 0.5836573839187622, "metrics/particle/matching/f1": 0.627499520778656, "metrics/particle/matching/purity": 0.6784631609916687, "metrics/particle/matching/target_coverage_dr0p05": 0.5301310420036316, "metrics/particle/matching/target_coverage_dr0p10": 0.6631867289543152, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.37939295172691345, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.639846682548523, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03064907155930996, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.18673205375671387, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.18028150498867035, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.007947355508804321, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.2218354195356369, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.013101109303534031, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.0427353382110596, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.1516922116279602, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.543842077255249, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6488.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3082340657711029, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.3100082874298096, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4980665445327759, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.322503685951233, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.202693372964859, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.53476333618164, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02785451151430607, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.540770173072815, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.11419989168643951, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.8322126865386963, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 8688.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6023022532463074, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 10.146142959594727, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 13.206928253173828, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.008203983306885, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.12445101141929626, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.1869595050811768, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.297953605651855, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.4104044437408447, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 19.682849884033203, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03178083896636963, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1973300576210022, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18925581872463226, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.010136181488633156, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.23070783913135529, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.04277286306023598, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.9012161493301392, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.40788283944129944, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.435370445251465, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1356.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.4694501757621765, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7115689516067505, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.234639644622803, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.032791614532471, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2511415481567383, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 63.57649612426758, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07508878409862518, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0001978874206543, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.3414565324783325, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.22172737121582, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 1971.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.49166062474250793, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.063802719116211, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.404858589172363, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.5087922811508179, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.019939275458455086, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.0014317063614726067, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.227030754089355, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.594186544418335, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.8724422454834, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.03233311325311661, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.19071930646896362, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.18298572301864624, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010069104842841625, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.22352834045886993, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0017656146083027124, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.37715259194374084, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.021196521818637848, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 1.0069952011108398, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 13593.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.45141732692718506, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.5558269023895264, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.9283809661865234, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.7509275674819946, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.27201077342033386, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.874917984008789, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.009659746661782265, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.8256852626800537, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.022542500868439674, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.8460471630096436, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 20808.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.5841677784919739, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 9.91953182220459, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 16.245393753051758, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 13.887876510620117, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 3.920691728591919, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 5.709978103637695, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.976323127746582, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.5803923606872559, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 13.926426887512207, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.03178849816322327, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.18993070721626282, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.1825639009475708, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.008148860186338425, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.22347934544086456, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0077902693301439285, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6749988198280334, "metrics/particle/momentum/all/accepted/pz_bias": -0.08515148609876633, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.9610955715179443, "metrics/particle/momentum/all/accepted/samples": 21437.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.41301414370536804, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.1589763164520264, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.0166659355163574, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.8382067680358887, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2515651285648346, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.6628360748291, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01878158003091812, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1593244075775146, "metrics/particle/momentum/all/assigned/pz_bias": -0.0678248181939125, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.6939194202423096, "metrics/particle/momentum/all/assigned/samples": 31467.0, "metrics/particle/momentum/all/efficiency": 0.5825430154800415, "metrics/particle/momentum/all/event/energy_bias": 15.001873016357422, "metrics/particle/momentum/all/event/p3_error_mean": 28.779510498046875, "metrics/particle/momentum/all/event/pt_vector_error_mean": 16.20130729675293, "metrics/particle/momentum/all/event/px_bias": 4.025203227996826, "metrics/particle/momentum/all/event/py_bias": 6.895504951477051, "metrics/particle/momentum/all/event/pz_abs_mean": 19.774982452392578, "metrics/particle/momentum/all/event/pz_bias": -5.584983825683594, "metrics/particle/momentum/all/event/pz_residual_rms": 30.655675888061523, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.034263916313648224, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.19825862348079681, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.1908767819404602, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.010843578726053238, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.23127688467502594, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.007024938706308603, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.7219525575637817, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.1094941571354866, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.8048771619796753, "metrics/particle/momentum/class_1/accepted/samples": 11388.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.3023656904697418, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 1.0841189622879028, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.1826647520065308, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.9958760738372803, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.21464678645133972, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.667388916015625, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01510173175483942, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.0627355575561523, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.10340751707553864, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.0782976150512695, "metrics/particle/momentum/class_1/assigned/samples": 15826.0, "metrics/particle/momentum/class_1/efficiency": 0.6419029235839844, "metrics/particle/momentum/class_1/event/energy_bias": 33.3725700378418, "metrics/particle/momentum/class_1/event/p3_error_mean": 26.11905288696289, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.896830558776855, "metrics/particle/momentum/class_1/event/px_bias": 3.606221914291382, "metrics/particle/momentum/class_1/event/py_bias": 7.192849636077881, "metrics/particle/momentum/class_1/event/pz_abs_mean": 17.149681091308594, "metrics/particle/momentum/class_1/event/pz_bias": -5.597615718841553, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.83997917175293, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.037047017365694046, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2195732742547989, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2164163887500763, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04882634058594704, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2554611563682556, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03727200627326965, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8463139533996582, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0803728997707367, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.216233730316162, "metrics/particle/momentum/class_2/accepted/samples": 1261.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.45783230662345886, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6885073781013489, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8508027791976929, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5163663625717163, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.26463231444358826, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.865956783294678, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06053026393055916, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.6128649711608887, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.09272760152816772, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.097269058227539, "metrics/particle/momentum/class_2/assigned/samples": 1999.0, "metrics/particle/momentum/class_2/efficiency": 0.42230409383773804, "metrics/particle/momentum/class_2/event/energy_bias": -17.26060676574707, "metrics/particle/momentum/class_2/event/p3_error_mean": 21.680213928222656, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.401601791381836, "metrics/particle/momentum/class_2/event/px_bias": -1.3387303352355957, "metrics/particle/momentum/class_2/event/py_bias": -1.5957019329071045, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.709437370300293, "metrics/particle/momentum/class_2/event/pz_bias": 2.323085069656372, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.609882354736328, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02772533893585205, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.17444881796836853, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16640698909759521, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.013531417585909367, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20719651877880096, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0037762566935271025, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.40852025151252747, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.05770917609333992, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.5027161836624146, "metrics/particle/momentum/class_3/accepted/samples": 8474.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5398916602134705, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.5888843536376953, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.1598453521728516, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.0143117904663086, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2923661172389984, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 27.07587242126465, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.016255222260951996, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.0292150974273682, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.04936635494232178, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.8623218536376953, "metrics/particle/momentum/class_3/assigned/samples": 13165.0, "metrics/particle/momentum/class_3/efficiency": 0.5455482006072998, "metrics/particle/momentum/class_3/event/energy_bias": 0.5276671051979065, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.711061477661133, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.70205020904541, "metrics/particle/momentum/class_3/event/px_bias": 1.390312910079956, "metrics/particle/momentum/class_3/event/py_bias": 0.8351251482963562, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.024015426635742, "metrics/particle/momentum/class_3/event/pz_bias": -2.27974271774292, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.856538772583008, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.032330382615327835, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18186914920806885, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1761087328195572, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.009286876767873764, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.21550092101097107, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.01005025114864111, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1227003335952759, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.17402246594429016, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.308154821395874, "metrics/particle/momentum/class_4/accepted/samples": 199.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.52507084608078, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.366423606872559, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 4.126511573791504, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 3.987682580947876, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3302469253540039, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 14.839649200439453, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.012345679104328156, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.2513974905014038, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.02658672444522381, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 2.5812909603118896, "metrics/particle/momentum/class_4/assigned/samples": 324.0, "metrics/particle/momentum/class_4/efficiency": 0.5306666493415833, "metrics/particle/momentum/class_4/event/energy_bias": -1.3037346601486206, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.1498141288757324, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.358015537261963, "metrics/particle/momentum/class_4/event/px_bias": 0.2577264904975891, "metrics/particle/momentum/class_4/event/py_bias": 0.14268484711647034, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6759569644927979, "metrics/particle/momentum/class_4/event/pz_bias": -0.23724207282066345, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.952495574951172, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.02746080793440342, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.1949741691350937, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.18989309668540955, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.03254668787121773, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.2316015362739563, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.052173912525177, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.042922258377075, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.4046538770198822, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 4.876744747161865, "metrics/particle/momentum/class_5/accepted/samples": 115.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.118122898042202, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.6047476530075073, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.6828205585479736, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.4867627024650574, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.2222222238779068, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 2.6997039318084717, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.08496732264757156, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 3.1596176624298096, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.16050173342227936, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.537120342254639, "metrics/particle/momentum/class_5/assigned/samples": 153.0, "metrics/particle/momentum/class_5/efficiency": 0.7012194991111755, "metrics/particle/momentum/class_5/event/energy_bias": -0.33402472734451294, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.834872245788574, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.1459240913391113, "metrics/particle/momentum/class_5/event/px_bias": 0.10967255383729935, "metrics/particle/momentum/class_5/event/py_bias": 0.32054823637008667, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.4555398225784302, "metrics/particle/momentum/class_5/event/pz_bias": 0.20653226971626282, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.6378655433654785, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5832335352897644, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5827953219413757, "metrics/set_slot/angular_recall_all_dr0p10": 0.8632677793502808, "metrics/set_slot/angular_recall_selected_dr0p10": 0.6808691024780273, "metrics/set_slot/soft_count_bias_mean": -3.6369800567626953}, "step_train_time": 7.188163995742798, "step_valid_time": 23961.664660453796, "step_total_time": 23968.852824687958, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.024870465430532, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2424731774369071, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3041, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2774, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2674, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.906832298136646, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9121999342321605, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9095081967213114, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08780006576783952, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8741418764302059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8793160144689247, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8767213114754099, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.906832298136646, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0255187001263772, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2252652124653447, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2267, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2080, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2050, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9102844638949672, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9175121305690339, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9138840070298769, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08248786943096603, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8971553610503282, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9042787825319806, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9007029876977153, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9102844638949672, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0355282444918474, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23178320249451922, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1302, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1172, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1153, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.886535552193646, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9001536098310292, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8932926829268293, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09984639016897082, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8721633888048411, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8855606758832565, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8788109756097561, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.886535552193646} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_9000.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..16cae435c804a6965926113811b47af7380c795b --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 0.4262978400909435, "train": {"Classification_binary": 0.31569141149520874, "Cardinality": 0.19420145452022552, "Classification": 0.1600499451160431, "Regression_pt": 0.2510662078857422, "Regression_eta": 0.00329016475006938, "Regression_sin_phi": 0.0017153791850432754, "Regression_cos_phi": 0.0015831838827580214, "Regression_energy": 0.2460937798023224, "HitGrouping": 0.19748999178409576, "Auxiliary": 0.3325729966163635, "Total": 1.7806622982025146, "Total_unweighted": 0.9794900721171871}, "train_group": {}, "valid": {"Classification_binary": 0.27635717391967773, "Cardinality": 0.19270789623260498, "Classification": 0.16062313318252563, "Regression_pt": 0.24944798151652017, "Regression_eta": 0.0032416280979911485, "Regression_sin_phi": 0.0017258678562939167, "Regression_cos_phi": 0.0017222391131023567, "Regression_energy": 0.2455300490061442, "HitGrouping": 0.20274847745895386, "Auxiliary": 0.32106999556223553, "Total": 1.7439443270365398, "Total_unweighted": 0.9386480726922551, "metrics/hit_grouping/calorimeter/abs_eta_1to2/best_slot_completeness": 0.8289323449134827, "metrics/hit_grouping/calorimeter/abs_eta_1to2/energy_completeness": 0.4587256908416748, "metrics/hit_grouping/calorimeter/abs_eta_1to2/merged_slot_fraction": 0.37853696942329407, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_completeness": 0.4655166268348694, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_purity": 0.4393564462661743, "metrics/hit_grouping/calorimeter/abs_eta_1to2/object_samples": 10567.0, "metrics/hit_grouping/calorimeter/abs_eta_1to2/split_fraction": 0.3970852792263031, "metrics/hit_grouping/calorimeter/abs_eta_ge2/best_slot_completeness": 0.7793786525726318, "metrics/hit_grouping/calorimeter/abs_eta_ge2/energy_completeness": 0.43066123127937317, "metrics/hit_grouping/calorimeter/abs_eta_ge2/merged_slot_fraction": 0.33308714628219604, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_completeness": 0.4350352883338928, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_purity": 0.42492133378982544, "metrics/hit_grouping/calorimeter/abs_eta_ge2/object_samples": 2708.0, "metrics/hit_grouping/calorimeter/abs_eta_ge2/split_fraction": 0.4689807891845703, "metrics/hit_grouping/calorimeter/abs_eta_lt1/best_slot_completeness": 0.8426177501678467, "metrics/hit_grouping/calorimeter/abs_eta_lt1/energy_completeness": 0.4183434247970581, "metrics/hit_grouping/calorimeter/abs_eta_lt1/merged_slot_fraction": 0.35084471106529236, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_completeness": 0.42142412066459656, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_purity": 0.3955671191215515, "metrics/hit_grouping/calorimeter/abs_eta_lt1/object_samples": 21605.0, "metrics/hit_grouping/calorimeter/abs_eta_lt1/split_fraction": 0.3781994879245758, "metrics/hit_grouping/calorimeter/all/best_slot_completeness": 0.8335619568824768, "metrics/hit_grouping/calorimeter/all/energy_completeness": 0.43153369426727295, "metrics/hit_grouping/calorimeter/all/merged_slot_fraction": 0.35785549879074097, "metrics/hit_grouping/calorimeter/all/object_completeness": 0.43583881855010986, "metrics/hit_grouping/calorimeter/all/object_purity": 0.4111122190952301, "metrics/hit_grouping/calorimeter/all/object_samples": 34880.0, "metrics/hit_grouping/calorimeter/all/split_fraction": 0.39096903800964355, "metrics/hit_grouping/calorimeter/background_assignment_fraction": 0.001743043540045619, "metrics/hit_grouping/calorimeter/background_recall": 0.08277426660060883, "metrics/hit_grouping/calorimeter/completeness": 0.5095558762550354, "metrics/hit_grouping/calorimeter/energy_completeness": 0.514825165271759, "metrics/hit_grouping/calorimeter/energy_purity": 0.5078612565994263, "metrics/hit_grouping/calorimeter/hit_samples": 4015964.0, "metrics/hit_grouping/calorimeter/merge_fraction": 0.44873568415641785, "metrics/hit_grouping/calorimeter/neighbors_0/best_slot_completeness": 0.86180180311203, "metrics/hit_grouping/calorimeter/neighbors_0/energy_completeness": 0.46026262640953064, "metrics/hit_grouping/calorimeter/neighbors_0/merged_slot_fraction": 0.2926149368286133, "metrics/hit_grouping/calorimeter/neighbors_0/object_completeness": 0.4674450159072876, "metrics/hit_grouping/calorimeter/neighbors_0/object_purity": 0.4373875856399536, "metrics/hit_grouping/calorimeter/neighbors_0/object_samples": 17576.0, "metrics/hit_grouping/calorimeter/neighbors_0/split_fraction": 0.32948338985443115, "metrics/hit_grouping/calorimeter/neighbors_1to2/best_slot_completeness": 0.827587366104126, "metrics/hit_grouping/calorimeter/neighbors_1to2/energy_completeness": 0.44759681820869446, "metrics/hit_grouping/calorimeter/neighbors_1to2/merged_slot_fraction": 0.3802395164966583, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_completeness": 0.451448529958725, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_purity": 0.430527925491333, "metrics/hit_grouping/calorimeter/neighbors_1to2/object_samples": 8684.0, "metrics/hit_grouping/calorimeter/neighbors_1to2/split_fraction": 0.4001612067222595, "metrics/hit_grouping/calorimeter/neighbors_ge3/best_slot_completeness": 0.7820004820823669, "metrics/hit_grouping/calorimeter/neighbors_ge3/energy_completeness": 0.35677361488342285, "metrics/hit_grouping/calorimeter/neighbors_ge3/merged_slot_fraction": 0.46832945942878723, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_completeness": 0.3556687831878662, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_purity": 0.3379773497581482, "metrics/hit_grouping/calorimeter/neighbors_ge3/object_samples": 8620.0, "metrics/hit_grouping/calorimeter/neighbors_ge3/split_fraction": 0.5070765614509583, "metrics/hit_grouping/calorimeter/owned_hit_samples": 3945012.0, "metrics/hit_grouping/calorimeter/pt_1to10/best_slot_completeness": 0.8088417053222656, "metrics/hit_grouping/calorimeter/pt_1to10/energy_completeness": 0.4632302522659302, "metrics/hit_grouping/calorimeter/pt_1to10/merged_slot_fraction": 0.4305388629436493, "metrics/hit_grouping/calorimeter/pt_1to10/object_completeness": 0.4700338840484619, "metrics/hit_grouping/calorimeter/pt_1to10/object_purity": 0.4526755213737488, "metrics/hit_grouping/calorimeter/pt_1to10/object_samples": 16628.0, "metrics/hit_grouping/calorimeter/pt_1to10/split_fraction": 0.4406422972679138, "metrics/hit_grouping/calorimeter/pt_ge10/best_slot_completeness": 0.7971879839897156, "metrics/hit_grouping/calorimeter/pt_ge10/energy_completeness": 0.5463292598724365, "metrics/hit_grouping/calorimeter/pt_ge10/merged_slot_fraction": 0.5573855042457581, "metrics/hit_grouping/calorimeter/pt_ge10/object_completeness": 0.5468083024024963, "metrics/hit_grouping/calorimeter/pt_ge10/object_purity": 0.5184643268585205, "metrics/hit_grouping/calorimeter/pt_ge10/object_samples": 1943.0, "metrics/hit_grouping/calorimeter/pt_ge10/split_fraction": 0.4379824995994568, "metrics/hit_grouping/calorimeter/pt_lt1/best_slot_completeness": 0.8630991578102112, "metrics/hit_grouping/calorimeter/pt_lt1/energy_completeness": 0.38554075360298157, "metrics/hit_grouping/calorimeter/pt_lt1/merged_slot_fraction": 0.25997915863990784, "metrics/hit_grouping/calorimeter/pt_lt1/object_completeness": 0.3877543807029724, "metrics/hit_grouping/calorimeter/pt_lt1/object_purity": 0.3559463322162628, "metrics/hit_grouping/calorimeter/pt_lt1/object_samples": 16309.0, "metrics/hit_grouping/calorimeter/pt_lt1/split_fraction": 0.3347231447696686, "metrics/hit_grouping/calorimeter/purity": 0.5014272928237915, "metrics/hit_grouping/calorimeter/surfaces_1to3/best_slot_completeness": 0.8739235997200012, "metrics/hit_grouping/calorimeter/surfaces_1to3/energy_completeness": 0.06464029103517532, "metrics/hit_grouping/calorimeter/surfaces_1to3/merged_slot_fraction": 0.1428571492433548, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_completeness": 0.06627282500267029, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_purity": 0.026043901219964027, "metrics/hit_grouping/calorimeter/surfaces_1to3/object_samples": 532.0, "metrics/hit_grouping/calorimeter/surfaces_1to3/split_fraction": 0.04887218028306961, "metrics/hit_grouping/calorimeter/surfaces_4to7/best_slot_completeness": 0.8595117330551147, "metrics/hit_grouping/calorimeter/surfaces_4to7/energy_completeness": 0.1686537265777588, "metrics/hit_grouping/calorimeter/surfaces_4to7/merged_slot_fraction": 0.12536786496639252, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_completeness": 0.16848115622997284, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_purity": 0.14941070973873138, "metrics/hit_grouping/calorimeter/surfaces_4to7/object_samples": 1699.0, "metrics/hit_grouping/calorimeter/surfaces_4to7/split_fraction": 0.27486756443977356, "metrics/hit_grouping/calorimeter/surfaces_ge8/best_slot_completeness": 0.8315538763999939, "metrics/hit_grouping/calorimeter/surfaces_ge8/energy_completeness": 0.45119187235832214, "metrics/hit_grouping/calorimeter/surfaces_ge8/merged_slot_fraction": 0.37345707416534424, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_completeness": 0.45577359199523926, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_purity": 0.43100523948669434, "metrics/hit_grouping/calorimeter/surfaces_ge8/object_samples": 32649.0, "metrics/hit_grouping/calorimeter/surfaces_ge8/split_fraction": 0.40258505940437317, "metrics/hit_grouping/tracker/abs_eta_1to2/best_slot_completeness": 0.6754899024963379, "metrics/hit_grouping/tracker/abs_eta_1to2/merged_slot_fraction": 0.4508184790611267, "metrics/hit_grouping/tracker/abs_eta_1to2/object_completeness": 0.39230117201805115, "metrics/hit_grouping/tracker/abs_eta_1to2/object_purity": 0.4000963568687439, "metrics/hit_grouping/tracker/abs_eta_1to2/object_samples": 6903.0, "metrics/hit_grouping/tracker/abs_eta_1to2/split_fraction": 0.6828914880752563, "metrics/hit_grouping/tracker/abs_eta_ge2/best_slot_completeness": 0.7413167357444763, "metrics/hit_grouping/tracker/abs_eta_ge2/merged_slot_fraction": 0.31766125559806824, "metrics/hit_grouping/tracker/abs_eta_ge2/object_completeness": 0.3978067636489868, "metrics/hit_grouping/tracker/abs_eta_ge2/object_purity": 0.38583049178123474, "metrics/hit_grouping/tracker/abs_eta_ge2/object_samples": 1659.0, "metrics/hit_grouping/tracker/abs_eta_ge2/split_fraction": 0.5009041428565979, "metrics/hit_grouping/tracker/abs_eta_lt1/best_slot_completeness": 0.5792323350906372, "metrics/hit_grouping/tracker/abs_eta_lt1/merged_slot_fraction": 0.4621528387069702, "metrics/hit_grouping/tracker/abs_eta_lt1/object_completeness": 0.29041653871536255, "metrics/hit_grouping/tracker/abs_eta_lt1/object_purity": 0.3063037097454071, "metrics/hit_grouping/tracker/abs_eta_lt1/object_samples": 13898.0, "metrics/hit_grouping/tracker/abs_eta_lt1/split_fraction": 0.7845733165740967, "metrics/hit_grouping/tracker/all/best_slot_completeness": 0.620789110660553, "metrics/hit_grouping/tracker/all/merged_slot_fraction": 0.44799643754959106, "metrics/hit_grouping/tracker/all/object_completeness": 0.3296627700328827, "metrics/hit_grouping/tracker/all/object_purity": 0.34100475907325745, "metrics/hit_grouping/tracker/all/object_samples": 22460.0, "metrics/hit_grouping/tracker/all/split_fraction": 0.7323686480522156, "metrics/hit_grouping/tracker/background_assignment_fraction": 0.00926778931170702, "metrics/hit_grouping/tracker/background_recall": 0.08363593369722366, "metrics/hit_grouping/tracker/completeness": 0.33219078183174133, "metrics/hit_grouping/tracker/hit_samples": 377760.0, "metrics/hit_grouping/tracker/merge_fraction": 0.5617681741714478, "metrics/hit_grouping/tracker/neighbors_0/best_slot_completeness": 0.6426365971565247, "metrics/hit_grouping/tracker/neighbors_0/merged_slot_fraction": 0.4402945935726166, "metrics/hit_grouping/tracker/neighbors_0/object_completeness": 0.3946307301521301, "metrics/hit_grouping/tracker/neighbors_0/object_purity": 0.4019515812397003, "metrics/hit_grouping/tracker/neighbors_0/object_samples": 11406.0, "metrics/hit_grouping/tracker/neighbors_0/split_fraction": 0.7226898074150085, "metrics/hit_grouping/tracker/neighbors_1to2/best_slot_completeness": 0.6042212247848511, "metrics/hit_grouping/tracker/neighbors_1to2/merged_slot_fraction": 0.4543474316596985, "metrics/hit_grouping/tracker/neighbors_1to2/object_completeness": 0.2969949245452881, "metrics/hit_grouping/tracker/neighbors_1to2/object_purity": 0.3190906345844269, "metrics/hit_grouping/tracker/neighbors_1to2/object_samples": 5509.0, "metrics/hit_grouping/tracker/neighbors_1to2/split_fraction": 0.7513160109519958, "metrics/hit_grouping/tracker/neighbors_ge3/best_slot_completeness": 0.5923091769218445, "metrics/hit_grouping/tracker/neighbors_ge3/merged_slot_fraction": 0.45752930641174316, "metrics/hit_grouping/tracker/neighbors_ge3/object_completeness": 0.22848020493984222, "metrics/hit_grouping/tracker/neighbors_ge3/object_purity": 0.23740969598293304, "metrics/hit_grouping/tracker/neighbors_ge3/object_samples": 5545.0, "metrics/hit_grouping/tracker/neighbors_ge3/split_fraction": 0.7334535717964172, "metrics/hit_grouping/tracker/owned_hit_samples": 352424.0, "metrics/hit_grouping/tracker/pt_1to10/best_slot_completeness": 0.6236371994018555, "metrics/hit_grouping/tracker/pt_1to10/merged_slot_fraction": 0.48025813698768616, "metrics/hit_grouping/tracker/pt_1to10/object_completeness": 0.338491290807724, "metrics/hit_grouping/tracker/pt_1to10/object_purity": 0.3569411039352417, "metrics/hit_grouping/tracker/pt_1to10/object_samples": 11777.0, "metrics/hit_grouping/tracker/pt_1to10/split_fraction": 0.7343126535415649, "metrics/hit_grouping/tracker/pt_ge10/best_slot_completeness": 0.6271321773529053, "metrics/hit_grouping/tracker/pt_ge10/merged_slot_fraction": 0.5166256427764893, "metrics/hit_grouping/tracker/pt_ge10/object_completeness": 0.3440619707107544, "metrics/hit_grouping/tracker/pt_ge10/object_purity": 0.35462287068367004, "metrics/hit_grouping/tracker/pt_ge10/object_samples": 1624.0, "metrics/hit_grouping/tracker/pt_ge10/split_fraction": 0.7222906351089478, "metrics/hit_grouping/tracker/pt_lt1/best_slot_completeness": 0.6159493327140808, "metrics/hit_grouping/tracker/pt_lt1/merged_slot_fraction": 0.3937520682811737, "metrics/hit_grouping/tracker/pt_lt1/object_completeness": 0.3156040608882904, "metrics/hit_grouping/tracker/pt_lt1/object_purity": 0.3178456425666809, "metrics/hit_grouping/tracker/pt_lt1/object_samples": 9059.0, "metrics/hit_grouping/tracker/pt_lt1/split_fraction": 0.7316480875015259, "metrics/hit_grouping/tracker/purity": 0.31281012296676636, "metrics/hit_grouping/tracker/surfaces_1to3/best_slot_completeness": 0.8796011209487915, "metrics/hit_grouping/tracker/surfaces_1to3/merged_slot_fraction": 0.02073906548321247, "metrics/hit_grouping/tracker/surfaces_1to3/object_completeness": 0.054146572947502136, "metrics/hit_grouping/tracker/surfaces_1to3/object_purity": 0.0567302331328392, "metrics/hit_grouping/tracker/surfaces_1to3/object_samples": 2652.0, "metrics/hit_grouping/tracker/surfaces_1to3/split_fraction": 0.06410256773233414, "metrics/hit_grouping/tracker/surfaces_4to7/best_slot_completeness": 0.6486417055130005, "metrics/hit_grouping/tracker/surfaces_4to7/merged_slot_fraction": 0.237996906042099, "metrics/hit_grouping/tracker/surfaces_4to7/object_completeness": 0.20651385188102722, "metrics/hit_grouping/tracker/surfaces_4to7/object_purity": 0.1894722431898117, "metrics/hit_grouping/tracker/surfaces_4to7/object_samples": 1937.0, "metrics/hit_grouping/tracker/surfaces_4to7/split_fraction": 0.5813112854957581, "metrics/hit_grouping/tracker/surfaces_ge8/best_slot_completeness": 0.5793632864952087, "metrics/hit_grouping/tracker/surfaces_ge8/merged_slot_fraction": 0.5341615080833435, "metrics/hit_grouping/tracker/surfaces_ge8/object_completeness": 0.3838963210582733, "metrics/hit_grouping/tracker/surfaces_ge8/object_purity": 0.39961445331573486, "metrics/hit_grouping/tracker/surfaces_ge8/object_samples": 17871.0, "metrics/hit_grouping/tracker/surfaces_ge8/split_fraction": 0.8479100465774536, "metrics/linear_probe/calorimeter/ari_calibration_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/ari_fit_event_samples": 64.0, "metrics/linear_probe/calorimeter/ari_fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/collected_event_samples": 128.0, "metrics/linear_probe/calorimeter/coverage/competing_candidates": 0.7885837554931641, "metrics/linear_probe/calorimeter/coverage/distance_matched_triplets": 0.24337653815746307, "metrics/linear_probe/calorimeter/coverage/positive_in_candidates": 0.932944118976593, "metrics/linear_probe/calorimeter/coverage/same_particle_in_target_layer": 0.9371230602264404, "metrics/linear_probe/calorimeter/coverage/target_layer_exists": 0.9856666326522827, "metrics/linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors": 0.8252502679824829, "metrics/linear_probe/calorimeter/direction_distance/top1_competing": 0.9597394466400146, "metrics/linear_probe/calorimeter/direction_distance/triplet_accuracy": 0.5734716057777405, "metrics/linear_probe/calorimeter/embedding/ari": 0.495576411485672, "metrics/linear_probe/calorimeter/embedding/ari_threshold": 0.7187499403953552, "metrics/linear_probe/calorimeter/embedding/top1_all_candidate_anchors": 0.43604281544685364, "metrics/linear_probe/calorimeter/embedding/top1_competing": 0.482732892036438, "metrics/linear_probe/calorimeter/embedding/triplet_accuracy": 0.5926330089569092, "metrics/linear_probe/calorimeter/embedding_minus_raw/ari": -0.053576309233903885, "metrics/linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy": 0.01815824955701828, "metrics/linear_probe/calorimeter/eval_event_samples": 32.0, "metrics/linear_probe/calorimeter/eval_triplet_samples": 56899.0, "metrics/linear_probe/calorimeter/fit_event_samples": 96.0, "metrics/linear_probe/calorimeter/fit_success_fraction": 1.0, "metrics/linear_probe/calorimeter/fit_triplet_samples": 192182.0, "metrics/linear_probe/calorimeter/oracle_graph/ari": 0.9774430990219116, "metrics/linear_probe/calorimeter/raw/ari": 0.5491527318954468, "metrics/linear_probe/calorimeter/raw/ari_threshold": 0.6999999284744263, "metrics/linear_probe/calorimeter/raw/top1_all_candidate_anchors": 0.722976803779602, "metrics/linear_probe/calorimeter/raw/top1_competing": 0.8391623497009277, "metrics/linear_probe/calorimeter/raw/triplet_accuracy": 0.5744748115539551, "metrics/linear_probe/calorimeter/seconds_per_rank": 21.35959815979004, "metrics/linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors": 0.8278332948684692, "metrics/linear_probe/calorimeter/xyz_distance/top1_competing": 0.9631138443946838, "metrics/linear_probe/calorimeter/xyz_distance/triplet_accuracy": 0.6267632246017456, "metrics/linear_probe/tracker/ari_calibration_event_samples": 32.0, "metrics/linear_probe/tracker/ari_eval_event_samples": 32.0, "metrics/linear_probe/tracker/ari_fit_event_samples": 64.0, "metrics/linear_probe/tracker/ari_fit_success_fraction": 1.0, "metrics/linear_probe/tracker/embedding/ari": 0.21704648435115814, "metrics/linear_probe/tracker/embedding/ari_threshold": 0.8499999642372131, "metrics/linear_probe/tracker/embedding_minus_raw/ari": 0.12542331218719482, "metrics/linear_probe/tracker/oracle_graph/ari": 0.8545429706573486, "metrics/linear_probe/tracker/raw/ari": 0.09162317216396332, "metrics/linear_probe/tracker/raw/ari_threshold": 0.6749999523162842, "metrics/particle/class_1/efficiency": 0.6766029596328735, "metrics/particle/class_1/pid_accuracy": 0.9408630728721619, "metrics/particle/class_2/efficiency": 0.46494507789611816, "metrics/particle/class_2/pid_accuracy": 0.5108404159545898, "metrics/particle/class_3/efficiency": 0.5777714848518372, "metrics/particle/class_3/pid_accuracy": 0.9329146146774292, "metrics/particle/class_4/efficiency": 0.5490981936454773, "metrics/particle/class_4/pid_accuracy": 0.29014599323272705, "metrics/particle/class_5/efficiency": 0.7375478744506836, "metrics/particle/class_5/pid_accuracy": 0.3896103799343109, "metrics/particle/count/bias_mean": -5.740234375, "metrics/particle/count/mae": 7.268880367279053, "metrics/particle/count/prediction_mean": 64.91471099853516, "metrics/particle/count/target_mean": 70.65494537353516, "metrics/particle/event/energy_relative_abs_error": 0.10298022627830505, "metrics/particle/event/energy_response_mean": 1.0062578916549683, "metrics/particle/event/met_abs_error": 10.282136917114258, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08779758214950562, "metrics/particle/event/scalar_pt_response_mean": 1.0296369791030884, "metrics/particle/event/vector_pt_closure": 0.0915655568242073, "metrics/particle/matched/delta_eta_abs_mean": 0.017908640205860138, "metrics/particle/matched/delta_phi_abs_mean": 0.019640082493424416, "metrics/particle/matched/delta_r_mean": 0.029700040817260742, "metrics/particle/matched/energy_relative_abs_mean": 0.1762191653251648, "metrics/particle/matched/pid_accuracy": 0.902859091758728, "metrics/particle/matched/pt_relative_abs_mean": 0.17700688540935516, "metrics/particle/matching/duplicate_fraction": 0.10387226939201355, "metrics/particle/matching/efficiency": 0.616847574710846, "metrics/particle/matching/f1": 0.6429658532142639, "metrics/particle/matching/purity": 0.6713937520980835, "metrics/particle/matching/target_coverage_dr0p05": 0.56563401222229, "metrics/particle/matching/target_coverage_dr0p10": 0.6979525685310364, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.41114571690559387, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6780771613121033, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.027383772656321526, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.1714145690202713, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.17528757452964783, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03152511268854141, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.21637484431266785, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.012249776162207127, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 1.0107570886611938, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.036462362855672836, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 2.509812593460083, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 6694.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3865644037723541, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4898561239242554, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.8870534896850586, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.7327672243118286, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.23212578892707825, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 15.047871589660645, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.021353447809815407, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.4269269704818726, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06035038083791733, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.340336799621582, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 9413.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6214259266853333, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.996612548828125, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.629383087158203, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.847259521484375, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -1.4406834840774536, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.978615939617157, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.787589073181152, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.3945794105529785, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 17.01202392578125, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10772.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026777340099215508, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.17365363240242004, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.18731100857257843, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.052339158952236176, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.22934970259666443, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.034356724470853806, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.7568598985671997, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.02096862718462944, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 4.508480548858643, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1368.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.5829511284828186, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.9438917636871338, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.925346851348877, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6.777065277099609, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.30465444922447205, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 111.22830963134766, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07522331923246384, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1986820697784424, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.02030026540160179, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.42023754119873, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2127.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4960116147994995, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -5.27147912979126, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.732393264770508, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.442275881767273, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.18138878047466278, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.02188970521092415, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.563352584838867, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.850700855255127, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.809036254882812, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 2758.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.031408119946718216, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.17956790328025818, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.17811420559883118, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010541621595621109, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.21910794079303741, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0014489754103124142, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.3510551452636719, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.006771883927285671, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.9561558365821838, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 14493.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.44478556513786316, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 2.2655491828918457, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.8761169910430908, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 1.6943926811218262, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.26357966661453247, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.795472145080566, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.008924099616706371, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.7436166405677795, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.016183700412511826, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.5445826053619385, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21963.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.6228458285331726, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.281430244445801, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 16.003732681274414, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 13.673458099365234, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -6.365053653717041, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 3.0261011123657227, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.963889122009277, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.46688103675842285, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.99468994140625, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 23269.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.029932884499430656, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.17678940296173096, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.17783311009407043, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019304312765598297, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.21893750131130219, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0066504101268947124, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.6321096420288086, "metrics/particle/momentum/all/accepted/pz_bias": 0.016444649547338486, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.920880913734436, "metrics/particle/momentum/all/accepted/samples": 22555.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.4371994137763977, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 2.0271894931793213, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2.19974946975708, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2.0278570652008057, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.2573500871658325, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 30.663854598999023, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.016625376418232918, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.0914638042449951, "metrics/particle/momentum/all/assigned/pz_bias": 0.026276513934135437, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.6074130535125732, "metrics/particle/momentum/all/assigned/samples": 33503.0, "metrics/particle/momentum/all/efficiency": 0.6129242777824402, "metrics/particle/momentum/all/event/energy_bias": 4.006563186645508, "metrics/particle/momentum/all/event/p3_error_mean": 29.037254333496094, "metrics/particle/momentum/all/event/pt_vector_error_mean": 16.704320907592773, "metrics/particle/momentum/all/event/px_bias": -7.987124919891357, "metrics/particle/momentum/all/event/py_bias": 3.982827663421631, "metrics/particle/momentum/all/event/pz_abs_mean": 19.9213924407959, "metrics/particle/momentum/all/event/pz_bias": 0.010759592056274414, "metrics/particle/momentum/all/event/pz_residual_rms": 29.55531120300293, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 36799.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.03441433236002922, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.1821098029613495, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.18249104917049408, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02118036337196827, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.22340378165245056, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0064575644209980965, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.6764043569564819, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.007110584061592817, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 1.7557851076126099, "metrics/particle/momentum/class_1/accepted/samples": 11924.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.3161347508430481, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.9686485528945923, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 1.1304466724395752, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.9495213627815247, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.21130844950675964, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 11.432341575622559, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.01250453107059002, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 1.0075078010559082, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.044865794479846954, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.2835187911987305, "metrics/particle/momentum/class_1/assigned/samples": 16554.0, "metrics/particle/momentum/class_1/efficiency": 0.6721154451370239, "metrics/particle/momentum/class_1/event/energy_bias": 23.4536075592041, "metrics/particle/momentum/class_1/event/p3_error_mean": 25.245737075805664, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.354740142822266, "metrics/particle/momentum/class_1/event/px_bias": -6.0938191413879395, "metrics/particle/momentum/class_1/event/py_bias": 1.566055417060852, "metrics/particle/momentum/class_1/event/pz_abs_mean": 16.605127334594727, "metrics/particle/momentum/class_1/event/pz_bias": 1.3435399532318115, "metrics/particle/momentum/class_1/event/pz_residual_rms": 24.651519775390625, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 17741.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03551770746707916, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21469643712043762, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21643908321857452, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05525946244597435, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2551795244216919, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.034840621054172516, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.796799898147583, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04126644879579544, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.096620559692383, "metrics/particle/momentum/class_2/accepted/samples": 1349.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.553928554058075, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6936545968055725, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.945968747138977, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6036608219146729, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.29058295488357544, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.342311859130859, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.05739910155534744, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.537776231765747, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0024029016494750977, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 5.791956424713135, "metrics/particle/momentum/class_2/assigned/samples": 2230.0, "metrics/particle/momentum/class_2/efficiency": 0.4517749547958374, "metrics/particle/momentum/class_2/event/energy_bias": -19.07074546813965, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.949939727783203, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.111540794372559, "metrics/particle/momentum/class_2/event/px_bias": -1.2993309497833252, "metrics/particle/momentum/class_2/event/py_bias": 1.6249942779541016, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.920307159423828, "metrics/particle/momentum/class_2/event/pz_bias": -2.7344107627868652, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.0262451171875, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2986.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023156462237238884, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1633075475692749, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16522254049777985, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.027781743556261063, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.20600265264511108, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018981688190251589, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.3605028986930847, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.014684242196381092, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.4260035753250122, "metrics/particle/momentum/class_3/accepted/samples": 8956.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5615900754928589, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.383478879928589, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.585672378540039, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.450385808944702, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.3042255640029907, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 45.26332092285156, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.014413273893296719, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9361705183982849, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.004156856331974268, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.488539695739746, "metrics/particle/momentum/class_3/assigned/samples": 14223.0, "metrics/particle/momentum/class_3/efficiency": 0.5765789151191711, "metrics/particle/momentum/class_3/event/energy_bias": 2.087834596633911, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.502266883850098, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.58874797821045, "metrics/particle/momentum/class_3/event/px_bias": -0.5680286288261414, "metrics/particle/momentum/class_3/event/py_bias": 1.3277637958526611, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.783659934997559, "metrics/particle/momentum/class_3/event/pz_bias": 0.8027883172035217, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.403013229370117, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15533.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02879013679921627, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.18861711025238037, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.1849755346775055, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.07060127705335617, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.2284533530473709, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0049019609577953815, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 1.1175185441970825, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.12965449690818787, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 2.2246196269989014, "metrics/particle/momentum/class_4/accepted/samples": 204.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.49256542325019836, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 6.233697891235352, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 5.1655778884887695, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 5.068868637084961, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.3421829044818878, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.76026153564453, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.014749262481927872, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 1.3193191289901733, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.15054379403591156, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 3.1947691440582275, "metrics/particle/momentum/class_4/assigned/samples": 339.0, "metrics/particle/momentum/class_4/efficiency": 0.5440000295639038, "metrics/particle/momentum/class_4/event/energy_bias": -1.5003865957260132, "metrics/particle/momentum/class_4/event/p3_error_mean": 3.4536519050598145, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 2.6338346004486084, "metrics/particle/momentum/class_4/event/px_bias": 0.14604923129081726, "metrics/particle/momentum/class_4/event/py_bias": -0.32714349031448364, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.7843637466430664, "metrics/particle/momentum/class_4/event/pz_bias": 0.14260704815387726, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.990036487579346, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 375.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.029540395364165306, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.20755571126937866, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.20949417352676392, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.04767901450395584, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.24743260443210602, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.06557376682758331, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 2.5513761043548584, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.5942001342773438, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 6.1425580978393555, "metrics/particle/momentum/class_5/accepted/samples": 122.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.15581543743610382, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.6281908750534058, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.7970329523086548, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.5196217894554138, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.21019108593463898, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 4.151170253753662, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.07643312215805054, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 2.9769845008850098, "metrics/particle/momentum/class_5/assigned/pz_bias": 0.8592586517333984, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 6.258808612823486, "metrics/particle/momentum/class_5/assigned/samples": 157.0, "metrics/particle/momentum/class_5/efficiency": 0.7439024448394775, "metrics/particle/momentum/class_5/event/energy_bias": -0.9637446403503418, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.616131067276001, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.0313684940338135, "metrics/particle/momentum/class_5/event/px_bias": -0.17199669778347015, "metrics/particle/momentum/class_5/event/py_bias": -0.20884254574775696, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.266560673713684, "metrics/particle/momentum/class_5/event/pz_bias": 0.45623528957366943, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.233432292938232, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 164.0, "metrics/set_slot/all_angular_matched/energy_abs_log_error": 0.5398198366165161, "metrics/set_slot/all_angular_matched/pt_abs_log_error": 0.5437624454498291, "metrics/set_slot/angular_recall_all_dr0p10": 0.872417688369751, "metrics/set_slot/angular_recall_selected_dr0p10": 0.7082173824310303, "metrics/set_slot/soft_count_bias_mean": 0.013626928441226482}, "step_train_time": 5.509145736694336, "step_valid_time": 26992.90683197975, "step_total_time": 26998.415977954865, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.054808340915424, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2357538385210115, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3059, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3050, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2796, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2721, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9140241909120628, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9167213114754098, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9153707644458995, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08327868852459017, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8895063746322328, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8921311475409837, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8908168276313636, "step/cld_edm_ttbar_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9140241909120628, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0319612163666045, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2119195140713932, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2285, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2290, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2110, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2085, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9234135667396062, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9213973799126638, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9224043715846995, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07860262008733625, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.912472647702407, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9104803493449781, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9114754098360656, "step/cld_edm_ww_fullhad_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9234135667396062, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0340240101571765, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21785664945113425, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1322, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1299, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1170, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1147, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8850226928895613, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9006928406466512, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8927890118275468, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09930715935334873, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.867624810892587, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8829869130100076, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8752384586035864, "step/cld_edm_qq_hits/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8850226928895613} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/hyperparameters.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..eabd5bac77b0fb87f5a2c706305196a28a24f965 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 12786964, "dataset": "cld_hits", "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", "model": {"type": "heptv2", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": false, "output_mode": "set", "binary_classification_focal_gamma": 2.0, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": true, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": {"num_slots": 256, "num_layers": 4, "num_heads": 8, "ffn_multiplier": 4.0, "dropout": 0.0, "query_init": "input-conditioned", "local_attention_radius": 0.4, "attention_mode": "global-flash", "num_sectors": 32, "sector_neighbors": 1, "tracker_query_fraction": 0.6, "proposal_mode": "grid-diverse", "proposal_grid_size": 0.1, "use_aggregate_anchors": true, "aggregate_grid_size": 0.1, "presence_threshold": 0.5, "no_object_weight": 1.0, "cardinality_loss_weight": 0.05, "auxiliary_loss_weight": 0.25, "momentum_loss": "mse", "momentum_huber_delta": 0.2, "hit_grouping": true, "hit_grouping_dim": 32, "hit_grouping_chunk_size": 1024, "hit_grouping_loss_weight": 0.1, "hit_grouping_background_weight": 0.1, "hit_grouping_detector_balanced": false, "hit_grouping_tracker_particle_balanced": false, "tracker_overlap_loss_weight": 0.0, "hit_grouping_sparse": false, "hit_grouping_topk": 4, "hit_grouping_temperature": 1.0, "tracker_momentum_readout": true, "tracker_momentum_max_log_correction": 0.4054651081081644, "tracker_geometry_loss_weight": 0.0, "tracker_curvature_loss_weight": 0.0, "tracker_loss_max_hits": 64, "tracker_group_readout": false, "tracker_fit_correction": false, "tracker_fit_max_log_correction": 0.4054651081081644, "tracker_group_magnetic_field": 2.0, "event_consistency_loss_weight": 0.0, "matcher": {"presence": 1.0, "pid": 1.0, "geometry": 2.0, "pt": 1.0, "energy": 0.0, "dr_scale": 0.1, "log_pt_scale": 0.6931471805599453, "log_energy_scale": 0.6931471805599453}}, "gnnlsh": null, "attention": null, "litept": null, "hept": null, "heptv2": {"activation_checkpointing": true, "embedding_dim": 256, "width": 256, "num_convs": 6, "num_heads": 16, "dropout_ff": 0.0, "activation": "elu", "pe_type": "learned", "block_size": 128, "n_hashes": 3, "num_regions": 140, "deterministic_hashing": true, "num_w_per_dist": 10, "mlp_ratio": 4.0}}, "conv_type": "heptv2", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0001}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 15, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 50000, "patience": 50000, "checkpoint_freq": 1000, "val_freq": 1000, "num_workers": 2, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.001, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "cosinedecay", "lr_schedule_config": {}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 4, "validation_linear_probe": true, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth", "relaxed_load": false, "load_weights_only": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 100, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld_hits": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld_hits": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "cld_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cld_edm_ww_fullhad_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "cld_edm_qq_hits": {"version": "3.2.2", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_hits", "cld_edm_ww_fullhad_hits", "cld_edm_qq_hits"], "ntrain": null, "nvalid": 512, "ntest": 512, "data_config": null, "gpus": 8} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/model_kwargs.pkl b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..4c58583af6a3eb6cf6b3ddc35e619e45376dad79 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e69f18484342dae306407316f3aae0085bd7f3db28e1fbb1153a839dc3d3c6eb +size 6750 diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/particleflow_spec.yaml b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..764890baecded6715d37cf35f42c4079f6e00e2c --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/particleflow_spec.yaml @@ -0,0 +1,2551 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.0 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.0 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id019 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id020 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id019 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id020 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_eta.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..62193f01419363a310a41aaff8b65ff3f3b11d97 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_eta.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_eta.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..d89fff430ec2bb217b0595bcda6ef83b4d949084 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_eta.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b128adcd326a914751abad114b4cc841774a2cc2 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..076733a69a842add54cdc80675cf9bad816fbdcc Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt_log.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8e036ca4c993f4cb4ef717065ce7d2cbb6e722c9 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt_log.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt_log.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..5af35e1e18d566904d7f3342a0c8d279bb7c1309 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_pt_log.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2178891f96d93b6ab32cdd000d0674ed914f54b8 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..877c56340f4e6f4822afc7490d2f14f6b44f04d9 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f954d8664081a95aaca5a73b480bbe76ac16f1ca Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..eed0ac96c4a705f906bd90730b88839850fbadd7 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/jet_res_vstarget.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/num_elements.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8fc8bc567db97ccdb70b97a5cbb45d2f2d172a9d Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/num_elements.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/num_elements.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..20c0094a268d5faa4f266572aecfbfd9e400f625 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_qq_hits/num_elements.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_eta.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..48c8b4a96375e35366f97a25daf1831aee08e399 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_eta.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_eta.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..a4b051210995afaaeebb7fda4a7d0d3a01ad554f Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_eta.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ece4622a76fd8ea6f5f8260df1472f1354462bb2 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..bed615f056d5d538ec0c1214806bc15b0a448ed9 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..abdee4d565d2d4d8d3e8efb339ef6bd0363da2c0 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..d70148ad304bc10f931603edf68130f4d46a64ee Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_pt_log.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..aac48760315b887f61997ed850d95ca1b975917f Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..ffc511b68e5f244643551fe476d62a604618c789 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3e1c2e183045dbeb1ecb136e4694279c17a315ac Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..38ec865897d0ccd9c7195f3646cdcc8e0b067d21 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/jet_res_vstarget.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/num_elements.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ab1aa471ec007d763c16f91a61348d0427ec040f Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/num_elements.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/num_elements.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..d7bd97d4d55dee442dcf6317d1adb5a85aa29a7c Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ttbar_hits/num_elements.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..428fcb28b45d37d42a73f872fd90284db67d9fc8 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..e697a664107521524206cbce483d07f1e2dc1125 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_eta.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dcbf8a0754b8045b95163ba36928fb4dedc79890 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..86dc9804cb3312040583932f2c79c4ac76bbd889 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0635dfec490211a624b0b50b08c0394d52f1b395 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..4b08cea7cda8bb5ad4a208aba6aa20bea951daad Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_pt_log.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..136f94beea302caa424184c9f3263d973f89649a Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..edbb9adfd5e60e10ac45448c0692b0bc97b4eed4 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c8baf28fdee7b21e5addf02c13b546de89142760 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..3701405cb6d90eafb9b0e3211aa8b6b9f1eae40d Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/jet_res_vstarget.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.pdf b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..baea4efdae3e1f13835995a11ee52fccb46cdf2f Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.pdf differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.png b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..cae4c4be74c24304d10c88941171169e157af474 Binary files /dev/null and b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/plots_step_50000/cld_edm_ww_fullhad_hits/num_elements.png differ diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/train/events.out.tfevents.1793178801.nid006495.97037.0 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/train/events.out.tfevents.1793178801.nid006495.97037.0 new file mode 100644 index 0000000000000000000000000000000000000000..84834983f5e95cf97a2bd4e86977695c3a7f8186 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/train/events.out.tfevents.1793178801.nid006495.97037.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:645b68e0082731ba25bfe794170c7bc75e4e9442a5a9f6413cf413593180175d +size 28618778 diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/train/events.out.tfevents.1793278187.nid005451.121603.0 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/train/events.out.tfevents.1793278187.nid005451.121603.0 new file mode 100644 index 0000000000000000000000000000000000000000..67ac6c9a14719359bb3870d108e05adce942f59a --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/train/events.out.tfevents.1793278187.nid005451.121603.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:638cf4bdb226a5d634248919fad0b405923c3aa66215e3f2e2b42d0e39cc243a +size 19187148 diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/valid/events.out.tfevents.1793178801.nid006495.97037.1 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/valid/events.out.tfevents.1793178801.nid006495.97037.1 new file mode 100644 index 0000000000000000000000000000000000000000..e8798c3d16e369784235fce519504f777e427a45 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/valid/events.out.tfevents.1793178801.nid006495.97037.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82907733e494fe149fb45be8d01b947a9e5cce07e82acb36d293b68bd69e1a24 +size 1824633 diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/valid/events.out.tfevents.1793278187.nid005451.121603.1 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/valid/events.out.tfevents.1793278187.nid005451.121603.1 new file mode 100644 index 0000000000000000000000000000000000000000..041e07f794e27eff5d7566f95168091ca6f63c6e --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/runs/valid/events.out.tfevents.1793278187.nid005451.121603.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14decc8db4cb9b59728c3068619a73cc08250359c1170fcd1813d2032e8d2d58 +size 1265308 diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/scenario-manifest.json b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3f2d90e92e9ca518c10c45a92a44cce5f4b6848d --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/scenario-manifest.json @@ -0,0 +1,1224 @@ +{ + "scenario": { + "name": "clic_cld_backbone_benchmark", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld_pf_attention": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": {} + }, + "cld_pf_heptv2": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "cld_hits_attention": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "cld_hits_heptv2": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_pf_attention": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": {} + }, + "clic_pf_heptv2": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "clic_hits_attention": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_hits_heptv2": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 128, + "parameters": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null + } + }, + "common_overrides": { + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true + }, + "allowed_variant_differences": [ + "conv_type", + "data_dir", + "dataset", + "elemtypes_nonzero", + "enabled_test_datasets", + "hyperparameters.lr", + "input_dim", + "model.type", + "model.attention", + "model.heptv2", + "model.binary_classification_focal_gamma", + "model.hit_feature_engineering.tracker_neighborhood", + "model.output_mode", + "model.set_decoder", + "train_dataset", + "valid_dataset", + "test_dataset" + ] + }, + "platform": { + "name": "lumi_mi250x", + "gpus": 8, + "python_command": [ + "python3" + ], + "data_dir": { + "cld": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "cms_run3": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/" + }, + "experiments_dir": "/scratch/project_465001293/patajoos/particleflow/experiments", + "environment": { + "MIOPEN_USER_DB_PATH": "/tmp/patajoos-23239274-miopen-cache", + "MIOPEN_CUSTOM_CACHE_DIR": "/tmp/patajoos-23239274-miopen-cache", + "ROCM_PATH": "/opt/rocm", + "KERAS_BACKEND": "torch", + "OMP_NUM_THREADS": "4", + "PYTHONUNBUFFERED": "1", + "NCCL_SOCKET_IFNAME": "hsn", + "NCCL_NET_GDR_LEVEL": "3" + }, + "runtime_overrides": { + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true + }, + "slurm": { + "partition": "standard-g", + "constraint": null, + "account": "project_465001293", + "time": "2-00:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 32, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "480G", + "memory_per_gpu": null, + "no_requeue": true + } + }, + "job": { + "scenario_name": "clic_cld_backbone_benchmark", + "platform_name": "lumi_mi250x", + "variant_name": "cld_hits_heptv2", + "model_name": "pyg-cld-hits-set-v1", + "production_name": "cld", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "seed": 12345, + "global_batch_size": 128, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null, + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true, + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true, + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld_hits", + "data_dir": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "model": { + "type": "heptv2", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": false, + "output_mode": "set", + "binary_classification_focal_gamma": 2.0, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": true, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": { + "num_slots": 256, + "num_layers": 4, + "num_heads": 8, + "ffn_multiplier": 4.0, + "dropout": 0.0, + "query_init": "input-conditioned", + "local_attention_radius": 0.4, + "attention_mode": "global-flash", + "num_sectors": 32, + "sector_neighbors": 1, + "tracker_query_fraction": 0.6, + "proposal_mode": "grid-diverse", + "proposal_grid_size": 0.1, + "use_aggregate_anchors": true, + "aggregate_grid_size": 0.1, + "presence_threshold": 0.5, + "no_object_weight": 1.0, + "cardinality_loss_weight": 0.05, + "auxiliary_loss_weight": 0.25, + "momentum_loss": "mse", + "momentum_huber_delta": 0.2, + "hit_grouping": true, + "hit_grouping_dim": 32, + "hit_grouping_chunk_size": 1024, + "hit_grouping_loss_weight": 0.1, + "hit_grouping_background_weight": 0.1, + "hit_grouping_detector_balanced": false, + "hit_grouping_tracker_particle_balanced": false, + "tracker_overlap_loss_weight": 0.0, + "hit_grouping_sparse": false, + "hit_grouping_topk": 4, + "hit_grouping_temperature": 1.0, + "tracker_momentum_readout": true, + "tracker_momentum_max_log_correction": 0.4054651081081644, + "tracker_geometry_loss_weight": 0.0, + "tracker_curvature_loss_weight": 0.0, + "tracker_loss_max_hits": 64, + "tracker_group_readout": false, + "tracker_fit_correction": false, + "tracker_fit_max_log_correction": 0.4054651081081644, + "tracker_group_magnetic_field": 2.0, + "event_consistency_loss_weight": 0.0, + "matcher": { + "presence": 1.0, + "pid": 1.0, + "geometry": 2.0, + "pt": 1.0, + "energy": 0.0, + "dr_scale": 0.1, + "log_pt_scale": 0.6931471805599453, + "log_energy_scale": 0.6931471805599453 + } + }, + "gnnlsh": null, + "attention": null, + "litept": null, + "hept": null, + "heptv2": { + "activation_checkpointing": true, + "embedding_dim": 256, + "width": 256, + "num_convs": 6, + "num_heads": 16, + "dropout_ff": 0.0, + "activation": "elu", + "pe_type": "learned", + "block_size": 128, + "n_hashes": 3, + "num_regions": 140, + "deterministic_hashing": true, + "num_w_per_dist": 10, + "mlp_ratio": 4.0 + } + }, + "conv_type": "heptv2", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0001 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 15, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 50000, + "patience": 50000, + "checkpoint_freq": 1000, + "val_freq": 1000, + "num_workers": 2, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.001, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "cosinedecay", + "lr_schedule_config": {}, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 100, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld_hits": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld_hits": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "cld_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cld_edm_ww_fullhad_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "cld_edm_qq_hits": { + "version": "3.2.2", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_hits", + "cld_edm_ww_fullhad_hits", + "cld_edm_qq_hits" + ], + "ntrain": null, + "nvalid": 512, + "ntest": 512, + "data_config": null, + "gpus": 8 + }, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-hits-set-v1", + "--production-name", + "cld", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252", + "train", + "--gpus", + "8", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.type", + "heptv2", + "--model.attention", + "null", + "--model.heptv2.embedding_dim", + "256", + "--model.heptv2.width", + "256", + "--model.heptv2.num_convs", + "6", + "--model.heptv2.num_heads", + "16", + "--model.heptv2.dropout_ff", + "0.0", + "--model.heptv2.block_size", + "128", + "--model.heptv2.deterministic_hashing", + "true", + "--model.heptv2.activation_checkpointing", + "true", + "--model.set_decoder.num_slots", + "256", + "--model.set_decoder.num_layers", + "4", + "--model.set_decoder.num_heads", + "8", + "--model.set_decoder.attention_mode", + "global-flash", + "--model.set_decoder.query_init", + "input-conditioned", + "--model.set_decoder.local_attention_radius", + "0.4", + "--model.set_decoder.tracker_query_fraction", + "0.6", + "--model.set_decoder.proposal_mode", + "grid-diverse", + "--model.set_decoder.proposal_grid_size", + "0.1", + "--model.set_decoder.use_aggregate_anchors", + "true", + "--model.set_decoder.aggregate_grid_size", + "0.1", + "--model.set_decoder.presence_threshold", + "0.5", + "--model.set_decoder.no_object_weight", + "1.0", + "--model.set_decoder.cardinality_loss_weight", + "0.05", + "--model.set_decoder.auxiliary_loss_weight", + "0.25", + "--model.set_decoder.matcher.presence", + "1.0", + "--model.set_decoder.matcher.pid", + "1.0", + "--model.set_decoder.matcher.geometry", + "2.0", + "--model.set_decoder.matcher.pt", + "1.0", + "--model.set_decoder.matcher.energy", + "0.0", + "--model.set_decoder.matcher.dr_scale", + "0.1", + "--model.set_decoder.matcher.log_pt_scale", + "0.6931471805599453", + "--model.set_decoder.matcher.log_energy_scale", + "0.6931471805599453", + "--model.set_decoder.hit_grouping", + "true", + "--model.set_decoder.hit_grouping_dim", + "32", + "--model.set_decoder.hit_grouping_chunk_size", + "1024", + "--model.set_decoder.hit_grouping_loss_weight", + "0.1", + "--model.set_decoder.hit_grouping_background_weight", + "0.1", + "--model.set_decoder.hit_grouping_detector_balanced", + "false", + "--model.set_decoder.hit_grouping_sparse", + "false", + "--model.set_decoder.hit_grouping_tracker_particle_balanced", + "false", + "--model.set_decoder.tracker_momentum_readout", + "true", + "--model.set_decoder.tracker_group_readout", + "false", + "--model.set_decoder.tracker_fit_correction", + "false", + "--model.set_decoder.tracker_overlap_loss_weight", + "0.0", + "--model.set_decoder.tracker_geometry_loss_weight", + "0.0", + "--model.set_decoder.tracker_curvature_loss_weight", + "0.0", + "--model.set_decoder.event_consistency_loss_weight", + "0.0", + "--model.hit_feature_engineering.enabled", + "true", + "--model.hit_feature_engineering.geometry", + "true", + "--model.hit_feature_engineering.calorimeter_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_momentum", + "false", + "--model.hit_feature_engineering.tracker_doublets", + "false", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "continuations": [ + { + "checkpoint": "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth", + "step": 30000, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-hits-set-v1", + "--production-name", + "cld", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252", + "train", + "--gpus", + "8", + "--dtype", + "bfloat16", + "--load", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.type", + "heptv2", + "--model.attention", + "null", + "--model.heptv2.embedding_dim", + "256", + "--model.heptv2.width", + "256", + "--model.heptv2.num_convs", + "6", + "--model.heptv2.num_heads", + "16", + "--model.heptv2.dropout_ff", + "0.0", + "--model.heptv2.block_size", + "128", + "--model.heptv2.deterministic_hashing", + "true", + "--model.heptv2.activation_checkpointing", + "true", + "--model.set_decoder.num_slots", + "256", + "--model.set_decoder.num_layers", + "4", + "--model.set_decoder.num_heads", + "8", + "--model.set_decoder.attention_mode", + "global-flash", + "--model.set_decoder.query_init", + "input-conditioned", + "--model.set_decoder.local_attention_radius", + "0.4", + "--model.set_decoder.tracker_query_fraction", + "0.6", + "--model.set_decoder.proposal_mode", + "grid-diverse", + "--model.set_decoder.proposal_grid_size", + "0.1", + "--model.set_decoder.use_aggregate_anchors", + "true", + "--model.set_decoder.aggregate_grid_size", + "0.1", + "--model.set_decoder.presence_threshold", + "0.5", + "--model.set_decoder.no_object_weight", + "1.0", + "--model.set_decoder.cardinality_loss_weight", + "0.05", + "--model.set_decoder.auxiliary_loss_weight", + "0.25", + "--model.set_decoder.matcher.presence", + "1.0", + "--model.set_decoder.matcher.pid", + "1.0", + "--model.set_decoder.matcher.geometry", + "2.0", + "--model.set_decoder.matcher.pt", + "1.0", + "--model.set_decoder.matcher.energy", + "0.0", + "--model.set_decoder.matcher.dr_scale", + "0.1", + "--model.set_decoder.matcher.log_pt_scale", + "0.6931471805599453", + "--model.set_decoder.matcher.log_energy_scale", + "0.6931471805599453", + "--model.set_decoder.hit_grouping", + "true", + "--model.set_decoder.hit_grouping_dim", + "32", + "--model.set_decoder.hit_grouping_chunk_size", + "1024", + "--model.set_decoder.hit_grouping_loss_weight", + "0.1", + "--model.set_decoder.hit_grouping_background_weight", + "0.1", + "--model.set_decoder.hit_grouping_detector_balanced", + "false", + "--model.set_decoder.hit_grouping_sparse", + "false", + "--model.set_decoder.hit_grouping_tracker_particle_balanced", + "false", + "--model.set_decoder.tracker_momentum_readout", + "true", + "--model.set_decoder.tracker_group_readout", + "false", + "--model.set_decoder.tracker_fit_correction", + "false", + "--model.set_decoder.tracker_overlap_loss_weight", + "0.0", + "--model.set_decoder.tracker_geometry_loss_weight", + "0.0", + "--model.set_decoder.tracker_curvature_loss_weight", + "0.0", + "--model.set_decoder.event_consistency_loss_weight", + "0.0", + "--model.hit_feature_engineering.enabled", + "true", + "--model.hit_feature_engineering.geometry", + "true", + "--model.hit_feature_engineering.calorimeter_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_neighborhood", + "true", + "--model.hit_feature_engineering.tracker_momentum", + "false", + "--model.hit_feature_engineering.tracker_doublets", + "false", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "submitted_at": "2026-10-29T14:48:30.433933" + } + ] +} \ No newline at end of file diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train-config.yaml b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9411cbad4a33d6a1ee583a476e714c73371d5416 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train-config.yaml @@ -0,0 +1,327 @@ +backend: pytorch +batch_size: 1 +checkpoint_freq: 1000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: heptv2 +data_config: null +data_dir: /scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds +dataset: cld_hits +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_hits +- cld_edm_ww_fullhad_hits +- cld_edm_qq_hits +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 8 +hyperparameters: + batch_size: 1 + lr: 0.0001 +input_dim: 15 +load: /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth +load_weights_only: false +lr: 0.001 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: cosinedecay +lr_schedule_config: {} +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: null + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: 2.0 + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + gnnlsh: null + hept: null + heptv2: + activation: elu + activation_checkpointing: true + block_size: 128 + deterministic_hashing: true + dropout_ff: 0.0 + embedding_dim: 256 + mlp_ratio: 4.0 + n_hashes: 3 + num_convs: 6 + num_heads: 16 + num_regions: 140 + num_w_per_dist: 10 + pe_type: learned + width: 256 + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: true + input_encoding: split + learned_representation_mode: last + litept: null + output_mode: set + pt_mode: direct-elemtype-split + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + dropout: 0.0 + event_consistency_loss_weight: 0.0 + ffn_multiplier: 4.0 + hit_grouping: true + hit_grouping_background_weight: 0.1 + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + hit_grouping_sparse: false + hit_grouping_temperature: 1.0 + hit_grouping_topk: 4 + hit_grouping_tracker_particle_balanced: false + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + momentum_huber_delta: 0.2 + momentum_loss: mse + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_curvature_loss_weight: 0.0 + tracker_fit_correction: false + tracker_fit_max_log_correction: 0.4054651081081644 + tracker_geometry_loss_weight: 0.0 + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_loss_max_hits: 64 + tracker_momentum_max_log_correction: 0.4054651081081644 + tracker_momentum_readout: true + tracker_overlap_loss_weight: 0.0 + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + sin_phi_mode: linear + task_queries: false + trainable: all + type: heptv2 +ntest: 512 +ntrain: null +num_classes: 6 +num_steps: 50000 +num_workers: 2 +nvalid: 512 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 50000 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 100 +test: true +test_dataset: + cld_edm_qq_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ttbar_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ww_fullhad_hits: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 +threads: 16 +train: true +train_dataset: + cld_hits: + physical: + batch_size: 1 + samples: + cld_edm_qq_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ttbar_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ww_fullhad_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + weight: null +val_freq: 1000 +valid_dataset: + cld_hits: + physical: + batch_size: 1 + samples: + cld_edm_qq_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ttbar_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + cld_edm_ww_fullhad_hits: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + weight: null +validation_diagnostics_batches: 4 +validation_linear_probe: true +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log new file mode 100644 index 0000000000000000000000000000000000000000..91c7f264a933957d0878bce236738b69aa8b13f4 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log @@ -0,0 +1,20 @@ +rank0 [2026-10-28 11:12:24,685] INFO : Will use torch.nn.parallel.DistributedDataParallel() and 8 gpus +rank0 [2026-10-28 11:12:24,876] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,876] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,877] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,877] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,877] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,877] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,877] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,877] INFO : AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,877] INFO : Spawning DDP processes +rank0 [2026-10-29 14:48:52,486] INFO : Will use torch.nn.parallel.DistributedDataParallel() and 8 gpus +rank0 [2026-10-29 14:48:52,695] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:52,695] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:52,695] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:52,695] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:52,695] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:52,695] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:52,695] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:52,695] INFO : AMD Instinct MI250X +rank0 [2026-10-29 14:48:52,695] INFO : Spawning DDP processes diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.0 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..121126a33c1330ffe61c8e868f1025aae654d636 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.0 @@ -0,0 +1,14943 @@ +rank0 [2026-10-28 11:12:32,403] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:32,404] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:54,158] INFO : Instantiating model +rank0 [2026-10-28 11:12:54,159] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank0 [2026-10-28 11:12:54,159] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:54,177] INFO : Input encoding initialization took 0.02s +rank0 [2026-10-28 11:12:54,177] INFO : nn0 parameters: 171776 +rank0 [2026-10-28 11:12:54,177] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:54,177] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:54,215] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:54,230] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:54,243] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:54,251] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:54,260] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:54,268] INFO : Backbone initialization took 0.09s +rank0 [2026-10-28 11:12:54,269] INFO : backbone parameters: 7901376 +rank0 [2026-10-28 11:12:54,269] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:54,302] INFO : Output DNNs initialization took 0.03s +rank0 [2026-10-28 11:12:54,302] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:54,302] INFO : jagged_attention enabled=False +rank0 [2026-10-28 11:12:54,302] INFO : flash_attn_varlen enabled=False +rank0 [2026-10-28 11:12:54,302] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:54,302] INFO : classification_norm parameters: 0 +rank0 [2026-10-28 11:12:54,303] INFO : regression_norm parameters: 0 +rank0 [2026-10-28 11:12:54,303] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:54,303] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:54,303] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:54,303] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:54,303] INFO : output_mode=set +rank0 [2026-10-28 11:12:54,303] INFO : set_decoder parameters: 4713812 +rank0 [2026-10-28 11:12:54,303] INFO : nn_binary_particle parameters: 0 +rank0 [2026-10-28 11:12:54,303] INFO : nn_pid parameters: 0 +rank0 [2026-10-28 11:12:54,303] INFO : nn_pt parameters: 0 +rank0 [2026-10-28 11:12:54,304] INFO : nn_eta parameters: 0 +rank0 [2026-10-28 11:12:54,304] INFO : nn_sin_phi parameters: 0 +rank0 [2026-10-28 11:12:54,304] INFO : nn_cos_phi parameters: 0 +rank0 [2026-10-28 11:12:54,304] INFO : nn_energy parameters: 0 +rank0 [2026-10-28 11:12:54,305] INFO : Total MLPF parameters: 12786964 +rank0 [2026-10-28 11:12:54,305] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:54,306] INFO : Instantiated model +rank0 [2026-10-28 11:12:54,306] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:54,350] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:54,352] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:54,353] INFO : Configured model for SyncBatchNorm rank=0 +rank0 [2026-10-28 11:12:54,648] INFO : Configured model for DistributedDataParallel rank=0 +rank0 [2026-10-28 11:12:54,810] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank0 [2026-10-28 11:12:54,810] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank0 [2026-10-28 11:12:54,811] INFO : Trainable parameters: 12786964 +rank0 [2026-10-28 11:12:54,811] INFO : Non-trainable parameters: 23268 +rank0 [2026-10-28 11:12:54,811] INFO : Total parameters: 12810232 +rank0 [2026-10-28 11:12:54,815] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank0 [2026-10-28 11:12:54,819] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252 +rank0 [2026-10-28 11:12:54,819] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252 +rank0 [2026-10-28 11:12:54,819] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:54,819] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:02,562] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,678] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,778] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,872] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:02,978] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,094] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,196] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,584] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,666] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-28 11:13:03,750] INFO : train_dataset: cld_edm_ttbar_hits, 90900 +rank0 [2026-10-28 11:13:03,750] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:03,864] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:03,956] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,059] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,161] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,256] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,350] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,463] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,573] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,693] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-28 11:13:04,811] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90900 +rank0 [2026-10-28 11:13:04,812] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:04,899] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,279] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-28 11:13:05,362] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,456] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,564] INFO : train_dataset: cld_edm_qq_hits, 89998 +rank0 [2026-10-28 11:13:05,672] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:05,863] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-28 11:13:05,965] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-28 11:13:06,048] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-28 11:13:06,146] INFO : train_dataset: cld_edm_qq_hits, 90599 +rank0 [2026-10-28 11:13:06,146] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:13:06,146] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:13:06,147] INFO : train mix: physical: weight=21112 step_share=1.000 batches_per_epoch=21112 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:13:06,147] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,174] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,202] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,228] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,256] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,283] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,311] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,337] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,364] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,391] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,418] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-28 11:13:06,418] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,445] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,470] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,496] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,521] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,547] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,574] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,603] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,632] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,659] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,684] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-28 11:13:06,684] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:13:06,710] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,737] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,762] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,790] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,817] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,842] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,868] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,895] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,921] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,946] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-28 11:13:06,947] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:13:06,947] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:13:21,238] INFO : Got dataloaders +rank0 [2026-10-28 11:13:21,239] INFO : loader split=train rank=0 len=21112 +rank0 [2026-10-28 11:13:21,239] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:13:21,239] INFO : Starting training from step 1 to 50000 on rank 0 of 8 +rank0 [2026-10-28 11:13:21,244] INFO : Creating train_iterator +rank0 [2026-10-28 11:13:21,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:13:35,773] INFO : Created train_iterator +rank0 [2026-10-28 11:18:11,208] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5408100029081107, 'Classification': 0.8863177470117808, 'Regression_pt': 0.9386234817653895, 'Regression_eta': 0.019238746587652712, 'Regression_sin_phi': 0.006576384851359762, 'Regression_cos_phi': 0.006405904629500583, 'Regression_energy': 1.0145655398070812}, weights={'Classification_binary': 1.0, 'Classification': 0.6101762056350708, 'Regression_pt': 0.5761735439300537, 'Regression_eta': 28.110458374023438, 'Regression_sin_phi': 82.23515319824219, 'Regression_cos_phi': 84.42367553710938, 'Regression_energy': 0.5330458879470825} +rank0 [2026-10-28 11:18:14,384] INFO : Step 100/50000 rank0 | Train Loss: 4.2631 | Train Loss Unweighted: 2.7366 | Train Loss Components: Auxiliary: 0.7941 | Cardinality: 0.3942 | Classification: 0.6312 | Classification_binary: 0.4549 | HitGrouping: 0.3381 | Regression_cos_phi: 0.0047 | Regression_energy: 0.8263 | Regression_eta: 0.0122 | Regression_pt: 0.8026 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.6564 | Classification_binary: 0.4953 | Regression_cos_phi: 0.0044 | Regression_energy: 0.7260 | Regression_eta: 0.0124 | Regression_pt: 0.6899 | Regression_sin_phi: 0.0043 | LR: 1.00e-03 | DataLoad Time: 0.0163s | Model Forward Time: 4.2013s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6509 | max_loss: 0.1961 | max_backward: 1.4166 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4769.1 | uss_MB: 3475.5 | child_rss_MB: 6372.1 | allocated_MB: 446.7 | reserved_MB: 24804.0 | inactive_split_MB: 1063.6 +rank0 [2026-10-28 11:18:14,620] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 193.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 54.0°C 200.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 51.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 53.0°C 194.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 49.0°C 200.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:22:43,227] INFO : Step 200/50000 rank0 | Train Loss: 4.2029 | Train Loss Unweighted: 2.5447 | Train Loss Components: Auxiliary: 0.8074 | Cardinality: 0.4500 | Classification: 0.5373 | Classification_binary: 0.5135 | HitGrouping: 0.3101 | Regression_cos_phi: 0.0042 | Regression_energy: 0.7508 | Regression_eta: 0.0093 | Regression_pt: 0.7252 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.3404 | Classification_binary: 0.5321 | Regression_cos_phi: 0.3583 | Regression_energy: 0.3414 | Regression_eta: 0.3042 | Regression_pt: 0.3535 | Regression_sin_phi: 0.3667 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 5.3515s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9214 | max_loss: 0.1502 | max_backward: 2.1220 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 4772.0 | uss_MB: 3480.3 | child_rss_MB: 6374.7 | allocated_MB: 449.0 | reserved_MB: 26252.0 | inactive_split_MB: 1054.4 +rank0 [2026-10-28 11:22:43,463] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:27:11,063] INFO : Step 300/50000 rank0 | Train Loss: 3.5114 | Train Loss Unweighted: 2.0975 | Train Loss Components: Auxiliary: 0.6791 | Cardinality: 0.3510 | Classification: 0.4201 | Classification_binary: 0.4975 | HitGrouping: 0.2993 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5752 | Regression_eta: 0.0073 | Regression_pt: 0.5905 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2494 | Classification_binary: 0.4127 | Regression_cos_phi: 0.3112 | Regression_energy: 0.3092 | Regression_eta: 0.2438 | Regression_pt: 0.3301 | Regression_sin_phi: 0.2595 | LR: 1.00e-03 | DataLoad Time: 0.0161s | Model Forward Time: 4.8629s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.7369 | max_loss: 0.1526 | max_backward: 1.7330 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.9030 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 4773.0 | uss_MB: 3484.3 | child_rss_MB: 6364.6 | allocated_MB: 446.1 | reserved_MB: 26252.0 | inactive_split_MB: 1068.0 +rank0 [2026-10-28 11:27:11,312] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 53.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:31:37,795] INFO : Step 400/50000 rank0 | Train Loss: 3.3615 | Train Loss Unweighted: 1.9456 | Train Loss Components: Auxiliary: 0.6310 | Cardinality: 0.3555 | Classification: 0.4058 | Classification_binary: 0.4436 | HitGrouping: 0.3006 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5363 | Regression_eta: 0.0080 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2573 | Classification_binary: 0.4237 | Regression_cos_phi: 0.3088 | Regression_energy: 0.2503 | Regression_eta: 0.2205 | Regression_pt: 0.2803 | Regression_sin_phi: 0.2336 | LR: 1.00e-03 | DataLoad Time: 0.0171s | Model Forward Time: 5.1330s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1151 | max_forward: 0.7642 | max_loss: 0.1384 | max_backward: 1.7906 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9359 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0880 | Memory: rss_MB: 4771.8 | uss_MB: 3484.8 | child_rss_MB: 6366.3 | allocated_MB: 446.8 | reserved_MB: 26252.0 | inactive_split_MB: 1064.0 +rank0 [2026-10-28 11:31:38,044] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:36:09,304] INFO : Step 500/50000 rank0 | Train Loss: 3.8257 | Train Loss Unweighted: 2.0630 | Train Loss Components: Auxiliary: 0.7010 | Cardinality: 0.4740 | Classification: 0.4380 | Classification_binary: 0.4817 | HitGrouping: 0.2961 | Regression_cos_phi: 0.0044 | Regression_energy: 0.5611 | Regression_eta: 0.0087 | Regression_pt: 0.5645 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2736 | Classification_binary: 0.5270 | Regression_cos_phi: 0.4856 | Regression_energy: 0.2616 | Regression_eta: 0.3240 | Regression_pt: 0.2884 | Regression_sin_phi: 0.4697 | LR: 1.00e-03 | DataLoad Time: 0.0165s | Model Forward Time: 5.7697s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0077 | max_forward: 1.2433 | max_loss: 0.1536 | max_backward: 2.4885 | max_optimizer_step: 0.1562 | max_tensorboard_logging: 1.0374 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0034 | Memory: rss_MB: 4768.9 | uss_MB: 3486.1 | child_rss_MB: 6361.5 | allocated_MB: 449.6 | reserved_MB: 26256.0 | inactive_split_MB: 929.2 +rank0 [2026-10-28 11:36:09,542] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 53.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:40:42,774] INFO : Step 600/50000 rank0 | Train Loss: 3.4702 | Train Loss Unweighted: 1.9733 | Train Loss Components: Auxiliary: 0.6509 | Cardinality: 0.3585 | Classification: 0.3825 | Classification_binary: 0.5061 | HitGrouping: 0.2915 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5321 | Regression_eta: 0.0074 | Regression_pt: 0.5377 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2148 | Classification_binary: 0.5029 | Regression_cos_phi: 0.2236 | Regression_energy: 0.2495 | Regression_eta: 0.1794 | Regression_pt: 0.2736 | Regression_sin_phi: 0.2681 | LR: 1.00e-03 | DataLoad Time: 0.0152s | Model Forward Time: 5.1873s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8098 | max_loss: 0.1544 | max_backward: 1.9541 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 0.9377 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 4772.7 | uss_MB: 3486.1 | child_rss_MB: 6366.7 | allocated_MB: 449.5 | reserved_MB: 27396.0 | inactive_split_MB: 1022.5 +rank0 [2026-10-28 11:40:43,025] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:45:17,201] INFO : Step 700/50000 rank0 | Train Loss: 2.9693 | Train Loss Unweighted: 1.8072 | Train Loss Components: Auxiliary: 0.5907 | Cardinality: 0.2768 | Classification: 0.3175 | Classification_binary: 0.3969 | HitGrouping: 0.2848 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5321 | Regression_eta: 0.0061 | Regression_pt: 0.5492 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1955 | Classification_binary: 0.4146 | Regression_cos_phi: 0.2373 | Regression_energy: 0.2540 | Regression_eta: 0.1705 | Regression_pt: 0.2810 | Regression_sin_phi: 0.2064 | LR: 1.00e-03 | DataLoad Time: 0.0212s | Model Forward Time: 4.5412s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7910 | max_loss: 0.2436 | max_backward: 1.8672 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.8 | uss_MB: 3484.3 | child_rss_MB: 6369.5 | allocated_MB: 446.1 | reserved_MB: 27396.0 | inactive_split_MB: 1066.2 +rank0 [2026-10-28 11:45:17,440] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 48.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:49:48,832] INFO : Step 800/50000 rank0 | Train Loss: 3.1527 | Train Loss Unweighted: 1.8424 | Train Loss Components: Auxiliary: 0.6214 | Cardinality: 0.3518 | Classification: 0.3758 | Classification_binary: 0.3967 | HitGrouping: 0.2962 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5290 | Regression_eta: 0.0063 | Regression_pt: 0.5287 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2353 | Classification_binary: 0.4148 | Regression_cos_phi: 0.2387 | Regression_energy: 0.2821 | Regression_eta: 0.1933 | Regression_pt: 0.3148 | Regression_sin_phi: 0.3100 | LR: 9.99e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.2992s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0093 | max_forward: 0.8507 | max_loss: 0.1493 | max_backward: 1.9706 | max_optimizer_step: 0.1582 | max_tensorboard_logging: 1.0840 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0072 | Memory: rss_MB: 4754.0 | uss_MB: 3484.3 | child_rss_MB: 6293.3 | allocated_MB: 450.9 | reserved_MB: 27396.0 | inactive_split_MB: 920.9 +rank0 [2026-10-28 11:49:49,072] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:54:31,297] INFO : Step 900/50000 rank0 | Train Loss: 3.2552 | Train Loss Unweighted: 1.9216 | Train Loss Components: Auxiliary: 0.6208 | Cardinality: 0.2901 | Classification: 0.3818 | Classification_binary: 0.4734 | HitGrouping: 0.2881 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5239 | Regression_eta: 0.0075 | Regression_pt: 0.5284 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2344 | Classification_binary: 0.4985 | Regression_cos_phi: 0.3712 | Regression_energy: 0.2746 | Regression_eta: 0.1874 | Regression_pt: 0.2976 | Regression_sin_phi: 0.3017 | LR: 9.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.3986s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8050 | max_loss: 0.1486 | max_backward: 1.9075 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.9729 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4751.7 | uss_MB: 3485.4 | child_rss_MB: 6307.4 | allocated_MB: 449.7 | reserved_MB: 27406.0 | inactive_split_MB: 927.3 +rank0 [2026-10-28 11:54:31,536] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:59:04,312] INFO : Step 1000/50000 rank0 | Train Loss: 2.8964 | Train Loss Unweighted: 1.7179 | Train Loss Components: Auxiliary: 0.5694 | Cardinality: 0.2509 | Classification: 0.3299 | Classification_binary: 0.4129 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4804 | Regression_eta: 0.0063 | Regression_pt: 0.4827 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1848 | Classification_binary: 0.3581 | Regression_cos_phi: 0.1339 | Regression_energy: 0.2675 | Regression_eta: 0.1178 | Regression_pt: 0.2910 | Regression_sin_phi: 0.1588 | LR: 9.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1136s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7367 | max_loss: 0.2419 | max_backward: 1.8137 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 1.0408 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 4746.9 | uss_MB: 3485.7 | child_rss_MB: 6281.8 | allocated_MB: 449.1 | reserved_MB: 27406.0 | inactive_split_MB: 959.2 +rank0 [2026-10-28 11:59:04,551] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:59:04,573] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:59:05,101] INFO : Running validation on rank0 +rank0 [2026-10-28 11:59:05,102] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:59:05,102] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:59:57,611] INFO : VALIDATION | Step=1000/50000 | Train Loss=2.8964 | Valid Loss=2.9880 | Train Loss Unweighted=1.7179 | Valid Loss Unweighted=1.7335 | Stale=0 | ETA=2283.7m +rank0 [2026-10-28 11:59:57,611] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 11:59:57,611] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 11:59:57,611] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 11:59:57,611] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 11:59:57,611] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 11:59:57,611] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/ari=0.428511 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.234442 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/top1_competing=0.238793 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.527314 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.120642 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.0471611 +rank0 [2026-10-28 11:59:57,612] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/seconds_per_rank=21.6398 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 11:59:57,613] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 11:59:57,614] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 11:59:57,614] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 11:59:57,614] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 11:59:57,614] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding/ari=0.100303 +rank0 [2026-10-28 11:59:57,614] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding/ari_threshold=0.8125 +rank0 [2026-10-28 11:59:57,614] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/embedding_minus_raw/ari=0.00867965 +rank0 [2026-10-28 11:59:57,614] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 11:59:57,614] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 11:59:57,614] INFO : VALIDATION_PROBE | Step=1000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 11:59:57,619] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:59:57,875] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 11:59:57,886] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 12:00:26,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 12:00:28,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 12:00:29,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 12:00:30,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 12:00:40,513] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-28 12:00:40,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:00:41,019] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 12:00:41,022] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 12:00:57,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 12:00:58,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 12:00:59,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 12:01:00,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 12:01:10,599] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 12:01:10,610] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:11,133] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 12:01:11,138] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 12:01:26,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 12:01:27,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 12:01:28,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 12:01:29,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 12:01:39,657] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 12:01:39,673] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits +rank0 [2026-10-28 12:01:42,908] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits +rank0 [2026-10-28 12:01:44,191] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:45,791] INFO : Plotted jets +rank0 [2026-10-28 12:01:46,920] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:46,994] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 12:01:50,242] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 12:01:51,246] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:52,842] INFO : Plotted jets +rank0 [2026-10-28 12:01:54,449] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:54,505] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits +rank0 [2026-10-28 12:01:56,864] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits +rank0 [2026-10-28 12:01:57,710] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:59,187] INFO : Plotted jets +rank0 [2026-10-28 12:02:00,296] INFO : Plotted jet ratio +rank0 [2026-10-28 12:06:48,090] INFO : Step 1100/50000 rank0 | Train Loss: 2.7351 | Train Loss Unweighted: 1.6531 | Train Loss Components: Auxiliary: 0.5524 | Cardinality: 0.2470 | Classification: 0.3238 | Classification_binary: 0.3516 | HitGrouping: 0.2744 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4831 | Regression_eta: 0.0054 | Regression_pt: 0.4842 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2361 | Classification_binary: 0.3642 | Regression_cos_phi: 0.2624 | Regression_energy: 0.2419 | Regression_eta: 0.1557 | Regression_pt: 0.2609 | Regression_sin_phi: 0.2125 | LR: 9.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.3526s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7345 | max_loss: 0.2408 | max_backward: 1.8906 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 1.1618 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6205.6 | uss_MB: 4949.9 | child_rss_MB: 6308.4 | allocated_MB: 448.2 | reserved_MB: 24590.0 | inactive_split_MB: 1051.2 +rank0 [2026-10-28 12:06:48,326] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:11:29,359] INFO : Step 1200/50000 rank0 | Train Loss: 2.9931 | Train Loss Unweighted: 1.7517 | Train Loss Components: Auxiliary: 0.5886 | Cardinality: 0.2847 | Classification: 0.3460 | Classification_binary: 0.4151 | HitGrouping: 0.2795 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4874 | Regression_eta: 0.0065 | Regression_pt: 0.4908 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2068 | Classification_binary: 0.4325 | Regression_cos_phi: 0.2972 | Regression_energy: 0.2629 | Regression_eta: 0.1753 | Regression_pt: 0.2842 | Regression_sin_phi: 0.2180 | LR: 9.99e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.6863s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0061 | max_forward: 0.9910 | max_loss: 0.1662 | max_backward: 2.3256 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.8355 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 6205.6 | uss_MB: 4943.2 | child_rss_MB: 6333.1 | allocated_MB: 449.6 | reserved_MB: 24590.0 | inactive_split_MB: 1155.8 +rank0 [2026-10-28 12:11:29,596] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:16:08,946] INFO : Step 1300/50000 rank0 | Train Loss: 3.0304 | Train Loss Unweighted: 1.7607 | Train Loss Components: Auxiliary: 0.5900 | Cardinality: 0.2909 | Classification: 0.3397 | Classification_binary: 0.4400 | HitGrouping: 0.2683 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4807 | Regression_eta: 0.0062 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2055 | Classification_binary: 0.4384 | Regression_cos_phi: 0.3046 | Regression_energy: 0.2122 | Regression_eta: 0.2124 | Regression_pt: 0.2285 | Regression_sin_phi: 0.2911 | LR: 9.98e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9684s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8206 | max_loss: 0.1504 | max_backward: 1.9515 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5688 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5595.4 | uss_MB: 4349.7 | child_rss_MB: 6286.2 | allocated_MB: 448.6 | reserved_MB: 25778.0 | inactive_split_MB: 1050.0 +rank0 [2026-10-28 12:16:09,181] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 48.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:20:48,898] INFO : Step 1400/50000 rank0 | Train Loss: 2.5732 | Train Loss Unweighted: 1.5137 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2349 | Classification: 0.2906 | Classification_binary: 0.3573 | HitGrouping: 0.2641 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4252 | Regression_eta: 0.0055 | Regression_pt: 0.4303 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1800 | Classification_binary: 0.3419 | Regression_cos_phi: 0.1878 | Regression_energy: 0.2440 | Regression_eta: 0.1382 | Regression_pt: 0.2686 | Regression_sin_phi: 0.1692 | LR: 9.98e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.9474s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.8166 | max_loss: 0.1962 | max_backward: 1.8388 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4964 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5588.2 | uss_MB: 3765.0 | child_rss_MB: 6275.0 | allocated_MB: 454.5 | reserved_MB: 28272.0 | inactive_split_MB: 1143.7 +rank0 [2026-10-28 12:20:49,138] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 24% +4 8 0x7408, 13395 52.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 38% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:25:38,495] INFO : Step 1500/50000 rank0 | Train Loss: 2.8564 | Train Loss Unweighted: 1.5902 | Train Loss Components: Auxiliary: 0.5576 | Cardinality: 0.3788 | Classification: 0.3167 | Classification_binary: 0.3534 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4517 | Regression_eta: 0.0055 | Regression_pt: 0.4576 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2033 | Classification_binary: 0.3643 | Regression_cos_phi: 0.1955 | Regression_energy: 0.2233 | Regression_eta: 0.1168 | Regression_pt: 0.2481 | Regression_sin_phi: 0.1994 | LR: 9.98e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2528s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.0272 | max_loss: 0.2459 | max_backward: 1.8682 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.0883 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0105 | Memory: rss_MB: 5566.9 | uss_MB: 4329.3 | child_rss_MB: 6244.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 989.3 +rank0 [2026-10-28 12:25:38,729] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:30:21,527] INFO : Step 1600/50000 rank0 | Train Loss: 2.8293 | Train Loss Unweighted: 1.6754 | Train Loss Components: Auxiliary: 0.5537 | Cardinality: 0.2456 | Classification: 0.3224 | Classification_binary: 0.4168 | HitGrouping: 0.2726 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4572 | Regression_eta: 0.0058 | Regression_pt: 0.4676 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1928 | Classification_binary: 0.4323 | Regression_cos_phi: 0.2364 | Regression_energy: 0.2306 | Regression_eta: 0.1465 | Regression_pt: 0.2536 | Regression_sin_phi: 0.2038 | LR: 9.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.1748s | Phase Times: max_pre_step_barrier: 0.1094 | max_batch_to_device: 0.0066 | max_forward: 0.7720 | max_loss: 0.1510 | max_backward: 1.7462 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5578.6 | uss_MB: 4348.9 | child_rss_MB: 6226.8 | allocated_MB: 448.1 | reserved_MB: 28272.0 | inactive_split_MB: 989.9 +rank0 [2026-10-28 12:30:21,766] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:35:04,834] INFO : Step 1700/50000 rank0 | Train Loss: 2.7295 | Train Loss Unweighted: 1.5885 | Train Loss Components: Auxiliary: 0.5261 | Cardinality: 0.2505 | Classification: 0.2942 | Classification_binary: 0.3852 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4488 | Regression_eta: 0.0059 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1795 | Classification_binary: 0.3692 | Regression_cos_phi: 0.1788 | Regression_energy: 0.2509 | Regression_eta: 0.1362 | Regression_pt: 0.2753 | Regression_sin_phi: 0.1691 | LR: 9.97e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.6582s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 1.0304 | max_loss: 0.4846 | max_backward: 1.9957 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0278 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5571.8 | uss_MB: 4349.5 | child_rss_MB: 6226.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 999.4 +rank0 [2026-10-28 12:35:05,074] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 92% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:39:47,142] INFO : Step 1800/50000 rank0 | Train Loss: 2.5541 | Train Loss Unweighted: 1.4989 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2504 | Classification: 0.2799 | Classification_binary: 0.3357 | HitGrouping: 0.2658 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4350 | Regression_eta: 0.0052 | Regression_pt: 0.4384 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1840 | Classification_binary: 0.3381 | Regression_cos_phi: 0.2956 | Regression_energy: 0.2124 | Regression_eta: 0.1209 | Regression_pt: 0.2306 | Regression_sin_phi: 0.2399 | LR: 9.97e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9869s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7316 | max_loss: 0.1436 | max_backward: 1.6570 | max_optimizer_step: 0.2625 | max_tensorboard_logging: 0.9220 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5563.3 | uss_MB: 4348.3 | child_rss_MB: 6170.2 | allocated_MB: 444.7 | reserved_MB: 28272.0 | inactive_split_MB: 991.9 +rank0 [2026-10-28 12:39:47,380] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:44:29,939] INFO : Step 1900/50000 rank0 | Train Loss: 2.7201 | Train Loss Unweighted: 1.5780 | Train Loss Components: Auxiliary: 0.5257 | Cardinality: 0.2710 | Classification: 0.3152 | Classification_binary: 0.3657 | HitGrouping: 0.2653 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4371 | Regression_eta: 0.0058 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1964 | Classification_binary: 0.3453 | Regression_cos_phi: 0.2264 | Regression_energy: 0.2430 | Regression_eta: 0.1346 | Regression_pt: 0.2710 | Regression_sin_phi: 0.1855 | LR: 9.97e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2124s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.7723 | max_loss: 0.1448 | max_backward: 2.0316 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5561.2 | uss_MB: 4335.4 | child_rss_MB: 6153.7 | allocated_MB: 448.9 | reserved_MB: 28272.0 | inactive_split_MB: 1166.2 +rank0 [2026-10-28 12:44:30,191] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 48% +2 6 0x7408, 15961 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 9% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 22% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:49:12,683] INFO : Step 2000/50000 rank0 | Train Loss: 2.7893 | Train Loss Unweighted: 1.5772 | Train Loss Components: Auxiliary: 0.5342 | Cardinality: 0.3149 | Classification: 0.3056 | Classification_binary: 0.4233 | HitGrouping: 0.2623 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4169 | Regression_eta: 0.0058 | Regression_pt: 0.4204 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1633 | Classification_binary: 0.4077 | Regression_cos_phi: 0.1503 | Regression_energy: 0.2206 | Regression_eta: 0.1467 | Regression_pt: 0.2443 | Regression_sin_phi: 0.2461 | LR: 9.96e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.2307s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0070 | max_forward: 1.0424 | max_loss: 0.2160 | max_backward: 2.3184 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.9849 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0132 | Memory: rss_MB: 5555.4 | uss_MB: 4316.1 | child_rss_MB: 6148.5 | allocated_MB: 451.3 | reserved_MB: 28272.0 | inactive_split_MB: 992.4 +rank0 [2026-10-28 12:49:12,924] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 8% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 66% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 69% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 98% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:49:12,947] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 12:49:13,395] INFO : Running validation on rank0 +rank0 [2026-10-28 12:49:13,395] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:49:13,395] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:50:08,371] INFO : VALIDATION | Step=2000/50000 | Train Loss=2.7893 | Valid Loss=2.7240 | Train Loss Unweighted=1.5772 | Valid Loss Unweighted=1.5544 | Stale=0 | ETA=2322.9m +rank0 [2026-10-28 12:50:08,371] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 12:50:08,371] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 12:50:08,371] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 12:50:08,371] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 12:50:08,371] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 12:50:08,371] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 12:50:08,371] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/ari=0.445182 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.262491 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/top1_competing=0.273956 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.523939 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.10397 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.0505357 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 12:50:08,372] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/seconds_per_rank=22.0836 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 12:50:08,373] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 12:50:08,374] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 12:50:08,374] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 12:50:08,374] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding/ari=0.147463 +rank0 [2026-10-28 12:50:08,374] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 12:50:08,374] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/embedding_minus_raw/ari=0.05584 +rank0 [2026-10-28 12:50:08,374] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 12:50:08,374] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 12:50:08,374] INFO : VALIDATION_PROBE | Step=2000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 12:50:08,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:50:09,068] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 12:50:09,073] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 12:50:34,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 12:50:35,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 12:50:36,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 12:50:38,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 12:50:48,321] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 12:50:48,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:50:48,899] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 12:50:48,903] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 12:51:04,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 12:51:06,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 12:51:07,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 12:51:08,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 12:51:18,304] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 12:51:18,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:51:18,822] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 12:51:18,825] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 12:51:34,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 12:51:35,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 12:51:36,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 12:51:37,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 12:51:47,729] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 12:51:47,849] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits +rank0 [2026-10-28 12:51:50,962] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits +rank0 [2026-10-28 12:51:51,952] INFO : Plotted number of elements +rank0 [2026-10-28 12:51:53,509] INFO : Plotted jets +rank0 [2026-10-28 12:51:54,662] INFO : Plotted jet ratio +rank0 [2026-10-28 12:51:54,735] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 12:51:57,911] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 12:51:58,879] INFO : Plotted number of elements +rank0 [2026-10-28 12:52:00,372] INFO : Plotted jets +rank0 [2026-10-28 12:52:01,467] INFO : Plotted jet ratio +rank0 [2026-10-28 12:52:01,519] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits +rank0 [2026-10-28 12:52:03,967] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits +rank0 [2026-10-28 12:52:05,257] INFO : Plotted number of elements +rank0 [2026-10-28 12:52:06,778] INFO : Plotted jets +rank0 [2026-10-28 12:52:07,924] INFO : Plotted jet ratio +rank0 [2026-10-28 12:57:02,568] INFO : Step 2100/50000 rank0 | Train Loss: 2.3990 | Train Loss Unweighted: 1.4049 | Train Loss Components: Auxiliary: 0.4671 | Cardinality: 0.2199 | Classification: 0.2491 | Classification_binary: 0.3399 | HitGrouping: 0.2605 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4015 | Regression_eta: 0.0046 | Regression_pt: 0.4053 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1566 | Classification_binary: 0.3386 | Regression_cos_phi: 0.1647 | Regression_energy: 0.2134 | Regression_eta: 0.1274 | Regression_pt: 0.2325 | Regression_sin_phi: 0.1713 | LR: 9.96e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.4718s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7238 | max_loss: 0.1490 | max_backward: 1.8098 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.2800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6608.5 | uss_MB: 5401.4 | child_rss_MB: 6187.1 | allocated_MB: 444.8 | reserved_MB: 23654.0 | inactive_split_MB: 964.6 +rank0 [2026-10-28 12:57:02,806] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:01:41,891] INFO : Step 2200/50000 rank0 | Train Loss: 2.6105 | Train Loss Unweighted: 1.4907 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2489 | Classification: 0.2976 | Classification_binary: 0.3552 | HitGrouping: 0.2519 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4108 | Regression_eta: 0.0054 | Regression_pt: 0.4161 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1871 | Classification_binary: 0.3689 | Regression_cos_phi: 0.3260 | Regression_energy: 0.2166 | Regression_eta: 0.1952 | Regression_pt: 0.2394 | Regression_sin_phi: 0.3201 | LR: 9.96e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.7502s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 1.0006 | max_loss: 0.1446 | max_backward: 1.7671 | max_optimizer_step: 0.2923 | max_tensorboard_logging: 1.2733 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6609.6 | uss_MB: 5403.3 | child_rss_MB: 6178.9 | allocated_MB: 445.7 | reserved_MB: 24780.0 | inactive_split_MB: 1132.5 +rank0 [2026-10-28 13:01:42,130] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 49.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:06:13,370] INFO : Step 2300/50000 rank0 | Train Loss: 2.7462 | Train Loss Unweighted: 1.5643 | Train Loss Components: Auxiliary: 0.5175 | Cardinality: 0.2724 | Classification: 0.3064 | Classification_binary: 0.4106 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4145 | Regression_eta: 0.0056 | Regression_pt: 0.4213 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1940 | Classification_binary: 0.4194 | Regression_cos_phi: 0.2996 | Regression_energy: 0.2382 | Regression_eta: 0.1622 | Regression_pt: 0.2537 | Regression_sin_phi: 0.2286 | LR: 9.95e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.3280s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0353 | max_forward: 0.7930 | max_loss: 0.1572 | max_backward: 1.9535 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0140 | Memory: rss_MB: 5702.4 | uss_MB: 4498.0 | child_rss_MB: 6178.9 | allocated_MB: 451.2 | reserved_MB: 26626.0 | inactive_split_MB: 1036.8 +rank0 [2026-10-28 13:06:13,618] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:10:49,806] INFO : Step 2400/50000 rank0 | Train Loss: 2.5721 | Train Loss Unweighted: 1.4496 | Train Loss Components: Auxiliary: 0.4832 | Cardinality: 0.2667 | Classification: 0.2677 | Classification_binary: 0.3686 | HitGrouping: 0.2521 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3975 | Regression_eta: 0.0053 | Regression_pt: 0.4051 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1718 | Classification_binary: 0.3481 | Regression_cos_phi: 0.2194 | Regression_energy: 0.2003 | Regression_eta: 0.1951 | Regression_pt: 0.2235 | Regression_sin_phi: 0.2014 | LR: 9.95e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.6624s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7367 | max_loss: 0.1564 | max_backward: 1.6902 | max_optimizer_step: 0.2413 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5703.1 | uss_MB: 4496.2 | child_rss_MB: 6167.5 | allocated_MB: 448.4 | reserved_MB: 26626.0 | inactive_split_MB: 1138.1 +rank0 [2026-10-28 13:10:50,044] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 53.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:15:34,186] INFO : Step 2500/50000 rank0 | Train Loss: 2.3709 | Train Loss Unweighted: 1.3478 | Train Loss Components: Auxiliary: 0.4634 | Cardinality: 0.2742 | Classification: 0.2528 | Classification_binary: 0.3100 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3861 | Regression_eta: 0.0042 | Regression_pt: 0.3903 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1538 | Classification_binary: 0.3321 | Regression_cos_phi: 0.1619 | Regression_energy: 0.2188 | Regression_eta: 0.1107 | Regression_pt: 0.2381 | Regression_sin_phi: 0.2044 | LR: 9.94e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.9959s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7598 | max_loss: 0.2442 | max_backward: 1.9617 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5783 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5689.7 | uss_MB: 4483.6 | child_rss_MB: 6173.2 | allocated_MB: 447.2 | reserved_MB: 27052.0 | inactive_split_MB: 1262.0 +rank0 [2026-10-28 13:15:34,424] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:20:26,709] INFO : Step 2600/50000 rank0 | Train Loss: 2.6216 | Train Loss Unweighted: 1.5111 | Train Loss Components: Auxiliary: 0.4988 | Cardinality: 0.2823 | Classification: 0.2863 | Classification_binary: 0.3723 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4174 | Regression_eta: 0.0052 | Regression_pt: 0.4247 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2068 | Classification_binary: 0.4480 | Regression_cos_phi: 0.2875 | Regression_energy: 0.2716 | Regression_eta: 0.1563 | Regression_pt: 0.3015 | Regression_sin_phi: 0.2376 | LR: 9.94e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9517s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8125 | max_loss: 0.1494 | max_backward: 1.9790 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5698.8 | uss_MB: 4500.2 | child_rss_MB: 6171.1 | allocated_MB: 448.2 | reserved_MB: 28734.0 | inactive_split_MB: 1300.2 +rank0 [2026-10-28 13:20:26,945] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 53.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:25:16,653] INFO : Step 2700/50000 rank0 | Train Loss: 2.5735 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4801 | Cardinality: 0.2457 | Classification: 0.2692 | Classification_binary: 0.4021 | HitGrouping: 0.2514 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3948 | Regression_eta: 0.0060 | Regression_pt: 0.3979 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1635 | Classification_binary: 0.3870 | Regression_cos_phi: 0.2241 | Regression_energy: 0.2044 | Regression_eta: 0.1648 | Regression_pt: 0.2225 | Regression_sin_phi: 0.2042 | LR: 9.94e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0891s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0070 | max_forward: 0.8435 | max_loss: 0.1556 | max_backward: 1.9409 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5695.7 | uss_MB: 4500.5 | child_rss_MB: 6182.0 | allocated_MB: 447.4 | reserved_MB: 28734.0 | inactive_split_MB: 980.7 +rank0 [2026-10-28 13:25:16,890] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:29:50,553] INFO : Step 2800/50000 rank0 | Train Loss: 2.3517 | Train Loss Unweighted: 1.3315 | Train Loss Components: Auxiliary: 0.4396 | Cardinality: 0.2218 | Classification: 0.2269 | Classification_binary: 0.3206 | HitGrouping: 0.2464 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3829 | Regression_eta: 0.0052 | Regression_pt: 0.3909 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1508 | Classification_binary: 0.2984 | Regression_cos_phi: 0.2467 | Regression_energy: 0.1816 | Regression_eta: 0.1771 | Regression_pt: 0.2042 | Regression_sin_phi: 0.2290 | LR: 9.93e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.0266s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0062 | max_forward: 0.8898 | max_loss: 0.1952 | max_backward: 1.8882 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0163 | Memory: rss_MB: 5687.7 | uss_MB: 4493.8 | child_rss_MB: 6179.6 | allocated_MB: 445.9 | reserved_MB: 28734.0 | inactive_split_MB: 1208.5 +rank0 [2026-10-28 13:29:50,793] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 3% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +2 6 0x7408, 15961 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:34:35,094] INFO : Step 2900/50000 rank0 | Train Loss: 2.4505 | Train Loss Unweighted: 1.3674 | Train Loss Components: Auxiliary: 0.4603 | Cardinality: 0.2746 | Classification: 0.2611 | Classification_binary: 0.3388 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3750 | Regression_eta: 0.0049 | Regression_pt: 0.3827 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1772 | Classification_binary: 0.3590 | Regression_cos_phi: 0.2389 | Regression_energy: 0.2144 | Regression_eta: 0.1486 | Regression_pt: 0.2362 | Regression_sin_phi: 0.1947 | LR: 9.93e-04 | DataLoad Time: 0.0167s | Model Forward Time: 7.1900s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7484 | max_loss: 0.6089 | max_backward: 2.2353 | max_optimizer_step: 0.1775 | max_tensorboard_logging: 2.4313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0155 | Memory: rss_MB: 5691.8 | uss_MB: 4497.3 | child_rss_MB: 6175.9 | allocated_MB: 446.7 | reserved_MB: 28734.0 | inactive_split_MB: 1133.2 +rank0 [2026-10-28 13:34:35,334] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 52.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 50.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 47.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:39:22,890] INFO : Step 3000/50000 rank0 | Train Loss: 2.6179 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4760 | Cardinality: 0.2600 | Classification: 0.3170 | Classification_binary: 0.3907 | HitGrouping: 0.2509 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3743 | Regression_eta: 0.0054 | Regression_pt: 0.3819 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1931 | Classification_binary: 0.4135 | Regression_cos_phi: 0.2733 | Regression_energy: 0.1923 | Regression_eta: 0.1427 | Regression_pt: 0.2109 | Regression_sin_phi: 0.2309 | LR: 9.92e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.4500s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0842 | max_forward: 0.8203 | max_loss: 0.1511 | max_backward: 1.9714 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.0307 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5691.4 | uss_MB: 4499.3 | child_rss_MB: 6154.8 | allocated_MB: 451.4 | reserved_MB: 28740.0 | inactive_split_MB: 1256.0 +rank0 [2026-10-28 13:39:23,127] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 47% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:39:23,149] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 13:39:23,542] INFO : Running validation on rank0 +rank0 [2026-10-28 13:39:23,542] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:39:23,542] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION | Step=3000/50000 | Train Loss=2.6179 | Valid Loss=2.4543 | Train Loss Unweighted=1.4750 | Valid Loss Unweighted=1.4099 | Stale=0 | ETA=2301.7m +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 13:40:16,318] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/ari=0.463831 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.295137 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/top1_competing=0.312749 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.544799 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0853216 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.0296755 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 13:40:16,319] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/seconds_per_rank=21.5177 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 13:40:16,320] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding/ari=0.155079 +rank0 [2026-10-28 13:40:16,321] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 13:40:16,321] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/embedding_minus_raw/ari=0.0634556 +rank0 [2026-10-28 13:40:16,321] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 13:40:16,321] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 13:40:16,321] INFO : VALIDATION_PROBE | Step=3000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 13:40:16,326] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:40:16,584] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 13:40:16,599] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 13:40:44,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 13:40:45,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 13:40:46,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 13:40:47,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 13:40:57,869] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 13:40:57,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:40:58,579] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 13:40:58,582] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 13:41:14,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 13:41:15,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 13:41:16,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 13:41:18,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 13:41:28,071] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 13:41:28,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:41:28,597] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 13:41:28,601] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 13:41:44,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 13:41:45,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 13:41:46,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 13:41:47,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 13:41:57,222] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 13:41:57,321] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits +rank0 [2026-10-28 13:42:00,490] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits +rank0 [2026-10-28 13:42:01,478] INFO : Plotted number of elements +rank0 [2026-10-28 13:42:03,422] INFO : Plotted jets +rank0 [2026-10-28 13:42:04,579] INFO : Plotted jet ratio +rank0 [2026-10-28 13:42:04,631] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 13:42:07,978] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 13:42:08,942] INFO : Plotted number of elements +rank0 [2026-10-28 13:42:10,443] INFO : Plotted jets +rank0 [2026-10-28 13:42:11,551] INFO : Plotted jet ratio +rank0 [2026-10-28 13:42:11,604] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits +rank0 [2026-10-28 13:42:14,086] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits +rank0 [2026-10-28 13:42:14,908] INFO : Plotted number of elements +rank0 [2026-10-28 13:42:16,385] INFO : Plotted jets +rank0 [2026-10-28 13:42:17,518] INFO : Plotted jet ratio +rank0 [2026-10-28 13:46:52,083] INFO : Step 3100/50000 rank0 | Train Loss: 2.3456 | Train Loss Unweighted: 1.3820 | Train Loss Components: Auxiliary: 0.4397 | Cardinality: 0.2017 | Classification: 0.2549 | Classification_binary: 0.3432 | HitGrouping: 0.2399 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3852 | Regression_eta: 0.0046 | Regression_pt: 0.3892 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1393 | Classification_binary: 0.2604 | Regression_cos_phi: 0.1312 | Regression_energy: 0.2138 | Regression_eta: 0.1098 | Regression_pt: 0.2365 | Regression_sin_phi: 0.1537 | LR: 9.91e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1574s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0425 | max_forward: 0.7824 | max_loss: 0.1439 | max_backward: 1.7183 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.8666 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6905.9 | uss_MB: 5708.9 | child_rss_MB: 6176.0 | allocated_MB: 447.6 | reserved_MB: 24468.0 | inactive_split_MB: 1269.5 +rank0 [2026-10-28 13:46:52,332] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:51:37,664] INFO : Step 3200/50000 rank0 | Train Loss: 2.2453 | Train Loss Unweighted: 1.2729 | Train Loss Components: Auxiliary: 0.4171 | Cardinality: 0.2143 | Classification: 0.2366 | Classification_binary: 0.3116 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3569 | Regression_eta: 0.0047 | Regression_pt: 0.3584 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1782 | Classification_binary: 0.3476 | Regression_cos_phi: 0.2224 | Regression_energy: 0.2048 | Regression_eta: 0.1600 | Regression_pt: 0.2233 | Regression_sin_phi: 0.1932 | LR: 9.91e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.3152s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7985 | max_loss: 0.1379 | max_backward: 1.7895 | max_optimizer_step: 0.2027 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6908.5 | uss_MB: 5710.9 | child_rss_MB: 6167.1 | allocated_MB: 448.6 | reserved_MB: 26448.0 | inactive_split_MB: 1391.4 +rank0 [2026-10-28 13:51:37,913] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:56:17,247] INFO : Step 3300/50000 rank0 | Train Loss: 2.5474 | Train Loss Unweighted: 1.4472 | Train Loss Components: Auxiliary: 0.4779 | Cardinality: 0.2732 | Classification: 0.2824 | Classification_binary: 0.3640 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3911 | Regression_eta: 0.0054 | Regression_pt: 0.3992 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2009 | Classification_binary: 0.4294 | Regression_cos_phi: 0.2743 | Regression_energy: 0.2150 | Regression_eta: 0.2134 | Regression_pt: 0.2352 | Regression_sin_phi: 0.2560 | LR: 9.90e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2944s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8323 | max_loss: 0.1479 | max_backward: 2.0543 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.7834 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5785.1 | uss_MB: 4587.8 | child_rss_MB: 6143.0 | allocated_MB: 448.7 | reserved_MB: 26460.0 | inactive_split_MB: 1087.4 +rank0 [2026-10-28 13:56:17,487] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:00:49,832] INFO : Step 3400/50000 rank0 | Train Loss: 2.5743 | Train Loss Unweighted: 1.4358 | Train Loss Components: Auxiliary: 0.4720 | Cardinality: 0.2664 | Classification: 0.2699 | Classification_binary: 0.3915 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3784 | Regression_eta: 0.0053 | Regression_pt: 0.3851 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1667 | Classification_binary: 0.3868 | Regression_cos_phi: 0.2516 | Regression_energy: 0.2241 | Regression_eta: 0.1671 | Regression_pt: 0.2462 | Regression_sin_phi: 0.2203 | LR: 9.90e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8442s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7827 | max_loss: 0.1630 | max_backward: 1.9191 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5606 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5794.1 | uss_MB: 4597.7 | child_rss_MB: 6161.5 | allocated_MB: 450.4 | reserved_MB: 28138.0 | inactive_split_MB: 1261.0 +rank0 [2026-10-28 14:00:50,070] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:05:24,224] INFO : Step 3500/50000 rank0 | Train Loss: 2.1916 | Train Loss Unweighted: 1.2418 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2204 | Classification: 0.2212 | Classification_binary: 0.3041 | HitGrouping: 0.2426 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3517 | Regression_eta: 0.0047 | Regression_pt: 0.3558 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0847 | Classification_binary: 0.2762 | Regression_cos_phi: 0.1293 | Regression_energy: 0.2086 | Regression_eta: 0.0837 | Regression_pt: 0.2273 | Regression_sin_phi: 0.1185 | LR: 9.89e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.8480s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7972 | max_loss: 0.1441 | max_backward: 1.7398 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6845 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5780.0 | uss_MB: 4584.6 | child_rss_MB: 6167.3 | allocated_MB: 443.7 | reserved_MB: 28532.0 | inactive_split_MB: 1176.3 +rank0 [2026-10-28 14:05:24,462] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:10:07,300] INFO : Step 3600/50000 rank0 | Train Loss: 2.3195 | Train Loss Unweighted: 1.3024 | Train Loss Components: Auxiliary: 0.4248 | Cardinality: 0.2221 | Classification: 0.2518 | Classification_binary: 0.3172 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3610 | Regression_eta: 0.0052 | Regression_pt: 0.3623 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1598 | Classification_binary: 0.3556 | Regression_cos_phi: 0.2086 | Regression_energy: 0.1825 | Regression_eta: 0.1382 | Regression_pt: 0.1957 | Regression_sin_phi: 0.2051 | LR: 9.89e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9146s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8456 | max_loss: 0.1388 | max_backward: 1.9466 | max_optimizer_step: 0.1811 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5787.8 | uss_MB: 4598.8 | child_rss_MB: 6163.2 | allocated_MB: 449.1 | reserved_MB: 28532.0 | inactive_split_MB: 1277.8 +rank0 [2026-10-28 14:10:07,538] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:14:48,540] INFO : Step 3700/50000 rank0 | Train Loss: 2.5861 | Train Loss Unweighted: 1.4427 | Train Loss Components: Auxiliary: 0.4700 | Cardinality: 0.2678 | Classification: 0.2978 | Classification_binary: 0.3844 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3719 | Regression_eta: 0.0058 | Regression_pt: 0.3773 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1623 | Classification_binary: 0.4335 | Regression_cos_phi: 0.2122 | Regression_energy: 0.1811 | Regression_eta: 0.1403 | Regression_pt: 0.1965 | Regression_sin_phi: 0.1870 | LR: 9.88e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2436s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.8001 | max_loss: 0.1519 | max_backward: 1.8239 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.7916 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0181 | Memory: rss_MB: 5775.1 | uss_MB: 4600.5 | child_rss_MB: 6165.4 | allocated_MB: 450.6 | reserved_MB: 30118.0 | inactive_split_MB: 1097.6 +rank0 [2026-10-28 14:14:48,781] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:19:25,255] INFO : Step 3800/50000 rank0 | Train Loss: 2.3269 | Train Loss Unweighted: 1.3016 | Train Loss Components: Auxiliary: 0.4225 | Cardinality: 0.2364 | Classification: 0.2400 | Classification_binary: 0.3389 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3534 | Regression_eta: 0.0054 | Regression_pt: 0.3591 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1240 | Classification_binary: 0.3056 | Regression_cos_phi: 0.1467 | Regression_energy: 0.1756 | Regression_eta: 0.1266 | Regression_pt: 0.1924 | Regression_sin_phi: 0.1936 | LR: 9.87e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.1268s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7555 | max_loss: 0.1601 | max_backward: 1.7612 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.9394 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5772.0 | uss_MB: 4595.7 | child_rss_MB: 6158.5 | allocated_MB: 447.4 | reserved_MB: 30118.0 | inactive_split_MB: 1355.2 +rank0 [2026-10-28 14:19:25,491] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 2% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 48% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 66% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:24:05,988] INFO : Step 3900/50000 rank0 | Train Loss: 2.0276 | Train Loss Unweighted: 1.1874 | Train Loss Components: Auxiliary: 0.3861 | Cardinality: 0.1825 | Classification: 0.2116 | Classification_binary: 0.2707 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3485 | Regression_eta: 0.0038 | Regression_pt: 0.3489 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1566 | Classification_binary: 0.3176 | Regression_cos_phi: 0.2291 | Regression_energy: 0.1855 | Regression_eta: 0.1030 | Regression_pt: 0.2038 | Regression_sin_phi: 0.1768 | LR: 9.87e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.1737s | Phase Times: max_pre_step_barrier: 0.0095 | max_batch_to_device: 0.0140 | max_forward: 1.1007 | max_loss: 0.1867 | max_backward: 2.0952 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 0.5091 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5772.2 | uss_MB: 4598.5 | child_rss_MB: 6165.7 | allocated_MB: 448.6 | reserved_MB: 30118.0 | inactive_split_MB: 1119.4 +rank0 [2026-10-28 14:24:06,227] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:28:53,065] INFO : Step 4000/50000 rank0 | Train Loss: 2.3921 | Train Loss Unweighted: 1.3683 | Train Loss Components: Auxiliary: 0.4366 | Cardinality: 0.2254 | Classification: 0.2720 | Classification_binary: 0.3524 | HitGrouping: 0.2405 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3647 | Regression_eta: 0.0048 | Regression_pt: 0.3692 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1536 | Classification_binary: 0.3728 | Regression_cos_phi: 0.3175 | Regression_energy: 0.1941 | Regression_eta: 0.1575 | Regression_pt: 0.2134 | Regression_sin_phi: 0.2764 | LR: 9.86e-04 | DataLoad Time: 0.0140s | Model Forward Time: 5.6325s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.8532 | max_loss: 0.1476 | max_backward: 2.4362 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.6138 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5773.5 | uss_MB: 4601.5 | child_rss_MB: 6164.9 | allocated_MB: 449.7 | reserved_MB: 30118.0 | inactive_split_MB: 899.8 +rank0 [2026-10-28 14:28:53,303] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 95% +2 6 0x7408, 15961 53.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +4 8 0x7408, 13395 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 99% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 93% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:28:53,437] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 14:28:53,773] INFO : Running validation on rank0 +rank0 [2026-10-28 14:28:53,773] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:28:53,773] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:29:46,666] INFO : VALIDATION | Step=4000/50000 | Train Loss=2.3921 | Valid Loss=2.3126 | Train Loss Unweighted=1.3683 | Valid Loss Unweighted=1.3135 | Stale=0 | ETA=2258.9m +rank0 [2026-10-28 14:29:46,666] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 14:29:46,666] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 14:29:46,666] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 14:29:46,666] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 14:29:46,666] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/ari=0.473108 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.295168 +rank0 [2026-10-28 14:29:46,667] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/top1_competing=0.31359 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.580614 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0760443 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.00613917 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 14:29:46,668] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/seconds_per_rank=21.5241 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding/ari=0.160502 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/embedding_minus_raw/ari=0.0688793 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 14:29:46,669] INFO : VALIDATION_PROBE | Step=4000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 14:29:46,672] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:29:46,929] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 14:29:46,939] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 14:30:13,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 14:30:14,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 14:30:15,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 14:30:16,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 14:30:26,915] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 14:30:26,980] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:30:27,512] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 14:30:27,516] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 14:30:43,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 14:30:44,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 14:30:45,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 14:30:47,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 14:30:57,208] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 14:30:57,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:30:57,799] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 14:30:57,803] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 14:31:13,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 14:31:14,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 14:31:15,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 14:31:16,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 14:31:26,396] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 14:31:26,412] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits +rank0 [2026-10-28 14:31:29,593] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits +rank0 [2026-10-28 14:31:30,976] INFO : Plotted number of elements +rank0 [2026-10-28 14:31:32,528] INFO : Plotted jets +rank0 [2026-10-28 14:31:33,663] INFO : Plotted jet ratio +rank0 [2026-10-28 14:31:33,797] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 14:31:37,129] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 14:31:38,156] INFO : Plotted number of elements +rank0 [2026-10-28 14:31:39,712] INFO : Plotted jets +rank0 [2026-10-28 14:31:40,878] INFO : Plotted jet ratio +rank0 [2026-10-28 14:31:40,929] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits +rank0 [2026-10-28 14:31:43,426] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits +rank0 [2026-10-28 14:31:44,263] INFO : Plotted number of elements +rank0 [2026-10-28 14:31:45,727] INFO : Plotted jets +rank0 [2026-10-28 14:31:47,306] INFO : Plotted jet ratio +rank0 [2026-10-28 14:36:30,847] INFO : Step 4100/50000 rank0 | Train Loss: 2.5062 | Train Loss Unweighted: 1.3582 | Train Loss Components: Auxiliary: 0.4559 | Cardinality: 0.3208 | Classification: 0.2565 | Classification_binary: 0.3823 | HitGrouping: 0.2386 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3543 | Regression_eta: 0.0046 | Regression_pt: 0.3553 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1204 | Classification_binary: 0.3482 | Regression_cos_phi: 0.1803 | Regression_energy: 0.2030 | Regression_eta: 0.1321 | Regression_pt: 0.2168 | Regression_sin_phi: 0.2030 | LR: 9.85e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.8901s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0069 | max_forward: 0.8943 | max_loss: 0.2518 | max_backward: 1.9865 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 1.1537 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6976.0 | uss_MB: 5798.7 | child_rss_MB: 6190.7 | allocated_MB: 449.4 | reserved_MB: 25148.0 | inactive_split_MB: 1028.6 +rank0 [2026-10-28 14:36:31,086] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:41:18,812] INFO : Step 4200/50000 rank0 | Train Loss: 2.0961 | Train Loss Unweighted: 1.2232 | Train Loss Components: Auxiliary: 0.3927 | Cardinality: 0.2101 | Classification: 0.2242 | Classification_binary: 0.2902 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3497 | Regression_eta: 0.0042 | Regression_pt: 0.3510 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1166 | Classification_binary: 0.2525 | Regression_cos_phi: 0.1309 | Regression_energy: 0.2194 | Regression_eta: 0.0935 | Regression_pt: 0.2362 | Regression_sin_phi: 0.1613 | LR: 9.84e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.3113s | Phase Times: max_pre_step_barrier: 0.1131 | max_batch_to_device: 0.0199 | max_forward: 1.5911 | max_loss: 0.3393 | max_backward: 3.0707 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5197 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6973.9 | uss_MB: 5797.7 | child_rss_MB: 6186.7 | allocated_MB: 446.0 | reserved_MB: 25296.0 | inactive_split_MB: 922.1 +rank0 [2026-10-28 14:41:19,049] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:46:19,379] INFO : Step 4300/50000 rank0 | Train Loss: 2.2792 | Train Loss Unweighted: 1.3065 | Train Loss Components: Auxiliary: 0.4224 | Cardinality: 0.2158 | Classification: 0.2339 | Classification_binary: 0.3250 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3669 | Regression_eta: 0.0044 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1420 | Classification_binary: 0.3565 | Regression_cos_phi: 0.1837 | Regression_energy: 0.1977 | Regression_eta: 0.1197 | Regression_pt: 0.2157 | Regression_sin_phi: 0.1323 | LR: 9.84e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.1886s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0938 | max_forward: 0.8451 | max_loss: 0.1396 | max_backward: 1.9669 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.6639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5862.7 | uss_MB: 4685.4 | child_rss_MB: 6195.4 | allocated_MB: 448.4 | reserved_MB: 28580.0 | inactive_split_MB: 1346.0 +rank0 [2026-10-28 14:46:19,637] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:50:59,581] INFO : Step 4400/50000 rank0 | Train Loss: 2.4955 | Train Loss Unweighted: 1.3385 | Train Loss Components: Auxiliary: 0.4510 | Cardinality: 0.3020 | Classification: 0.2394 | Classification_binary: 0.3842 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3482 | Regression_eta: 0.0051 | Regression_pt: 0.3563 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1530 | Classification_binary: 0.4104 | Regression_cos_phi: 0.1790 | Regression_energy: 0.1938 | Regression_eta: 0.1432 | Regression_pt: 0.2146 | Regression_sin_phi: 0.1881 | LR: 9.83e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9001s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0083 | max_forward: 0.7889 | max_loss: 0.1505 | max_backward: 1.8572 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4868 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5870.1 | uss_MB: 4697.1 | child_rss_MB: 6175.1 | allocated_MB: 449.1 | reserved_MB: 28582.0 | inactive_split_MB: 1105.9 +rank0 [2026-10-28 14:50:59,819] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 54.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 53.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 14:55:40,979] INFO : Step 4500/50000 rank0 | Train Loss: 2.3283 | Train Loss Unweighted: 1.2750 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.3015 | Classification: 0.2318 | Classification_binary: 0.3495 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3409 | Regression_eta: 0.0044 | Regression_pt: 0.3441 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1300 | Classification_binary: 0.3352 | Regression_cos_phi: 0.1250 | Regression_energy: 0.1790 | Regression_eta: 0.1111 | Regression_pt: 0.1973 | Regression_sin_phi: 0.1565 | LR: 9.82e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7716s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7326 | max_loss: 0.2428 | max_backward: 1.7841 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5868.2 | uss_MB: 4696.0 | child_rss_MB: 6178.6 | allocated_MB: 447.8 | reserved_MB: 28582.0 | inactive_split_MB: 1258.0 +rank0 [2026-10-28 14:55:41,229] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 53.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:00:17,114] INFO : Step 4600/50000 rank0 | Train Loss: 2.0353 | Train Loss Unweighted: 1.1777 | Train Loss Components: Auxiliary: 0.3804 | Cardinality: 0.1920 | Classification: 0.2053 | Classification_binary: 0.2642 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3490 | Regression_eta: 0.0039 | Regression_pt: 0.3512 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1105 | Classification_binary: 0.2608 | Regression_cos_phi: 0.1249 | Regression_energy: 0.1833 | Regression_eta: 0.1320 | Regression_pt: 0.2060 | Regression_sin_phi: 0.1745 | LR: 9.81e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7430s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0402 | max_forward: 0.7143 | max_loss: 0.1412 | max_backward: 1.6726 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 5861.4 | uss_MB: 4694.6 | child_rss_MB: 6186.8 | allocated_MB: 446.4 | reserved_MB: 28582.0 | inactive_split_MB: 981.4 +rank0 [2026-10-28 15:00:17,353] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 52.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:05:12,863] INFO : Step 4700/50000 rank0 | Train Loss: 2.2426 | Train Loss Unweighted: 1.2576 | Train Loss Components: Auxiliary: 0.4073 | Cardinality: 0.2222 | Classification: 0.2485 | Classification_binary: 0.3333 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3307 | Regression_eta: 0.0049 | Regression_pt: 0.3356 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1654 | Classification_binary: 0.3906 | Regression_cos_phi: 0.2059 | Regression_energy: 0.1645 | Regression_eta: 0.1275 | Regression_pt: 0.1811 | Regression_sin_phi: 0.1900 | LR: 9.81e-04 | DataLoad Time: 0.0144s | Model Forward Time: 5.3947s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.1471 | max_backward: 2.0627 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.7062 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0769 | Memory: rss_MB: 5860.5 | uss_MB: 4699.2 | child_rss_MB: 6169.2 | allocated_MB: 447.7 | reserved_MB: 28652.0 | inactive_split_MB: 1164.9 +rank0 [2026-10-28 15:05:13,104] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +2 6 0x7408, 15961 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:10:00,551] INFO : Step 4800/50000 rank0 | Train Loss: 2.2697 | Train Loss Unweighted: 1.2637 | Train Loss Components: Auxiliary: 0.4132 | Cardinality: 0.2493 | Classification: 0.2262 | Classification_binary: 0.3667 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3287 | Regression_eta: 0.0044 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1061 | Classification_binary: 0.3095 | Regression_cos_phi: 0.1445 | Regression_energy: 0.1731 | Regression_eta: 0.0959 | Regression_pt: 0.1912 | Regression_sin_phi: 0.1177 | LR: 9.80e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.7128s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0065 | max_forward: 0.7626 | max_loss: 0.1571 | max_backward: 1.6465 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5855.3 | uss_MB: 4693.4 | child_rss_MB: 6188.9 | allocated_MB: 450.7 | reserved_MB: 28652.0 | inactive_split_MB: 1088.6 +rank0 [2026-10-28 15:10:00,786] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +2 6 0x7408, 15961 53.0°C 193.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 52.0°C 191.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 97% +6 10 0x7408, 62036 49.0°C 192.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 41% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:14:52,635] INFO : Step 4900/50000 rank0 | Train Loss: 2.0529 | Train Loss Unweighted: 1.1598 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.1942 | Classification: 0.2269 | Classification_binary: 0.2874 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3172 | Regression_eta: 0.0041 | Regression_pt: 0.3199 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1462 | Classification_binary: 0.2683 | Regression_cos_phi: 0.1487 | Regression_energy: 0.1841 | Regression_eta: 0.1055 | Regression_pt: 0.2038 | Regression_sin_phi: 0.1708 | LR: 9.79e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1692s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7293 | max_loss: 0.1974 | max_backward: 1.8633 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5858.6 | uss_MB: 4697.0 | child_rss_MB: 6189.0 | allocated_MB: 447.4 | reserved_MB: 28652.0 | inactive_split_MB: 1266.1 +rank0 [2026-10-28 15:14:52,874] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +2 6 0x7408, 15961 53.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 41% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:20:00,090] INFO : Step 5000/50000 rank0 | Train Loss: 2.0827 | Train Loss Unweighted: 1.1872 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2154 | Classification: 0.2462 | Classification_binary: 0.2941 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3169 | Regression_eta: 0.0042 | Regression_pt: 0.3219 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1756 | Classification_binary: 0.3342 | Regression_cos_phi: 0.2591 | Regression_energy: 0.1631 | Regression_eta: 0.1382 | Regression_pt: 0.1793 | Regression_sin_phi: 0.1452 | LR: 9.78e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9506s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.8382 | max_loss: 0.1450 | max_backward: 1.9472 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0203 | Memory: rss_MB: 5858.9 | uss_MB: 4697.5 | child_rss_MB: 6177.0 | allocated_MB: 446.9 | reserved_MB: 29316.0 | inactive_split_MB: 980.8 +rank0 [2026-10-28 15:20:00,327] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:20:00,348] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 15:20:00,733] INFO : Running validation on rank0 +rank0 [2026-10-28 15:20:00,733] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:20:00,733] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:20:56,512] INFO : VALIDATION | Step=5000/50000 | Train Loss=2.0827 | Valid Loss=2.2621 | Train Loss Unweighted=1.1872 | Valid Loss Unweighted=1.2436 | Stale=0 | ETA=2228.3m +rank0 [2026-10-28 15:20:56,512] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 15:20:56,512] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/ari=0.477157 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 15:20:56,513] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.340501 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/top1_competing=0.364385 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.573334 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0719963 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=-0.00114085 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 15:20:56,514] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/seconds_per_rank=21.4134 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding/ari=0.16769 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/embedding_minus_raw/ari=0.076067 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 15:20:56,515] INFO : VALIDATION_PROBE | Step=5000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 15:20:56,521] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:20:56,775] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 15:20:56,779] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 15:21:22,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 15:21:24,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 15:21:25,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 15:21:26,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 15:21:36,429] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 15:21:36,445] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:21:36,966] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 15:21:36,970] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 15:21:52,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 15:21:54,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 15:21:55,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 15:21:56,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 15:22:06,290] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 15:22:06,552] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:22:07,050] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 15:22:07,054] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 15:22:23,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 15:22:24,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 15:22:25,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 15:22:26,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 15:22:36,710] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 15:22:36,730] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits +rank0 [2026-10-28 15:22:39,928] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits +rank0 [2026-10-28 15:22:40,989] INFO : Plotted number of elements +rank0 [2026-10-28 15:22:42,544] INFO : Plotted jets +rank0 [2026-10-28 15:22:43,695] INFO : Plotted jet ratio +rank0 [2026-10-28 15:22:43,764] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 15:22:47,057] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 15:22:48,048] INFO : Plotted number of elements +rank0 [2026-10-28 15:22:49,969] INFO : Plotted jets +rank0 [2026-10-28 15:22:51,079] INFO : Plotted jet ratio +rank0 [2026-10-28 15:22:51,168] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits +rank0 [2026-10-28 15:22:53,584] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits +rank0 [2026-10-28 15:22:54,441] INFO : Plotted number of elements +rank0 [2026-10-28 15:22:55,968] INFO : Plotted jets +rank0 [2026-10-28 15:22:57,137] INFO : Plotted jet ratio +rank0 [2026-10-28 15:28:09,767] INFO : Step 5100/50000 rank0 | Train Loss: 2.4098 | Train Loss Unweighted: 1.3457 | Train Loss Components: Auxiliary: 0.4318 | Cardinality: 0.2397 | Classification: 0.2711 | Classification_binary: 0.3798 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3391 | Regression_eta: 0.0051 | Regression_pt: 0.3453 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1959 | Classification_binary: 0.3704 | Regression_cos_phi: 0.2444 | Regression_energy: 0.1889 | Regression_eta: 0.1433 | Regression_pt: 0.2077 | Regression_sin_phi: 0.2068 | LR: 9.77e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.4540s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1511 | max_backward: 1.7914 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 1.0544 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5949.7 | uss_MB: 4786.0 | child_rss_MB: 6159.3 | allocated_MB: 447.6 | reserved_MB: 25650.0 | inactive_split_MB: 1122.4 +rank0 [2026-10-28 15:28:10,025] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:32:59,338] INFO : Step 5200/50000 rank0 | Train Loss: 2.0929 | Train Loss Unweighted: 1.1974 | Train Loss Components: Auxiliary: 0.3875 | Cardinality: 0.2007 | Classification: 0.2193 | Classification_binary: 0.3278 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3194 | Regression_eta: 0.0041 | Regression_pt: 0.3227 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1214 | Classification_binary: 0.2800 | Regression_cos_phi: 0.1798 | Regression_energy: 0.1832 | Regression_eta: 0.1128 | Regression_pt: 0.1993 | Regression_sin_phi: 0.1592 | LR: 9.76e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.1670s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7458 | max_loss: 0.2423 | max_backward: 1.7123 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9648 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5957.0 | uss_MB: 4796.7 | child_rss_MB: 6179.0 | allocated_MB: 450.7 | reserved_MB: 28708.0 | inactive_split_MB: 1139.3 +rank0 [2026-10-28 15:32:59,578] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:37:53,222] INFO : Step 5300/50000 rank0 | Train Loss: 1.9905 | Train Loss Unweighted: 1.1352 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.2083 | Classification: 0.2239 | Classification_binary: 0.2818 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0037 | Regression_pt: 0.3135 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1101 | Classification_binary: 0.2397 | Regression_cos_phi: 0.1170 | Regression_energy: 0.1650 | Regression_eta: 0.0762 | Regression_pt: 0.1850 | Regression_sin_phi: 0.0846 | LR: 9.75e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.7286s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 1.2453 | max_loss: 0.2326 | max_backward: 2.1392 | max_optimizer_step: 0.1883 | max_tensorboard_logging: 1.0947 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5945.0 | uss_MB: 4784.6 | child_rss_MB: 6156.3 | allocated_MB: 445.9 | reserved_MB: 30688.0 | inactive_split_MB: 925.3 +rank0 [2026-10-28 15:37:53,462] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:42:40,332] INFO : Step 5400/50000 rank0 | Train Loss: 2.2932 | Train Loss Unweighted: 1.2957 | Train Loss Components: Auxiliary: 0.4189 | Cardinality: 0.2411 | Classification: 0.2689 | Classification_binary: 0.3420 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3351 | Regression_eta: 0.0045 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1372 | Classification_binary: 0.3596 | Regression_cos_phi: 0.2016 | Regression_energy: 0.1942 | Regression_eta: 0.1366 | Regression_pt: 0.2131 | Regression_sin_phi: 0.1630 | LR: 9.74e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2388s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7688 | max_loss: 0.1697 | max_backward: 1.8189 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8641 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5955.2 | uss_MB: 4796.8 | child_rss_MB: 6163.2 | allocated_MB: 450.8 | reserved_MB: 30688.0 | inactive_split_MB: 915.1 +rank0 [2026-10-28 15:42:40,593] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 53.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:47:43,007] INFO : Step 5500/50000 rank0 | Train Loss: 2.1734 | Train Loss Unweighted: 1.2295 | Train Loss Components: Auxiliary: 0.3970 | Cardinality: 0.2269 | Classification: 0.2218 | Classification_binary: 0.3571 | HitGrouping: 0.2200 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3188 | Regression_eta: 0.0041 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1089 | Classification_binary: 0.3063 | Regression_cos_phi: 0.1871 | Regression_energy: 0.1688 | Regression_eta: 0.1010 | Regression_pt: 0.1846 | Regression_sin_phi: 0.1605 | LR: 9.73e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.7073s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0153 | max_forward: 1.1913 | max_loss: 0.6135 | max_backward: 3.0599 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.6982 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5952.5 | uss_MB: 4797.4 | child_rss_MB: 6170.1 | allocated_MB: 446.9 | reserved_MB: 32804.0 | inactive_split_MB: 913.2 +rank0 [2026-10-28 15:47:43,249] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 53% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:52:31,001] INFO : Step 5600/50000 rank0 | Train Loss: 2.0307 | Train Loss Unweighted: 1.1169 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.1966 | Classification: 0.2085 | Classification_binary: 0.3031 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2975 | Regression_eta: 0.0047 | Regression_pt: 0.2988 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0977 | Classification_binary: 0.2561 | Regression_cos_phi: 0.1595 | Regression_energy: 0.1554 | Regression_eta: 0.0925 | Regression_pt: 0.1697 | Regression_sin_phi: 0.1594 | LR: 9.72e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.5066s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7794 | max_loss: 0.1497 | max_backward: 1.9410 | max_optimizer_step: 0.5154 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0061 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5920.9 | uss_MB: 4797.0 | child_rss_MB: 6174.4 | allocated_MB: 445.4 | reserved_MB: 32804.0 | inactive_split_MB: 1164.2 +rank0 [2026-10-28 15:52:31,241] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 53.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 53% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 15:57:17,744] INFO : Step 5700/50000 rank0 | Train Loss: 2.2064 | Train Loss Unweighted: 1.2224 | Train Loss Components: Auxiliary: 0.4023 | Cardinality: 0.2320 | Classification: 0.2526 | Classification_binary: 0.3174 | HitGrouping: 0.2343 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3173 | Regression_eta: 0.0045 | Regression_pt: 0.3259 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1443 | Classification_binary: 0.3192 | Regression_cos_phi: 0.2175 | Regression_energy: 0.1589 | Regression_eta: 0.1472 | Regression_pt: 0.1761 | Regression_sin_phi: 0.1627 | LR: 9.71e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.4473s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7746 | max_loss: 0.4928 | max_backward: 1.9590 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8719 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 5893.1 | uss_MB: 3755.9 | child_rss_MB: 6168.2 | allocated_MB: 448.4 | reserved_MB: 32804.0 | inactive_split_MB: 913.0 +rank0 [2026-10-28 15:57:17,993] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 98% +2 6 0x7408, 15961 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 42% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 53% 98% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:02:01,811] INFO : Step 5800/50000 rank0 | Train Loss: 2.1438 | Train Loss Unweighted: 1.2123 | Train Loss Components: Auxiliary: 0.3898 | Cardinality: 0.2335 | Classification: 0.2404 | Classification_binary: 0.3575 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3006 | Regression_eta: 0.0042 | Regression_pt: 0.3054 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1542 | Classification_binary: 0.3711 | Regression_cos_phi: 0.1754 | Regression_energy: 0.1569 | Regression_eta: 0.1227 | Regression_pt: 0.1698 | Regression_sin_phi: 0.1750 | LR: 9.70e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.2234s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8028 | max_loss: 0.1540 | max_backward: 1.8645 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.8379 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5883.5 | uss_MB: 4797.1 | child_rss_MB: 6169.4 | allocated_MB: 448.3 | reserved_MB: 32804.0 | inactive_split_MB: 916.5 +rank0 [2026-10-28 16:02:02,049] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 49% 0% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 3% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 87% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 98% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 53% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 45% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:06:46,838] INFO : Step 5900/50000 rank0 | Train Loss: 2.2094 | Train Loss Unweighted: 1.1453 | Train Loss Components: Auxiliary: 0.3936 | Cardinality: 0.2891 | Classification: 0.2198 | Classification_binary: 0.3337 | HitGrouping: 0.2238 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2890 | Regression_eta: 0.0046 | Regression_pt: 0.2935 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1025 | Classification_binary: 0.2719 | Regression_cos_phi: 0.1443 | Regression_energy: 0.1525 | Regression_eta: 0.1064 | Regression_pt: 0.1687 | Regression_sin_phi: 0.2335 | LR: 9.69e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.3189s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0065 | max_forward: 1.0922 | max_loss: 0.1477 | max_backward: 1.8468 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8251 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0991 | Memory: rss_MB: 5881.4 | uss_MB: 4798.1 | child_rss_MB: 6148.2 | allocated_MB: 446.5 | reserved_MB: 32804.0 | inactive_split_MB: 927.1 +rank0 [2026-10-28 16:06:47,077] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 53% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:11:32,608] INFO : Step 6000/50000 rank0 | Train Loss: 1.8458 | Train Loss Unweighted: 1.0363 | Train Loss Components: Auxiliary: 0.3398 | Cardinality: 0.1796 | Classification: 0.1819 | Classification_binary: 0.2710 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2865 | Regression_eta: 0.0032 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1250 | Classification_binary: 0.3117 | Regression_cos_phi: 0.2060 | Regression_energy: 0.1581 | Regression_eta: 0.1224 | Regression_pt: 0.1743 | Regression_sin_phi: 0.2131 | LR: 9.68e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1170s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7380 | max_loss: 0.2483 | max_backward: 1.8094 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.9243 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5850.9 | uss_MB: 4795.2 | child_rss_MB: 6173.4 | allocated_MB: 448.1 | reserved_MB: 32804.0 | inactive_split_MB: 1232.3 +rank0 [2026-10-28 16:11:32,847] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 49.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 53% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:11:32,869] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 16:11:33,248] INFO : Running validation on rank0 +rank0 [2026-10-28 16:11:33,248] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:11:33,248] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION | Step=6000/50000 | Train Loss=1.8458 | Valid Loss=2.1188 | Train Loss Unweighted=1.0363 | Valid Loss Unweighted=1.1521 | Stale=0 | ETA=2193.4m +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 16:12:26,913] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/ari=0.490926 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/ari_threshold=0.7 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.333832 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/top1_competing=0.356213 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.576574 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.058227 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.00209953 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 16:12:26,914] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/seconds_per_rank=21.4793 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding/ari=0.175922 +rank0 [2026-10-28 16:12:26,915] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 16:12:26,916] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/embedding_minus_raw/ari=0.0842987 +rank0 [2026-10-28 16:12:26,916] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 16:12:26,916] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 16:12:26,916] INFO : VALIDATION_PROBE | Step=6000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 16:12:26,921] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:12:27,176] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 16:12:27,180] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 16:12:53,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 16:12:55,370] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 16:12:56,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 16:12:57,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 16:13:07,801] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 16:13:07,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:13:08,323] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 16:13:08,327] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 16:13:24,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 16:13:25,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 16:13:26,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 16:13:28,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 16:13:38,088] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 16:13:38,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:13:38,687] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 16:13:38,691] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 16:13:54,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 16:13:55,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 16:13:56,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 16:13:57,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 16:14:07,223] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 16:14:07,239] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits +rank0 [2026-10-28 16:14:10,337] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits +rank0 [2026-10-28 16:14:11,350] INFO : Plotted number of elements +rank0 [2026-10-28 16:14:13,257] INFO : Plotted jets +rank0 [2026-10-28 16:14:14,481] INFO : Plotted jet ratio +rank0 [2026-10-28 16:14:14,549] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 16:14:17,782] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 16:14:18,829] INFO : Plotted number of elements +rank0 [2026-10-28 16:14:20,380] INFO : Plotted jets +rank0 [2026-10-28 16:14:21,514] INFO : Plotted jet ratio +rank0 [2026-10-28 16:14:21,566] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits +rank0 [2026-10-28 16:14:23,995] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits +rank0 [2026-10-28 16:14:24,830] INFO : Plotted number of elements +rank0 [2026-10-28 16:14:26,312] INFO : Plotted jets +rank0 [2026-10-28 16:14:27,910] INFO : Plotted jet ratio +rank0 [2026-10-28 16:19:14,582] INFO : Step 6100/50000 rank0 | Train Loss: 2.0818 | Train Loss Unweighted: 1.1297 | Train Loss Components: Auxiliary: 0.3734 | Cardinality: 0.2287 | Classification: 0.2079 | Classification_binary: 0.3249 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2929 | Regression_eta: 0.0044 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1199 | Classification_binary: 0.3660 | Regression_cos_phi: 0.1973 | Regression_energy: 0.1520 | Regression_eta: 0.1243 | Regression_pt: 0.1643 | Regression_sin_phi: 0.1510 | LR: 9.67e-04 | DataLoad Time: 0.0148s | Model Forward Time: 7.1647s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8130 | max_loss: 0.1524 | max_backward: 3.1921 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 1.4719 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5941.1 | uss_MB: 4883.2 | child_rss_MB: 6189.9 | allocated_MB: 448.4 | reserved_MB: 24434.0 | inactive_split_MB: 1287.8 +rank0 [2026-10-28 16:19:14,819] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 50.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:23:58,778] INFO : Step 6200/50000 rank0 | Train Loss: 2.3716 | Train Loss Unweighted: 1.2065 | Train Loss Components: Auxiliary: 0.4163 | Cardinality: 0.3927 | Classification: 0.2343 | Classification_binary: 0.3679 | HitGrouping: 0.2165 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2973 | Regression_eta: 0.0043 | Regression_pt: 0.2981 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1762 | Classification_binary: 0.3464 | Regression_cos_phi: 0.2062 | Regression_energy: 0.1536 | Regression_eta: 0.1387 | Regression_pt: 0.1670 | Regression_sin_phi: 0.1570 | LR: 9.66e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.2471s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8556 | max_loss: 0.1996 | max_backward: 1.9593 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.5823 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5946.6 | uss_MB: 3767.3 | child_rss_MB: 6181.8 | allocated_MB: 447.9 | reserved_MB: 27020.0 | inactive_split_MB: 1185.4 +rank0 [2026-10-28 16:23:59,018] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 0% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:28:39,275] INFO : Step 6300/50000 rank0 | Train Loss: 1.9346 | Train Loss Unweighted: 1.0816 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.2071 | Classification: 0.2053 | Classification_binary: 0.2781 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2941 | Regression_eta: 0.0037 | Regression_pt: 0.2966 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1497 | Classification_binary: 0.2625 | Regression_cos_phi: 0.1497 | Regression_energy: 0.1426 | Regression_eta: 0.0849 | Regression_pt: 0.1560 | Regression_sin_phi: 0.1684 | LR: 9.65e-04 | DataLoad Time: 0.0135s | Model Forward Time: 5.3583s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0068 | max_forward: 0.7412 | max_loss: 0.1444 | max_backward: 1.7431 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.9045 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5941.6 | uss_MB: 4896.5 | child_rss_MB: 6164.9 | allocated_MB: 444.6 | reserved_MB: 27022.0 | inactive_split_MB: 1185.4 +rank0 [2026-10-28 16:28:39,533] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:33:23,838] INFO : Step 6400/50000 rank0 | Train Loss: 2.0123 | Train Loss Unweighted: 1.0992 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2082 | Classification: 0.2291 | Classification_binary: 0.2885 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2842 | Regression_eta: 0.0040 | Regression_pt: 0.2891 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1555 | Classification_binary: 0.3446 | Regression_cos_phi: 0.1676 | Regression_energy: 0.1641 | Regression_eta: 0.1103 | Regression_pt: 0.1770 | Regression_sin_phi: 0.1900 | LR: 9.64e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.4508s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0069 | max_forward: 1.1063 | max_loss: 0.2471 | max_backward: 1.9949 | max_optimizer_step: 0.2726 | max_tensorboard_logging: 0.8233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5936.1 | uss_MB: 4896.3 | child_rss_MB: 6176.5 | allocated_MB: 451.6 | reserved_MB: 27022.0 | inactive_split_MB: 1113.0 +rank0 [2026-10-28 16:33:24,077] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:37:58,007] INFO : Step 6500/50000 rank0 | Train Loss: 2.1791 | Train Loss Unweighted: 1.2220 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2499 | Classification: 0.2271 | Classification_binary: 0.3508 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3177 | Regression_eta: 0.0042 | Regression_pt: 0.3181 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0956 | Classification_binary: 0.3375 | Regression_cos_phi: 0.1518 | Regression_energy: 0.1445 | Regression_eta: 0.0902 | Regression_pt: 0.1576 | Regression_sin_phi: 0.1462 | LR: 9.63e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.8204s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0548 | max_forward: 0.8069 | max_loss: 0.1518 | max_backward: 2.0692 | max_optimizer_step: 0.1860 | max_tensorboard_logging: 0.9414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5926.6 | uss_MB: 4895.5 | child_rss_MB: 6160.0 | allocated_MB: 449.6 | reserved_MB: 28336.0 | inactive_split_MB: 1199.2 +rank0 [2026-10-28 16:37:58,251] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 53.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 85% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 42% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:42:33,691] INFO : Step 6600/50000 rank0 | Train Loss: 1.9690 | Train Loss Unweighted: 1.0745 | Train Loss Components: Auxiliary: 0.3537 | Cardinality: 0.2132 | Classification: 0.1982 | Classification_binary: 0.3095 | HitGrouping: 0.2136 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2771 | Regression_eta: 0.0040 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1002 | Classification_binary: 0.2717 | Regression_cos_phi: 0.1663 | Regression_energy: 0.1357 | Regression_eta: 0.1044 | Regression_pt: 0.1495 | Regression_sin_phi: 0.1819 | LR: 9.62e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.4734s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7382 | max_loss: 0.1467 | max_backward: 1.8335 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 1.1664 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5923.1 | uss_MB: 4893.3 | child_rss_MB: 6174.8 | allocated_MB: 447.4 | reserved_MB: 28340.0 | inactive_split_MB: 1012.8 +rank0 [2026-10-28 16:42:33,929] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 53.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:47:07,697] INFO : Step 6700/50000 rank0 | Train Loss: 1.9470 | Train Loss Unweighted: 1.0781 | Train Loss Components: Auxiliary: 0.3616 | Cardinality: 0.2363 | Classification: 0.1923 | Classification_binary: 0.2661 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3028 | Regression_eta: 0.0035 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1150 | Classification_binary: 0.2747 | Regression_cos_phi: 0.1628 | Regression_energy: 0.1688 | Regression_eta: 0.1309 | Regression_pt: 0.1855 | Regression_sin_phi: 0.2250 | LR: 9.61e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.4542s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7926 | max_loss: 0.2421 | max_backward: 2.0577 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.7621 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5925.5 | uss_MB: 4898.4 | child_rss_MB: 6176.2 | allocated_MB: 450.1 | reserved_MB: 29806.0 | inactive_split_MB: 1002.0 +rank0 [2026-10-28 16:47:07,960] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 53.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 49.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:51:51,564] INFO : Step 6800/50000 rank0 | Train Loss: 2.1609 | Train Loss Unweighted: 1.1507 | Train Loss Components: Auxiliary: 0.3771 | Cardinality: 0.2180 | Classification: 0.2315 | Classification_binary: 0.3225 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0025 | Regression_energy: 0.2916 | Regression_eta: 0.0048 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1702 | Classification_binary: 0.3778 | Regression_cos_phi: 0.2273 | Regression_energy: 0.1398 | Regression_eta: 0.1556 | Regression_pt: 0.1528 | Regression_sin_phi: 0.1847 | LR: 9.60e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.8406s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0070 | max_forward: 0.9144 | max_loss: 0.4866 | max_backward: 2.1484 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 1.2438 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5920.8 | uss_MB: 4896.0 | child_rss_MB: 6174.1 | allocated_MB: 451.3 | reserved_MB: 29806.0 | inactive_split_MB: 1098.9 +rank0 [2026-10-28 16:51:51,824] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 16:56:33,143] INFO : Step 6900/50000 rank0 | Train Loss: 1.9936 | Train Loss Unweighted: 1.1069 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.1991 | Classification: 0.2085 | Classification_binary: 0.3385 | HitGrouping: 0.2125 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2742 | Regression_eta: 0.0041 | Regression_pt: 0.2777 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1235 | Classification_binary: 0.3053 | Regression_cos_phi: 0.1756 | Regression_energy: 0.1348 | Regression_eta: 0.0962 | Regression_pt: 0.1487 | Regression_sin_phi: 0.1420 | LR: 9.58e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.4129s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7764 | max_loss: 0.2225 | max_backward: 1.6637 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1184 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5918.2 | uss_MB: 4895.4 | child_rss_MB: 6170.7 | allocated_MB: 450.8 | reserved_MB: 29806.0 | inactive_split_MB: 1193.4 +rank0 [2026-10-28 16:56:33,396] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:01:17,196] INFO : Step 7000/50000 rank0 | Train Loss: 1.9256 | Train Loss Unweighted: 1.0803 | Train Loss Components: Auxiliary: 0.3608 | Cardinality: 0.2036 | Classification: 0.1946 | Classification_binary: 0.2738 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2998 | Regression_eta: 0.0039 | Regression_pt: 0.3044 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0875 | Classification_binary: 0.2229 | Regression_cos_phi: 0.1040 | Regression_energy: 0.1652 | Regression_eta: 0.0635 | Regression_pt: 0.1779 | Regression_sin_phi: 0.1205 | LR: 9.57e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.3870s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7896 | max_loss: 0.1557 | max_backward: 1.7415 | max_optimizer_step: 0.2946 | max_tensorboard_logging: 1.0959 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 5913.7 | uss_MB: 4893.6 | child_rss_MB: 6162.0 | allocated_MB: 450.3 | reserved_MB: 29806.0 | inactive_split_MB: 1283.4 +rank0 [2026-10-28 17:01:17,463] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:01:17,486] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 17:01:17,768] INFO : Running validation on rank0 +rank0 [2026-10-28 17:01:17,768] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:01:17,769] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:02:12,481] INFO : VALIDATION | Step=7000/50000 | Train Loss=1.9256 | Valid Loss=2.1012 | Train Loss Unweighted=1.0803 | Valid Loss Unweighted=1.1402 | Stale=0 | ETA=2143.0m +rank0 [2026-10-28 17:02:12,481] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 17:02:12,481] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 17:02:12,481] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 17:02:12,481] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 17:02:12,481] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 17:02:12,481] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/ari=0.495174 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.354727 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/top1_competing=0.380401 +rank0 [2026-10-28 17:02:12,482] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.580276 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0539789 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.00580122 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 17:02:12,483] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/seconds_per_rank=21.3678 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding/ari=0.199337 +rank0 [2026-10-28 17:02:12,484] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 17:02:12,485] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/embedding_minus_raw/ari=0.107714 +rank0 [2026-10-28 17:02:12,485] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 17:02:12,485] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 17:02:12,485] INFO : VALIDATION_PROBE | Step=7000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 17:02:12,490] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:02:12,761] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 17:02:12,766] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 17:02:39,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 17:02:40,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 17:02:41,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 17:02:42,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 17:02:52,579] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 17:02:52,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:02:53,110] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 17:02:53,114] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 17:03:09,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 17:03:10,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 17:03:11,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 17:03:12,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 17:03:22,761] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 17:03:22,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:03:23,342] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 17:03:23,346] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 17:03:38,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 17:03:40,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 17:03:40,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 17:03:42,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 17:03:52,010] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 17:03:52,026] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits +rank0 [2026-10-28 17:03:55,174] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits +rank0 [2026-10-28 17:03:56,234] INFO : Plotted number of elements +rank0 [2026-10-28 17:03:58,167] INFO : Plotted jets +rank0 [2026-10-28 17:03:59,361] INFO : Plotted jet ratio +rank0 [2026-10-28 17:03:59,416] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 17:04:02,678] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 17:04:03,708] INFO : Plotted number of elements +rank0 [2026-10-28 17:04:05,209] INFO : Plotted jets +rank0 [2026-10-28 17:04:06,356] INFO : Plotted jet ratio +rank0 [2026-10-28 17:04:06,411] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits +rank0 [2026-10-28 17:04:08,862] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits +rank0 [2026-10-28 17:04:09,728] INFO : Plotted number of elements +rank0 [2026-10-28 17:04:11,211] INFO : Plotted jets +rank0 [2026-10-28 17:04:12,359] INFO : Plotted jet ratio +rank0 [2026-10-28 17:08:59,619] INFO : Step 7100/50000 rank0 | Train Loss: 1.8251 | Train Loss Unweighted: 1.0448 | Train Loss Components: Auxiliary: 0.3403 | Cardinality: 0.1731 | Classification: 0.2040 | Classification_binary: 0.2675 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2809 | Regression_eta: 0.0035 | Regression_pt: 0.2853 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1140 | Classification_binary: 0.2579 | Regression_cos_phi: 0.1100 | Regression_energy: 0.1567 | Regression_eta: 0.0869 | Regression_pt: 0.1692 | Regression_sin_phi: 0.1276 | LR: 9.56e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.8804s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0622 | max_forward: 0.8282 | max_loss: 0.1540 | max_backward: 1.9352 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.3584 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6015.2 | uss_MB: 4988.2 | child_rss_MB: 6155.9 | allocated_MB: 449.9 | reserved_MB: 22498.0 | inactive_split_MB: 1193.4 +rank0 [2026-10-28 17:08:59,877] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 47.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:13:32,432] INFO : Step 7200/50000 rank0 | Train Loss: 2.1146 | Train Loss Unweighted: 1.1506 | Train Loss Components: Auxiliary: 0.3887 | Cardinality: 0.2411 | Classification: 0.2243 | Classification_binary: 0.3469 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2844 | Regression_eta: 0.0044 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1268 | Classification_binary: 0.3569 | Regression_cos_phi: 0.1736 | Regression_energy: 0.1361 | Regression_eta: 0.1036 | Regression_pt: 0.1470 | Regression_sin_phi: 0.1533 | LR: 9.55e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.2781s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0369 | max_forward: 0.7969 | max_loss: 0.1517 | max_backward: 1.9380 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6030.2 | uss_MB: 5003.5 | child_rss_MB: 6152.6 | allocated_MB: 448.6 | reserved_MB: 22906.0 | inactive_split_MB: 1087.6 +rank0 [2026-10-28 17:13:32,673] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 53.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 2% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 20% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 97% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:18:12,794] INFO : Step 7300/50000 rank0 | Train Loss: 1.8657 | Train Loss Unweighted: 1.0129 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2046 | Classification: 0.1759 | Classification_binary: 0.2802 | HitGrouping: 0.2112 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2746 | Regression_eta: 0.0037 | Regression_pt: 0.2747 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1101 | Classification_binary: 0.2616 | Regression_cos_phi: 0.1581 | Regression_energy: 0.1603 | Regression_eta: 0.0870 | Regression_pt: 0.1760 | Regression_sin_phi: 0.1543 | LR: 9.53e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2914s | Phase Times: max_pre_step_barrier: 0.0082 | max_batch_to_device: 0.0150 | max_forward: 0.7713 | max_loss: 0.1507 | max_backward: 1.8303 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.6113 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6029.9 | uss_MB: 5003.9 | child_rss_MB: 6151.8 | allocated_MB: 449.9 | reserved_MB: 25044.0 | inactive_split_MB: 918.5 +rank0 [2026-10-28 17:18:13,033] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 32% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:22:48,110] INFO : Step 7400/50000 rank0 | Train Loss: 1.8326 | Train Loss Unweighted: 1.0006 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2227 | Classification: 0.1795 | Classification_binary: 0.2563 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2767 | Regression_eta: 0.0034 | Regression_pt: 0.2813 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0900 | Classification_binary: 0.2355 | Regression_cos_phi: 0.0891 | Regression_energy: 0.1577 | Regression_eta: 0.0911 | Regression_pt: 0.1768 | Regression_sin_phi: 0.1282 | LR: 9.52e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.8379s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0684 | max_forward: 0.7762 | max_loss: 0.1384 | max_backward: 1.7881 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6026.9 | uss_MB: 5000.5 | child_rss_MB: 6151.8 | allocated_MB: 447.4 | reserved_MB: 25044.0 | inactive_split_MB: 1152.4 +rank0 [2026-10-28 17:22:48,348] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:27:29,663] INFO : Step 7500/50000 rank0 | Train Loss: 2.1479 | Train Loss Unweighted: 1.1510 | Train Loss Components: Auxiliary: 0.3847 | Cardinality: 0.2329 | Classification: 0.2181 | Classification_binary: 0.3323 | HitGrouping: 0.2143 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2947 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1123 | Classification_binary: 0.2913 | Regression_cos_phi: 0.1359 | Regression_energy: 0.1576 | Regression_eta: 0.0843 | Regression_pt: 0.1693 | Regression_sin_phi: 0.1567 | LR: 9.51e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0653s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7726 | max_loss: 0.2518 | max_backward: 1.8458 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1040 | Memory: rss_MB: 6026.8 | uss_MB: 5002.5 | child_rss_MB: 6153.2 | allocated_MB: 452.3 | reserved_MB: 26846.0 | inactive_split_MB: 1117.1 +rank0 [2026-10-28 17:27:29,901] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:32:30,274] INFO : Step 7600/50000 rank0 | Train Loss: 1.9984 | Train Loss Unweighted: 1.0873 | Train Loss Components: Auxiliary: 0.3667 | Cardinality: 0.2088 | Classification: 0.2053 | Classification_binary: 0.3405 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2647 | Regression_eta: 0.0038 | Regression_pt: 0.2689 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1053 | Classification_binary: 0.3026 | Regression_cos_phi: 0.1507 | Regression_energy: 0.1321 | Regression_eta: 0.1110 | Regression_pt: 0.1436 | Regression_sin_phi: 0.1109 | LR: 9.50e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1966s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7834 | max_loss: 0.1641 | max_backward: 1.8407 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.8255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6029.1 | uss_MB: 5003.6 | child_rss_MB: 6150.1 | allocated_MB: 448.9 | reserved_MB: 26846.0 | inactive_split_MB: 1232.3 +rank0 [2026-10-28 17:32:30,512] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 49.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:37:20,619] INFO : Step 7700/50000 rank0 | Train Loss: 1.7524 | Train Loss Unweighted: 0.9761 | Train Loss Components: Auxiliary: 0.3200 | Cardinality: 0.1783 | Classification: 0.1756 | Classification_binary: 0.2521 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2705 | Regression_eta: 0.0035 | Regression_pt: 0.2709 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0902 | Classification_binary: 0.2421 | Regression_cos_phi: 0.1822 | Regression_energy: 0.1330 | Regression_eta: 0.0897 | Regression_pt: 0.1453 | Regression_sin_phi: 0.1146 | LR: 9.48e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.2513s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7659 | max_loss: 0.1551 | max_backward: 1.8103 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6027.2 | uss_MB: 5004.4 | child_rss_MB: 6154.6 | allocated_MB: 448.8 | reserved_MB: 26846.0 | inactive_split_MB: 1042.2 +rank0 [2026-10-28 17:37:20,860] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:42:11,383] INFO : Step 7800/50000 rank0 | Train Loss: 1.8039 | Train Loss Unweighted: 1.0226 | Train Loss Components: Auxiliary: 0.3378 | Cardinality: 0.1686 | Classification: 0.1822 | Classification_binary: 0.2556 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2863 | Regression_eta: 0.0034 | Regression_pt: 0.2914 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1022 | Classification_binary: 0.3008 | Regression_cos_phi: 0.1674 | Regression_energy: 0.1548 | Regression_eta: 0.0905 | Regression_pt: 0.1717 | Regression_sin_phi: 0.1735 | LR: 9.47e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.3357s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.8680 | max_loss: 0.1424 | max_backward: 1.9949 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.8163 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 6024.0 | uss_MB: 5003.5 | child_rss_MB: 6156.5 | allocated_MB: 448.6 | reserved_MB: 26846.0 | inactive_split_MB: 1396.9 +rank0 [2026-10-28 17:42:11,639] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:47:05,827] INFO : Step 7900/50000 rank0 | Train Loss: 2.0641 | Train Loss Unweighted: 1.0960 | Train Loss Components: Auxiliary: 0.3855 | Cardinality: 0.2401 | Classification: 0.2031 | Classification_binary: 0.3367 | HitGrouping: 0.2085 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2721 | Regression_eta: 0.0041 | Regression_pt: 0.2759 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1211 | Classification_binary: 0.3290 | Regression_cos_phi: 0.1707 | Regression_energy: 0.1370 | Regression_eta: 0.1309 | Regression_pt: 0.1520 | Regression_sin_phi: 0.2153 | LR: 9.46e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.3621s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7971 | max_loss: 0.2430 | max_backward: 1.9131 | max_optimizer_step: 0.2681 | max_tensorboard_logging: 0.7971 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6025.3 | uss_MB: 5004.3 | child_rss_MB: 6161.5 | allocated_MB: 451.8 | reserved_MB: 27044.0 | inactive_split_MB: 827.4 +rank0 [2026-10-28 17:47:06,065] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:51:56,739] INFO : Step 8000/50000 rank0 | Train Loss: 1.8391 | Train Loss Unweighted: 0.9813 | Train Loss Components: Auxiliary: 0.3355 | Cardinality: 0.2153 | Classification: 0.1625 | Classification_binary: 0.2896 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2605 | Regression_eta: 0.0035 | Regression_pt: 0.2616 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0939 | Classification_binary: 0.2251 | Regression_cos_phi: 0.1269 | Regression_energy: 0.1118 | Regression_eta: 0.0762 | Regression_pt: 0.1221 | Regression_sin_phi: 0.1642 | LR: 9.44e-04 | DataLoad Time: 0.0177s | Model Forward Time: 7.1704s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 1.2188 | max_loss: 0.4570 | max_backward: 3.9408 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.5566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6013.9 | uss_MB: 4999.4 | child_rss_MB: 6155.0 | allocated_MB: 448.4 | reserved_MB: 27044.0 | inactive_split_MB: 1156.9 +rank0 [2026-10-28 17:51:56,977] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 17:51:57,000] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 17:51:57,380] INFO : Running validation on rank0 +rank0 [2026-10-28 17:51:57,381] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:51:57,381] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:52:50,625] INFO : VALIDATION | Step=8000/50000 | Train Loss=1.8391 | Valid Loss=1.9817 | Train Loss Unweighted=0.9813 | Valid Loss Unweighted=1.0261 | Stale=0 | ETA=2097.3m +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 17:52:50,626] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/ari=0.491441 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.393893 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/top1_competing=0.428208 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.579607 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0577115 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.00513234 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 17:52:50,627] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/seconds_per_rank=21.4732 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding/ari=0.194276 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/embedding_minus_raw/ari=0.102653 +rank0 [2026-10-28 17:52:50,628] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 17:52:50,629] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 17:52:50,629] INFO : VALIDATION_PROBE | Step=8000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 17:52:50,634] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:52:50,893] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 17:52:50,904] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 17:53:16,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 17:53:17,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 17:53:19,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 17:53:20,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 17:53:30,237] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 17:53:30,250] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:53:30,806] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 17:53:30,810] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 17:53:47,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 17:53:49,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 17:53:50,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 17:53:51,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 17:54:01,619] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 17:54:01,630] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:54:02,177] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 17:54:02,181] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 17:54:17,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 17:54:18,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 17:54:19,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 17:54:20,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 17:54:30,834] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 17:54:30,909] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits +rank0 [2026-10-28 17:54:34,197] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits +rank0 [2026-10-28 17:54:35,287] INFO : Plotted number of elements +rank0 [2026-10-28 17:54:36,858] INFO : Plotted jets +rank0 [2026-10-28 17:54:38,021] INFO : Plotted jet ratio +rank0 [2026-10-28 17:54:38,092] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 17:54:41,747] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 17:54:42,742] INFO : Plotted number of elements +rank0 [2026-10-28 17:54:44,234] INFO : Plotted jets +rank0 [2026-10-28 17:54:45,360] INFO : Plotted jet ratio +rank0 [2026-10-28 17:54:45,420] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits +rank0 [2026-10-28 17:54:47,817] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits +rank0 [2026-10-28 17:54:48,672] INFO : Plotted number of elements +rank0 [2026-10-28 17:54:50,155] INFO : Plotted jets +rank0 [2026-10-28 17:54:51,300] INFO : Plotted jet ratio +rank0 [2026-10-28 17:59:44,595] INFO : Step 8100/50000 rank0 | Train Loss: 1.7101 | Train Loss Unweighted: 0.9276 | Train Loss Components: Auxiliary: 0.3166 | Cardinality: 0.1787 | Classification: 0.1664 | Classification_binary: 0.2381 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2564 | Regression_eta: 0.0032 | Regression_pt: 0.2599 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1235 | Classification_binary: 0.2488 | Regression_cos_phi: 0.1996 | Regression_energy: 0.1216 | Regression_eta: 0.1015 | Regression_pt: 0.1310 | Regression_sin_phi: 0.1755 | LR: 9.43e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.5464s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7628 | max_loss: 0.1353 | max_backward: 2.0123 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.8300 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6970.2 | uss_MB: 5951.6 | child_rss_MB: 6190.8 | allocated_MB: 447.2 | reserved_MB: 25306.0 | inactive_split_MB: 1261.3 +rank0 [2026-10-28 17:59:44,846] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 47.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:04:56,783] INFO : Step 8200/50000 rank0 | Train Loss: 1.8745 | Train Loss Unweighted: 1.0293 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.1946 | Classification: 0.1986 | Classification_binary: 0.2966 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2609 | Regression_eta: 0.0035 | Regression_pt: 0.2657 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1007 | Classification_binary: 0.3151 | Regression_cos_phi: 0.1642 | Regression_energy: 0.1308 | Regression_eta: 0.0866 | Regression_pt: 0.1419 | Regression_sin_phi: 0.1334 | LR: 9.42e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.9798s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7790 | max_loss: 0.1535 | max_backward: 1.9181 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.5352 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6080.6 | uss_MB: 5064.4 | child_rss_MB: 6184.5 | allocated_MB: 448.2 | reserved_MB: 25360.0 | inactive_split_MB: 1159.7 +rank0 [2026-10-28 18:04:57,024] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:09:38,048] INFO : Step 8300/50000 rank0 | Train Loss: 1.8727 | Train Loss Unweighted: 1.0119 | Train Loss Components: Auxiliary: 0.3401 | Cardinality: 0.1867 | Classification: 0.1728 | Classification_binary: 0.3272 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2513 | Regression_eta: 0.0033 | Regression_pt: 0.2533 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1256 | Classification_binary: 0.3297 | Regression_cos_phi: 0.1807 | Regression_energy: 0.1520 | Regression_eta: 0.1072 | Regression_pt: 0.1636 | Regression_sin_phi: 0.2009 | LR: 9.40e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2948s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0436 | max_forward: 1.1303 | max_loss: 0.1503 | max_backward: 2.0213 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6082.4 | uss_MB: 5067.6 | child_rss_MB: 6182.5 | allocated_MB: 449.6 | reserved_MB: 27362.0 | inactive_split_MB: 1355.2 +rank0 [2026-10-28 18:09:38,290] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:14:11,170] INFO : Step 8400/50000 rank0 | Train Loss: 1.6705 | Train Loss Unweighted: 0.9205 | Train Loss Components: Auxiliary: 0.3113 | Cardinality: 0.1938 | Classification: 0.1656 | Classification_binary: 0.2512 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2485 | Regression_eta: 0.0028 | Regression_pt: 0.2494 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0854 | Classification_binary: 0.2326 | Regression_cos_phi: 0.1279 | Regression_energy: 0.1528 | Regression_eta: 0.0646 | Regression_pt: 0.1666 | Regression_sin_phi: 0.0860 | LR: 9.39e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0978s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8079 | max_loss: 0.1514 | max_backward: 1.6942 | max_optimizer_step: 0.2670 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6074.8 | uss_MB: 5060.4 | child_rss_MB: 6188.9 | allocated_MB: 446.4 | reserved_MB: 27366.0 | inactive_split_MB: 1386.9 +rank0 [2026-10-28 18:14:11,413] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 8% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 93% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 48.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:18:45,919] INFO : Step 8500/50000 rank0 | Train Loss: 1.8447 | Train Loss Unweighted: 0.9684 | Train Loss Components: Auxiliary: 0.3299 | Cardinality: 0.2197 | Classification: 0.1758 | Classification_binary: 0.2823 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2478 | Regression_eta: 0.0040 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.2461 | Regression_cos_phi: 0.1189 | Regression_energy: 0.1179 | Regression_eta: 0.1009 | Regression_pt: 0.1321 | Regression_sin_phi: 0.1315 | LR: 9.37e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.9942s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0066 | max_forward: 0.7185 | max_loss: 0.1465 | max_backward: 1.6783 | max_optimizer_step: 0.2239 | max_tensorboard_logging: 0.6269 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 6077.8 | uss_MB: 5064.0 | child_rss_MB: 6185.3 | allocated_MB: 448.4 | reserved_MB: 27368.0 | inactive_split_MB: 1257.8 +rank0 [2026-10-28 18:18:46,158] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 98% +2 6 0x7408, 15961 53.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 98% +4 8 0x7408, 13395 51.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 1% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 0% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:23:30,835] INFO : Step 8600/50000 rank0 | Train Loss: 1.9431 | Train Loss Unweighted: 1.0237 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.2483 | Classification: 0.1814 | Classification_binary: 0.3328 | HitGrouping: 0.2024 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2495 | Regression_eta: 0.0034 | Regression_pt: 0.2527 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1151 | Classification_binary: 0.3589 | Regression_cos_phi: 0.1914 | Regression_energy: 0.1272 | Regression_eta: 0.1045 | Regression_pt: 0.1406 | Regression_sin_phi: 0.1750 | LR: 9.36e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.2187s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1011 | max_forward: 0.9340 | max_loss: 0.1508 | max_backward: 1.7704 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.7686 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 6082.0 | uss_MB: 5062.1 | child_rss_MB: 6191.5 | allocated_MB: 448.9 | reserved_MB: 27368.0 | inactive_split_MB: 1268.3 +rank0 [2026-10-28 18:23:31,086] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:28:09,205] INFO : Step 8700/50000 rank0 | Train Loss: 1.9739 | Train Loss Unweighted: 1.0707 | Train Loss Components: Auxiliary: 0.3725 | Cardinality: 0.2322 | Classification: 0.1854 | Classification_binary: 0.2934 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2900 | Regression_eta: 0.0037 | Regression_pt: 0.2944 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1087 | Classification_binary: 0.2653 | Regression_cos_phi: 0.1608 | Regression_energy: 0.1460 | Regression_eta: 0.0982 | Regression_pt: 0.1561 | Regression_sin_phi: 0.1786 | LR: 9.34e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.9227s | Phase Times: max_pre_step_barrier: 0.0890 | max_batch_to_device: 0.0067 | max_forward: 0.7726 | max_loss: 0.1574 | max_backward: 1.6332 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 2.5948 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 6082.7 | uss_MB: 5062.6 | child_rss_MB: 6185.4 | allocated_MB: 448.0 | reserved_MB: 29420.0 | inactive_split_MB: 876.0 +rank0 [2026-10-28 18:28:09,448] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:32:56,231] INFO : Step 8800/50000 rank0 | Train Loss: 1.5247 | Train Loss Unweighted: 0.8421 | Train Loss Components: Auxiliary: 0.2808 | Cardinality: 0.1524 | Classification: 0.1581 | Classification_binary: 0.2323 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2217 | Regression_eta: 0.0029 | Regression_pt: 0.2242 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1198 | Classification_binary: 0.2185 | Regression_cos_phi: 0.1037 | Regression_energy: 0.1170 | Regression_eta: 0.0702 | Regression_pt: 0.1270 | Regression_sin_phi: 0.1236 | LR: 9.33e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6363s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7382 | max_loss: 0.2414 | max_backward: 1.7741 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 6078.3 | uss_MB: 5061.0 | child_rss_MB: 6166.0 | allocated_MB: 447.9 | reserved_MB: 29420.0 | inactive_split_MB: 1260.3 +rank0 [2026-10-28 18:32:56,471] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 52.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:37:37,563] INFO : Step 8900/50000 rank0 | Train Loss: 1.7540 | Train Loss Unweighted: 0.9436 | Train Loss Components: Auxiliary: 0.3227 | Cardinality: 0.2072 | Classification: 0.1816 | Classification_binary: 0.2735 | HitGrouping: 0.1904 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2393 | Regression_eta: 0.0031 | Regression_pt: 0.2426 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1341 | Classification_binary: 0.3026 | Regression_cos_phi: 0.1941 | Regression_energy: 0.1240 | Regression_eta: 0.1007 | Regression_pt: 0.1358 | Regression_sin_phi: 0.1605 | LR: 9.31e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.1214s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8077 | max_loss: 0.2089 | max_backward: 1.9976 | max_optimizer_step: 0.2211 | max_tensorboard_logging: 0.5747 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 6080.9 | uss_MB: 5000.4 | child_rss_MB: 6188.7 | allocated_MB: 451.8 | reserved_MB: 29420.0 | inactive_split_MB: 1133.8 +rank0 [2026-10-28 18:37:37,803] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 5% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:42:21,382] INFO : Step 9000/50000 rank0 | Train Loss: 1.7807 | Train Loss Unweighted: 0.9795 | Train Loss Components: Auxiliary: 0.3326 | Cardinality: 0.1942 | Classification: 0.1600 | Classification_binary: 0.3157 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2461 | Regression_eta: 0.0033 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2544 | Regression_cos_phi: 0.0966 | Regression_energy: 0.1325 | Regression_eta: 0.0636 | Regression_pt: 0.1463 | Regression_sin_phi: 0.1172 | LR: 9.30e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.4925s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.8696 | max_loss: 0.1559 | max_backward: 1.8245 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.9520 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 6064.6 | uss_MB: 5051.1 | child_rss_MB: 6187.5 | allocated_MB: 448.9 | reserved_MB: 29420.0 | inactive_split_MB: 1051.6 +rank0 [2026-10-28 18:42:21,623] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:42:21,647] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 18:42:22,034] INFO : Running validation on rank0 +rank0 [2026-10-28 18:42:22,035] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 18:42:22,035] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 18:43:20,182] INFO : VALIDATION | Step=9000/50000 | Train Loss=1.7807 | Valid Loss=1.7439 | Train Loss Unweighted=0.9795 | Valid Loss Unweighted=0.9386 | Stale=0 | ETA=2049.9m +rank0 [2026-10-28 18:43:20,182] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:43:20,182] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 18:43:20,182] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 18:43:20,182] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:43:20,182] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 18:43:20,182] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/ari=0.495576 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/ari_threshold=0.71875 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.436043 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/top1_competing=0.482733 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.592633 +rank0 [2026-10-28 18:43:20,183] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0535763 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0181582 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/seconds_per_rank=21.3596 +rank0 [2026-10-28 18:43:20,184] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding/ari=0.217046 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/embedding_minus_raw/ari=0.125423 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 18:43:20,185] INFO : VALIDATION_PROBE | Step=9000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 18:43:20,191] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:43:20,450] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 18:43:20,453] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 18:43:47,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 18:43:48,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 18:43:49,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 18:43:50,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 18:44:00,776] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 18:44:00,792] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:44:01,305] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 18:44:01,308] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 18:44:17,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 18:44:18,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 18:44:19,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 18:44:21,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 18:44:31,023] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 18:44:31,056] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 18:44:31,604] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 18:44:31,608] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 18:44:47,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 18:44:48,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 18:44:49,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 18:44:50,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 18:45:00,851] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 18:45:00,867] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits +rank0 [2026-10-28 18:45:04,076] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits +rank0 [2026-10-28 18:45:05,122] INFO : Plotted number of elements +rank0 [2026-10-28 18:45:06,678] INFO : Plotted jets +rank0 [2026-10-28 18:45:07,879] INFO : Plotted jet ratio +rank0 [2026-10-28 18:45:07,947] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 18:45:11,520] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 18:45:12,509] INFO : Plotted number of elements +rank0 [2026-10-28 18:45:13,999] INFO : Plotted jets +rank0 [2026-10-28 18:45:15,124] INFO : Plotted jet ratio +rank0 [2026-10-28 18:45:15,177] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits +rank0 [2026-10-28 18:45:17,557] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits +rank0 [2026-10-28 18:45:18,406] INFO : Plotted number of elements +rank0 [2026-10-28 18:45:19,895] INFO : Plotted jets +rank0 [2026-10-28 18:45:21,069] INFO : Plotted jet ratio +rank0 [2026-10-28 18:50:15,832] INFO : Step 9100/50000 rank0 | Train Loss: 1.6002 | Train Loss Unweighted: 0.8685 | Train Loss Components: Auxiliary: 0.3013 | Cardinality: 0.1975 | Classification: 0.1658 | Classification_binary: 0.2349 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2304 | Regression_eta: 0.0027 | Regression_pt: 0.2318 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0888 | Classification_binary: 0.1885 | Regression_cos_phi: 0.1023 | Regression_energy: 0.1083 | Regression_eta: 0.0490 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0865 | LR: 9.28e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.3714s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7981 | max_loss: 0.1460 | max_backward: 1.6475 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.9178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7065.6 | uss_MB: 6052.1 | child_rss_MB: 6167.4 | allocated_MB: 449.7 | reserved_MB: 25788.0 | inactive_split_MB: 957.3 +rank0 [2026-10-28 18:50:16,072] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:55:02,956] INFO : Step 9200/50000 rank0 | Train Loss: 1.6859 | Train Loss Unweighted: 0.8867 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.1906 | Classification: 0.1447 | Classification_binary: 0.2542 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2389 | Regression_eta: 0.0033 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0776 | Classification_binary: 0.2955 | Regression_cos_phi: 0.1928 | Regression_energy: 0.1374 | Regression_eta: 0.1146 | Regression_pt: 0.1500 | Regression_sin_phi: 0.1758 | LR: 9.27e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9832s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7865 | max_loss: 0.1441 | max_backward: 1.8922 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6128.5 | uss_MB: 5123.6 | child_rss_MB: 6161.7 | allocated_MB: 450.8 | reserved_MB: 25788.0 | inactive_split_MB: 1040.1 +rank0 [2026-10-28 18:55:03,196] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 53.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 18:59:50,689] INFO : Step 9300/50000 rank0 | Train Loss: 1.8980 | Train Loss Unweighted: 0.9669 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.3104 | Classification: 0.1729 | Classification_binary: 0.3073 | HitGrouping: 0.1936 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2397 | Regression_eta: 0.0030 | Regression_pt: 0.2407 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1150 | Classification_binary: 0.3203 | Regression_cos_phi: 0.1771 | Regression_energy: 0.1392 | Regression_eta: 0.1185 | Regression_pt: 0.1504 | Regression_sin_phi: 0.1451 | LR: 9.25e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.9627s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0143 | max_forward: 0.7819 | max_loss: 0.1515 | max_backward: 1.8209 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6071.1 | uss_MB: 5121.9 | child_rss_MB: 5951.4 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1031.1 +rank0 [2026-10-28 18:59:50,931] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 14% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 48.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:04:47,479] INFO : Step 9400/50000 rank0 | Train Loss: 1.5878 | Train Loss Unweighted: 0.8563 | Train Loss Components: Auxiliary: 0.2880 | Cardinality: 0.1552 | Classification: 0.1580 | Classification_binary: 0.2625 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2141 | Regression_eta: 0.0030 | Regression_pt: 0.2153 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.2290 | Regression_cos_phi: 0.1206 | Regression_energy: 0.1126 | Regression_eta: 0.0712 | Regression_pt: 0.1218 | Regression_sin_phi: 0.0949 | LR: 9.24e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6538s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7342 | max_loss: 0.2471 | max_backward: 1.7320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.4671 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6067.3 | uss_MB: 5118.7 | child_rss_MB: 5945.7 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1121.1 +rank0 [2026-10-28 19:04:47,717] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 1% +2 6 0x7408, 15961 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 91% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 48.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:09:48,078] INFO : Step 9500/50000 rank0 | Train Loss: 1.5519 | Train Loss Unweighted: 0.8141 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1719 | Classification: 0.1393 | Classification_binary: 0.2430 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2129 | Regression_eta: 0.0030 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0856 | Classification_binary: 0.2188 | Regression_cos_phi: 0.1384 | Regression_energy: 0.1094 | Regression_eta: 0.0617 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0874 | LR: 9.22e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8646s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0076 | max_forward: 0.7261 | max_loss: 0.1376 | max_backward: 1.8267 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6069.1 | uss_MB: 5122.8 | child_rss_MB: 5952.7 | allocated_MB: 446.7 | reserved_MB: 27752.0 | inactive_split_MB: 1128.4 +rank0 [2026-10-28 19:09:48,319] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:14:56,202] INFO : Step 9600/50000 rank0 | Train Loss: 1.7379 | Train Loss Unweighted: 0.9171 | Train Loss Components: Auxiliary: 0.3221 | Cardinality: 0.2199 | Classification: 0.1679 | Classification_binary: 0.2886 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2245 | Regression_eta: 0.0032 | Regression_pt: 0.2297 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.2972 | Regression_cos_phi: 0.1410 | Regression_energy: 0.1071 | Regression_eta: 0.0825 | Regression_pt: 0.1223 | Regression_sin_phi: 0.1344 | LR: 9.21e-04 | DataLoad Time: 0.0153s | Model Forward Time: 8.4787s | Phase Times: max_pre_step_barrier: 0.0081 | max_batch_to_device: 0.0183 | max_forward: 3.1116 | max_loss: 0.7194 | max_backward: 4.4829 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6068.7 | uss_MB: 5120.4 | child_rss_MB: 5924.4 | allocated_MB: 449.4 | reserved_MB: 29732.0 | inactive_split_MB: 1322.6 +rank0 [2026-10-28 19:14:56,445] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 2% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 47.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:20:03,695] INFO : Step 9700/50000 rank0 | Train Loss: 1.8683 | Train Loss Unweighted: 0.9602 | Train Loss Components: Auxiliary: 0.3449 | Cardinality: 0.2461 | Classification: 0.1512 | Classification_binary: 0.3271 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2364 | Regression_eta: 0.0037 | Regression_pt: 0.2383 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0881 | Classification_binary: 0.2922 | Regression_cos_phi: 0.1526 | Regression_energy: 0.1254 | Regression_eta: 0.0741 | Regression_pt: 0.1372 | Regression_sin_phi: 0.1032 | LR: 9.19e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.7867s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0069 | max_forward: 0.8299 | max_loss: 0.7776 | max_backward: 2.5571 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5726 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6067.5 | uss_MB: 5123.7 | child_rss_MB: 5943.1 | allocated_MB: 450.3 | reserved_MB: 29734.0 | inactive_split_MB: 1128.1 +rank0 [2026-10-28 19:20:03,933] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 50.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:25:01,197] INFO : Step 9800/50000 rank0 | Train Loss: 1.6119 | Train Loss Unweighted: 0.8200 | Train Loss Components: Auxiliary: 0.3036 | Cardinality: 0.2266 | Classification: 0.1371 | Classification_binary: 0.2410 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2176 | Regression_eta: 0.0027 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0902 | Classification_binary: 0.2330 | Regression_cos_phi: 0.1220 | Regression_energy: 0.1005 | Regression_eta: 0.0758 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0842 | LR: 9.17e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.2182s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7430 | max_loss: 0.2467 | max_backward: 1.8961 | max_optimizer_step: 0.2008 | max_tensorboard_logging: 0.9056 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6064.8 | uss_MB: 5122.0 | child_rss_MB: 5925.9 | allocated_MB: 446.9 | reserved_MB: 29734.0 | inactive_split_MB: 905.1 +rank0 [2026-10-28 19:25:01,440] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:29:40,809] INFO : Step 9900/50000 rank0 | Train Loss: 1.5408 | Train Loss Unweighted: 0.8391 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1631 | Classification: 0.1532 | Classification_binary: 0.2416 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2178 | Regression_eta: 0.0029 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1036 | Classification_binary: 0.2744 | Regression_cos_phi: 0.1626 | Regression_energy: 0.1202 | Regression_eta: 0.0734 | Regression_pt: 0.1307 | Regression_sin_phi: 0.1351 | LR: 9.16e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.2026s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7611 | max_loss: 0.1425 | max_backward: 1.8994 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6066.1 | uss_MB: 5122.8 | child_rss_MB: 5945.5 | allocated_MB: 448.3 | reserved_MB: 29734.0 | inactive_split_MB: 1119.7 +rank0 [2026-10-28 19:29:41,049] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 49.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:34:23,986] INFO : Step 10000/50000 rank0 | Train Loss: 1.7147 | Train Loss Unweighted: 0.9225 | Train Loss Components: Auxiliary: 0.3261 | Cardinality: 0.1969 | Classification: 0.1590 | Classification_binary: 0.2912 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2305 | Regression_eta: 0.0029 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0931 | Classification_binary: 0.2937 | Regression_cos_phi: 0.1290 | Regression_energy: 0.1330 | Regression_eta: 0.0901 | Regression_pt: 0.1439 | Regression_sin_phi: 0.1146 | LR: 9.14e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.4868s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1140 | max_forward: 0.8951 | max_loss: 0.1695 | max_backward: 1.8837 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 1.0339 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 6055.8 | uss_MB: 5112.2 | child_rss_MB: 5922.2 | allocated_MB: 449.4 | reserved_MB: 29734.0 | inactive_split_MB: 1320.6 +rank0 [2026-10-28 19:34:24,223] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:34:24,246] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 19:34:24,636] INFO : Running validation on rank0 +rank0 [2026-10-28 19:34:24,636] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 19:34:24,637] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 19:35:22,213] INFO : VALIDATION | Step=10000/50000 | Train Loss=1.7147 | Valid Loss=1.6300 | Train Loss Unweighted=0.9225 | Valid Loss Unweighted=0.8629 | Stale=0 | ETA=2008.1m +rank0 [2026-10-28 19:35:22,213] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 19:35:22,213] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/ari=0.515843 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.427356 +rank0 [2026-10-28 19:35:22,214] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/top1_competing=0.471137 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.617111 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0333098 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.042636 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 19:35:22,215] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/seconds_per_rank=21.4081 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding/ari=0.237163 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/embedding_minus_raw/ari=0.14554 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 19:35:22,216] INFO : VALIDATION_PROBE | Step=10000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 19:35:22,221] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:35:22,476] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 19:35:22,489] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 19:35:48,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 19:35:50,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 19:35:51,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 19:35:52,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 19:36:02,443] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 19:36:02,455] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:36:02,954] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 19:36:02,957] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 19:36:19,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 19:36:20,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 19:36:22,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 19:36:23,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 19:36:33,413] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 19:36:36,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 19:36:36,973] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 19:36:36,976] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 19:36:53,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 19:36:54,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 19:36:54,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 19:36:55,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 19:37:05,871] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 19:37:05,949] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits +rank0 [2026-10-28 19:37:09,180] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits +rank0 [2026-10-28 19:37:10,219] INFO : Plotted number of elements +rank0 [2026-10-28 19:37:12,168] INFO : Plotted jets +rank0 [2026-10-28 19:37:13,289] INFO : Plotted jet ratio +rank0 [2026-10-28 19:37:13,352] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 19:37:16,584] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 19:37:17,599] INFO : Plotted number of elements +rank0 [2026-10-28 19:37:19,090] INFO : Plotted jets +rank0 [2026-10-28 19:37:20,210] INFO : Plotted jet ratio +rank0 [2026-10-28 19:37:20,264] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits +rank0 [2026-10-28 19:37:22,687] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits +rank0 [2026-10-28 19:37:23,535] INFO : Plotted number of elements +rank0 [2026-10-28 19:37:25,013] INFO : Plotted jets +rank0 [2026-10-28 19:37:26,166] INFO : Plotted jet ratio +rank0 [2026-10-28 19:42:24,284] INFO : Step 10100/50000 rank0 | Train Loss: 1.6697 | Train Loss Unweighted: 0.8627 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.2287 | Classification: 0.1283 | Classification_binary: 0.2543 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2359 | Regression_eta: 0.0032 | Regression_pt: 0.2378 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2177 | Regression_cos_phi: 0.1432 | Regression_energy: 0.1269 | Regression_eta: 0.1179 | Regression_pt: 0.1401 | Regression_sin_phi: 0.1282 | LR: 9.12e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.6429s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7652 | max_loss: 0.1496 | max_backward: 1.8791 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 0.9326 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 7065.4 | uss_MB: 6119.6 | child_rss_MB: 5927.2 | allocated_MB: 448.1 | reserved_MB: 26360.0 | inactive_split_MB: 1080.0 +rank0 [2026-10-28 19:42:24,523] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:47:11,271] INFO : Step 10200/50000 rank0 | Train Loss: 1.3968 | Train Loss Unweighted: 0.7619 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1483 | Classification: 0.1293 | Classification_binary: 0.2070 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2090 | Regression_eta: 0.0026 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.2228 | Regression_cos_phi: 0.0944 | Regression_energy: 0.1182 | Regression_eta: 0.0621 | Regression_pt: 0.1270 | Regression_sin_phi: 0.1007 | LR: 9.11e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.1527s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0330 | max_forward: 0.8960 | max_loss: 0.5080 | max_backward: 1.9040 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.6816 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 6137.8 | uss_MB: 5194.1 | child_rss_MB: 5912.5 | allocated_MB: 450.3 | reserved_MB: 27130.0 | inactive_split_MB: 1088.5 +rank0 [2026-10-28 19:47:11,529] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:51:56,971] INFO : Step 10300/50000 rank0 | Train Loss: 1.6039 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.2943 | Cardinality: 0.1656 | Classification: 0.1487 | Classification_binary: 0.2679 | HitGrouping: 0.1881 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2132 | Regression_eta: 0.0030 | Regression_pt: 0.2158 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0986 | Classification_binary: 0.2842 | Regression_cos_phi: 0.2126 | Regression_energy: 0.1062 | Regression_eta: 0.0910 | Regression_pt: 0.1152 | Regression_sin_phi: 0.1236 | LR: 9.09e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.5991s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8152 | max_loss: 0.2478 | max_backward: 2.0483 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6143.7 | uss_MB: 5198.5 | child_rss_MB: 5930.4 | allocated_MB: 448.7 | reserved_MB: 28404.0 | inactive_split_MB: 991.9 +rank0 [2026-10-28 19:51:57,229] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 1% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 19:56:40,090] INFO : Step 10400/50000 rank0 | Train Loss: 1.7188 | Train Loss Unweighted: 0.8798 | Train Loss Components: Auxiliary: 0.3209 | Cardinality: 0.2578 | Classification: 0.1544 | Classification_binary: 0.2910 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2139 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0870 | Classification_binary: 0.2581 | Regression_cos_phi: 0.1244 | Regression_energy: 0.1179 | Regression_eta: 0.0878 | Regression_pt: 0.1271 | Regression_sin_phi: 0.1209 | LR: 9.07e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.3133s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7821 | max_loss: 0.2448 | max_backward: 1.8782 | max_optimizer_step: 0.2932 | max_tensorboard_logging: 0.8061 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 6134.3 | uss_MB: 5193.2 | child_rss_MB: 5938.4 | allocated_MB: 448.9 | reserved_MB: 28404.0 | inactive_split_MB: 983.3 +rank0 [2026-10-28 19:56:40,328] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:01:30,296] INFO : Step 10500/50000 rank0 | Train Loss: 1.4652 | Train Loss Unweighted: 0.7986 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1697 | Classification: 0.1280 | Classification_binary: 0.2159 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2232 | Regression_eta: 0.0026 | Regression_pt: 0.2263 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2051 | Regression_cos_phi: 0.1038 | Regression_energy: 0.1214 | Regression_eta: 0.1095 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0787 | LR: 9.06e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9469s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.7113 | max_loss: 0.1455 | max_backward: 1.6308 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.7760 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6134.1 | uss_MB: 5192.0 | child_rss_MB: 5916.0 | allocated_MB: 444.7 | reserved_MB: 28404.0 | inactive_split_MB: 1307.3 +rank0 [2026-10-28 20:01:30,538] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 53.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 48.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 6% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:06:15,222] INFO : Step 10600/50000 rank0 | Train Loss: 1.5657 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2802 | Cardinality: 0.1297 | Classification: 0.1617 | Classification_binary: 0.2471 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2084 | Regression_eta: 0.0037 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0861 | Classification_binary: 0.2388 | Regression_cos_phi: 0.1026 | Regression_energy: 0.0922 | Regression_eta: 0.0714 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0937 | LR: 9.04e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.6415s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0173 | max_forward: 0.7875 | max_loss: 0.2211 | max_backward: 1.9019 | max_optimizer_step: 0.2814 | max_tensorboard_logging: 0.8518 | max_tensorboard_flush: 0.0072 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6133.2 | uss_MB: 5193.0 | child_rss_MB: 5934.7 | allocated_MB: 449.8 | reserved_MB: 28404.0 | inactive_split_MB: 1130.2 +rank0 [2026-10-28 20:06:15,467] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 2% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:11:00,167] INFO : Step 10700/50000 rank0 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8676 | Train Loss Components: Auxiliary: 0.3003 | Cardinality: 0.1749 | Classification: 0.1453 | Classification_binary: 0.2956 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2085 | Regression_eta: 0.0033 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2967 | Regression_cos_phi: 0.1056 | Regression_energy: 0.1127 | Regression_eta: 0.0644 | Regression_pt: 0.1213 | Regression_sin_phi: 0.1345 | LR: 9.02e-04 | DataLoad Time: 0.0120s | Model Forward Time: 5.8378s | Phase Times: max_pre_step_barrier: 0.0305 | max_batch_to_device: 0.0154 | max_forward: 2.0246 | max_loss: 0.2864 | max_backward: 2.7424 | max_optimizer_step: 0.1980 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6136.9 | uss_MB: 5195.8 | child_rss_MB: 5906.4 | allocated_MB: 446.2 | reserved_MB: 29148.0 | inactive_split_MB: 1134.5 +rank0 [2026-10-28 20:11:00,404] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:15:42,404] INFO : Step 10800/50000 rank0 | Train Loss: 1.5849 | Train Loss Unweighted: 0.7936 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1942 | Classification: 0.1383 | Classification_binary: 0.2480 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2007 | Regression_eta: 0.0032 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2036 | Regression_cos_phi: 0.0987 | Regression_energy: 0.0918 | Regression_eta: 0.0860 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0936 | LR: 9.00e-04 | DataLoad Time: 0.0175s | Model Forward Time: 8.9634s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2408 | max_backward: 1.7977 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 1.7470 | max_tensorboard_flush: 0.0028 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 6126.4 | uss_MB: 4363.9 | child_rss_MB: 5910.3 | allocated_MB: 448.6 | reserved_MB: 30628.0 | inactive_split_MB: 1292.6 +rank0 [2026-10-28 20:15:42,646] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 52.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 50.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 88% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 47.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:20:33,584] INFO : Step 10900/50000 rank0 | Train Loss: 1.3552 | Train Loss Unweighted: 0.7247 | Train Loss Components: Auxiliary: 0.2531 | Cardinality: 0.1449 | Classification: 0.1275 | Classification_binary: 0.2074 | HitGrouping: 0.1874 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1917 | Regression_eta: 0.0024 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0762 | Regression_energy: 0.1045 | Regression_eta: 0.0769 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0974 | LR: 8.99e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.2039s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7076 | max_loss: 0.2425 | max_backward: 1.6536 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6131.8 | uss_MB: 5191.3 | child_rss_MB: 5932.5 | allocated_MB: 446.4 | reserved_MB: 30628.0 | inactive_split_MB: 984.9 +rank0 [2026-10-28 20:20:33,825] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:25:37,235] INFO : Step 11000/50000 rank0 | Train Loss: 1.5999 | Train Loss Unweighted: 0.8322 | Train Loss Components: Auxiliary: 0.2959 | Cardinality: 0.1737 | Classification: 0.1477 | Classification_binary: 0.2877 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0017 | Regression_energy: 0.1943 | Regression_eta: 0.0030 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0862 | Classification_binary: 0.3009 | Regression_cos_phi: 0.1150 | Regression_energy: 0.0853 | Regression_eta: 0.0868 | Regression_pt: 0.0942 | Regression_sin_phi: 0.1078 | LR: 8.97e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.5585s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8260 | max_loss: 0.1503 | max_backward: 1.9603 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6124.9 | uss_MB: 5185.7 | child_rss_MB: 5918.3 | allocated_MB: 451.8 | reserved_MB: 30628.0 | inactive_split_MB: 996.6 +rank0 [2026-10-28 20:25:37,477] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 47.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:25:37,501] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 20:25:37,893] INFO : Running validation on rank0 +rank0 [2026-10-28 20:25:37,893] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 20:25:37,893] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 20:26:33,393] INFO : VALIDATION | Step=11000/50000 | Train Loss=1.5999 | Valid Loss=1.5236 | Train Loss Unweighted=0.8322 | Valid Loss Unweighted=0.8003 | Stale=0 | ETA=1961.4m +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 20:26:33,394] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/ari=0.525316 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/ari_threshold=0.7375 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.424151 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/top1_competing=0.467391 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.612272 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0238372 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0377967 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 20:26:33,395] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/seconds_per_rank=21.7244 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding/ari=0.251079 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/embedding_minus_raw/ari=0.159456 +rank0 [2026-10-28 20:26:33,396] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 20:26:33,397] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 20:26:33,397] INFO : VALIDATION_PROBE | Step=11000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 20:26:33,402] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:26:33,660] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 20:26:33,671] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 20:27:00,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 20:27:01,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 20:27:03,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 20:27:04,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 20:27:14,303] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-28 20:27:14,319] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:27:14,833] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 20:27:14,835] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 20:27:31,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 20:27:32,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 20:27:34,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 20:27:36,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 20:27:46,099] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-28 20:27:46,110] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 20:27:46,648] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 20:27:46,651] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 20:28:02,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 20:28:03,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 20:28:04,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 20:28:05,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 20:28:15,426] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 20:28:15,447] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits +rank0 [2026-10-28 20:28:19,097] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits +rank0 [2026-10-28 20:28:20,131] INFO : Plotted number of elements +rank0 [2026-10-28 20:28:21,705] INFO : Plotted jets +rank0 [2026-10-28 20:28:22,823] INFO : Plotted jet ratio +rank0 [2026-10-28 20:28:22,878] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 20:28:26,144] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 20:28:27,188] INFO : Plotted number of elements +rank0 [2026-10-28 20:28:28,744] INFO : Plotted jets +rank0 [2026-10-28 20:28:29,907] INFO : Plotted jet ratio +rank0 [2026-10-28 20:28:29,961] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits +rank0 [2026-10-28 20:28:32,428] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits +rank0 [2026-10-28 20:28:33,284] INFO : Plotted number of elements +rank0 [2026-10-28 20:28:35,235] INFO : Plotted jets +rank0 [2026-10-28 20:28:36,396] INFO : Plotted jet ratio +rank0 [2026-10-28 20:33:30,042] INFO : Step 11100/50000 rank0 | Train Loss: 1.6214 | Train Loss Unweighted: 0.8328 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.2186 | Classification: 0.1350 | Classification_binary: 0.2802 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2042 | Regression_eta: 0.0030 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0826 | Classification_binary: 0.2652 | Regression_cos_phi: 0.1125 | Regression_energy: 0.1190 | Regression_eta: 0.0810 | Regression_pt: 0.1310 | Regression_sin_phi: 0.1115 | LR: 8.95e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.7350s | Phase Times: max_pre_step_barrier: 0.0088 | max_batch_to_device: 0.0068 | max_forward: 0.8488 | max_loss: 0.1548 | max_backward: 1.7645 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 1.1046 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7069.9 | uss_MB: 6125.7 | child_rss_MB: 5940.1 | allocated_MB: 448.1 | reserved_MB: 24340.0 | inactive_split_MB: 1081.9 +rank0 [2026-10-28 20:33:30,284] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:38:19,387] INFO : Step 11200/50000 rank0 | Train Loss: 1.7050 | Train Loss Unweighted: 0.8927 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2640 | Classification: 0.1534 | Classification_binary: 0.2523 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2404 | Regression_eta: 0.0030 | Regression_pt: 0.2410 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0995 | Classification_binary: 0.2381 | Regression_cos_phi: 0.0965 | Regression_energy: 0.1211 | Regression_eta: 0.0826 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0981 | LR: 8.93e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.5653s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8707 | max_loss: 0.2001 | max_backward: 2.2037 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.7912 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 6163.4 | uss_MB: 5218.5 | child_rss_MB: 5949.2 | allocated_MB: 447.8 | reserved_MB: 26392.0 | inactive_split_MB: 938.7 +rank0 [2026-10-28 20:38:19,639] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:43:02,447] INFO : Step 11300/50000 rank0 | Train Loss: 1.3885 | Train Loss Unweighted: 0.7511 | Train Loss Components: Auxiliary: 0.2594 | Cardinality: 0.1357 | Classification: 0.1316 | Classification_binary: 0.2357 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1874 | Regression_eta: 0.0024 | Regression_pt: 0.1915 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0672 | Classification_binary: 0.2296 | Regression_cos_phi: 0.1414 | Regression_energy: 0.0872 | Regression_eta: 0.0762 | Regression_pt: 0.0966 | Regression_sin_phi: 0.1387 | LR: 8.91e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.1525s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8053 | max_loss: 0.1469 | max_backward: 2.0062 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.4618 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1345 | Memory: rss_MB: 6165.0 | uss_MB: 5222.0 | child_rss_MB: 5948.8 | allocated_MB: 450.6 | reserved_MB: 26400.0 | inactive_split_MB: 1092.9 +rank0 [2026-10-28 20:43:02,688] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:48:06,172] INFO : Step 11400/50000 rank0 | Train Loss: 1.5865 | Train Loss Unweighted: 0.8323 | Train Loss Components: Auxiliary: 0.2981 | Cardinality: 0.1866 | Classification: 0.1338 | Classification_binary: 0.2899 | HitGrouping: 0.1840 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1999 | Regression_eta: 0.0028 | Regression_pt: 0.2030 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0986 | Classification_binary: 0.3331 | Regression_cos_phi: 0.1105 | Regression_energy: 0.1166 | Regression_eta: 0.0801 | Regression_pt: 0.1269 | Regression_sin_phi: 0.1251 | LR: 8.89e-04 | DataLoad Time: 0.0181s | Model Forward Time: 6.4162s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0769 | max_forward: 1.3800 | max_loss: 0.6715 | max_backward: 3.0339 | max_optimizer_step: 0.2660 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6163.4 | uss_MB: 5219.6 | child_rss_MB: 5946.4 | allocated_MB: 448.1 | reserved_MB: 27144.0 | inactive_split_MB: 1081.9 +rank0 [2026-10-28 20:48:06,413] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:52:58,763] INFO : Step 11500/50000 rank0 | Train Loss: 1.4825 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2779 | Cardinality: 0.1999 | Classification: 0.1273 | Classification_binary: 0.2463 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1956 | Regression_eta: 0.0026 | Regression_pt: 0.1972 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2197 | Regression_cos_phi: 0.1072 | Regression_energy: 0.1074 | Regression_eta: 0.0589 | Regression_pt: 0.1201 | Regression_sin_phi: 0.0820 | LR: 8.88e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.8427s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.7822 | max_loss: 0.1471 | max_backward: 1.7308 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 1.4583 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6164.2 | uss_MB: 5219.4 | child_rss_MB: 5944.4 | allocated_MB: 450.8 | reserved_MB: 27144.0 | inactive_split_MB: 1342.3 +rank0 [2026-10-28 20:52:59,001] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 53.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 20:57:41,891] INFO : Step 11600/50000 rank0 | Train Loss: 1.4219 | Train Loss Unweighted: 0.7444 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1679 | Classification: 0.1415 | Classification_binary: 0.2106 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1945 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0994 | Classification_binary: 0.2156 | Regression_cos_phi: 0.1440 | Regression_energy: 0.1031 | Regression_eta: 0.0696 | Regression_pt: 0.1125 | Regression_sin_phi: 0.1098 | LR: 8.86e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.3317s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8362 | max_loss: 0.1428 | max_backward: 2.0109 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6156.9 | uss_MB: 5220.2 | child_rss_MB: 5947.3 | allocated_MB: 448.6 | reserved_MB: 27146.0 | inactive_split_MB: 1315.4 +rank0 [2026-10-28 20:57:42,144] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:02:35,784] INFO : Step 11700/50000 rank0 | Train Loss: 1.5203 | Train Loss Unweighted: 0.8017 | Train Loss Components: Auxiliary: 0.2866 | Cardinality: 0.1889 | Classification: 0.1302 | Classification_binary: 0.2577 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2024 | Regression_eta: 0.0026 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2786 | Regression_cos_phi: 0.0981 | Regression_energy: 0.0975 | Regression_eta: 0.0623 | Regression_pt: 0.1079 | Regression_sin_phi: 0.1089 | LR: 8.84e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.9845s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 1.0014 | max_loss: 0.1975 | max_backward: 2.5347 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0688 | Memory: rss_MB: 6157.0 | uss_MB: 5144.6 | child_rss_MB: 5922.6 | allocated_MB: 448.7 | reserved_MB: 27266.0 | inactive_split_MB: 1090.9 +rank0 [2026-10-28 21:02:36,034] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 95% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:07:43,237] INFO : Step 11800/50000 rank0 | Train Loss: 1.5591 | Train Loss Unweighted: 0.7907 | Train Loss Components: Auxiliary: 0.2873 | Cardinality: 0.2051 | Classification: 0.1346 | Classification_binary: 0.2655 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1918 | Regression_eta: 0.0030 | Regression_pt: 0.1929 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2108 | Regression_cos_phi: 0.0887 | Regression_energy: 0.0913 | Regression_eta: 0.0673 | Regression_pt: 0.0965 | Regression_sin_phi: 0.1012 | LR: 8.82e-04 | DataLoad Time: 0.0195s | Model Forward Time: 8.1309s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.0966 | max_forward: 1.7907 | max_loss: 0.6496 | max_backward: 2.7103 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 2.3587 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6136.9 | uss_MB: 5209.7 | child_rss_MB: 5946.4 | allocated_MB: 450.4 | reserved_MB: 27266.0 | inactive_split_MB: 1187.2 +rank0 [2026-10-28 21:07:43,494] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 49.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 47.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:12:51,903] INFO : Step 11900/50000 rank0 | Train Loss: 1.3283 | Train Loss Unweighted: 0.7113 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1476 | Classification: 0.1193 | Classification_binary: 0.2094 | HitGrouping: 0.1824 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0021 | Regression_pt: 0.1901 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.2013 | Regression_cos_phi: 0.1054 | Regression_energy: 0.1037 | Regression_eta: 0.0660 | Regression_pt: 0.1116 | Regression_sin_phi: 0.1189 | LR: 8.80e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.8141s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0785 | max_forward: 0.7917 | max_loss: 0.1465 | max_backward: 1.5595 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6137.9 | uss_MB: 5216.4 | child_rss_MB: 5947.1 | allocated_MB: 443.9 | reserved_MB: 27266.0 | inactive_split_MB: 955.3 +rank0 [2026-10-28 21:12:52,143] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:17:49,474] INFO : Step 12000/50000 rank0 | Train Loss: 1.4074 | Train Loss Unweighted: 0.7415 | Train Loss Components: Auxiliary: 0.2683 | Cardinality: 0.1719 | Classification: 0.1235 | Classification_binary: 0.2283 | HitGrouping: 0.1812 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1909 | Regression_eta: 0.0022 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0843 | Classification_binary: 0.2697 | Regression_cos_phi: 0.1526 | Regression_energy: 0.1158 | Regression_eta: 0.0859 | Regression_pt: 0.1279 | Regression_sin_phi: 0.1100 | LR: 8.78e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9900s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0303 | max_forward: 0.7589 | max_loss: 0.1459 | max_backward: 1.9073 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6136.6 | uss_MB: 5219.4 | child_rss_MB: 5952.0 | allocated_MB: 447.6 | reserved_MB: 27266.0 | inactive_split_MB: 1310.4 +rank0 [2026-10-28 21:17:49,716] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:17:49,738] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 21:17:50,020] INFO : Running validation on rank0 +rank0 [2026-10-28 21:17:50,020] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 21:17:50,020] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 21:18:45,474] INFO : VALIDATION | Step=12000/50000 | Train Loss=1.4074 | Valid Loss=1.4392 | Train Loss Unweighted=0.7415 | Valid Loss Unweighted=0.7532 | Stale=0 | ETA=1917.1m +rank0 [2026-10-28 21:18:45,474] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:18:45,474] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/ari=0.517243 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/ari_threshold=0.775 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.516879 +rank0 [2026-10-28 21:18:45,475] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/top1_competing=0.579289 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.585578 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0319094 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0111035 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/seconds_per_rank=21.3375 +rank0 [2026-10-28 21:18:45,476] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding/ari=0.24632 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/embedding_minus_raw/ari=0.154697 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 21:18:45,477] INFO : VALIDATION_PROBE | Step=12000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 21:18:45,482] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:18:45,745] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 21:18:45,748] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 21:19:11,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 21:19:12,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 21:19:14,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 21:19:15,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 21:19:25,258] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 21:19:25,274] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:19:25,828] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 21:19:25,831] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 21:19:42,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 21:19:43,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 21:19:44,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 21:19:45,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 21:19:55,861] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 21:19:55,979] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 21:19:56,382] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 21:19:56,386] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 21:20:12,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 21:20:12,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 21:20:13,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 21:20:14,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 21:20:24,942] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 21:20:25,049] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits +rank0 [2026-10-28 21:20:28,265] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits +rank0 [2026-10-28 21:20:29,299] INFO : Plotted number of elements +rank0 [2026-10-28 21:20:30,847] INFO : Plotted jets +rank0 [2026-10-28 21:20:31,974] INFO : Plotted jet ratio +rank0 [2026-10-28 21:20:32,048] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 21:20:35,269] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 21:20:36,322] INFO : Plotted number of elements +rank0 [2026-10-28 21:20:37,841] INFO : Plotted jets +rank0 [2026-10-28 21:20:38,972] INFO : Plotted jet ratio +rank0 [2026-10-28 21:20:39,023] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits +rank0 [2026-10-28 21:20:41,885] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits +rank0 [2026-10-28 21:20:42,729] INFO : Plotted number of elements +rank0 [2026-10-28 21:20:44,216] INFO : Plotted jets +rank0 [2026-10-28 21:20:45,405] INFO : Plotted jet ratio +rank0 [2026-10-28 21:25:34,000] INFO : Step 12100/50000 rank0 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8521 | Train Loss Components: Auxiliary: 0.3093 | Cardinality: 0.2088 | Classification: 0.1337 | Classification_binary: 0.2971 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0029 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0845 | Classification_binary: 0.3049 | Regression_cos_phi: 0.1002 | Regression_energy: 0.1138 | Regression_eta: 0.0821 | Regression_pt: 0.1255 | Regression_sin_phi: 0.1353 | LR: 8.76e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.7440s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8139 | max_loss: 0.1586 | max_backward: 1.9999 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9439 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7027.3 | uss_MB: 6104.0 | child_rss_MB: 5925.8 | allocated_MB: 450.8 | reserved_MB: 28008.0 | inactive_split_MB: 1125.6 +rank0 [2026-10-28 21:25:34,240] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:30:26,075] INFO : Step 12200/50000 rank0 | Train Loss: 1.4655 | Train Loss Unweighted: 0.7431 | Train Loss Components: Auxiliary: 0.2757 | Cardinality: 0.1845 | Classification: 0.1184 | Classification_binary: 0.2450 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1862 | Regression_eta: 0.0026 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.2183 | Regression_cos_phi: 0.0985 | Regression_energy: 0.1077 | Regression_eta: 0.0486 | Regression_pt: 0.1192 | Regression_sin_phi: 0.1053 | LR: 8.74e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.1119s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0364 | max_forward: 0.7684 | max_loss: 0.1438 | max_backward: 1.8240 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.7243 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0697 | Memory: rss_MB: 6148.5 | uss_MB: 5233.4 | child_rss_MB: 5949.1 | allocated_MB: 445.9 | reserved_MB: 28008.0 | inactive_split_MB: 995.3 +rank0 [2026-10-28 21:30:26,329] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:35:21,437] INFO : Step 12300/50000 rank0 | Train Loss: 1.3845 | Train Loss Unweighted: 0.7270 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1523 | Classification: 0.1341 | Classification_binary: 0.2109 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1869 | Regression_eta: 0.0026 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2014 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0973 | Regression_eta: 0.0638 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0767 | LR: 8.72e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.3375s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7264 | max_loss: 0.1976 | max_backward: 1.8534 | max_optimizer_step: 0.2993 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6147.3 | uss_MB: 5233.4 | child_rss_MB: 5944.1 | allocated_MB: 448.1 | reserved_MB: 29988.0 | inactive_split_MB: 1298.2 +rank0 [2026-10-28 21:35:21,680] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 107.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 19% +2 6 0x7408, 15961 51.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 79% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 49.0°C 126.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 10% +6 10 0x7408, 62036 47.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 2% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:40:15,760] INFO : Step 12400/50000 rank0 | Train Loss: 1.4006 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.2654 | Cardinality: 0.1602 | Classification: 0.1310 | Classification_binary: 0.2363 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.2216 | Regression_cos_phi: 0.1185 | Regression_energy: 0.1043 | Regression_eta: 0.0687 | Regression_pt: 0.1160 | Regression_sin_phi: 0.1024 | LR: 8.70e-04 | DataLoad Time: 0.0159s | Model Forward Time: 7.7623s | Phase Times: max_pre_step_barrier: 0.0107 | max_batch_to_device: 0.0207 | max_forward: 1.2284 | max_loss: 0.2740 | max_backward: 2.6428 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8714 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 6147.7 | uss_MB: 5022.1 | child_rss_MB: 5936.9 | allocated_MB: 448.8 | reserved_MB: 29988.0 | inactive_split_MB: 1332.2 +rank0 [2026-10-28 21:40:16,016] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 3% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 99% +2 6 0x7408, 15961 52.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 50.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 47.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 89% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:45:08,220] INFO : Step 12500/50000 rank0 | Train Loss: 1.4931 | Train Loss Unweighted: 0.7691 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1768 | Classification: 0.1197 | Classification_binary: 0.2587 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1910 | Regression_eta: 0.0024 | Regression_pt: 0.1944 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.2294 | Regression_cos_phi: 0.1196 | Regression_energy: 0.1170 | Regression_eta: 0.0600 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0896 | LR: 8.68e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.3665s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0333 | max_forward: 0.8806 | max_loss: 0.1518 | max_backward: 2.1554 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.5983 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6144.5 | uss_MB: 5232.3 | child_rss_MB: 5946.8 | allocated_MB: 452.8 | reserved_MB: 29990.0 | inactive_split_MB: 1046.7 +rank0 [2026-10-28 21:45:08,463] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:50:04,277] INFO : Step 12600/50000 rank0 | Train Loss: 1.3547 | Train Loss Unweighted: 0.7028 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1629 | Classification: 0.1033 | Classification_binary: 0.2112 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1912 | Regression_eta: 0.0024 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.2157 | Regression_cos_phi: 0.0882 | Regression_energy: 0.0729 | Regression_eta: 0.0540 | Regression_pt: 0.0812 | Regression_sin_phi: 0.0841 | LR: 8.66e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1434s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.7639 | max_loss: 0.2516 | max_backward: 1.8947 | max_optimizer_step: 0.2049 | max_tensorboard_logging: 0.7350 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6141.1 | uss_MB: 5233.0 | child_rss_MB: 5920.5 | allocated_MB: 446.9 | reserved_MB: 29990.0 | inactive_split_MB: 1034.9 +rank0 [2026-10-28 21:50:04,521] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:54:55,286] INFO : Step 12700/50000 rank0 | Train Loss: 1.3557 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1573 | Classification: 0.1262 | Classification_binary: 0.2132 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1863 | Regression_eta: 0.0022 | Regression_pt: 0.1887 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0772 | Classification_binary: 0.2148 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0941 | Regression_eta: 0.0565 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0818 | LR: 8.64e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.7675s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7499 | max_loss: 0.2453 | max_backward: 3.4334 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6139.6 | uss_MB: 5232.1 | child_rss_MB: 5948.4 | allocated_MB: 450.2 | reserved_MB: 29990.0 | inactive_split_MB: 1245.6 +rank0 [2026-10-28 21:54:55,527] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +2 6 0x7408, 15961 53.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 51.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 47.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 21:59:40,027] INFO : Step 12800/50000 rank0 | Train Loss: 1.5298 | Train Loss Unweighted: 0.7754 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1939 | Classification: 0.1237 | Classification_binary: 0.2821 | HitGrouping: 0.1822 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1810 | Regression_eta: 0.0027 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.2822 | Regression_cos_phi: 0.1255 | Regression_energy: 0.0982 | Regression_eta: 0.0613 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0839 | LR: 8.62e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.9489s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.5299 | max_backward: 2.3185 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6133.4 | uss_MB: 4813.1 | child_rss_MB: 5927.2 | allocated_MB: 449.9 | reserved_MB: 32006.0 | inactive_split_MB: 889.3 +rank0 [2026-10-28 21:59:40,265] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 99% +2 6 0x7408, 15961 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 47.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:04:40,433] INFO : Step 12900/50000 rank0 | Train Loss: 1.3302 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1576 | Classification: 0.1054 | Classification_binary: 0.2343 | HitGrouping: 0.1811 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1758 | Regression_eta: 0.0020 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.2102 | Regression_cos_phi: 0.0812 | Regression_energy: 0.1074 | Regression_eta: 0.0502 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0796 | LR: 8.60e-04 | DataLoad Time: 0.0202s | Model Forward Time: 6.3030s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0137 | max_forward: 0.9836 | max_loss: 0.2293 | max_backward: 2.2888 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.9302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6139.6 | uss_MB: 5231.4 | child_rss_MB: 5950.5 | allocated_MB: 445.9 | reserved_MB: 32006.0 | inactive_split_MB: 1247.6 +rank0 [2026-10-28 22:04:40,674] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +2 6 0x7408, 15961 52.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 47.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:09:26,089] INFO : Step 13000/50000 rank0 | Train Loss: 1.3124 | Train Loss Unweighted: 0.7073 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1406 | Classification: 0.1147 | Classification_binary: 0.1973 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1944 | Regression_eta: 0.0023 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1887 | Regression_cos_phi: 0.0725 | Regression_energy: 0.1038 | Regression_eta: 0.0568 | Regression_pt: 0.1114 | Regression_sin_phi: 0.1266 | LR: 8.58e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.0296s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0065 | max_forward: 0.7321 | max_loss: 0.1616 | max_backward: 1.7374 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6140.2 | uss_MB: 5231.6 | child_rss_MB: 5949.6 | allocated_MB: 444.7 | reserved_MB: 32006.0 | inactive_split_MB: 1300.2 +rank0 [2026-10-28 22:09:26,327] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 52% 100% +2 6 0x7408, 15961 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 47.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:09:26,351] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 22:09:26,732] INFO : Running validation on rank0 +rank0 [2026-10-28 22:09:26,732] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 22:09:26,733] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 22:10:22,437] INFO : VALIDATION | Step=13000/50000 | Train Loss=1.3124 | Valid Loss=1.4079 | Train Loss Unweighted=0.7073 | Valid Loss Unweighted=0.7330 | Stale=0 | ETA=1870.0m +rank0 [2026-10-28 22:10:22,437] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 22:10:22,437] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 22:10:22,437] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/ari=0.509623 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.546947 +rank0 [2026-10-28 22:10:22,438] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/top1_competing=0.61758 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.612299 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0395296 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0378241 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 22:10:22,439] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/seconds_per_rank=21.2594 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding/ari=0.272343 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/embedding_minus_raw/ari=0.18072 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 22:10:22,440] INFO : VALIDATION_PROBE | Step=13000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 22:10:22,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:10:22,704] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 22:10:22,708] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 22:10:48,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 22:10:50,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 22:10:51,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 22:10:52,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 22:11:02,338] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 22:11:02,351] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:11:02,875] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 22:11:02,878] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 22:11:19,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 22:11:20,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 22:11:21,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 22:11:22,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 22:11:32,679] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 22:11:32,690] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 22:11:33,218] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 22:11:33,221] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 22:11:49,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 22:11:50,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 22:11:51,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 22:11:52,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 22:12:02,163] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 22:12:02,179] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits +rank0 [2026-10-28 22:12:05,426] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits +rank0 [2026-10-28 22:12:06,482] INFO : Plotted number of elements +rank0 [2026-10-28 22:12:08,032] INFO : Plotted jets +rank0 [2026-10-28 22:12:09,155] INFO : Plotted jet ratio +rank0 [2026-10-28 22:12:09,207] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 22:12:12,372] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 22:12:13,360] INFO : Plotted number of elements +rank0 [2026-10-28 22:12:14,860] INFO : Plotted jets +rank0 [2026-10-28 22:12:16,009] INFO : Plotted jet ratio +rank0 [2026-10-28 22:12:16,064] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits +rank0 [2026-10-28 22:12:18,510] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits +rank0 [2026-10-28 22:12:19,874] INFO : Plotted number of elements +rank0 [2026-10-28 22:12:21,415] INFO : Plotted jets +rank0 [2026-10-28 22:12:22,563] INFO : Plotted jet ratio +rank0 [2026-10-28 22:17:09,298] INFO : Step 13100/50000 rank0 | Train Loss: 1.6165 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3082 | Cardinality: 0.2635 | Classification: 0.1364 | Classification_binary: 0.2628 | HitGrouping: 0.1835 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0026 | Regression_pt: 0.1957 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0881 | Classification_binary: 0.3150 | Regression_cos_phi: 0.1161 | Regression_energy: 0.1091 | Regression_eta: 0.0779 | Regression_pt: 0.1204 | Regression_sin_phi: 0.1391 | LR: 8.56e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.0845s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0479 | max_forward: 0.8994 | max_loss: 0.2807 | max_backward: 2.1108 | max_optimizer_step: 0.2760 | max_tensorboard_logging: 0.9690 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 7057.8 | uss_MB: 6143.5 | child_rss_MB: 5948.1 | allocated_MB: 446.5 | reserved_MB: 23120.0 | inactive_split_MB: 920.0 +rank0 [2026-10-28 22:17:09,542] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 5% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +6 10 0x7408, 62036 47.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:22:02,354] INFO : Step 13200/50000 rank0 | Train Loss: 1.4928 | Train Loss Unweighted: 0.7469 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.2211 | Classification: 0.1278 | Classification_binary: 0.2505 | HitGrouping: 0.1746 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1808 | Regression_eta: 0.0025 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0989 | Classification_binary: 0.2628 | Regression_cos_phi: 0.1168 | Regression_energy: 0.0975 | Regression_eta: 0.0808 | Regression_pt: 0.1061 | Regression_sin_phi: 0.1455 | LR: 8.54e-04 | DataLoad Time: 0.0197s | Model Forward Time: 7.2965s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.7755 | max_loss: 0.2864 | max_backward: 3.4589 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6222.1 | uss_MB: 5074.0 | child_rss_MB: 5943.3 | allocated_MB: 448.3 | reserved_MB: 25156.0 | inactive_split_MB: 978.0 +rank0 [2026-10-28 22:22:02,594] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 132.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 50.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +6 10 0x7408, 62036 47.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 92% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:26:53,721] INFO : Step 13300/50000 rank0 | Train Loss: 1.3251 | Train Loss Unweighted: 0.6741 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1607 | Classification: 0.1132 | Classification_binary: 0.1927 | HitGrouping: 0.1773 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1805 | Regression_eta: 0.0024 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1702 | Regression_cos_phi: 0.1631 | Regression_energy: 0.0981 | Regression_eta: 0.0916 | Regression_pt: 0.1116 | Regression_sin_phi: 0.1580 | LR: 8.52e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1260s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8118 | max_loss: 0.1467 | max_backward: 1.5503 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.9039 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6221.5 | uss_MB: 5307.8 | child_rss_MB: 5952.8 | allocated_MB: 446.0 | reserved_MB: 26736.0 | inactive_split_MB: 965.8 +rank0 [2026-10-28 22:26:53,963] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:31:44,782] INFO : Step 13400/50000 rank0 | Train Loss: 1.2635 | Train Loss Unweighted: 0.6770 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1436 | Classification: 0.1317 | Classification_binary: 0.2016 | HitGrouping: 0.1790 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1692 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.2041 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0854 | Regression_eta: 0.0447 | Regression_pt: 0.0925 | Regression_sin_phi: 0.1099 | LR: 8.50e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.3987s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8396 | max_loss: 0.1412 | max_backward: 2.0130 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.8047 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6225.0 | uss_MB: 5310.9 | child_rss_MB: 5925.4 | allocated_MB: 445.8 | reserved_MB: 26736.0 | inactive_split_MB: 1014.4 +rank0 [2026-10-28 22:31:45,043] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:36:28,303] INFO : Step 13500/50000 rank0 | Train Loss: 1.3935 | Train Loss Unweighted: 0.7401 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1713 | Classification: 0.1245 | Classification_binary: 0.2541 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0021 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2402 | Regression_cos_phi: 0.0853 | Regression_energy: 0.0825 | Regression_eta: 0.0569 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0814 | LR: 8.48e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2120s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0859 | max_forward: 0.8721 | max_loss: 0.1498 | max_backward: 1.7581 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.7855 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6221.6 | uss_MB: 5308.9 | child_rss_MB: 5951.2 | allocated_MB: 449.1 | reserved_MB: 26736.0 | inactive_split_MB: 922.5 +rank0 [2026-10-28 22:36:28,555] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 47.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:41:19,983] INFO : Step 13600/50000 rank0 | Train Loss: 1.3997 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1870 | Classification: 0.1206 | Classification_binary: 0.2286 | HitGrouping: 0.1765 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1702 | Regression_eta: 0.0026 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0947 | Classification_binary: 0.2267 | Regression_cos_phi: 0.0794 | Regression_energy: 0.0858 | Regression_eta: 0.0611 | Regression_pt: 0.0959 | Regression_sin_phi: 0.1080 | LR: 8.45e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8557s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7703 | max_loss: 0.2484 | max_backward: 1.6943 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 6210.7 | uss_MB: 5299.9 | child_rss_MB: 5953.4 | allocated_MB: 450.3 | reserved_MB: 26736.0 | inactive_split_MB: 922.0 +rank0 [2026-10-28 22:41:20,237] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 48.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:46:13,011] INFO : Step 13700/50000 rank0 | Train Loss: 1.2648 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2420 | Cardinality: 0.1315 | Classification: 0.1136 | Classification_binary: 0.1964 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1793 | Regression_eta: 0.0020 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1923 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0864 | Regression_eta: 0.0383 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0822 | LR: 8.43e-04 | DataLoad Time: 0.0098s | Model Forward Time: 6.9233s | Phase Times: max_pre_step_barrier: 0.0406 | max_batch_to_device: 0.0119 | max_forward: 1.2281 | max_loss: 0.2480 | max_backward: 3.6472 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.6460 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0767 | Memory: rss_MB: 6216.0 | uss_MB: 5307.4 | child_rss_MB: 5948.4 | allocated_MB: 446.9 | reserved_MB: 26742.0 | inactive_split_MB: 1058.7 +rank0 [2026-10-28 22:46:13,265] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 50.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 47.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:51:09,977] INFO : Step 13800/50000 rank0 | Train Loss: 1.3469 | Train Loss Unweighted: 0.7266 | Train Loss Components: Auxiliary: 0.2604 | Cardinality: 0.1436 | Classification: 0.1184 | Classification_binary: 0.2382 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1812 | Regression_eta: 0.0021 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2401 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0833 | Regression_eta: 0.0495 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0817 | LR: 8.41e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.5959s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7995 | max_loss: 0.1458 | max_backward: 3.4735 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6221.0 | uss_MB: 5310.1 | child_rss_MB: 5919.7 | allocated_MB: 446.4 | reserved_MB: 27082.0 | inactive_split_MB: 1370.3 +rank0 [2026-10-28 22:51:10,219] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 52.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +6 10 0x7408, 62036 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 22:56:04,981] INFO : Step 13900/50000 rank0 | Train Loss: 1.4882 | Train Loss Unweighted: 0.7682 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1631 | Classification: 0.1199 | Classification_binary: 0.2624 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1900 | Regression_eta: 0.0029 | Regression_pt: 0.1902 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.2100 | Regression_cos_phi: 0.1021 | Regression_energy: 0.0964 | Regression_eta: 0.0612 | Regression_pt: 0.1042 | Regression_sin_phi: 0.0872 | LR: 8.39e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0848s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.7750 | max_loss: 0.1563 | max_backward: 1.5716 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.8868 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6217.9 | uss_MB: 5307.2 | child_rss_MB: 5946.5 | allocated_MB: 449.9 | reserved_MB: 27316.0 | inactive_split_MB: 918.0 +rank0 [2026-10-28 22:56:05,240] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 50.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:00:55,503] INFO : Step 14000/50000 rank0 | Train Loss: 1.2239 | Train Loss Unweighted: 0.6384 | Train Loss Components: Auxiliary: 0.2362 | Cardinality: 0.1573 | Classification: 0.0898 | Classification_binary: 0.1853 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0021 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1876 | Regression_cos_phi: 0.0674 | Regression_energy: 0.1089 | Regression_eta: 0.0393 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0577 | LR: 8.37e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2548s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7611 | max_loss: 0.1442 | max_backward: 1.7887 | max_optimizer_step: 0.2976 | max_tensorboard_logging: 0.7418 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6219.2 | uss_MB: 5309.9 | child_rss_MB: 5948.9 | allocated_MB: 446.8 | reserved_MB: 29264.0 | inactive_split_MB: 1150.9 +rank0 [2026-10-28 23:00:55,743] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:00:55,766] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 23:00:56,080] INFO : Running validation on rank0 +rank0 [2026-10-28 23:00:56,080] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 23:00:56,081] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 23:01:51,134] INFO : VALIDATION | Step=14000/50000 | Train Loss=1.2239 | Valid Loss=1.3363 | Train Loss Unweighted=0.6384 | Valid Loss Unweighted=0.6859 | Stale=0 | ETA=1821.9m +rank0 [2026-10-28 23:01:51,134] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 23:01:51,134] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 23:01:51,134] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 23:01:51,134] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 23:01:51,134] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 23:01:51,134] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 23:01:51,134] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 23:01:51,134] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/ari=0.521367 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.55007 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/top1_competing=0.621131 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.608022 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0277861 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0335473 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 23:01:51,135] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/seconds_per_rank=21.4317 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 23:01:51,136] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 23:01:51,137] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 23:01:51,137] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 23:01:51,137] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding/ari=0.267351 +rank0 [2026-10-28 23:01:51,137] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 23:01:51,137] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/embedding_minus_raw/ari=0.175728 +rank0 [2026-10-28 23:01:51,137] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 23:01:51,137] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 23:01:51,137] INFO : VALIDATION_PROBE | Step=14000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 23:01:51,142] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:01:51,400] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 23:01:51,403] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 23:02:17,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 23:02:20,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 23:02:22,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 23:02:23,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 23:02:33,432] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-28 23:02:33,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:02:33,998] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 23:02:34,002] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 23:02:49,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 23:02:51,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 23:02:52,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 23:02:53,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 23:03:03,395] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 23:03:03,406] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:03:03,862] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 23:03:03,865] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 23:03:19,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 23:03:20,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 23:03:21,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 23:03:22,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 23:03:32,758] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-28 23:03:32,897] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits +rank0 [2026-10-28 23:03:36,139] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits +rank0 [2026-10-28 23:03:37,205] INFO : Plotted number of elements +rank0 [2026-10-28 23:03:38,752] INFO : Plotted jets +rank0 [2026-10-28 23:03:39,894] INFO : Plotted jet ratio +rank0 [2026-10-28 23:03:39,962] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 23:03:43,164] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 23:03:44,605] INFO : Plotted number of elements +rank0 [2026-10-28 23:03:46,104] INFO : Plotted jets +rank0 [2026-10-28 23:03:47,255] INFO : Plotted jet ratio +rank0 [2026-10-28 23:03:47,314] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits +rank0 [2026-10-28 23:03:49,650] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits +rank0 [2026-10-28 23:03:50,501] INFO : Plotted number of elements +rank0 [2026-10-28 23:03:51,981] INFO : Plotted jets +rank0 [2026-10-28 23:03:53,091] INFO : Plotted jet ratio +rank0 [2026-10-28 23:08:23,114] INFO : Step 14100/50000 rank0 | Train Loss: 1.3472 | Train Loss Unweighted: 0.6964 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1511 | Classification: 0.1164 | Classification_binary: 0.2090 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1819 | Regression_eta: 0.0023 | Regression_pt: 0.1843 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1058 | Classification_binary: 0.2225 | Regression_cos_phi: 0.1349 | Regression_energy: 0.1049 | Regression_eta: 0.0738 | Regression_pt: 0.1143 | Regression_sin_phi: 0.1487 | LR: 8.35e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.0176s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8885 | max_loss: 0.1430 | max_backward: 2.1784 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 1.1214 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7149.4 | uss_MB: 6238.2 | child_rss_MB: 5960.3 | allocated_MB: 451.8 | reserved_MB: 25710.0 | inactive_split_MB: 1103.7 +rank0 [2026-10-28 23:08:23,373] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:12:53,496] INFO : Step 14200/50000 rank0 | Train Loss: 1.4358 | Train Loss Unweighted: 0.7354 | Train Loss Components: Auxiliary: 0.2744 | Cardinality: 0.1685 | Classification: 0.1248 | Classification_binary: 0.2623 | HitGrouping: 0.1814 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1709 | Regression_eta: 0.0023 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.2686 | Regression_cos_phi: 0.1134 | Regression_energy: 0.1063 | Regression_eta: 0.0680 | Regression_pt: 0.1141 | Regression_sin_phi: 0.1249 | LR: 8.32e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9777s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0875 | max_forward: 0.8579 | max_loss: 0.1577 | max_backward: 1.8303 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6333.7 | uss_MB: 5422.0 | child_rss_MB: 5960.5 | allocated_MB: 449.2 | reserved_MB: 27638.0 | inactive_split_MB: 1319.6 +rank0 [2026-10-28 23:12:53,736] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 48.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:17:27,174] INFO : Step 14300/50000 rank0 | Train Loss: 1.2642 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2452 | Cardinality: 0.1652 | Classification: 0.1069 | Classification_binary: 0.2108 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0018 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0662 | Regression_energy: 0.1037 | Regression_eta: 0.0391 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0752 | LR: 8.30e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9460s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7599 | max_loss: 0.1477 | max_backward: 1.7369 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6331.9 | uss_MB: 5419.4 | child_rss_MB: 5959.7 | allocated_MB: 446.3 | reserved_MB: 29582.0 | inactive_split_MB: 949.7 +rank0 [2026-10-28 23:17:27,416] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:22:06,561] INFO : Step 14400/50000 rank0 | Train Loss: 1.2457 | Train Loss Unweighted: 0.6253 | Train Loss Components: Auxiliary: 0.2300 | Cardinality: 0.1408 | Classification: 0.1072 | Classification_binary: 0.1847 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1629 | Regression_eta: 0.0022 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0880 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0809 | Regression_eta: 0.0429 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0737 | LR: 8.28e-04 | DataLoad Time: 0.0202s | Model Forward Time: 6.6542s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.9667 | max_loss: 0.2306 | max_backward: 2.7434 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6331.8 | uss_MB: 4969.8 | child_rss_MB: 5956.2 | allocated_MB: 450.7 | reserved_MB: 29584.0 | inactive_split_MB: 1252.8 +rank0 [2026-10-28 23:22:06,804] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 95% +2 6 0x7408, 15961 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 50.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +6 10 0x7408, 62036 48.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:26:51,219] INFO : Step 14500/50000 rank0 | Train Loss: 1.4093 | Train Loss Unweighted: 0.7194 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1803 | Classification: 0.1211 | Classification_binary: 0.2406 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0862 | Classification_binary: 0.2793 | Regression_cos_phi: 0.1434 | Regression_energy: 0.0933 | Regression_eta: 0.0805 | Regression_pt: 0.1026 | Regression_sin_phi: 0.1497 | LR: 8.26e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.4946s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8204 | max_loss: 0.1455 | max_backward: 2.0838 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7592 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1066 | Memory: rss_MB: 6329.0 | uss_MB: 5423.5 | child_rss_MB: 5932.6 | allocated_MB: 449.1 | reserved_MB: 29584.0 | inactive_split_MB: 1312.1 +rank0 [2026-10-28 23:26:51,472] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 47.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:31:26,741] INFO : Step 14600/50000 rank0 | Train Loss: 1.4466 | Train Loss Unweighted: 0.7628 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1607 | Classification: 0.1257 | Classification_binary: 0.2761 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1767 | Regression_eta: 0.0023 | Regression_pt: 0.1796 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0976 | Classification_binary: 0.2754 | Regression_cos_phi: 0.1446 | Regression_energy: 0.0953 | Regression_eta: 0.0955 | Regression_pt: 0.1057 | Regression_sin_phi: 0.1057 | LR: 8.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.4673s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0622 | max_forward: 0.8658 | max_loss: 0.1496 | max_backward: 1.9094 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.9894 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6320.2 | uss_MB: 5414.6 | child_rss_MB: 5962.0 | allocated_MB: 450.2 | reserved_MB: 29584.0 | inactive_split_MB: 939.8 +rank0 [2026-10-28 23:31:26,979] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:36:00,436] INFO : Step 14700/50000 rank0 | Train Loss: 1.2298 | Train Loss Unweighted: 0.6544 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1321 | Classification: 0.0995 | Classification_binary: 0.1882 | HitGrouping: 0.1788 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1813 | Regression_eta: 0.0021 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1977 | Regression_cos_phi: 0.0910 | Regression_energy: 0.1052 | Regression_eta: 0.0561 | Regression_pt: 0.1112 | Regression_sin_phi: 0.0775 | LR: 8.21e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.6827s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8361 | max_loss: 0.2462 | max_backward: 1.9700 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.8314 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6323.9 | uss_MB: 5421.4 | child_rss_MB: 5964.3 | allocated_MB: 445.8 | reserved_MB: 31548.0 | inactive_split_MB: 1127.1 +rank0 [2026-10-28 23:36:00,675] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 77% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 97% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 99% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 1% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:40:35,303] INFO : Step 14800/50000 rank0 | Train Loss: 1.2682 | Train Loss Unweighted: 0.6574 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1516 | Classification: 0.1063 | Classification_binary: 0.2120 | HitGrouping: 0.1767 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1664 | Regression_eta: 0.0020 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2314 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0831 | Regression_eta: 0.0477 | Regression_pt: 0.0962 | Regression_sin_phi: 0.0991 | LR: 8.19e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.4862s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7530 | max_loss: 0.1476 | max_backward: 1.8229 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6323.4 | uss_MB: 5420.5 | child_rss_MB: 5957.9 | allocated_MB: 449.4 | reserved_MB: 33554.0 | inactive_split_MB: 1332.6 +rank0 [2026-10-28 23:40:35,548] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 99% +6 10 0x7408, 62036 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:45:12,247] INFO : Step 14900/50000 rank0 | Train Loss: 1.2753 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2409 | Cardinality: 0.1305 | Classification: 0.1126 | Classification_binary: 0.2387 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1497 | Regression_eta: 0.0022 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.2425 | Regression_cos_phi: 0.0999 | Regression_energy: 0.0660 | Regression_eta: 0.0530 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0883 | LR: 8.17e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.4990s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.7955 | max_loss: 0.1633 | max_backward: 1.8376 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.9623 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1206 | Memory: rss_MB: 6322.3 | uss_MB: 5420.8 | child_rss_MB: 5961.8 | allocated_MB: 449.7 | reserved_MB: 33554.0 | inactive_split_MB: 1310.1 +rank0 [2026-10-28 23:45:12,500] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:49:46,041] INFO : Step 15000/50000 rank0 | Train Loss: 1.2701 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1516 | Classification: 0.1012 | Classification_binary: 0.2121 | HitGrouping: 0.1755 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1837 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0801 | Regression_eta: 0.0595 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0856 | LR: 8.15e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8360s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7505 | max_loss: 0.2449 | max_backward: 1.7335 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.4870 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6318.2 | uss_MB: 5419.0 | child_rss_MB: 5953.6 | allocated_MB: 447.3 | reserved_MB: 33554.0 | inactive_split_MB: 1038.0 +rank0 [2026-10-28 23:49:46,284] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +6 10 0x7408, 62036 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 23:49:46,308] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 23:49:46,594] INFO : Running validation on rank0 +rank0 [2026-10-28 23:49:46,595] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 23:49:46,595] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 23:50:37,974] INFO : VALIDATION | Step=15000/50000 | Train Loss=1.2701 | Valid Loss=1.3039 | Train Loss Unweighted=0.6495 | Valid Loss Unweighted=0.6725 | Stale=0 | ETA=1767.0m +rank0 [2026-10-28 23:50:37,974] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-28 23:50:37,974] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/ari=0.530853 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.549156 +rank0 [2026-10-28 23:50:37,975] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/top1_competing=0.619296 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.638586 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0183 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0641112 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-28 23:50:37,976] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/seconds_per_rank=21.3427 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding/ari=0.268791 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/embedding_minus_raw/ari=0.177168 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-28 23:50:37,977] INFO : VALIDATION_PROBE | Step=15000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-28 23:50:37,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:50:38,251] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-28 23:50:38,262] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-28 23:51:04,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-28 23:51:05,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-28 23:51:07,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-28 23:51:08,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-28 23:51:18,407] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 23:51:18,491] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:51:18,974] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-28 23:51:18,977] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-28 23:51:35,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-28 23:51:36,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-28 23:51:37,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-28 23:51:38,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-28 23:51:48,537] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-28 23:51:48,548] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 23:51:49,026] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-28 23:51:49,029] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-28 23:52:04,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-28 23:52:05,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-28 23:52:06,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-28 23:52:07,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-28 23:52:17,897] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-28 23:52:18,017] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits +rank0 [2026-10-28 23:52:21,188] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits +rank0 [2026-10-28 23:52:22,224] INFO : Plotted number of elements +rank0 [2026-10-28 23:52:23,777] INFO : Plotted jets +rank0 [2026-10-28 23:52:25,324] INFO : Plotted jet ratio +rank0 [2026-10-28 23:52:25,373] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 23:52:28,590] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits +rank0 [2026-10-28 23:52:29,597] INFO : Plotted number of elements +rank0 [2026-10-28 23:52:31,106] INFO : Plotted jets +rank0 [2026-10-28 23:52:32,258] INFO : Plotted jet ratio +rank0 [2026-10-28 23:52:32,309] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits +rank0 [2026-10-28 23:52:34,762] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits +rank0 [2026-10-28 23:52:35,598] INFO : Plotted number of elements +rank0 [2026-10-28 23:52:37,078] INFO : Plotted jets +rank0 [2026-10-28 23:52:38,195] INFO : Plotted jet ratio +rank0 [2026-10-28 23:57:19,719] INFO : Step 15100/50000 rank0 | Train Loss: 1.2388 | Train Loss Unweighted: 0.6590 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1366 | Classification: 0.1178 | Classification_binary: 0.1841 | HitGrouping: 0.1810 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1743 | Regression_eta: 0.0018 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.2012 | Regression_cos_phi: 0.1019 | Regression_energy: 0.0948 | Regression_eta: 0.0644 | Regression_pt: 0.1069 | Regression_sin_phi: 0.1036 | LR: 8.12e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.6617s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.8010 | max_loss: 0.1660 | max_backward: 1.9832 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.8565 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7422.7 | uss_MB: 6518.8 | child_rss_MB: 5938.1 | allocated_MB: 446.2 | reserved_MB: 25230.0 | inactive_split_MB: 1087.3 +rank0 [2026-10-28 23:57:19,958] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +2 6 0x7408, 15961 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 99% +4 8 0x7408, 13395 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 47.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:02:06,219] INFO : Step 15200/50000 rank0 | Train Loss: 1.3362 | Train Loss Unweighted: 0.6611 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1596 | Classification: 0.1086 | Classification_binary: 0.2256 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1600 | Regression_eta: 0.0022 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.2428 | Regression_cos_phi: 0.1295 | Regression_energy: 0.0711 | Regression_eta: 0.0733 | Regression_pt: 0.0769 | Regression_sin_phi: 0.1267 | LR: 8.10e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2660s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1077 | max_forward: 0.8572 | max_loss: 0.1505 | max_backward: 1.8951 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7089 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6402.3 | uss_MB: 5500.4 | child_rss_MB: 5939.7 | allocated_MB: 449.9 | reserved_MB: 25448.0 | inactive_split_MB: 1183.4 +rank0 [2026-10-29 00:02:06,472] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +4 8 0x7408, 13395 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:06:55,372] INFO : Step 15300/50000 rank0 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6783 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1400 | Classification: 0.1062 | Classification_binary: 0.2357 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1646 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2270 | Regression_cos_phi: 0.1036 | Regression_energy: 0.1016 | Regression_eta: 0.0602 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0882 | LR: 8.08e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.3715s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7962 | max_loss: 0.2606 | max_backward: 1.9831 | max_optimizer_step: 0.2491 | max_tensorboard_logging: 0.7840 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6402.1 | uss_MB: 5500.3 | child_rss_MB: 5940.3 | allocated_MB: 446.7 | reserved_MB: 27282.0 | inactive_split_MB: 1189.4 +rank0 [2026-10-29 00:06:55,633] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:11:35,112] INFO : Step 15400/50000 rank0 | Train Loss: 1.2329 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1583 | Classification: 0.1127 | Classification_binary: 0.1891 | HitGrouping: 0.1750 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1648 | Regression_eta: 0.0018 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0858 | Classification_binary: 0.1821 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0932 | Regression_eta: 0.0611 | Regression_pt: 0.1015 | Regression_sin_phi: 0.1052 | LR: 8.05e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9219s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.7153 | max_loss: 0.1508 | max_backward: 1.6501 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.5965 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6395.9 | uss_MB: 5491.2 | child_rss_MB: 5941.0 | allocated_MB: 444.7 | reserved_MB: 27520.0 | inactive_split_MB: 1102.9 +rank0 [2026-10-29 00:11:35,351] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 49.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:16:24,060] INFO : Step 15500/50000 rank0 | Train Loss: 1.2944 | Train Loss Unweighted: 0.6657 | Train Loss Components: Auxiliary: 0.2480 | Cardinality: 0.1645 | Classification: 0.1174 | Classification_binary: 0.2036 | HitGrouping: 0.1739 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1691 | Regression_eta: 0.0021 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2677 | Regression_cos_phi: 0.1139 | Regression_energy: 0.0939 | Regression_eta: 0.0611 | Regression_pt: 0.1038 | Regression_sin_phi: 0.1117 | LR: 8.03e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.9579s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.9970 | max_loss: 0.1740 | max_backward: 2.3916 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.8759 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 6406.1 | uss_MB: 5472.3 | child_rss_MB: 5938.5 | allocated_MB: 449.9 | reserved_MB: 27520.0 | inactive_split_MB: 1174.5 +rank0 [2026-10-29 00:16:24,302] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 99% +4 8 0x7408, 13395 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 0% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:21:08,003] INFO : Step 15600/50000 rank0 | Train Loss: 1.4613 | Train Loss Unweighted: 0.7617 | Train Loss Components: Auxiliary: 0.2872 | Cardinality: 0.1801 | Classification: 0.1276 | Classification_binary: 0.2641 | HitGrouping: 0.1785 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1795 | Regression_eta: 0.0025 | Regression_pt: 0.1856 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0798 | Classification_binary: 0.2518 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0859 | Regression_eta: 0.0537 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0644 | LR: 8.01e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.4617s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.9893 | max_loss: 0.1680 | max_backward: 2.1731 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.0419 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6396.7 | uss_MB: 5034.1 | child_rss_MB: 5938.2 | allocated_MB: 448.1 | reserved_MB: 27520.0 | inactive_split_MB: 1080.9 +rank0 [2026-10-29 00:21:08,258] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 99% +4 8 0x7408, 13395 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 3% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:25:54,872] INFO : Step 15700/50000 rank0 | Train Loss: 1.3299 | Train Loss Unweighted: 0.6547 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1880 | Classification: 0.1192 | Classification_binary: 0.2203 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1547 | Regression_eta: 0.0020 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1709 | Regression_cos_phi: 0.1218 | Regression_energy: 0.0877 | Regression_eta: 0.0377 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0970 | LR: 7.98e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.7324s | Phase Times: max_pre_step_barrier: 0.0061 | max_batch_to_device: 0.0068 | max_forward: 0.9761 | max_loss: 0.2543 | max_backward: 2.0225 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8016 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.1020 | Memory: rss_MB: 6402.6 | uss_MB: 5496.7 | child_rss_MB: 5941.3 | allocated_MB: 448.3 | reserved_MB: 27520.0 | inactive_split_MB: 1390.5 +rank0 [2026-10-29 00:25:55,128] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:30:34,313] INFO : Step 15800/50000 rank0 | Train Loss: 1.2079 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1572 | Classification: 0.0909 | Classification_binary: 0.1814 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1704 | Regression_eta: 0.0019 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0679 | Regression_energy: 0.1287 | Regression_eta: 0.0504 | Regression_pt: 0.1425 | Regression_sin_phi: 0.1384 | LR: 7.96e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1286s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1066 | max_forward: 0.8154 | max_loss: 0.1440 | max_backward: 1.6020 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.7740 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6405.3 | uss_MB: 5497.2 | child_rss_MB: 5941.3 | allocated_MB: 447.1 | reserved_MB: 27520.0 | inactive_split_MB: 1038.3 +rank0 [2026-10-29 00:30:34,571] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:35:19,976] INFO : Step 15900/50000 rank0 | Train Loss: 1.4055 | Train Loss Unweighted: 0.7313 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1808 | Classification: 0.1304 | Classification_binary: 0.2426 | HitGrouping: 0.1763 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1752 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2447 | Regression_cos_phi: 0.0954 | Regression_energy: 0.0851 | Regression_eta: 0.0468 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0780 | LR: 7.94e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.4828s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 1.0653 | max_loss: 0.1515 | max_backward: 2.0024 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.6957 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6402.5 | uss_MB: 5500.0 | child_rss_MB: 5940.1 | allocated_MB: 447.4 | reserved_MB: 27520.0 | inactive_split_MB: 1172.5 +rank0 [2026-10-29 00:35:20,215] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +4 8 0x7408, 13395 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:40:01,435] INFO : Step 16000/50000 rank0 | Train Loss: 1.3107 | Train Loss Unweighted: 0.6580 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1760 | Classification: 0.1019 | Classification_binary: 0.2391 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1548 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2312 | Regression_cos_phi: 0.1438 | Regression_energy: 0.0829 | Regression_eta: 0.0901 | Regression_pt: 0.0988 | Regression_sin_phi: 0.1054 | LR: 7.91e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.5147s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.1053 | max_forward: 1.1703 | max_loss: 0.1516 | max_backward: 1.8974 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6403.0 | uss_MB: 5502.7 | child_rss_MB: 5941.0 | allocated_MB: 447.2 | reserved_MB: 27520.0 | inactive_split_MB: 1195.6 +rank0 [2026-10-29 00:40:01,694] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:40:01,717] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-16000.pth +rank0 [2026-10-29 00:40:02,104] INFO : Running validation on rank0 +rank0 [2026-10-29 00:40:02,104] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 00:40:02,104] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 00:40:58,597] INFO : VALIDATION | Step=16000/50000 | Train Loss=1.3107 | Valid Loss=1.2521 | Train Loss Unweighted=0.6580 | Valid Loss Unweighted=0.6443 | Stale=0 | ETA=1716.2m +rank0 [2026-10-29 00:40:58,597] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:40:58,597] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 00:40:58,597] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/ari=0.546496 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.56426 +rank0 [2026-10-29 00:40:58,598] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/top1_competing=0.637887 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.619376 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.0026564 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0449013 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/seconds_per_rank=21.3753 +rank0 [2026-10-29 00:40:58,599] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding/ari=0.265513 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/embedding_minus_raw/ari=0.17389 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 00:40:58,600] INFO : VALIDATION_PROBE | Step=16000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 00:40:58,606] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:40:58,862] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 00:40:58,872] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 00:41:23,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 00:41:24,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 00:41:25,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 00:41:26,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 00:41:36,978] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 00:41:36,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:41:37,529] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 00:41:37,534] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 00:41:53,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 00:41:54,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 00:41:55,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 00:41:56,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 00:42:06,771] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 00:42:07,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 00:42:07,840] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 00:42:07,843] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 00:42:23,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 00:42:24,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 00:42:25,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 00:42:26,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 00:42:36,406] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 00:42:36,423] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits +rank0 [2026-10-29 00:42:39,663] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits +rank0 [2026-10-29 00:42:40,709] INFO : Plotted number of elements +rank0 [2026-10-29 00:42:42,270] INFO : Plotted jets +rank0 [2026-10-29 00:42:43,404] INFO : Plotted jet ratio +rank0 [2026-10-29 00:42:43,498] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 00:42:46,680] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 00:42:47,673] INFO : Plotted number of elements +rank0 [2026-10-29 00:42:49,176] INFO : Plotted jets +rank0 [2026-10-29 00:42:50,322] INFO : Plotted jet ratio +rank0 [2026-10-29 00:42:50,389] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits +rank0 [2026-10-29 00:42:52,816] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits +rank0 [2026-10-29 00:42:54,152] INFO : Plotted number of elements +rank0 [2026-10-29 00:42:55,639] INFO : Plotted jets +rank0 [2026-10-29 00:42:56,814] INFO : Plotted jet ratio +rank0 [2026-10-29 00:47:34,970] INFO : Step 16100/50000 rank0 | Train Loss: 1.2037 | Train Loss Unweighted: 0.6025 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1467 | Classification: 0.1080 | Classification_binary: 0.1878 | HitGrouping: 0.1747 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1512 | Regression_eta: 0.0018 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0883 | Regression_energy: 0.0727 | Regression_eta: 0.0538 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0747 | LR: 7.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9316s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7665 | max_loss: 0.1405 | max_backward: 1.4602 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7547.3 | uss_MB: 6643.5 | child_rss_MB: 5941.5 | allocated_MB: 446.1 | reserved_MB: 27680.0 | inactive_split_MB: 1038.2 +rank0 [2026-10-29 00:47:35,211] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:52:15,823] INFO : Step 16200/50000 rank0 | Train Loss: 1.2083 | Train Loss Unweighted: 0.6192 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1499 | Classification: 0.1107 | Classification_binary: 0.2017 | HitGrouping: 0.1711 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1506 | Regression_eta: 0.0018 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0845 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0889 | Regression_eta: 0.0583 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0835 | LR: 7.86e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.6184s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0473 | max_forward: 0.8343 | max_loss: 0.2530 | max_backward: 2.1136 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6458.0 | uss_MB: 5555.6 | child_rss_MB: 5961.3 | allocated_MB: 446.7 | reserved_MB: 27682.0 | inactive_split_MB: 1157.2 +rank0 [2026-10-29 00:52:16,062] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 00:56:52,358] INFO : Step 16300/50000 rank0 | Train Loss: 1.2745 | Train Loss Unweighted: 0.6291 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1604 | Classification: 0.1062 | Classification_binary: 0.2417 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1373 | Regression_eta: 0.0022 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2445 | Regression_cos_phi: 0.1309 | Regression_energy: 0.0708 | Regression_eta: 0.0482 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0813 | LR: 7.84e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.4510s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.8059 | max_loss: 0.2506 | max_backward: 2.1763 | max_optimizer_step: 0.5213 | max_tensorboard_logging: 0.5923 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6459.9 | uss_MB: 5555.3 | child_rss_MB: 5936.4 | allocated_MB: 450.9 | reserved_MB: 27682.0 | inactive_split_MB: 950.8 +rank0 [2026-10-29 00:56:52,596] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:01:27,242] INFO : Step 16400/50000 rank0 | Train Loss: 1.2654 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2380 | Cardinality: 0.1554 | Classification: 0.1095 | Classification_binary: 0.2018 | HitGrouping: 0.1699 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1561 | Regression_eta: 0.0023 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0905 | Regression_eta: 0.0652 | Regression_pt: 0.1046 | Regression_sin_phi: 0.0691 | LR: 7.81e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9381s | Phase Times: max_pre_step_barrier: 0.0533 | max_batch_to_device: 0.0064 | max_forward: 0.7459 | max_loss: 0.1486 | max_backward: 1.5352 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.6677 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6452.2 | uss_MB: 5545.1 | child_rss_MB: 5962.3 | allocated_MB: 446.4 | reserved_MB: 27682.0 | inactive_split_MB: 949.1 +rank0 [2026-10-29 01:01:27,503] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 53.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:06:05,711] INFO : Step 16500/50000 rank0 | Train Loss: 1.1728 | Train Loss Unweighted: 0.5918 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1444 | Classification: 0.0927 | Classification_binary: 0.1686 | HitGrouping: 0.1723 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0021 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0767 | Classification_binary: 0.2457 | Regression_cos_phi: 0.1105 | Regression_energy: 0.0862 | Regression_eta: 0.0680 | Regression_pt: 0.0950 | Regression_sin_phi: 0.1030 | LR: 7.79e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1514s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8043 | max_loss: 0.1772 | max_backward: 2.0250 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6458.1 | uss_MB: 5554.8 | child_rss_MB: 5927.7 | allocated_MB: 448.6 | reserved_MB: 29698.0 | inactive_split_MB: 1167.6 +rank0 [2026-10-29 01:06:05,953] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 47.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:10:54,499] INFO : Step 16600/50000 rank0 | Train Loss: 1.2920 | Train Loss Unweighted: 0.6427 | Train Loss Components: Auxiliary: 0.2455 | Cardinality: 0.1794 | Classification: 0.1148 | Classification_binary: 0.2098 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1561 | Regression_eta: 0.0021 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.2081 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0777 | Regression_eta: 0.0515 | Regression_pt: 0.0848 | Regression_sin_phi: 0.1016 | LR: 7.77e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.0511s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0377 | max_forward: 1.0830 | max_loss: 0.1480 | max_backward: 1.7919 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6457.2 | uss_MB: 5552.9 | child_rss_MB: 5951.6 | allocated_MB: 449.3 | reserved_MB: 29698.0 | inactive_split_MB: 1014.7 +rank0 [2026-10-29 01:10:54,739] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:15:46,549] INFO : Step 16700/50000 rank0 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6325 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1731 | Classification: 0.1072 | Classification_binary: 0.2252 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1469 | Regression_eta: 0.0019 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0856 | Regression_eta: 0.0543 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0726 | LR: 7.74e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1555s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0722 | max_loss: 0.1506 | max_backward: 2.0365 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5067 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6456.8 | uss_MB: 5555.5 | child_rss_MB: 5918.8 | allocated_MB: 448.3 | reserved_MB: 29698.0 | inactive_split_MB: 893.3 +rank0 [2026-10-29 01:15:46,795] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 98% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 14% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 2% +6 10 0x7408, 62036 48.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 79% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:20:21,550] INFO : Step 16800/50000 rank0 | Train Loss: 1.1947 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1542 | Classification: 0.1036 | Classification_binary: 0.1800 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0023 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 0.1867 | Regression_cos_phi: 0.1085 | Regression_energy: 0.0770 | Regression_eta: 0.0968 | Regression_pt: 0.0835 | Regression_sin_phi: 0.1080 | LR: 7.72e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.0019s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0166 | max_forward: 0.8224 | max_loss: 0.1466 | max_backward: 1.6608 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6428.0 | uss_MB: 5552.9 | child_rss_MB: 5812.3 | allocated_MB: 446.4 | reserved_MB: 29698.0 | inactive_split_MB: 1161.6 +rank0 [2026-10-29 01:20:21,790] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 98% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 23% +4 8 0x7408, 13395 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 88% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 98% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:25:07,549] INFO : Step 16900/50000 rank0 | Train Loss: 1.2025 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1406 | Classification: 0.1120 | Classification_binary: 0.1884 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1446 | Regression_eta: 0.0023 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.2044 | Regression_cos_phi: 0.0930 | Regression_energy: 0.0717 | Regression_eta: 0.0621 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0988 | LR: 7.69e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.0083s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7445 | max_loss: 0.1453 | max_backward: 1.8211 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6418.8 | uss_MB: 5553.4 | child_rss_MB: 5783.4 | allocated_MB: 448.9 | reserved_MB: 29698.0 | inactive_split_MB: 901.3 +rank0 [2026-10-29 01:25:07,791] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:29:52,829] INFO : Step 17000/50000 rank0 | Train Loss: 1.2049 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2349 | Cardinality: 0.1467 | Classification: 0.1018 | Classification_binary: 0.2228 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2400 | Regression_cos_phi: 0.0989 | Regression_energy: 0.0657 | Regression_eta: 0.0651 | Regression_pt: 0.0730 | Regression_sin_phi: 0.1002 | LR: 7.67e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.8884s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0141 | max_forward: 1.3980 | max_loss: 0.5129 | max_backward: 2.5664 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.5255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1218 | Memory: rss_MB: 6382.7 | uss_MB: 5556.2 | child_rss_MB: 5576.1 | allocated_MB: 448.2 | reserved_MB: 29698.0 | inactive_split_MB: 955.8 +rank0 [2026-10-29 01:29:53,072] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:29:53,097] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-17000.pth +rank0 [2026-10-29 01:29:53,499] INFO : Running validation on rank0 +rank0 [2026-10-29 01:29:53,499] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 01:29:53,500] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 01:30:46,605] INFO : VALIDATION | Step=17000/50000 | Train Loss=1.2049 | Valid Loss=1.2036 | Train Loss Unweighted=0.6194 | Valid Loss Unweighted=0.6086 | Stale=0 | ETA=1664.4m +rank0 [2026-10-29 01:30:46,605] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 01:30:46,605] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 01:30:46,605] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 01:30:46,605] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 01:30:46,605] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 01:30:46,605] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 01:30:46,605] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/ari=0.546001 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.592943 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/top1_competing=0.675382 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.636362 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.00315199 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0618875 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 01:30:46,606] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/seconds_per_rank=21.8556 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 01:30:46,607] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 01:30:46,608] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding/ari=0.292313 +rank0 [2026-10-29 01:30:46,608] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 01:30:46,608] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/embedding_minus_raw/ari=0.200689 +rank0 [2026-10-29 01:30:46,608] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 01:30:46,608] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 01:30:46,608] INFO : VALIDATION_PROBE | Step=17000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 01:30:46,613] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:30:46,869] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 01:30:46,879] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 01:31:13,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 01:31:15,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 01:31:16,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 01:31:17,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 01:31:27,982] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 01:31:28,036] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:31:28,588] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 01:31:28,592] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 01:31:44,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 01:31:45,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 01:31:47,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 01:31:48,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 01:31:58,237] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 01:31:58,251] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 01:31:58,744] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 01:31:58,748] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 01:32:14,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 01:32:15,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 01:32:16,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 01:32:17,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 01:32:27,481] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 01:32:27,585] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits +rank0 [2026-10-29 01:32:30,881] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits +rank0 [2026-10-29 01:32:31,892] INFO : Plotted number of elements +rank0 [2026-10-29 01:32:33,443] INFO : Plotted jets +rank0 [2026-10-29 01:32:34,568] INFO : Plotted jet ratio +rank0 [2026-10-29 01:32:34,626] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 01:32:37,833] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 01:32:38,813] INFO : Plotted number of elements +rank0 [2026-10-29 01:32:40,321] INFO : Plotted jets +rank0 [2026-10-29 01:32:41,501] INFO : Plotted jet ratio +rank0 [2026-10-29 01:32:41,565] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits +rank0 [2026-10-29 01:32:44,020] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits +rank0 [2026-10-29 01:32:44,865] INFO : Plotted number of elements +rank0 [2026-10-29 01:32:46,860] INFO : Plotted jets +rank0 [2026-10-29 01:32:48,021] INFO : Plotted jet ratio +rank0 [2026-10-29 01:37:36,845] INFO : Step 17100/50000 rank0 | Train Loss: 1.2386 | Train Loss Unweighted: 0.5919 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1931 | Classification: 0.0995 | Classification_binary: 0.1969 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1859 | Regression_cos_phi: 0.0880 | Regression_energy: 0.0838 | Regression_eta: 0.0607 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0936 | LR: 7.64e-04 | DataLoad Time: 0.0147s | Model Forward Time: 6.0139s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0067 | max_forward: 1.0304 | max_loss: 0.1869 | max_backward: 2.3467 | max_optimizer_step: 0.3175 | max_tensorboard_logging: 0.7405 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 7631.0 | uss_MB: 6798.9 | child_rss_MB: 5579.4 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 1373.9 +rank0 [2026-10-29 01:37:37,085] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:42:27,837] INFO : Step 17200/50000 rank0 | Train Loss: 1.1810 | Train Loss Unweighted: 0.5844 | Train Loss Components: Auxiliary: 0.2270 | Cardinality: 0.1495 | Classification: 0.1085 | Classification_binary: 0.1683 | HitGrouping: 0.1731 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0022 | Regression_pt: 0.1515 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0868 | Regression_energy: 0.0704 | Regression_eta: 0.0506 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0966 | LR: 7.62e-04 | DataLoad Time: 0.0156s | Model Forward Time: 6.8094s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7638 | max_loss: 0.2424 | max_backward: 1.9393 | max_optimizer_step: 0.2921 | max_tensorboard_logging: 2.1267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6444.8 | uss_MB: 5633.5 | child_rss_MB: 5577.8 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 988.1 +rank0 [2026-10-29 01:42:28,077] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 52.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:47:11,986] INFO : Step 17300/50000 rank0 | Train Loss: 1.2271 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2397 | Cardinality: 0.1372 | Classification: 0.1102 | Classification_binary: 0.2127 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1512 | Regression_eta: 0.0022 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0801 | Classification_binary: 0.2648 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0937 | Regression_eta: 0.0653 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0791 | LR: 7.59e-04 | DataLoad Time: 0.0180s | Model Forward Time: 6.7989s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0243 | max_forward: 1.4273 | max_loss: 0.6053 | max_backward: 3.4810 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6426.5 | uss_MB: 5175.0 | child_rss_MB: 5586.3 | allocated_MB: 449.6 | reserved_MB: 26752.0 | inactive_split_MB: 1120.1 +rank0 [2026-10-29 01:47:12,231] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 53.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 45% +4 8 0x7408, 13395 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 98% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 97% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:52:04,867] INFO : Step 17400/50000 rank0 | Train Loss: 1.2297 | Train Loss Unweighted: 0.6013 | Train Loss Components: Auxiliary: 0.2404 | Cardinality: 0.1709 | Classification: 0.1017 | Classification_binary: 0.2232 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1354 | Regression_eta: 0.0019 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2079 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0673 | Regression_eta: 0.0633 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0873 | LR: 7.57e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.3013s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0185 | max_forward: 1.0000 | max_loss: 0.1504 | max_backward: 1.9673 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6331 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6409.7 | uss_MB: 5631.7 | child_rss_MB: 5583.3 | allocated_MB: 447.1 | reserved_MB: 28732.0 | inactive_split_MB: 1125.6 +rank0 [2026-10-29 01:52:05,107] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 01:56:49,440] INFO : Step 17500/50000 rank0 | Train Loss: 1.2152 | Train Loss Unweighted: 0.6044 | Train Loss Components: Auxiliary: 0.2290 | Cardinality: 0.1485 | Classification: 0.1023 | Classification_binary: 0.1789 | HitGrouping: 0.1757 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1590 | Regression_eta: 0.0026 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0923 | Regression_eta: 0.0453 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0935 | LR: 7.54e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.8836s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0167 | max_forward: 0.7711 | max_loss: 0.1449 | max_backward: 1.8153 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.4947 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6388.7 | uss_MB: 5638.8 | child_rss_MB: 5515.3 | allocated_MB: 450.1 | reserved_MB: 28732.0 | inactive_split_MB: 1375.8 +rank0 [2026-10-29 01:56:49,679] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 53.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:01:40,533] INFO : Step 17600/50000 rank0 | Train Loss: 1.1560 | Train Loss Unweighted: 0.6042 | Train Loss Components: Auxiliary: 0.2286 | Cardinality: 0.1294 | Classification: 0.1057 | Classification_binary: 0.1993 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1471 | Regression_eta: 0.0018 | Regression_pt: 0.1484 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2097 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0685 | Regression_eta: 0.0429 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0888 | LR: 7.52e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.5559s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.8868 | max_loss: 0.1705 | max_backward: 2.0183 | max_optimizer_step: 0.2066 | max_tensorboard_logging: 0.7222 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1349 | Memory: rss_MB: 6384.6 | uss_MB: 5639.3 | child_rss_MB: 5506.2 | allocated_MB: 448.3 | reserved_MB: 28732.0 | inactive_split_MB: 1319.7 +rank0 [2026-10-29 02:01:40,774] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 6% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 42% 4% +2 6 0x7408, 15961 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:06:29,567] INFO : Step 17700/50000 rank0 | Train Loss: 1.7848 | Train Loss Unweighted: 0.7433 | Train Loss Components: Auxiliary: 0.3415 | Cardinality: 0.4306 | Classification: 0.1200 | Classification_binary: 0.2341 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1920 | Regression_eta: 0.0037 | Regression_pt: 0.1910 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.2030 | Regression_cos_phi: 0.0977 | Regression_energy: 0.1045 | Regression_eta: 0.0925 | Regression_pt: 0.1127 | Regression_sin_phi: 0.1041 | LR: 7.49e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.2553s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8788 | max_loss: 0.4738 | max_backward: 1.9859 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.5305 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6359.7 | uss_MB: 5640.4 | child_rss_MB: 5400.4 | allocated_MB: 450.1 | reserved_MB: 28738.0 | inactive_split_MB: 1125.6 +rank0 [2026-10-29 02:06:29,811] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 53.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:11:15,505] INFO : Step 17800/50000 rank0 | Train Loss: 1.1828 | Train Loss Unweighted: 0.6014 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1456 | Classification: 0.0982 | Classification_binary: 0.1996 | HitGrouping: 0.1749 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1503 | Regression_eta: 0.0019 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.2029 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0809 | Regression_eta: 0.0684 | Regression_pt: 0.0879 | Regression_sin_phi: 0.1076 | LR: 7.47e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.9261s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.3167 | max_backward: 2.2751 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 6343.6 | uss_MB: 5261.8 | child_rss_MB: 5331.5 | allocated_MB: 444.9 | reserved_MB: 28738.0 | inactive_split_MB: 1155.9 +rank0 [2026-10-29 02:11:15,772] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:16:10,940] INFO : Step 17900/50000 rank0 | Train Loss: 1.0765 | Train Loss Unweighted: 0.5646 | Train Loss Components: Auxiliary: 0.2108 | Cardinality: 0.1097 | Classification: 0.0932 | Classification_binary: 0.1585 | HitGrouping: 0.1686 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1536 | Regression_eta: 0.0016 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.1745 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0853 | Regression_eta: 0.0514 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0737 | LR: 7.44e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9240s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7593 | max_loss: 0.4918 | max_backward: 1.7987 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.6052 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6290.9 | uss_MB: 5640.2 | child_rss_MB: 5109.0 | allocated_MB: 447.1 | reserved_MB: 28738.0 | inactive_split_MB: 1154.4 +rank0 [2026-10-29 02:16:11,181] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 53.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:20:56,712] INFO : Step 18000/50000 rank0 | Train Loss: 1.2227 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2303 | Cardinality: 0.1450 | Classification: 0.1088 | Classification_binary: 0.2099 | HitGrouping: 0.1650 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1447 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2419 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0693 | Regression_eta: 0.0608 | Regression_pt: 0.0755 | Regression_sin_phi: 0.1007 | LR: 7.42e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.4761s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9293 | max_loss: 0.1567 | max_backward: 2.2476 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6283.0 | uss_MB: 5642.9 | child_rss_MB: 5063.5 | allocated_MB: 449.4 | reserved_MB: 28738.0 | inactive_split_MB: 1324.2 +rank0 [2026-10-29 02:20:56,964] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:20:56,989] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-18000.pth +rank0 [2026-10-29 02:20:57,299] INFO : Running validation on rank0 +rank0 [2026-10-29 02:20:57,299] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 02:20:57,299] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 02:21:58,041] INFO : VALIDATION | Step=18000/50000 | Train Loss=1.2227 | Valid Loss=1.2214 | Train Loss Unweighted=0.6143 | Valid Loss Unweighted=0.6037 | Stale=1 | ETA=1615.3m +rank0 [2026-10-29 02:21:58,041] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:21:58,041] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 02:21:58,041] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 02:21:58,041] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:21:58,041] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 02:21:58,041] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 02:21:58,041] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 02:21:58,041] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/ari=0.534259 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.588679 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/top1_competing=0.668089 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.63471 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.014894 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0602349 +rank0 [2026-10-29 02:21:58,042] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/seconds_per_rank=21.4407 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 02:21:58,043] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 02:21:58,044] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 02:21:58,044] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 02:21:58,044] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 02:21:58,044] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 02:21:58,044] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding/ari=0.269027 +rank0 [2026-10-29 02:21:58,044] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 02:21:58,044] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/embedding_minus_raw/ari=0.177404 +rank0 [2026-10-29 02:21:58,044] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 02:21:58,044] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 02:21:58,044] INFO : VALIDATION_PROBE | Step=18000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 02:21:58,049] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:21:58,301] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 02:21:58,311] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 02:22:25,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 02:22:26,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 02:22:27,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 02:22:28,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 02:22:38,897] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 02:22:38,913] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:22:39,417] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 02:22:39,421] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 02:22:55,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 02:22:56,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 02:22:57,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 02:22:58,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 02:23:08,983] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 02:23:08,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 02:23:09,542] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 02:23:09,546] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 02:23:25,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 02:23:26,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 02:23:27,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 02:23:28,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 02:23:38,411] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 02:23:38,427] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits +rank0 [2026-10-29 02:23:41,570] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits +rank0 [2026-10-29 02:23:42,618] INFO : Plotted number of elements +rank0 [2026-10-29 02:23:44,162] INFO : Plotted jets +rank0 [2026-10-29 02:23:45,300] INFO : Plotted jet ratio +rank0 [2026-10-29 02:23:45,380] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 02:23:48,567] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 02:23:49,566] INFO : Plotted number of elements +rank0 [2026-10-29 02:23:51,068] INFO : Plotted jets +rank0 [2026-10-29 02:23:52,217] INFO : Plotted jet ratio +rank0 [2026-10-29 02:23:52,270] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits +rank0 [2026-10-29 02:23:54,695] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits +rank0 [2026-10-29 02:23:55,527] INFO : Plotted number of elements +rank0 [2026-10-29 02:23:57,523] INFO : Plotted jets +rank0 [2026-10-29 02:23:58,674] INFO : Plotted jet ratio +rank0 [2026-10-29 02:28:43,685] INFO : Step 18100/50000 rank0 | Train Loss: 1.2922 | Train Loss Unweighted: 0.6330 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1763 | Classification: 0.1061 | Classification_binary: 0.2257 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1467 | Regression_eta: 0.0022 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2037 | Regression_cos_phi: 0.1233 | Regression_energy: 0.0714 | Regression_eta: 0.0788 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0876 | LR: 7.39e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.7480s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.9150 | max_loss: 0.1573 | max_backward: 2.1915 | max_optimizer_step: 0.2256 | max_tensorboard_logging: 0.8426 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7551.6 | uss_MB: 6907.4 | child_rss_MB: 5063.5 | allocated_MB: 448.8 | reserved_MB: 25396.0 | inactive_split_MB: 1192.4 +rank0 [2026-10-29 02:28:43,924] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:33:22,240] INFO : Step 18200/50000 rank0 | Train Loss: 1.1403 | Train Loss Unweighted: 0.5693 | Train Loss Components: Auxiliary: 0.2140 | Cardinality: 0.1362 | Classification: 0.1099 | Classification_binary: 0.1658 | HitGrouping: 0.1698 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1444 | Regression_eta: 0.0021 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0837 | Classification_binary: 0.1562 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0743 | Regression_eta: 0.0639 | Regression_pt: 0.0792 | Regression_sin_phi: 0.1037 | LR: 7.36e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.1789s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0059 | max_forward: 0.7843 | max_loss: 0.1427 | max_backward: 1.4948 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 1.0509 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6418.0 | uss_MB: 5773.9 | child_rss_MB: 5038.4 | allocated_MB: 446.0 | reserved_MB: 27340.0 | inactive_split_MB: 922.1 +rank0 [2026-10-29 02:33:22,504] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:37:54,267] INFO : Step 18300/50000 rank0 | Train Loss: 1.1938 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2339 | Cardinality: 0.1526 | Classification: 0.1133 | Classification_binary: 0.1878 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1493 | Regression_eta: 0.0019 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0689 | Classification_binary: 0.2338 | Regression_cos_phi: 0.0915 | Regression_energy: 0.0944 | Regression_eta: 0.0470 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0897 | LR: 7.34e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.5607s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.9058 | max_loss: 0.1527 | max_backward: 2.1689 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6422.1 | uss_MB: 5784.1 | child_rss_MB: 5032.3 | allocated_MB: 448.4 | reserved_MB: 27340.0 | inactive_split_MB: 1185.8 +rank0 [2026-10-29 02:37:54,523] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:42:25,236] INFO : Step 18400/50000 rank0 | Train Loss: 1.1959 | Train Loss Unweighted: 0.6109 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1130 | Classification: 0.1042 | Classification_binary: 0.2213 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1400 | Regression_eta: 0.0022 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.2498 | Regression_cos_phi: 0.0950 | Regression_energy: 0.0807 | Regression_eta: 0.0603 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0896 | LR: 7.31e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.5049s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8434 | max_loss: 0.1474 | max_backward: 1.9441 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6423.0 | uss_MB: 5783.9 | child_rss_MB: 5008.1 | allocated_MB: 448.8 | reserved_MB: 29306.0 | inactive_split_MB: 1091.4 +rank0 [2026-10-29 02:42:25,488] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:47:17,872] INFO : Step 18500/50000 rank0 | Train Loss: 1.2719 | Train Loss Unweighted: 0.6036 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1990 | Classification: 0.1087 | Classification_binary: 0.1793 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1538 | Regression_eta: 0.0024 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1309 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0938 | Regression_eta: 0.0668 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0698 | LR: 7.29e-04 | DataLoad Time: 0.0164s | Model Forward Time: 7.1028s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 1.6245 | max_loss: 0.3527 | max_backward: 3.4112 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.7932 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 6423.2 | uss_MB: 5781.7 | child_rss_MB: 5035.9 | allocated_MB: 447.9 | reserved_MB: 29306.0 | inactive_split_MB: 1193.4 +rank0 [2026-10-29 02:47:18,125] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 134.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:52:03,168] INFO : Step 18600/50000 rank0 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5476 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1351 | Classification: 0.0870 | Classification_binary: 0.1697 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1433 | Regression_eta: 0.0017 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0754 | Regression_eta: 0.0411 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0593 | LR: 7.26e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0216s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0527 | max_forward: 0.7639 | max_loss: 0.1373 | max_backward: 1.5273 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.7424 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6420.8 | uss_MB: 5779.1 | child_rss_MB: 5038.0 | allocated_MB: 445.6 | reserved_MB: 29306.0 | inactive_split_MB: 1194.4 +rank0 [2026-10-29 02:52:03,409] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 52.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 02:56:36,036] INFO : Step 18700/50000 rank0 | Train Loss: 1.2376 | Train Loss Unweighted: 0.5906 | Train Loss Components: Auxiliary: 0.2405 | Cardinality: 0.1841 | Classification: 0.1039 | Classification_binary: 0.2012 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1400 | Regression_eta: 0.0020 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.2211 | Regression_cos_phi: 0.0963 | Regression_energy: 0.0804 | Regression_eta: 0.0470 | Regression_pt: 0.0894 | Regression_sin_phi: 0.0706 | LR: 7.24e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.4523s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8903 | max_loss: 0.1469 | max_backward: 2.1433 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4822 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1128 | Memory: rss_MB: 6419.7 | uss_MB: 5784.3 | child_rss_MB: 5013.1 | allocated_MB: 449.2 | reserved_MB: 31324.0 | inactive_split_MB: 1093.4 +rank0 [2026-10-29 02:56:36,278] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 49.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:01:09,760] INFO : Step 18800/50000 rank0 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1585 | Classification: 0.0943 | Classification_binary: 0.2038 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1342 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.2173 | Regression_cos_phi: 0.0952 | Regression_energy: 0.0652 | Regression_eta: 0.0563 | Regression_pt: 0.0721 | Regression_sin_phi: 0.1185 | LR: 7.21e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1748s | Phase Times: max_pre_step_barrier: 0.1142 | max_batch_to_device: 0.0068 | max_forward: 0.7808 | max_loss: 0.2502 | max_backward: 1.8229 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0996 | Memory: rss_MB: 6421.2 | uss_MB: 5781.7 | child_rss_MB: 5036.2 | allocated_MB: 450.4 | reserved_MB: 31336.0 | inactive_split_MB: 1098.4 +rank0 [2026-10-29 03:01:10,001] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +4 8 0x7408, 13395 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:05:46,407] INFO : Step 18900/50000 rank0 | Train Loss: 1.0747 | Train Loss Unweighted: 0.5385 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.1306 | Classification: 0.0887 | Classification_binary: 0.1613 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1414 | Regression_eta: 0.0017 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0585 | Regression_eta: 0.0495 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0633 | LR: 7.18e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.0308s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7400 | max_loss: 0.1656 | max_backward: 1.8003 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.7596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6421.5 | uss_MB: 5781.2 | child_rss_MB: 5039.7 | allocated_MB: 443.5 | reserved_MB: 31336.0 | inactive_split_MB: 928.5 +rank0 [2026-10-29 03:05:46,646] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:10:21,739] INFO : Step 19000/50000 rank0 | Train Loss: 1.1726 | Train Loss Unweighted: 0.5966 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1418 | Classification: 0.1102 | Classification_binary: 0.1870 | HitGrouping: 0.1705 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1463 | Regression_eta: 0.0019 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.1972 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0840 | Regression_eta: 0.0503 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0703 | LR: 7.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.6732s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7637 | max_loss: 0.5203 | max_backward: 1.8720 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 1.1055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6422.5 | uss_MB: 5783.8 | child_rss_MB: 5041.8 | allocated_MB: 446.6 | reserved_MB: 31336.0 | inactive_split_MB: 1097.4 +rank0 [2026-10-29 03:10:21,981] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +4 8 0x7408, 13395 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:10:22,004] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-19000.pth +rank0 [2026-10-29 03:10:22,291] INFO : Running validation on rank0 +rank0 [2026-10-29 03:10:22,291] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 03:10:22,291] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 03:11:19,222] INFO : VALIDATION | Step=19000/50000 | Train Loss=1.1726 | Valid Loss=1.2094 | Train Loss Unweighted=0.5966 | Valid Loss Unweighted=0.6165 | Stale=2 | ETA=1563.0m +rank0 [2026-10-29 03:11:19,222] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 03:11:19,222] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 03:11:19,222] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 03:11:19,222] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/ari=0.54966 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/ari_threshold=0.79375 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.590017 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/top1_competing=0.667504 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.632584 +rank0 [2026-10-29 03:11:19,223] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.000507108 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0581095 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/seconds_per_rank=21.3165 +rank0 [2026-10-29 03:11:19,224] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding/ari=0.287483 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/embedding_minus_raw/ari=0.19586 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 03:11:19,225] INFO : VALIDATION_PROBE | Step=19000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 03:11:19,230] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:11:19,490] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 03:11:19,494] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 03:11:45,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 03:11:47,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 03:11:49,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 03:11:51,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 03:12:01,369] INFO : Time taken to make predictions on device 0 is: 0.30 min +rank0 [2026-10-29 03:12:01,383] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:12:01,927] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 03:12:01,931] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 03:12:18,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 03:12:19,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 03:12:20,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 03:12:21,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 03:12:31,435] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 03:12:31,455] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 03:12:31,972] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 03:12:31,975] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 03:12:48,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 03:12:49,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 03:12:50,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 03:12:51,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 03:13:01,027] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 03:13:01,043] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits +rank0 [2026-10-29 03:13:04,240] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits +rank0 [2026-10-29 03:13:05,308] INFO : Plotted number of elements +rank0 [2026-10-29 03:13:06,871] INFO : Plotted jets +rank0 [2026-10-29 03:13:08,432] INFO : Plotted jet ratio +rank0 [2026-10-29 03:13:08,520] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 03:13:11,777] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 03:13:12,862] INFO : Plotted number of elements +rank0 [2026-10-29 03:13:14,454] INFO : Plotted jets +rank0 [2026-10-29 03:13:15,666] INFO : Plotted jet ratio +rank0 [2026-10-29 03:13:15,717] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits +rank0 [2026-10-29 03:13:18,249] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits +rank0 [2026-10-29 03:13:19,143] INFO : Plotted number of elements +rank0 [2026-10-29 03:13:20,637] INFO : Plotted jets +rank0 [2026-10-29 03:13:21,764] INFO : Plotted jet ratio +rank0 [2026-10-29 03:17:58,907] INFO : Step 19100/50000 rank0 | Train Loss: 1.2150 | Train Loss Unweighted: 0.6107 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1439 | Classification: 0.1037 | Classification_binary: 0.2230 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1395 | Regression_eta: 0.0020 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2309 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0760 | Regression_eta: 0.0520 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0866 | LR: 7.13e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.4512s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8630 | max_loss: 0.1532 | max_backward: 1.9622 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.7474 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7697.6 | uss_MB: 7053.9 | child_rss_MB: 5049.6 | allocated_MB: 449.4 | reserved_MB: 23972.0 | inactive_split_MB: 1371.8 +rank0 [2026-10-29 03:17:59,162] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:22:36,043] INFO : Step 19200/50000 rank0 | Train Loss: 1.1075 | Train Loss Unweighted: 0.5303 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1382 | Classification: 0.0958 | Classification_binary: 0.1758 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1270 | Regression_eta: 0.0019 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1736 | Regression_cos_phi: 0.0942 | Regression_energy: 0.0759 | Regression_eta: 0.0558 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0878 | LR: 7.10e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.6353s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8287 | max_loss: 0.1556 | max_backward: 1.8127 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6504.0 | uss_MB: 5863.3 | child_rss_MB: 5048.7 | allocated_MB: 445.3 | reserved_MB: 27640.0 | inactive_split_MB: 940.7 +rank0 [2026-10-29 03:22:36,313] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:27:10,632] INFO : Step 19300/50000 rank0 | Train Loss: 1.0792 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2111 | Cardinality: 0.1254 | Classification: 0.0911 | Classification_binary: 0.1561 | HitGrouping: 0.1662 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.1733 | Regression_cos_phi: 0.1065 | Regression_energy: 0.0677 | Regression_eta: 0.0606 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0694 | LR: 7.08e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.1479s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8891 | max_loss: 0.1513 | max_backward: 1.7637 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6508.5 | uss_MB: 5869.2 | child_rss_MB: 5044.9 | allocated_MB: 445.1 | reserved_MB: 27640.0 | inactive_split_MB: 1366.9 +rank0 [2026-10-29 03:27:10,894] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:31:46,763] INFO : Step 19400/50000 rank0 | Train Loss: 1.2411 | Train Loss Unweighted: 0.6183 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1555 | Classification: 0.1055 | Classification_binary: 0.2145 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1463 | Regression_eta: 0.0020 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0811 | Regression_eta: 0.0416 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0634 | LR: 7.05e-04 | DataLoad Time: 0.0167s | Model Forward Time: 7.4798s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0092 | max_forward: 0.8079 | max_loss: 0.1550 | max_backward: 1.7710 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 2.8282 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6511.0 | uss_MB: 5872.2 | child_rss_MB: 5045.1 | allocated_MB: 452.5 | reserved_MB: 28010.0 | inactive_split_MB: 1195.9 +rank0 [2026-10-29 03:31:47,021] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 111.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 20% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +4 8 0x7408, 13395 51.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 48.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:36:22,108] INFO : Step 19500/50000 rank0 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6008 | Train Loss Components: Auxiliary: 0.2448 | Cardinality: 0.1836 | Classification: 0.1087 | Classification_binary: 0.2123 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1374 | Regression_eta: 0.0023 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0777 | Classification_binary: 0.1920 | Regression_cos_phi: 0.0842 | Regression_energy: 0.0625 | Regression_eta: 0.0455 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0799 | LR: 7.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2540s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7880 | max_loss: 0.1506 | max_backward: 1.7760 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.7620 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6510.7 | uss_MB: 5870.2 | child_rss_MB: 5041.2 | allocated_MB: 449.4 | reserved_MB: 29680.0 | inactive_split_MB: 1368.8 +rank0 [2026-10-29 03:36:22,360] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 48.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:40:55,363] INFO : Step 19600/50000 rank0 | Train Loss: 1.1451 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1464 | Classification: 0.0933 | Classification_binary: 0.1736 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1533 | Regression_eta: 0.0018 | Regression_pt: 0.1557 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0818 | Regression_eta: 0.0448 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0469 | LR: 7.00e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.0929s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8082 | max_loss: 0.1437 | max_backward: 1.6649 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.6567 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6510.9 | uss_MB: 5869.4 | child_rss_MB: 5045.0 | allocated_MB: 444.5 | reserved_MB: 29680.0 | inactive_split_MB: 1016.8 +rank0 [2026-10-29 03:40:55,608] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:45:30,927] INFO : Step 19700/50000 rank0 | Train Loss: 1.1424 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2236 | Cardinality: 0.1385 | Classification: 0.1093 | Classification_binary: 0.1797 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1379 | Regression_eta: 0.0020 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1893 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0773 | Regression_eta: 0.0639 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0628 | LR: 6.97e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.1037s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7612 | max_loss: 0.2297 | max_backward: 1.9668 | max_optimizer_step: 0.5229 | max_tensorboard_logging: 0.5756 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6508.5 | uss_MB: 5869.4 | child_rss_MB: 5047.4 | allocated_MB: 449.1 | reserved_MB: 29680.0 | inactive_split_MB: 1100.9 +rank0 [2026-10-29 03:45:31,166] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 49.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:50:09,540] INFO : Step 19800/50000 rank0 | Train Loss: 1.2806 | Train Loss Unweighted: 0.6390 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1432 | Classification: 0.1110 | Classification_binary: 0.2282 | HitGrouping: 0.1668 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0030 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2350 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0802 | Regression_eta: 0.0861 | Regression_pt: 0.0875 | Regression_sin_phi: 0.1273 | LR: 6.94e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2663s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8034 | max_loss: 0.1812 | max_backward: 1.9502 | max_optimizer_step: 0.2119 | max_tensorboard_logging: 0.5857 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6510.7 | uss_MB: 5873.6 | child_rss_MB: 5045.7 | allocated_MB: 452.3 | reserved_MB: 29680.0 | inactive_split_MB: 1110.9 +rank0 [2026-10-29 03:50:09,780] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:54:47,300] INFO : Step 19900/50000 rank0 | Train Loss: 1.0692 | Train Loss Unweighted: 0.5407 | Train Loss Components: Auxiliary: 0.2077 | Cardinality: 0.1171 | Classification: 0.0922 | Classification_binary: 0.1812 | HitGrouping: 0.1610 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1454 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0834 | Regression_eta: 0.0315 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0653 | LR: 6.92e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.2489s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7533 | max_loss: 0.2412 | max_backward: 2.0720 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5848 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6505.8 | uss_MB: 5870.0 | child_rss_MB: 5057.5 | allocated_MB: 444.7 | reserved_MB: 29682.0 | inactive_split_MB: 1000.0 +rank0 [2026-10-29 03:54:47,540] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:59:25,968] INFO : Step 20000/50000 rank0 | Train Loss: 1.0304 | Train Loss Unweighted: 0.5275 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1060 | Classification: 0.0940 | Classification_binary: 0.1604 | HitGrouping: 0.1656 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1344 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1842 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0825 | Regression_eta: 0.0532 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0879 | LR: 6.89e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.6366s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.8721 | max_loss: 0.1507 | max_backward: 2.1861 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6505.7 | uss_MB: 5870.7 | child_rss_MB: 5049.5 | allocated_MB: 445.9 | reserved_MB: 30966.0 | inactive_split_MB: 1179.2 +rank0 [2026-10-29 03:59:26,235] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 49% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 03:59:26,261] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-20000.pth +rank0 [2026-10-29 03:59:26,665] INFO : Running validation on rank0 +rank0 [2026-10-29 03:59:26,665] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 03:59:26,665] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 04:00:19,557] INFO : VALIDATION | Step=20000/50000 | Train Loss=1.0304 | Valid Loss=1.1349 | Train Loss Unweighted=0.5275 | Valid Loss Unweighted=0.5666 | Stale=0 | ETA=1510.5m +rank0 [2026-10-29 04:00:19,557] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:00:19,557] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 04:00:19,557] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 04:00:19,557] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:00:19,557] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/ari=0.563289 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.594839 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/top1_competing=0.673177 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.62922 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0141361 +rank0 [2026-10-29 04:00:19,558] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0547449 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/seconds_per_rank=21.5125 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 04:00:19,559] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:00:19,560] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 04:00:19,560] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 04:00:19,560] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:00:19,560] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding/ari=0.29442 +rank0 [2026-10-29 04:00:19,560] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 04:00:19,560] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/embedding_minus_raw/ari=0.202797 +rank0 [2026-10-29 04:00:19,560] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 04:00:19,560] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 04:00:19,560] INFO : VALIDATION_PROBE | Step=20000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 04:00:19,565] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:00:19,817] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 04:00:19,821] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 04:00:45,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 04:00:46,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 04:00:47,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 04:00:48,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 04:00:58,945] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 04:00:59,044] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:00:59,502] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 04:00:59,505] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:01:15,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 04:01:16,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 04:01:17,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 04:01:18,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 04:01:28,859] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 04:01:28,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:01:29,372] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 04:01:29,376] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 04:01:45,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 04:01:46,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 04:01:47,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 04:01:48,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 04:01:58,577] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 04:01:58,657] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits +rank0 [2026-10-29 04:02:01,902] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits +rank0 [2026-10-29 04:02:02,943] INFO : Plotted number of elements +rank0 [2026-10-29 04:02:04,503] INFO : Plotted jets +rank0 [2026-10-29 04:02:05,655] INFO : Plotted jet ratio +rank0 [2026-10-29 04:02:05,708] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:02:08,947] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:02:09,966] INFO : Plotted number of elements +rank0 [2026-10-29 04:02:11,498] INFO : Plotted jets +rank0 [2026-10-29 04:02:13,126] INFO : Plotted jet ratio +rank0 [2026-10-29 04:02:13,199] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits +rank0 [2026-10-29 04:02:15,601] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits +rank0 [2026-10-29 04:02:16,465] INFO : Plotted number of elements +rank0 [2026-10-29 04:02:17,989] INFO : Plotted jets +rank0 [2026-10-29 04:02:19,140] INFO : Plotted jet ratio +rank0 [2026-10-29 04:06:54,015] INFO : Step 20100/50000 rank0 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2437 | Cardinality: 0.1586 | Classification: 0.1055 | Classification_binary: 0.2238 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1360 | Regression_eta: 0.0021 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.2245 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0615 | Regression_eta: 0.0539 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0836 | LR: 6.86e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2300s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0372 | max_forward: 0.8293 | max_loss: 0.1584 | max_backward: 1.7738 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.7139 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7674.6 | uss_MB: 7034.7 | child_rss_MB: 5034.6 | allocated_MB: 448.4 | reserved_MB: 29328.0 | inactive_split_MB: 969.2 +rank0 [2026-10-29 04:06:54,269] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:11:26,643] INFO : Step 20200/50000 rank0 | Train Loss: 1.3163 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.1695 | Classification: 0.1000 | Classification_binary: 0.2250 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1433 | Regression_eta: 0.0029 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0658 | Regression_eta: 0.0568 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0776 | LR: 6.84e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.4487s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8823 | max_loss: 0.1595 | max_backward: 2.1030 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5136 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7674.6 | uss_MB: 7036.8 | child_rss_MB: 5027.8 | allocated_MB: 449.6 | reserved_MB: 29340.0 | inactive_split_MB: 938.7 +rank0 [2026-10-29 04:11:26,886] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:16:04,286] INFO : Step 20300/50000 rank0 | Train Loss: 1.1420 | Train Loss Unweighted: 0.5734 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1487 | Classification: 0.1087 | Classification_binary: 0.1713 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1438 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0839 | Regression_eta: 0.0485 | Regression_pt: 0.0901 | Regression_sin_phi: 0.0836 | LR: 6.81e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.5755s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.9650 | max_loss: 0.1904 | max_backward: 2.0801 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.8679 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6555.4 | uss_MB: 5913.2 | child_rss_MB: 5021.5 | allocated_MB: 444.8 | reserved_MB: 29344.0 | inactive_split_MB: 986.2 +rank0 [2026-10-29 04:16:04,547] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:20:45,707] INFO : Step 20400/50000 rank0 | Train Loss: 1.1926 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1642 | Classification: 0.1091 | Classification_binary: 0.1922 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1329 | Regression_eta: 0.0018 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0972 | Classification_binary: 0.2094 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0744 | Regression_eta: 0.0405 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0881 | LR: 6.78e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.9818s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0167 | max_forward: 1.2639 | max_loss: 0.1873 | max_backward: 2.4261 | max_optimizer_step: 0.3105 | max_tensorboard_logging: 0.7562 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6556.4 | uss_MB: 5916.7 | child_rss_MB: 5031.4 | allocated_MB: 447.7 | reserved_MB: 29344.0 | inactive_split_MB: 985.2 +rank0 [2026-10-29 04:20:45,948] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:25:27,722] INFO : Step 20500/50000 rank0 | Train Loss: 1.2110 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1591 | Classification: 0.1022 | Classification_binary: 0.2117 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1333 | Regression_eta: 0.0017 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.2106 | Regression_cos_phi: 0.1096 | Regression_energy: 0.0802 | Regression_eta: 0.0410 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0992 | LR: 6.76e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.1545s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7705 | max_loss: 0.1525 | max_backward: 1.8060 | max_optimizer_step: 0.2622 | max_tensorboard_logging: 0.7595 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6553.9 | uss_MB: 5915.2 | child_rss_MB: 5013.6 | allocated_MB: 450.6 | reserved_MB: 29344.0 | inactive_split_MB: 1123.4 +rank0 [2026-10-29 04:25:27,963] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:30:08,601] INFO : Step 20600/50000 rank0 | Train Loss: 1.1161 | Train Loss Unweighted: 0.5564 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1436 | Classification: 0.0978 | Classification_binary: 0.1795 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1580 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0779 | Regression_eta: 0.0671 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0799 | LR: 6.73e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.1713s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8358 | max_loss: 0.5148 | max_backward: 1.8393 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6556.9 | uss_MB: 5916.8 | child_rss_MB: 5024.7 | allocated_MB: 445.4 | reserved_MB: 31380.0 | inactive_split_MB: 917.0 +rank0 [2026-10-29 04:30:08,842] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 49.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:34:52,340] INFO : Step 20700/50000 rank0 | Train Loss: 1.0740 | Train Loss Unweighted: 0.5459 | Train Loss Components: Auxiliary: 0.2086 | Cardinality: 0.1252 | Classification: 0.0962 | Classification_binary: 0.1560 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0637 | Regression_eta: 0.0418 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0507 | LR: 6.70e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6830s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7111 | max_loss: 0.2396 | max_backward: 1.6829 | max_optimizer_step: 0.2121 | max_tensorboard_logging: 0.4996 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6553.0 | uss_MB: 5912.9 | child_rss_MB: 5017.8 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1185.7 +rank0 [2026-10-29 04:34:52,582] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 53.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 49.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:39:30,097] INFO : Step 20800/50000 rank0 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5426 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1263 | Classification: 0.0965 | Classification_binary: 0.1999 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1207 | Regression_eta: 0.0017 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2183 | Regression_cos_phi: 0.0882 | Regression_energy: 0.0645 | Regression_eta: 0.0590 | Regression_pt: 0.0715 | Regression_sin_phi: 0.1009 | LR: 6.67e-04 | DataLoad Time: 0.0148s | Model Forward Time: 5.5330s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.9079 | max_loss: 0.1550 | max_backward: 2.1083 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.5897 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6553.9 | uss_MB: 5918.9 | child_rss_MB: 5012.5 | allocated_MB: 448.2 | reserved_MB: 31380.0 | inactive_split_MB: 1157.8 +rank0 [2026-10-29 04:39:30,339] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:44:10,519] INFO : Step 20900/50000 rank0 | Train Loss: 1.2188 | Train Loss Unweighted: 0.5970 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1525 | Classification: 0.1062 | Classification_binary: 0.2022 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0022 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.2031 | Regression_cos_phi: 0.1183 | Regression_energy: 0.0759 | Regression_eta: 0.1112 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0791 | LR: 6.65e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.4312s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0138 | max_forward: 0.8922 | max_loss: 0.1535 | max_backward: 2.0483 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6550.7 | uss_MB: 5916.4 | child_rss_MB: 5027.7 | allocated_MB: 447.8 | reserved_MB: 31380.0 | inactive_split_MB: 1126.3 +rank0 [2026-10-29 04:44:10,774] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:48:49,041] INFO : Step 21000/50000 rank0 | Train Loss: 1.0503 | Train Loss Unweighted: 0.5338 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1267 | Classification: 0.0900 | Classification_binary: 0.1573 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1405 | Regression_eta: 0.0016 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0710 | Regression_eta: 0.0398 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0612 | LR: 6.62e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.4778s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7596 | max_loss: 0.2536 | max_backward: 1.8713 | max_optimizer_step: 0.3177 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6551.7 | uss_MB: 5915.9 | child_rss_MB: 5032.5 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1148.0 +rank0 [2026-10-29 04:48:49,283] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 53.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 49.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:48:49,304] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-21000.pth +rank0 [2026-10-29 04:48:49,612] INFO : Running validation on rank0 +rank0 [2026-10-29 04:48:49,612] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 04:48:49,612] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 04:49:44,981] INFO : VALIDATION | Step=21000/50000 | Train Loss=1.0503 | Valid Loss=1.1355 | Train Loss Unweighted=0.5338 | Valid Loss Unweighted=0.5619 | Stale=1 | ETA=1458.8m +rank0 [2026-10-29 04:49:44,981] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/ari=0.553475 +rank0 [2026-10-29 04:49:44,982] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.604609 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/top1_competing=0.687786 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.635576 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0043218 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0611008 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 04:49:44,983] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/seconds_per_rank=21.415 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding/ari=0.292134 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/embedding_minus_raw/ari=0.200511 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 04:49:44,984] INFO : VALIDATION_PROBE | Step=21000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 04:49:44,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:49:45,249] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 04:49:45,254] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 04:50:11,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 04:50:12,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 04:50:13,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 04:50:15,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 04:50:25,129] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 04:50:25,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:50:25,767] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 04:50:25,771] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:50:41,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 04:50:42,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 04:50:43,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 04:50:45,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 04:50:55,022] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 04:50:55,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 04:50:55,775] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 04:50:55,779] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 04:51:11,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 04:51:12,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 04:51:13,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 04:51:14,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 04:51:24,439] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 04:51:25,580] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits +rank0 [2026-10-29 04:51:28,818] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits +rank0 [2026-10-29 04:51:29,868] INFO : Plotted number of elements +rank0 [2026-10-29 04:51:31,860] INFO : Plotted jets +rank0 [2026-10-29 04:51:33,012] INFO : Plotted jet ratio +rank0 [2026-10-29 04:51:33,070] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:51:36,320] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 04:51:37,354] INFO : Plotted number of elements +rank0 [2026-10-29 04:51:38,891] INFO : Plotted jets +rank0 [2026-10-29 04:51:39,976] INFO : Plotted jet ratio +rank0 [2026-10-29 04:51:40,025] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits +rank0 [2026-10-29 04:51:42,498] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits +rank0 [2026-10-29 04:51:43,362] INFO : Plotted number of elements +rank0 [2026-10-29 04:51:44,832] INFO : Plotted jets +rank0 [2026-10-29 04:51:45,942] INFO : Plotted jet ratio +rank0 [2026-10-29 04:56:26,511] INFO : Step 21100/50000 rank0 | Train Loss: 1.1646 | Train Loss Unweighted: 0.5785 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1401 | Classification: 0.1095 | Classification_binary: 0.1927 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1344 | Regression_eta: 0.0022 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.2180 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0674 | Regression_eta: 0.0645 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0686 | LR: 6.59e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.1093s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.9258 | max_loss: 0.1506 | max_backward: 2.4347 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.8145 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7847.0 | uss_MB: 5890.3 | child_rss_MB: 5018.3 | allocated_MB: 448.7 | reserved_MB: 23438.0 | inactive_split_MB: 911.7 +rank0 [2026-10-29 04:56:26,753] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +2 6 0x7408, 15961 53.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 80% +4 8 0x7408, 13395 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 04:56:59,983] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-29 04:56:59,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-29 05:01:23,516] INFO : Step 21200/50000 rank0 | Train Loss: 1.1437 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1413 | Classification: 0.0985 | Classification_binary: 0.1768 | HitGrouping: 0.1669 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0020 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1721 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0685 | Regression_eta: 0.0566 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0788 | LR: 6.56e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.4675s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7993 | max_loss: 0.1450 | max_backward: 1.7427 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7842.9 | uss_MB: 7208.8 | child_rss_MB: 5012.9 | allocated_MB: 446.9 | reserved_MB: 23590.0 | inactive_split_MB: 1086.9 +rank0 [2026-10-29 05:01:23,756] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:06:02,899] INFO : Step 21300/50000 rank0 | Train Loss: 1.1586 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1648 | Classification: 0.1118 | Classification_binary: 0.1760 | HitGrouping: 0.1681 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1392 | Regression_eta: 0.0021 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.2043 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0708 | Regression_eta: 0.0370 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0700 | LR: 6.54e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.9343s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0558 | max_forward: 0.9498 | max_loss: 0.1699 | max_backward: 2.4288 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.7788 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1270 | Memory: rss_MB: 6605.4 | uss_MB: 5978.0 | child_rss_MB: 5012.8 | allocated_MB: 447.4 | reserved_MB: 25440.0 | inactive_split_MB: 1003.9 +rank0 [2026-10-29 05:06:03,157] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:10:42,563] INFO : Step 21400/50000 rank0 | Train Loss: 1.1843 | Train Loss Unweighted: 0.5823 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1516 | Classification: 0.1038 | Classification_binary: 0.2065 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1335 | Regression_eta: 0.0020 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.2150 | Regression_cos_phi: 0.1204 | Regression_energy: 0.0678 | Regression_eta: 0.0632 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0763 | LR: 6.51e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.9213s | Phase Times: max_pre_step_barrier: 0.0085 | max_batch_to_device: 0.0640 | max_forward: 0.9920 | max_loss: 0.5036 | max_backward: 2.0711 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.8200 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6606.3 | uss_MB: 5978.8 | child_rss_MB: 5018.5 | allocated_MB: 448.9 | reserved_MB: 25440.0 | inactive_split_MB: 994.4 +rank0 [2026-10-29 05:10:42,830] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 53.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:15:15,846] INFO : Step 21500/50000 rank0 | Train Loss: 1.1484 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1478 | Classification: 0.0953 | Classification_binary: 0.2062 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1299 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.2270 | Regression_cos_phi: 0.0890 | Regression_energy: 0.0773 | Regression_eta: 0.0572 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0849 | LR: 6.48e-04 | DataLoad Time: 0.0205s | Model Forward Time: 6.6913s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8371 | max_loss: 0.1537 | max_backward: 2.1150 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.8633 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6603.8 | uss_MB: 5979.6 | child_rss_MB: 5017.0 | allocated_MB: 447.2 | reserved_MB: 27056.0 | inactive_split_MB: 1468.9 +rank0 [2026-10-29 05:15:16,107] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 52.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 1% +6 10 0x7408, 62036 48.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:19:52,514] INFO : Step 21600/50000 rank0 | Train Loss: 1.0171 | Train Loss Unweighted: 0.5098 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1167 | Classification: 0.0973 | Classification_binary: 0.1559 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1265 | Regression_eta: 0.0014 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1570 | Regression_cos_phi: 0.1015 | Regression_energy: 0.0759 | Regression_eta: 0.0396 | Regression_pt: 0.0826 | Regression_sin_phi: 0.1470 | LR: 6.45e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.3661s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0062 | max_forward: 0.7285 | max_loss: 0.1940 | max_backward: 1.8148 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.8819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6602.2 | uss_MB: 5978.2 | child_rss_MB: 5014.0 | allocated_MB: 445.7 | reserved_MB: 27056.0 | inactive_split_MB: 1196.5 +rank0 [2026-10-29 05:19:52,782] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 53.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 49.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:24:31,204] INFO : Step 21700/50000 rank0 | Train Loss: 1.1623 | Train Loss Unweighted: 0.5535 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1498 | Classification: 0.1025 | Classification_binary: 0.1819 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2151 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0720 | Regression_eta: 0.0481 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0880 | LR: 6.43e-04 | DataLoad Time: 0.0145s | Model Forward Time: 5.2224s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7947 | max_loss: 0.1379 | max_backward: 1.9808 | max_optimizer_step: 0.2157 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6600.2 | uss_MB: 5978.2 | child_rss_MB: 5010.8 | allocated_MB: 448.2 | reserved_MB: 27056.0 | inactive_split_MB: 1045.2 +rank0 [2026-10-29 05:24:31,443] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:29:09,286] INFO : Step 21800/50000 rank0 | Train Loss: 1.0898 | Train Loss Unweighted: 0.5518 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1233 | Classification: 0.0977 | Classification_binary: 0.1967 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1269 | Regression_eta: 0.0015 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0653 | Regression_eta: 0.0465 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0689 | LR: 6.40e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.5389s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8952 | max_loss: 0.1765 | max_backward: 2.1628 | max_optimizer_step: 0.2138 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6602.0 | uss_MB: 5872.4 | child_rss_MB: 5012.0 | allocated_MB: 448.4 | reserved_MB: 27056.0 | inactive_split_MB: 1001.4 +rank0 [2026-10-29 05:29:09,528] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 88% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 83% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:33:47,139] INFO : Step 21900/50000 rank0 | Train Loss: 1.0699 | Train Loss Unweighted: 0.5306 | Train Loss Components: Auxiliary: 0.2121 | Cardinality: 0.1436 | Classification: 0.0991 | Classification_binary: 0.1704 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1276 | Regression_eta: 0.0015 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0932 | Classification_binary: 0.1729 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0649 | Regression_eta: 0.0414 | Regression_pt: 0.0710 | Regression_sin_phi: 0.1033 | LR: 6.37e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1101s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7813 | max_loss: 0.1495 | max_backward: 1.8703 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 6600.0 | uss_MB: 5976.9 | child_rss_MB: 5013.0 | allocated_MB: 447.4 | reserved_MB: 27056.0 | inactive_split_MB: 1002.4 +rank0 [2026-10-29 05:33:47,394] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 53.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 49.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:38:24,589] INFO : Step 22000/50000 rank0 | Train Loss: 1.0659 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2100 | Cardinality: 0.1294 | Classification: 0.0955 | Classification_binary: 0.1521 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1360 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1821 | Regression_cos_phi: 0.1044 | Regression_energy: 0.0682 | Regression_eta: 0.0563 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0870 | LR: 6.34e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1044s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7855 | max_loss: 0.1383 | max_backward: 1.9648 | max_optimizer_step: 0.2132 | max_tensorboard_logging: 0.5884 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6605.4 | uss_MB: 5979.9 | child_rss_MB: 5015.2 | allocated_MB: 449.1 | reserved_MB: 27056.0 | inactive_split_MB: 1180.9 +rank0 [2026-10-29 05:38:24,833] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 49.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:38:24,855] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-22000.pth +rank0 [2026-10-29 05:38:25,241] INFO : Running validation on rank0 +rank0 [2026-10-29 05:38:25,241] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 05:38:25,241] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 05:39:17,356] INFO : VALIDATION | Step=22000/50000 | Train Loss=1.0659 | Valid Loss=1.2746 | Train Loss Unweighted=0.5232 | Valid Loss Unweighted=0.6347 | Stale=2 | ETA=1407.6m +rank0 [2026-10-29 05:39:17,356] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 05:39:17,356] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 05:39:17,356] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 05:39:17,356] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 05:39:17,356] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 05:39:17,356] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 05:39:17,356] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/ari=0.545235 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.590531 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/top1_competing=0.670257 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.617649 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding_minus_raw/ari=-0.00391733 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0431738 +rank0 [2026-10-29 05:39:17,357] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/seconds_per_rank=21.3304 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 05:39:17,358] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 05:39:17,359] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 05:39:17,359] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 05:39:17,359] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding/ari=0.268764 +rank0 [2026-10-29 05:39:17,359] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 05:39:17,359] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/embedding_minus_raw/ari=0.177141 +rank0 [2026-10-29 05:39:17,359] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 05:39:17,359] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 05:39:17,359] INFO : VALIDATION_PROBE | Step=22000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 05:39:17,364] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:39:17,619] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 05:39:17,624] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 05:39:42,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 05:39:44,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 05:39:45,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 05:39:46,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 05:39:56,577] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 05:39:56,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:39:57,074] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 05:39:57,078] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 05:40:13,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 05:40:14,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 05:40:16,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 05:40:17,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 05:40:27,466] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 05:40:27,478] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 05:40:27,967] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 05:40:27,970] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 05:40:43,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 05:40:44,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 05:40:45,576] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 05:40:46,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 05:40:56,569] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 05:40:56,589] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits +rank0 [2026-10-29 05:41:00,324] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits +rank0 [2026-10-29 05:41:01,452] INFO : Plotted number of elements +rank0 [2026-10-29 05:41:03,078] INFO : Plotted jets +rank0 [2026-10-29 05:41:04,208] INFO : Plotted jet ratio +rank0 [2026-10-29 05:41:04,263] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 05:41:07,482] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 05:41:08,515] INFO : Plotted number of elements +rank0 [2026-10-29 05:41:10,028] INFO : Plotted jets +rank0 [2026-10-29 05:41:11,184] INFO : Plotted jet ratio +rank0 [2026-10-29 05:41:11,241] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits +rank0 [2026-10-29 05:41:13,693] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits +rank0 [2026-10-29 05:41:14,558] INFO : Plotted number of elements +rank0 [2026-10-29 05:41:16,040] INFO : Plotted jets +rank0 [2026-10-29 05:41:17,699] INFO : Plotted jet ratio +rank0 [2026-10-29 05:45:57,344] INFO : Step 22100/50000 rank0 | Train Loss: 1.1235 | Train Loss Unweighted: 0.5530 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1528 | Classification: 0.0997 | Classification_binary: 0.1905 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1304 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.2114 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0732 | Regression_eta: 0.0423 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0825 | LR: 6.32e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.1799s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8135 | max_loss: 0.1488 | max_backward: 1.7877 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5979 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7900.1 | uss_MB: 7271.0 | child_rss_MB: 5019.5 | allocated_MB: 451.8 | reserved_MB: 29136.0 | inactive_split_MB: 1136.2 +rank0 [2026-10-29 05:45:57,584] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 53.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +4 8 0x7408, 13395 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:50:46,424] INFO : Step 22200/50000 rank0 | Train Loss: 1.1922 | Train Loss Unweighted: 0.5684 | Train Loss Components: Auxiliary: 0.2361 | Cardinality: 0.1699 | Classification: 0.0906 | Classification_binary: 0.2085 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0922 | Regression_energy: 0.0541 | Regression_eta: 0.0637 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0732 | LR: 6.29e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.5566s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8616 | max_loss: 0.1545 | max_backward: 1.6969 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 0.7120 | max_tensorboard_flush: 0.0150 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7899.1 | uss_MB: 7269.8 | child_rss_MB: 5025.9 | allocated_MB: 448.1 | reserved_MB: 29136.0 | inactive_split_MB: 1371.0 +rank0 [2026-10-29 05:50:46,682] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 05:55:22,580] INFO : Step 22300/50000 rank0 | Train Loss: 1.0221 | Train Loss Unweighted: 0.5006 | Train Loss Components: Auxiliary: 0.1973 | Cardinality: 0.1169 | Classification: 0.0876 | Classification_binary: 0.1576 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1252 | Regression_eta: 0.0021 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1539 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0642 | Regression_eta: 0.0864 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0491 | LR: 6.26e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1425s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7903 | max_loss: 0.1446 | max_backward: 1.7078 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 0.8083 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6690.1 | uss_MB: 6062.0 | child_rss_MB: 5023.7 | allocated_MB: 446.4 | reserved_MB: 29136.0 | inactive_split_MB: 1159.3 +rank0 [2026-10-29 05:55:22,828] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 31% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +2 6 0x7408, 15961 53.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:00:09,243] INFO : Step 22400/50000 rank0 | Train Loss: 1.1170 | Train Loss Unweighted: 0.5609 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1391 | Classification: 0.1065 | Classification_binary: 0.1807 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1348 | Regression_eta: 0.0019 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0780 | Classification_binary: 0.1867 | Regression_cos_phi: 0.0970 | Regression_energy: 0.0717 | Regression_eta: 0.0424 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0687 | LR: 6.23e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.7270s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7658 | max_loss: 0.2479 | max_backward: 1.9808 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9928 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6691.0 | uss_MB: 6065.0 | child_rss_MB: 5027.0 | allocated_MB: 447.1 | reserved_MB: 31152.0 | inactive_split_MB: 925.7 +rank0 [2026-10-29 06:00:09,501] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 53.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:04:42,207] INFO : Step 22500/50000 rank0 | Train Loss: 1.1245 | Train Loss Unweighted: 0.5482 | Train Loss Components: Auxiliary: 0.2231 | Cardinality: 0.1463 | Classification: 0.0856 | Classification_binary: 0.2083 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1244 | Regression_eta: 0.0016 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.2158 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0687 | Regression_eta: 0.0409 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0744 | LR: 6.20e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.6566s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 1.1916 | max_loss: 0.1613 | max_backward: 2.1225 | max_optimizer_step: 0.2176 | max_tensorboard_logging: 0.7664 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6685.2 | uss_MB: 6063.6 | child_rss_MB: 5003.5 | allocated_MB: 447.4 | reserved_MB: 31152.0 | inactive_split_MB: 959.7 +rank0 [2026-10-29 06:04:42,452] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 49.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:09:14,357] INFO : Step 22600/50000 rank0 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5177 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1260 | Classification: 0.0928 | Classification_binary: 0.1789 | HitGrouping: 0.1619 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1208 | Regression_eta: 0.0019 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1697 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0587 | Regression_eta: 0.0433 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0722 | LR: 6.18e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.6082s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0651 | max_forward: 1.1064 | max_loss: 0.1531 | max_backward: 1.9236 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.8515 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6676.3 | uss_MB: 6063.0 | child_rss_MB: 5024.2 | allocated_MB: 448.3 | reserved_MB: 31154.0 | inactive_split_MB: 961.5 +rank0 [2026-10-29 06:09:14,601] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:13:52,419] INFO : Step 22700/50000 rank0 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5090 | Train Loss Components: Auxiliary: 0.1959 | Cardinality: 0.1101 | Classification: 0.0877 | Classification_binary: 0.1479 | HitGrouping: 0.1634 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1339 | Regression_eta: 0.0015 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0852 | Regression_eta: 0.0607 | Regression_pt: 0.0928 | Regression_sin_phi: 0.1177 | LR: 6.15e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.1581s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 1.0743 | max_loss: 0.1383 | max_backward: 1.7235 | max_optimizer_step: 0.2204 | max_tensorboard_logging: 0.5170 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 6672.9 | uss_MB: 6061.8 | child_rss_MB: 5030.8 | allocated_MB: 446.5 | reserved_MB: 31154.0 | inactive_split_MB: 972.7 +rank0 [2026-10-29 06:13:52,666] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:18:33,349] INFO : Step 22800/50000 rank0 | Train Loss: 1.0730 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2215 | Cardinality: 0.1332 | Classification: 0.1095 | Classification_binary: 0.1782 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1289 | Regression_eta: 0.0014 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1884 | Regression_cos_phi: 0.0794 | Regression_energy: 0.0761 | Regression_eta: 0.0402 | Regression_pt: 0.0826 | Regression_sin_phi: 0.0906 | LR: 6.12e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.5174s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.2500 | max_backward: 2.0107 | max_optimizer_step: 0.2761 | max_tensorboard_logging: 0.6168 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 6673.9 | uss_MB: 6062.6 | child_rss_MB: 5029.6 | allocated_MB: 447.2 | reserved_MB: 31154.0 | inactive_split_MB: 922.1 +rank0 [2026-10-29 06:18:33,594] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:23:13,895] INFO : Step 22900/50000 rank0 | Train Loss: 1.1830 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2389 | Cardinality: 0.1788 | Classification: 0.0948 | Classification_binary: 0.2023 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1393 | Regression_eta: 0.0018 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0751 | Classification_binary: 0.1875 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0654 | Regression_eta: 0.0448 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0927 | LR: 6.09e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.6836s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0073 | max_forward: 0.7992 | max_loss: 0.1478 | max_backward: 1.9051 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.9858 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 6664.2 | uss_MB: 6055.9 | child_rss_MB: 5028.8 | allocated_MB: 448.8 | reserved_MB: 31154.0 | inactive_split_MB: 1109.2 +rank0 [2026-10-29 06:23:14,139] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 2% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 31% +2 6 0x7408, 15961 53.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 56% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 21% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:27:52,502] INFO : Step 23000/50000 rank0 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5014 | Train Loss Components: Auxiliary: 0.1951 | Cardinality: 0.1184 | Classification: 0.0781 | Classification_binary: 0.1455 | HitGrouping: 0.1653 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0760 | Regression_eta: 0.0472 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0810 | LR: 6.06e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.2412s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7528 | max_loss: 0.2292 | max_backward: 1.7818 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.8884 | max_tensorboard_flush: 0.0060 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6672.0 | uss_MB: 6063.2 | child_rss_MB: 5028.5 | allocated_MB: 443.9 | reserved_MB: 31154.0 | inactive_split_MB: 985.1 +rank0 [2026-10-29 06:27:52,748] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 1% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 49.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:27:52,771] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-23000.pth +rank0 [2026-10-29 06:27:53,240] INFO : Running validation on rank0 +rank0 [2026-10-29 06:27:53,241] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 06:27:53,241] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 06:28:45,912] INFO : VALIDATION | Step=23000/50000 | Train Loss=0.9959 | Valid Loss=1.0941 | Train Loss Unweighted=0.5014 | Valid Loss Unweighted=0.5398 | Stale=0 | ETA=1356.4m +rank0 [2026-10-29 06:28:45,912] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 06:28:45,912] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 06:28:45,912] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 06:28:45,912] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 06:28:45,912] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 06:28:45,912] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 06:28:45,912] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/ari=0.563599 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.649787 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/top1_competing=0.744839 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.636017 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0144465 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0615421 +rank0 [2026-10-29 06:28:45,913] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/seconds_per_rank=21.5631 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 06:28:45,914] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 06:28:45,915] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 06:28:45,915] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 06:28:45,915] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 06:28:45,915] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding/ari=0.269434 +rank0 [2026-10-29 06:28:45,915] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 06:28:45,915] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/embedding_minus_raw/ari=0.177811 +rank0 [2026-10-29 06:28:45,915] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 06:28:45,915] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 06:28:45,915] INFO : VALIDATION_PROBE | Step=23000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 06:28:45,920] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:28:46,180] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 06:28:46,184] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 06:29:12,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 06:29:13,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 06:29:14,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 06:29:16,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 06:29:26,149] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 06:29:26,162] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:29:26,632] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 06:29:26,636] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 06:29:42,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 06:29:43,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 06:29:45,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 06:29:46,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 06:29:56,153] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 06:29:56,164] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 06:29:56,644] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 06:29:56,647] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 06:30:12,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 06:30:13,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 06:30:14,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 06:30:15,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 06:30:25,680] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 06:30:25,785] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits +rank0 [2026-10-29 06:30:29,490] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits +rank0 [2026-10-29 06:30:30,552] INFO : Plotted number of elements +rank0 [2026-10-29 06:30:32,096] INFO : Plotted jets +rank0 [2026-10-29 06:30:33,243] INFO : Plotted jet ratio +rank0 [2026-10-29 06:30:33,317] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 06:30:36,545] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 06:30:37,570] INFO : Plotted number of elements +rank0 [2026-10-29 06:30:39,063] INFO : Plotted jets +rank0 [2026-10-29 06:30:40,161] INFO : Plotted jet ratio +rank0 [2026-10-29 06:30:40,211] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits +rank0 [2026-10-29 06:30:42,717] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits +rank0 [2026-10-29 06:30:43,596] INFO : Plotted number of elements +rank0 [2026-10-29 06:30:45,105] INFO : Plotted jets +rank0 [2026-10-29 06:30:46,813] INFO : Plotted jet ratio +rank0 [2026-10-29 06:35:30,140] INFO : Step 23100/50000 rank0 | Train Loss: 1.1240 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1320 | Classification: 0.1122 | Classification_binary: 0.1745 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1274 | Regression_eta: 0.0021 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1680 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0670 | Regression_eta: 0.0711 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0551 | LR: 6.04e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.4759s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7914 | max_loss: 0.1430 | max_backward: 1.7255 | max_optimizer_step: 0.3295 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0695 | Memory: rss_MB: 7843.1 | uss_MB: 7233.5 | child_rss_MB: 5030.9 | allocated_MB: 449.4 | reserved_MB: 25680.0 | inactive_split_MB: 1046.0 +rank0 [2026-10-29 06:35:30,393] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 53.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:40:12,824] INFO : Step 23200/50000 rank0 | Train Loss: 1.1736 | Train Loss Unweighted: 0.5745 | Train Loss Components: Auxiliary: 0.2341 | Cardinality: 0.1586 | Classification: 0.1057 | Classification_binary: 0.2033 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1304 | Regression_eta: 0.0018 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2051 | Regression_cos_phi: 0.0995 | Regression_energy: 0.0614 | Regression_eta: 0.0767 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0549 | LR: 6.01e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.9813s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9381 | max_loss: 0.1692 | max_backward: 2.2545 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6161 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 7846.1 | uss_MB: 5861.1 | child_rss_MB: 5025.2 | allocated_MB: 447.8 | reserved_MB: 25680.0 | inactive_split_MB: 1169.7 +rank0 [2026-10-29 06:40:13,068] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 53.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:44:53,472] INFO : Step 23300/50000 rank0 | Train Loss: 1.0647 | Train Loss Unweighted: 0.5262 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1268 | Classification: 0.0958 | Classification_binary: 0.1665 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1301 | Regression_eta: 0.0017 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1527 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0604 | Regression_eta: 0.0417 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0600 | LR: 5.98e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2539s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8613 | max_loss: 0.1501 | max_backward: 2.0035 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 0.6274 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6729.0 | uss_MB: 6118.5 | child_rss_MB: 5031.7 | allocated_MB: 447.3 | reserved_MB: 29702.0 | inactive_split_MB: 1252.4 +rank0 [2026-10-29 06:44:53,715] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:49:33,497] INFO : Step 23400/50000 rank0 | Train Loss: 0.9916 | Train Loss Unweighted: 0.5023 | Train Loss Components: Auxiliary: 0.1957 | Cardinality: 0.1164 | Classification: 0.1006 | Classification_binary: 0.1374 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0697 | Regression_eta: 0.0449 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0628 | LR: 5.95e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.2711s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7229 | max_loss: 0.2412 | max_backward: 1.6798 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.4992 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1172 | Memory: rss_MB: 6727.8 | uss_MB: 6117.0 | child_rss_MB: 5031.9 | allocated_MB: 445.7 | reserved_MB: 29702.0 | inactive_split_MB: 1158.3 +rank0 [2026-10-29 06:49:33,738] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 2% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 49.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:54:43,048] INFO : Step 23500/50000 rank0 | Train Loss: 1.0349 | Train Loss Unweighted: 0.5234 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1083 | Classification: 0.0851 | Classification_binary: 0.1870 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1235 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0766 | Regression_eta: 0.0314 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0497 | LR: 5.92e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.4680s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7924 | max_loss: 0.2020 | max_backward: 1.9787 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.7987 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6728.4 | uss_MB: 6111.2 | child_rss_MB: 5026.6 | allocated_MB: 446.9 | reserved_MB: 31774.0 | inactive_split_MB: 1042.0 +rank0 [2026-10-29 06:54:43,295] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 06:59:39,070] INFO : Step 23600/50000 rank0 | Train Loss: 1.1326 | Train Loss Unweighted: 0.5418 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1458 | Classification: 0.0933 | Classification_binary: 0.1900 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1267 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1752 | Regression_cos_phi: 0.0856 | Regression_energy: 0.0808 | Regression_eta: 0.0457 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0804 | LR: 5.90e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.7737s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0063 | max_forward: 0.7946 | max_loss: 0.1855 | max_backward: 1.9114 | max_optimizer_step: 0.2664 | max_tensorboard_logging: 0.9547 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6730.1 | uss_MB: 6120.3 | child_rss_MB: 5028.3 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1333.8 +rank0 [2026-10-29 06:59:39,315] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 3% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 95% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 47.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 66% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:04:20,715] INFO : Step 23700/50000 rank0 | Train Loss: 1.0167 | Train Loss Unweighted: 0.4968 | Train Loss Components: Auxiliary: 0.1932 | Cardinality: 0.1158 | Classification: 0.0853 | Classification_binary: 0.1475 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1285 | Regression_eta: 0.0017 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1216 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0744 | Regression_eta: 0.0407 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0743 | LR: 5.87e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.3658s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0221 | max_forward: 0.7952 | max_loss: 0.1461 | max_backward: 1.8308 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7718 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6725.6 | uss_MB: 6118.0 | child_rss_MB: 5028.2 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1160.4 +rank0 [2026-10-29 07:04:20,957] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:08:58,595] INFO : Step 23800/50000 rank0 | Train Loss: 1.0651 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1325 | Classification: 0.0886 | Classification_binary: 0.1616 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1318 | Regression_eta: 0.0015 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0707 | Regression_eta: 0.0408 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0494 | LR: 5.84e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.3005s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8562 | max_loss: 0.1426 | max_backward: 1.8700 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6730.2 | uss_MB: 6120.2 | child_rss_MB: 5026.7 | allocated_MB: 449.4 | reserved_MB: 31776.0 | inactive_split_MB: 1046.7 +rank0 [2026-10-29 07:08:58,853] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 47.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:13:42,226] INFO : Step 23900/50000 rank0 | Train Loss: 1.1515 | Train Loss Unweighted: 0.5718 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1646 | Classification: 0.0962 | Classification_binary: 0.2086 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0691 | Regression_eta: 0.0382 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0518 | LR: 5.81e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.4678s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.8887 | max_loss: 0.2019 | max_backward: 2.1185 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6224 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6714.4 | uss_MB: 6108.3 | child_rss_MB: 5032.9 | allocated_MB: 447.1 | reserved_MB: 31776.0 | inactive_split_MB: 1168.8 +rank0 [2026-10-29 07:13:42,475] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 51% 1% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 90% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:18:21,375] INFO : Step 24000/50000 rank0 | Train Loss: 1.0174 | Train Loss Unweighted: 0.5187 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1133 | Classification: 0.0873 | Classification_binary: 0.1674 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0014 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0726 | Regression_eta: 0.0363 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0571 | LR: 5.78e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.2942s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 1.3237 | max_loss: 0.2403 | max_backward: 1.8319 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5068 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6721.2 | uss_MB: 6114.9 | child_rss_MB: 5029.4 | allocated_MB: 446.4 | reserved_MB: 31776.0 | inactive_split_MB: 957.1 +rank0 [2026-10-29 07:18:21,622] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 53.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:18:21,645] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-24000.pth +rank0 [2026-10-29 07:18:21,962] INFO : Running validation on rank0 +rank0 [2026-10-29 07:18:21,962] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 07:18:21,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 07:19:14,164] INFO : VALIDATION | Step=24000/50000 | Train Loss=1.0174 | Valid Loss=1.0877 | Train Loss Unweighted=0.5187 | Valid Loss Unweighted=0.5353 | Stale=0 | ETA=1306.4m +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 07:19:14,165] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/ari=0.570391 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.638406 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/top1_competing=0.73094 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.639398 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0212379 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0649232 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 07:19:14,166] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/seconds_per_rank=21.6361 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding/ari=0.28122 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/embedding_minus_raw/ari=0.189597 +rank0 [2026-10-29 07:19:14,167] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 07:19:14,168] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 07:19:14,168] INFO : VALIDATION_PROBE | Step=24000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 07:19:14,174] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:19:14,440] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 07:19:14,444] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 07:19:41,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 07:19:42,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 07:19:43,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 07:19:45,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 07:19:55,107] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 07:19:55,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:19:55,706] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 07:19:55,710] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 07:20:11,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 07:20:12,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 07:20:13,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 07:20:14,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 07:20:25,010] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 07:20:25,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 07:20:25,604] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 07:20:25,606] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 07:20:41,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 07:20:42,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 07:20:43,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 07:20:44,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 07:20:54,697] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 07:20:54,830] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits +rank0 [2026-10-29 07:20:58,132] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits +rank0 [2026-10-29 07:20:59,254] INFO : Plotted number of elements +rank0 [2026-10-29 07:21:00,908] INFO : Plotted jets +rank0 [2026-10-29 07:21:02,133] INFO : Plotted jet ratio +rank0 [2026-10-29 07:21:02,183] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 07:21:05,449] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 07:21:06,462] INFO : Plotted number of elements +rank0 [2026-10-29 07:21:08,426] INFO : Plotted jets +rank0 [2026-10-29 07:21:09,511] INFO : Plotted jet ratio +rank0 [2026-10-29 07:21:09,562] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits +rank0 [2026-10-29 07:21:11,948] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits +rank0 [2026-10-29 07:21:12,785] INFO : Plotted number of elements +rank0 [2026-10-29 07:21:14,276] INFO : Plotted jets +rank0 [2026-10-29 07:21:15,401] INFO : Plotted jet ratio +rank0 [2026-10-29 07:26:07,255] INFO : Step 24100/50000 rank0 | Train Loss: 1.0669 | Train Loss Unweighted: 0.5331 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1324 | Classification: 0.0928 | Classification_binary: 0.1578 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1390 | Regression_eta: 0.0017 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1828 | Regression_cos_phi: 0.0908 | Regression_energy: 0.0696 | Regression_eta: 0.0439 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0831 | LR: 5.75e-04 | DataLoad Time: 0.0170s | Model Forward Time: 8.1374s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.7836 | max_loss: 0.1442 | max_backward: 1.8747 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 3.6030 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0626 | Memory: rss_MB: 7857.5 | uss_MB: 7243.1 | child_rss_MB: 5051.0 | allocated_MB: 448.1 | reserved_MB: 24490.0 | inactive_split_MB: 1198.5 +rank0 [2026-10-29 07:26:07,516] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 115.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +4 8 0x7408, 13395 50.0°C 132.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +6 10 0x7408, 62036 47.0°C 134.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:31:00,707] INFO : Step 24200/50000 rank0 | Train Loss: 1.1691 | Train Loss Unweighted: 0.5645 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1546 | Classification: 0.1031 | Classification_binary: 0.1915 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1325 | Regression_eta: 0.0019 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.2096 | Regression_cos_phi: 0.0975 | Regression_energy: 0.0679 | Regression_eta: 0.0506 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0804 | LR: 5.73e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.0149s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.9207 | max_loss: 0.1533 | max_backward: 2.2288 | max_optimizer_step: 0.2436 | max_tensorboard_logging: 0.8522 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7858.2 | uss_MB: 7247.2 | child_rss_MB: 5039.0 | allocated_MB: 450.2 | reserved_MB: 26490.0 | inactive_split_MB: 976.9 +rank0 [2026-10-29 07:31:00,949] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 48.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:35:42,876] INFO : Step 24300/50000 rank0 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5336 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1380 | Classification: 0.0927 | Classification_binary: 0.1863 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1247 | Regression_eta: 0.0020 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0652 | Regression_eta: 0.0386 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0846 | LR: 5.70e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.3397s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8648 | max_loss: 0.1481 | max_backward: 1.7152 | max_optimizer_step: 0.2781 | max_tensorboard_logging: 0.7831 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6810.0 | uss_MB: 6199.1 | child_rss_MB: 5043.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 983.3 +rank0 [2026-10-29 07:35:43,120] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 48.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:40:20,015] INFO : Step 24400/50000 rank0 | Train Loss: 0.9707 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0879 | Classification_binary: 0.1454 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0014 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0747 | Regression_eta: 0.0263 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0484 | LR: 5.67e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.7197s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7628 | max_loss: 0.1940 | max_backward: 2.4253 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.9481 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6811.6 | uss_MB: 5803.5 | child_rss_MB: 5042.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 1129.3 +rank0 [2026-10-29 07:40:20,257] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 133.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 53.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +4 8 0x7408, 13395 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 97% +6 10 0x7408, 62036 48.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:45:00,697] INFO : Step 24500/50000 rank0 | Train Loss: 1.0937 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1319 | Classification: 0.0941 | Classification_binary: 0.1676 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1334 | Regression_eta: 0.0016 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.1799 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0673 | Regression_eta: 0.0379 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0670 | LR: 5.64e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.0758s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7565 | max_loss: 0.5105 | max_backward: 1.8057 | max_optimizer_step: 0.2534 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6807.0 | uss_MB: 6198.6 | child_rss_MB: 5045.1 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 945.2 +rank0 [2026-10-29 07:45:00,937] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 53.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 48.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:49:44,906] INFO : Step 24600/50000 rank0 | Train Loss: 1.0924 | Train Loss Unweighted: 0.5511 | Train Loss Components: Auxiliary: 0.2237 | Cardinality: 0.1233 | Classification: 0.0963 | Classification_binary: 0.1991 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1255 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2196 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0620 | Regression_eta: 0.0392 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0612 | LR: 5.61e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.3173s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.9060 | max_loss: 0.1523 | max_backward: 2.0894 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.5102 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6811.1 | uss_MB: 6200.6 | child_rss_MB: 5046.3 | allocated_MB: 449.5 | reserved_MB: 26492.0 | inactive_split_MB: 943.2 +rank0 [2026-10-29 07:49:45,148] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:54:30,011] INFO : Step 24700/50000 rank0 | Train Loss: 0.9972 | Train Loss Unweighted: 0.4984 | Train Loss Components: Auxiliary: 0.1950 | Cardinality: 0.1160 | Classification: 0.0782 | Classification_binary: 0.1692 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0590 | Regression_eta: 0.0673 | Regression_pt: 0.0644 | Regression_sin_phi: 0.1134 | LR: 5.58e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2432s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0060 | max_forward: 0.7847 | max_loss: 0.1519 | max_backward: 1.7159 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.8325 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6813.0 | uss_MB: 6199.7 | child_rss_MB: 5050.6 | allocated_MB: 447.9 | reserved_MB: 27708.0 | inactive_split_MB: 1169.6 +rank0 [2026-10-29 07:54:30,258] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 07:59:09,704] INFO : Step 24800/50000 rank0 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4722 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1202 | Classification: 0.0808 | Classification_binary: 0.1484 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0016 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1541 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0601 | Regression_eta: 0.0573 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0732 | LR: 5.56e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.3538s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0127 | max_forward: 0.7903 | max_loss: 0.1427 | max_backward: 1.7798 | max_optimizer_step: 0.2260 | max_tensorboard_logging: 0.8943 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6813.0 | uss_MB: 6199.9 | child_rss_MB: 5051.5 | allocated_MB: 447.6 | reserved_MB: 28508.0 | inactive_split_MB: 989.1 +rank0 [2026-10-29 07:59:09,949] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 53.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:03:51,492] INFO : Step 24900/50000 rank0 | Train Loss: 1.0773 | Train Loss Unweighted: 0.5300 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1506 | Classification: 0.0964 | Classification_binary: 0.1776 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1257 | Regression_eta: 0.0016 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0585 | Regression_eta: 0.0476 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0595 | LR: 5.53e-04 | DataLoad Time: 0.0147s | Model Forward Time: 6.0018s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8100 | max_loss: 0.1483 | max_backward: 1.9423 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 1.0834 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6802.2 | uss_MB: 5593.7 | child_rss_MB: 5050.0 | allocated_MB: 449.1 | reserved_MB: 28508.0 | inactive_split_MB: 993.6 +rank0 [2026-10-29 08:03:51,738] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 53.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:08:38,354] INFO : Step 25000/50000 rank0 | Train Loss: 1.1315 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1671 | Classification: 0.0915 | Classification_binary: 0.1899 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1201 | Regression_eta: 0.0020 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0645 | Regression_eta: 0.1188 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0629 | LR: 5.50e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.3804s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8152 | max_loss: 0.2476 | max_backward: 1.7290 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.8750 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6807.0 | uss_MB: 6195.6 | child_rss_MB: 5052.2 | allocated_MB: 448.5 | reserved_MB: 28508.0 | inactive_split_MB: 953.9 +rank0 [2026-10-29 08:08:38,601] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 41% +2 6 0x7408, 15961 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 98% +4 8 0x7408, 13395 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 95% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 55% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:08:38,695] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-25000.pth +rank0 [2026-10-29 08:08:39,050] INFO : Running validation on rank0 +rank0 [2026-10-29 08:08:39,051] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 08:08:39,051] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 08:09:31,488] INFO : VALIDATION | Step=25000/50000 | Train Loss=1.1315 | Valid Loss=1.0818 | Train Loss Unweighted=0.5264 | Valid Loss Unweighted=0.5274 | Stale=0 | ETA=1256.2m +rank0 [2026-10-29 08:09:31,489] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 08:09:31,489] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 08:09:31,489] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 08:09:31,489] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 08:09:31,489] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 08:09:31,489] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 08:09:31,490] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 08:09:31,490] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 08:09:31,490] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 08:09:31,490] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 08:09:31,490] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 08:09:31,490] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 08:09:31,490] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 08:09:31,490] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/ari=0.577244 +rank0 [2026-10-29 08:09:31,490] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-29 08:09:31,491] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.617664 +rank0 [2026-10-29 08:09:31,491] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/top1_competing=0.70391 +rank0 [2026-10-29 08:09:31,491] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.638432 +rank0 [2026-10-29 08:09:31,491] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0280913 +rank0 [2026-10-29 08:09:31,491] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0639571 +rank0 [2026-10-29 08:09:31,491] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 08:09:31,491] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 08:09:31,491] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 08:09:31,491] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 08:09:31,492] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 08:09:31,492] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 08:09:31,492] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 08:09:31,492] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 08:09:31,492] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 08:09:31,492] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 08:09:31,492] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 08:09:31,492] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/seconds_per_rank=21.6045 +rank0 [2026-10-29 08:09:31,492] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 08:09:31,492] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 08:09:31,493] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 08:09:31,493] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 08:09:31,493] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 08:09:31,493] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 08:09:31,493] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 08:09:31,493] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding/ari=0.294443 +rank0 [2026-10-29 08:09:31,493] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 08:09:31,493] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/embedding_minus_raw/ari=0.20282 +rank0 [2026-10-29 08:09:31,493] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 08:09:31,494] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 08:09:31,494] INFO : VALIDATION_PROBE | Step=25000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 08:09:31,498] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:09:31,755] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 08:09:31,765] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 08:09:58,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 08:09:59,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 08:10:00,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 08:10:01,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 08:10:11,648] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 08:10:11,660] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:10:12,183] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 08:10:12,186] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 08:10:28,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 08:10:29,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 08:10:31,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 08:10:32,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 08:10:42,207] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 08:10:42,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 08:10:42,694] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 08:10:42,698] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 08:10:59,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 08:11:00,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 08:11:01,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 08:11:02,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 08:11:12,097] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 08:11:12,117] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits +rank0 [2026-10-29 08:11:15,554] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits +rank0 [2026-10-29 08:11:16,695] INFO : Plotted number of elements +rank0 [2026-10-29 08:11:18,262] INFO : Plotted jets +rank0 [2026-10-29 08:11:19,452] INFO : Plotted jet ratio +rank0 [2026-10-29 08:11:19,527] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 08:11:22,758] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 08:11:23,794] INFO : Plotted number of elements +rank0 [2026-10-29 08:11:25,790] INFO : Plotted jets +rank0 [2026-10-29 08:11:26,883] INFO : Plotted jet ratio +rank0 [2026-10-29 08:11:26,932] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits +rank0 [2026-10-29 08:11:29,314] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits +rank0 [2026-10-29 08:11:30,150] INFO : Plotted number of elements +rank0 [2026-10-29 08:11:31,650] INFO : Plotted jets +rank0 [2026-10-29 08:11:32,804] INFO : Plotted jet ratio +rank0 [2026-10-29 08:16:20,472] INFO : Step 25100/50000 rank0 | Train Loss: 1.0264 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1288 | Classification: 0.0850 | Classification_binary: 0.1497 | HitGrouping: 0.1664 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1378 | Regression_eta: 0.0016 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1395 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0584 | Regression_eta: 0.0539 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0491 | LR: 5.47e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.2403s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0059 | max_forward: 0.7878 | max_loss: 0.1437 | max_backward: 2.2383 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9692 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7903.8 | uss_MB: 7286.7 | child_rss_MB: 5042.9 | allocated_MB: 446.8 | reserved_MB: 24062.0 | inactive_split_MB: 1159.3 +rank0 [2026-10-29 08:16:20,715] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 52.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 1% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 95% +4 8 0x7408, 13395 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 10% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 48.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:21:21,611] INFO : Step 25200/50000 rank0 | Train Loss: 1.0775 | Train Loss Unweighted: 0.5434 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1397 | Classification: 0.0996 | Classification_binary: 0.1706 | HitGrouping: 0.1584 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1340 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.1893 | Regression_cos_phi: 0.1072 | Regression_energy: 0.0833 | Regression_eta: 0.0401 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0715 | LR: 5.44e-04 | DataLoad Time: 0.0154s | Model Forward Time: 6.1434s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8831 | max_loss: 0.1456 | max_backward: 2.1320 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7903.3 | uss_MB: 7287.2 | child_rss_MB: 5042.2 | allocated_MB: 448.0 | reserved_MB: 26042.0 | inactive_split_MB: 1093.3 +rank0 [2026-10-29 08:21:21,858] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:26:17,982] INFO : Step 25300/50000 rank0 | Train Loss: 1.0761 | Train Loss Unweighted: 0.5109 | Train Loss Components: Auxiliary: 0.2132 | Cardinality: 0.1361 | Classification: 0.0961 | Classification_binary: 0.1821 | HitGrouping: 0.1562 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1138 | Regression_eta: 0.0018 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1983 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0588 | Regression_eta: 0.0321 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0635 | LR: 5.42e-04 | DataLoad Time: 0.0189s | Model Forward Time: 6.8924s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.9122 | max_loss: 0.1821 | max_backward: 2.1499 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.9506 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6897.2 | uss_MB: 5222.8 | child_rss_MB: 5036.8 | allocated_MB: 446.7 | reserved_MB: 28062.0 | inactive_split_MB: 1098.3 +rank0 [2026-10-29 08:26:18,227] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 4% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 0% +2 6 0x7408, 15961 52.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 86% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 25% +4 8 0x7408, 13395 51.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 8% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 48.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 0% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 43% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:31:11,434] INFO : Step 25400/50000 rank0 | Train Loss: 1.0222 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1238 | Classification: 0.0928 | Classification_binary: 0.1653 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1221 | Regression_eta: 0.0015 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0627 | Regression_eta: 0.0352 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0478 | LR: 5.39e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.4481s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1954 | max_backward: 1.8415 | max_optimizer_step: 0.3277 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6896.2 | uss_MB: 6278.8 | child_rss_MB: 5052.2 | allocated_MB: 449.3 | reserved_MB: 28062.0 | inactive_split_MB: 945.6 +rank0 [2026-10-29 08:31:11,677] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 53.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:36:14,230] INFO : Step 25500/50000 rank0 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4596 | Train Loss Components: Auxiliary: 0.1817 | Cardinality: 0.1001 | Classification: 0.0953 | Classification_binary: 0.1334 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0016 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0987 | Regression_energy: 0.0593 | Regression_eta: 0.0619 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0865 | LR: 5.36e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.4362s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7884 | max_loss: 0.1475 | max_backward: 2.0272 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.7862 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0483 | Memory: rss_MB: 6898.2 | uss_MB: 6283.2 | child_rss_MB: 5050.7 | allocated_MB: 450.7 | reserved_MB: 28062.0 | inactive_split_MB: 1166.9 +rank0 [2026-10-29 08:36:14,474] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:41:09,669] INFO : Step 25600/50000 rank0 | Train Loss: 1.0724 | Train Loss Unweighted: 0.5330 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1190 | Classification: 0.0985 | Classification_binary: 0.1843 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1231 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2160 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0672 | Regression_eta: 0.0509 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0789 | LR: 5.33e-04 | DataLoad Time: 0.0092s | Model Forward Time: 6.8911s | Phase Times: max_pre_step_barrier: 0.0194 | max_batch_to_device: 0.0083 | max_forward: 1.0622 | max_loss: 0.3311 | max_backward: 2.5999 | max_optimizer_step: 0.3019 | max_tensorboard_logging: 0.8123 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6895.2 | uss_MB: 5585.2 | child_rss_MB: 5023.7 | allocated_MB: 450.4 | reserved_MB: 28064.0 | inactive_split_MB: 1163.5 +rank0 [2026-10-29 08:41:09,913] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 52.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 9% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 50.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 99% +6 10 0x7408, 62036 48.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:46:10,326] INFO : Step 25700/50000 rank0 | Train Loss: 1.0595 | Train Loss Unweighted: 0.5120 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.1394 | Classification: 0.0920 | Classification_binary: 0.1853 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0754 | Regression_eta: 0.0387 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0467 | LR: 5.30e-04 | DataLoad Time: 0.0216s | Model Forward Time: 6.4554s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.8320 | max_loss: 0.2520 | max_backward: 2.6962 | max_optimizer_step: 0.2699 | max_tensorboard_logging: 0.7057 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6885.0 | uss_MB: 6272.6 | child_rss_MB: 5051.6 | allocated_MB: 448.6 | reserved_MB: 29942.0 | inactive_split_MB: 1165.2 +rank0 [2026-10-29 08:46:10,569] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 50.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 47.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:51:40,856] INFO : Step 25800/50000 rank0 | Train Loss: 0.9592 | Train Loss Unweighted: 0.4733 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1038 | Classification: 0.0787 | Classification_binary: 0.1531 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1182 | Regression_eta: 0.0016 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0723 | Regression_eta: 0.0672 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0666 | LR: 5.27e-04 | DataLoad Time: 0.0145s | Model Forward Time: 5.1812s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7549 | max_loss: 0.1480 | max_backward: 1.5296 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 0.8796 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1361 | Memory: rss_MB: 6889.1 | uss_MB: 6277.5 | child_rss_MB: 5023.6 | allocated_MB: 445.9 | reserved_MB: 30100.0 | inactive_split_MB: 961.8 +rank0 [2026-10-29 08:51:41,104] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 08:56:50,871] INFO : Step 25900/50000 rank0 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4613 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1142 | Classification: 0.0768 | Classification_binary: 0.1501 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1145 | Regression_eta: 0.0015 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1874 | Regression_cos_phi: 0.0932 | Regression_energy: 0.0735 | Regression_eta: 0.0504 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0584 | LR: 5.25e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.5930s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0124 | max_forward: 0.7901 | max_loss: 0.1430 | max_backward: 1.9362 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8933 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 6880.5 | uss_MB: 6281.6 | child_rss_MB: 5008.5 | allocated_MB: 447.7 | reserved_MB: 30100.0 | inactive_split_MB: 1167.6 +rank0 [2026-10-29 08:56:51,115] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:01:48,168] INFO : Step 26000/50000 rank0 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5169 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1347 | Classification: 0.0896 | Classification_binary: 0.1938 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0015 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0594 | Regression_eta: 0.0335 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0532 | LR: 5.22e-04 | DataLoad Time: 0.0147s | Model Forward Time: 6.1442s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8125 | max_loss: 0.2612 | max_backward: 1.8771 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.8972 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6845.2 | uss_MB: 5700.0 | child_rss_MB: 4882.0 | allocated_MB: 450.3 | reserved_MB: 30108.0 | inactive_split_MB: 1161.3 +rank0 [2026-10-29 09:01:48,412] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 3% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 22% +2 6 0x7408, 15961 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 82% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +4 8 0x7408, 13395 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 84% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 2% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:01:48,436] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-26000.pth +rank0 [2026-10-29 09:01:48,954] INFO : Running validation on rank0 +rank0 [2026-10-29 09:01:48,954] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 09:01:48,954] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 09:02:43,145] INFO : VALIDATION | Step=26000/50000 | Train Loss=1.0575 | Valid Loss=1.0511 | Train Loss Unweighted=0.5169 | Valid Loss Unweighted=0.5140 | Stale=0 | ETA=1208.6m +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 09:02:43,146] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/ari=0.566465 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.646957 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/top1_competing=0.740669 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.646341 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0173124 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0718658 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 09:02:43,147] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/seconds_per_rank=21.5535 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding/ari=0.288733 +rank0 [2026-10-29 09:02:43,148] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 09:02:43,149] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/embedding_minus_raw/ari=0.19711 +rank0 [2026-10-29 09:02:43,149] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 09:02:43,149] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 09:02:43,149] INFO : VALIDATION_PROBE | Step=26000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 09:02:43,154] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:02:43,425] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 09:02:43,434] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 09:03:09,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 09:03:10,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 09:03:11,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 09:03:12,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 09:03:22,706] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 09:03:22,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:03:23,245] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 09:03:23,248] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 09:03:39,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 09:03:40,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 09:03:41,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 09:03:42,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 09:03:52,905] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 09:03:52,917] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:03:53,429] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 09:03:53,433] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 09:04:09,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 09:04:10,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 09:04:11,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 09:04:12,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 09:04:22,049] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 09:04:22,065] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits +rank0 [2026-10-29 09:04:25,343] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits +rank0 [2026-10-29 09:04:26,389] INFO : Plotted number of elements +rank0 [2026-10-29 09:04:27,984] INFO : Plotted jets +rank0 [2026-10-29 09:04:29,619] INFO : Plotted jet ratio +rank0 [2026-10-29 09:04:29,720] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 09:04:33,018] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 09:04:34,055] INFO : Plotted number of elements +rank0 [2026-10-29 09:04:35,653] INFO : Plotted jets +rank0 [2026-10-29 09:04:36,745] INFO : Plotted jet ratio +rank0 [2026-10-29 09:04:36,806] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits +rank0 [2026-10-29 09:04:39,273] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits +rank0 [2026-10-29 09:04:40,139] INFO : Plotted number of elements +rank0 [2026-10-29 09:04:41,627] INFO : Plotted jets +rank0 [2026-10-29 09:04:42,756] INFO : Plotted jet ratio +rank0 [2026-10-29 09:09:38,266] INFO : Step 26100/50000 rank0 | Train Loss: 0.9519 | Train Loss Unweighted: 0.4738 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1091 | Classification: 0.0871 | Classification_binary: 0.1547 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1147 | Regression_eta: 0.0013 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0684 | Regression_eta: 0.0388 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0694 | LR: 5.19e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.8817s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0321 | max_forward: 0.8143 | max_loss: 0.1540 | max_backward: 1.7669 | max_optimizer_step: 0.2382 | max_tensorboard_logging: 1.1185 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 8060.0 | uss_MB: 7433.5 | child_rss_MB: 4850.5 | allocated_MB: 445.1 | reserved_MB: 25850.0 | inactive_split_MB: 1376.9 +rank0 [2026-10-29 09:09:38,514] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 29% 11% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:14:37,032] INFO : Step 26200/50000 rank0 | Train Loss: 0.9758 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.0970 | Classification: 0.0858 | Classification_binary: 0.1501 | HitGrouping: 0.1665 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0573 | Regression_eta: 0.0359 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0788 | LR: 5.16e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.5614s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7728 | max_loss: 0.1384 | max_backward: 1.8195 | max_optimizer_step: 0.3036 | max_tensorboard_logging: 0.7010 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0602 | Memory: rss_MB: 8060.1 | uss_MB: 7488.7 | child_rss_MB: 4848.5 | allocated_MB: 447.4 | reserved_MB: 27830.0 | inactive_split_MB: 1117.7 +rank0 [2026-10-29 09:14:37,288] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:19:52,350] INFO : Step 26300/50000 rank0 | Train Loss: 1.0568 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2146 | Cardinality: 0.1416 | Classification: 0.0935 | Classification_binary: 0.1772 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2184 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0576 | Regression_eta: 0.0395 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0847 | LR: 5.13e-04 | DataLoad Time: 0.0192s | Model Forward Time: 6.8754s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.8205 | max_loss: 0.1808 | max_backward: 2.8872 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 1.1024 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6927.6 | uss_MB: 6354.4 | child_rss_MB: 4842.0 | allocated_MB: 450.8 | reserved_MB: 29830.0 | inactive_split_MB: 1114.5 +rank0 [2026-10-29 09:19:52,594] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +2 6 0x7408, 15961 53.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 99% +6 10 0x7408, 62036 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:24:56,311] INFO : Step 26400/50000 rank0 | Train Loss: 1.0536 | Train Loss Unweighted: 0.5239 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1323 | Classification: 0.0937 | Classification_binary: 0.1845 | HitGrouping: 0.1559 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1209 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0521 | Regression_eta: 0.0367 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0463 | LR: 5.10e-04 | DataLoad Time: 0.0169s | Model Forward Time: 6.1716s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8639 | max_loss: 0.1569 | max_backward: 2.4710 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 1.0049 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6924.7 | uss_MB: 6353.2 | child_rss_MB: 4853.6 | allocated_MB: 447.6 | reserved_MB: 29832.0 | inactive_split_MB: 1186.4 +rank0 [2026-10-29 09:24:56,568] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 53.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 50.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +6 10 0x7408, 62036 48.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:30:00,294] INFO : Step 26500/50000 rank0 | Train Loss: 0.9812 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1299 | Classification: 0.0807 | Classification_binary: 0.1400 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1217 | Regression_eta: 0.0017 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1331 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0605 | Regression_eta: 0.0578 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0739 | LR: 5.08e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.5794s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7436 | max_loss: 0.1579 | max_backward: 1.8417 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.9265 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6921.4 | uss_MB: 6354.1 | child_rss_MB: 4851.7 | allocated_MB: 444.4 | reserved_MB: 29832.0 | inactive_split_MB: 1229.5 +rank0 [2026-10-29 09:30:00,539] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 45% 0% +2 6 0x7408, 15961 52.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 2% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 92% +4 8 0x7408, 13395 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 10% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +6 10 0x7408, 62036 48.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 16% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 94% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:34:59,204] INFO : Step 26600/50000 rank0 | Train Loss: 1.0108 | Train Loss Unweighted: 0.5066 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.1182 | Classification: 0.0957 | Classification_binary: 0.1588 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1242 | Regression_eta: 0.0014 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0510 | Regression_eta: 0.0336 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0585 | LR: 5.05e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.3942s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7561 | max_loss: 0.1430 | max_backward: 1.8427 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6921.4 | uss_MB: 6356.0 | child_rss_MB: 4827.0 | allocated_MB: 447.9 | reserved_MB: 29832.0 | inactive_split_MB: 1037.2 +rank0 [2026-10-29 09:34:59,447] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:40:07,809] INFO : Step 26700/50000 rank0 | Train Loss: 1.0878 | Train Loss Unweighted: 0.5266 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1269 | Classification: 0.0956 | Classification_binary: 0.1961 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1144 | Regression_eta: 0.0018 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1937 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0566 | Regression_eta: 0.0368 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0775 | LR: 5.02e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.8569s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.9788 | max_loss: 0.1540 | max_backward: 1.9407 | max_optimizer_step: 0.2295 | max_tensorboard_logging: 1.0076 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6911.8 | uss_MB: 6344.9 | child_rss_MB: 4859.9 | allocated_MB: 450.1 | reserved_MB: 29832.0 | inactive_split_MB: 1190.4 +rank0 [2026-10-29 09:40:08,053] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 47.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:48:42,196] INFO : Step 26800/50000 rank0 | Train Loss: 1.0077 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1102 | Classification: 0.0787 | Classification_binary: 0.1553 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1359 | Regression_eta: 0.0015 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0915 | Regression_energy: 0.0774 | Regression_eta: 0.0355 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0689 | LR: 4.99e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.4989s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.7774 | max_loss: 0.2523 | max_backward: 1.8741 | max_optimizer_step: 0.2873 | max_tensorboard_logging: 0.8206 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6909.2 | uss_MB: 6351.0 | child_rss_MB: 4816.3 | allocated_MB: 444.7 | reserved_MB: 29832.0 | inactive_split_MB: 1121.3 +rank0 [2026-10-29 09:48:42,441] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:53:40,919] INFO : Step 26900/50000 rank0 | Train Loss: 0.9524 | Train Loss Unweighted: 0.4793 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1099 | Classification: 0.0833 | Classification_binary: 0.1305 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1313 | Regression_eta: 0.0016 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0476 | Regression_energy: 0.1038 | Regression_eta: 0.0380 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0867 | LR: 4.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.2957s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7292 | max_loss: 0.1373 | max_backward: 1.7442 | max_optimizer_step: 0.3152 | max_tensorboard_logging: 0.6764 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 6905.2 | uss_MB: 5520.6 | child_rss_MB: 4788.1 | allocated_MB: 446.9 | reserved_MB: 29832.0 | inactive_split_MB: 1016.7 +rank0 [2026-10-29 09:53:41,164] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 1% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 45% 0% +2 6 0x7408, 15961 52.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 8% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 51.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +6 10 0x7408, 62036 47.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 98% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 25% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:58:37,858] INFO : Step 27000/50000 rank0 | Train Loss: 1.0285 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1243 | Classification: 0.0862 | Classification_binary: 0.1888 | HitGrouping: 0.1539 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0016 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1944 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0613 | Regression_eta: 0.0469 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0650 | LR: 4.94e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.8568s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.9134 | max_loss: 0.1497 | max_backward: 2.1803 | max_optimizer_step: 0.2254 | max_tensorboard_logging: 0.9490 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6904.1 | uss_MB: 6354.5 | child_rss_MB: 4772.3 | allocated_MB: 449.7 | reserved_MB: 29832.0 | inactive_split_MB: 1024.3 +rank0 [2026-10-29 09:58:38,106] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +2 6 0x7408, 15961 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 09:58:38,129] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-27000.pth +rank0 [2026-10-29 09:58:38,523] INFO : Running validation on rank0 +rank0 [2026-10-29 09:58:38,523] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 09:58:38,523] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION | Step=27000/50000 | Train Loss=1.0285 | Valid Loss=1.0246 | Train Loss Unweighted=0.4948 | Valid Loss Unweighted=0.4959 | Stale=0 | ETA=1163.8m +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 09:59:33,492] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/ari=0.575684 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.67293 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/top1_competing=0.774638 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.647623 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0265311 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0731481 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 09:59:33,493] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/seconds_per_rank=21.6351 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding/ari=0.284019 +rank0 [2026-10-29 09:59:33,494] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 09:59:33,495] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/embedding_minus_raw/ari=0.192396 +rank0 [2026-10-29 09:59:33,495] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 09:59:33,495] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 09:59:33,495] INFO : VALIDATION_PROBE | Step=27000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 09:59:33,500] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 09:59:33,763] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 09:59:33,773] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 10:00:00,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 10:00:01,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 10:00:02,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 10:00:03,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 10:00:13,937] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 10:00:14,008] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:00:14,538] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 10:00:14,542] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 10:00:31,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 10:00:33,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 10:00:35,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 10:00:36,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 10:00:46,253] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-29 10:00:46,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:00:46,867] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 10:00:46,871] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 10:01:02,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 10:01:03,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 10:01:04,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 10:01:05,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 10:01:15,567] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 10:01:15,585] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits +rank0 [2026-10-29 10:01:18,908] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits +rank0 [2026-10-29 10:01:19,926] INFO : Plotted number of elements +rank0 [2026-10-29 10:01:21,485] INFO : Plotted jets +rank0 [2026-10-29 10:01:22,655] INFO : Plotted jet ratio +rank0 [2026-10-29 10:01:22,726] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 10:01:26,023] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 10:01:27,085] INFO : Plotted number of elements +rank0 [2026-10-29 10:01:29,084] INFO : Plotted jets +rank0 [2026-10-29 10:01:30,206] INFO : Plotted jet ratio +rank0 [2026-10-29 10:01:30,259] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits +rank0 [2026-10-29 10:01:32,680] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits +rank0 [2026-10-29 10:01:33,536] INFO : Plotted number of elements +rank0 [2026-10-29 10:01:35,032] INFO : Plotted jets +rank0 [2026-10-29 10:01:36,172] INFO : Plotted jet ratio +rank0 [2026-10-29 10:06:38,125] INFO : Step 27100/50000 rank0 | Train Loss: 1.0614 | Train Loss Unweighted: 0.4989 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1449 | Classification: 0.0752 | Classification_binary: 0.1877 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1965 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0642 | Regression_eta: 0.0342 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0630 | LR: 4.91e-04 | DataLoad Time: 0.0214s | Model Forward Time: 7.7640s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0966 | max_forward: 2.1724 | max_loss: 0.6705 | max_backward: 3.1723 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 1.5888 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 8047.0 | uss_MB: 7496.7 | child_rss_MB: 4778.7 | allocated_MB: 447.0 | reserved_MB: 27292.0 | inactive_split_MB: 983.4 +rank0 [2026-10-29 10:06:38,414] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 50.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:11:20,388] INFO : Step 27200/50000 rank0 | Train Loss: 0.9045 | Train Loss Unweighted: 0.4389 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1028 | Classification: 0.0770 | Classification_binary: 0.1287 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1318 | Regression_cos_phi: 0.0804 | Regression_energy: 0.0702 | Regression_eta: 0.0881 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0902 | LR: 4.88e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.5931s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7725 | max_loss: 0.5032 | max_backward: 1.6788 | max_optimizer_step: 0.2815 | max_tensorboard_logging: 0.9686 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8047.0 | uss_MB: 7493.7 | child_rss_MB: 4786.7 | allocated_MB: 445.7 | reserved_MB: 29236.0 | inactive_split_MB: 1232.8 +rank0 [2026-10-29 10:11:20,646] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:15:59,329] INFO : Step 27300/50000 rank0 | Train Loss: 1.0726 | Train Loss Unweighted: 0.5095 | Train Loss Components: Auxiliary: 0.2138 | Cardinality: 0.1337 | Classification: 0.1081 | Classification_binary: 0.1640 | HitGrouping: 0.1596 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1159 | Regression_eta: 0.0019 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1752 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0667 | Regression_eta: 0.0451 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0551 | LR: 4.85e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.8332s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8896 | max_loss: 0.1460 | max_backward: 2.0153 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6974.0 | uss_MB: 6424.6 | child_rss_MB: 4784.6 | allocated_MB: 450.9 | reserved_MB: 29330.0 | inactive_split_MB: 1103.1 +rank0 [2026-10-29 10:15:59,572] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 48.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:20:32,418] INFO : Step 27400/50000 rank0 | Train Loss: 1.0758 | Train Loss Unweighted: 0.5282 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1344 | Classification: 0.0930 | Classification_binary: 0.1899 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1207 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1866 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0618 | Regression_eta: 0.0496 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0731 | LR: 4.82e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.6249s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8446 | max_loss: 0.1488 | max_backward: 2.0519 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.8725 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6977.0 | uss_MB: 5627.2 | child_rss_MB: 4777.8 | allocated_MB: 448.4 | reserved_MB: 31310.0 | inactive_split_MB: 895.7 +rank0 [2026-10-29 10:20:32,660] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 51% 4% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:25:08,058] INFO : Step 27500/50000 rank0 | Train Loss: 0.9697 | Train Loss Unweighted: 0.4574 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1111 | Classification: 0.0831 | Classification_binary: 0.1473 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1114 | Regression_eta: 0.0018 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1096 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0601 | Regression_eta: 0.0473 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0640 | LR: 4.80e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.1775s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.7904 | max_loss: 0.1563 | max_backward: 1.6291 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.7457 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6966.1 | uss_MB: 4710.9 | child_rss_MB: 4769.8 | allocated_MB: 447.3 | reserved_MB: 31310.0 | inactive_split_MB: 986.8 +rank0 [2026-10-29 10:25:08,299] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +4 8 0x7408, 13395 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +6 10 0x7408, 62036 48.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:30:04,647] INFO : Step 27600/50000 rank0 | Train Loss: 0.9616 | Train Loss Unweighted: 0.4618 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1092 | Classification: 0.0865 | Classification_binary: 0.1284 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0019 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1308 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0681 | Regression_eta: 0.0372 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0560 | LR: 4.77e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1811s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0380 | max_forward: 0.7674 | max_loss: 0.1463 | max_backward: 1.9164 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.5145 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6974.1 | uss_MB: 6422.9 | child_rss_MB: 4778.4 | allocated_MB: 446.7 | reserved_MB: 31310.0 | inactive_split_MB: 1182.0 +rank0 [2026-10-29 10:30:04,906] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 51% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 47.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:34:47,704] INFO : Step 27700/50000 rank0 | Train Loss: 1.0693 | Train Loss Unweighted: 0.5190 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1414 | Classification: 0.0939 | Classification_binary: 0.1783 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0015 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0634 | Regression_eta: 0.0349 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0857 | LR: 4.74e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.3830s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.8151 | max_loss: 0.1750 | max_backward: 2.0398 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.5213 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0659 | Memory: rss_MB: 6974.1 | uss_MB: 6424.8 | child_rss_MB: 4756.0 | allocated_MB: 448.4 | reserved_MB: 33312.0 | inactive_split_MB: 986.3 +rank0 [2026-10-29 10:34:47,960] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +2 6 0x7408, 15961 53.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 54% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 48.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:39:29,984] INFO : Step 27800/50000 rank0 | Train Loss: 1.0875 | Train Loss Unweighted: 0.5294 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1226 | Classification: 0.0924 | Classification_binary: 0.1842 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1237 | Regression_eta: 0.0019 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0720 | Regression_eta: 0.0429 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0762 | LR: 4.71e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.4267s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0152 | max_forward: 0.8463 | max_loss: 0.1744 | max_backward: 1.7379 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.9397 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 6971.2 | uss_MB: 6421.2 | child_rss_MB: 4773.7 | allocated_MB: 452.2 | reserved_MB: 33312.0 | inactive_split_MB: 1320.1 +rank0 [2026-10-29 10:39:30,240] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 54% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:44:12,752] INFO : Step 27900/50000 rank0 | Train Loss: 0.9483 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0988 | Classification: 0.0859 | Classification_binary: 0.1439 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1244 | Regression_eta: 0.0014 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1364 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0884 | Regression_eta: 0.0250 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0646 | LR: 4.68e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.2863s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.7626 | max_loss: 0.1370 | max_backward: 1.8424 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6972.2 | uss_MB: 6423.3 | child_rss_MB: 4773.1 | allocated_MB: 447.1 | reserved_MB: 33312.0 | inactive_split_MB: 1112.6 +rank0 [2026-10-29 10:44:13,030] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 54% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:48:56,592] INFO : Step 28000/50000 rank0 | Train Loss: 0.9840 | Train Loss Unweighted: 0.4736 | Train Loss Components: Auxiliary: 0.1949 | Cardinality: 0.1114 | Classification: 0.0927 | Classification_binary: 0.1564 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1099 | Regression_eta: 0.0015 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1761 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0604 | Regression_eta: 0.0568 | Regression_pt: 0.0665 | Regression_sin_phi: 0.0772 | LR: 4.66e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.5627s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8192 | max_loss: 0.2552 | max_backward: 2.1299 | max_optimizer_step: 0.2920 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 6969.4 | uss_MB: 6424.2 | child_rss_MB: 4776.8 | allocated_MB: 448.2 | reserved_MB: 33312.0 | inactive_split_MB: 1178.0 +rank0 [2026-10-29 10:48:56,833] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 54% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 48.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 54% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 10:48:56,950] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-28000.pth +rank0 [2026-10-29 10:48:57,236] INFO : Running validation on rank0 +rank0 [2026-10-29 10:48:57,237] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 10:48:57,237] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 10:49:51,984] INFO : VALIDATION | Step=28000/50000 | Train Loss=0.9840 | Valid Loss=1.0041 | Train Loss Unweighted=0.4736 | Valid Loss Unweighted=0.4893 | Stale=0 | ETA=1113.0m +rank0 [2026-10-29 10:49:51,984] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 10:49:51,984] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/ari=0.57101 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.655574 +rank0 [2026-10-29 10:49:51,985] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/top1_competing=0.749576 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.659603 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0218576 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0851287 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 10:49:51,986] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/seconds_per_rank=21.221 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding/ari=0.276025 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/embedding_minus_raw/ari=0.184402 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 10:49:51,987] INFO : VALIDATION_PROBE | Step=28000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 10:49:51,993] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:49:52,410] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 10:49:52,414] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 10:50:18,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 10:50:19,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 10:50:21,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 10:50:22,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 10:50:32,385] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 10:50:32,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:50:32,998] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 10:50:33,002] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 10:50:49,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 10:50:50,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 10:50:51,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 10:50:52,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 10:51:02,893] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 10:51:02,905] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 10:51:03,448] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 10:51:03,453] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 10:51:19,655] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 10:51:20,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 10:51:21,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 10:51:22,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 10:51:32,588] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 10:51:32,606] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits +rank0 [2026-10-29 10:51:36,362] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits +rank0 [2026-10-29 10:51:37,365] INFO : Plotted number of elements +rank0 [2026-10-29 10:51:38,927] INFO : Plotted jets +rank0 [2026-10-29 10:51:40,092] INFO : Plotted jet ratio +rank0 [2026-10-29 10:51:40,163] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 10:51:43,437] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 10:51:44,446] INFO : Plotted number of elements +rank0 [2026-10-29 10:51:45,948] INFO : Plotted jets +rank0 [2026-10-29 10:51:47,045] INFO : Plotted jet ratio +rank0 [2026-10-29 10:51:47,098] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits +rank0 [2026-10-29 10:51:49,564] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits +rank0 [2026-10-29 10:51:50,429] INFO : Plotted number of elements +rank0 [2026-10-29 10:51:51,907] INFO : Plotted jets +rank0 [2026-10-29 10:51:53,591] INFO : Plotted jet ratio +rank0 [2026-10-29 10:56:33,377] INFO : Step 28100/50000 rank0 | Train Loss: 1.0915 | Train Loss Unweighted: 0.5085 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1649 | Classification: 0.0833 | Classification_binary: 0.1834 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1187 | Regression_eta: 0.0017 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1617 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0685 | Regression_eta: 0.0589 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0596 | LR: 4.63e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.5003s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8571 | max_loss: 0.1603 | max_backward: 1.9112 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 0.7670 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0838 | Memory: rss_MB: 8288.3 | uss_MB: 7733.7 | child_rss_MB: 4773.7 | allocated_MB: 448.9 | reserved_MB: 25946.0 | inactive_split_MB: 1313.1 +rank0 [2026-10-29 10:56:33,658] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:01:19,027] INFO : Step 28200/50000 rank0 | Train Loss: 0.9241 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1144 | Classification: 0.0770 | Classification_binary: 0.1441 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1163 | Regression_eta: 0.0014 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0308 | Classification_binary: 0.1328 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0604 | Regression_eta: 0.0403 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0487 | LR: 4.60e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.7507s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0751 | max_forward: 0.8691 | max_loss: 0.5236 | max_backward: 1.8483 | max_optimizer_step: 0.2480 | max_tensorboard_logging: 0.8546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 8289.0 | uss_MB: 7733.3 | child_rss_MB: 4770.2 | allocated_MB: 445.0 | reserved_MB: 25948.0 | inactive_split_MB: 976.4 +rank0 [2026-10-29 11:01:19,306] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 47.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:07:41,390] INFO : Step 28300/50000 rank0 | Train Loss: 0.9513 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1028 | Classification: 0.0882 | Classification_binary: 0.1368 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1147 | Regression_eta: 0.0016 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1529 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0604 | Regression_eta: 0.0387 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0725 | LR: 4.57e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.6332s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.7916 | max_loss: 0.1449 | max_backward: 2.0850 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.8583 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7055.4 | uss_MB: 6496.0 | child_rss_MB: 4774.9 | allocated_MB: 447.2 | reserved_MB: 25952.0 | inactive_split_MB: 1165.3 +rank0 [2026-10-29 11:07:41,685] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 47.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:12:28,255] INFO : Step 28400/50000 rank0 | Train Loss: 1.0604 | Train Loss Unweighted: 0.4982 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1382 | Classification: 0.0847 | Classification_binary: 0.1862 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1111 | Regression_eta: 0.0018 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1810 | Regression_cos_phi: 0.1063 | Regression_energy: 0.0546 | Regression_eta: 0.0457 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0687 | LR: 4.55e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.6707s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9139 | max_loss: 0.1474 | max_backward: 2.1323 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.6846 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0517 | Memory: rss_MB: 7055.5 | uss_MB: 6496.2 | child_rss_MB: 4770.0 | allocated_MB: 448.2 | reserved_MB: 26968.0 | inactive_split_MB: 835.9 +rank0 [2026-10-29 11:12:28,499] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 48.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:17:12,556] INFO : Step 28500/50000 rank0 | Train Loss: 1.0282 | Train Loss Unweighted: 0.5005 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1237 | Classification: 0.0911 | Classification_binary: 0.1756 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0013 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0617 | Regression_eta: 0.0344 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0675 | LR: 4.52e-04 | DataLoad Time: 0.0208s | Model Forward Time: 10.2991s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0886 | max_forward: 0.9758 | max_loss: 0.5123 | max_backward: 2.0811 | max_optimizer_step: 0.2797 | max_tensorboard_logging: 1.5644 | max_tensorboard_flush: 0.0047 | max_post_step_barrier: 0.0889 | Memory: rss_MB: 7042.2 | uss_MB: 4349.9 | child_rss_MB: 4774.7 | allocated_MB: 452.0 | reserved_MB: 27968.0 | inactive_split_MB: 923.2 +rank0 [2026-10-29 11:17:12,803] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 109.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 52.0°C 130.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 50.0°C 129.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 47.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:21:56,928] INFO : Step 28600/50000 rank0 | Train Loss: 0.9753 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1202 | Classification: 0.0804 | Classification_binary: 0.1446 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1132 | Regression_eta: 0.0017 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0323 | Classification_binary: 0.1220 | Regression_cos_phi: 0.1130 | Regression_energy: 0.0761 | Regression_eta: 0.0693 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0576 | LR: 4.49e-04 | DataLoad Time: 0.0231s | Model Forward Time: 5.6218s | Phase Times: max_pre_step_barrier: 0.1127 | max_batch_to_device: 0.0146 | max_forward: 0.7731 | max_loss: 0.2404 | max_backward: 1.6352 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 1.0006 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7048.8 | uss_MB: 5469.7 | child_rss_MB: 4776.7 | allocated_MB: 446.8 | reserved_MB: 30056.0 | inactive_split_MB: 896.4 +rank0 [2026-10-29 11:21:57,170] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 2% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 57% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 48.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 96% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:26:35,179] INFO : Step 28700/50000 rank0 | Train Loss: 1.0558 | Train Loss Unweighted: 0.5173 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1082 | Classification: 0.1065 | Classification_binary: 0.1612 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1217 | Regression_eta: 0.0024 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1730 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0574 | Regression_eta: 0.0589 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0818 | LR: 4.46e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.7088s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0686 | max_forward: 0.8423 | max_loss: 0.1475 | max_backward: 2.0170 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.8830 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 7053.8 | uss_MB: 6496.4 | child_rss_MB: 4774.2 | allocated_MB: 447.9 | reserved_MB: 30056.0 | inactive_split_MB: 925.2 +rank0 [2026-10-29 11:26:35,427] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 53.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:31:18,948] INFO : Step 28800/50000 rank0 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5167 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1474 | Classification: 0.0969 | Classification_binary: 0.1939 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2021 | Regression_cos_phi: 0.1160 | Regression_energy: 0.0741 | Regression_eta: 0.0684 | Regression_pt: 0.0790 | Regression_sin_phi: 0.1294 | LR: 4.44e-04 | DataLoad Time: 0.0201s | Model Forward Time: 6.5530s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0075 | max_forward: 0.8427 | max_loss: 0.2661 | max_backward: 2.8464 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7050.1 | uss_MB: 4315.7 | child_rss_MB: 4755.7 | allocated_MB: 448.2 | reserved_MB: 30058.0 | inactive_split_MB: 976.8 +rank0 [2026-10-29 11:31:19,208] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:35:54,594] INFO : Step 28900/50000 rank0 | Train Loss: 0.9389 | Train Loss Unweighted: 0.4547 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.1078 | Classification: 0.0733 | Classification_binary: 0.1521 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0497 | Regression_eta: 0.0440 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0654 | LR: 4.41e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.3527s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0062 | max_forward: 0.7499 | max_loss: 0.2426 | max_backward: 1.8556 | max_optimizer_step: 0.3135 | max_tensorboard_logging: 0.5831 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7048.4 | uss_MB: 6494.0 | child_rss_MB: 4774.7 | allocated_MB: 444.3 | reserved_MB: 30058.0 | inactive_split_MB: 1133.2 +rank0 [2026-10-29 11:35:54,836] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 49.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:40:53,994] INFO : Step 29000/50000 rank0 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4535 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0939 | Classification: 0.0855 | Classification_binary: 0.1308 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0015 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0681 | Regression_eta: 0.0574 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0961 | LR: 4.38e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.5236s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0062 | max_forward: 0.9270 | max_loss: 0.2073 | max_backward: 2.1896 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6044 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7047.7 | uss_MB: 6496.6 | child_rss_MB: 4759.3 | allocated_MB: 450.2 | reserved_MB: 30060.0 | inactive_split_MB: 1402.9 +rank0 [2026-10-29 11:40:54,238] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 48.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:40:54,356] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-29000.pth +rank0 [2026-10-29 11:40:54,646] INFO : Running validation on rank0 +rank0 [2026-10-29 11:40:54,646] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 11:40:54,646] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 11:41:47,204] INFO : VALIDATION | Step=29000/50000 | Train Loss=0.9231 | Valid Loss=1.0109 | Train Loss Unweighted=0.4535 | Valid Loss Unweighted=0.4900 | Stale=1 | ETA=1063.3m +rank0 [2026-10-29 11:41:47,204] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/ari=0.58128 +rank0 [2026-10-29 11:41:47,205] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.670165 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/top1_competing=0.769327 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.647951 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0321268 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0734758 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 11:41:47,206] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/seconds_per_rank=21.5 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding/ari=0.281696 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/embedding_minus_raw/ari=0.190073 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 11:41:47,207] INFO : VALIDATION_PROBE | Step=29000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 11:41:47,212] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:41:47,471] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 11:41:47,476] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 11:42:16,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 11:42:18,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 11:42:19,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 11:42:20,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 11:42:30,913] INFO : Time taken to make predictions on device 0 is: 0.30 min +rank0 [2026-10-29 11:42:31,032] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:42:31,640] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 11:42:31,643] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 11:42:47,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 11:42:48,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 11:42:49,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 11:42:50,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 11:43:00,800] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 11:43:00,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 11:43:01,399] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 11:43:01,403] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 11:43:16,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 11:43:17,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 11:43:18,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 11:43:19,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 11:43:29,682] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 11:43:29,698] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits +rank0 [2026-10-29 11:43:33,008] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits +rank0 [2026-10-29 11:43:34,022] INFO : Plotted number of elements +rank0 [2026-10-29 11:43:35,617] INFO : Plotted jets +rank0 [2026-10-29 11:43:36,805] INFO : Plotted jet ratio +rank0 [2026-10-29 11:43:36,855] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 11:43:40,129] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 11:43:41,144] INFO : Plotted number of elements +rank0 [2026-10-29 11:43:42,648] INFO : Plotted jets +rank0 [2026-10-29 11:43:43,748] INFO : Plotted jet ratio +rank0 [2026-10-29 11:43:43,813] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits +rank0 [2026-10-29 11:43:46,282] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits +rank0 [2026-10-29 11:43:47,135] INFO : Plotted number of elements +rank0 [2026-10-29 11:43:49,149] INFO : Plotted jets +rank0 [2026-10-29 11:43:50,286] INFO : Plotted jet ratio +rank0 [2026-10-29 11:48:35,808] INFO : Step 29100/50000 rank0 | Train Loss: 1.0559 | Train Loss Unweighted: 0.5040 | Train Loss Components: Auxiliary: 0.2142 | Cardinality: 0.1307 | Classification: 0.0939 | Classification_binary: 0.1851 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1098 | Regression_eta: 0.0015 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.2103 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0551 | Regression_eta: 0.0515 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0655 | LR: 4.35e-04 | DataLoad Time: 0.0150s | Model Forward Time: 8.3734s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.6779 | max_loss: 0.3815 | max_backward: 4.2661 | max_optimizer_step: 0.3470 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8253.0 | uss_MB: 6395.3 | child_rss_MB: 4770.6 | allocated_MB: 448.8 | reserved_MB: 24188.0 | inactive_split_MB: 1162.1 +rank0 [2026-10-29 11:48:36,054] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 137.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 29% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 50.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 47.0°C 136.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:53:34,681] INFO : Step 29200/50000 rank0 | Train Loss: 0.9909 | Train Loss Unweighted: 0.4801 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1204 | Classification: 0.0923 | Classification_binary: 0.1746 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1054 | Regression_eta: 0.0014 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.1919 | Regression_cos_phi: 0.1060 | Regression_energy: 0.0690 | Regression_eta: 0.0591 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0937 | LR: 4.33e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.4081s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8354 | max_loss: 0.1558 | max_backward: 2.1201 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0677 | Memory: rss_MB: 8251.2 | uss_MB: 7692.4 | child_rss_MB: 4763.1 | allocated_MB: 447.9 | reserved_MB: 24190.0 | inactive_split_MB: 1166.5 +rank0 [2026-10-29 11:53:34,926] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 50.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 11:58:28,612] INFO : Step 29300/50000 rank0 | Train Loss: 0.9337 | Train Loss Unweighted: 0.4462 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1121 | Classification: 0.0773 | Classification_binary: 0.1359 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1146 | Regression_eta: 0.0015 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1250 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0612 | Regression_eta: 0.0336 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0489 | LR: 4.30e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.5433s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0277 | max_forward: 0.8212 | max_loss: 0.1457 | max_backward: 1.8414 | max_optimizer_step: 0.2550 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7135.1 | uss_MB: 5997.0 | child_rss_MB: 4761.5 | allocated_MB: 444.9 | reserved_MB: 24190.0 | inactive_split_MB: 1038.6 +rank0 [2026-10-29 11:58:28,855] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:03:08,160] INFO : Step 29400/50000 rank0 | Train Loss: 1.0186 | Train Loss Unweighted: 0.4675 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1372 | Classification: 0.0948 | Classification_binary: 0.1561 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0020 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0856 | Regression_energy: 0.0585 | Regression_eta: 0.0381 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0653 | LR: 4.27e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.4513s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8495 | max_loss: 0.1434 | max_backward: 1.9359 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7145.4 | uss_MB: 6588.5 | child_rss_MB: 4762.3 | allocated_MB: 447.9 | reserved_MB: 26208.0 | inactive_split_MB: 957.3 +rank0 [2026-10-29 12:03:08,427] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 53.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:08:16,887] INFO : Step 29500/50000 rank0 | Train Loss: 1.0909 | Train Loss Unweighted: 0.5362 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.1071 | Classification_binary: 0.1896 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1176 | Regression_eta: 0.0019 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0665 | Regression_eta: 0.0452 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0935 | LR: 4.24e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.6499s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0876 | max_forward: 0.8925 | max_loss: 0.1527 | max_backward: 1.9621 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.8162 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7137.1 | uss_MB: 6580.0 | child_rss_MB: 4760.9 | allocated_MB: 448.9 | reserved_MB: 26214.0 | inactive_split_MB: 1166.5 +rank0 [2026-10-29 12:08:17,127] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 47.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:13:13,279] INFO : Step 29600/50000 rank0 | Train Loss: 0.9608 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.1935 | Cardinality: 0.1093 | Classification: 0.0850 | Classification_binary: 0.1542 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0557 | Regression_eta: 0.0562 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0520 | LR: 4.22e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.4940s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7848 | max_loss: 0.1495 | max_backward: 1.8042 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.0178 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7115.2 | uss_MB: 6585.9 | child_rss_MB: 4743.4 | allocated_MB: 445.5 | reserved_MB: 26214.0 | inactive_split_MB: 1171.9 +rank0 [2026-10-29 12:13:13,524] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 48.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:18:08,233] INFO : Step 29700/50000 rank0 | Train Loss: 0.9511 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1260 | Classification: 0.0829 | Classification_binary: 0.1357 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1183 | Regression_eta: 0.0014 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1453 | Regression_cos_phi: 0.1061 | Regression_energy: 0.0553 | Regression_eta: 0.0436 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0799 | LR: 4.19e-04 | DataLoad Time: 0.0211s | Model Forward Time: 6.0265s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7531 | max_loss: 0.2308 | max_backward: 2.2313 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.9708 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7117.3 | uss_MB: 6587.7 | child_rss_MB: 4741.9 | allocated_MB: 446.6 | reserved_MB: 26214.0 | inactive_split_MB: 1160.6 +rank0 [2026-10-29 12:18:08,479] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 51.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +5 9 0x7408, 1553 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 48.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:23:09,624] INFO : Step 29800/50000 rank0 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4915 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1104 | Classification: 0.0864 | Classification_binary: 0.1759 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0548 | Regression_eta: 0.0406 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0633 | LR: 4.16e-04 | DataLoad Time: 0.0159s | Model Forward Time: 7.1219s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8210 | max_loss: 0.1528 | max_backward: 2.4753 | max_optimizer_step: 0.3576 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0038 | max_post_step_barrier: 0.1798 | Memory: rss_MB: 7115.5 | uss_MB: 5536.1 | child_rss_MB: 4741.9 | allocated_MB: 450.3 | reserved_MB: 26214.0 | inactive_split_MB: 938.8 +rank0 [2026-10-29 12:23:09,903] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 133.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 10% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 98% +2 6 0x7408, 15961 52.0°C 140.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 36% 0% +4 8 0x7408, 13395 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 98% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 8% +6 10 0x7408, 62036 47.0°C 138.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 63% +7 11 0x7408, 49174 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 40% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:28:10,920] INFO : Step 29900/50000 rank0 | Train Loss: 0.9604 | Train Loss Unweighted: 0.4559 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1243 | Classification: 0.0825 | Classification_binary: 0.1616 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1040 | Regression_eta: 0.0015 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1428 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0525 | Regression_eta: 0.0559 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0517 | LR: 4.14e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.7258s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1142 | max_forward: 0.9271 | max_loss: 0.1531 | max_backward: 1.8698 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.8620 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7091.0 | uss_MB: 6437.5 | child_rss_MB: 4745.0 | allocated_MB: 450.0 | reserved_MB: 26214.0 | inactive_split_MB: 1166.0 +rank0 [2026-10-29 12:28:11,167] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 2% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +4 8 0x7408, 13395 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 47.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:32:59,588] INFO : Step 30000/50000 rank0 | Train Loss: 0.9450 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.1132 | Classification: 0.0847 | Classification_binary: 0.1394 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1615 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0767 | Regression_eta: 0.0378 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0888 | LR: 4.11e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.8871s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 1.0327 | max_loss: 0.1419 | max_backward: 1.6503 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.4878 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7039.0 | uss_MB: 6586.5 | child_rss_MB: 4745.4 | allocated_MB: 443.8 | reserved_MB: 26214.0 | inactive_split_MB: 1152.2 +rank0 [2026-10-29 12:47:12,788] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 43.0°C 104.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 41.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 48.0°C 123.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 41.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +6 10 0x7408, 62036 44.0°C 123.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 12:47:12,811] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth +rank0 [2026-10-29 12:47:13,116] INFO : Running validation on rank0 +rank0 [2026-10-29 12:47:13,116] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 12:47:13,116] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 14:49:00,070] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-29 14:49:00,082] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-29 14:49:20,930] INFO : Instantiating model +rank0 [2026-10-29 14:49:20,930] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank0 [2026-10-29 14:49:20,930] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-29 14:49:20,948] INFO : Input encoding initialization took 0.02s +rank0 [2026-10-29 14:49:20,948] INFO : nn0 parameters: 171776 +rank0 [2026-10-29 14:49:20,948] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-29 14:49:20,948] INFO : Initializing backbone layer 0 +rank0 [2026-10-29 14:49:20,986] INFO : Initializing backbone layer 1 +rank0 [2026-10-29 14:49:21,001] INFO : Initializing backbone layer 2 +rank0 [2026-10-29 14:49:21,015] INFO : Initializing backbone layer 3 +rank0 [2026-10-29 14:49:21,029] INFO : Initializing backbone layer 4 +rank0 [2026-10-29 14:49:21,043] INFO : Initializing backbone layer 5 +rank0 [2026-10-29 14:49:21,057] INFO : Backbone initialization took 0.11s +rank0 [2026-10-29 14:49:21,058] INFO : backbone parameters: 7901376 +rank0 [2026-10-29 14:49:21,058] INFO : Initializing output DNNs +rank0 [2026-10-29 14:49:21,112] INFO : Output DNNs initialization took 0.05s +rank0 [2026-10-29 14:49:21,112] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-29 14:49:21,112] INFO : jagged_attention enabled=False +rank0 [2026-10-29 14:49:21,112] INFO : flash_attn_varlen enabled=False +rank0 [2026-10-29 14:49:21,112] INFO : task_queries enabled=False +rank0 [2026-10-29 14:49:21,112] INFO : classification_norm parameters: 0 +rank0 [2026-10-29 14:49:21,112] INFO : regression_norm parameters: 0 +rank0 [2026-10-29 14:49:21,112] INFO : classification_query parameters: 0 +rank0 [2026-10-29 14:49:21,112] INFO : regression_query parameters: 0 +rank0 [2026-10-29 14:49:21,113] INFO : classification_readout parameters: 0 +rank0 [2026-10-29 14:49:21,113] INFO : regression_readout parameters: 0 +rank0 [2026-10-29 14:49:21,113] INFO : output_mode=set +rank0 [2026-10-29 14:49:21,113] INFO : set_decoder parameters: 4713812 +rank0 [2026-10-29 14:49:21,113] INFO : nn_binary_particle parameters: 0 +rank0 [2026-10-29 14:49:21,113] INFO : nn_pid parameters: 0 +rank0 [2026-10-29 14:49:21,113] INFO : nn_pt parameters: 0 +rank0 [2026-10-29 14:49:21,113] INFO : nn_eta parameters: 0 +rank0 [2026-10-29 14:49:21,114] INFO : nn_sin_phi parameters: 0 +rank0 [2026-10-29 14:49:21,114] INFO : nn_cos_phi parameters: 0 +rank0 [2026-10-29 14:49:21,114] INFO : nn_energy parameters: 0 +rank0 [2026-10-29 14:49:21,115] INFO : Total MLPF parameters: 12786964 +rank0 [2026-10-29 14:49:21,115] INFO : MLPF __init__ done +rank0 [2026-10-29 14:49:21,115] INFO : Instantiated model +rank0 [2026-10-29 14:49:21,115] INFO : Moving model to device rank=0 +rank0 [2026-10-29 14:49:21,145] INFO : Moved model to device rank=0 +rank0 [2026-10-29 14:49:21,148] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-29 14:49:21,561] INFO : Restoring training from step 30001 +rank0 [2026-10-29 14:49:21,569] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth +rank0 [2026-10-29 14:49:21,569] INFO : Configured model for SyncBatchNorm rank=0 +rank0 [2026-10-29 14:49:22,135] INFO : Configured model for DistributedDataParallel rank=0 +rank0 [2026-10-29 14:49:22,288] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank0 [2026-10-29 14:49:22,288] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank0 [2026-10-29 14:49:22,290] INFO : Trainable parameters: 12786964 +rank0 [2026-10-29 14:49:22,290] INFO : Non-trainable parameters: 23268 +rank0 [2026-10-29 14:49:22,290] INFO : Total parameters: 12810232 +rank0 [2026-10-29 14:49:22,293] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank0 [2026-10-29 14:49:22,298] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252 +rank0 [2026-10-29 14:49:22,298] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252 +rank0 [2026-10-29 14:49:22,298] INFO : Getting dataloaders +rank0 [2026-10-29 14:49:22,298] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:49:27,493] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:49:27,596] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:49:27,696] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:49:27,821] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:49:27,961] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:49:28,066] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:49:28,439] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:49:28,519] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:49:28,600] INFO : train_dataset: cld_edm_ttbar_hits, 90000 +rank0 [2026-10-29 14:49:28,706] INFO : train_dataset: cld_edm_ttbar_hits, 90900 +rank0 [2026-10-29 14:49:28,706] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:49:28,810] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:49:28,920] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:49:29,024] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:49:29,122] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:49:29,221] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:49:29,353] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:49:29,431] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:49:29,530] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:49:29,655] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90000 +rank0 [2026-10-29 14:49:29,758] INFO : train_dataset: cld_edm_ww_fullhad_hits, 90900 +rank0 [2026-10-29 14:49:29,758] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:49:30,139] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-29 14:49:30,219] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-29 14:49:30,297] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-29 14:49:30,376] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-29 14:49:30,454] INFO : train_dataset: cld_edm_qq_hits, 89998 +rank0 [2026-10-29 14:49:30,585] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-29 14:49:30,686] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-29 14:49:30,764] INFO : train_dataset: cld_edm_qq_hits, 90000 +rank0 [2026-10-29 14:49:30,849] INFO : train_dataset: cld_edm_qq_hits, 89999 +rank0 [2026-10-29 14:49:30,957] INFO : train_dataset: cld_edm_qq_hits, 90599 +rank0 [2026-10-29 14:49:30,957] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-29 14:49:30,957] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-29 14:49:30,958] INFO : train mix: physical: weight=21112 step_share=1.000 batches_per_epoch=21112 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-29 14:49:30,958] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:49:30,985] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:49:31,014] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:49:31,041] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:49:31,068] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:49:31,096] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:49:31,122] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:49:31,148] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:49:31,175] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:49:31,201] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:49:31,228] INFO : valid_dataset: cld_edm_ttbar_hits, 51 +rank0 [2026-10-29 14:49:31,228] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:49:31,254] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:49:31,279] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:49:31,306] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:49:31,334] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:49:31,359] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:49:31,385] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:49:31,413] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:49:31,444] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:49:31,471] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:49:31,501] INFO : valid_dataset: cld_edm_ww_fullhad_hits, 51 +rank0 [2026-10-29 14:49:31,501] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 14:49:31,527] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:49:31,552] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:49:31,578] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:49:31,604] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:49:31,630] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:49:31,655] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:49:31,680] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:49:31,705] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:49:31,733] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:49:31,763] INFO : valid_dataset: cld_edm_qq_hits, 51 +rank0 [2026-10-29 14:49:31,763] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-29 14:49:31,763] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-29 14:49:45,479] INFO : Got dataloaders +rank0 [2026-10-29 14:49:45,480] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [8888] of epochs [1] +rank0 [2026-10-29 14:49:45,480] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank0 [2026-10-29 14:49:45,480] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank0 [2026-10-29 14:49:45,480] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank0 [2026-10-29 14:49:47,542] INFO : loader split=train rank=0 len=21112 +rank0 [2026-10-29 14:49:47,543] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-29 14:49:47,543] INFO : Starting training from step 30001 to 50000 on rank 0 of 8 +rank0 [2026-10-29 14:49:47,549] INFO : Creating train_iterator +rank0 [2026-10-29 14:49:47,549] INFO : ResumableSampler physical:train loading state: start_index=142208 +rank0 [2026-10-29 14:50:01,388] INFO : Created train_iterator +rank0 [2026-10-29 14:54:42,677] INFO : Step 30100/50000 rank0 | Train Loss: 0.9854 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.2024 | Cardinality: 0.1295 | Classification: 0.0965 | Classification_binary: 0.1561 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0619 | Regression_eta: 0.0297 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0566 | LR: 4.08e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3087s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7011 | max_loss: 0.1473 | max_backward: 1.5273 | max_optimizer_step: 0.1574 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0061 | Memory: rss_MB: 4775.9 | uss_MB: 3472.5 | child_rss_MB: 6376.2 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 850.1 +rank0 [2026-10-29 14:54:42,916] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 201.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 56.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 196.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 56.0°C 212.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 52.0°C 198.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 14:59:13,507] INFO : Step 30200/50000 rank0 | Train Loss: 1.0165 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2044 | Cardinality: 0.1520 | Classification: 0.0856 | Classification_binary: 0.1718 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1917 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0675 | Regression_eta: 0.0403 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0855 | LR: 4.06e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7009s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.7915 | max_loss: 0.2568 | max_backward: 1.8926 | max_optimizer_step: 0.1551 | max_tensorboard_logging: 0.4604 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4776.3 | uss_MB: 3476.7 | child_rss_MB: 6367.6 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 842.0 +rank0 [2026-10-29 14:59:13,742] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 187.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 53.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 200.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:03:43,600] INFO : Step 30300/50000 rank0 | Train Loss: 0.9183 | Train Loss Unweighted: 0.4449 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1232 | Classification: 0.0820 | Classification_binary: 0.1459 | HitGrouping: 0.1516 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0012 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0489 | Regression_eta: 0.0371 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0477 | LR: 4.03e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4914s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7525 | max_loss: 0.2476 | max_backward: 1.8262 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.4574 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0039 | Memory: rss_MB: 4773.5 | uss_MB: 3475.1 | child_rss_MB: 6385.6 | allocated_MB: 495.8 | reserved_MB: 28760.0 | inactive_split_MB: 848.2 +rank0 [2026-10-29 15:03:43,839] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 192.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 56.0°C 201.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 51.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:08:14,135] INFO : Step 30400/50000 rank0 | Train Loss: 0.9035 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1137 | Classification: 0.0848 | Classification_binary: 0.1358 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1717 | Regression_cos_phi: 0.0802 | Regression_energy: 0.0615 | Regression_eta: 0.0462 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0783 | LR: 4.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6317s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7769 | max_loss: 0.1398 | max_backward: 1.9002 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.4925 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4770.8 | uss_MB: 3475.5 | child_rss_MB: 6366.2 | allocated_MB: 498.8 | reserved_MB: 28760.0 | inactive_split_MB: 878.9 +rank0 [2026-10-29 15:08:14,374] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 194.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 52.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 195.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:12:46,292] INFO : Step 30500/50000 rank0 | Train Loss: 1.0399 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1239 | Classification: 0.0901 | Classification_binary: 0.1841 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0018 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.2005 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0599 | Regression_eta: 0.0398 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0580 | LR: 3.98e-04 | DataLoad Time: 0.0145s | Model Forward Time: 5.0711s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9688 | max_loss: 0.1627 | max_backward: 2.3054 | max_optimizer_step: 0.1555 | max_tensorboard_logging: 0.4908 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4768.4 | uss_MB: 3475.6 | child_rss_MB: 6405.5 | allocated_MB: 501.7 | reserved_MB: 28760.0 | inactive_split_MB: 944.3 +rank0 [2026-10-29 15:12:46,532] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 186.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:17:14,720] INFO : Step 30600/50000 rank0 | Train Loss: 0.9987 | Train Loss Unweighted: 0.4959 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1112 | Classification: 0.0932 | Classification_binary: 0.1720 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1137 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0610 | Regression_eta: 0.0334 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0690 | LR: 3.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.0765s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7736 | max_loss: 0.1566 | max_backward: 1.7922 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.6994 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4770.8 | uss_MB: 3476.1 | child_rss_MB: 6399.5 | allocated_MB: 500.1 | reserved_MB: 28760.0 | inactive_split_MB: 845.9 +rank0 [2026-10-29 15:17:14,960] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:21:46,487] INFO : Step 30700/50000 rank0 | Train Loss: 0.9490 | Train Loss Unweighted: 0.4575 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1237 | Classification: 0.0844 | Classification_binary: 0.1399 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0014 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1309 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0465 | Regression_eta: 0.0391 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0570 | LR: 3.92e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9182s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7215 | max_loss: 0.5560 | max_backward: 2.0906 | max_optimizer_step: 0.1553 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4768.7 | uss_MB: 3474.9 | child_rss_MB: 6379.2 | allocated_MB: 493.8 | reserved_MB: 28760.0 | inactive_split_MB: 827.5 +rank0 [2026-10-29 15:21:46,723] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:26:16,879] INFO : Step 30800/50000 rank0 | Train Loss: 1.0320 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1331 | Classification: 0.0916 | Classification_binary: 0.1632 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1173 | Regression_eta: 0.0015 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0577 | Regression_eta: 0.0399 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0656 | LR: 3.90e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.3823s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1016 | max_forward: 0.8041 | max_loss: 0.1462 | max_backward: 1.9351 | max_optimizer_step: 0.1552 | max_tensorboard_logging: 1.1007 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4769.5 | uss_MB: 3475.6 | child_rss_MB: 6407.8 | allocated_MB: 497.9 | reserved_MB: 28760.0 | inactive_split_MB: 842.0 +rank0 [2026-10-29 15:26:17,118] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:30:49,836] INFO : Step 30900/50000 rank0 | Train Loss: 1.0226 | Train Loss Unweighted: 0.4952 | Train Loss Components: Auxiliary: 0.2139 | Cardinality: 0.1339 | Classification: 0.0875 | Classification_binary: 0.1859 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1092 | Regression_eta: 0.0013 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0649 | Regression_eta: 0.0380 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0644 | LR: 3.87e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0486s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7610 | max_loss: 0.2067 | max_backward: 1.7028 | max_optimizer_step: 0.1599 | max_tensorboard_logging: 1.0327 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4771.1 | uss_MB: 3476.3 | child_rss_MB: 6385.8 | allocated_MB: 497.2 | reserved_MB: 28760.0 | inactive_split_MB: 838.0 +rank0 [2026-10-29 15:30:50,075] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 56.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:35:27,173] INFO : Step 31000/50000 rank0 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4630 | Train Loss Components: Auxiliary: 0.2006 | Cardinality: 0.1217 | Classification: 0.0821 | Classification_binary: 0.1592 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1085 | Regression_eta: 0.0018 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0502 | Regression_eta: 0.0681 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0765 | LR: 3.84e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.1229s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7780 | max_loss: 0.2475 | max_backward: 1.8803 | max_optimizer_step: 0.1588 | max_tensorboard_logging: 0.9947 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4772.1 | uss_MB: 3477.0 | child_rss_MB: 6402.3 | allocated_MB: 498.2 | reserved_MB: 28760.0 | inactive_split_MB: 820.4 +rank0 [2026-10-29 15:35:27,412] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:35:27,436] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-31000.pth +rank0 [2026-10-29 15:35:27,892] INFO : Running validation on rank0 +rank0 [2026-10-29 15:35:27,892] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 15:35:27,893] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 15:36:20,287] INFO : VALIDATION | Step=31000/50000 | Train Loss=0.9922 | Valid Loss=0.9880 | Train Loss Unweighted=0.4630 | Valid Loss Unweighted=0.4761 | Stale=0 | ETA=28.5m +rank0 [2026-10-29 15:36:20,287] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 15:36:20,287] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/ari=0.600466 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.664344 +rank0 [2026-10-29 15:36:20,288] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/top1_competing=0.761917 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.647994 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0513134 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0735189 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 15:36:20,289] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/seconds_per_rank=21.2504 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding/ari=0.302711 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 15:36:20,290] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/embedding_minus_raw/ari=0.211088 +rank0 [2026-10-29 15:36:20,291] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 15:36:20,291] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 15:36:20,291] INFO : VALIDATION_PROBE | Step=31000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 15:36:20,296] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:36:20,555] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 15:36:20,571] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 15:36:45,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 15:36:47,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 15:36:48,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 15:36:49,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 15:36:59,549] INFO : Time taken to make predictions on device 0 is: 0.28 min +rank0 [2026-10-29 15:36:59,566] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:37:00,103] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 15:37:00,106] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 15:37:16,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 15:37:17,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 15:37:18,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 15:37:19,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 15:37:29,378] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 15:37:29,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 15:37:29,837] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 15:37:29,841] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 15:37:45,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 15:37:46,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 15:37:47,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 15:37:48,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 15:37:58,405] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 15:37:58,422] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits +rank0 [2026-10-29 15:38:01,563] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits +rank0 [2026-10-29 15:38:02,803] INFO : Plotted number of elements +rank0 [2026-10-29 15:38:04,412] INFO : Plotted jets +rank0 [2026-10-29 15:38:05,914] INFO : Plotted jet ratio +rank0 [2026-10-29 15:38:05,967] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 15:38:09,157] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 15:38:10,132] INFO : Plotted number of elements +rank0 [2026-10-29 15:38:11,633] INFO : Plotted jets +rank0 [2026-10-29 15:38:12,735] INFO : Plotted jet ratio +rank0 [2026-10-29 15:38:12,788] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits +rank0 [2026-10-29 15:38:15,225] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits +rank0 [2026-10-29 15:38:16,055] INFO : Plotted number of elements +rank0 [2026-10-29 15:38:17,538] INFO : Plotted jets +rank0 [2026-10-29 15:38:18,689] INFO : Plotted jet ratio +rank0 [2026-10-29 15:42:50,995] INFO : Step 31100/50000 rank0 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4264 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1054 | Classification: 0.0853 | Classification_binary: 0.1288 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1064 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0634 | Regression_eta: 0.0349 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0444 | LR: 3.82e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2344s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7353 | max_loss: 0.1410 | max_backward: 1.7959 | max_optimizer_step: 0.2754 | max_tensorboard_logging: 0.9599 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 6463.6 | uss_MB: 5188.4 | child_rss_MB: 6394.5 | allocated_MB: 495.8 | reserved_MB: 27508.0 | inactive_split_MB: 851.2 +rank0 [2026-10-29 15:42:51,237] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +2 6 0x7408, 15961 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 30% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:47:35,286] INFO : Step 31200/50000 rank0 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4709 | Train Loss Components: Auxiliary: 0.2045 | Cardinality: 0.1286 | Classification: 0.0862 | Classification_binary: 0.1646 | HitGrouping: 0.1524 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1077 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1826 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0629 | Regression_eta: 0.0373 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0621 | LR: 3.79e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.0976s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8441 | max_loss: 0.1469 | max_backward: 2.0813 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 6462.4 | uss_MB: 5189.4 | child_rss_MB: 6422.3 | allocated_MB: 496.9 | reserved_MB: 27508.0 | inactive_split_MB: 935.6 +rank0 [2026-10-29 15:47:35,525] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:52:20,675] INFO : Step 31300/50000 rank0 | Train Loss: 1.0631 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1394 | Classification: 0.0917 | Classification_binary: 0.1731 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0578 | Regression_eta: 0.0933 | Regression_pt: 0.0607 | Regression_sin_phi: 0.1486 | LR: 3.76e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9613s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7950 | max_loss: 0.1940 | max_backward: 1.8810 | max_optimizer_step: 0.2519 | max_tensorboard_logging: 0.5719 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5602.7 | uss_MB: 4330.1 | child_rss_MB: 6421.1 | allocated_MB: 498.6 | reserved_MB: 29580.0 | inactive_split_MB: 773.5 +rank0 [2026-10-29 15:52:20,913] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 15:56:53,203] INFO : Step 31400/50000 rank0 | Train Loss: 0.9019 | Train Loss Unweighted: 0.4258 | Train Loss Components: Auxiliary: 0.1823 | Cardinality: 0.1101 | Classification: 0.0746 | Classification_binary: 0.1337 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1070 | Regression_eta: 0.0014 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0624 | Regression_eta: 0.0607 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0625 | LR: 3.74e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8916s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8478 | max_loss: 0.2460 | max_backward: 2.0403 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.4770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5595.4 | uss_MB: 4331.0 | child_rss_MB: 6422.8 | allocated_MB: 495.2 | reserved_MB: 29580.0 | inactive_split_MB: 986.6 +rank0 [2026-10-29 15:56:53,453] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:01:34,887] INFO : Step 31500/50000 rank0 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4434 | Train Loss Components: Auxiliary: 0.1867 | Cardinality: 0.1036 | Classification: 0.0834 | Classification_binary: 0.1437 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1064 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1593 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0583 | Regression_eta: 0.0396 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0533 | LR: 3.71e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.3877s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7930 | max_loss: 0.1458 | max_backward: 1.7855 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 2.2800 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5572.8 | uss_MB: 4310.0 | child_rss_MB: 6426.4 | allocated_MB: 496.8 | reserved_MB: 29580.0 | inactive_split_MB: 850.8 +rank0 [2026-10-29 16:01:35,126] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:06:06,145] INFO : Step 31600/50000 rank0 | Train Loss: 1.0288 | Train Loss Unweighted: 0.4730 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1317 | Classification: 0.0906 | Classification_binary: 0.1743 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1012 | Regression_eta: 0.0017 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0397 | Regression_energy: 0.0539 | Regression_eta: 0.0398 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0529 | LR: 3.69e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8623s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1021 | max_forward: 0.8041 | max_loss: 0.5000 | max_backward: 1.8881 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0133 | Memory: rss_MB: 5567.6 | uss_MB: 4330.2 | child_rss_MB: 6421.3 | allocated_MB: 494.5 | reserved_MB: 29580.0 | inactive_split_MB: 850.5 +rank0 [2026-10-29 16:06:06,383] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 186.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 53.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 199.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:10:48,800] INFO : Step 31700/50000 rank0 | Train Loss: 0.9206 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1106 | Classification: 0.0729 | Classification_binary: 0.1465 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1217 | Regression_cos_phi: 0.0338 | Regression_energy: 0.0571 | Regression_eta: 0.0267 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0419 | LR: 3.66e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4274s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0062 | max_forward: 0.7097 | max_loss: 0.1806 | max_backward: 1.6117 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0100 | Memory: rss_MB: 5567.0 | uss_MB: 4328.6 | child_rss_MB: 6424.1 | allocated_MB: 495.3 | reserved_MB: 29580.0 | inactive_split_MB: 969.6 +rank0 [2026-10-29 16:10:49,038] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 187.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 206.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 52.0°C 200.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:15:56,946] INFO : Step 31800/50000 rank0 | Train Loss: 0.9090 | Train Loss Unweighted: 0.4310 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1140 | Classification: 0.0748 | Classification_binary: 0.1332 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1094 | Regression_eta: 0.0013 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0533 | Regression_eta: 0.0495 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0701 | LR: 3.64e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5542s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7102 | max_loss: 0.1436 | max_backward: 1.6093 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5562.0 | uss_MB: 4326.9 | child_rss_MB: 6424.4 | allocated_MB: 499.4 | reserved_MB: 29584.0 | inactive_split_MB: 1068.5 +rank0 [2026-10-29 16:15:57,181] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 184.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 54.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:20:40,353] INFO : Step 31900/50000 rank0 | Train Loss: 0.9919 | Train Loss Unweighted: 0.4645 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1269 | Classification: 0.0911 | Classification_binary: 0.1745 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0975 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1845 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0493 | Regression_eta: 0.0293 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0563 | LR: 3.61e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9827s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.2538 | max_backward: 1.9768 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.5597 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5568.8 | uss_MB: 4333.1 | child_rss_MB: 6433.0 | allocated_MB: 496.2 | reserved_MB: 29584.0 | inactive_split_MB: 829.8 +rank0 [2026-10-29 16:20:40,590] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:25:22,216] INFO : Step 32000/50000 rank0 | Train Loss: 0.9662 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1172 | Classification: 0.0861 | Classification_binary: 0.1650 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1010 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0510 | Regression_eta: 0.0367 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0879 | LR: 3.58e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9356s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7801 | max_loss: 0.1534 | max_backward: 1.8397 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5561.3 | uss_MB: 4329.3 | child_rss_MB: 6422.4 | allocated_MB: 497.7 | reserved_MB: 29584.0 | inactive_split_MB: 927.6 +rank0 [2026-10-29 16:25:22,455] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 192.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +6 10 0x7408, 62036 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:25:22,479] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-32000.pth +rank0 [2026-10-29 16:25:22,879] INFO : Running validation on rank0 +rank0 [2026-10-29 16:25:22,879] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 16:25:22,879] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 16:26:16,731] INFO : VALIDATION | Step=32000/50000 | Train Loss=0.9662 | Valid Loss=0.9601 | Train Loss Unweighted=0.4570 | Valid Loss Unweighted=0.4602 | Stale=0 | ETA=54.3m +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 16:26:16,732] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/ari=0.581932 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.652459 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/top1_competing=0.74879 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.654348 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.032779 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0798729 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 16:26:16,733] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/seconds_per_rank=21.2511 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding/ari=0.264991 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/embedding_minus_raw/ari=0.173368 +rank0 [2026-10-29 16:26:16,734] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 16:26:16,735] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 16:26:16,735] INFO : VALIDATION_PROBE | Step=32000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 16:26:16,739] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:26:17,076] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 16:26:17,092] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 16:26:42,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 16:26:44,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 16:26:45,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 16:26:46,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 16:26:56,481] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 16:26:56,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:26:57,010] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 16:26:57,014] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 16:27:13,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 16:27:14,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 16:27:15,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 16:27:16,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 16:27:26,900] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 16:27:26,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 16:27:27,382] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 16:27:27,386] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 16:27:43,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 16:27:44,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 16:27:45,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 16:27:46,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 16:27:56,016] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 16:27:56,117] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits +rank0 [2026-10-29 16:27:59,352] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits +rank0 [2026-10-29 16:28:00,711] INFO : Plotted number of elements +rank0 [2026-10-29 16:28:02,287] INFO : Plotted jets +rank0 [2026-10-29 16:28:03,459] INFO : Plotted jet ratio +rank0 [2026-10-29 16:28:03,527] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 16:28:06,761] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 16:28:07,768] INFO : Plotted number of elements +rank0 [2026-10-29 16:28:09,272] INFO : Plotted jets +rank0 [2026-10-29 16:28:10,401] INFO : Plotted jet ratio +rank0 [2026-10-29 16:28:10,453] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits +rank0 [2026-10-29 16:28:12,903] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits +rank0 [2026-10-29 16:28:13,753] INFO : Plotted number of elements +rank0 [2026-10-29 16:28:15,683] INFO : Plotted jets +rank0 [2026-10-29 16:28:16,859] INFO : Plotted jet ratio +rank0 [2026-10-29 16:33:00,466] INFO : Step 32100/50000 rank0 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4572 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1125 | Classification: 0.0829 | Classification_binary: 0.1389 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1159 | Regression_eta: 0.0016 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1241 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0635 | Regression_eta: 0.0301 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0633 | LR: 3.56e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.4195s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0431 | max_forward: 0.7694 | max_loss: 0.1379 | max_backward: 1.5662 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 1.3675 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6985.8 | uss_MB: 5757.8 | child_rss_MB: 6447.8 | allocated_MB: 499.0 | reserved_MB: 25862.0 | inactive_split_MB: 892.0 +rank0 [2026-10-29 16:33:00,707] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 56.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 29% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:37:34,011] INFO : Step 32200/50000 rank0 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1193 | Classification: 0.0958 | Classification_binary: 0.1508 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1092 | Regression_eta: 0.0016 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0693 | Regression_eta: 0.0370 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0771 | LR: 3.53e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.9710s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8207 | max_loss: 0.1504 | max_backward: 2.0613 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.9148 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6989.2 | uss_MB: 5763.2 | child_rss_MB: 6457.2 | allocated_MB: 499.4 | reserved_MB: 25864.0 | inactive_split_MB: 778.6 +rank0 [2026-10-29 16:37:34,250] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 52.0°C 151.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:42:05,155] INFO : Step 32300/50000 rank0 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4909 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1295 | Classification: 0.0875 | Classification_binary: 0.1834 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0017 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0579 | Regression_eta: 0.0379 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0581 | LR: 3.51e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.4320s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8382 | max_loss: 0.2412 | max_backward: 2.0637 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.9540 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5664.5 | uss_MB: 4450.0 | child_rss_MB: 6462.2 | allocated_MB: 499.0 | reserved_MB: 27716.0 | inactive_split_MB: 775.0 +rank0 [2026-10-29 16:42:05,393] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 55.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:46:38,139] INFO : Step 32400/50000 rank0 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1089 | Classification: 0.0860 | Classification_binary: 0.1494 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1059 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1369 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0531 | Regression_eta: 0.0388 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0521 | LR: 3.48e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.1855s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1137 | max_forward: 0.8338 | max_loss: 0.1463 | max_backward: 1.8285 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 0.8970 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5664.1 | uss_MB: 4449.3 | child_rss_MB: 6459.6 | allocated_MB: 497.5 | reserved_MB: 27716.0 | inactive_split_MB: 964.6 +rank0 [2026-10-29 16:46:38,390] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:51:12,496] INFO : Step 32500/50000 rank0 | Train Loss: 0.9126 | Train Loss Unweighted: 0.4568 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.0989 | Classification: 0.0980 | Classification_binary: 0.1302 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1128 | Regression_eta: 0.0017 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1149 | Regression_cos_phi: 0.0366 | Regression_energy: 0.0488 | Regression_eta: 0.0366 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0689 | LR: 3.46e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.8684s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7619 | max_loss: 0.1497 | max_backward: 1.5269 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.7665 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5643.1 | uss_MB: 4429.2 | child_rss_MB: 6454.9 | allocated_MB: 495.6 | reserved_MB: 27718.0 | inactive_split_MB: 1084.2 +rank0 [2026-10-29 16:51:12,742] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 170.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 55.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 16:55:49,305] INFO : Step 32600/50000 rank0 | Train Loss: 0.9821 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2025 | Cardinality: 0.1270 | Classification: 0.0917 | Classification_binary: 0.1687 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1000 | Regression_eta: 0.0014 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.2051 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0534 | Regression_eta: 0.0324 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0528 | LR: 3.43e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9684s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1505 | max_backward: 1.8823 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 5658.5 | uss_MB: 4448.5 | child_rss_MB: 6453.0 | allocated_MB: 496.5 | reserved_MB: 27844.0 | inactive_split_MB: 835.5 +rank0 [2026-10-29 16:55:49,544] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 56.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:00:17,707] INFO : Step 32700/50000 rank0 | Train Loss: 1.0218 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2103 | Cardinality: 0.1308 | Classification: 0.0908 | Classification_binary: 0.1761 | HitGrouping: 0.1579 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0695 | Regression_eta: 0.0453 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0449 | LR: 3.41e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.9458s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7654 | max_loss: 0.2523 | max_backward: 1.8639 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4904 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.1101 | Memory: rss_MB: 5657.8 | uss_MB: 4448.3 | child_rss_MB: 6459.1 | allocated_MB: 495.8 | reserved_MB: 27844.0 | inactive_split_MB: 1177.8 +rank0 [2026-10-29 17:00:17,946] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 55.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:04:51,346] INFO : Step 32800/50000 rank0 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4798 | Train Loss Components: Auxiliary: 0.1977 | Cardinality: 0.1139 | Classification: 0.0957 | Classification_binary: 0.1480 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1163 | Regression_eta: 0.0019 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0563 | Regression_eta: 0.0542 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0509 | LR: 3.38e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7467s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7437 | max_loss: 0.1443 | max_backward: 1.7603 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.5989 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5618.6 | uss_MB: 4447.1 | child_rss_MB: 6328.3 | allocated_MB: 497.6 | reserved_MB: 27846.0 | inactive_split_MB: 1089.8 +rank0 [2026-10-29 17:04:51,584] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 53.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 56.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:09:25,895] INFO : Step 32900/50000 rank0 | Train Loss: 1.0585 | Train Loss Unweighted: 0.4917 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1472 | Classification: 0.0991 | Classification_binary: 0.1566 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0020 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.1743 | Regression_cos_phi: 0.1071 | Regression_energy: 0.0585 | Regression_eta: 0.0676 | Regression_pt: 0.0662 | Regression_sin_phi: 0.1104 | LR: 3.36e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.0176s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0141 | max_forward: 0.7853 | max_loss: 0.1442 | max_backward: 1.9716 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.5450 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5622.7 | uss_MB: 4451.0 | child_rss_MB: 6346.8 | allocated_MB: 498.5 | reserved_MB: 27846.0 | inactive_split_MB: 844.8 +rank0 [2026-10-29 17:09:26,147] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 56.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:13:58,182] INFO : Step 33000/50000 rank0 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4896 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1109 | Classification: 0.0902 | Classification_binary: 0.1757 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1096 | Regression_eta: 0.0015 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1672 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0477 | Regression_eta: 0.0379 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0521 | LR: 3.33e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7768s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0372 | max_forward: 0.7798 | max_loss: 0.1504 | max_backward: 1.8342 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5614.6 | uss_MB: 4448.7 | child_rss_MB: 6348.5 | allocated_MB: 497.7 | reserved_MB: 27846.0 | inactive_split_MB: 857.6 +rank0 [2026-10-29 17:13:58,423] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 184.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 56.0°C 198.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:13:58,445] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-33000.pth +rank0 [2026-10-29 17:13:58,853] INFO : Running validation on rank0 +rank0 [2026-10-29 17:13:58,853] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 17:13:58,853] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION | Step=33000/50000 | Train Loss=0.9900 | Valid Loss=0.9457 | Train Loss Unweighted=0.4896 | Valid Loss Unweighted=0.4576 | Stale=0 | ETA=74.7m +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 17:14:50,060] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/ari=0.592042 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.672651 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/top1_competing=0.774116 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.664115 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0428894 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0896404 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 17:14:50,061] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/seconds_per_rank=21.2216 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 17:14:50,062] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 17:14:50,063] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 17:14:50,063] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 17:14:50,063] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding/ari=0.285482 +rank0 [2026-10-29 17:14:50,063] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 17:14:50,063] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/embedding_minus_raw/ari=0.193859 +rank0 [2026-10-29 17:14:50,063] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 17:14:50,063] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 17:14:50,063] INFO : VALIDATION_PROBE | Step=33000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 17:14:50,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:14:50,319] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 17:14:50,323] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 17:15:17,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 17:15:18,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 17:15:19,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 17:15:20,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 17:15:31,006] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 17:15:31,022] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:15:31,568] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 17:15:31,571] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 17:15:47,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 17:15:48,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 17:15:50,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 17:15:51,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 17:16:01,255] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 17:16:01,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 17:16:01,832] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 17:16:01,837] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 17:16:17,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 17:16:18,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 17:16:19,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 17:16:19,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 17:16:29,926] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 17:16:29,942] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits +rank0 [2026-10-29 17:16:33,171] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits +rank0 [2026-10-29 17:16:34,206] INFO : Plotted number of elements +rank0 [2026-10-29 17:16:35,770] INFO : Plotted jets +rank0 [2026-10-29 17:16:36,968] INFO : Plotted jet ratio +rank0 [2026-10-29 17:16:37,058] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 17:16:40,284] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 17:16:41,296] INFO : Plotted number of elements +rank0 [2026-10-29 17:16:42,797] INFO : Plotted jets +rank0 [2026-10-29 17:16:43,904] INFO : Plotted jet ratio +rank0 [2026-10-29 17:16:44,018] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits +rank0 [2026-10-29 17:16:46,443] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits +rank0 [2026-10-29 17:16:47,277] INFO : Plotted number of elements +rank0 [2026-10-29 17:16:49,170] INFO : Plotted jets +rank0 [2026-10-29 17:16:50,342] INFO : Plotted jet ratio +rank0 [2026-10-29 17:21:40,875] INFO : Step 33100/50000 rank0 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4666 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1297 | Classification: 0.0842 | Classification_binary: 0.1537 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1125 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1776 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0703 | Regression_eta: 0.0399 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0556 | LR: 3.31e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7979s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7607 | max_loss: 0.1464 | max_backward: 1.8352 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6888.0 | uss_MB: 5749.0 | child_rss_MB: 6329.7 | allocated_MB: 496.1 | reserved_MB: 24572.0 | inactive_split_MB: 941.9 +rank0 [2026-10-29 17:21:41,114] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 181.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 55.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:26:33,730] INFO : Step 33200/50000 rank0 | Train Loss: 0.9345 | Train Loss Unweighted: 0.4417 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.1121 | Classification: 0.0825 | Classification_binary: 0.1333 | HitGrouping: 0.1549 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0017 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0604 | Regression_eta: 0.0296 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0820 | LR: 3.28e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.9202s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0447 | max_forward: 0.7619 | max_loss: 0.4589 | max_backward: 1.9181 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.4922 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6887.2 | uss_MB: 5750.0 | child_rss_MB: 6329.3 | allocated_MB: 497.6 | reserved_MB: 24578.0 | inactive_split_MB: 835.2 +rank0 [2026-10-29 17:26:33,969] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:31:32,251] INFO : Step 33300/50000 rank0 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1270 | Classification: 0.0780 | Classification_binary: 0.1518 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0985 | Regression_eta: 0.0012 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1310 | Regression_cos_phi: 0.0424 | Regression_energy: 0.0404 | Regression_eta: 0.0245 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0427 | LR: 3.26e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.3769s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0334 | max_forward: 0.7851 | max_loss: 0.1434 | max_backward: 2.0284 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.7975 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5654.9 | uss_MB: 4170.1 | child_rss_MB: 6324.3 | allocated_MB: 496.9 | reserved_MB: 24578.0 | inactive_split_MB: 903.1 +rank0 [2026-10-29 17:31:32,489] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 9% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 73% +2 6 0x7408, 15961 52.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 68% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 38% +4 8 0x7408, 13395 55.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 58% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 89% +6 10 0x7408, 62036 50.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 50% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:36:23,391] INFO : Step 33400/50000 rank0 | Train Loss: 0.9537 | Train Loss Unweighted: 0.4634 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1239 | Classification: 0.0872 | Classification_binary: 0.1563 | HitGrouping: 0.1500 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0013 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1594 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0619 | Regression_eta: 0.0442 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0474 | LR: 3.23e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9663s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7951 | max_loss: 0.2522 | max_backward: 1.9349 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0134 | Memory: rss_MB: 5664.7 | uss_MB: 4533.0 | child_rss_MB: 6325.6 | allocated_MB: 500.7 | reserved_MB: 26578.0 | inactive_split_MB: 843.2 +rank0 [2026-10-29 17:36:23,642] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:41:05,973] INFO : Step 33500/50000 rank0 | Train Loss: 0.9039 | Train Loss Unweighted: 0.4494 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.1057 | Classification: 0.0795 | Classification_binary: 0.1278 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0011 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1232 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0575 | Regression_eta: 0.0350 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0571 | LR: 3.21e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8521s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0177 | max_forward: 0.7696 | max_loss: 0.1421 | max_backward: 1.9122 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 0.5562 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5654.7 | uss_MB: 4519.8 | child_rss_MB: 6324.9 | allocated_MB: 494.9 | reserved_MB: 26578.0 | inactive_split_MB: 839.3 +rank0 [2026-10-29 17:41:06,209] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:45:57,428] INFO : Step 33600/50000 rank0 | Train Loss: 0.9179 | Train Loss Unweighted: 0.4415 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.0997 | Classification: 0.0812 | Classification_binary: 0.1440 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0633 | Regression_eta: 0.0374 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0541 | LR: 3.19e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1198s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7783 | max_loss: 0.2418 | max_backward: 2.0392 | max_optimizer_step: 0.2787 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5667.7 | uss_MB: 4535.9 | child_rss_MB: 6324.1 | allocated_MB: 497.9 | reserved_MB: 28594.0 | inactive_split_MB: 1127.0 +rank0 [2026-10-29 17:45:57,671] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 55.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:50:44,182] INFO : Step 33700/50000 rank0 | Train Loss: 1.0564 | Train Loss Unweighted: 0.4962 | Train Loss Components: Auxiliary: 0.2235 | Cardinality: 0.1517 | Classification: 0.0917 | Classification_binary: 0.1786 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1782 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0572 | Regression_eta: 0.0354 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0547 | LR: 3.16e-04 | DataLoad Time: 0.0138s | Model Forward Time: 4.8750s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0121 | max_forward: 0.7981 | max_loss: 0.1554 | max_backward: 1.9054 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5667.6 | uss_MB: 4535.1 | child_rss_MB: 6300.3 | allocated_MB: 501.8 | reserved_MB: 28594.0 | inactive_split_MB: 1103.2 +rank0 [2026-10-29 17:50:44,422] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 191.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 56.0°C 196.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 17:55:25,308] INFO : Step 33800/50000 rank0 | Train Loss: 0.9305 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1894 | Cardinality: 0.1080 | Classification: 0.0751 | Classification_binary: 0.1369 | HitGrouping: 0.1527 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1201 | Regression_eta: 0.0014 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0243 | Classification_binary: 0.1197 | Regression_cos_phi: 0.0875 | Regression_energy: 0.0654 | Regression_eta: 0.0380 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0477 | LR: 3.14e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.7739s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8033 | max_loss: 0.1441 | max_backward: 1.7197 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0176 | Memory: rss_MB: 5662.9 | uss_MB: 4531.8 | child_rss_MB: 6320.7 | allocated_MB: 496.9 | reserved_MB: 28594.0 | inactive_split_MB: 909.5 +rank0 [2026-10-29 17:55:25,544] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 56.0°C 195.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:00:02,357] INFO : Step 33900/50000 rank0 | Train Loss: 0.9360 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.1942 | Cardinality: 0.1035 | Classification: 0.0805 | Classification_binary: 0.1361 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1124 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1317 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0439 | Regression_eta: 0.0357 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0568 | LR: 3.11e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.0662s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7557 | max_loss: 0.2439 | max_backward: 1.8923 | max_optimizer_step: 0.2623 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5665.6 | uss_MB: 4533.9 | child_rss_MB: 6306.4 | allocated_MB: 496.5 | reserved_MB: 28598.0 | inactive_split_MB: 941.5 +rank0 [2026-10-29 18:00:02,596] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 56.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:04:39,730] INFO : Step 34000/50000 rank0 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0930 | Classification: 0.0875 | Classification_binary: 0.1606 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1639 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0480 | Regression_eta: 0.0259 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0430 | LR: 3.09e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.0238s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0943 | max_forward: 0.8289 | max_loss: 0.4616 | max_backward: 2.0219 | max_optimizer_step: 0.1767 | max_tensorboard_logging: 0.4773 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5666.0 | uss_MB: 4534.2 | child_rss_MB: 6326.4 | allocated_MB: 496.0 | reserved_MB: 28598.0 | inactive_split_MB: 1123.0 +rank0 [2026-10-29 18:04:39,966] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 55.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:04:39,990] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-34000.pth +rank0 [2026-10-29 18:04:40,284] INFO : Running validation on rank0 +rank0 [2026-10-29 18:04:40,284] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 18:04:40,284] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 18:05:31,164] INFO : VALIDATION | Step=34000/50000 | Train Loss=0.9048 | Valid Loss=0.9384 | Train Loss Unweighted=0.4437 | Valid Loss Unweighted=0.4508 | Stale=0 | ETA=92.1m +rank0 [2026-10-29 18:05:31,164] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:05:31,164] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 18:05:31,164] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/ari=0.606474 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.672448 +rank0 [2026-10-29 18:05:31,165] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/top1_competing=0.772464 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.661089 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0573213 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0866143 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 18:05:31,166] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/seconds_per_rank=21.5378 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding/ari=0.297285 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/embedding_minus_raw/ari=0.205662 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 18:05:31,167] INFO : VALIDATION_PROBE | Step=34000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 18:05:31,172] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:05:31,424] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 18:05:31,429] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 18:05:55,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 18:05:56,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 18:05:57,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 18:05:58,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 18:06:08,694] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:06:08,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:06:09,213] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 18:06:09,216] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 18:06:25,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 18:06:26,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 18:06:27,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 18:06:29,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 18:06:39,018] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:06:39,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:06:39,483] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 18:06:39,487] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 18:06:55,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 18:06:56,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 18:06:57,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 18:06:58,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 18:07:08,177] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 18:07:08,236] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits +rank0 [2026-10-29 18:07:11,396] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits +rank0 [2026-10-29 18:07:12,385] INFO : Plotted number of elements +rank0 [2026-10-29 18:07:13,966] INFO : Plotted jets +rank0 [2026-10-29 18:07:15,172] INFO : Plotted jet ratio +rank0 [2026-10-29 18:07:15,324] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 18:07:18,634] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 18:07:19,643] INFO : Plotted number of elements +rank0 [2026-10-29 18:07:21,175] INFO : Plotted jets +rank0 [2026-10-29 18:07:22,303] INFO : Plotted jet ratio +rank0 [2026-10-29 18:07:22,357] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits +rank0 [2026-10-29 18:07:24,828] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits +rank0 [2026-10-29 18:07:25,669] INFO : Plotted number of elements +rank0 [2026-10-29 18:07:27,627] INFO : Plotted jets +rank0 [2026-10-29 18:07:28,810] INFO : Plotted jet ratio +rank0 [2026-10-29 18:12:12,729] INFO : Step 34100/50000 rank0 | Train Loss: 1.0240 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2153 | Cardinality: 0.1202 | Classification: 0.0979 | Classification_binary: 0.1668 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1210 | Regression_eta: 0.0013 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1750 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0588 | Regression_eta: 0.0372 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0489 | LR: 3.06e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1221s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0078 | max_forward: 0.9085 | max_loss: 0.1531 | max_backward: 2.1727 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.4680 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6900.4 | uss_MB: 5761.6 | child_rss_MB: 6326.4 | allocated_MB: 496.4 | reserved_MB: 24560.0 | inactive_split_MB: 1172.2 +rank0 [2026-10-29 18:12:12,971] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 55.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:17:12,287] INFO : Step 34200/50000 rank0 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4225 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.1030 | Classification: 0.0694 | Classification_binary: 0.1279 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0016 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0279 | Classification_binary: 0.1315 | Regression_cos_phi: 0.0406 | Regression_energy: 0.0565 | Regression_eta: 0.0268 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0293 | LR: 3.04e-04 | DataLoad Time: 0.0149s | Model Forward Time: 6.4953s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 1.1401 | max_loss: 1.0887 | max_backward: 3.3565 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5402 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6897.3 | uss_MB: 5759.3 | child_rss_MB: 6328.4 | allocated_MB: 493.1 | reserved_MB: 27668.0 | inactive_split_MB: 842.0 +rank0 [2026-10-29 18:17:12,523] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 55.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:21:54,018] INFO : Step 34300/50000 rank0 | Train Loss: 0.8999 | Train Loss Unweighted: 0.4291 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.1046 | Classification: 0.0782 | Classification_binary: 0.1421 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0015 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1600 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0563 | Regression_eta: 0.0424 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0555 | LR: 3.02e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2350s | Phase Times: max_pre_step_barrier: 0.1089 | max_batch_to_device: 0.0070 | max_forward: 0.7738 | max_loss: 0.4754 | max_backward: 1.8888 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5721.4 | uss_MB: 4584.0 | child_rss_MB: 6315.8 | allocated_MB: 498.9 | reserved_MB: 29668.0 | inactive_split_MB: 777.1 +rank0 [2026-10-29 18:21:54,257] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:26:43,130] INFO : Step 34400/50000 rank0 | Train Loss: 0.9382 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2026 | Cardinality: 0.1181 | Classification: 0.0846 | Classification_binary: 0.1681 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1870 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0527 | Regression_eta: 0.0342 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0451 | LR: 2.99e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.4541s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.8411 | max_loss: 0.2739 | max_backward: 2.1385 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5734.9 | uss_MB: 4598.3 | child_rss_MB: 6315.2 | allocated_MB: 496.8 | reserved_MB: 29668.0 | inactive_split_MB: 858.8 +rank0 [2026-10-29 18:26:43,367] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +4 8 0x7408, 13395 55.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 56% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 93% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:31:41,613] INFO : Step 34500/50000 rank0 | Train Loss: 0.9189 | Train Loss Unweighted: 0.4456 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1063 | Classification: 0.0767 | Classification_binary: 0.1421 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1455 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0518 | Regression_eta: 0.0312 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0637 | LR: 2.97e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0031s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.7430 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5727.7 | uss_MB: 4595.8 | child_rss_MB: 6313.8 | allocated_MB: 494.3 | reserved_MB: 29668.0 | inactive_split_MB: 926.0 +rank0 [2026-10-29 18:31:41,851] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 55.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:36:27,823] INFO : Step 34600/50000 rank0 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4340 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1046 | Classification: 0.0794 | Classification_binary: 0.1340 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0016 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0659 | Regression_eta: 0.0395 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0396 | LR: 2.95e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.3346s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0170 | max_forward: 0.9895 | max_loss: 0.4622 | max_backward: 2.2541 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5731.7 | uss_MB: 4597.0 | child_rss_MB: 6311.8 | allocated_MB: 498.6 | reserved_MB: 29672.0 | inactive_split_MB: 765.1 +rank0 [2026-10-29 18:36:28,064] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +4 8 0x7408, 13395 55.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 0% +6 10 0x7408, 62036 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:40:59,525] INFO : Step 34700/50000 rank0 | Train Loss: 0.9586 | Train Loss Unweighted: 0.4539 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1185 | Classification: 0.0862 | Classification_binary: 0.1624 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0497 | Regression_eta: 0.0368 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0482 | LR: 2.92e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.0664s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7862 | max_loss: 0.2628 | max_backward: 1.9231 | max_optimizer_step: 0.2860 | max_tensorboard_logging: 0.4599 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5730.7 | uss_MB: 4596.4 | child_rss_MB: 6303.0 | allocated_MB: 499.0 | reserved_MB: 29672.0 | inactive_split_MB: 1074.3 +rank0 [2026-10-29 18:40:59,763] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 56.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:45:39,355] INFO : Step 34800/50000 rank0 | Train Loss: 0.9346 | Train Loss Unweighted: 0.4373 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1276 | Classification: 0.0867 | Classification_binary: 0.1551 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0959 | Regression_eta: 0.0013 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1205 | Regression_cos_phi: 0.0329 | Regression_energy: 0.0535 | Regression_eta: 0.0223 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0468 | LR: 2.90e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9191s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0770 | max_forward: 0.8404 | max_loss: 0.1556 | max_backward: 1.8035 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.5351 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5727.0 | uss_MB: 4594.2 | child_rss_MB: 6312.4 | allocated_MB: 497.4 | reserved_MB: 29672.0 | inactive_split_MB: 1113.6 +rank0 [2026-10-29 18:45:39,593] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 56.0°C 193.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:50:18,788] INFO : Step 34900/50000 rank0 | Train Loss: 0.8719 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1154 | Classification: 0.0681 | Classification_binary: 0.1220 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1093 | Regression_eta: 0.0013 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0572 | Regression_eta: 0.0376 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0537 | LR: 2.88e-04 | DataLoad Time: 0.0162s | Model Forward Time: 7.0030s | Phase Times: max_pre_step_barrier: 0.0080 | max_batch_to_device: 0.0693 | max_forward: 1.2127 | max_loss: 0.2834 | max_backward: 3.0996 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.6125 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 5724.6 | uss_MB: 4576.6 | child_rss_MB: 6292.2 | allocated_MB: 495.3 | reserved_MB: 29672.0 | inactive_split_MB: 864.7 +rank0 [2026-10-29 18:50:19,028] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 2% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 52.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 48% 0% +4 8 0x7408, 13395 55.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 78% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 0% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:55:19,885] INFO : Step 35000/50000 rank0 | Train Loss: 0.9157 | Train Loss Unweighted: 0.4220 | Train Loss Components: Auxiliary: 0.1897 | Cardinality: 0.1230 | Classification: 0.0771 | Classification_binary: 0.1455 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0980 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1978 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0532 | Regression_eta: 0.0461 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0735 | LR: 2.85e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.2020s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.8243 | max_loss: 0.4759 | max_backward: 2.1181 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5728.2 | uss_MB: 4598.3 | child_rss_MB: 6307.6 | allocated_MB: 498.3 | reserved_MB: 29672.0 | inactive_split_MB: 766.0 +rank0 [2026-10-29 18:55:20,125] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 54.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 18:55:20,150] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-35000.pth +rank0 [2026-10-29 18:55:20,575] INFO : Running validation on rank0 +rank0 [2026-10-29 18:55:20,575] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 18:55:20,575] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 18:56:12,136] INFO : VALIDATION | Step=35000/50000 | Train Loss=0.9157 | Valid Loss=0.9573 | Train Loss Unweighted=0.4220 | Valid Loss Unweighted=0.4600 | Stale=1 | ETA=105.6m +rank0 [2026-10-29 18:56:12,136] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:56:12,136] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 18:56:12,136] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 18:56:12,136] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:56:12,136] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 18:56:12,136] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 18:56:12,136] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/ari=0.602533 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.674842 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/top1_competing=0.775572 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.647454 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0533807 +rank0 [2026-10-29 18:56:12,137] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0729787 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 18:56:12,138] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/seconds_per_rank=21.4938 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding/ari=0.277772 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/embedding_minus_raw/ari=0.186149 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 18:56:12,139] INFO : VALIDATION_PROBE | Step=35000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 18:56:12,144] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:56:12,399] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 18:56:12,403] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 18:56:36,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 18:56:37,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 18:56:38,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 18:56:39,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 18:56:49,800] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:56:49,875] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:56:50,324] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 18:56:50,328] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 18:57:06,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 18:57:07,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 18:57:09,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 18:57:10,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 18:57:20,345] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 18:57:20,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 18:57:20,903] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 18:57:20,907] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 18:57:36,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 18:57:37,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 18:57:38,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 18:57:39,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 18:57:49,488] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 18:57:49,612] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits +rank0 [2026-10-29 18:57:52,812] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits +rank0 [2026-10-29 18:57:53,789] INFO : Plotted number of elements +rank0 [2026-10-29 18:57:55,706] INFO : Plotted jets +rank0 [2026-10-29 18:57:56,877] INFO : Plotted jet ratio +rank0 [2026-10-29 18:57:56,944] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 18:58:00,177] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 18:58:01,168] INFO : Plotted number of elements +rank0 [2026-10-29 18:58:02,685] INFO : Plotted jets +rank0 [2026-10-29 18:58:03,794] INFO : Plotted jet ratio +rank0 [2026-10-29 18:58:03,851] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits +rank0 [2026-10-29 18:58:06,301] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits +rank0 [2026-10-29 18:58:07,131] INFO : Plotted number of elements +rank0 [2026-10-29 18:58:08,622] INFO : Plotted jets +rank0 [2026-10-29 18:58:10,206] INFO : Plotted jet ratio +rank0 [2026-10-29 19:02:57,291] INFO : Step 35100/50000 rank0 | Train Loss: 0.9622 | Train Loss Unweighted: 0.4512 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1150 | Classification: 0.0857 | Classification_binary: 0.1724 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0948 | Regression_eta: 0.0014 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0406 | Regression_eta: 0.0470 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0586 | LR: 2.83e-04 | DataLoad Time: 0.0142s | Model Forward Time: 5.4993s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1018 | max_forward: 0.8921 | max_loss: 0.1518 | max_backward: 2.0328 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8633 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5815.2 | uss_MB: 4686.8 | child_rss_MB: 6305.0 | allocated_MB: 499.4 | reserved_MB: 23366.0 | inactive_split_MB: 1243.8 +rank0 [2026-10-29 19:02:57,529] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:07:43,921] INFO : Step 35200/50000 rank0 | Train Loss: 0.9331 | Train Loss Unweighted: 0.4385 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.1234 | Classification: 0.0814 | Classification_binary: 0.1441 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.1226 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0479 | Regression_eta: 0.0347 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0506 | LR: 2.81e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7898s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0577 | max_forward: 0.7693 | max_loss: 0.1423 | max_backward: 1.7209 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5822.3 | uss_MB: 4693.9 | child_rss_MB: 6304.7 | allocated_MB: 494.4 | reserved_MB: 25346.0 | inactive_split_MB: 1054.2 +rank0 [2026-10-29 19:07:44,158] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 55.0°C 196.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:12:31,332] INFO : Step 35300/50000 rank0 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4075 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1105 | Classification: 0.0819 | Classification_binary: 0.1245 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0989 | Regression_eta: 0.0014 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0510 | Regression_eta: 0.0340 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0748 | LR: 2.79e-04 | DataLoad Time: 0.0170s | Model Forward Time: 7.4768s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 1.2856 | max_loss: 0.4079 | max_backward: 4.4969 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5749 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5814.5 | uss_MB: 4685.2 | child_rss_MB: 6307.8 | allocated_MB: 498.3 | reserved_MB: 25346.0 | inactive_split_MB: 1140.5 +rank0 [2026-10-29 19:12:31,582] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:17:15,475] INFO : Step 35400/50000 rank0 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4888 | Train Loss Components: Auxiliary: 0.2130 | Cardinality: 0.1296 | Classification: 0.0908 | Classification_binary: 0.1718 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0014 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0581 | Regression_eta: 0.0370 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0566 | LR: 2.76e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0092s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8312 | max_loss: 0.1490 | max_backward: 2.0287 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.4708 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5820.8 | uss_MB: 4696.7 | child_rss_MB: 6308.5 | allocated_MB: 497.8 | reserved_MB: 25346.0 | inactive_split_MB: 1132.0 +rank0 [2026-10-29 19:17:15,713] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +2 6 0x7408, 15961 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 192.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:21:58,981] INFO : Step 35500/50000 rank0 | Train Loss: 0.9444 | Train Loss Unweighted: 0.4557 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1201 | Classification: 0.0924 | Classification_binary: 0.1605 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0873 | Regression_energy: 0.0772 | Regression_eta: 0.0305 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0505 | LR: 2.74e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.0912s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7670 | max_loss: 0.2318 | max_backward: 1.8405 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5820.9 | uss_MB: 4694.9 | child_rss_MB: 6312.3 | allocated_MB: 497.5 | reserved_MB: 26942.0 | inactive_split_MB: 1130.5 +rank0 [2026-10-29 19:21:59,219] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:26:42,710] INFO : Step 35600/50000 rank0 | Train Loss: 0.9280 | Train Loss Unweighted: 0.4451 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1095 | Classification: 0.0861 | Classification_binary: 0.1334 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0840 | Classification_binary: 0.1440 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0734 | Regression_eta: 0.0806 | Regression_pt: 0.0815 | Regression_sin_phi: 0.1307 | LR: 2.72e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.5096s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7460 | max_loss: 0.1440 | max_backward: 1.8206 | max_optimizer_step: 0.2911 | max_tensorboard_logging: 0.6036 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 5820.5 | uss_MB: 3919.7 | child_rss_MB: 6316.0 | allocated_MB: 496.1 | reserved_MB: 26942.0 | inactive_split_MB: 907.6 +rank0 [2026-10-29 19:26:42,955] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +2 6 0x7408, 15961 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 56.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 93% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 2% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:31:30,375] INFO : Step 35700/50000 rank0 | Train Loss: 0.9116 | Train Loss Unweighted: 0.4301 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1148 | Classification: 0.0829 | Classification_binary: 0.1432 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0998 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.1894 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0689 | Regression_eta: 0.0550 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0750 | LR: 2.70e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.3409s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8312 | max_loss: 0.2568 | max_backward: 2.2622 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5823.2 | uss_MB: 4699.5 | child_rss_MB: 6323.6 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1241.0 +rank0 [2026-10-29 19:31:30,615] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 99% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:36:12,524] INFO : Step 35800/50000 rank0 | Train Loss: 1.0095 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1252 | Classification: 0.0857 | Classification_binary: 0.1786 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1066 | Regression_eta: 0.0014 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0476 | Regression_eta: 0.0361 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0526 | LR: 2.68e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8019s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7644 | max_loss: 0.5090 | max_backward: 1.8855 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4861 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5816.8 | uss_MB: 4694.8 | child_rss_MB: 6319.7 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1129.0 +rank0 [2026-10-29 19:36:12,764] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 187.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 53.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 193.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 52.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:40:49,455] INFO : Step 35900/50000 rank0 | Train Loss: 0.9596 | Train Loss Unweighted: 0.4571 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1173 | Classification: 0.0913 | Classification_binary: 0.1468 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0014 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.1280 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0537 | Regression_eta: 0.0324 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0741 | LR: 2.65e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7247s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7217 | max_loss: 0.1443 | max_backward: 1.6894 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.4586 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5816.9 | uss_MB: 4695.3 | child_rss_MB: 6319.9 | allocated_MB: 495.4 | reserved_MB: 29024.0 | inactive_split_MB: 934.6 +rank0 [2026-10-29 19:40:49,703] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 53.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 195.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 52.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:45:19,740] INFO : Step 36000/50000 rank0 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0912 | Classification: 0.0938 | Classification_binary: 0.1314 | HitGrouping: 0.1587 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1135 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1240 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0487 | Regression_eta: 0.0302 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0342 | LR: 2.63e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9475s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0073 | max_forward: 0.7630 | max_loss: 0.4769 | max_backward: 1.8869 | max_optimizer_step: 0.2587 | max_tensorboard_logging: 0.5842 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5815.2 | uss_MB: 4695.9 | child_rss_MB: 6324.3 | allocated_MB: 495.0 | reserved_MB: 29024.0 | inactive_split_MB: 937.5 +rank0 [2026-10-29 19:45:19,979] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 39% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +2 6 0x7408, 15961 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:45:20,002] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-36000.pth +rank0 [2026-10-29 19:45:20,398] INFO : Running validation on rank0 +rank0 [2026-10-29 19:45:20,398] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 19:45:20,398] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 19:46:11,123] INFO : VALIDATION | Step=36000/50000 | Train Loss=0.9231 | Valid Loss=0.9339 | Train Loss Unweighted=0.4578 | Valid Loss Unweighted=0.4458 | Stale=0 | ETA=115.3m +rank0 [2026-10-29 19:46:11,123] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 19:46:11,123] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 19:46:11,124] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 19:46:11,124] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 19:46:11,124] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 19:46:11,124] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 19:46:11,124] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 19:46:11,124] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 19:46:11,124] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 19:46:11,124] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 19:46:11,124] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 19:46:11,124] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/ari=0.6029 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.700462 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/top1_competing=0.806811 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.674852 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0537471 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.100377 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 19:46:11,125] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/seconds_per_rank=21.3704 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 19:46:11,126] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 19:46:11,127] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding/ari=0.295615 +rank0 [2026-10-29 19:46:11,127] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 19:46:11,127] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/embedding_minus_raw/ari=0.203992 +rank0 [2026-10-29 19:46:11,127] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 19:46:11,127] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 19:46:11,127] INFO : VALIDATION_PROBE | Step=36000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 19:46:11,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:46:11,389] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 19:46:11,393] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 19:46:35,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 19:46:37,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 19:46:38,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 19:46:39,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 19:46:49,441] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 19:46:49,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:46:49,932] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 19:46:49,936] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 19:47:06,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 19:47:07,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 19:47:08,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 19:47:09,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 19:47:19,418] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 19:47:19,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 19:47:20,161] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 19:47:20,164] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 19:47:35,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 19:47:36,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 19:47:37,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 19:47:38,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 19:47:48,725] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 19:47:48,836] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits +rank0 [2026-10-29 19:47:52,052] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits +rank0 [2026-10-29 19:47:53,101] INFO : Plotted number of elements +rank0 [2026-10-29 19:47:54,685] INFO : Plotted jets +rank0 [2026-10-29 19:47:55,929] INFO : Plotted jet ratio +rank0 [2026-10-29 19:47:56,021] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 19:47:59,266] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 19:48:00,324] INFO : Plotted number of elements +rank0 [2026-10-29 19:48:01,861] INFO : Plotted jets +rank0 [2026-10-29 19:48:03,003] INFO : Plotted jet ratio +rank0 [2026-10-29 19:48:03,078] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits +rank0 [2026-10-29 19:48:05,562] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits +rank0 [2026-10-29 19:48:06,427] INFO : Plotted number of elements +rank0 [2026-10-29 19:48:08,367] INFO : Plotted jets +rank0 [2026-10-29 19:48:09,529] INFO : Plotted jet ratio +rank0 [2026-10-29 19:53:12,654] INFO : Step 36100/50000 rank0 | Train Loss: 0.9292 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1100 | Classification: 0.0906 | Classification_binary: 0.1490 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0548 | Regression_eta: 0.0286 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0562 | LR: 2.61e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.4306s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7768 | max_loss: 0.2455 | max_backward: 1.8077 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 1.0653 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5880.7 | uss_MB: 4756.1 | child_rss_MB: 6293.3 | allocated_MB: 499.2 | reserved_MB: 26012.0 | inactive_split_MB: 1080.2 +rank0 [2026-10-29 19:53:12,892] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 31% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 19:58:22,131] INFO : Step 36200/50000 rank0 | Train Loss: 0.9030 | Train Loss Unweighted: 0.4308 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1163 | Classification: 0.0762 | Classification_binary: 0.1596 | HitGrouping: 0.1506 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0530 | Regression_eta: 0.0288 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0614 | LR: 2.59e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1955s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0777 | max_forward: 0.8125 | max_loss: 0.1541 | max_backward: 1.8492 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 5884.8 | uss_MB: 4767.2 | child_rss_MB: 6290.6 | allocated_MB: 494.4 | reserved_MB: 26016.0 | inactive_split_MB: 1377.4 +rank0 [2026-10-29 19:58:22,382] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:03:21,355] INFO : Step 36300/50000 rank0 | Train Loss: 0.9576 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1236 | Classification: 0.0876 | Classification_binary: 0.1347 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1100 | Regression_eta: 0.0017 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0689 | Regression_eta: 0.0439 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0701 | LR: 2.57e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.1484s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.8065 | max_loss: 0.1484 | max_backward: 1.9315 | max_optimizer_step: 0.1930 | max_tensorboard_logging: 0.6801 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5878.7 | uss_MB: 4768.0 | child_rss_MB: 6273.7 | allocated_MB: 496.0 | reserved_MB: 28008.0 | inactive_split_MB: 1156.5 +rank0 [2026-10-29 20:03:21,595] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 53.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:08:11,542] INFO : Step 36400/50000 rank0 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4147 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.0999 | Classification: 0.0796 | Classification_binary: 0.1372 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0513 | Regression_eta: 0.0229 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0430 | LR: 2.55e-04 | DataLoad Time: 0.0211s | Model Forward Time: 6.5467s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0192 | max_forward: 1.1829 | max_loss: 0.3790 | max_backward: 3.0917 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 5880.4 | uss_MB: 4768.9 | child_rss_MB: 6270.8 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 845.2 +rank0 [2026-10-29 20:08:11,783] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 50.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:12:42,695] INFO : Step 36500/50000 rank0 | Train Loss: 0.9762 | Train Loss Unweighted: 0.4543 | Train Loss Components: Auxiliary: 0.2088 | Cardinality: 0.1239 | Classification: 0.0805 | Classification_binary: 0.1682 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0653 | Regression_eta: 0.0373 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0768 | LR: 2.52e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0383s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7782 | max_loss: 0.1554 | max_backward: 1.9378 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.5103 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5877.6 | uss_MB: 4769.9 | child_rss_MB: 6273.0 | allocated_MB: 499.0 | reserved_MB: 30008.0 | inactive_split_MB: 913.8 +rank0 [2026-10-29 20:12:42,932] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:17:47,861] INFO : Step 36600/50000 rank0 | Train Loss: 0.8699 | Train Loss Unweighted: 0.4190 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.1138 | Classification: 0.0734 | Classification_binary: 0.1317 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0010 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1268 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0575 | Regression_eta: 0.0361 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0576 | LR: 2.50e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.9289s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7561 | max_loss: 0.1454 | max_backward: 1.9017 | max_optimizer_step: 0.5115 | max_tensorboard_logging: 0.5672 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5872.6 | uss_MB: 4766.8 | child_rss_MB: 6277.6 | allocated_MB: 494.4 | reserved_MB: 30008.0 | inactive_split_MB: 856.0 +rank0 [2026-10-29 20:17:48,101] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:22:57,133] INFO : Step 36700/50000 rank0 | Train Loss: 0.9065 | Train Loss Unweighted: 0.4327 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.1102 | Classification: 0.0849 | Classification_binary: 0.1262 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1087 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0566 | Regression_eta: 0.0314 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0448 | LR: 2.48e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7601s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.7169 | max_loss: 0.1507 | max_backward: 1.7104 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5871.4 | uss_MB: 4766.4 | child_rss_MB: 6270.7 | allocated_MB: 496.5 | reserved_MB: 30008.0 | inactive_split_MB: 1154.6 +rank0 [2026-10-29 20:22:57,372] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 193.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:28:04,717] INFO : Step 36800/50000 rank0 | Train Loss: 0.9068 | Train Loss Unweighted: 0.4227 | Train Loss Components: Auxiliary: 0.1925 | Cardinality: 0.1100 | Classification: 0.0739 | Classification_binary: 0.1571 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0945 | Regression_eta: 0.0013 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1796 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0533 | Regression_eta: 0.0294 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0732 | LR: 2.46e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.1344s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8487 | max_loss: 0.1525 | max_backward: 2.1121 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5877.1 | uss_MB: 4770.1 | child_rss_MB: 6274.6 | allocated_MB: 499.3 | reserved_MB: 30008.0 | inactive_split_MB: 939.6 +rank0 [2026-10-29 20:28:04,958] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 51.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:33:10,373] INFO : Step 36900/50000 rank0 | Train Loss: 0.8934 | Train Loss Unweighted: 0.4126 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1194 | Classification: 0.0721 | Classification_binary: 0.1507 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0013 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0410 | Regression_eta: 0.0379 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0395 | LR: 2.44e-04 | DataLoad Time: 0.0148s | Model Forward Time: 6.3981s | Phase Times: max_pre_step_barrier: 0.0139 | max_batch_to_device: 0.0068 | max_forward: 1.0975 | max_loss: 0.2241 | max_backward: 2.7278 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.6216 | max_tensorboard_flush: 0.0085 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5871.7 | uss_MB: 4750.7 | child_rss_MB: 6260.8 | allocated_MB: 498.9 | reserved_MB: 30008.0 | inactive_split_MB: 771.1 +rank0 [2026-10-29 20:33:10,615] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 8% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 51.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +4 8 0x7408, 13395 55.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 12% +6 10 0x7408, 62036 50.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 93% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:38:06,688] INFO : Step 37000/50000 rank0 | Train Loss: 0.9040 | Train Loss Unweighted: 0.4182 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1192 | Classification: 0.0807 | Classification_binary: 0.1318 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1217 | Regression_cos_phi: 0.0370 | Regression_energy: 0.0489 | Regression_eta: 0.0231 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0527 | LR: 2.42e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5784s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7055 | max_loss: 0.1505 | max_backward: 1.5664 | max_optimizer_step: 0.1900 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5866.9 | uss_MB: 4763.2 | child_rss_MB: 6273.0 | allocated_MB: 493.9 | reserved_MB: 30008.0 | inactive_split_MB: 1169.8 +rank0 [2026-10-29 20:38:06,929] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 55.0°C 192.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 51.0°C 192.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:38:06,953] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-37000.pth +rank0 [2026-10-29 20:38:07,242] INFO : Running validation on rank0 +rank0 [2026-10-29 20:38:07,242] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 20:38:07,242] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 20:39:06,518] INFO : VALIDATION | Step=37000/50000 | Train Loss=0.9040 | Valid Loss=0.9336 | Train Loss Unweighted=0.4182 | Valid Loss Unweighted=0.4421 | Stale=0 | ETA=122.7m +rank0 [2026-10-29 20:39:06,518] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 20:39:06,519] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/ari=0.598293 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.693147 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/top1_competing=0.797814 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.669026 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0491402 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0945516 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 20:39:06,520] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/seconds_per_rank=21.5355 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding/ari=0.279987 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/embedding_minus_raw/ari=0.188364 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 20:39:06,521] INFO : VALIDATION_PROBE | Step=37000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 20:39:06,532] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:39:06,789] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 20:39:06,792] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 20:39:31,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 20:39:32,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 20:39:33,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 20:39:35,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 20:39:45,027] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 20:39:45,042] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:39:45,533] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 20:39:45,536] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 20:40:02,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 20:40:03,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 20:40:04,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 20:40:05,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 20:40:15,993] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 20:40:16,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 20:40:16,608] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 20:40:16,611] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 20:40:31,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 20:40:33,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 20:40:34,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 20:40:35,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 20:40:45,591] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 20:40:45,611] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits +rank0 [2026-10-29 20:40:48,882] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits +rank0 [2026-10-29 20:40:49,956] INFO : Plotted number of elements +rank0 [2026-10-29 20:40:51,614] INFO : Plotted jets +rank0 [2026-10-29 20:40:52,850] INFO : Plotted jet ratio +rank0 [2026-10-29 20:40:52,904] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 20:40:56,277] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 20:40:57,270] INFO : Plotted number of elements +rank0 [2026-10-29 20:40:58,804] INFO : Plotted jets +rank0 [2026-10-29 20:40:59,952] INFO : Plotted jet ratio +rank0 [2026-10-29 20:41:00,045] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits +rank0 [2026-10-29 20:41:02,565] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits +rank0 [2026-10-29 20:41:03,428] INFO : Plotted number of elements +rank0 [2026-10-29 20:41:04,944] INFO : Plotted jets +rank0 [2026-10-29 20:41:06,112] INFO : Plotted jet ratio +rank0 [2026-10-29 20:45:59,815] INFO : Step 37100/50000 rank0 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4614 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1255 | Classification: 0.0999 | Classification_binary: 0.1492 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1041 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0511 | Regression_eta: 0.0357 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0462 | LR: 2.40e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.4830s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0061 | max_forward: 0.8553 | max_loss: 0.1461 | max_backward: 2.1723 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5959.4 | uss_MB: 4852.0 | child_rss_MB: 6311.9 | allocated_MB: 499.3 | reserved_MB: 25668.0 | inactive_split_MB: 852.4 +rank0 [2026-10-29 20:46:00,054] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 28% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +4 8 0x7408, 13395 54.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:50:40,215] INFO : Step 37200/50000 rank0 | Train Loss: 0.9288 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.1928 | Cardinality: 0.1108 | Classification: 0.0836 | Classification_binary: 0.1643 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0015 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1898 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0600 | Regression_eta: 0.0380 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0656 | LR: 2.38e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.2766s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8460 | max_loss: 0.2603 | max_backward: 2.0959 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5966.0 | uss_MB: 4861.5 | child_rss_MB: 6312.1 | allocated_MB: 497.9 | reserved_MB: 25674.0 | inactive_split_MB: 846.0 +rank0 [2026-10-29 20:50:40,453] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 20:55:12,335] INFO : Step 37300/50000 rank0 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1275 | Classification: 0.0781 | Classification_binary: 0.1424 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0982 | Regression_eta: 0.0013 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0571 | Regression_eta: 0.0480 | Regression_pt: 0.0615 | Regression_sin_phi: 0.1137 | LR: 2.36e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7686s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7653 | max_loss: 0.1472 | max_backward: 1.7105 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.4778 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0692 | Memory: rss_MB: 5962.2 | uss_MB: 4859.4 | child_rss_MB: 6302.2 | allocated_MB: 497.3 | reserved_MB: 27616.0 | inactive_split_MB: 974.4 +rank0 [2026-10-29 20:55:12,574] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 197.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:00:10,588] INFO : Step 37400/50000 rank0 | Train Loss: 0.8836 | Train Loss Unweighted: 0.4228 | Train Loss Components: Auxiliary: 0.1825 | Cardinality: 0.0983 | Classification: 0.0851 | Classification_binary: 0.1289 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1025 | Regression_eta: 0.0015 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.1364 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0533 | Regression_eta: 0.0430 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0525 | LR: 2.34e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.1788s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7611 | max_loss: 0.1956 | max_backward: 1.9276 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7505 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5960.0 | uss_MB: 4860.1 | child_rss_MB: 6301.8 | allocated_MB: 495.0 | reserved_MB: 27690.0 | inactive_split_MB: 915.5 +rank0 [2026-10-29 21:00:10,824] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 55.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:05:02,520] INFO : Step 37500/50000 rank0 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1260 | Classification: 0.0804 | Classification_binary: 0.1577 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1005 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1457 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0480 | Regression_eta: 0.0320 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0568 | LR: 2.32e-04 | DataLoad Time: 0.0144s | Model Forward Time: 5.2025s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.9030 | max_loss: 0.1494 | max_backward: 1.9871 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5938.4 | uss_MB: 4859.4 | child_rss_MB: 6224.4 | allocated_MB: 496.1 | reserved_MB: 27690.0 | inactive_split_MB: 923.5 +rank0 [2026-10-29 21:05:02,759] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 55.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:09:54,823] INFO : Step 37600/50000 rank0 | Train Loss: 0.9091 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.0995 | Classification: 0.0683 | Classification_binary: 0.1534 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0957 | Regression_eta: 0.0015 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1375 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0627 | Regression_eta: 0.0597 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0694 | LR: 2.30e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9739s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0070 | max_forward: 0.8280 | max_loss: 0.1484 | max_backward: 1.8328 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5933.4 | uss_MB: 4856.7 | child_rss_MB: 6230.2 | allocated_MB: 497.1 | reserved_MB: 27690.0 | inactive_split_MB: 927.5 +rank0 [2026-10-29 21:09:55,064] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 55.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:14:46,671] INFO : Step 37700/50000 rank0 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3995 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.0864 | Classification: 0.0813 | Classification_binary: 0.1231 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0012 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1214 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0490 | Regression_eta: 0.0340 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0418 | LR: 2.28e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1706s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7582 | max_loss: 0.2486 | max_backward: 1.9197 | max_optimizer_step: 0.3005 | max_tensorboard_logging: 0.5767 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5802.8 | uss_MB: 4857.6 | child_rss_MB: 5767.0 | allocated_MB: 495.2 | reserved_MB: 29766.0 | inactive_split_MB: 925.5 +rank0 [2026-10-29 21:14:46,907] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 55.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:19:50,784] INFO : Step 37800/50000 rank0 | Train Loss: 0.8717 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1057 | Classification: 0.0822 | Classification_binary: 0.1363 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0014 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1326 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0627 | Regression_eta: 0.0339 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0508 | LR: 2.26e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.2555s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 1.2105 | max_loss: 0.2449 | max_backward: 2.0574 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 5800.5 | uss_MB: 4856.8 | child_rss_MB: 5763.9 | allocated_MB: 498.3 | reserved_MB: 29766.0 | inactive_split_MB: 930.0 +rank0 [2026-10-29 21:19:51,041] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 55.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:24:49,999] INFO : Step 37900/50000 rank0 | Train Loss: 0.9843 | Train Loss Unweighted: 0.4684 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1353 | Classification: 0.0912 | Classification_binary: 0.1705 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0014 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1934 | Regression_cos_phi: 0.0566 | Regression_energy: 0.0483 | Regression_eta: 0.0306 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0585 | LR: 2.24e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.9431s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0297 | max_forward: 0.8081 | max_loss: 0.1547 | max_backward: 2.6708 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5096 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5782.9 | uss_MB: 4859.7 | child_rss_MB: 5732.7 | allocated_MB: 495.6 | reserved_MB: 29766.0 | inactive_split_MB: 912.0 +rank0 [2026-10-29 21:24:50,238] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 54.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:29:45,187] INFO : Step 38000/50000 rank0 | Train Loss: 0.8556 | Train Loss Unweighted: 0.4002 | Train Loss Components: Auxiliary: 0.1833 | Cardinality: 0.1090 | Classification: 0.0730 | Classification_binary: 0.1293 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0972 | Regression_eta: 0.0011 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0322 | Classification_binary: 0.1084 | Regression_cos_phi: 0.0441 | Regression_energy: 0.0520 | Regression_eta: 0.0290 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0524 | LR: 2.22e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8216s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7330 | max_loss: 0.2284 | max_backward: 1.8082 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5778.8 | uss_MB: 4857.4 | child_rss_MB: 5727.0 | allocated_MB: 494.5 | reserved_MB: 29766.0 | inactive_split_MB: 847.3 +rank0 [2026-10-29 21:29:45,439] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 45% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 55.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:29:45,462] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-38000.pth +rank0 [2026-10-29 21:29:45,790] INFO : Running validation on rank0 +rank0 [2026-10-29 21:29:45,791] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 21:29:45,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 21:30:42,134] INFO : VALIDATION | Step=38000/50000 | Train Loss=0.8556 | Valid Loss=0.9173 | Train Loss Unweighted=0.4002 | Valid Loss Unweighted=0.4324 | Stale=0 | ETA=126.6m +rank0 [2026-10-29 21:30:42,134] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:30:42,134] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 21:30:42,134] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 21:30:42,134] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:30:42,134] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 21:30:42,134] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/ari=0.574862 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/ari_threshold=0.8125 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.695934 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/top1_competing=0.800249 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.661998 +rank0 [2026-10-29 21:30:42,135] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0257091 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0875232 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 21:30:42,136] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/seconds_per_rank=21.5528 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding/ari=0.297938 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/embedding_minus_raw/ari=0.206315 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 21:30:42,137] INFO : VALIDATION_PROBE | Step=38000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 21:30:42,142] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:30:42,391] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 21:30:42,398] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 21:31:09,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 21:31:10,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 21:31:11,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 21:31:13,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 21:31:23,181] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 21:31:23,194] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:31:23,629] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 21:31:23,632] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 21:31:39,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 21:31:40,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 21:31:41,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 21:31:42,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 21:31:52,722] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 21:31:52,733] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 21:31:53,235] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 21:31:53,238] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 21:32:08,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 21:32:09,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 21:32:10,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 21:32:11,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 21:32:21,725] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 21:32:21,741] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits +rank0 [2026-10-29 21:32:24,969] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits +rank0 [2026-10-29 21:32:26,027] INFO : Plotted number of elements +rank0 [2026-10-29 21:32:27,613] INFO : Plotted jets +rank0 [2026-10-29 21:32:28,804] INFO : Plotted jet ratio +rank0 [2026-10-29 21:32:28,898] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 21:32:32,132] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 21:32:33,157] INFO : Plotted number of elements +rank0 [2026-10-29 21:32:34,686] INFO : Plotted jets +rank0 [2026-10-29 21:32:36,295] INFO : Plotted jet ratio +rank0 [2026-10-29 21:32:36,376] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits +rank0 [2026-10-29 21:32:38,787] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits +rank0 [2026-10-29 21:32:39,654] INFO : Plotted number of elements +rank0 [2026-10-29 21:32:41,232] INFO : Plotted jets +rank0 [2026-10-29 21:32:42,441] INFO : Plotted jet ratio +rank0 [2026-10-29 21:37:40,019] INFO : Step 38100/50000 rank0 | Train Loss: 0.9020 | Train Loss Unweighted: 0.4337 | Train Loss Components: Auxiliary: 0.1907 | Cardinality: 0.1231 | Classification: 0.0888 | Classification_binary: 0.1339 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0014 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1199 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0573 | Regression_eta: 0.0749 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0674 | LR: 2.20e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.8207s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0963 | max_forward: 0.8445 | max_loss: 0.1406 | max_backward: 1.9197 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.9697 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7028.7 | uss_MB: 6098.3 | child_rss_MB: 5734.5 | allocated_MB: 496.0 | reserved_MB: 27516.0 | inactive_split_MB: 970.5 +rank0 [2026-10-29 21:37:40,260] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:42:38,396] INFO : Step 38200/50000 rank0 | Train Loss: 0.9174 | Train Loss Unweighted: 0.4351 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.0971 | Classification: 0.0888 | Classification_binary: 0.1526 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0948 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1645 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0443 | Regression_eta: 0.0475 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0715 | LR: 2.18e-04 | DataLoad Time: 0.0165s | Model Forward Time: 7.1540s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.8480 | max_loss: 0.2651 | max_backward: 3.1905 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.9151 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5881.2 | uss_MB: 4342.7 | child_rss_MB: 5733.1 | allocated_MB: 499.7 | reserved_MB: 27516.0 | inactive_split_MB: 848.5 +rank0 [2026-10-29 21:42:38,635] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 148.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 7% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 98% +2 6 0x7408, 15961 51.0°C 142.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +4 8 0x7408, 13395 54.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 24% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 32% 0% +6 10 0x7408, 62036 50.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 74% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:47:31,238] INFO : Step 38300/50000 rank0 | Train Loss: 0.9342 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1998 | Cardinality: 0.1099 | Classification: 0.0869 | Classification_binary: 0.1470 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0013 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0638 | Regression_eta: 0.0468 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0695 | LR: 2.16e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.4917s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0664 | max_forward: 0.8187 | max_loss: 0.1464 | max_backward: 1.8154 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.9517 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5875.2 | uss_MB: 4949.5 | child_rss_MB: 5725.9 | allocated_MB: 495.5 | reserved_MB: 27516.0 | inactive_split_MB: 1085.5 +rank0 [2026-10-29 21:47:31,476] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 21% +4 8 0x7408, 13395 55.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 58% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 31% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:52:15,083] INFO : Step 38400/50000 rank0 | Train Loss: 0.8281 | Train Loss Unweighted: 0.4036 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0915 | Classification: 0.0723 | Classification_binary: 0.1160 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1069 | Regression_eta: 0.0012 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0595 | Regression_eta: 0.0429 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0567 | LR: 2.14e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.4140s | Phase Times: max_pre_step_barrier: 0.0868 | max_batch_to_device: 0.0062 | max_forward: 0.9937 | max_loss: 0.2500 | max_backward: 1.9690 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5872.2 | uss_MB: 4950.8 | child_rss_MB: 5730.9 | allocated_MB: 496.1 | reserved_MB: 27516.0 | inactive_split_MB: 972.6 +rank0 [2026-10-29 21:52:15,325] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 55.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 21:57:10,050] INFO : Step 38500/50000 rank0 | Train Loss: 0.9386 | Train Loss Unweighted: 0.4484 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1248 | Classification: 0.1044 | Classification_binary: 0.1389 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0467 | Regression_eta: 0.0364 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0498 | LR: 2.12e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9354s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.7200 | max_loss: 0.1386 | max_backward: 1.5323 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5868.3 | uss_MB: 4952.0 | child_rss_MB: 5707.9 | allocated_MB: 496.0 | reserved_MB: 27520.0 | inactive_split_MB: 776.0 +rank0 [2026-10-29 21:57:13,304] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 48.0°C 131.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 50.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 47.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:01:44,240] INFO : Step 38600/50000 rank0 | Train Loss: 0.9227 | Train Loss Unweighted: 0.4436 | Train Loss Components: Auxiliary: 0.1984 | Cardinality: 0.1056 | Classification: 0.0788 | Classification_binary: 0.1660 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0979 | Regression_eta: 0.0013 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1395 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0494 | Regression_eta: 0.0377 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0421 | LR: 2.11e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.0625s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8785 | max_loss: 0.1473 | max_backward: 2.0276 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5861.9 | uss_MB: 4951.4 | child_rss_MB: 5677.1 | allocated_MB: 499.0 | reserved_MB: 29592.0 | inactive_split_MB: 827.5 +rank0 [2026-10-29 22:01:44,478] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 185.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +2 6 0x7408, 15961 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +4 8 0x7408, 13395 55.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:06:20,188] INFO : Step 38700/50000 rank0 | Train Loss: 0.8153 | Train Loss Unweighted: 0.3795 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0947 | Classification: 0.0722 | Classification_binary: 0.1256 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0893 | Regression_eta: 0.0013 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.1197 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0496 | Regression_eta: 0.0457 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0642 | LR: 2.09e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9595s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7251 | max_loss: 0.2433 | max_backward: 1.8091 | max_optimizer_step: 0.1965 | max_tensorboard_logging: 0.4744 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5858.3 | uss_MB: 4950.6 | child_rss_MB: 5664.7 | allocated_MB: 493.1 | reserved_MB: 29592.0 | inactive_split_MB: 1080.0 +rank0 [2026-10-29 22:06:20,426] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 55.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 32% 100% +6 10 0x7408, 62036 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:11:06,758] INFO : Step 38800/50000 rank0 | Train Loss: 0.8386 | Train Loss Unweighted: 0.4077 | Train Loss Components: Auxiliary: 0.1776 | Cardinality: 0.0905 | Classification: 0.0849 | Classification_binary: 0.1308 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0012 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1146 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0460 | Regression_eta: 0.0306 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0377 | LR: 2.07e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8333s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.7346 | max_loss: 0.1426 | max_backward: 1.8320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5857.6 | uss_MB: 4949.0 | child_rss_MB: 5646.6 | allocated_MB: 497.9 | reserved_MB: 29592.0 | inactive_split_MB: 1086.4 +rank0 [2026-10-29 22:11:06,995] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 183.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +4 8 0x7408, 13395 55.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +6 10 0x7408, 62036 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:15:55,363] INFO : Step 38900/50000 rank0 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1916 | Cardinality: 0.1035 | Classification: 0.0839 | Classification_binary: 0.1479 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0014 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0447 | Regression_eta: 0.0404 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0655 | LR: 2.05e-04 | DataLoad Time: 0.0208s | Model Forward Time: 6.5269s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0144 | max_forward: 2.1610 | max_loss: 0.2966 | max_backward: 2.8203 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4876 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 5858.5 | uss_MB: 4951.7 | child_rss_MB: 5647.2 | allocated_MB: 500.0 | reserved_MB: 29592.0 | inactive_split_MB: 849.6 +rank0 [2026-10-29 22:15:55,615] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 55.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 50.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:20:37,390] INFO : Step 39000/50000 rank0 | Train Loss: 0.8505 | Train Loss Unweighted: 0.4196 | Train Loss Components: Auxiliary: 0.1813 | Cardinality: 0.0941 | Classification: 0.0808 | Classification_binary: 0.1368 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0993 | Regression_eta: 0.0015 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1246 | Regression_cos_phi: 0.0344 | Regression_energy: 0.0464 | Regression_eta: 0.0304 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0472 | LR: 2.03e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9510s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7392 | max_loss: 0.2445 | max_backward: 1.8584 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5844.6 | uss_MB: 4939.8 | child_rss_MB: 5638.4 | allocated_MB: 496.8 | reserved_MB: 29592.0 | inactive_split_MB: 1086.0 +rank0 [2026-10-29 22:20:37,628] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +2 6 0x7408, 15961 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 55.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:20:37,651] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-39000.pth +rank0 [2026-10-29 22:20:37,940] INFO : Running validation on rank0 +rank0 [2026-10-29 22:20:37,941] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 22:20:37,941] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 22:21:31,759] INFO : VALIDATION | Step=39000/50000 | Train Loss=0.8505 | Valid Loss=0.8924 | Train Loss Unweighted=0.4196 | Valid Loss Unweighted=0.4221 | Stale=0 | ETA=127.4m +rank0 [2026-10-29 22:21:31,759] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 22:21:31,759] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 22:21:31,759] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 22:21:31,759] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/ari=0.59948 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.707207 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/top1_competing=0.815106 +rank0 [2026-10-29 22:21:31,760] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.660145 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.050327 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0856704 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/seconds_per_rank=21.4438 +rank0 [2026-10-29 22:21:31,761] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding/ari=0.292362 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/embedding_minus_raw/ari=0.200738 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 22:21:31,762] INFO : VALIDATION_PROBE | Step=39000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 22:21:31,767] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:21:32,014] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 22:21:32,018] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 22:21:59,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 22:22:00,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 22:22:01,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 22:22:02,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 22:22:12,666] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 22:22:12,679] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:22:13,174] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 22:22:13,178] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 22:22:29,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 22:22:30,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 22:22:31,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 22:22:33,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 22:22:43,209] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-29 22:22:43,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 22:22:43,800] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 22:22:43,803] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 22:22:59,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 22:23:00,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 22:23:01,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 22:23:02,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 22:23:12,194] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-29 22:23:12,284] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits +rank0 [2026-10-29 22:23:15,559] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits +rank0 [2026-10-29 22:23:16,616] INFO : Plotted number of elements +rank0 [2026-10-29 22:23:18,180] INFO : Plotted jets +rank0 [2026-10-29 22:23:19,355] INFO : Plotted jet ratio +rank0 [2026-10-29 22:23:19,411] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 22:23:22,614] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 22:23:23,593] INFO : Plotted number of elements +rank0 [2026-10-29 22:23:25,130] INFO : Plotted jets +rank0 [2026-10-29 22:23:26,250] INFO : Plotted jet ratio +rank0 [2026-10-29 22:23:26,345] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits +rank0 [2026-10-29 22:23:29,134] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits +rank0 [2026-10-29 22:23:29,970] INFO : Plotted number of elements +rank0 [2026-10-29 22:23:31,473] INFO : Plotted jets +rank0 [2026-10-29 22:23:32,634] INFO : Plotted jet ratio +rank0 [2026-10-29 22:28:31,274] INFO : Step 39100/50000 rank0 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1110 | Classification: 0.0839 | Classification_binary: 0.1198 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1150 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0500 | Regression_eta: 0.0294 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0399 | LR: 2.01e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.5016s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7800 | max_loss: 0.2457 | max_backward: 1.9719 | max_optimizer_step: 0.3248 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 7010.9 | uss_MB: 6103.0 | child_rss_MB: 5608.2 | allocated_MB: 497.5 | reserved_MB: 24374.0 | inactive_split_MB: 923.6 +rank0 [2026-10-29 22:28:31,517] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 50.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:33:30,547] INFO : Step 39200/50000 rank0 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4242 | Train Loss Components: Auxiliary: 0.1863 | Cardinality: 0.1210 | Classification: 0.0912 | Classification_binary: 0.1382 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0454 | Regression_eta: 0.0419 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0660 | LR: 2.00e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9972s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0324 | max_forward: 0.7817 | max_loss: 0.1452 | max_backward: 1.9286 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5940.0 | uss_MB: 5036.4 | child_rss_MB: 5609.8 | allocated_MB: 497.9 | reserved_MB: 26338.0 | inactive_split_MB: 912.4 +rank0 [2026-10-29 22:33:30,790] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 56.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:38:29,739] INFO : Step 39300/50000 rank0 | Train Loss: 0.9298 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1080 | Classification: 0.0882 | Classification_binary: 0.1594 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0951 | Regression_eta: 0.0013 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1422 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0526 | Regression_eta: 0.0405 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0684 | LR: 1.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.9405s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7922 | max_loss: 0.2572 | max_backward: 1.8558 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 5927.1 | uss_MB: 5036.1 | child_rss_MB: 5613.3 | allocated_MB: 499.4 | reserved_MB: 26338.0 | inactive_split_MB: 1408.0 +rank0 [2026-10-29 22:38:29,978] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 54.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:43:22,226] INFO : Step 39400/50000 rank0 | Train Loss: 0.8421 | Train Loss Unweighted: 0.3964 | Train Loss Components: Auxiliary: 0.1755 | Cardinality: 0.1006 | Classification: 0.0679 | Classification_binary: 0.1346 | HitGrouping: 0.1534 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0279 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0438 | Regression_eta: 0.0214 | Regression_pt: 0.0478 | Regression_sin_phi: 0.0313 | LR: 1.96e-04 | DataLoad Time: 0.0202s | Model Forward Time: 7.0506s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1084 | max_forward: 1.1740 | max_loss: 0.3046 | max_backward: 2.6778 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8613 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5928.1 | uss_MB: 4351.2 | child_rss_MB: 5609.1 | allocated_MB: 495.0 | reserved_MB: 26338.0 | inactive_split_MB: 1095.6 +rank0 [2026-10-29 22:43:22,467] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 6% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 64% +2 6 0x7408, 15961 51.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 42% 2% +4 8 0x7408, 13395 55.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 99% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:48:18,131] INFO : Step 39500/50000 rank0 | Train Loss: 0.8539 | Train Loss Unweighted: 0.4179 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0879 | Classification: 0.0889 | Classification_binary: 0.1243 | HitGrouping: 0.1565 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1351 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0340 | Regression_eta: 0.0273 | Regression_pt: 0.0372 | Regression_sin_phi: 0.0502 | LR: 1.94e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.4547s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7373 | max_loss: 0.1406 | max_backward: 1.8719 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.8502 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5922.7 | uss_MB: 5037.5 | child_rss_MB: 5598.5 | allocated_MB: 492.8 | reserved_MB: 28318.0 | inactive_split_MB: 1078.3 +rank0 [2026-10-29 22:48:18,371] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 12% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 52.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 42% 0% +4 8 0x7408, 13395 55.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 1% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 91% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:53:41,889] INFO : Step 39600/50000 rank0 | Train Loss: 0.9371 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1161 | Classification: 0.0813 | Classification_binary: 0.1596 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1039 | Regression_eta: 0.0013 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1771 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0646 | Regression_eta: 0.0491 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0543 | LR: 1.93e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.0076s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0173 | max_forward: 1.4879 | max_loss: 0.2520 | max_backward: 1.9863 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.8346 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5921.1 | uss_MB: 5038.4 | child_rss_MB: 5602.9 | allocated_MB: 496.0 | reserved_MB: 28318.0 | inactive_split_MB: 858.4 +rank0 [2026-10-29 22:53:42,132] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 55.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 22:58:48,783] INFO : Step 39700/50000 rank0 | Train Loss: 0.9204 | Train Loss Unweighted: 0.4427 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1101 | Classification: 0.0841 | Classification_binary: 0.1567 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0997 | Regression_eta: 0.0013 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1510 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0611 | Regression_eta: 0.0431 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0579 | LR: 1.91e-04 | DataLoad Time: 0.0131s | Model Forward Time: 5.0023s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.7571 | max_loss: 0.4946 | max_backward: 1.9147 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 5909.3 | uss_MB: 5037.4 | child_rss_MB: 5602.2 | allocated_MB: 497.0 | reserved_MB: 30324.0 | inactive_split_MB: 1175.5 +rank0 [2026-10-29 22:58:49,027] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 52.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 42% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 85% +4 8 0x7408, 13395 55.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 85% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 51.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 6% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 39% 1% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:04:38,863] INFO : Step 39800/50000 rank0 | Train Loss: 0.8155 | Train Loss Unweighted: 0.3780 | Train Loss Components: Auxiliary: 0.1674 | Cardinality: 0.0963 | Classification: 0.0643 | Classification_binary: 0.1275 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0285 | Classification_binary: 0.1066 | Regression_cos_phi: 0.0330 | Regression_energy: 0.0511 | Regression_eta: 0.0186 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0391 | LR: 1.89e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6339s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7197 | max_loss: 0.1489 | max_backward: 1.6690 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.4760 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1068 | Memory: rss_MB: 5897.7 | uss_MB: 5034.7 | child_rss_MB: 5594.3 | allocated_MB: 494.4 | reserved_MB: 30324.0 | inactive_split_MB: 897.3 +rank0 [2026-10-29 23:04:39,121] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 55.0°C 194.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:09:44,099] INFO : Step 39900/50000 rank0 | Train Loss: 0.8844 | Train Loss Unweighted: 0.3998 | Train Loss Components: Auxiliary: 0.1871 | Cardinality: 0.1077 | Classification: 0.0758 | Classification_binary: 0.1443 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0883 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0360 | Regression_eta: 0.0364 | Regression_pt: 0.0396 | Regression_sin_phi: 0.0608 | LR: 1.88e-04 | DataLoad Time: 0.0125s | Model Forward Time: 5.6426s | Phase Times: max_pre_step_barrier: 0.0268 | max_batch_to_device: 0.0727 | max_forward: 1.7016 | max_loss: 0.4968 | max_backward: 2.0829 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.4776 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5898.5 | uss_MB: 5037.5 | child_rss_MB: 5601.6 | allocated_MB: 496.1 | reserved_MB: 30324.0 | inactive_split_MB: 1077.6 +rank0 [2026-10-29 23:09:44,340] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +2 6 0x7408, 15961 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 55.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:14:38,878] INFO : Step 40000/50000 rank0 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1170 | Classification: 0.0871 | Classification_binary: 0.1550 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1379 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0584 | Regression_eta: 0.0343 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0359 | LR: 1.86e-04 | DataLoad Time: 0.0077s | Model Forward Time: 6.1485s | Phase Times: max_pre_step_barrier: 0.1084 | max_batch_to_device: 0.0070 | max_forward: 1.1725 | max_loss: 0.2555 | max_backward: 2.4297 | max_optimizer_step: 0.2969 | max_tensorboard_logging: 0.6818 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5889.3 | uss_MB: 5028.2 | child_rss_MB: 5597.1 | allocated_MB: 501.7 | reserved_MB: 30324.0 | inactive_split_MB: 1368.0 +rank0 [2026-10-29 23:14:39,116] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +4 8 0x7408, 13395 55.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:14:39,139] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-40000.pth +rank0 [2026-10-29 23:14:39,546] INFO : Running validation on rank0 +rank0 [2026-10-29 23:14:39,547] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-29 23:14:39,547] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-29 23:15:32,352] INFO : VALIDATION | Step=40000/50000 | Train Loss=0.9365 | Valid Loss=0.9015 | Train Loss Unweighted=0.4585 | Valid Loss Unweighted=0.4200 | Stale=1 | ETA=126.4m +rank0 [2026-10-29 23:15:32,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-29 23:15:32,352] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/ari=0.600773 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.692906 +rank0 [2026-10-29 23:15:32,353] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/top1_competing=0.799021 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.653784 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0516204 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0793088 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/seconds_per_rank=21.7012 +rank0 [2026-10-29 23:15:32,354] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding/ari=0.296834 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/embedding_minus_raw/ari=0.20521 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-29 23:15:32,355] INFO : VALIDATION_PROBE | Step=40000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-29 23:15:32,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:15:32,622] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-29 23:15:32,633] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-29 23:16:03,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-29 23:16:06,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-29 23:16:15,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-29 23:16:18,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-29 23:16:28,281] INFO : Time taken to make predictions on device 0 is: 0.45 min +rank0 [2026-10-29 23:16:28,294] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:16:28,835] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-29 23:16:28,839] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-29 23:16:45,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-29 23:16:46,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-29 23:16:47,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-29 23:16:48,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-29 23:16:58,521] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-29 23:16:58,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-29 23:16:59,025] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-29 23:16:59,029] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-29 23:17:14,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-29 23:17:15,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-29 23:17:16,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-29 23:17:17,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-29 23:17:27,974] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-29 23:17:27,993] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits +rank0 [2026-10-29 23:17:31,235] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits +rank0 [2026-10-29 23:17:32,270] INFO : Plotted number of elements +rank0 [2026-10-29 23:17:33,902] INFO : Plotted jets +rank0 [2026-10-29 23:17:35,622] INFO : Plotted jet ratio +rank0 [2026-10-29 23:17:35,682] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 23:17:38,945] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits +rank0 [2026-10-29 23:17:39,945] INFO : Plotted number of elements +rank0 [2026-10-29 23:17:41,507] INFO : Plotted jets +rank0 [2026-10-29 23:17:42,673] INFO : Plotted jet ratio +rank0 [2026-10-29 23:17:42,800] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits +rank0 [2026-10-29 23:17:45,284] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits +rank0 [2026-10-29 23:17:46,112] INFO : Plotted number of elements +rank0 [2026-10-29 23:17:47,637] INFO : Plotted jets +rank0 [2026-10-29 23:17:48,869] INFO : Plotted jet ratio +rank0 [2026-10-29 23:22:42,722] INFO : Step 40100/50000 rank0 | Train Loss: 0.8115 | Train Loss Unweighted: 0.3794 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.1028 | Classification: 0.0708 | Classification_binary: 0.1239 | HitGrouping: 0.1460 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0909 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1075 | Regression_cos_phi: 0.0338 | Regression_energy: 0.0452 | Regression_eta: 0.0245 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0321 | LR: 1.84e-04 | DataLoad Time: 0.0175s | Model Forward Time: 6.0559s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.1076 | max_forward: 0.8426 | max_loss: 0.1437 | max_backward: 1.8847 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 7273.7 | uss_MB: 5477.4 | child_rss_MB: 5617.3 | allocated_MB: 493.6 | reserved_MB: 28004.0 | inactive_split_MB: 975.8 +rank0 [2026-10-29 23:22:42,972] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +2 6 0x7408, 15961 51.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 21% +4 8 0x7408, 13395 54.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 78% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 50.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:27:53,946] INFO : Step 40200/50000 rank0 | Train Loss: 0.8264 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1679 | Cardinality: 0.0984 | Classification: 0.0703 | Classification_binary: 0.1210 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0913 | Regression_eta: 0.0013 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0415 | Regression_eta: 0.0415 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0570 | LR: 1.83e-04 | DataLoad Time: 0.0088s | Model Forward Time: 5.6128s | Phase Times: max_pre_step_barrier: 0.0270 | max_batch_to_device: 0.0062 | max_forward: 0.8324 | max_loss: 0.1802 | max_backward: 2.0569 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0152 | max_post_step_barrier: 0.0867 | Memory: rss_MB: 5955.5 | uss_MB: 5094.1 | child_rss_MB: 5625.1 | allocated_MB: 497.1 | reserved_MB: 28008.0 | inactive_split_MB: 935.5 +rank0 [2026-10-29 23:27:54,188] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +2 6 0x7408, 15961 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 34% 0% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:33:23,758] INFO : Step 40300/50000 rank0 | Train Loss: 0.9351 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1181 | Classification: 0.0959 | Classification_binary: 0.1652 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0014 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1621 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0459 | Regression_eta: 0.0369 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0629 | LR: 1.81e-04 | DataLoad Time: 0.0175s | Model Forward Time: 6.2576s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8191 | max_loss: 0.1494 | max_backward: 1.9817 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 0.9693 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 5955.1 | uss_MB: 5096.4 | child_rss_MB: 5610.7 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 927.5 +rank0 [2026-10-29 23:33:24,002] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 51.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 46% 1% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 54.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 42% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 11% +6 10 0x7408, 62036 50.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:38:39,333] INFO : Step 40400/50000 rank0 | Train Loss: 0.9379 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1359 | Classification: 0.0771 | Classification_binary: 0.1554 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0944 | Regression_eta: 0.0014 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0440 | Regression_eta: 0.0625 | Regression_pt: 0.0426 | Regression_sin_phi: 0.0502 | LR: 1.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.3278s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0500 | max_forward: 0.8111 | max_loss: 0.5067 | max_backward: 1.8144 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5951.4 | uss_MB: 5092.9 | child_rss_MB: 5616.0 | allocated_MB: 497.0 | reserved_MB: 28008.0 | inactive_split_MB: 931.6 +rank0 [2026-10-29 23:38:39,575] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 54.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:43:44,557] INFO : Step 40500/50000 rank0 | Train Loss: 0.8053 | Train Loss Unweighted: 0.3782 | Train Loss Components: Auxiliary: 0.1649 | Cardinality: 0.0788 | Classification: 0.0749 | Classification_binary: 0.1246 | HitGrouping: 0.1502 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0436 | Regression_eta: 0.0397 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0481 | LR: 1.78e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1542s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.8420 | max_loss: 0.1414 | max_backward: 1.5396 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.9215 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5948.4 | uss_MB: 5092.0 | child_rss_MB: 5597.4 | allocated_MB: 494.5 | reserved_MB: 28012.0 | inactive_split_MB: 979.3 +rank0 [2026-10-29 23:43:44,807] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 55.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:48:44,352] INFO : Step 40600/50000 rank0 | Train Loss: 0.8980 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1108 | Classification: 0.0752 | Classification_binary: 0.1417 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0955 | Regression_eta: 0.0015 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0334 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0448 | Regression_eta: 0.0325 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0581 | LR: 1.76e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9730s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7855 | max_loss: 0.1488 | max_backward: 1.8742 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 5948.2 | uss_MB: 5094.3 | child_rss_MB: 5593.1 | allocated_MB: 499.7 | reserved_MB: 28012.0 | inactive_split_MB: 1102.0 +rank0 [2026-10-29 23:48:44,596] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 182.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 56.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-29 23:53:40,220] INFO : Step 40700/50000 rank0 | Train Loss: 0.9220 | Train Loss Unweighted: 0.4304 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1148 | Classification: 0.0767 | Classification_binary: 0.1645 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0496 | Regression_eta: 0.0330 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0519 | LR: 1.75e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1840s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.8091 | max_loss: 0.2475 | max_backward: 2.0011 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.5972 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5949.6 | uss_MB: 5096.1 | child_rss_MB: 5593.6 | allocated_MB: 498.9 | reserved_MB: 28012.0 | inactive_split_MB: 776.0 +rank0 [2026-10-29 23:53:40,460] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 55.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:01:00,773] INFO : Step 40800/50000 rank0 | Train Loss: 0.8642 | Train Loss Unweighted: 0.4082 | Train Loss Components: Auxiliary: 0.1815 | Cardinality: 0.1013 | Classification: 0.0834 | Classification_binary: 0.1260 | HitGrouping: 0.1507 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1216 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0582 | Regression_eta: 0.0395 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0738 | LR: 1.73e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.8435s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7033 | max_loss: 0.1373 | max_backward: 1.5042 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1068 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5935.0 | uss_MB: 4327.0 | child_rss_MB: 5586.6 | allocated_MB: 493.1 | reserved_MB: 28012.0 | inactive_split_MB: 897.3 +rank0 [2026-10-30 00:01:01,010] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 56% +2 6 0x7408, 15961 51.0°C 147.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 72% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 43% 0% +4 8 0x7408, 13395 54.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 1% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 40% 0% +6 10 0x7408, 62036 50.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 98% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:06:15,261] INFO : Step 40900/50000 rank0 | Train Loss: 0.8749 | Train Loss Unweighted: 0.4038 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1070 | Classification: 0.0854 | Classification_binary: 0.1300 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0923 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0372 | Regression_energy: 0.0550 | Regression_eta: 0.0281 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0607 | LR: 1.72e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.2405s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7515 | max_loss: 0.2142 | max_backward: 1.7950 | max_optimizer_step: 0.4960 | max_tensorboard_logging: 0.9322 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5926.9 | uss_MB: 5094.2 | child_rss_MB: 5563.3 | allocated_MB: 497.9 | reserved_MB: 30012.0 | inactive_split_MB: 893.0 +rank0 [2026-10-30 00:06:15,502] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:11:08,132] INFO : Step 41000/50000 rank0 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.0956 | Classification: 0.0893 | Classification_binary: 0.1497 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0014 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1587 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0473 | Regression_eta: 0.0300 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0498 | LR: 1.70e-04 | DataLoad Time: 0.0128s | Model Forward Time: 5.7787s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0071 | max_forward: 1.1963 | max_loss: 0.1519 | max_backward: 2.0474 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.0118 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 5918.6 | uss_MB: 5085.3 | child_rss_MB: 5547.4 | allocated_MB: 498.7 | reserved_MB: 30012.0 | inactive_split_MB: 1108.0 +rank0 [2026-10-30 00:11:08,371] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 34% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 54.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:11:08,395] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-41000.pth +rank0 [2026-10-30 00:11:08,806] INFO : Running validation on rank0 +rank0 [2026-10-30 00:11:08,806] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 00:11:08,806] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION | Step=41000/50000 | Train Loss=0.9032 | Valid Loss=0.8973 | Train Loss Unweighted=0.4353 | Valid Loss Unweighted=0.4242 | Stale=2 | ETA=123.4m +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 00:11:59,817] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/ari=0.611082 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.707468 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/top1_competing=0.8141 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.665695 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0619295 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0912198 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 00:11:59,818] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/seconds_per_rank=21.5338 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding/ari=0.278327 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/embedding_minus_raw/ari=0.186704 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 00:11:59,819] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 00:11:59,820] INFO : VALIDATION_PROBE | Step=41000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 00:11:59,820] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:12:00,093] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 00:12:00,104] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 00:12:26,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 00:12:27,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 00:12:28,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 00:12:29,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 00:12:39,665] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 00:12:39,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:12:40,281] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 00:12:40,285] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 00:12:56,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 00:12:57,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 00:12:58,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 00:12:59,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 00:13:09,690] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 00:13:09,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 00:13:10,646] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 00:13:10,650] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 00:13:27,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 00:13:28,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 00:13:29,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 00:13:30,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 00:13:40,270] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 00:13:40,285] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits +rank0 [2026-10-30 00:13:43,481] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits +rank0 [2026-10-30 00:13:44,516] INFO : Plotted number of elements +rank0 [2026-10-30 00:13:46,073] INFO : Plotted jets +rank0 [2026-10-30 00:13:47,249] INFO : Plotted jet ratio +rank0 [2026-10-30 00:13:47,320] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 00:13:50,515] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 00:13:51,527] INFO : Plotted number of elements +rank0 [2026-10-30 00:13:53,040] INFO : Plotted jets +rank0 [2026-10-30 00:13:54,584] INFO : Plotted jet ratio +rank0 [2026-10-30 00:13:54,681] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits +rank0 [2026-10-30 00:13:57,049] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits +rank0 [2026-10-30 00:13:57,874] INFO : Plotted number of elements +rank0 [2026-10-30 00:13:59,370] INFO : Plotted jets +rank0 [2026-10-30 00:14:00,528] INFO : Plotted jet ratio +rank0 [2026-10-30 00:19:16,287] INFO : Step 41100/50000 rank0 | Train Loss: 0.9430 | Train Loss Unweighted: 0.4319 | Train Loss Components: Auxiliary: 0.1995 | Cardinality: 0.1330 | Classification: 0.0721 | Classification_binary: 0.1579 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0013 | Regression_pt: 0.0998 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1652 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0607 | Regression_eta: 0.0390 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0761 | LR: 1.69e-04 | DataLoad Time: 0.0153s | Model Forward Time: 7.2774s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8221 | max_loss: 0.4800 | max_backward: 2.2615 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7242.0 | uss_MB: 5759.2 | child_rss_MB: 5549.6 | allocated_MB: 497.6 | reserved_MB: 23400.0 | inactive_split_MB: 991.8 +rank0 [2026-10-30 00:19:16,525] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 51.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 6% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 98% +4 8 0x7408, 13395 54.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 30% 0% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 35% 0% +6 10 0x7408, 62036 51.0°C 139.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 88% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:24:27,791] INFO : Step 41200/50000 rank0 | Train Loss: 0.8544 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1097 | Classification: 0.0734 | Classification_binary: 0.1131 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1015 | Regression_eta: 0.0015 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1112 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0563 | Regression_eta: 0.0367 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0789 | LR: 1.67e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9641s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7030 | max_loss: 0.2408 | max_backward: 1.7750 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5316 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6039.9 | uss_MB: 5208.8 | child_rss_MB: 5554.6 | allocated_MB: 496.1 | reserved_MB: 27308.0 | inactive_split_MB: 984.0 +rank0 [2026-10-30 00:24:28,030] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 54.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 50.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:29:21,726] INFO : Step 41300/50000 rank0 | Train Loss: 0.8221 | Train Loss Unweighted: 0.3852 | Train Loss Components: Auxiliary: 0.1744 | Cardinality: 0.0906 | Classification: 0.0754 | Classification_binary: 0.1369 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0849 | Regression_eta: 0.0012 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0430 | Regression_eta: 0.0495 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0609 | LR: 1.66e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2426s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8083 | max_loss: 0.1794 | max_backward: 1.9993 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.5906 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6040.0 | uss_MB: 5213.3 | child_rss_MB: 5551.2 | allocated_MB: 497.0 | reserved_MB: 27308.0 | inactive_split_MB: 1229.0 +rank0 [2026-10-30 00:29:21,967] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +4 8 0x7408, 13395 55.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 30% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:34:26,308] INFO : Step 41400/50000 rank0 | Train Loss: 0.9216 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1171 | Classification: 0.0901 | Classification_binary: 0.1523 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1010 | Regression_eta: 0.0011 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1503 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0634 | Regression_eta: 0.0353 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0631 | LR: 1.64e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.3236s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0065 | max_forward: 0.7831 | max_loss: 0.4937 | max_backward: 1.9271 | max_optimizer_step: 0.3071 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6033.3 | uss_MB: 5208.0 | child_rss_MB: 5553.7 | allocated_MB: 498.4 | reserved_MB: 29288.0 | inactive_split_MB: 1180.4 +rank0 [2026-10-30 00:34:26,547] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 55.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:39:30,714] INFO : Step 41500/50000 rank0 | Train Loss: 0.8584 | Train Loss Unweighted: 0.4111 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1030 | Classification: 0.0769 | Classification_binary: 0.1210 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1085 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0712 | Regression_eta: 0.0330 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0457 | LR: 1.63e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.7955s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.1908 | max_loss: 0.3017 | max_backward: 3.1310 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 6009.7 | uss_MB: 5150.0 | child_rss_MB: 5552.7 | allocated_MB: 494.5 | reserved_MB: 29288.0 | inactive_split_MB: 976.9 +rank0 [2026-10-30 00:39:30,958] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 52.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 54.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 50.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:44:20,080] INFO : Step 41600/50000 rank0 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4026 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1032 | Classification: 0.0762 | Classification_binary: 0.1246 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0992 | Regression_eta: 0.0013 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1195 | Regression_cos_phi: 0.0441 | Regression_energy: 0.0497 | Regression_eta: 0.0435 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0732 | LR: 1.61e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.6613s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0065 | max_forward: 0.7279 | max_loss: 0.2339 | max_backward: 2.5338 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.4866 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1404 | Memory: rss_MB: 6001.6 | uss_MB: 5206.5 | child_rss_MB: 5546.1 | allocated_MB: 496.4 | reserved_MB: 29288.0 | inactive_split_MB: 761.6 +rank0 [2026-10-30 00:44:20,333] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +2 6 0x7408, 15961 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +4 8 0x7408, 13395 53.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 50.0°C 155.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:49:11,531] INFO : Step 41700/50000 rank0 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1941 | Cardinality: 0.1112 | Classification: 0.0767 | Classification_binary: 0.1493 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0988 | Regression_eta: 0.0013 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0464 | Regression_eta: 0.0324 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0772 | LR: 1.60e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.1604s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7998 | max_loss: 0.1827 | max_backward: 1.9774 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6004.5 | uss_MB: 5210.8 | child_rss_MB: 5548.7 | allocated_MB: 497.0 | reserved_MB: 29288.0 | inactive_split_MB: 1220.0 +rank0 [2026-10-30 00:49:11,772] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 70% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 99% +4 8 0x7408, 13395 55.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:53:59,652] INFO : Step 41800/50000 rank0 | Train Loss: 0.9125 | Train Loss Unweighted: 0.4285 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.1148 | Classification: 0.0784 | Classification_binary: 0.1537 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0291 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0557 | Regression_eta: 0.0313 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0383 | LR: 1.58e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.4837s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7952 | max_loss: 0.1582 | max_backward: 1.8899 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9392 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1143 | Memory: rss_MB: 5984.7 | uss_MB: 5200.7 | child_rss_MB: 5542.4 | allocated_MB: 499.3 | reserved_MB: 31288.0 | inactive_split_MB: 932.9 +rank0 [2026-10-30 00:53:59,917] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +4 8 0x7408, 13395 55.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 45% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 00:59:01,435] INFO : Step 41900/50000 rank0 | Train Loss: 0.8303 | Train Loss Unweighted: 0.3927 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1132 | Classification: 0.0782 | Classification_binary: 0.1167 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0010 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1043 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0429 | Regression_eta: 0.0245 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0508 | LR: 1.57e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.7955s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7413 | max_loss: 0.1970 | max_backward: 1.7614 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5208 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5992.5 | uss_MB: 5207.2 | child_rss_MB: 5522.2 | allocated_MB: 497.1 | reserved_MB: 31288.0 | inactive_split_MB: 1188.0 +rank0 [2026-10-30 00:59:01,689] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 178.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 40% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +4 8 0x7408, 13395 55.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:03:52,354] INFO : Step 42000/50000 rank0 | Train Loss: 0.8724 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1847 | Cardinality: 0.0893 | Classification: 0.0874 | Classification_binary: 0.1377 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0958 | Regression_eta: 0.0012 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0615 | Regression_eta: 0.0259 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0747 | LR: 1.56e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.4951s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1147 | max_forward: 0.8308 | max_loss: 0.1392 | max_backward: 1.8336 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5996.6 | uss_MB: 5209.4 | child_rss_MB: 5540.2 | allocated_MB: 496.3 | reserved_MB: 31288.0 | inactive_split_MB: 930.5 +rank0 [2026-10-30 01:03:52,595] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 43% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +2 6 0x7408, 15961 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 51% 100% +4 8 0x7408, 13395 55.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 48% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:03:52,616] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-42000.pth +rank0 [2026-10-30 01:03:53,023] INFO : Running validation on rank0 +rank0 [2026-10-30 01:03:53,024] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 01:03:53,024] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 01:04:46,655] INFO : VALIDATION | Step=42000/50000 | Train Loss=0.8724 | Valid Loss=0.8738 | Train Loss Unweighted=0.4208 | Valid Loss Unweighted=0.4126 | Stale=0 | ETA=117.1m +rank0 [2026-10-30 01:04:46,655] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 01:04:46,655] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 01:04:46,655] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/ari=0.595752 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/ari_threshold=0.85 +rank0 [2026-10-30 01:04:46,656] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.725006 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/top1_competing=0.836039 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.665773 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0465989 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0912984 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 01:04:46,657] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/seconds_per_rank=21.2111 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding/ari=0.31117 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/embedding_minus_raw/ari=0.219546 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 01:04:46,658] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 01:04:46,659] INFO : VALIDATION_PROBE | Step=42000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 01:04:46,665] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:04:46,916] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 01:04:46,926] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 01:05:13,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 01:05:14,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 01:05:16,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 01:05:17,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 01:05:27,355] INFO : Time taken to make predictions on device 0 is: 0.27 min +rank0 [2026-10-30 01:05:27,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:05:27,884] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 01:05:27,888] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 01:05:43,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 01:05:44,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 01:05:46,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 01:05:47,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 01:05:57,110] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 01:05:57,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:05:57,664] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 01:05:57,667] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 01:06:13,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 01:06:15,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 01:06:16,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 01:06:17,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 01:06:27,449] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 01:06:27,468] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits +rank0 [2026-10-30 01:06:30,821] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits +rank0 [2026-10-30 01:06:31,944] INFO : Plotted number of elements +rank0 [2026-10-30 01:06:33,992] INFO : Plotted jets +rank0 [2026-10-30 01:06:35,252] INFO : Plotted jet ratio +rank0 [2026-10-30 01:06:35,332] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 01:06:38,659] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 01:06:39,732] INFO : Plotted number of elements +rank0 [2026-10-30 01:06:41,349] INFO : Plotted jets +rank0 [2026-10-30 01:06:42,475] INFO : Plotted jet ratio +rank0 [2026-10-30 01:06:42,526] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits +rank0 [2026-10-30 01:06:45,038] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits +rank0 [2026-10-30 01:06:45,904] INFO : Plotted number of elements +rank0 [2026-10-30 01:06:47,399] INFO : Plotted jets +rank0 [2026-10-30 01:06:48,558] INFO : Plotted jet ratio +rank0 [2026-10-30 01:11:43,990] INFO : Step 42100/50000 rank0 | Train Loss: 0.9390 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1200 | Classification: 0.0884 | Classification_binary: 0.1619 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0926 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0417 | Regression_eta: 0.0335 | Regression_pt: 0.0444 | Regression_sin_phi: 0.0480 | LR: 1.54e-04 | DataLoad Time: 0.0163s | Model Forward Time: 7.2149s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0976 | max_forward: 1.0189 | max_loss: 0.3859 | max_backward: 2.5922 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 1.0368 | max_tensorboard_flush: 0.0149 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7501.8 | uss_MB: 4866.0 | child_rss_MB: 5544.9 | allocated_MB: 497.0 | reserved_MB: 25850.0 | inactive_split_MB: 892.8 +rank0 [2026-10-30 01:11:44,228] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 141.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 4% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 99% +2 6 0x7408, 15961 51.0°C 143.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 99% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 54.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 99% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 99% +6 10 0x7408, 62036 50.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:16:54,622] INFO : Step 42200/50000 rank0 | Train Loss: 0.9057 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1144 | Classification: 0.0879 | Classification_binary: 0.1381 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0786 | Classification_binary: 0.1174 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0726 | Regression_eta: 0.0323 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0567 | LR: 1.53e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.8177s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7457 | max_loss: 0.1999 | max_backward: 1.6824 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.5916 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6096.0 | uss_MB: 5306.7 | child_rss_MB: 5537.7 | allocated_MB: 497.3 | reserved_MB: 27866.0 | inactive_split_MB: 959.8 +rank0 [2026-10-30 01:16:54,862] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 50.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:18:06,040] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-30 01:18:06,040] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-30 01:21:58,267] INFO : Step 42300/50000 rank0 | Train Loss: 0.9052 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1933 | Cardinality: 0.1127 | Classification: 0.0787 | Classification_binary: 0.1569 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0868 | Regression_eta: 0.0013 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1574 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0482 | Regression_eta: 0.0326 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0701 | LR: 1.52e-04 | DataLoad Time: 0.0149s | Model Forward Time: 6.9207s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.9764 | max_loss: 0.1904 | max_backward: 2.7671 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.9997 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6118.1 | uss_MB: 5325.9 | child_rss_MB: 5544.6 | allocated_MB: 499.1 | reserved_MB: 27872.0 | inactive_split_MB: 1033.6 +rank0 [2026-10-30 01:21:58,518] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 52.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 51.0°C 145.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 54.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 49.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:26:56,280] INFO : Step 42400/50000 rank0 | Train Loss: 0.9358 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.1976 | Cardinality: 0.1220 | Classification: 0.0815 | Classification_binary: 0.1533 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0577 | Regression_eta: 0.0334 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0399 | LR: 1.50e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.5553s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8701 | max_loss: 0.1465 | max_backward: 1.9416 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.8192 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6099.3 | uss_MB: 5308.3 | child_rss_MB: 5546.3 | allocated_MB: 499.8 | reserved_MB: 29908.0 | inactive_split_MB: 939.2 +rank0 [2026-10-30 01:26:56,520] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 55.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:31:49,655] INFO : Step 42500/50000 rank0 | Train Loss: 0.8126 | Train Loss Unweighted: 0.3791 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0975 | Classification: 0.0765 | Classification_binary: 0.1172 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0918 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0562 | Regression_eta: 0.0306 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0554 | LR: 1.49e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2169s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0858 | max_forward: 0.8156 | max_loss: 0.1407 | max_backward: 1.8570 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7997 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6101.2 | uss_MB: 5308.6 | child_rss_MB: 5538.8 | allocated_MB: 497.5 | reserved_MB: 29908.0 | inactive_split_MB: 943.4 +rank0 [2026-10-30 01:31:49,907] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 55.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:36:45,147] INFO : Step 42600/50000 rank0 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0772 | Classification_binary: 0.1347 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0012 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1286 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0367 | Regression_eta: 0.0303 | Regression_pt: 0.0405 | Regression_sin_phi: 0.0370 | LR: 1.48e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.6670s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7519 | max_loss: 0.1393 | max_backward: 1.8021 | max_optimizer_step: 0.3023 | max_tensorboard_logging: 0.7973 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6101.5 | uss_MB: 4501.2 | child_rss_MB: 5544.2 | allocated_MB: 497.0 | reserved_MB: 29908.0 | inactive_split_MB: 877.2 +rank0 [2026-10-30 01:36:45,384] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 1% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 39% +2 6 0x7408, 15961 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 33% 1% +4 8 0x7408, 13395 55.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 98% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 88% +6 10 0x7408, 62036 50.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 21% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 78% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:41:48,989] INFO : Step 42700/50000 rank0 | Train Loss: 0.9129 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1178 | Classification: 0.0751 | Classification_binary: 0.1632 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0858 | Regression_eta: 0.0014 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0429 | Regression_eta: 0.0883 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0476 | LR: 1.47e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2504s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 1.0869 | max_loss: 0.1547 | max_backward: 1.6854 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7949 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6101.6 | uss_MB: 5308.9 | child_rss_MB: 5545.8 | allocated_MB: 497.3 | reserved_MB: 29908.0 | inactive_split_MB: 953.4 +rank0 [2026-10-30 01:41:49,246] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 171.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 55.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 50.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:46:39,823] INFO : Step 42800/50000 rank0 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1179 | Classification: 0.0767 | Classification_binary: 0.1362 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0013 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0327 | Classification_binary: 0.1136 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0676 | Regression_eta: 0.0260 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0709 | LR: 1.45e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.0583s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8438 | max_loss: 0.1478 | max_backward: 1.6710 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.6821 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6089.4 | uss_MB: 5295.4 | child_rss_MB: 5546.6 | allocated_MB: 499.0 | reserved_MB: 30126.0 | inactive_split_MB: 875.7 +rank0 [2026-10-30 01:46:40,067] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 3% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 98% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 99% +4 8 0x7408, 13395 55.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 99% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 99% +6 10 0x7408, 62036 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:51:41,578] INFO : Step 42900/50000 rank0 | Train Loss: 0.7857 | Train Loss Unweighted: 0.3758 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0951 | Classification: 0.0820 | Classification_binary: 0.1113 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0898 | Regression_eta: 0.0010 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1393 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0478 | Regression_eta: 0.0251 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0514 | LR: 1.44e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.6721s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8792 | max_loss: 0.1461 | max_backward: 1.9928 | max_optimizer_step: 0.1903 | max_tensorboard_logging: 0.9695 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6098.2 | uss_MB: 5308.5 | child_rss_MB: 5544.6 | allocated_MB: 496.5 | reserved_MB: 30126.0 | inactive_split_MB: 951.8 +rank0 [2026-10-30 01:51:41,818] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 55.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:56:37,892] INFO : Step 43000/50000 rank0 | Train Loss: 0.8899 | Train Loss Unweighted: 0.4207 | Train Loss Components: Auxiliary: 0.1911 | Cardinality: 0.1035 | Classification: 0.0847 | Classification_binary: 0.1529 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0512 | Regression_eta: 0.0337 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0646 | LR: 1.43e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.5184s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1951 | max_backward: 1.9694 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.8418 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6086.1 | uss_MB: 5311.0 | child_rss_MB: 5508.6 | allocated_MB: 495.5 | reserved_MB: 30126.0 | inactive_split_MB: 1120.4 +rank0 [2026-10-30 01:56:38,135] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 49% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +4 8 0x7408, 13395 55.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 49% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 01:56:38,158] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-43000.pth +rank0 [2026-10-30 01:56:38,460] INFO : Running validation on rank0 +rank0 [2026-10-30 01:56:38,460] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 01:56:38,460] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 01:57:33,688] INFO : VALIDATION | Step=43000/50000 | Train Loss=0.8899 | Valid Loss=0.8857 | Train Loss Unweighted=0.4207 | Valid Loss Unweighted=0.4189 | Stale=1 | ETA=108.7m +rank0 [2026-10-30 01:57:33,688] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 01:57:33,688] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 01:57:33,688] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 01:57:33,688] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 01:57:33,688] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 01:57:33,688] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 01:57:33,688] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/ari=0.617432 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/ari_threshold=0.85 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.710018 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/top1_competing=0.818244 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.668901 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0682793 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0944266 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 01:57:33,689] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/seconds_per_rank=21.2826 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 01:57:33,690] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 01:57:33,691] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 01:57:33,691] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 01:57:33,691] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding/ari=0.308082 +rank0 [2026-10-30 01:57:33,691] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-30 01:57:33,691] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/embedding_minus_raw/ari=0.216459 +rank0 [2026-10-30 01:57:33,691] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 01:57:33,691] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 01:57:33,691] INFO : VALIDATION_PROBE | Step=43000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 01:57:33,696] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:57:33,954] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 01:57:33,962] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 01:57:59,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 01:58:00,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 01:58:01,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 01:58:03,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 01:58:13,023] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 01:58:13,037] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:58:13,515] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 01:58:13,518] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 01:58:29,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 01:58:30,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 01:58:32,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 01:58:33,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 01:58:43,291] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 01:58:43,304] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 01:58:43,760] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 01:58:43,763] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 01:58:59,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 01:59:00,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 01:59:01,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 01:59:02,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 01:59:12,498] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 01:59:12,514] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits +rank0 [2026-10-30 01:59:15,870] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits +rank0 [2026-10-30 01:59:16,978] INFO : Plotted number of elements +rank0 [2026-10-30 01:59:18,931] INFO : Plotted jets +rank0 [2026-10-30 01:59:20,135] INFO : Plotted jet ratio +rank0 [2026-10-30 01:59:20,222] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 01:59:23,476] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 01:59:24,527] INFO : Plotted number of elements +rank0 [2026-10-30 01:59:26,087] INFO : Plotted jets +rank0 [2026-10-30 01:59:27,245] INFO : Plotted jet ratio +rank0 [2026-10-30 01:59:27,296] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits +rank0 [2026-10-30 01:59:29,832] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits +rank0 [2026-10-30 01:59:30,709] INFO : Plotted number of elements +rank0 [2026-10-30 01:59:32,202] INFO : Plotted jets +rank0 [2026-10-30 01:59:33,362] INFO : Plotted jet ratio +rank0 [2026-10-30 02:04:25,545] INFO : Step 43100/50000 rank0 | Train Loss: 0.8965 | Train Loss Unweighted: 0.4137 | Train Loss Components: Auxiliary: 0.1901 | Cardinality: 0.1074 | Classification: 0.0705 | Classification_binary: 0.1472 | HitGrouping: 0.1435 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0493 | Regression_eta: 0.0397 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0702 | LR: 1.42e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.1331s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.2396 | max_backward: 1.9954 | max_optimizer_step: 0.3209 | max_tensorboard_logging: 1.0803 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7631.1 | uss_MB: 6853.7 | child_rss_MB: 5519.2 | allocated_MB: 496.0 | reserved_MB: 26194.0 | inactive_split_MB: 843.4 +rank0 [2026-10-30 02:04:25,785] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +2 6 0x7408, 15961 52.0°C 152.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 54.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +6 10 0x7408, 62036 50.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:09:07,358] INFO : Step 43200/50000 rank0 | Train Loss: 0.8164 | Train Loss Unweighted: 0.3744 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0957 | Classification: 0.0763 | Classification_binary: 0.1197 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0287 | Classification_binary: 0.1038 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0450 | Regression_eta: 0.0298 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0474 | LR: 1.40e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0503s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7823 | max_loss: 0.4822 | max_backward: 1.6766 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6123.7 | uss_MB: 5349.6 | child_rss_MB: 5511.0 | allocated_MB: 495.8 | reserved_MB: 26686.0 | inactive_split_MB: 848.9 +rank0 [2026-10-30 02:09:07,595] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 43% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +4 8 0x7408, 13395 55.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:13:49,528] INFO : Step 43300/50000 rank0 | Train Loss: 0.9151 | Train Loss Unweighted: 0.4260 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1202 | Classification: 0.0906 | Classification_binary: 0.1467 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0013 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0681 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0623 | Regression_eta: 0.0385 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0452 | LR: 1.39e-04 | DataLoad Time: 0.0110s | Model Forward Time: 5.4968s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0155 | max_forward: 0.8938 | max_loss: 0.1464 | max_backward: 2.1010 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.5477 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1243 | Memory: rss_MB: 6097.4 | uss_MB: 5352.9 | child_rss_MB: 5427.7 | allocated_MB: 497.9 | reserved_MB: 28278.0 | inactive_split_MB: 1309.2 +rank0 [2026-10-30 02:13:49,778] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:18:30,738] INFO : Step 43400/50000 rank0 | Train Loss: 0.9723 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2082 | Cardinality: 0.1222 | Classification: 0.0880 | Classification_binary: 0.1684 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0015 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1598 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0449 | Regression_eta: 0.0205 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0397 | LR: 1.38e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.0607s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.5043 | max_backward: 2.0661 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.4824 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6077.9 | uss_MB: 5350.3 | child_rss_MB: 5335.1 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 851.9 +rank0 [2026-10-30 02:18:30,980] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 183.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 193.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:23:13,739] INFO : Step 43500/50000 rank0 | Train Loss: 0.8402 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.0947 | Classification: 0.0776 | Classification_binary: 0.1278 | HitGrouping: 0.1501 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0999 | Regression_eta: 0.0012 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1337 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0548 | Regression_eta: 0.0382 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0364 | LR: 1.37e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.7077s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8086 | max_loss: 0.1489 | max_backward: 2.0887 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.7978 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1094 | Memory: rss_MB: 6077.6 | uss_MB: 5352.8 | child_rss_MB: 5340.6 | allocated_MB: 498.6 | reserved_MB: 28278.0 | inactive_split_MB: 909.7 +rank0 [2026-10-30 02:23:13,978] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 51.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:28:10,286] INFO : Step 43600/50000 rank0 | Train Loss: 0.7985 | Train Loss Unweighted: 0.3653 | Train Loss Components: Auxiliary: 0.1621 | Cardinality: 0.0887 | Classification: 0.0728 | Classification_binary: 0.1184 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0851 | Regression_eta: 0.0013 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0436 | Regression_eta: 0.0282 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0594 | LR: 1.36e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.9252s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0170 | max_forward: 1.2777 | max_loss: 0.2064 | max_backward: 1.9835 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 1.1084 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6044.0 | uss_MB: 4957.3 | child_rss_MB: 5260.0 | allocated_MB: 493.5 | reserved_MB: 28278.0 | inactive_split_MB: 1313.2 +rank0 [2026-10-30 02:28:10,530] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 55.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 99% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:33:07,984] INFO : Step 43700/50000 rank0 | Train Loss: 0.8599 | Train Loss Unweighted: 0.3884 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1089 | Classification: 0.0797 | Classification_binary: 0.1434 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0812 | Regression_eta: 0.0013 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0335 | Regression_eta: 0.0367 | Regression_pt: 0.0366 | Regression_sin_phi: 0.0506 | LR: 1.35e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2929s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7902 | max_loss: 0.1752 | max_backward: 1.8774 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8044 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6046.0 | uss_MB: 5350.4 | child_rss_MB: 5241.5 | allocated_MB: 498.0 | reserved_MB: 28278.0 | inactive_split_MB: 1046.0 +rank0 [2026-10-30 02:33:08,224] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:37:46,433] INFO : Step 43800/50000 rank0 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1875 | Cardinality: 0.1049 | Classification: 0.0814 | Classification_binary: 0.1423 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0907 | Regression_eta: 0.0012 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1273 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0562 | Regression_eta: 0.0258 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0368 | LR: 1.34e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2808s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7940 | max_loss: 0.4805 | max_backward: 1.8159 | max_optimizer_step: 0.2941 | max_tensorboard_logging: 0.7850 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6040.0 | uss_MB: 5349.3 | child_rss_MB: 5218.8 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 839.4 +rank0 [2026-10-30 02:37:46,674] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:42:42,411] INFO : Step 43900/50000 rank0 | Train Loss: 0.7987 | Train Loss Unweighted: 0.3885 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.0999 | Classification: 0.0753 | Classification_binary: 0.1166 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0968 | Regression_eta: 0.0010 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1089 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0593 | Regression_eta: 0.0445 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0646 | LR: 1.33e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.2683s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0846 | max_forward: 0.8891 | max_loss: 0.1768 | max_backward: 1.7357 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7516 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6030.6 | uss_MB: 5347.7 | child_rss_MB: 5179.6 | allocated_MB: 494.5 | reserved_MB: 28278.0 | inactive_split_MB: 1129.5 +rank0 [2026-10-30 02:42:42,668] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +2 6 0x7408, 15961 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 99% +4 8 0x7408, 13395 55.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:47:34,603] INFO : Step 44000/50000 rank0 | Train Loss: 0.8263 | Train Loss Unweighted: 0.3926 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.1027 | Classification: 0.0840 | Classification_binary: 0.1318 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0868 | Regression_eta: 0.0011 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1478 | Regression_cos_phi: 0.0383 | Regression_energy: 0.0511 | Regression_eta: 0.0347 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0601 | LR: 1.32e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2284s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8816 | max_loss: 0.1426 | max_backward: 1.9616 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.4846 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6020.6 | uss_MB: 5352.2 | child_rss_MB: 5134.9 | allocated_MB: 496.3 | reserved_MB: 28762.0 | inactive_split_MB: 1307.2 +rank0 [2026-10-30 02:47:34,858] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 37% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +2 6 0x7408, 15961 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 46% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 40% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 02:47:34,881] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-44000.pth +rank0 [2026-10-30 02:47:35,190] INFO : Running validation on rank0 +rank0 [2026-10-30 02:47:35,190] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 02:47:35,190] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 02:48:28,683] INFO : VALIDATION | Step=44000/50000 | Train Loss=0.8263 | Valid Loss=0.8607 | Train Loss Unweighted=0.3926 | Valid Loss Unweighted=0.4029 | Stale=0 | ETA=98.0m +rank0 [2026-10-30 02:48:28,683] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 02:48:28,683] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 02:48:28,683] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/ari=0.608055 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.720537 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/top1_competing=0.83159 +rank0 [2026-10-30 02:48:28,684] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.664773 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0589023 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0902979 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/seconds_per_rank=21.1812 +rank0 [2026-10-30 02:48:28,685] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding/ari=0.299615 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/embedding_minus_raw/ari=0.207992 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 02:48:28,686] INFO : VALIDATION_PROBE | Step=44000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 02:48:28,691] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 02:48:28,944] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 02:48:28,948] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 02:48:56,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 02:48:57,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 02:48:58,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 02:48:59,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 02:49:09,605] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 02:49:09,617] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 02:49:10,105] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 02:49:10,109] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 02:49:26,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 02:49:27,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 02:49:28,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 02:49:29,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 02:49:39,569] INFO : Time taken to make predictions on device 0 is: 0.25 min +rank0 [2026-10-30 02:49:39,648] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 02:49:40,180] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 02:49:40,184] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 02:49:55,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 02:49:56,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 02:49:57,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 02:49:58,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 02:50:08,365] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 02:50:08,513] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits +rank0 [2026-10-30 02:50:12,150] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits +rank0 [2026-10-30 02:50:13,194] INFO : Plotted number of elements +rank0 [2026-10-30 02:50:14,761] INFO : Plotted jets +rank0 [2026-10-30 02:50:15,938] INFO : Plotted jet ratio +rank0 [2026-10-30 02:50:15,996] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 02:50:19,281] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 02:50:20,333] INFO : Plotted number of elements +rank0 [2026-10-30 02:50:21,911] INFO : Plotted jets +rank0 [2026-10-30 02:50:23,093] INFO : Plotted jet ratio +rank0 [2026-10-30 02:50:23,157] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits +rank0 [2026-10-30 02:50:25,682] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits +rank0 [2026-10-30 02:50:26,563] INFO : Plotted number of elements +rank0 [2026-10-30 02:50:28,122] INFO : Plotted jets +rank0 [2026-10-30 02:50:29,863] INFO : Plotted jet ratio +rank0 [2026-10-30 02:55:21,079] INFO : Step 44100/50000 rank0 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1986 | Cardinality: 0.1334 | Classification: 0.0764 | Classification_binary: 0.1516 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0362 | Regression_energy: 0.0419 | Regression_eta: 0.0243 | Regression_pt: 0.0461 | Regression_sin_phi: 0.0474 | LR: 1.31e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.0173s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0079 | max_forward: 0.8221 | max_loss: 0.1579 | max_backward: 2.0192 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 1.0889 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7539.1 | uss_MB: 6869.4 | child_rss_MB: 5111.3 | allocated_MB: 498.6 | reserved_MB: 25920.0 | inactive_split_MB: 1073.8 +rank0 [2026-10-30 02:55:21,332] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 54.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +6 10 0x7408, 62036 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:00:12,981] INFO : Step 44200/50000 rank0 | Train Loss: 0.9018 | Train Loss Unweighted: 0.4273 | Train Loss Components: Auxiliary: 0.1909 | Cardinality: 0.1109 | Classification: 0.0847 | Classification_binary: 0.1354 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0012 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.1150 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0822 | Regression_eta: 0.0353 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0706 | LR: 1.30e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9765s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.1013 | max_forward: 0.8608 | max_loss: 0.1485 | max_backward: 1.6010 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 6119.2 | uss_MB: 5452.1 | child_rss_MB: 5083.8 | allocated_MB: 496.1 | reserved_MB: 27956.0 | inactive_split_MB: 1248.1 +rank0 [2026-10-30 03:00:13,222] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +4 8 0x7408, 13395 56.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:04:47,900] INFO : Step 44300/50000 rank0 | Train Loss: 0.8239 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1722 | Cardinality: 0.1007 | Classification: 0.0789 | Classification_binary: 0.1118 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.1018 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0348 | Regression_eta: 0.0240 | Regression_pt: 0.0382 | Regression_sin_phi: 0.0379 | LR: 1.29e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9476s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8120 | max_loss: 0.1377 | max_backward: 1.4700 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9055 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6110.2 | uss_MB: 5452.8 | child_rss_MB: 5031.6 | allocated_MB: 492.8 | reserved_MB: 27956.0 | inactive_split_MB: 1035.8 +rank0 [2026-10-30 03:04:48,142] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 56.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:09:18,715] INFO : Step 44400/50000 rank0 | Train Loss: 0.8683 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.0849 | Classification: 0.0836 | Classification_binary: 0.1500 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0602 | Regression_eta: 0.0339 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0437 | LR: 1.28e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.5849s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8907 | max_loss: 0.1476 | max_backward: 2.0623 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6106.5 | uss_MB: 5454.6 | child_rss_MB: 5002.7 | allocated_MB: 499.5 | reserved_MB: 27956.0 | inactive_split_MB: 926.0 +rank0 [2026-10-30 03:09:18,953] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 162.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:14:02,510] INFO : Step 44500/50000 rank0 | Train Loss: 0.8672 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.0989 | Classification: 0.0738 | Classification_binary: 0.1455 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0568 | Regression_eta: 0.0442 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0825 | LR: 1.27e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8899s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0597 | max_forward: 0.8669 | max_loss: 0.1547 | max_backward: 1.7327 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.5628 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6105.7 | uss_MB: 5452.5 | child_rss_MB: 5004.4 | allocated_MB: 498.3 | reserved_MB: 27956.0 | inactive_split_MB: 860.5 +rank0 [2026-10-30 03:14:02,752] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 179.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 53.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 195.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 52.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:19:00,047] INFO : Step 44600/50000 rank0 | Train Loss: 0.7900 | Train Loss Unweighted: 0.3743 | Train Loss Components: Auxiliary: 0.1688 | Cardinality: 0.0956 | Classification: 0.0724 | Classification_binary: 0.1157 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0919 | Regression_eta: 0.0010 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1128 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0499 | Regression_eta: 0.0266 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0462 | LR: 1.26e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2171s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0745 | max_forward: 0.7797 | max_loss: 0.1409 | max_backward: 1.9334 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6022 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6098.6 | uss_MB: 5447.3 | child_rss_MB: 4994.6 | allocated_MB: 494.5 | reserved_MB: 27958.0 | inactive_split_MB: 927.6 +rank0 [2026-10-30 03:19:00,299] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:23:45,494] INFO : Step 44700/50000 rank0 | Train Loss: 0.8669 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1878 | Cardinality: 0.1160 | Classification: 0.0768 | Classification_binary: 0.1380 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0863 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1403 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0421 | Regression_eta: 0.0274 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0359 | LR: 1.25e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1990s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8629 | max_loss: 0.1449 | max_backward: 2.0281 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.5242 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6099.2 | uss_MB: 5434.3 | child_rss_MB: 5003.0 | allocated_MB: 498.4 | reserved_MB: 27958.0 | inactive_split_MB: 781.6 +rank0 [2026-10-30 03:23:45,736] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 99% +2 6 0x7408, 15961 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +4 8 0x7408, 13395 56.0°C 183.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:28:43,005] INFO : Step 44800/50000 rank0 | Train Loss: 0.8585 | Train Loss Unweighted: 0.4003 | Train Loss Components: Auxiliary: 0.1869 | Cardinality: 0.1024 | Classification: 0.0769 | Classification_binary: 0.1550 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0823 | Regression_eta: 0.0011 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0335 | Classification_binary: 0.1379 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0392 | Regression_eta: 0.0304 | Regression_pt: 0.0433 | Regression_sin_phi: 0.0507 | LR: 1.24e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.5706s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0072 | max_forward: 0.7883 | max_loss: 0.5033 | max_backward: 1.9727 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6090.2 | uss_MB: 5455.2 | child_rss_MB: 4956.4 | allocated_MB: 496.5 | reserved_MB: 27958.0 | inactive_split_MB: 778.0 +rank0 [2026-10-30 03:28:43,247] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 41% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 51.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:33:13,151] INFO : Step 44900/50000 rank0 | Train Loss: 0.8358 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.1024 | Classification: 0.0797 | Classification_binary: 0.1294 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0859 | Regression_eta: 0.0011 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1179 | Regression_cos_phi: 0.0307 | Regression_energy: 0.0491 | Regression_eta: 0.0256 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0337 | LR: 1.23e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.0275s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7360 | max_loss: 0.1482 | max_backward: 1.6886 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7825 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6084.1 | uss_MB: 5452.9 | child_rss_MB: 4956.9 | allocated_MB: 494.3 | reserved_MB: 27958.0 | inactive_split_MB: 922.0 +rank0 [2026-10-30 03:33:13,391] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:37:55,471] INFO : Step 45000/50000 rank0 | Train Loss: 0.8678 | Train Loss Unweighted: 0.3957 | Train Loss Components: Auxiliary: 0.1790 | Cardinality: 0.1036 | Classification: 0.0801 | Classification_binary: 0.1237 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0941 | Regression_eta: 0.0013 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0435 | Regression_eta: 0.0444 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0813 | LR: 1.22e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.3346s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7189 | max_loss: 0.2183 | max_backward: 1.8172 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.8812 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6058.9 | uss_MB: 5454.9 | child_rss_MB: 4844.5 | allocated_MB: 495.0 | reserved_MB: 27960.0 | inactive_split_MB: 978.5 +rank0 [2026-10-30 03:37:55,712] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 44% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 46% 100% +6 10 0x7408, 62036 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:37:55,733] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-45000.pth +rank0 [2026-10-30 03:37:56,031] INFO : Running validation on rank0 +rank0 [2026-10-30 03:37:56,031] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 03:37:56,031] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 03:38:46,797] INFO : VALIDATION | Step=45000/50000 | Train Loss=0.8678 | Valid Loss=0.8551 | Train Loss Unweighted=0.3957 | Valid Loss Unweighted=0.4021 | Stale=0 | ETA=85.4m +rank0 [2026-10-30 03:38:46,797] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 03:38:46,797] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 03:38:46,797] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 03:38:46,797] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 03:38:46,797] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 03:38:46,797] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 03:38:46,797] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 03:38:46,797] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 03:38:46,797] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/ari=0.606583 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.709733 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/top1_competing=0.817274 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.67096 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0574299 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0964855 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 03:38:46,798] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/seconds_per_rank=21.3284 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 03:38:46,799] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding/ari=0.299854 +rank0 [2026-10-30 03:38:46,800] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-30 03:38:46,800] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/embedding_minus_raw/ari=0.20823 +rank0 [2026-10-30 03:38:46,800] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 03:38:46,800] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 03:38:46,800] INFO : VALIDATION_PROBE | Step=45000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 03:38:46,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 03:38:47,063] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 03:38:47,067] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 03:39:14,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 03:39:15,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 03:39:17,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 03:39:19,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 03:39:29,781] INFO : Time taken to make predictions on device 0 is: 0.29 min +rank0 [2026-10-30 03:39:29,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 03:39:30,322] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 03:39:30,326] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 03:39:46,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 03:39:47,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 03:39:49,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 03:39:50,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 03:40:00,132] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 03:40:00,143] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 03:40:00,674] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 03:40:00,678] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 03:40:17,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 03:40:18,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 03:40:18,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 03:40:19,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 03:40:29,896] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 03:40:29,912] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits +rank0 [2026-10-30 03:40:33,131] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits +rank0 [2026-10-30 03:40:34,149] INFO : Plotted number of elements +rank0 [2026-10-30 03:40:35,713] INFO : Plotted jets +rank0 [2026-10-30 03:40:36,917] INFO : Plotted jet ratio +rank0 [2026-10-30 03:40:36,995] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 03:40:40,252] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 03:40:41,267] INFO : Plotted number of elements +rank0 [2026-10-30 03:40:42,776] INFO : Plotted jets +rank0 [2026-10-30 03:40:43,927] INFO : Plotted jet ratio +rank0 [2026-10-30 03:40:43,983] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits +rank0 [2026-10-30 03:40:46,885] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits +rank0 [2026-10-30 03:40:47,672] INFO : Plotted number of elements +rank0 [2026-10-30 03:40:49,245] INFO : Plotted jets +rank0 [2026-10-30 03:40:50,445] INFO : Plotted jet ratio +rank0 [2026-10-30 03:45:40,228] INFO : Step 45100/50000 rank0 | Train Loss: 0.8712 | Train Loss Unweighted: 0.4129 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0997 | Classification: 0.0837 | Classification_binary: 0.1445 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0910 | Regression_eta: 0.0012 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1493 | Regression_cos_phi: 0.0633 | Regression_energy: 0.0515 | Regression_eta: 0.0421 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0602 | LR: 1.21e-04 | DataLoad Time: 0.0147s | Model Forward Time: 6.0654s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8448 | max_loss: 0.1523 | max_backward: 2.1445 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 1.0003 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7489.1 | uss_MB: 6883.8 | child_rss_MB: 4866.9 | allocated_MB: 499.4 | reserved_MB: 24746.0 | inactive_split_MB: 784.5 +rank0 [2026-10-30 03:45:40,471] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 32% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 50.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:50:47,066] INFO : Step 45200/50000 rank0 | Train Loss: 0.9130 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1058 | Classification: 0.0776 | Classification_binary: 0.1464 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0963 | Regression_eta: 0.0015 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1059 | Regression_cos_phi: 0.0296 | Regression_energy: 0.0736 | Regression_eta: 0.0440 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0466 | LR: 1.20e-04 | DataLoad Time: 0.0156s | Model Forward Time: 7.2298s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1574 | max_backward: 1.6440 | max_optimizer_step: 0.2111 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1112 | Memory: rss_MB: 6132.8 | uss_MB: 4597.7 | child_rss_MB: 4866.1 | allocated_MB: 499.7 | reserved_MB: 26730.0 | inactive_split_MB: 1073.5 +rank0 [2026-10-30 03:50:47,308] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 52.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 71% +2 6 0x7408, 15961 51.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 98% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 44% 3% +4 8 0x7408, 13395 54.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 37% +6 10 0x7408, 62036 51.0°C 135.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 72% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 03:55:44,657] INFO : Step 45300/50000 rank0 | Train Loss: 0.8299 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.1126 | Classification: 0.0744 | Classification_binary: 0.1207 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0312 | Classification_binary: 0.1087 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0474 | Regression_eta: 0.0454 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0473 | LR: 1.19e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8344s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0827 | max_forward: 0.7966 | max_loss: 0.1432 | max_backward: 1.5141 | max_optimizer_step: 0.2646 | max_tensorboard_logging: 0.6448 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6128.3 | uss_MB: 5538.9 | child_rss_MB: 4864.4 | allocated_MB: 495.4 | reserved_MB: 27170.0 | inactive_split_MB: 881.8 +rank0 [2026-10-30 03:55:44,899] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 176.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 55.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:00:48,010] INFO : Step 45400/50000 rank0 | Train Loss: 0.8255 | Train Loss Unweighted: 0.4011 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.0984 | Classification: 0.0907 | Classification_binary: 0.1338 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0870 | Regression_eta: 0.0011 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1801 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0417 | Regression_eta: 0.0412 | Regression_pt: 0.0459 | Regression_sin_phi: 0.0462 | LR: 1.19e-04 | DataLoad Time: 0.0157s | Model Forward Time: 6.0138s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7846 | max_loss: 0.1934 | max_backward: 1.9857 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 0.7880 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6124.9 | uss_MB: 4519.9 | child_rss_MB: 4862.5 | allocated_MB: 499.2 | reserved_MB: 27170.0 | inactive_split_MB: 989.3 +rank0 [2026-10-30 04:00:48,257] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 99% +2 6 0x7408, 15961 52.0°C 150.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 25% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 14% +4 8 0x7408, 13395 55.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 95% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 149.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 13% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 20% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:05:45,569] INFO : Step 45500/50000 rank0 | Train Loss: 0.8945 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1090 | Classification: 0.0825 | Classification_binary: 0.1567 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0012 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0435 | Regression_eta: 0.0300 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0280 | LR: 1.18e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1734s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8686 | max_loss: 0.1508 | max_backward: 1.9581 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.4909 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6130.5 | uss_MB: 5543.1 | child_rss_MB: 4864.9 | allocated_MB: 498.4 | reserved_MB: 28766.0 | inactive_split_MB: 883.2 +rank0 [2026-10-30 04:05:45,808] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 55.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:10:40,400] INFO : Step 45600/50000 rank0 | Train Loss: 0.8704 | Train Loss Unweighted: 0.4121 | Train Loss Components: Auxiliary: 0.1840 | Cardinality: 0.1126 | Classification: 0.0802 | Classification_binary: 0.1324 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0983 | Regression_eta: 0.0010 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1251 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0680 | Regression_eta: 0.0267 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0496 | LR: 1.17e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.5406s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0163 | max_forward: 0.8161 | max_loss: 0.2729 | max_backward: 2.1846 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6122.7 | uss_MB: 5488.3 | child_rss_MB: 4871.5 | allocated_MB: 495.6 | reserved_MB: 28766.0 | inactive_split_MB: 1080.4 +rank0 [2026-10-30 04:10:40,643] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 2% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 42% 3% +2 6 0x7408, 15961 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 99% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 55.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 69% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:15:37,875] INFO : Step 45700/50000 rank0 | Train Loss: 0.8006 | Train Loss Unweighted: 0.3784 | Train Loss Components: Auxiliary: 0.1681 | Cardinality: 0.0873 | Classification: 0.0773 | Classification_binary: 0.1111 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0012 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1130 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0483 | Regression_eta: 0.0410 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0753 | LR: 1.16e-04 | DataLoad Time: 0.0105s | Model Forward Time: 5.4672s | Phase Times: max_pre_step_barrier: 0.0327 | max_batch_to_device: 0.0149 | max_forward: 1.1858 | max_loss: 0.1429 | max_backward: 1.6968 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1366 | Memory: rss_MB: 6126.5 | uss_MB: 5540.3 | child_rss_MB: 4873.0 | allocated_MB: 496.6 | reserved_MB: 28766.0 | inactive_split_MB: 903.7 +rank0 [2026-10-30 04:15:38,130] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 54.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 61.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:20:29,583] INFO : Step 45800/50000 rank0 | Train Loss: 0.8566 | Train Loss Unweighted: 0.3982 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1097 | Classification: 0.0794 | Classification_binary: 0.1398 | HitGrouping: 0.1406 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0631 | Regression_eta: 0.0369 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0492 | LR: 1.16e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.9042s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.9805 | max_loss: 0.5234 | max_backward: 2.3872 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.8153 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6131.1 | uss_MB: 5547.8 | child_rss_MB: 4874.9 | allocated_MB: 501.7 | reserved_MB: 28766.0 | inactive_split_MB: 1064.3 +rank0 [2026-10-30 04:20:29,835] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 160.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 55.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:25:17,511] INFO : Step 45900/50000 rank0 | Train Loss: 0.8754 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1042 | Classification: 0.0696 | Classification_binary: 0.1484 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1231 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0592 | Regression_eta: 0.0249 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0432 | LR: 1.15e-04 | DataLoad Time: 0.0157s | Model Forward Time: 6.5481s | Phase Times: max_pre_step_barrier: 0.0308 | max_batch_to_device: 0.0265 | max_forward: 1.7626 | max_loss: 0.1678 | max_backward: 2.6283 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 6124.4 | uss_MB: 5544.6 | child_rss_MB: 4869.8 | allocated_MB: 496.0 | reserved_MB: 29170.0 | inactive_split_MB: 983.4 +rank0 [2026-10-30 04:25:17,769] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 55.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:29:52,807] INFO : Step 46000/50000 rank0 | Train Loss: 0.7531 | Train Loss Unweighted: 0.3556 | Train Loss Components: Auxiliary: 0.1535 | Cardinality: 0.0870 | Classification: 0.0690 | Classification_binary: 0.1026 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0903 | Regression_eta: 0.0011 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0297 | Classification_binary: 0.0840 | Regression_cos_phi: 0.0341 | Regression_energy: 0.0388 | Regression_eta: 0.0281 | Regression_pt: 0.0416 | Regression_sin_phi: 0.0514 | LR: 1.14e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.1289s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0329 | max_forward: 0.7497 | max_loss: 0.1389 | max_backward: 1.9140 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5104 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6122.4 | uss_MB: 5543.2 | child_rss_MB: 4869.8 | allocated_MB: 492.4 | reserved_MB: 29170.0 | inactive_split_MB: 973.8 +rank0 [2026-10-30 04:29:53,060] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 48% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 56.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:29:53,081] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-46000.pth +rank0 [2026-10-30 04:29:53,390] INFO : Running validation on rank0 +rank0 [2026-10-30 04:29:53,390] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 04:29:53,390] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION | Step=46000/50000 | Train Loss=0.7531 | Valid Loss=0.8554 | Train Loss Unweighted=0.3556 | Valid Loss Unweighted=0.4004 | Stale=1 | ETA=71.4m +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 04:30:44,845] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/ari=0.604303 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/ari_threshold=0.85 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.721918 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/top1_competing=0.833642 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.669466 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0551504 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0949913 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 04:30:44,846] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/seconds_per_rank=21.3543 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 04:30:44,847] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 04:30:44,848] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding/ari=0.313443 +rank0 [2026-10-30 04:30:44,848] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-30 04:30:44,848] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/embedding_minus_raw/ari=0.22182 +rank0 [2026-10-30 04:30:44,848] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 04:30:44,848] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 04:30:44,848] INFO : VALIDATION_PROBE | Step=46000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 04:30:44,852] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 04:30:45,122] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 04:30:45,130] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 04:31:11,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 04:31:12,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 04:31:13,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 04:31:15,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 04:31:25,038] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 04:31:25,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 04:31:25,579] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 04:31:25,582] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 04:31:41,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 04:31:42,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 04:31:43,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 04:31:44,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 04:31:54,967] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 04:31:54,978] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 04:31:55,463] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 04:31:55,468] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 04:32:10,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 04:32:11,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 04:32:12,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 04:32:13,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 04:32:23,731] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 04:32:23,854] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits +rank0 [2026-10-30 04:32:27,722] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits +rank0 [2026-10-30 04:32:28,870] INFO : Plotted number of elements +rank0 [2026-10-30 04:32:30,588] INFO : Plotted jets +rank0 [2026-10-30 04:32:31,856] INFO : Plotted jet ratio +rank0 [2026-10-30 04:32:31,907] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 04:32:35,227] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 04:32:36,300] INFO : Plotted number of elements +rank0 [2026-10-30 04:32:38,296] INFO : Plotted jets +rank0 [2026-10-30 04:32:39,427] INFO : Plotted jet ratio +rank0 [2026-10-30 04:32:39,482] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits +rank0 [2026-10-30 04:32:41,877] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits +rank0 [2026-10-30 04:32:42,725] INFO : Plotted number of elements +rank0 [2026-10-30 04:32:44,228] INFO : Plotted jets +rank0 [2026-10-30 04:32:45,381] INFO : Plotted jet ratio +rank0 [2026-10-30 04:37:28,626] INFO : Step 46100/50000 rank0 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3769 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.0999 | Classification: 0.0743 | Classification_binary: 0.1253 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0010 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0509 | Regression_eta: 0.0290 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0499 | LR: 1.13e-04 | DataLoad Time: 0.0138s | Model Forward Time: 5.7609s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.9293 | max_loss: 0.1452 | max_backward: 2.3008 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.4968 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1219 | Memory: rss_MB: 7472.5 | uss_MB: 6889.9 | child_rss_MB: 4882.8 | allocated_MB: 498.3 | reserved_MB: 23298.0 | inactive_split_MB: 776.1 +rank0 [2026-10-30 04:37:28,868] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 164.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +2 6 0x7408, 15961 51.0°C 157.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +4 8 0x7408, 13395 55.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:42:20,259] INFO : Step 46200/50000 rank0 | Train Loss: 0.8661 | Train Loss Unweighted: 0.3994 | Train Loss Components: Auxiliary: 0.1947 | Cardinality: 0.1164 | Classification: 0.0793 | Classification_binary: 0.1511 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0833 | Regression_eta: 0.0011 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1578 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0432 | Regression_eta: 0.0310 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0428 | LR: 1.13e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.3706s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8070 | max_loss: 0.2423 | max_backward: 1.9860 | max_optimizer_step: 0.2482 | max_tensorboard_logging: 0.7539 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6199.1 | uss_MB: 5617.8 | child_rss_MB: 4859.8 | allocated_MB: 499.3 | reserved_MB: 25288.0 | inactive_split_MB: 1325.5 +rank0 [2026-10-30 04:42:20,502] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 99% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 99% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 99% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:47:00,079] INFO : Step 46300/50000 rank0 | Train Loss: 0.8634 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0916 | Classification: 0.0803 | Classification_binary: 0.1320 | HitGrouping: 0.1536 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1031 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1273 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0632 | Regression_eta: 0.0295 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0706 | LR: 1.12e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1173s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7891 | max_loss: 0.1470 | max_backward: 1.7900 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.7709 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6201.3 | uss_MB: 5618.4 | child_rss_MB: 4887.4 | allocated_MB: 498.4 | reserved_MB: 25290.0 | inactive_split_MB: 977.6 +rank0 [2026-10-30 04:47:00,331] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:51:37,600] INFO : Step 46400/50000 rank0 | Train Loss: 0.8499 | Train Loss Unweighted: 0.4102 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.0956 | Classification: 0.0755 | Classification_binary: 0.1172 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0258 | Regression_energy: 0.0537 | Regression_eta: 0.0330 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0343 | LR: 1.11e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.5938s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7847 | max_loss: 0.2487 | max_backward: 1.9181 | max_optimizer_step: 0.3148 | max_tensorboard_logging: 0.9408 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6188.6 | uss_MB: 5608.9 | child_rss_MB: 4878.3 | allocated_MB: 501.6 | reserved_MB: 25290.0 | inactive_split_MB: 992.4 +rank0 [2026-10-30 04:51:37,861] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 155.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 50.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 04:56:11,590] INFO : Step 46500/50000 rank0 | Train Loss: 0.8687 | Train Loss Unweighted: 0.4055 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1002 | Classification: 0.0822 | Classification_binary: 0.1448 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0872 | Regression_eta: 0.0013 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1437 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0485 | Regression_eta: 0.0417 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0447 | LR: 1.11e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2286s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1839 | max_backward: 1.9629 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6193.9 | uss_MB: 5618.8 | child_rss_MB: 4880.0 | allocated_MB: 498.8 | reserved_MB: 29306.0 | inactive_split_MB: 924.4 +rank0 [2026-10-30 04:56:11,831] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:00:44,861] INFO : Step 46600/50000 rank0 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4049 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1082 | Classification: 0.0757 | Classification_binary: 0.1508 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1358 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0468 | Regression_eta: 0.0430 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0316 | LR: 1.10e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9833s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7939 | max_loss: 0.2001 | max_backward: 1.6859 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6188.5 | uss_MB: 5615.4 | child_rss_MB: 4846.0 | allocated_MB: 496.0 | reserved_MB: 29306.0 | inactive_split_MB: 901.7 +rank0 [2026-10-30 05:00:45,106] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 176.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 189.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 182.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:05:19,083] INFO : Step 46700/50000 rank0 | Train Loss: 0.8417 | Train Loss Unweighted: 0.3855 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0982 | Classification: 0.0807 | Classification_binary: 0.1184 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0916 | Regression_eta: 0.0015 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1131 | Regression_cos_phi: 0.0361 | Regression_energy: 0.0574 | Regression_eta: 0.0324 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0629 | LR: 1.10e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.4079s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8551 | max_loss: 0.5124 | max_backward: 1.9004 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.7762 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1375 | Memory: rss_MB: 6185.4 | uss_MB: 5616.8 | child_rss_MB: 4868.3 | allocated_MB: 495.0 | reserved_MB: 29308.0 | inactive_split_MB: 1000.4 +rank0 [2026-10-30 05:05:19,337] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:09:49,136] INFO : Step 46800/50000 rank0 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3911 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0920 | Classification: 0.0920 | Classification_binary: 0.1296 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0828 | Regression_eta: 0.0013 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1170 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0447 | Regression_eta: 0.0261 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0388 | LR: 1.09e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2089s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7934 | max_loss: 0.1440 | max_backward: 1.7630 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.7219 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6183.1 | uss_MB: 5617.4 | child_rss_MB: 4871.4 | allocated_MB: 495.6 | reserved_MB: 29308.0 | inactive_split_MB: 1332.0 +rank0 [2026-10-30 05:09:49,390] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:14:20,430] INFO : Step 46900/50000 rank0 | Train Loss: 0.8496 | Train Loss Unweighted: 0.4142 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0976 | Classification: 0.0776 | Classification_binary: 0.1448 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0946 | Regression_eta: 0.0010 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1529 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0484 | Regression_eta: 0.0262 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0540 | LR: 1.09e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1422s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.9007 | max_loss: 0.1576 | max_backward: 1.8470 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6183.1 | uss_MB: 5616.1 | child_rss_MB: 4872.4 | allocated_MB: 495.4 | reserved_MB: 29310.0 | inactive_split_MB: 850.5 +rank0 [2026-10-30 05:14:20,672] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:18:50,441] INFO : Step 47000/50000 rank0 | Train Loss: 0.7980 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0982 | Classification: 0.0774 | Classification_binary: 0.1196 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1208 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0705 | Regression_eta: 0.0321 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0384 | LR: 1.08e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.5564s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 1.2405 | max_loss: 0.1502 | max_backward: 2.1273 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.7105 | max_tensorboard_flush: 0.0082 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6184.1 | uss_MB: 5617.4 | child_rss_MB: 4877.5 | allocated_MB: 494.9 | reserved_MB: 29310.0 | inactive_split_MB: 852.6 +rank0 [2026-10-30 05:18:50,701] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 163.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 48% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:18:50,827] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-47000.pth +rank0 [2026-10-30 05:18:51,147] INFO : Running validation on rank0 +rank0 [2026-10-30 05:18:51,147] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 05:18:51,147] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 05:19:42,309] INFO : VALIDATION | Step=47000/50000 | Train Loss=0.7980 | Valid Loss=0.8508 | Train Loss Unweighted=0.3866 | Valid Loss Unweighted=0.3965 | Stale=0 | ETA=55.5m +rank0 [2026-10-30 05:19:42,309] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 05:19:42,309] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 05:19:42,309] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/ari=0.618525 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/ari_threshold=0.85 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.716494 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/top1_competing=0.826962 +rank0 [2026-10-30 05:19:42,310] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.678309 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0693723 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.103834 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 05:19:42,311] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/seconds_per_rank=21.4257 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding/ari=0.299485 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/embedding_minus_raw/ari=0.207861 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 05:19:42,312] INFO : VALIDATION_PROBE | Step=47000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 05:19:42,318] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 05:19:42,587] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 05:19:42,590] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 05:20:07,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 05:20:09,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 05:20:10,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 05:20:11,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 05:20:21,383] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 05:20:21,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 05:20:22,014] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 05:20:22,018] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 05:20:38,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 05:20:39,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 05:20:40,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 05:20:41,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 05:20:51,694] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 05:20:51,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 05:20:52,202] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 05:20:52,205] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 05:21:07,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 05:21:08,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 05:21:09,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 05:21:10,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 05:21:20,330] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 05:21:20,475] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits +rank0 [2026-10-30 05:21:23,687] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits +rank0 [2026-10-30 05:21:24,689] INFO : Plotted number of elements +rank0 [2026-10-30 05:21:26,261] INFO : Plotted jets +rank0 [2026-10-30 05:21:27,457] INFO : Plotted jet ratio +rank0 [2026-10-30 05:21:27,536] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 05:21:30,764] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 05:21:31,771] INFO : Plotted number of elements +rank0 [2026-10-30 05:21:33,276] INFO : Plotted jets +rank0 [2026-10-30 05:21:34,403] INFO : Plotted jet ratio +rank0 [2026-10-30 05:21:34,471] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits +rank0 [2026-10-30 05:21:36,954] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits +rank0 [2026-10-30 05:21:37,801] INFO : Plotted number of elements +rank0 [2026-10-30 05:21:39,758] INFO : Plotted jets +rank0 [2026-10-30 05:21:40,934] INFO : Plotted jet ratio +rank0 [2026-10-30 05:26:13,016] INFO : Step 47100/50000 rank0 | Train Loss: 0.8046 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0851 | Classification: 0.0882 | Classification_binary: 0.1121 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0936 | Regression_eta: 0.0010 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1117 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0362 | Regression_eta: 0.0197 | Regression_pt: 0.0398 | Regression_sin_phi: 0.0315 | LR: 1.07e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.2084s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8250 | max_loss: 0.1453 | max_backward: 1.6267 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8108 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 7449.0 | uss_MB: 6878.6 | child_rss_MB: 4837.4 | allocated_MB: 494.1 | reserved_MB: 25416.0 | inactive_split_MB: 894.5 +rank0 [2026-10-30 05:26:13,263] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 31% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 55.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 33% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:30:45,207] INFO : Step 47200/50000 rank0 | Train Loss: 0.8884 | Train Loss Unweighted: 0.4191 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1138 | Classification: 0.0793 | Classification_binary: 0.1489 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0010 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1526 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0541 | Regression_eta: 0.0256 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0784 | LR: 1.07e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.5396s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7960 | max_loss: 0.1482 | max_backward: 1.9261 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.9435 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0661 | Memory: rss_MB: 6249.9 | uss_MB: 5682.5 | child_rss_MB: 4852.8 | allocated_MB: 496.8 | reserved_MB: 26534.0 | inactive_split_MB: 847.6 +rank0 [2026-10-30 05:30:45,466] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 161.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:35:15,897] INFO : Step 47300/50000 rank0 | Train Loss: 0.8547 | Train Loss Unweighted: 0.3920 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1049 | Classification: 0.0798 | Classification_binary: 0.1368 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0012 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0550 | Regression_eta: 0.0553 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0517 | LR: 1.06e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2306s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7649 | max_loss: 0.2313 | max_backward: 1.8030 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8438 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6253.9 | uss_MB: 5688.5 | child_rss_MB: 4855.9 | allocated_MB: 496.9 | reserved_MB: 28534.0 | inactive_split_MB: 933.0 +rank0 [2026-10-30 05:35:16,150] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 33% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 34% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +4 8 0x7408, 13395 56.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:39:47,695] INFO : Step 47400/50000 rank0 | Train Loss: 0.7606 | Train Loss Unweighted: 0.3755 | Train Loss Components: Auxiliary: 0.1640 | Cardinality: 0.0704 | Classification: 0.0737 | Classification_binary: 0.1072 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0960 | Regression_eta: 0.0010 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1017 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0511 | Regression_eta: 0.0237 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0416 | LR: 1.06e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2045s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0275 | max_forward: 0.8465 | max_loss: 0.1482 | max_backward: 1.7103 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.8388 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 6241.7 | uss_MB: 5678.8 | child_rss_MB: 4860.9 | allocated_MB: 496.4 | reserved_MB: 28534.0 | inactive_split_MB: 913.7 +rank0 [2026-10-30 05:39:47,947] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:44:18,683] INFO : Step 47500/50000 rank0 | Train Loss: 0.8134 | Train Loss Unweighted: 0.3681 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1057 | Classification: 0.0739 | Classification_binary: 0.1303 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0805 | Regression_eta: 0.0011 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1355 | Regression_cos_phi: 0.0316 | Regression_energy: 0.0448 | Regression_eta: 0.0197 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0610 | LR: 1.06e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.3843s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1751 | max_backward: 1.9857 | max_optimizer_step: 0.3263 | max_tensorboard_logging: 0.7363 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6249.7 | uss_MB: 5690.9 | child_rss_MB: 4845.9 | allocated_MB: 498.3 | reserved_MB: 28534.0 | inactive_split_MB: 851.5 +rank0 [2026-10-30 05:44:18,934] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 55.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:48:49,269] INFO : Step 47600/50000 rank0 | Train Loss: 0.8663 | Train Loss Unweighted: 0.4076 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1090 | Classification: 0.0820 | Classification_binary: 0.1491 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0865 | Regression_eta: 0.0011 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0375 | Regression_energy: 0.0424 | Regression_eta: 0.0230 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0540 | LR: 1.05e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.3673s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.9385 | max_loss: 0.1530 | max_backward: 2.1678 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6249.7 | uss_MB: 5691.7 | child_rss_MB: 4853.8 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 843.5 +rank0 [2026-10-30 05:48:49,513] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 164.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +4 8 0x7408, 13395 56.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:53:20,519] INFO : Step 47700/50000 rank0 | Train Loss: 0.8615 | Train Loss Unweighted: 0.4088 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0954 | Classification: 0.0764 | Classification_binary: 0.1291 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1002 | Regression_eta: 0.0011 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1160 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0719 | Regression_eta: 0.0441 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0523 | LR: 1.05e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7943s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7406 | max_loss: 0.2391 | max_backward: 1.7059 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1444 | Memory: rss_MB: 6248.7 | uss_MB: 5687.5 | child_rss_MB: 4851.8 | allocated_MB: 493.9 | reserved_MB: 28534.0 | inactive_split_MB: 895.3 +rank0 [2026-10-30 05:53:20,779] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 53.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 56.0°C 186.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 05:57:53,660] INFO : Step 47800/50000 rank0 | Train Loss: 0.8267 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.0942 | Classification: 0.0882 | Classification_binary: 0.1186 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0870 | Regression_eta: 0.0012 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0466 | Regression_eta: 0.0298 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0540 | LR: 1.04e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.8261s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.7869 | max_loss: 0.5030 | max_backward: 2.0522 | max_optimizer_step: 0.2267 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6249.7 | uss_MB: 5692.3 | child_rss_MB: 4857.3 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 916.9 +rank0 [2026-10-30 05:57:53,913] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 55.0°C 162.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:02:25,767] INFO : Step 47900/50000 rank0 | Train Loss: 0.8423 | Train Loss Unweighted: 0.4000 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.0845 | Classification: 0.0794 | Classification_binary: 0.1416 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1521 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0494 | Regression_eta: 0.0320 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0426 | LR: 1.04e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.1097s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7994 | max_loss: 0.1511 | max_backward: 1.9354 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.6685 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6247.4 | uss_MB: 5692.2 | child_rss_MB: 4860.2 | allocated_MB: 496.8 | reserved_MB: 30642.0 | inactive_split_MB: 779.5 +rank0 [2026-10-30 06:02:26,020] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 56.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 52.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:06:57,389] INFO : Step 48000/50000 rank0 | Train Loss: 0.8641 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0972 | Classification: 0.0768 | Classification_binary: 0.1396 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0890 | Regression_eta: 0.0015 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1303 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0548 | Regression_eta: 0.0312 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0466 | LR: 1.04e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.1511s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0057 | max_forward: 0.8017 | max_loss: 0.1550 | max_backward: 1.6990 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.7498 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6247.1 | uss_MB: 5690.2 | child_rss_MB: 4855.9 | allocated_MB: 495.1 | reserved_MB: 30642.0 | inactive_split_MB: 896.2 +rank0 [2026-10-30 06:06:57,652] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 36% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +2 6 0x7408, 15961 52.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 37% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 43% 100% +4 8 0x7408, 13395 56.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:06:57,674] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-48000.pth +rank0 [2026-10-30 06:06:57,985] INFO : Running validation on rank0 +rank0 [2026-10-30 06:06:57,985] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 06:06:57,985] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 06:07:49,559] INFO : VALIDATION | Step=48000/50000 | Train Loss=0.8641 | Valid Loss=0.8629 | Train Loss Unweighted=0.3968 | Valid Loss Unweighted=0.3989 | Stale=1 | ETA=38.3m +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 06:07:49,560] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/ari=0.619435 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/ari_threshold=0.85 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.728491 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/top1_competing=0.84163 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.672039 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0702824 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0975644 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 06:07:49,561] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/seconds_per_rank=21.4561 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding/ari=0.318958 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/embedding_minus_raw/ari=0.227335 +rank0 [2026-10-30 06:07:49,562] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 06:07:49,563] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 06:07:49,563] INFO : VALIDATION_PROBE | Step=48000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 06:07:49,567] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:07:49,814] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 06:07:49,818] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 06:08:14,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 06:08:15,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 06:08:16,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 06:08:17,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 06:08:28,002] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 06:08:28,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:08:28,620] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 06:08:28,623] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 06:08:44,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 06:08:45,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 06:08:47,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 06:08:48,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 06:08:58,192] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 06:08:58,203] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:08:58,714] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 06:08:58,718] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 06:09:14,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 06:09:15,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 06:09:16,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 06:09:16,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 06:09:26,998] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 06:09:27,015] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits +rank0 [2026-10-30 06:09:30,362] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits +rank0 [2026-10-30 06:09:31,400] INFO : Plotted number of elements +rank0 [2026-10-30 06:09:33,407] INFO : Plotted jets +rank0 [2026-10-30 06:09:34,599] INFO : Plotted jet ratio +rank0 [2026-10-30 06:09:34,668] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 06:09:37,937] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 06:09:38,950] INFO : Plotted number of elements +rank0 [2026-10-30 06:09:40,469] INFO : Plotted jets +rank0 [2026-10-30 06:09:41,592] INFO : Plotted jet ratio +rank0 [2026-10-30 06:09:41,658] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits +rank0 [2026-10-30 06:09:44,148] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits +rank0 [2026-10-30 06:09:45,003] INFO : Plotted number of elements +rank0 [2026-10-30 06:09:46,495] INFO : Plotted jets +rank0 [2026-10-30 06:09:47,646] INFO : Plotted jet ratio +rank0 [2026-10-30 06:14:17,264] INFO : Step 48100/50000 rank0 | Train Loss: 0.8226 | Train Loss Unweighted: 0.3895 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.1043 | Classification: 0.0717 | Classification_binary: 0.1116 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1023 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1133 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0661 | Regression_eta: 0.0253 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0415 | LR: 1.03e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1219s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7477 | max_loss: 0.1426 | max_backward: 1.7972 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.6718 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1303 | Memory: rss_MB: 7752.6 | uss_MB: 7191.3 | child_rss_MB: 4880.7 | allocated_MB: 494.1 | reserved_MB: 23078.0 | inactive_split_MB: 1096.3 +rank0 [2026-10-30 06:14:17,524] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 167.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 38% 100% +4 8 0x7408, 13395 55.0°C 177.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 32% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 35% 100% +6 10 0x7408, 62036 51.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:18:47,867] INFO : Step 48200/50000 rank0 | Train Loss: 0.8500 | Train Loss Unweighted: 0.3835 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1027 | Classification: 0.0771 | Classification_binary: 0.1374 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0827 | Regression_eta: 0.0012 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0388 | Regression_eta: 0.0330 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0521 | LR: 1.03e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.7981s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8132 | max_loss: 0.2545 | max_backward: 2.1167 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1241 | Memory: rss_MB: 6310.4 | uss_MB: 5751.6 | child_rss_MB: 4867.3 | allocated_MB: 496.6 | reserved_MB: 25094.0 | inactive_split_MB: 913.4 +rank0 [2026-10-30 06:18:48,121] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 156.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 33% 100% +2 6 0x7408, 15961 52.0°C 153.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 169.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 159.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:23:19,053] INFO : Step 48300/50000 rank0 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4303 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1089 | Classification: 0.0832 | Classification_binary: 0.1567 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0013 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0463 | Regression_eta: 0.0369 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0498 | LR: 1.03e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.0889s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8079 | max_loss: 0.1526 | max_backward: 1.9733 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6317.4 | uss_MB: 5760.9 | child_rss_MB: 4872.1 | allocated_MB: 498.5 | reserved_MB: 25098.0 | inactive_split_MB: 771.0 +rank0 [2026-10-30 06:23:19,295] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 180.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 53.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 191.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:27:49,767] INFO : Step 48400/50000 rank0 | Train Loss: 0.7961 | Train Loss Unweighted: 0.3733 | Train Loss Components: Auxiliary: 0.1682 | Cardinality: 0.0877 | Classification: 0.0685 | Classification_binary: 0.1191 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0012 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0325 | Classification_binary: 0.1013 | Regression_cos_phi: 0.0356 | Regression_energy: 0.0511 | Regression_eta: 0.0416 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0304 | LR: 1.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9776s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7916 | max_loss: 0.1471 | max_backward: 1.6943 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5940 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6314.4 | uss_MB: 5759.4 | child_rss_MB: 4842.4 | allocated_MB: 496.3 | reserved_MB: 25098.0 | inactive_split_MB: 1087.5 +rank0 [2026-10-30 06:27:50,011] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 192.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:32:21,693] INFO : Step 48500/50000 rank0 | Train Loss: 0.8668 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0816 | Classification: 0.0832 | Classification_binary: 0.1257 | HitGrouping: 0.1542 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0597 | Regression_eta: 0.0382 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0543 | LR: 1.02e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0489s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7587 | max_loss: 0.1606 | max_backward: 1.7573 | max_optimizer_step: 0.3024 | max_tensorboard_logging: 0.5851 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6309.3 | uss_MB: 5759.8 | child_rss_MB: 4851.4 | allocated_MB: 498.9 | reserved_MB: 25098.0 | inactive_split_MB: 1192.4 +rank0 [2026-10-30 06:32:21,932] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 53.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:36:53,745] INFO : Step 48600/50000 rank0 | Train Loss: 0.8434 | Train Loss Unweighted: 0.4010 | Train Loss Components: Auxiliary: 0.1850 | Cardinality: 0.0964 | Classification: 0.0791 | Classification_binary: 0.1425 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0489 | Regression_energy: 0.0382 | Regression_eta: 0.0237 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0413 | LR: 1.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2426s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7760 | max_loss: 0.1505 | max_backward: 1.8991 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0654 | Memory: rss_MB: 6311.3 | uss_MB: 5762.6 | child_rss_MB: 4856.0 | allocated_MB: 497.0 | reserved_MB: 26848.0 | inactive_split_MB: 777.5 +rank0 [2026-10-30 06:36:53,989] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 179.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:41:30,358] INFO : Step 48700/50000 rank0 | Train Loss: 0.9079 | Train Loss Unweighted: 0.4266 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1195 | Classification: 0.0920 | Classification_binary: 0.1456 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0478 | Regression_eta: 0.0335 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0555 | LR: 1.02e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.2181s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7797 | max_loss: 0.1578 | max_backward: 1.7107 | max_optimizer_step: 0.5280 | max_tensorboard_logging: 0.7650 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6308.9 | uss_MB: 5759.1 | child_rss_MB: 4857.7 | allocated_MB: 496.9 | reserved_MB: 26848.0 | inactive_split_MB: 891.8 +rank0 [2026-10-30 06:41:30,601] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 54.0°C 168.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 36% 100% +2 6 0x7408, 15961 53.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:46:03,317] INFO : Step 48800/50000 rank0 | Train Loss: 0.8011 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1685 | Cardinality: 0.0863 | Classification: 0.0837 | Classification_binary: 0.1116 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0965 | Regression_eta: 0.0012 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1288 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0497 | Regression_eta: 0.0273 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0537 | LR: 1.01e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2194s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7374 | max_loss: 0.2432 | max_backward: 1.8459 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.7427 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6306.9 | uss_MB: 5759.1 | child_rss_MB: 4857.0 | allocated_MB: 493.9 | reserved_MB: 26848.0 | inactive_split_MB: 991.0 +rank0 [2026-10-30 06:46:03,558] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 165.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 163.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 64.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:50:45,722] INFO : Step 48900/50000 rank0 | Train Loss: 0.8551 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.1109 | Classification: 0.0760 | Classification_binary: 0.1310 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0562 | Regression_eta: 0.0273 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0547 | LR: 1.01e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1796s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8768 | max_loss: 0.1470 | max_backward: 2.0816 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6306.9 | uss_MB: 5761.7 | child_rss_MB: 4858.9 | allocated_MB: 499.5 | reserved_MB: 26852.0 | inactive_split_MB: 1398.0 +rank0 [2026-10-30 06:50:45,966] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 175.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 188.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 51.0°C 175.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:55:21,411] INFO : Step 49000/50000 rank0 | Train Loss: 0.9005 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.2010 | Cardinality: 0.1248 | Classification: 0.0800 | Classification_binary: 0.1499 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0939 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1405 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0553 | Regression_eta: 0.0431 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0807 | LR: 1.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.7395s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0439 | max_forward: 0.8365 | max_loss: 0.1536 | max_backward: 1.9893 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8890 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6309.3 | uss_MB: 5762.6 | child_rss_MB: 4865.2 | allocated_MB: 497.5 | reserved_MB: 26852.0 | inactive_split_MB: 943.6 +rank0 [2026-10-30 06:55:21,667] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 38% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +2 6 0x7408, 15961 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 56.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 39% 100% +6 10 0x7408, 62036 52.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 42% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 06:55:21,689] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-49000.pth +rank0 [2026-10-30 06:55:21,995] INFO : Running validation on rank0 +rank0 [2026-10-30 06:55:21,995] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 06:55:21,995] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 06:56:13,849] INFO : VALIDATION | Step=49000/50000 | Train Loss=0.9005 | Valid Loss=0.8581 | Train Loss Unweighted=0.4208 | Valid Loss Unweighted=0.3984 | Stale=2 | ETA=19.7m +rank0 [2026-10-30 06:56:13,850] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 06:56:13,850] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 06:56:13,850] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 06:56:13,850] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 06:56:13,850] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 06:56:13,850] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 06:56:13,850] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 06:56:13,850] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 06:56:13,850] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 06:56:13,850] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/ari=0.611569 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/ari_threshold=0.83125 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.721722 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/top1_competing=0.833353 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.671108 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0624162 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0966333 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 06:56:13,851] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/seconds_per_rank=21.4842 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 06:56:13,852] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding/ari=0.301986 +rank0 [2026-10-30 06:56:13,853] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-30 06:56:13,853] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/embedding_minus_raw/ari=0.210362 +rank0 [2026-10-30 06:56:13,853] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 06:56:13,853] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 06:56:13,853] INFO : VALIDATION_PROBE | Step=49000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 06:56:13,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:56:14,117] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 06:56:14,121] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 06:56:38,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 06:56:40,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 06:56:41,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 06:56:42,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 06:56:52,536] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 06:56:52,549] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:56:53,028] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 06:56:53,032] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 06:57:09,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 06:57:10,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 06:57:11,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 06:57:12,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 06:57:22,621] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 06:57:22,632] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 06:57:23,151] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 06:57:23,154] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 06:57:38,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 06:57:39,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 06:57:40,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 06:57:41,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 06:57:51,684] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 06:57:51,700] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits +rank0 [2026-10-30 06:57:54,908] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits +rank0 [2026-10-30 06:57:55,907] INFO : Plotted number of elements +rank0 [2026-10-30 06:57:57,494] INFO : Plotted jets +rank0 [2026-10-30 06:57:58,690] INFO : Plotted jet ratio +rank0 [2026-10-30 06:57:58,761] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 06:58:01,925] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 06:58:02,946] INFO : Plotted number of elements +rank0 [2026-10-30 06:58:04,464] INFO : Plotted jets +rank0 [2026-10-30 06:58:05,605] INFO : Plotted jet ratio +rank0 [2026-10-30 06:58:05,660] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits +rank0 [2026-10-30 06:58:08,419] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits +rank0 [2026-10-30 06:58:09,159] INFO : Plotted number of elements +rank0 [2026-10-30 06:58:10,672] INFO : Plotted jets +rank0 [2026-10-30 06:58:11,831] INFO : Plotted jet ratio +rank0 [2026-10-30 07:02:46,292] INFO : Step 49100/50000 rank0 | Train Loss: 0.8219 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.0880 | Classification: 0.0841 | Classification_binary: 0.1303 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0855 | Regression_eta: 0.0012 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1089 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0545 | Regression_eta: 0.0278 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0350 | LR: 1.01e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0330s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7738 | max_loss: 0.1431 | max_backward: 1.6939 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6276 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 7720.3 | uss_MB: 7170.2 | child_rss_MB: 4829.0 | allocated_MB: 495.9 | reserved_MB: 24730.0 | inactive_split_MB: 936.0 +rank0 [2026-10-30 07:02:46,532] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 41% 100% +4 8 0x7408, 13395 55.0°C 187.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 36% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 181.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 38% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 99% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:07:19,288] INFO : Step 49200/50000 rank0 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3843 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0856 | Classification: 0.0829 | Classification_binary: 0.1188 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0892 | Regression_eta: 0.0011 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1199 | Regression_cos_phi: 0.0407 | Regression_energy: 0.0398 | Regression_eta: 0.0318 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0472 | LR: 1.01e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.3835s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7590 | max_loss: 0.5032 | max_backward: 1.9826 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.7926 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6365.4 | uss_MB: 5817.0 | child_rss_MB: 4830.4 | allocated_MB: 494.5 | reserved_MB: 26752.0 | inactive_split_MB: 1117.5 +rank0 [2026-10-30 07:07:19,532] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 166.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 160.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 56.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 41% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:11:51,482] INFO : Step 49300/50000 rank0 | Train Loss: 0.9397 | Train Loss Unweighted: 0.4282 | Train Loss Components: Auxiliary: 0.2065 | Cardinality: 0.1351 | Classification: 0.0889 | Classification_binary: 0.1552 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0901 | Regression_eta: 0.0012 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0574 | Regression_eta: 0.0302 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0579 | LR: 1.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1115s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0070 | max_forward: 0.7807 | max_loss: 0.1520 | max_backward: 1.8619 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6076 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6376.6 | uss_MB: 5827.0 | child_rss_MB: 4814.1 | allocated_MB: 496.4 | reserved_MB: 26752.0 | inactive_split_MB: 1120.6 +rank0 [2026-10-30 07:11:51,721] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 177.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 174.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +4 8 0x7408, 13395 56.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:16:23,056] INFO : Step 49400/50000 rank0 | Train Loss: 0.8870 | Train Loss Unweighted: 0.4213 | Train Loss Components: Auxiliary: 0.1937 | Cardinality: 0.1005 | Classification: 0.0815 | Classification_binary: 0.1465 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0274 | Regression_energy: 0.0467 | Regression_eta: 0.0280 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0496 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0747s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8046 | max_loss: 0.1540 | max_backward: 1.7244 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.6030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6373.7 | uss_MB: 5825.4 | child_rss_MB: 4813.8 | allocated_MB: 498.5 | reserved_MB: 28804.0 | inactive_split_MB: 886.2 +rank0 [2026-10-30 07:16:23,300] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 169.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 55.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 99% +2 6 0x7408, 15961 52.0°C 178.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 99% +3 7 0x7408, 3099 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 56.0°C 185.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 180.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 44% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:20:55,909] INFO : Step 49500/50000 rank0 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3949 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0991 | Classification: 0.0800 | Classification_binary: 0.1101 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0013 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0506 | Regression_eta: 0.0206 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0291 | LR: 1.00e-04 | DataLoad Time: 0.0148s | Model Forward Time: 5.8498s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0074 | max_forward: 0.9924 | max_loss: 0.1865 | max_backward: 2.5853 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.6095 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0429 | Memory: rss_MB: 6377.1 | uss_MB: 5695.1 | child_rss_MB: 4830.9 | allocated_MB: 495.9 | reserved_MB: 28804.0 | inactive_split_MB: 849.6 +rank0 [2026-10-30 07:20:56,150] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 159.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 53.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 55.0°C 166.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 40% 100% +6 10 0x7408, 62036 51.0°C 161.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:25:48,860] INFO : Step 49600/50000 rank0 | Train Loss: 0.9075 | Train Loss Unweighted: 0.4421 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1021 | Classification: 0.0939 | Classification_binary: 0.1401 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1529 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0512 | Regression_eta: 0.0336 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0640 | LR: 1.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1678s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7852 | max_loss: 0.1445 | max_backward: 2.0967 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4754 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6378.7 | uss_MB: 5829.2 | child_rss_MB: 4808.2 | allocated_MB: 497.8 | reserved_MB: 28804.0 | inactive_split_MB: 845.2 +rank0 [2026-10-30 07:25:49,102] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 174.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 172.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 55.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 52.0°C 171.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 47% 100% +7 11 0x7408, 49174 50.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:30:32,903] INFO : Step 49700/50000 rank0 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4307 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1074 | Classification: 0.0931 | Classification_binary: 0.1611 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0865 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0444 | Regression_eta: 0.0451 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0593 | LR: 1.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2362s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.8169 | max_loss: 0.1522 | max_backward: 2.0553 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.4937 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1402 | Memory: rss_MB: 6379.2 | uss_MB: 5828.6 | child_rss_MB: 4829.3 | allocated_MB: 499.3 | reserved_MB: 30718.0 | inactive_split_MB: 838.8 +rank0 [2026-10-30 07:30:33,158] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 173.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 56.0°C 184.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 173.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:35:17,875] INFO : Step 49800/50000 rank0 | Train Loss: 0.8342 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.1036 | Classification: 0.0769 | Classification_binary: 0.1292 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.0998 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0465 | Regression_eta: 0.0360 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0437 | LR: 1.00e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.9211s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0329 | max_forward: 0.7490 | max_loss: 0.1889 | max_backward: 2.2249 | max_optimizer_step: 0.2452 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6374.9 | uss_MB: 5812.4 | child_rss_MB: 4828.7 | allocated_MB: 494.4 | reserved_MB: 30718.0 | inactive_split_MB: 1134.4 +rank0 [2026-10-30 07:35:18,128] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 156.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 35% 1% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 46% +2 6 0x7408, 15961 52.0°C 157.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 42% 0% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 99% +4 8 0x7408, 13395 55.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 99% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 99% +6 10 0x7408, 62036 51.0°C 158.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 50% 0% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 37% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:39:58,932] INFO : Step 49900/50000 rank0 | Train Loss: 0.8436 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.0947 | Classification: 0.0943 | Classification_binary: 0.1179 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0982 | Regression_eta: 0.0012 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1320 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0558 | Regression_eta: 0.0392 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0640 | LR: 1.00e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.0439s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7496 | max_loss: 0.5142 | max_backward: 1.9365 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6373.9 | uss_MB: 5825.7 | child_rss_MB: 4831.8 | allocated_MB: 494.8 | reserved_MB: 30718.0 | inactive_split_MB: 1090.4 +rank0 [2026-10-30 07:39:59,174] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 172.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 170.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 55.0°C 190.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 39% 100% +5 9 0x7408, 1553 63.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 168.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:44:46,674] INFO : Step 50000/50000 rank0 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4040 | Train Loss Components: Auxiliary: 0.1921 | Cardinality: 0.1113 | Classification: 0.0818 | Classification_binary: 0.1436 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0872 | Regression_eta: 0.0014 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0483 | Regression_eta: 0.0249 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0441 | LR: 1.00e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.4230s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 1.3997 | max_loss: 0.5678 | max_backward: 3.0655 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6373.9 | uss_MB: 5825.4 | child_rss_MB: 4827.6 | allocated_MB: 497.9 | reserved_MB: 30720.0 | inactive_split_MB: 906.1 +rank0 [2026-10-30 07:44:46,917] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 53.0°C 154.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 35% 0% +1 5 0x7408, 51740 54.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 34% 100% +2 6 0x7408, 15961 52.0°C 165.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 45% 100% +3 7 0x7408, 3099 48.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 47% 100% +4 8 0x7408, 13395 55.0°C 167.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 42% 100% +5 9 0x7408, 1553 62.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 44% 100% +6 10 0x7408, 62036 51.0°C 154.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 50% 100% +7 11 0x7408, 49174 49.0°C N/A N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 0.0W 37% 100% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-30 07:44:46,939] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-50000.pth +rank0 [2026-10-30 07:44:47,248] INFO : Running validation on rank0 +rank0 [2026-10-30 07:44:47,248] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-30 07:44:47,248] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-30 07:45:39,094] INFO : VALIDATION | Step=50000/50000 | Train Loss=0.8857 | Valid Loss=0.8535 | Train Loss Unweighted=0.4040 | Valid Loss Unweighted=0.3940 | Stale=3 | ETA=0.0m +rank0 [2026-10-30 07:45:39,094] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_calibration_event_samples=32 +rank0 [2026-10-30 07:45:39,094] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_eval_event_samples=32 +rank0 [2026-10-30 07:45:39,094] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_fit_event_samples=64 +rank0 [2026-10-30 07:45:39,094] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/ari_fit_success_fraction=1 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/collected_event_samples=128 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/competing_candidates=0.788584 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/distance_matched_triplets=0.243377 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/positive_in_candidates=0.932944 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/same_particle_in_target_layer=0.937123 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/coverage/target_layer_exists=0.985667 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/top1_all_candidate_anchors=0.82525 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/top1_competing=0.959739 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/direction_distance/triplet_accuracy=0.573472 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/ari=0.61416 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/ari_threshold=0.85 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/top1_all_candidate_anchors=0.729185 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/top1_competing=0.842113 +rank0 [2026-10-30 07:45:39,095] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding/triplet_accuracy=0.6653 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding_minus_raw/ari=0.0650068 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/embedding_minus_raw/triplet_accuracy=0.0908251 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/eval_event_samples=32 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/eval_triplet_samples=56899 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_event_samples=96 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_success_fraction=1 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/fit_triplet_samples=192182 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/oracle_graph/ari=0.977443 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/ari=0.549153 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/ari_threshold=0.7 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/top1_all_candidate_anchors=0.722977 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/top1_competing=0.839162 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/raw/triplet_accuracy=0.574475 +rank0 [2026-10-30 07:45:39,096] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/seconds_per_rank=22.1968 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/top1_all_candidate_anchors=0.827833 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/top1_competing=0.963114 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/calorimeter/xyz_distance/triplet_accuracy=0.626763 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_calibration_event_samples=32 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_eval_event_samples=32 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_fit_event_samples=64 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/ari_fit_success_fraction=1 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding/ari=0.31102 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding/ari_threshold=0.85 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/embedding_minus_raw/ari=0.219397 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/oracle_graph/ari=0.854543 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/raw/ari=0.0916232 +rank0 [2026-10-30 07:45:39,097] INFO : VALIDATION_PROBE | Step=50000 | linear_probe/tracker/raw/ari_threshold=0.675 +rank0 [2026-10-30 07:45:39,103] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 07:45:39,368] INFO : test_dataset: cld_edm_ttbar_hits, 510 +rank0 [2026-10-30 07:45:39,372] INFO : Running predictions on cld_edm_ttbar_hits +rank0 [2026-10-30 07:46:04,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_0_0.parquet +rank0 [2026-10-30 07:46:05,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_0_1.parquet +rank0 [2026-10-30 07:46:06,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_0_2.parquet +rank0 [2026-10-30 07:46:07,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_0_3.parquet +rank0 [2026-10-30 07:46:17,498] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 07:46:17,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 07:46:17,996] INFO : test_dataset: cld_edm_ww_fullhad_hits, 510 +rank0 [2026-10-30 07:46:17,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank0 [2026-10-30 07:46:34,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_0_0.parquet +rank0 [2026-10-30 07:46:35,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_0_1.parquet +rank0 [2026-10-30 07:46:36,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_0_2.parquet +rank0 [2026-10-30 07:46:37,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_0_3.parquet +rank0 [2026-10-30 07:46:47,603] INFO : Time taken to make predictions on device 0 is: 0.26 min +rank0 [2026-10-30 07:46:47,615] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-30 07:46:48,154] INFO : test_dataset: cld_edm_qq_hits, 510 +rank0 [2026-10-30 07:46:48,158] INFO : Running predictions on cld_edm_qq_hits +rank0 [2026-10-30 07:47:04,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_0_0.parquet +rank0 [2026-10-30 07:47:05,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_0_1.parquet +rank0 [2026-10-30 07:47:05,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_0_2.parquet +rank0 [2026-10-30 07:47:06,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_0_3.parquet +rank0 [2026-10-30 07:47:16,861] INFO : Time taken to make predictions on device 0 is: 0.24 min +rank0 [2026-10-30 07:47:16,878] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits +rank0 [2026-10-30 07:47:20,111] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits +rank0 [2026-10-30 07:47:21,142] INFO : Plotted number of elements +rank0 [2026-10-30 07:47:22,732] INFO : Plotted jets +rank0 [2026-10-30 07:47:23,944] INFO : Plotted jet ratio +rank0 [2026-10-30 07:47:24,012] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 07:47:27,269] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits +rank0 [2026-10-30 07:47:28,319] INFO : Plotted number of elements +rank0 [2026-10-30 07:47:30,340] INFO : Plotted jets +rank0 [2026-10-30 07:47:31,488] INFO : Plotted jet ratio +rank0 [2026-10-30 07:47:31,541] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits +rank0 [2026-10-30 07:47:33,955] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits +rank0 [2026-10-30 07:47:34,804] INFO : Plotted number of elements +rank0 [2026-10-30 07:47:36,312] INFO : Plotted jets +rank0 [2026-10-30 07:47:37,487] INFO : Plotted jet ratio +rank0 [2026-10-30 07:47:38,023] INFO : Training completed. Total time on device 0: 1017.841min diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.1 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.1 new file mode 100644 index 0000000000000000000000000000000000000000..10447568f5367d69e0bcd06fc1f8f4e67f717574 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.1 @@ -0,0 +1,2631 @@ +rank1 [2026-10-28 11:12:32,434] INFO : Initializing model with seed=12345; process seed=12346 +rank1 [2026-10-28 11:12:32,435] INFO : configured dtype=torch.bfloat16 for autocast +rank1 [2026-10-28 11:12:54,158] INFO : Instantiating model +rank1 [2026-10-28 11:12:54,158] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank1 [2026-10-28 11:12:54,158] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank1 [2026-10-28 11:12:54,177] INFO : Input encoding initialization took 0.02s +rank1 [2026-10-28 11:12:54,177] INFO : nn0 parameters: 171776 +rank1 [2026-10-28 11:12:54,177] INFO : Initializing shared backbone layers, num_convs=6 +rank1 [2026-10-28 11:12:54,177] INFO : Initializing backbone layer 0 +rank1 [2026-10-28 11:12:54,215] INFO : Initializing backbone layer 1 +rank1 [2026-10-28 11:12:54,228] INFO : Initializing backbone layer 2 +rank1 [2026-10-28 11:12:54,236] INFO : Initializing backbone layer 3 +rank1 [2026-10-28 11:12:54,245] INFO : Initializing backbone layer 4 +rank1 [2026-10-28 11:12:54,253] INFO : Initializing backbone layer 5 +rank1 [2026-10-28 11:12:54,262] INFO : Backbone initialization took 0.08s +rank1 [2026-10-28 11:12:54,262] INFO : backbone parameters: 7901376 +rank1 [2026-10-28 11:12:54,262] INFO : Initializing output DNNs +rank1 [2026-10-28 11:12:54,294] INFO : Output DNNs initialization took 0.03s +rank1 [2026-10-28 11:12:54,295] INFO : backbone_mode=BackboneMode.SHARED +rank1 [2026-10-28 11:12:54,295] INFO : jagged_attention enabled=False +rank1 [2026-10-28 11:12:54,295] INFO : flash_attn_varlen enabled=False +rank1 [2026-10-28 11:12:54,295] INFO : task_queries enabled=False +rank1 [2026-10-28 11:12:54,295] INFO : classification_norm parameters: 0 +rank1 [2026-10-28 11:12:54,295] INFO : regression_norm parameters: 0 +rank1 [2026-10-28 11:12:54,295] INFO : classification_query parameters: 0 +rank1 [2026-10-28 11:12:54,295] INFO : regression_query parameters: 0 +rank1 [2026-10-28 11:12:54,295] INFO : classification_readout parameters: 0 +rank1 [2026-10-28 11:12:54,295] INFO : regression_readout parameters: 0 +rank1 [2026-10-28 11:12:54,295] INFO : output_mode=set +rank1 [2026-10-28 11:12:54,295] INFO : set_decoder parameters: 4713812 +rank1 [2026-10-28 11:12:54,295] INFO : nn_binary_particle parameters: 0 +rank1 [2026-10-28 11:12:54,296] INFO : nn_pid parameters: 0 +rank1 [2026-10-28 11:12:54,296] INFO : nn_pt parameters: 0 +rank1 [2026-10-28 11:12:54,296] INFO : nn_eta parameters: 0 +rank1 [2026-10-28 11:12:54,296] INFO : nn_sin_phi parameters: 0 +rank1 [2026-10-28 11:12:54,296] INFO : nn_cos_phi parameters: 0 +rank1 [2026-10-28 11:12:54,296] INFO : nn_energy parameters: 0 +rank1 [2026-10-28 11:12:54,296] INFO : Total MLPF parameters: 12786964 +rank1 [2026-10-28 11:12:54,296] INFO : MLPF __init__ done +rank1 [2026-10-28 11:12:54,297] INFO : Instantiated model +rank1 [2026-10-28 11:12:54,297] INFO : Moving model to device rank=1 +rank1 [2026-10-28 11:12:54,344] INFO : Moved model to device rank=1 +rank1 [2026-10-28 11:12:54,347] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank1 [2026-10-28 11:12:54,347] INFO : Configured model for SyncBatchNorm rank=1 +rank1 [2026-10-28 11:12:54,648] INFO : Configured model for DistributedDataParallel rank=1 +rank1 [2026-10-28 11:12:54,810] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank1 [2026-10-28 11:12:54,810] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank1 [2026-10-28 11:12:54,811] INFO : Trainable parameters: 12786964 +rank1 [2026-10-28 11:12:54,811] INFO : Non-trainable parameters: 23268 +rank1 [2026-10-28 11:12:54,811] INFO : Total parameters: 12810232 +rank1 [2026-10-28 11:12:54,814] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank1 [2026-10-28 11:12:54,815] INFO : Getting dataloaders +rank1 [2026-10-28 11:12:54,815] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:03,741] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:04,810] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,149] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-28 11:13:06,149] INFO : Creating WeightedMixIterator with 1 data loaders. +rank1 [2026-10-28 11:13:06,149] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,418] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,680] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:13:06,933] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-28 11:13:06,933] INFO : Creating InterleavedIterator with 1 data loaders. +rank1 [2026-10-28 11:13:21,079] INFO : Got dataloaders +rank1 [2026-10-28 11:13:21,079] INFO : loader split=train rank=1 len=21112 +rank1 [2026-10-28 11:13:21,079] INFO : loader split=valid rank=1 len=12 +rank1 [2026-10-28 11:13:21,079] INFO : Starting training from step 1 to 50000 on rank 1 of 8 +rank1 [2026-10-28 11:13:21,079] INFO : Creating train_iterator +rank1 [2026-10-28 11:13:21,079] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-28 11:13:35,657] INFO : Created train_iterator +rank1 [2026-10-28 11:18:11,208] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5408100029081107, 'Classification': 0.8863177470117808, 'Regression_pt': 0.9386234817653895, 'Regression_eta': 0.019238746587652712, 'Regression_sin_phi': 0.006576384851359762, 'Regression_cos_phi': 0.006405904629500583, 'Regression_energy': 1.0145655398070812}, weights={'Classification_binary': 1.0, 'Classification': 0.6101762056350708, 'Regression_pt': 0.5761735439300537, 'Regression_eta': 28.110458374023438, 'Regression_sin_phi': 82.23515319824219, 'Regression_cos_phi': 84.42367553710938, 'Regression_energy': 0.5330458879470825} +rank1 [2026-10-28 11:18:13,719] INFO : Step 100/50000 rank1 | Train Loss: 4.2631 | Train Loss Unweighted: 2.7366 | Train Loss Components: Auxiliary: 0.7941 | Cardinality: 0.3942 | Classification: 0.6312 | Classification_binary: 0.4549 | HitGrouping: 0.3381 | Regression_cos_phi: 0.0047 | Regression_energy: 0.8263 | Regression_eta: 0.0122 | Regression_pt: 0.8026 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.6043 | Classification_binary: 0.4936 | Regression_cos_phi: 0.0051 | Regression_energy: 0.7039 | Regression_eta: 0.0106 | Regression_pt: 0.7190 | Regression_sin_phi: 0.0048 | LR: 1.00e-03 | DataLoad Time: 0.0169s | Model Forward Time: 3.5375s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6509 | max_loss: 0.1961 | max_backward: 1.4166 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4769.1 | uss_MB: 3475.5 | child_rss_MB: 6372.1 | allocated_MB: 446.7 | reserved_MB: 24804.0 | inactive_split_MB: 1063.6 +rank1 [2026-10-28 11:22:42,538] INFO : Step 200/50000 rank1 | Train Loss: 4.2029 | Train Loss Unweighted: 2.5447 | Train Loss Components: Auxiliary: 0.8074 | Cardinality: 0.4500 | Classification: 0.5373 | Classification_binary: 0.5135 | HitGrouping: 0.3101 | Regression_cos_phi: 0.0042 | Regression_energy: 0.7508 | Regression_eta: 0.0093 | Regression_pt: 0.7252 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.3009 | Classification_binary: 0.5517 | Regression_cos_phi: 0.3719 | Regression_energy: 0.4001 | Regression_eta: 0.2494 | Regression_pt: 0.4249 | Regression_sin_phi: 0.4074 | LR: 1.00e-03 | DataLoad Time: 0.0161s | Model Forward Time: 4.6624s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9214 | max_loss: 0.1502 | max_backward: 2.1220 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 4772.0 | uss_MB: 3480.3 | child_rss_MB: 6374.7 | allocated_MB: 449.0 | reserved_MB: 26252.0 | inactive_split_MB: 1054.4 +rank1 [2026-10-28 11:27:10,389] INFO : Step 300/50000 rank1 | Train Loss: 3.5114 | Train Loss Unweighted: 2.0975 | Train Loss Components: Auxiliary: 0.6791 | Cardinality: 0.3510 | Classification: 0.4201 | Classification_binary: 0.4975 | HitGrouping: 0.2993 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5752 | Regression_eta: 0.0073 | Regression_pt: 0.5905 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2591 | Classification_binary: 0.4779 | Regression_cos_phi: 0.2622 | Regression_energy: 0.3018 | Regression_eta: 0.1637 | Regression_pt: 0.3413 | Regression_sin_phi: 0.2553 | LR: 1.00e-03 | DataLoad Time: 0.0134s | Model Forward Time: 4.2620s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.7369 | max_loss: 0.1526 | max_backward: 1.7330 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.9030 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 4773.0 | uss_MB: 3484.3 | child_rss_MB: 6364.6 | allocated_MB: 446.1 | reserved_MB: 26252.0 | inactive_split_MB: 1068.0 +rank1 [2026-10-28 11:31:37,117] INFO : Step 400/50000 rank1 | Train Loss: 3.3615 | Train Loss Unweighted: 1.9456 | Train Loss Components: Auxiliary: 0.6310 | Cardinality: 0.3555 | Classification: 0.4058 | Classification_binary: 0.4436 | HitGrouping: 0.3006 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5363 | Regression_eta: 0.0080 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2568 | Classification_binary: 0.4431 | Regression_cos_phi: 0.2852 | Regression_energy: 0.3116 | Regression_eta: 0.1885 | Regression_pt: 0.3371 | Regression_sin_phi: 0.2344 | LR: 1.00e-03 | DataLoad Time: 0.0173s | Model Forward Time: 4.5025s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1151 | max_forward: 0.7642 | max_loss: 0.1384 | max_backward: 1.7906 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9359 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0880 | Memory: rss_MB: 4771.8 | uss_MB: 3484.8 | child_rss_MB: 6366.3 | allocated_MB: 446.8 | reserved_MB: 26252.0 | inactive_split_MB: 1064.0 +rank1 [2026-10-28 11:36:08,659] INFO : Step 500/50000 rank1 | Train Loss: 3.8257 | Train Loss Unweighted: 2.0630 | Train Loss Components: Auxiliary: 0.7010 | Cardinality: 0.4740 | Classification: 0.4380 | Classification_binary: 0.4817 | HitGrouping: 0.2961 | Regression_cos_phi: 0.0044 | Regression_energy: 0.5611 | Regression_eta: 0.0087 | Regression_pt: 0.5645 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2921 | Classification_binary: 0.5381 | Regression_cos_phi: 0.4189 | Regression_energy: 0.3260 | Regression_eta: 0.2974 | Regression_pt: 0.3547 | Regression_sin_phi: 0.4874 | LR: 1.00e-03 | DataLoad Time: 0.0161s | Model Forward Time: 5.1261s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0077 | max_forward: 1.2433 | max_loss: 0.1536 | max_backward: 2.4885 | max_optimizer_step: 0.1562 | max_tensorboard_logging: 1.0374 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0034 | Memory: rss_MB: 4768.9 | uss_MB: 3486.1 | child_rss_MB: 6361.5 | allocated_MB: 449.6 | reserved_MB: 26256.0 | inactive_split_MB: 929.2 +rank1 [2026-10-28 11:40:42,093] INFO : Step 600/50000 rank1 | Train Loss: 3.4702 | Train Loss Unweighted: 1.9733 | Train Loss Components: Auxiliary: 0.6509 | Cardinality: 0.3585 | Classification: 0.3825 | Classification_binary: 0.5061 | HitGrouping: 0.2915 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5321 | Regression_eta: 0.0074 | Regression_pt: 0.5377 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2393 | Classification_binary: 0.5025 | Regression_cos_phi: 0.3608 | Regression_energy: 0.2694 | Regression_eta: 0.2403 | Regression_pt: 0.2919 | Regression_sin_phi: 0.3586 | LR: 1.00e-03 | DataLoad Time: 0.0156s | Model Forward Time: 4.5576s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8098 | max_loss: 0.1544 | max_backward: 1.9541 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 0.9377 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 4772.7 | uss_MB: 3486.1 | child_rss_MB: 6366.7 | allocated_MB: 449.5 | reserved_MB: 27396.0 | inactive_split_MB: 1022.5 +rank1 [2026-10-28 11:45:16,604] INFO : Step 700/50000 rank1 | Train Loss: 2.9693 | Train Loss Unweighted: 1.8072 | Train Loss Components: Auxiliary: 0.5907 | Cardinality: 0.2768 | Classification: 0.3175 | Classification_binary: 0.3969 | HitGrouping: 0.2848 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5321 | Regression_eta: 0.0061 | Regression_pt: 0.5492 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1903 | Classification_binary: 0.3679 | Regression_cos_phi: 0.1796 | Regression_energy: 0.2987 | Regression_eta: 0.1428 | Regression_pt: 0.3311 | Regression_sin_phi: 0.1477 | LR: 1.00e-03 | DataLoad Time: 0.0215s | Model Forward Time: 3.9456s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7910 | max_loss: 0.2436 | max_backward: 1.8672 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.8 | uss_MB: 3484.3 | child_rss_MB: 6369.5 | allocated_MB: 446.1 | reserved_MB: 27396.0 | inactive_split_MB: 1066.2 +rank1 [2026-10-28 11:49:48,175] INFO : Step 800/50000 rank1 | Train Loss: 3.1527 | Train Loss Unweighted: 1.8424 | Train Loss Components: Auxiliary: 0.6214 | Cardinality: 0.3518 | Classification: 0.3758 | Classification_binary: 0.3967 | HitGrouping: 0.2962 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5290 | Regression_eta: 0.0063 | Regression_pt: 0.5287 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2378 | Classification_binary: 0.4166 | Regression_cos_phi: 0.2195 | Regression_energy: 0.2695 | Regression_eta: 0.1228 | Regression_pt: 0.2924 | Regression_sin_phi: 0.1430 | LR: 9.99e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6444s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0093 | max_forward: 0.8507 | max_loss: 0.1493 | max_backward: 1.9706 | max_optimizer_step: 0.1582 | max_tensorboard_logging: 1.0840 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0072 | Memory: rss_MB: 4754.0 | uss_MB: 3484.3 | child_rss_MB: 6293.3 | allocated_MB: 450.9 | reserved_MB: 27396.0 | inactive_split_MB: 920.9 +rank1 [2026-10-28 11:54:30,570] INFO : Step 900/50000 rank1 | Train Loss: 3.2552 | Train Loss Unweighted: 1.9216 | Train Loss Components: Auxiliary: 0.6208 | Cardinality: 0.2901 | Classification: 0.3818 | Classification_binary: 0.4734 | HitGrouping: 0.2881 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5239 | Regression_eta: 0.0075 | Regression_pt: 0.5284 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2196 | Classification_binary: 0.5007 | Regression_cos_phi: 0.2732 | Regression_energy: 0.2968 | Regression_eta: 0.2404 | Regression_pt: 0.3160 | Regression_sin_phi: 0.2498 | LR: 9.99e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6738s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8050 | max_loss: 0.1486 | max_backward: 1.9075 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.9729 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4751.7 | uss_MB: 3485.4 | child_rss_MB: 6307.4 | allocated_MB: 449.7 | reserved_MB: 27406.0 | inactive_split_MB: 927.3 +rank1 [2026-10-28 11:59:03,649] INFO : Step 1000/50000 rank1 | Train Loss: 2.8964 | Train Loss Unweighted: 1.7179 | Train Loss Components: Auxiliary: 0.5694 | Cardinality: 0.2509 | Classification: 0.3299 | Classification_binary: 0.4129 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4804 | Regression_eta: 0.0063 | Regression_pt: 0.4827 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1663 | Classification_binary: 0.3924 | Regression_cos_phi: 0.2360 | Regression_energy: 0.2294 | Regression_eta: 0.1953 | Regression_pt: 0.2447 | Regression_sin_phi: 0.2349 | LR: 9.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4521s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7367 | max_loss: 0.2419 | max_backward: 1.8137 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 1.0408 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 4746.9 | uss_MB: 3485.7 | child_rss_MB: 6281.8 | allocated_MB: 449.1 | reserved_MB: 27406.0 | inactive_split_MB: 959.2 +rank1 [2026-10-28 11:59:04,552] INFO : Running validation on rank1 +rank1 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 11:59:57,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 11:59:57,577] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 12:00:26,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 12:00:27,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 12:00:28,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 12:00:29,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 12:00:39,987] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-28 12:00:40,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:00:41,020] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 12:00:56,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 12:00:57,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 12:00:58,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 12:00:59,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 12:01:09,903] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 12:01:10,610] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:01:11,135] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 12:01:26,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 12:01:27,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 12:01:28,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 12:01:29,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 12:01:39,340] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 12:06:47,466] INFO : Step 1100/50000 rank1 | Train Loss: 2.7351 | Train Loss Unweighted: 1.6531 | Train Loss Components: Auxiliary: 0.5524 | Cardinality: 0.2470 | Classification: 0.3238 | Classification_binary: 0.3516 | HitGrouping: 0.2744 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4831 | Regression_eta: 0.0054 | Regression_pt: 0.4842 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1786 | Classification_binary: 0.3236 | Regression_cos_phi: 0.1858 | Regression_energy: 0.2632 | Regression_eta: 0.1375 | Regression_pt: 0.2897 | Regression_sin_phi: 0.1810 | LR: 9.99e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7321s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7345 | max_loss: 0.2408 | max_backward: 1.8906 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 1.1618 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6205.6 | uss_MB: 4949.9 | child_rss_MB: 6308.4 | allocated_MB: 448.2 | reserved_MB: 24590.0 | inactive_split_MB: 1051.2 +rank1 [2026-10-28 12:11:28,614] INFO : Step 1200/50000 rank1 | Train Loss: 2.9931 | Train Loss Unweighted: 1.7517 | Train Loss Components: Auxiliary: 0.5886 | Cardinality: 0.2847 | Classification: 0.3460 | Classification_binary: 0.4151 | HitGrouping: 0.2795 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4874 | Regression_eta: 0.0065 | Regression_pt: 0.4908 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2294 | Classification_binary: 0.4409 | Regression_cos_phi: 0.3000 | Regression_energy: 0.2712 | Regression_eta: 0.1831 | Regression_pt: 0.3007 | Regression_sin_phi: 0.2403 | LR: 9.99e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.9385s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0061 | max_forward: 0.9910 | max_loss: 0.1662 | max_backward: 2.3256 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.8355 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 6205.6 | uss_MB: 4943.2 | child_rss_MB: 6333.1 | allocated_MB: 449.6 | reserved_MB: 24590.0 | inactive_split_MB: 1155.8 +rank1 [2026-10-28 12:16:08,306] INFO : Step 1300/50000 rank1 | Train Loss: 3.0304 | Train Loss Unweighted: 1.7607 | Train Loss Components: Auxiliary: 0.5900 | Cardinality: 0.2909 | Classification: 0.3397 | Classification_binary: 0.4400 | HitGrouping: 0.2683 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4807 | Regression_eta: 0.0062 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2177 | Classification_binary: 0.4178 | Regression_cos_phi: 0.2394 | Regression_energy: 0.2582 | Regression_eta: 0.1857 | Regression_pt: 0.2821 | Regression_sin_phi: 0.2467 | LR: 9.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3295s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8206 | max_loss: 0.1504 | max_backward: 1.9515 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5688 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5595.4 | uss_MB: 4349.7 | child_rss_MB: 6286.2 | allocated_MB: 448.6 | reserved_MB: 25778.0 | inactive_split_MB: 1050.0 +rank1 [2026-10-28 12:20:48,118] INFO : Step 1400/50000 rank1 | Train Loss: 2.5732 | Train Loss Unweighted: 1.5137 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2349 | Classification: 0.2906 | Classification_binary: 0.3573 | HitGrouping: 0.2641 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4252 | Regression_eta: 0.0055 | Regression_pt: 0.4303 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1730 | Classification_binary: 0.3487 | Regression_cos_phi: 0.2112 | Regression_energy: 0.2208 | Regression_eta: 0.1229 | Regression_pt: 0.2439 | Regression_sin_phi: 0.1986 | LR: 9.98e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.1757s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.8166 | max_loss: 0.1962 | max_backward: 1.8388 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4964 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5588.2 | uss_MB: 3765.0 | child_rss_MB: 6275.0 | allocated_MB: 454.5 | reserved_MB: 28272.0 | inactive_split_MB: 1143.7 +rank1 [2026-10-28 12:25:37,856] INFO : Step 1500/50000 rank1 | Train Loss: 2.8564 | Train Loss Unweighted: 1.5902 | Train Loss Components: Auxiliary: 0.5576 | Cardinality: 0.3788 | Classification: 0.3167 | Classification_binary: 0.3534 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4517 | Regression_eta: 0.0055 | Regression_pt: 0.4576 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1992 | Classification_binary: 0.3201 | Regression_cos_phi: 0.2195 | Regression_energy: 0.2338 | Regression_eta: 0.1396 | Regression_pt: 0.2573 | Regression_sin_phi: 0.1630 | LR: 9.98e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6153s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.0272 | max_loss: 0.2459 | max_backward: 1.8682 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.0883 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0105 | Memory: rss_MB: 5566.9 | uss_MB: 4329.3 | child_rss_MB: 6244.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 989.3 +rank1 [2026-10-28 12:30:20,922] INFO : Step 1600/50000 rank1 | Train Loss: 2.8293 | Train Loss Unweighted: 1.6754 | Train Loss Components: Auxiliary: 0.5537 | Cardinality: 0.2456 | Classification: 0.3224 | Classification_binary: 0.4168 | HitGrouping: 0.2726 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4572 | Regression_eta: 0.0058 | Regression_pt: 0.4676 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1872 | Classification_binary: 0.4661 | Regression_cos_phi: 0.2678 | Regression_energy: 0.2251 | Regression_eta: 0.1647 | Regression_pt: 0.2477 | Regression_sin_phi: 0.2886 | LR: 9.98e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.5707s | Phase Times: max_pre_step_barrier: 0.1094 | max_batch_to_device: 0.0066 | max_forward: 0.7720 | max_loss: 0.1510 | max_backward: 1.7462 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5578.6 | uss_MB: 4348.9 | child_rss_MB: 6226.8 | allocated_MB: 448.1 | reserved_MB: 28272.0 | inactive_split_MB: 989.9 +rank1 [2026-10-28 12:35:04,110] INFO : Step 1700/50000 rank1 | Train Loss: 2.7295 | Train Loss Unweighted: 1.5885 | Train Loss Components: Auxiliary: 0.5261 | Cardinality: 0.2505 | Classification: 0.2942 | Classification_binary: 0.3852 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4488 | Regression_eta: 0.0059 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1876 | Classification_binary: 0.3472 | Regression_cos_phi: 0.2051 | Regression_energy: 0.2284 | Regression_eta: 0.1416 | Regression_pt: 0.2468 | Regression_sin_phi: 0.1694 | LR: 9.97e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9355s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 1.0304 | max_loss: 0.4846 | max_backward: 1.9957 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0278 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5571.8 | uss_MB: 4349.5 | child_rss_MB: 6226.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 999.4 +rank1 [2026-10-28 12:39:46,515] INFO : Step 1800/50000 rank1 | Train Loss: 2.5541 | Train Loss Unweighted: 1.4989 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2504 | Classification: 0.2799 | Classification_binary: 0.3357 | HitGrouping: 0.2658 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4350 | Regression_eta: 0.0052 | Regression_pt: 0.4384 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1948 | Classification_binary: 0.3336 | Regression_cos_phi: 0.1515 | Regression_energy: 0.2459 | Regression_eta: 0.1271 | Regression_pt: 0.2632 | Regression_sin_phi: 0.1259 | LR: 9.97e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3595s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7316 | max_loss: 0.1436 | max_backward: 1.6570 | max_optimizer_step: 0.2625 | max_tensorboard_logging: 0.9220 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5563.3 | uss_MB: 4348.3 | child_rss_MB: 6170.2 | allocated_MB: 444.7 | reserved_MB: 28272.0 | inactive_split_MB: 991.9 +rank1 [2026-10-28 12:44:29,073] INFO : Step 1900/50000 rank1 | Train Loss: 2.7201 | Train Loss Unweighted: 1.5780 | Train Loss Components: Auxiliary: 0.5257 | Cardinality: 0.2710 | Classification: 0.3152 | Classification_binary: 0.3657 | HitGrouping: 0.2653 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4371 | Regression_eta: 0.0058 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1856 | Classification_binary: 0.3867 | Regression_cos_phi: 0.2608 | Regression_energy: 0.2143 | Regression_eta: 0.1790 | Regression_pt: 0.2387 | Regression_sin_phi: 0.2455 | LR: 9.97e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4050s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.7723 | max_loss: 0.1448 | max_backward: 2.0316 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5561.2 | uss_MB: 4335.4 | child_rss_MB: 6153.7 | allocated_MB: 448.9 | reserved_MB: 28272.0 | inactive_split_MB: 1166.2 +rank1 [2026-10-28 12:49:11,972] INFO : Step 2000/50000 rank1 | Train Loss: 2.7893 | Train Loss Unweighted: 1.5772 | Train Loss Components: Auxiliary: 0.5342 | Cardinality: 0.3149 | Classification: 0.3056 | Classification_binary: 0.4233 | HitGrouping: 0.2623 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4169 | Regression_eta: 0.0058 | Regression_pt: 0.4204 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1699 | Classification_binary: 0.3907 | Regression_cos_phi: 0.2460 | Regression_energy: 0.2431 | Regression_eta: 0.1315 | Regression_pt: 0.2627 | Regression_sin_phi: 0.1927 | LR: 9.96e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.5207s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0070 | max_forward: 1.0424 | max_loss: 0.2160 | max_backward: 2.3184 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.9849 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0132 | Memory: rss_MB: 5555.4 | uss_MB: 4316.1 | child_rss_MB: 6148.5 | allocated_MB: 451.3 | reserved_MB: 28272.0 | inactive_split_MB: 992.4 +rank1 [2026-10-28 12:49:12,925] INFO : Running validation on rank1 +rank1 [2026-10-28 12:49:12,926] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 12:49:12,926] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 12:50:07,789] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:50:08,360] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 12:50:34,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 12:50:35,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 12:50:36,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 12:50:37,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 12:50:47,435] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 12:50:48,334] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:50:48,898] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 12:51:05,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 12:51:06,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 12:51:07,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 12:51:08,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 12:51:18,141] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 12:51:18,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 12:51:18,823] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 12:51:35,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 12:51:35,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 12:51:36,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 12:51:37,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 12:51:47,621] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 12:57:01,909] INFO : Step 2100/50000 rank1 | Train Loss: 2.3990 | Train Loss Unweighted: 1.4049 | Train Loss Components: Auxiliary: 0.4671 | Cardinality: 0.2199 | Classification: 0.2491 | Classification_binary: 0.3399 | HitGrouping: 0.2605 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4015 | Regression_eta: 0.0046 | Regression_pt: 0.4053 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1489 | Classification_binary: 0.3260 | Regression_cos_phi: 0.1790 | Regression_energy: 0.2392 | Regression_eta: 0.1124 | Regression_pt: 0.2634 | Regression_sin_phi: 0.1701 | LR: 9.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8143s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7238 | max_loss: 0.1490 | max_backward: 1.8098 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.2800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6608.5 | uss_MB: 5401.4 | child_rss_MB: 6187.1 | allocated_MB: 444.8 | reserved_MB: 23654.0 | inactive_split_MB: 964.6 +rank1 [2026-10-28 13:01:41,080] INFO : Step 2200/50000 rank1 | Train Loss: 2.6105 | Train Loss Unweighted: 1.4907 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2489 | Classification: 0.2976 | Classification_binary: 0.3552 | HitGrouping: 0.2519 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4108 | Regression_eta: 0.0054 | Regression_pt: 0.4161 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1794 | Classification_binary: 0.3503 | Regression_cos_phi: 0.1922 | Regression_energy: 0.2771 | Regression_eta: 0.1483 | Regression_pt: 0.3055 | Regression_sin_phi: 0.2197 | LR: 9.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9411s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 1.0006 | max_loss: 0.1446 | max_backward: 1.7671 | max_optimizer_step: 0.2923 | max_tensorboard_logging: 1.2733 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6609.6 | uss_MB: 5403.3 | child_rss_MB: 6178.9 | allocated_MB: 445.7 | reserved_MB: 24780.0 | inactive_split_MB: 1132.5 +rank1 [2026-10-28 13:06:12,758] INFO : Step 2300/50000 rank1 | Train Loss: 2.7462 | Train Loss Unweighted: 1.5643 | Train Loss Components: Auxiliary: 0.5175 | Cardinality: 0.2724 | Classification: 0.3064 | Classification_binary: 0.4106 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4145 | Regression_eta: 0.0056 | Regression_pt: 0.4213 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2138 | Classification_binary: 0.4422 | Regression_cos_phi: 0.2965 | Regression_energy: 0.2169 | Regression_eta: 0.2079 | Regression_pt: 0.2392 | Regression_sin_phi: 0.2887 | LR: 9.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7168s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0353 | max_forward: 0.7930 | max_loss: 0.1572 | max_backward: 1.9535 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0140 | Memory: rss_MB: 5702.4 | uss_MB: 4498.0 | child_rss_MB: 6178.9 | allocated_MB: 451.2 | reserved_MB: 26626.0 | inactive_split_MB: 1036.8 +rank1 [2026-10-28 13:10:49,216] INFO : Step 2400/50000 rank1 | Train Loss: 2.5721 | Train Loss Unweighted: 1.4496 | Train Loss Components: Auxiliary: 0.4832 | Cardinality: 0.2667 | Classification: 0.2677 | Classification_binary: 0.3686 | HitGrouping: 0.2521 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3975 | Regression_eta: 0.0053 | Regression_pt: 0.4051 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1857 | Classification_binary: 0.3464 | Regression_cos_phi: 0.2609 | Regression_energy: 0.2093 | Regression_eta: 0.1569 | Regression_pt: 0.2252 | Regression_sin_phi: 0.2882 | LR: 9.95e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.0745s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7367 | max_loss: 0.1564 | max_backward: 1.6902 | max_optimizer_step: 0.2413 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5703.1 | uss_MB: 4496.2 | child_rss_MB: 6167.5 | allocated_MB: 448.4 | reserved_MB: 26626.0 | inactive_split_MB: 1138.1 +rank1 [2026-10-28 13:15:33,528] INFO : Step 2500/50000 rank1 | Train Loss: 2.3709 | Train Loss Unweighted: 1.3478 | Train Loss Components: Auxiliary: 0.4634 | Cardinality: 0.2742 | Classification: 0.2528 | Classification_binary: 0.3100 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3861 | Regression_eta: 0.0042 | Regression_pt: 0.3903 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1487 | Classification_binary: 0.3201 | Regression_cos_phi: 0.2172 | Regression_energy: 0.2183 | Regression_eta: 0.1178 | Regression_pt: 0.2391 | Regression_sin_phi: 0.1678 | LR: 9.94e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3398s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7598 | max_loss: 0.2442 | max_backward: 1.9617 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5783 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5689.7 | uss_MB: 4483.6 | child_rss_MB: 6173.2 | allocated_MB: 447.2 | reserved_MB: 27052.0 | inactive_split_MB: 1262.0 +rank1 [2026-10-28 13:20:26,044] INFO : Step 2600/50000 rank1 | Train Loss: 2.6216 | Train Loss Unweighted: 1.5111 | Train Loss Components: Auxiliary: 0.4988 | Cardinality: 0.2823 | Classification: 0.2863 | Classification_binary: 0.3723 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4174 | Regression_eta: 0.0052 | Regression_pt: 0.4247 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2197 | Classification_binary: 0.4009 | Regression_cos_phi: 0.2581 | Regression_energy: 0.2310 | Regression_eta: 0.2154 | Regression_pt: 0.2516 | Regression_sin_phi: 0.2514 | LR: 9.94e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2882s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8125 | max_loss: 0.1494 | max_backward: 1.9790 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5698.8 | uss_MB: 4500.2 | child_rss_MB: 6171.1 | allocated_MB: 448.2 | reserved_MB: 28734.0 | inactive_split_MB: 1300.2 +rank1 [2026-10-28 13:25:15,884] INFO : Step 2700/50000 rank1 | Train Loss: 2.5735 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4801 | Cardinality: 0.2457 | Classification: 0.2692 | Classification_binary: 0.4021 | HitGrouping: 0.2514 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3948 | Regression_eta: 0.0060 | Regression_pt: 0.3979 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1499 | Classification_binary: 0.3918 | Regression_cos_phi: 0.2370 | Regression_energy: 0.1984 | Regression_eta: 0.1574 | Regression_pt: 0.2169 | Regression_sin_phi: 0.1717 | LR: 9.94e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3204s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0070 | max_forward: 0.8435 | max_loss: 0.1556 | max_backward: 1.9409 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5695.7 | uss_MB: 4500.5 | child_rss_MB: 6182.0 | allocated_MB: 447.4 | reserved_MB: 28734.0 | inactive_split_MB: 980.7 +rank1 [2026-10-28 13:29:49,852] INFO : Step 2800/50000 rank1 | Train Loss: 2.3517 | Train Loss Unweighted: 1.3315 | Train Loss Components: Auxiliary: 0.4396 | Cardinality: 0.2218 | Classification: 0.2269 | Classification_binary: 0.3206 | HitGrouping: 0.2464 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3829 | Regression_eta: 0.0052 | Regression_pt: 0.3909 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1353 | Classification_binary: 0.2810 | Regression_cos_phi: 0.1960 | Regression_energy: 0.1974 | Regression_eta: 0.1539 | Regression_pt: 0.2176 | Regression_sin_phi: 0.1589 | LR: 9.93e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.3252s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0062 | max_forward: 0.8898 | max_loss: 0.1952 | max_backward: 1.8882 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0163 | Memory: rss_MB: 5687.7 | uss_MB: 4493.8 | child_rss_MB: 6179.6 | allocated_MB: 445.9 | reserved_MB: 28734.0 | inactive_split_MB: 1208.5 +rank1 [2026-10-28 13:34:34,297] INFO : Step 2900/50000 rank1 | Train Loss: 2.4505 | Train Loss Unweighted: 1.3674 | Train Loss Components: Auxiliary: 0.4603 | Cardinality: 0.2746 | Classification: 0.2611 | Classification_binary: 0.3388 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3750 | Regression_eta: 0.0049 | Regression_pt: 0.3827 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1753 | Classification_binary: 0.3566 | Regression_cos_phi: 0.1912 | Regression_energy: 0.1906 | Regression_eta: 0.2015 | Regression_pt: 0.2132 | Regression_sin_phi: 0.2856 | LR: 9.93e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.3929s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7484 | max_loss: 0.6089 | max_backward: 2.2353 | max_optimizer_step: 0.1775 | max_tensorboard_logging: 2.4313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0155 | Memory: rss_MB: 5691.8 | uss_MB: 4497.3 | child_rss_MB: 6175.9 | allocated_MB: 446.7 | reserved_MB: 28734.0 | inactive_split_MB: 1133.2 +rank1 [2026-10-28 13:39:22,258] INFO : Step 3000/50000 rank1 | Train Loss: 2.6179 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4760 | Cardinality: 0.2600 | Classification: 0.3170 | Classification_binary: 0.3907 | HitGrouping: 0.2509 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3743 | Regression_eta: 0.0054 | Regression_pt: 0.3819 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1927 | Classification_binary: 0.4426 | Regression_cos_phi: 0.2308 | Regression_energy: 0.1848 | Regression_eta: 0.1674 | Regression_pt: 0.2041 | Regression_sin_phi: 0.2463 | LR: 9.92e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8195s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0842 | max_forward: 0.8203 | max_loss: 0.1511 | max_backward: 1.9714 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.0307 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5691.4 | uss_MB: 4499.3 | child_rss_MB: 6154.8 | allocated_MB: 451.4 | reserved_MB: 28740.0 | inactive_split_MB: 1256.0 +rank1 [2026-10-28 13:39:23,128] INFO : Running validation on rank1 +rank1 [2026-10-28 13:39:23,128] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 13:39:23,128] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 13:40:15,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:40:16,489] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 13:40:44,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 13:40:45,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 13:40:46,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 13:40:47,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 13:40:57,190] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 13:40:57,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:40:58,512] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 13:41:14,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 13:41:15,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 13:41:16,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 13:41:17,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 13:41:27,666] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 13:41:28,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 13:41:28,597] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 13:41:43,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 13:41:44,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 13:41:45,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 13:41:46,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 13:41:56,430] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 13:46:51,408] INFO : Step 3100/50000 rank1 | Train Loss: 2.3456 | Train Loss Unweighted: 1.3820 | Train Loss Components: Auxiliary: 0.4397 | Cardinality: 0.2017 | Classification: 0.2549 | Classification_binary: 0.3432 | HitGrouping: 0.2399 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3852 | Regression_eta: 0.0046 | Regression_pt: 0.3892 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1440 | Classification_binary: 0.3445 | Regression_cos_phi: 0.1842 | Regression_energy: 0.1716 | Regression_eta: 0.0879 | Regression_pt: 0.1853 | Regression_sin_phi: 0.1711 | LR: 9.91e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4920s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0425 | max_forward: 0.7824 | max_loss: 0.1439 | max_backward: 1.7183 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.8666 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6905.9 | uss_MB: 5708.9 | child_rss_MB: 6176.0 | allocated_MB: 447.6 | reserved_MB: 24468.0 | inactive_split_MB: 1269.5 +rank1 [2026-10-28 13:51:36,981] INFO : Step 3200/50000 rank1 | Train Loss: 2.2453 | Train Loss Unweighted: 1.2729 | Train Loss Components: Auxiliary: 0.4171 | Cardinality: 0.2143 | Classification: 0.2366 | Classification_binary: 0.3116 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3569 | Regression_eta: 0.0047 | Regression_pt: 0.3584 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1446 | Classification_binary: 0.3159 | Regression_cos_phi: 0.2199 | Regression_energy: 0.2076 | Regression_eta: 0.1425 | Regression_pt: 0.2238 | Regression_sin_phi: 0.2995 | LR: 9.91e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6657s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7985 | max_loss: 0.1379 | max_backward: 1.7895 | max_optimizer_step: 0.2027 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6908.5 | uss_MB: 5710.9 | child_rss_MB: 6167.1 | allocated_MB: 448.6 | reserved_MB: 26448.0 | inactive_split_MB: 1391.4 +rank1 [2026-10-28 13:56:16,618] INFO : Step 3300/50000 rank1 | Train Loss: 2.5474 | Train Loss Unweighted: 1.4472 | Train Loss Components: Auxiliary: 0.4779 | Cardinality: 0.2732 | Classification: 0.2824 | Classification_binary: 0.3640 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3911 | Regression_eta: 0.0054 | Regression_pt: 0.3992 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1529 | Classification_binary: 0.3644 | Regression_cos_phi: 0.1991 | Regression_energy: 0.2165 | Regression_eta: 0.1144 | Regression_pt: 0.2392 | Regression_sin_phi: 0.2094 | LR: 9.90e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6659s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8323 | max_loss: 0.1479 | max_backward: 2.0543 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.7834 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5785.1 | uss_MB: 4587.8 | child_rss_MB: 6143.0 | allocated_MB: 448.7 | reserved_MB: 26460.0 | inactive_split_MB: 1087.4 +rank1 [2026-10-28 14:00:49,247] INFO : Step 3400/50000 rank1 | Train Loss: 2.5743 | Train Loss Unweighted: 1.4358 | Train Loss Components: Auxiliary: 0.4720 | Cardinality: 0.2664 | Classification: 0.2699 | Classification_binary: 0.3915 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3784 | Regression_eta: 0.0053 | Regression_pt: 0.3851 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1531 | Classification_binary: 0.3362 | Regression_cos_phi: 0.2387 | Regression_energy: 0.1995 | Regression_eta: 0.1507 | Regression_pt: 0.2186 | Regression_sin_phi: 0.1904 | LR: 9.90e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2593s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7827 | max_loss: 0.1630 | max_backward: 1.9191 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5606 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5794.1 | uss_MB: 4597.7 | child_rss_MB: 6161.5 | allocated_MB: 450.4 | reserved_MB: 28138.0 | inactive_split_MB: 1261.0 +rank1 [2026-10-28 14:05:23,603] INFO : Step 3500/50000 rank1 | Train Loss: 2.1916 | Train Loss Unweighted: 1.2418 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2204 | Classification: 0.2212 | Classification_binary: 0.3041 | HitGrouping: 0.2426 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3517 | Regression_eta: 0.0047 | Regression_pt: 0.3558 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1253 | Classification_binary: 0.2695 | Regression_cos_phi: 0.1766 | Regression_energy: 0.1996 | Regression_eta: 0.1564 | Regression_pt: 0.2184 | Regression_sin_phi: 0.1865 | LR: 9.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2270s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7972 | max_loss: 0.1441 | max_backward: 1.7398 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6845 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5780.0 | uss_MB: 4584.6 | child_rss_MB: 6167.3 | allocated_MB: 443.7 | reserved_MB: 28532.0 | inactive_split_MB: 1176.3 +rank1 [2026-10-28 14:10:06,692] INFO : Step 3600/50000 rank1 | Train Loss: 2.3195 | Train Loss Unweighted: 1.3024 | Train Loss Components: Auxiliary: 0.4248 | Cardinality: 0.2221 | Classification: 0.2518 | Classification_binary: 0.3172 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3610 | Regression_eta: 0.0052 | Regression_pt: 0.3623 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1582 | Classification_binary: 0.3394 | Regression_cos_phi: 0.2583 | Regression_energy: 0.2270 | Regression_eta: 0.1640 | Regression_pt: 0.2515 | Regression_sin_phi: 0.1823 | LR: 9.89e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3067s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8456 | max_loss: 0.1388 | max_backward: 1.9466 | max_optimizer_step: 0.1811 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5787.8 | uss_MB: 4598.8 | child_rss_MB: 6163.2 | allocated_MB: 449.1 | reserved_MB: 28532.0 | inactive_split_MB: 1277.8 +rank1 [2026-10-28 14:14:47,797] INFO : Step 3700/50000 rank1 | Train Loss: 2.5861 | Train Loss Unweighted: 1.4427 | Train Loss Components: Auxiliary: 0.4700 | Cardinality: 0.2678 | Classification: 0.2978 | Classification_binary: 0.3844 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3719 | Regression_eta: 0.0058 | Regression_pt: 0.3773 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1465 | Classification_binary: 0.3785 | Regression_cos_phi: 0.2283 | Regression_energy: 0.2086 | Regression_eta: 0.1786 | Regression_pt: 0.2297 | Regression_sin_phi: 0.2056 | LR: 9.88e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5007s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.8001 | max_loss: 0.1519 | max_backward: 1.8239 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.7916 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0181 | Memory: rss_MB: 5775.1 | uss_MB: 4600.5 | child_rss_MB: 6165.4 | allocated_MB: 450.6 | reserved_MB: 30118.0 | inactive_split_MB: 1097.6 +rank1 [2026-10-28 14:19:24,599] INFO : Step 3800/50000 rank1 | Train Loss: 2.3269 | Train Loss Unweighted: 1.3016 | Train Loss Components: Auxiliary: 0.4225 | Cardinality: 0.2364 | Classification: 0.2400 | Classification_binary: 0.3389 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3534 | Regression_eta: 0.0054 | Regression_pt: 0.3591 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1788 | Classification_binary: 0.3182 | Regression_cos_phi: 0.2458 | Regression_energy: 0.1784 | Regression_eta: 0.3052 | Regression_pt: 0.1919 | Regression_sin_phi: 0.2209 | LR: 9.87e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.4709s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7555 | max_loss: 0.1601 | max_backward: 1.7612 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.9394 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5772.0 | uss_MB: 4595.7 | child_rss_MB: 6158.5 | allocated_MB: 447.4 | reserved_MB: 30118.0 | inactive_split_MB: 1355.2 +rank1 [2026-10-28 14:24:05,269] INFO : Step 3900/50000 rank1 | Train Loss: 2.0276 | Train Loss Unweighted: 1.1874 | Train Loss Components: Auxiliary: 0.3861 | Cardinality: 0.1825 | Classification: 0.2116 | Classification_binary: 0.2707 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3485 | Regression_eta: 0.0038 | Regression_pt: 0.3489 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1902 | Classification_binary: 0.3092 | Regression_cos_phi: 0.2843 | Regression_energy: 0.1885 | Regression_eta: 0.1650 | Regression_pt: 0.2015 | Regression_sin_phi: 0.2153 | LR: 9.87e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.4537s | Phase Times: max_pre_step_barrier: 0.0095 | max_batch_to_device: 0.0140 | max_forward: 1.1007 | max_loss: 0.1867 | max_backward: 2.0952 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 0.5091 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5772.2 | uss_MB: 4598.5 | child_rss_MB: 6165.7 | allocated_MB: 448.6 | reserved_MB: 30118.0 | inactive_split_MB: 1119.4 +rank1 [2026-10-28 14:28:52,325] INFO : Step 4000/50000 rank1 | Train Loss: 2.3921 | Train Loss Unweighted: 1.3683 | Train Loss Components: Auxiliary: 0.4366 | Cardinality: 0.2254 | Classification: 0.2720 | Classification_binary: 0.3524 | HitGrouping: 0.2405 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3647 | Regression_eta: 0.0048 | Regression_pt: 0.3692 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1752 | Classification_binary: 0.3618 | Regression_cos_phi: 0.1549 | Regression_energy: 0.2218 | Regression_eta: 0.1263 | Regression_pt: 0.2427 | Regression_sin_phi: 0.1778 | LR: 9.86e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.8926s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.8532 | max_loss: 0.1476 | max_backward: 2.4362 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.6138 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5773.5 | uss_MB: 4601.5 | child_rss_MB: 6164.9 | allocated_MB: 449.7 | reserved_MB: 30118.0 | inactive_split_MB: 899.8 +rank1 [2026-10-28 14:28:53,307] INFO : Running validation on rank1 +rank1 [2026-10-28 14:28:53,307] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 14:28:53,307] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 14:29:46,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:29:46,700] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 14:30:13,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 14:30:14,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 14:30:15,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 14:30:16,316] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 14:30:26,325] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 14:30:26,980] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:30:27,511] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 14:30:43,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 14:30:44,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 14:30:45,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 14:30:46,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 14:30:56,087] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 14:30:57,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 14:30:57,800] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 14:31:13,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 14:31:14,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 14:31:15,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 14:31:15,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 14:31:25,950] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 14:36:30,160] INFO : Step 4100/50000 rank1 | Train Loss: 2.5062 | Train Loss Unweighted: 1.3582 | Train Loss Components: Auxiliary: 0.4559 | Cardinality: 0.3208 | Classification: 0.2565 | Classification_binary: 0.3823 | HitGrouping: 0.2386 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3543 | Regression_eta: 0.0046 | Regression_pt: 0.3553 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1518 | Classification_binary: 0.3544 | Regression_cos_phi: 0.2014 | Regression_energy: 0.1953 | Regression_eta: 0.0984 | Regression_pt: 0.2129 | Regression_sin_phi: 0.1868 | LR: 9.85e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.2042s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0069 | max_forward: 0.8943 | max_loss: 0.2518 | max_backward: 1.9865 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 1.1537 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6976.0 | uss_MB: 5798.7 | child_rss_MB: 6190.7 | allocated_MB: 449.4 | reserved_MB: 25148.0 | inactive_split_MB: 1028.6 +rank1 [2026-10-28 14:41:18,155] INFO : Step 4200/50000 rank1 | Train Loss: 2.0961 | Train Loss Unweighted: 1.2232 | Train Loss Components: Auxiliary: 0.3927 | Cardinality: 0.2101 | Classification: 0.2242 | Classification_binary: 0.2902 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3497 | Regression_eta: 0.0042 | Regression_pt: 0.3510 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1451 | Classification_binary: 0.2808 | Regression_cos_phi: 0.1635 | Regression_energy: 0.1804 | Regression_eta: 0.1143 | Regression_pt: 0.1933 | Regression_sin_phi: 0.1227 | LR: 9.84e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.6589s | Phase Times: max_pre_step_barrier: 0.1131 | max_batch_to_device: 0.0199 | max_forward: 1.5911 | max_loss: 0.3393 | max_backward: 3.0707 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5197 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6973.9 | uss_MB: 5797.7 | child_rss_MB: 6186.7 | allocated_MB: 446.0 | reserved_MB: 25296.0 | inactive_split_MB: 922.1 +rank1 [2026-10-28 14:46:18,702] INFO : Step 4300/50000 rank1 | Train Loss: 2.2792 | Train Loss Unweighted: 1.3065 | Train Loss Components: Auxiliary: 0.4224 | Cardinality: 0.2158 | Classification: 0.2339 | Classification_binary: 0.3250 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3669 | Regression_eta: 0.0044 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1685 | Classification_binary: 0.3554 | Regression_cos_phi: 0.2222 | Regression_energy: 0.1604 | Regression_eta: 0.1513 | Regression_pt: 0.1785 | Regression_sin_phi: 0.2155 | LR: 9.84e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5474s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0938 | max_forward: 0.8451 | max_loss: 0.1396 | max_backward: 1.9669 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.6639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5862.7 | uss_MB: 4685.4 | child_rss_MB: 6195.4 | allocated_MB: 448.4 | reserved_MB: 28580.0 | inactive_split_MB: 1346.0 +rank1 [2026-10-28 14:50:58,885] INFO : Step 4400/50000 rank1 | Train Loss: 2.4955 | Train Loss Unweighted: 1.3385 | Train Loss Components: Auxiliary: 0.4510 | Cardinality: 0.3020 | Classification: 0.2394 | Classification_binary: 0.3842 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3482 | Regression_eta: 0.0051 | Regression_pt: 0.3563 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1429 | Classification_binary: 0.4105 | Regression_cos_phi: 0.2511 | Regression_energy: 0.1933 | Regression_eta: 0.1391 | Regression_pt: 0.2130 | Regression_sin_phi: 0.1937 | LR: 9.83e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2054s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0083 | max_forward: 0.7889 | max_loss: 0.1505 | max_backward: 1.8572 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4868 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5870.1 | uss_MB: 4697.1 | child_rss_MB: 6175.1 | allocated_MB: 449.1 | reserved_MB: 28582.0 | inactive_split_MB: 1105.9 +rank1 [2026-10-28 14:55:40,223] INFO : Step 4500/50000 rank1 | Train Loss: 2.3283 | Train Loss Unweighted: 1.2750 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.3015 | Classification: 0.2318 | Classification_binary: 0.3495 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3409 | Regression_eta: 0.0044 | Regression_pt: 0.3441 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1252 | Classification_binary: 0.3030 | Regression_cos_phi: 0.1537 | Regression_energy: 0.1835 | Regression_eta: 0.1149 | Regression_pt: 0.1992 | Regression_sin_phi: 0.1261 | LR: 9.82e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0175s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7326 | max_loss: 0.2428 | max_backward: 1.7841 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5868.2 | uss_MB: 4696.0 | child_rss_MB: 6178.6 | allocated_MB: 447.8 | reserved_MB: 28582.0 | inactive_split_MB: 1258.0 +rank1 [2026-10-28 15:00:16,448] INFO : Step 4600/50000 rank1 | Train Loss: 2.0353 | Train Loss Unweighted: 1.1777 | Train Loss Components: Auxiliary: 0.3804 | Cardinality: 0.1920 | Classification: 0.2053 | Classification_binary: 0.2642 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3490 | Regression_eta: 0.0039 | Regression_pt: 0.3512 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1019 | Classification_binary: 0.2501 | Regression_cos_phi: 0.1526 | Regression_energy: 0.1669 | Regression_eta: 0.0791 | Regression_pt: 0.1848 | Regression_sin_phi: 0.1979 | LR: 9.81e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.0860s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0402 | max_forward: 0.7143 | max_loss: 0.1412 | max_backward: 1.6726 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 5861.4 | uss_MB: 4694.6 | child_rss_MB: 6186.8 | allocated_MB: 446.4 | reserved_MB: 28582.0 | inactive_split_MB: 981.4 +rank1 [2026-10-28 15:05:12,090] INFO : Step 4700/50000 rank1 | Train Loss: 2.2426 | Train Loss Unweighted: 1.2576 | Train Loss Components: Auxiliary: 0.4073 | Cardinality: 0.2222 | Classification: 0.2485 | Classification_binary: 0.3333 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3307 | Regression_eta: 0.0049 | Regression_pt: 0.3356 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1521 | Classification_binary: 0.3459 | Regression_cos_phi: 0.1800 | Regression_energy: 0.1826 | Regression_eta: 0.1050 | Regression_pt: 0.1995 | Regression_sin_phi: 0.1773 | LR: 9.81e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6196s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.1471 | max_backward: 2.0627 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.7062 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0769 | Memory: rss_MB: 5860.5 | uss_MB: 4699.2 | child_rss_MB: 6169.2 | allocated_MB: 447.7 | reserved_MB: 28652.0 | inactive_split_MB: 1164.9 +rank1 [2026-10-28 15:09:59,906] INFO : Step 4800/50000 rank1 | Train Loss: 2.2697 | Train Loss Unweighted: 1.2637 | Train Loss Components: Auxiliary: 0.4132 | Cardinality: 0.2493 | Classification: 0.2262 | Classification_binary: 0.3667 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3287 | Regression_eta: 0.0044 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1283 | Classification_binary: 0.3414 | Regression_cos_phi: 0.1935 | Regression_energy: 0.1724 | Regression_eta: 0.1155 | Regression_pt: 0.1899 | Regression_sin_phi: 0.1783 | LR: 9.80e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.0687s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0065 | max_forward: 0.7626 | max_loss: 0.1571 | max_backward: 1.6465 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5855.3 | uss_MB: 4693.4 | child_rss_MB: 6188.9 | allocated_MB: 450.7 | reserved_MB: 28652.0 | inactive_split_MB: 1088.6 +rank1 [2026-10-28 15:14:51,789] INFO : Step 4900/50000 rank1 | Train Loss: 2.0529 | Train Loss Unweighted: 1.1598 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.1942 | Classification: 0.2269 | Classification_binary: 0.2874 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3172 | Regression_eta: 0.0041 | Regression_pt: 0.3199 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1274 | Classification_binary: 0.2960 | Regression_cos_phi: 0.2149 | Regression_energy: 0.1705 | Regression_eta: 0.1565 | Regression_pt: 0.1881 | Regression_sin_phi: 0.2202 | LR: 9.79e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.3235s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7293 | max_loss: 0.1974 | max_backward: 1.8633 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5858.6 | uss_MB: 4697.0 | child_rss_MB: 6189.0 | allocated_MB: 447.4 | reserved_MB: 28652.0 | inactive_split_MB: 1266.1 +rank1 [2026-10-28 15:19:59,464] INFO : Step 5000/50000 rank1 | Train Loss: 2.0827 | Train Loss Unweighted: 1.1872 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2154 | Classification: 0.2462 | Classification_binary: 0.2941 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3169 | Regression_eta: 0.0042 | Regression_pt: 0.3219 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1377 | Classification_binary: 0.2937 | Regression_cos_phi: 0.1432 | Regression_energy: 0.1571 | Regression_eta: 0.1173 | Regression_pt: 0.1715 | Regression_sin_phi: 0.1309 | LR: 9.78e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3261s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.8382 | max_loss: 0.1450 | max_backward: 1.9472 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0203 | Memory: rss_MB: 5858.9 | uss_MB: 4697.5 | child_rss_MB: 6177.0 | allocated_MB: 446.9 | reserved_MB: 29316.0 | inactive_split_MB: 980.8 +rank1 [2026-10-28 15:20:00,328] INFO : Running validation on rank1 +rank1 [2026-10-28 15:20:00,328] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 15:20:00,328] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 15:20:55,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:20:56,482] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 15:21:22,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 15:21:23,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 15:21:24,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 15:21:26,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 15:21:36,012] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 15:21:36,445] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:21:36,967] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 15:21:53,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 15:21:54,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 15:21:55,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 15:21:56,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 15:22:06,350] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 15:22:06,552] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 15:22:07,050] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 15:22:23,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 15:22:24,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 15:22:24,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 15:22:25,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 15:22:35,859] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 15:28:09,086] INFO : Step 5100/50000 rank1 | Train Loss: 2.4098 | Train Loss Unweighted: 1.3457 | Train Loss Components: Auxiliary: 0.4318 | Cardinality: 0.2397 | Classification: 0.2711 | Classification_binary: 0.3798 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3391 | Regression_eta: 0.0051 | Regression_pt: 0.3453 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1630 | Classification_binary: 0.3891 | Regression_cos_phi: 0.1857 | Regression_energy: 0.1854 | Regression_eta: 0.1247 | Regression_pt: 0.2060 | Regression_sin_phi: 0.1944 | LR: 9.77e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8405s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1511 | max_backward: 1.7914 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 1.0544 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5949.7 | uss_MB: 4786.0 | child_rss_MB: 6159.3 | allocated_MB: 447.6 | reserved_MB: 25650.0 | inactive_split_MB: 1122.4 +rank1 [2026-10-28 15:32:58,662] INFO : Step 5200/50000 rank1 | Train Loss: 2.0929 | Train Loss Unweighted: 1.1974 | Train Loss Components: Auxiliary: 0.3875 | Cardinality: 0.2007 | Classification: 0.2193 | Classification_binary: 0.3278 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3194 | Regression_eta: 0.0041 | Regression_pt: 0.3227 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1287 | Classification_binary: 0.3382 | Regression_cos_phi: 0.1243 | Regression_energy: 0.1765 | Regression_eta: 0.1102 | Regression_pt: 0.1903 | Regression_sin_phi: 0.1500 | LR: 9.76e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.4918s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7458 | max_loss: 0.2423 | max_backward: 1.7123 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9648 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5957.0 | uss_MB: 4796.7 | child_rss_MB: 6179.0 | allocated_MB: 450.7 | reserved_MB: 28708.0 | inactive_split_MB: 1139.3 +rank1 [2026-10-28 15:37:52,597] INFO : Step 5300/50000 rank1 | Train Loss: 1.9905 | Train Loss Unweighted: 1.1352 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.2083 | Classification: 0.2239 | Classification_binary: 0.2818 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0037 | Regression_pt: 0.3135 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1578 | Classification_binary: 0.2994 | Regression_cos_phi: 0.1946 | Regression_energy: 0.1804 | Regression_eta: 0.1505 | Regression_pt: 0.1955 | Regression_sin_phi: 0.1849 | LR: 9.75e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.1044s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 1.2453 | max_loss: 0.2326 | max_backward: 2.1392 | max_optimizer_step: 0.1883 | max_tensorboard_logging: 1.0947 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5945.0 | uss_MB: 4784.6 | child_rss_MB: 6156.3 | allocated_MB: 445.9 | reserved_MB: 30688.0 | inactive_split_MB: 925.3 +rank1 [2026-10-28 15:42:39,663] INFO : Step 5400/50000 rank1 | Train Loss: 2.2932 | Train Loss Unweighted: 1.2957 | Train Loss Components: Auxiliary: 0.4189 | Cardinality: 0.2411 | Classification: 0.2689 | Classification_binary: 0.3420 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3351 | Regression_eta: 0.0045 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1711 | Classification_binary: 0.4001 | Regression_cos_phi: 0.2354 | Regression_energy: 0.1453 | Regression_eta: 0.1331 | Regression_pt: 0.1604 | Regression_sin_phi: 0.2321 | LR: 9.74e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5697s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7688 | max_loss: 0.1697 | max_backward: 1.8189 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8641 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5955.2 | uss_MB: 4796.8 | child_rss_MB: 6163.2 | allocated_MB: 450.8 | reserved_MB: 30688.0 | inactive_split_MB: 915.1 +rank1 [2026-10-28 15:47:42,229] INFO : Step 5500/50000 rank1 | Train Loss: 2.1734 | Train Loss Unweighted: 1.2295 | Train Loss Components: Auxiliary: 0.3970 | Cardinality: 0.2269 | Classification: 0.2218 | Classification_binary: 0.3571 | HitGrouping: 0.2200 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3188 | Regression_eta: 0.0041 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1509 | Classification_binary: 0.3482 | Regression_cos_phi: 0.1993 | Regression_energy: 0.1494 | Regression_eta: 0.1272 | Regression_pt: 0.1637 | Regression_sin_phi: 0.1540 | LR: 9.73e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.9276s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0153 | max_forward: 1.1913 | max_loss: 0.6135 | max_backward: 3.0599 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.6982 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5952.5 | uss_MB: 4797.4 | child_rss_MB: 6170.1 | allocated_MB: 446.9 | reserved_MB: 32804.0 | inactive_split_MB: 913.2 +rank1 [2026-10-28 15:52:30,288] INFO : Step 5600/50000 rank1 | Train Loss: 2.0307 | Train Loss Unweighted: 1.1169 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.1966 | Classification: 0.2085 | Classification_binary: 0.3031 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2975 | Regression_eta: 0.0047 | Regression_pt: 0.2988 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1143 | Classification_binary: 0.2775 | Regression_cos_phi: 0.1530 | Regression_energy: 0.1490 | Regression_eta: 0.1114 | Regression_pt: 0.1623 | Regression_sin_phi: 0.1308 | LR: 9.72e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.7948s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7794 | max_loss: 0.1497 | max_backward: 1.9410 | max_optimizer_step: 0.5154 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0061 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5920.9 | uss_MB: 4797.0 | child_rss_MB: 6174.4 | allocated_MB: 445.4 | reserved_MB: 32804.0 | inactive_split_MB: 1164.2 +rank1 [2026-10-28 15:57:16,862] INFO : Step 5700/50000 rank1 | Train Loss: 2.2064 | Train Loss Unweighted: 1.2224 | Train Loss Components: Auxiliary: 0.4023 | Cardinality: 0.2320 | Classification: 0.2526 | Classification_binary: 0.3174 | HitGrouping: 0.2343 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3173 | Regression_eta: 0.0045 | Regression_pt: 0.3259 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1293 | Classification_binary: 0.2972 | Regression_cos_phi: 0.1909 | Regression_energy: 0.1524 | Regression_eta: 0.1150 | Regression_pt: 0.1686 | Regression_sin_phi: 0.1804 | LR: 9.71e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6668s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7746 | max_loss: 0.4928 | max_backward: 1.9590 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8719 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 5893.1 | uss_MB: 3755.9 | child_rss_MB: 6168.2 | allocated_MB: 448.4 | reserved_MB: 32804.0 | inactive_split_MB: 913.0 +rank1 [2026-10-28 16:02:01,174] INFO : Step 5800/50000 rank1 | Train Loss: 2.1438 | Train Loss Unweighted: 1.2123 | Train Loss Components: Auxiliary: 0.3898 | Cardinality: 0.2335 | Classification: 0.2404 | Classification_binary: 0.3575 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3006 | Regression_eta: 0.0042 | Regression_pt: 0.3054 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1710 | Classification_binary: 0.3675 | Regression_cos_phi: 0.2294 | Regression_energy: 0.1768 | Regression_eta: 0.1371 | Regression_pt: 0.1951 | Regression_sin_phi: 0.1755 | LR: 9.70e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.5894s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8028 | max_loss: 0.1540 | max_backward: 1.8645 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.8379 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5883.5 | uss_MB: 4797.1 | child_rss_MB: 6169.4 | allocated_MB: 448.3 | reserved_MB: 32804.0 | inactive_split_MB: 916.5 +rank1 [2026-10-28 16:06:46,106] INFO : Step 5900/50000 rank1 | Train Loss: 2.2094 | Train Loss Unweighted: 1.1453 | Train Loss Components: Auxiliary: 0.3936 | Cardinality: 0.2891 | Classification: 0.2198 | Classification_binary: 0.3337 | HitGrouping: 0.2238 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2890 | Regression_eta: 0.0046 | Regression_pt: 0.2935 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1141 | Classification_binary: 0.3089 | Regression_cos_phi: 0.1416 | Regression_energy: 0.1728 | Regression_eta: 0.1091 | Regression_pt: 0.1891 | Regression_sin_phi: 0.1755 | LR: 9.69e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5878s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0065 | max_forward: 1.0922 | max_loss: 0.1477 | max_backward: 1.8468 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8251 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0991 | Memory: rss_MB: 5881.4 | uss_MB: 4798.1 | child_rss_MB: 6148.2 | allocated_MB: 446.5 | reserved_MB: 32804.0 | inactive_split_MB: 927.1 +rank1 [2026-10-28 16:11:31,999] INFO : Step 6000/50000 rank1 | Train Loss: 1.8458 | Train Loss Unweighted: 1.0363 | Train Loss Components: Auxiliary: 0.3398 | Cardinality: 0.1796 | Classification: 0.1819 | Classification_binary: 0.2710 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2865 | Regression_eta: 0.0032 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1120 | Classification_binary: 0.2394 | Regression_cos_phi: 0.1419 | Regression_energy: 0.1393 | Regression_eta: 0.0958 | Regression_pt: 0.1533 | Regression_sin_phi: 0.2028 | LR: 9.68e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5095s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7380 | max_loss: 0.2483 | max_backward: 1.8094 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.9243 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5850.9 | uss_MB: 4795.2 | child_rss_MB: 6173.4 | allocated_MB: 448.1 | reserved_MB: 32804.0 | inactive_split_MB: 1232.3 +rank1 [2026-10-28 16:11:32,848] INFO : Running validation on rank1 +rank1 [2026-10-28 16:11:32,848] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 16:11:32,848] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 16:12:26,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:12:26,924] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 16:12:54,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 16:12:55,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 16:12:56,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 16:12:57,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 16:13:07,681] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-28 16:13:07,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:13:08,324] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 16:13:24,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 16:13:25,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 16:13:26,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 16:13:27,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 16:13:37,591] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 16:13:38,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 16:13:38,691] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 16:13:54,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 16:13:55,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 16:13:55,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 16:13:56,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 16:14:06,834] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 16:19:13,933] INFO : Step 6100/50000 rank1 | Train Loss: 2.0818 | Train Loss Unweighted: 1.1297 | Train Loss Components: Auxiliary: 0.3734 | Cardinality: 0.2287 | Classification: 0.2079 | Classification_binary: 0.3249 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2929 | Regression_eta: 0.0044 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1296 | Classification_binary: 0.3091 | Regression_cos_phi: 0.2441 | Regression_energy: 0.1732 | Regression_eta: 0.1396 | Regression_pt: 0.1885 | Regression_sin_phi: 0.1680 | LR: 9.67e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.5155s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8130 | max_loss: 0.1524 | max_backward: 3.1921 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 1.4719 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5941.1 | uss_MB: 4883.2 | child_rss_MB: 6189.9 | allocated_MB: 448.4 | reserved_MB: 24434.0 | inactive_split_MB: 1287.8 +rank1 [2026-10-28 16:23:58,090] INFO : Step 6200/50000 rank1 | Train Loss: 2.3716 | Train Loss Unweighted: 1.2065 | Train Loss Components: Auxiliary: 0.4163 | Cardinality: 0.3927 | Classification: 0.2343 | Classification_binary: 0.3679 | HitGrouping: 0.2165 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2973 | Regression_eta: 0.0043 | Regression_pt: 0.2981 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1133 | Classification_binary: 0.3319 | Regression_cos_phi: 0.1702 | Regression_energy: 0.1500 | Regression_eta: 0.1047 | Regression_pt: 0.1634 | Regression_sin_phi: 0.1533 | LR: 9.66e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.5591s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8556 | max_loss: 0.1996 | max_backward: 1.9593 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.5823 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5946.6 | uss_MB: 3767.3 | child_rss_MB: 6181.8 | allocated_MB: 447.9 | reserved_MB: 27020.0 | inactive_split_MB: 1185.4 +rank1 [2026-10-28 16:28:38,484] INFO : Step 6300/50000 rank1 | Train Loss: 1.9346 | Train Loss Unweighted: 1.0816 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.2071 | Classification: 0.2053 | Classification_binary: 0.2781 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2941 | Regression_eta: 0.0037 | Regression_pt: 0.2966 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1433 | Classification_binary: 0.3032 | Regression_cos_phi: 0.1745 | Regression_energy: 0.1588 | Regression_eta: 0.1178 | Regression_pt: 0.1741 | Regression_sin_phi: 0.1657 | LR: 9.65e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.5982s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0068 | max_forward: 0.7412 | max_loss: 0.1444 | max_backward: 1.7431 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.9045 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5941.6 | uss_MB: 4896.5 | child_rss_MB: 6164.9 | allocated_MB: 444.6 | reserved_MB: 27022.0 | inactive_split_MB: 1185.4 +rank1 [2026-10-28 16:33:23,144] INFO : Step 6400/50000 rank1 | Train Loss: 2.0123 | Train Loss Unweighted: 1.0992 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2082 | Classification: 0.2291 | Classification_binary: 0.2885 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2842 | Regression_eta: 0.0040 | Regression_pt: 0.2891 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1548 | Classification_binary: 0.3222 | Regression_cos_phi: 0.2286 | Regression_energy: 0.1334 | Regression_eta: 0.1262 | Regression_pt: 0.1466 | Regression_sin_phi: 0.3439 | LR: 9.64e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.7612s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0069 | max_forward: 1.1063 | max_loss: 0.2471 | max_backward: 1.9949 | max_optimizer_step: 0.2726 | max_tensorboard_logging: 0.8233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5936.1 | uss_MB: 4896.3 | child_rss_MB: 6176.5 | allocated_MB: 451.6 | reserved_MB: 27022.0 | inactive_split_MB: 1113.0 +rank1 [2026-10-28 16:37:57,182] INFO : Step 6500/50000 rank1 | Train Loss: 2.1791 | Train Loss Unweighted: 1.2220 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2499 | Classification: 0.2271 | Classification_binary: 0.3508 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3177 | Regression_eta: 0.0042 | Regression_pt: 0.3181 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1455 | Classification_binary: 0.3368 | Regression_cos_phi: 0.1475 | Regression_energy: 0.1662 | Regression_eta: 0.1035 | Regression_pt: 0.1805 | Regression_sin_phi: 0.1942 | LR: 9.63e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9961s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0548 | max_forward: 0.8069 | max_loss: 0.1518 | max_backward: 2.0692 | max_optimizer_step: 0.1860 | max_tensorboard_logging: 0.9414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5926.6 | uss_MB: 4895.5 | child_rss_MB: 6160.0 | allocated_MB: 449.6 | reserved_MB: 28336.0 | inactive_split_MB: 1199.2 +rank1 [2026-10-28 16:42:33,042] INFO : Step 6600/50000 rank1 | Train Loss: 1.9690 | Train Loss Unweighted: 1.0745 | Train Loss Components: Auxiliary: 0.3537 | Cardinality: 0.2132 | Classification: 0.1982 | Classification_binary: 0.3095 | HitGrouping: 0.2136 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2771 | Regression_eta: 0.0040 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1069 | Classification_binary: 0.3067 | Regression_cos_phi: 0.2134 | Regression_energy: 0.1479 | Regression_eta: 0.1401 | Regression_pt: 0.1632 | Regression_sin_phi: 0.1591 | LR: 9.62e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8255s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7382 | max_loss: 0.1467 | max_backward: 1.8335 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 1.1664 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5923.1 | uss_MB: 4893.3 | child_rss_MB: 6174.8 | allocated_MB: 447.4 | reserved_MB: 28340.0 | inactive_split_MB: 1012.8 +rank1 [2026-10-28 16:47:06,903] INFO : Step 6700/50000 rank1 | Train Loss: 1.9470 | Train Loss Unweighted: 1.0781 | Train Loss Components: Auxiliary: 0.3616 | Cardinality: 0.2363 | Classification: 0.1923 | Classification_binary: 0.2661 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3028 | Regression_eta: 0.0035 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1014 | Classification_binary: 0.2541 | Regression_cos_phi: 0.1587 | Regression_energy: 0.1606 | Regression_eta: 0.0858 | Regression_pt: 0.1765 | Regression_sin_phi: 0.1603 | LR: 9.61e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6596s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7926 | max_loss: 0.2421 | max_backward: 2.0577 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.7621 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5925.5 | uss_MB: 4898.4 | child_rss_MB: 6176.2 | allocated_MB: 450.1 | reserved_MB: 29806.0 | inactive_split_MB: 1002.0 +rank1 [2026-10-28 16:51:50,914] INFO : Step 6800/50000 rank1 | Train Loss: 2.1609 | Train Loss Unweighted: 1.1507 | Train Loss Components: Auxiliary: 0.3771 | Cardinality: 0.2180 | Classification: 0.2315 | Classification_binary: 0.3225 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0025 | Regression_energy: 0.2916 | Regression_eta: 0.0048 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1449 | Classification_binary: 0.3354 | Regression_cos_phi: 0.1782 | Regression_energy: 0.1575 | Regression_eta: 0.1364 | Regression_pt: 0.1713 | Regression_sin_phi: 0.2141 | LR: 9.60e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.1909s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0070 | max_forward: 0.9144 | max_loss: 0.4866 | max_backward: 2.1484 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 1.2438 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5920.8 | uss_MB: 4896.0 | child_rss_MB: 6174.1 | allocated_MB: 451.3 | reserved_MB: 29806.0 | inactive_split_MB: 1098.9 +rank1 [2026-10-28 16:56:32,287] INFO : Step 6900/50000 rank1 | Train Loss: 1.9936 | Train Loss Unweighted: 1.1069 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.1991 | Classification: 0.2085 | Classification_binary: 0.3385 | HitGrouping: 0.2125 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2742 | Regression_eta: 0.0041 | Regression_pt: 0.2777 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0930 | Classification_binary: 0.3066 | Regression_cos_phi: 0.1045 | Regression_energy: 0.1452 | Regression_eta: 0.0886 | Regression_pt: 0.1565 | Regression_sin_phi: 0.1303 | LR: 9.58e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6490s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7764 | max_loss: 0.2225 | max_backward: 1.6637 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1184 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5918.2 | uss_MB: 4895.4 | child_rss_MB: 6170.7 | allocated_MB: 450.8 | reserved_MB: 29806.0 | inactive_split_MB: 1193.4 +rank1 [2026-10-28 17:01:16,520] INFO : Step 7000/50000 rank1 | Train Loss: 1.9256 | Train Loss Unweighted: 1.0803 | Train Loss Components: Auxiliary: 0.3608 | Cardinality: 0.2036 | Classification: 0.1946 | Classification_binary: 0.2738 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2998 | Regression_eta: 0.0039 | Regression_pt: 0.3044 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1510 | Classification_binary: 0.2901 | Regression_cos_phi: 0.1610 | Regression_energy: 0.1544 | Regression_eta: 0.1255 | Regression_pt: 0.1702 | Regression_sin_phi: 0.1903 | LR: 9.57e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7642s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7896 | max_loss: 0.1557 | max_backward: 1.7415 | max_optimizer_step: 0.2946 | max_tensorboard_logging: 1.0959 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 5913.7 | uss_MB: 4893.6 | child_rss_MB: 6162.0 | allocated_MB: 450.3 | reserved_MB: 29806.0 | inactive_split_MB: 1283.4 +rank1 [2026-10-28 17:01:17,464] INFO : Running validation on rank1 +rank1 [2026-10-28 17:01:17,464] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 17:01:17,464] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 17:02:11,957] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:02:12,536] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 17:02:38,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 17:02:39,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 17:02:41,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 17:02:42,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 17:02:52,060] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 17:02:52,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:02:53,110] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 17:03:09,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 17:03:10,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 17:03:11,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 17:03:12,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 17:03:22,466] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 17:03:22,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:03:23,342] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 17:03:38,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 17:03:39,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 17:03:40,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 17:03:41,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 17:03:51,499] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 17:08:58,934] INFO : Step 7100/50000 rank1 | Train Loss: 1.8251 | Train Loss Unweighted: 1.0448 | Train Loss Components: Auxiliary: 0.3403 | Cardinality: 0.1731 | Classification: 0.2040 | Classification_binary: 0.2675 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2809 | Regression_eta: 0.0035 | Regression_pt: 0.2853 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1237 | Classification_binary: 0.2689 | Regression_cos_phi: 0.1306 | Regression_energy: 0.1555 | Regression_eta: 0.0894 | Regression_pt: 0.1718 | Regression_sin_phi: 0.1312 | LR: 9.56e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.2542s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0622 | max_forward: 0.8282 | max_loss: 0.1540 | max_backward: 1.9352 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.3584 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6015.2 | uss_MB: 4988.2 | child_rss_MB: 6155.9 | allocated_MB: 449.9 | reserved_MB: 22498.0 | inactive_split_MB: 1193.4 +rank1 [2026-10-28 17:13:31,725] INFO : Step 7200/50000 rank1 | Train Loss: 2.1146 | Train Loss Unweighted: 1.1506 | Train Loss Components: Auxiliary: 0.3887 | Cardinality: 0.2411 | Classification: 0.2243 | Classification_binary: 0.3469 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2844 | Regression_eta: 0.0044 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1637 | Classification_binary: 0.4046 | Regression_cos_phi: 0.2169 | Regression_energy: 0.1735 | Regression_eta: 0.1315 | Regression_pt: 0.1893 | Regression_sin_phi: 0.1857 | LR: 9.55e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.5737s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0369 | max_forward: 0.7969 | max_loss: 0.1517 | max_backward: 1.9380 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6030.2 | uss_MB: 5003.5 | child_rss_MB: 6152.6 | allocated_MB: 448.6 | reserved_MB: 22906.0 | inactive_split_MB: 1087.6 +rank1 [2026-10-28 17:18:11,974] INFO : Step 7300/50000 rank1 | Train Loss: 1.8657 | Train Loss Unweighted: 1.0129 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2046 | Classification: 0.1759 | Classification_binary: 0.2802 | HitGrouping: 0.2112 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2746 | Regression_eta: 0.0037 | Regression_pt: 0.2747 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1076 | Classification_binary: 0.2882 | Regression_cos_phi: 0.1704 | Regression_energy: 0.1389 | Regression_eta: 0.1205 | Regression_pt: 0.1491 | Regression_sin_phi: 0.1656 | LR: 9.53e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4782s | Phase Times: max_pre_step_barrier: 0.0082 | max_batch_to_device: 0.0150 | max_forward: 0.7713 | max_loss: 0.1507 | max_backward: 1.8303 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.6113 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6029.9 | uss_MB: 5003.9 | child_rss_MB: 6151.8 | allocated_MB: 449.9 | reserved_MB: 25044.0 | inactive_split_MB: 918.5 +rank1 [2026-10-28 17:22:47,474] INFO : Step 7400/50000 rank1 | Train Loss: 1.8326 | Train Loss Unweighted: 1.0006 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2227 | Classification: 0.1795 | Classification_binary: 0.2563 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2767 | Regression_eta: 0.0034 | Regression_pt: 0.2813 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1157 | Classification_binary: 0.2656 | Regression_cos_phi: 0.1593 | Regression_energy: 0.1257 | Regression_eta: 0.0984 | Regression_pt: 0.1401 | Regression_sin_phi: 0.1982 | LR: 9.52e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.2041s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0684 | max_forward: 0.7762 | max_loss: 0.1384 | max_backward: 1.7881 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6026.9 | uss_MB: 5000.5 | child_rss_MB: 6151.8 | allocated_MB: 447.4 | reserved_MB: 25044.0 | inactive_split_MB: 1152.4 +rank1 [2026-10-28 17:27:28,847] INFO : Step 7500/50000 rank1 | Train Loss: 2.1479 | Train Loss Unweighted: 1.1510 | Train Loss Components: Auxiliary: 0.3847 | Cardinality: 0.2329 | Classification: 0.2181 | Classification_binary: 0.3323 | HitGrouping: 0.2143 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2947 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1351 | Classification_binary: 0.3538 | Regression_cos_phi: 0.1864 | Regression_energy: 0.1512 | Regression_eta: 0.1245 | Regression_pt: 0.1673 | Regression_sin_phi: 0.2056 | LR: 9.51e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2515s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7726 | max_loss: 0.2518 | max_backward: 1.8458 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1040 | Memory: rss_MB: 6026.8 | uss_MB: 5002.5 | child_rss_MB: 6153.2 | allocated_MB: 452.3 | reserved_MB: 26846.0 | inactive_split_MB: 1117.1 +rank1 [2026-10-28 17:32:29,655] INFO : Step 7600/50000 rank1 | Train Loss: 1.9984 | Train Loss Unweighted: 1.0873 | Train Loss Components: Auxiliary: 0.3667 | Cardinality: 0.2088 | Classification: 0.2053 | Classification_binary: 0.3405 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2647 | Regression_eta: 0.0038 | Regression_pt: 0.2689 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1484 | Classification_binary: 0.3269 | Regression_cos_phi: 0.1708 | Regression_energy: 0.1446 | Regression_eta: 0.0997 | Regression_pt: 0.1576 | Regression_sin_phi: 0.1584 | LR: 9.50e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5779s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7834 | max_loss: 0.1641 | max_backward: 1.8407 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.8255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6029.1 | uss_MB: 5003.6 | child_rss_MB: 6150.1 | allocated_MB: 448.9 | reserved_MB: 26846.0 | inactive_split_MB: 1232.3 +rank1 [2026-10-28 17:37:19,852] INFO : Step 7700/50000 rank1 | Train Loss: 1.7524 | Train Loss Unweighted: 0.9761 | Train Loss Components: Auxiliary: 0.3200 | Cardinality: 0.1783 | Classification: 0.1756 | Classification_binary: 0.2521 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2705 | Regression_eta: 0.0035 | Regression_pt: 0.2709 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1018 | Classification_binary: 0.2136 | Regression_cos_phi: 0.1169 | Regression_energy: 0.1400 | Regression_eta: 0.0700 | Regression_pt: 0.1504 | Regression_sin_phi: 0.1253 | LR: 9.48e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4849s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7659 | max_loss: 0.1551 | max_backward: 1.8103 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6027.2 | uss_MB: 5004.4 | child_rss_MB: 6154.6 | allocated_MB: 448.8 | reserved_MB: 26846.0 | inactive_split_MB: 1042.2 +rank1 [2026-10-28 17:42:10,706] INFO : Step 7800/50000 rank1 | Train Loss: 1.8039 | Train Loss Unweighted: 1.0226 | Train Loss Components: Auxiliary: 0.3378 | Cardinality: 0.1686 | Classification: 0.1822 | Classification_binary: 0.2556 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2863 | Regression_eta: 0.0034 | Regression_pt: 0.2914 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1176 | Classification_binary: 0.2825 | Regression_cos_phi: 0.1718 | Regression_energy: 0.1405 | Regression_eta: 0.1095 | Regression_pt: 0.1558 | Regression_sin_phi: 0.1752 | LR: 9.47e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6881s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.8680 | max_loss: 0.1424 | max_backward: 1.9949 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.8163 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 6024.0 | uss_MB: 5003.5 | child_rss_MB: 6156.5 | allocated_MB: 448.6 | reserved_MB: 26846.0 | inactive_split_MB: 1396.9 +rank1 [2026-10-28 17:47:05,102] INFO : Step 7900/50000 rank1 | Train Loss: 2.0641 | Train Loss Unweighted: 1.0960 | Train Loss Components: Auxiliary: 0.3855 | Cardinality: 0.2401 | Classification: 0.2031 | Classification_binary: 0.3367 | HitGrouping: 0.2085 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2721 | Regression_eta: 0.0041 | Regression_pt: 0.2759 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1568 | Classification_binary: 0.3840 | Regression_cos_phi: 0.2798 | Regression_energy: 0.1447 | Regression_eta: 0.1436 | Regression_pt: 0.1597 | Regression_sin_phi: 0.1983 | LR: 9.46e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6373s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7971 | max_loss: 0.2430 | max_backward: 1.9131 | max_optimizer_step: 0.2681 | max_tensorboard_logging: 0.7971 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6025.3 | uss_MB: 5004.3 | child_rss_MB: 6161.5 | allocated_MB: 451.8 | reserved_MB: 27044.0 | inactive_split_MB: 827.4 +rank1 [2026-10-28 17:51:56,121] INFO : Step 8000/50000 rank1 | Train Loss: 1.8391 | Train Loss Unweighted: 0.9813 | Train Loss Components: Auxiliary: 0.3355 | Cardinality: 0.2153 | Classification: 0.1625 | Classification_binary: 0.2896 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2605 | Regression_eta: 0.0035 | Regression_pt: 0.2616 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1147 | Classification_binary: 0.2789 | Regression_cos_phi: 0.1624 | Regression_energy: 0.1331 | Regression_eta: 0.1002 | Regression_pt: 0.1410 | Regression_sin_phi: 0.1408 | LR: 9.44e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.5541s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 1.2188 | max_loss: 0.4570 | max_backward: 3.9408 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.5566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6013.9 | uss_MB: 4999.4 | child_rss_MB: 6155.0 | allocated_MB: 448.4 | reserved_MB: 27044.0 | inactive_split_MB: 1156.9 +rank1 [2026-10-28 17:51:56,978] INFO : Running validation on rank1 +rank1 [2026-10-28 17:51:56,978] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 17:51:56,978] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 17:52:50,097] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:52:50,645] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 17:53:16,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 17:53:17,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 17:53:18,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 17:53:19,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 17:53:29,990] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 17:53:30,250] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:53:30,806] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 17:53:47,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 17:53:48,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 17:53:49,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 17:53:50,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 17:54:00,308] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 17:54:01,630] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 17:54:02,177] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 17:54:17,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 17:54:18,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 17:54:19,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 17:54:20,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 17:54:30,021] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 17:59:43,744] INFO : Step 8100/50000 rank1 | Train Loss: 1.7101 | Train Loss Unweighted: 0.9276 | Train Loss Components: Auxiliary: 0.3166 | Cardinality: 0.1787 | Classification: 0.1664 | Classification_binary: 0.2381 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2564 | Regression_eta: 0.0032 | Regression_pt: 0.2599 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.2397 | Regression_cos_phi: 0.0995 | Regression_energy: 0.1355 | Regression_eta: 0.0611 | Regression_pt: 0.1508 | Regression_sin_phi: 0.1036 | LR: 9.43e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6988s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7628 | max_loss: 0.1353 | max_backward: 2.0123 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.8300 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6970.2 | uss_MB: 5951.6 | child_rss_MB: 6190.8 | allocated_MB: 447.2 | reserved_MB: 25306.0 | inactive_split_MB: 1261.3 +rank1 [2026-10-28 18:04:56,111] INFO : Step 8200/50000 rank1 | Train Loss: 1.8745 | Train Loss Unweighted: 1.0293 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.1946 | Classification: 0.1986 | Classification_binary: 0.2966 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2609 | Regression_eta: 0.0035 | Regression_pt: 0.2657 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1287 | Classification_binary: 0.3334 | Regression_cos_phi: 0.2103 | Regression_energy: 0.1456 | Regression_eta: 0.1279 | Regression_pt: 0.1585 | Regression_sin_phi: 0.1465 | LR: 9.42e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.3082s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7790 | max_loss: 0.1535 | max_backward: 1.9181 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.5352 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6080.6 | uss_MB: 5064.4 | child_rss_MB: 6184.5 | allocated_MB: 448.2 | reserved_MB: 25360.0 | inactive_split_MB: 1159.7 +rank1 [2026-10-28 18:09:37,297] INFO : Step 8300/50000 rank1 | Train Loss: 1.8727 | Train Loss Unweighted: 1.0119 | Train Loss Components: Auxiliary: 0.3401 | Cardinality: 0.1867 | Classification: 0.1728 | Classification_binary: 0.3272 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2513 | Regression_eta: 0.0033 | Regression_pt: 0.2533 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0907 | Classification_binary: 0.2959 | Regression_cos_phi: 0.1286 | Regression_energy: 0.1228 | Regression_eta: 0.0795 | Regression_pt: 0.1341 | Regression_sin_phi: 0.1337 | LR: 9.40e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5429s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0436 | max_forward: 1.1303 | max_loss: 0.1503 | max_backward: 2.0213 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6082.4 | uss_MB: 5067.6 | child_rss_MB: 6182.5 | allocated_MB: 449.6 | reserved_MB: 27362.0 | inactive_split_MB: 1355.2 +rank1 [2026-10-28 18:14:10,520] INFO : Step 8400/50000 rank1 | Train Loss: 1.6705 | Train Loss Unweighted: 0.9205 | Train Loss Components: Auxiliary: 0.3113 | Cardinality: 0.1938 | Classification: 0.1656 | Classification_binary: 0.2512 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2485 | Regression_eta: 0.0028 | Regression_pt: 0.2494 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1177 | Classification_binary: 0.2416 | Regression_cos_phi: 0.1735 | Regression_energy: 0.1248 | Regression_eta: 0.0867 | Regression_pt: 0.1328 | Regression_sin_phi: 0.1626 | LR: 9.39e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4479s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8079 | max_loss: 0.1514 | max_backward: 1.6942 | max_optimizer_step: 0.2670 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6074.8 | uss_MB: 5060.4 | child_rss_MB: 6188.9 | allocated_MB: 446.4 | reserved_MB: 27366.0 | inactive_split_MB: 1386.9 +rank1 [2026-10-28 18:18:45,092] INFO : Step 8500/50000 rank1 | Train Loss: 1.8447 | Train Loss Unweighted: 0.9684 | Train Loss Components: Auxiliary: 0.3299 | Cardinality: 0.2197 | Classification: 0.1758 | Classification_binary: 0.2823 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2478 | Regression_eta: 0.0040 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1253 | Classification_binary: 0.3136 | Regression_cos_phi: 0.1829 | Regression_energy: 0.1335 | Regression_eta: 0.1040 | Regression_pt: 0.1508 | Regression_sin_phi: 0.1513 | LR: 9.37e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.1677s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0066 | max_forward: 0.7185 | max_loss: 0.1465 | max_backward: 1.6783 | max_optimizer_step: 0.2239 | max_tensorboard_logging: 0.6269 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 6077.8 | uss_MB: 5064.0 | child_rss_MB: 6185.3 | allocated_MB: 448.4 | reserved_MB: 27368.0 | inactive_split_MB: 1257.8 +rank1 [2026-10-28 18:23:30,152] INFO : Step 8600/50000 rank1 | Train Loss: 1.9431 | Train Loss Unweighted: 1.0237 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.2483 | Classification: 0.1814 | Classification_binary: 0.3328 | HitGrouping: 0.2024 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2495 | Regression_eta: 0.0034 | Regression_pt: 0.2527 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0985 | Classification_binary: 0.3210 | Regression_cos_phi: 0.1613 | Regression_energy: 0.1282 | Regression_eta: 0.0873 | Regression_pt: 0.1408 | Regression_sin_phi: 0.1706 | LR: 9.36e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.5545s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1011 | max_forward: 0.9340 | max_loss: 0.1508 | max_backward: 1.7704 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.7686 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 6082.0 | uss_MB: 5062.1 | child_rss_MB: 6191.5 | allocated_MB: 448.9 | reserved_MB: 27368.0 | inactive_split_MB: 1268.3 +rank1 [2026-10-28 18:28:08,444] INFO : Step 8700/50000 rank1 | Train Loss: 1.9739 | Train Loss Unweighted: 1.0707 | Train Loss Components: Auxiliary: 0.3725 | Cardinality: 0.2322 | Classification: 0.1854 | Classification_binary: 0.2934 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2900 | Regression_eta: 0.0037 | Regression_pt: 0.2944 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1111 | Classification_binary: 0.2827 | Regression_cos_phi: 0.1645 | Regression_energy: 0.1428 | Regression_eta: 0.0978 | Regression_pt: 0.1597 | Regression_sin_phi: 0.1536 | LR: 9.34e-04 | DataLoad Time: 0.0193s | Model Forward Time: 6.1632s | Phase Times: max_pre_step_barrier: 0.0890 | max_batch_to_device: 0.0067 | max_forward: 0.7726 | max_loss: 0.1574 | max_backward: 1.6332 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 2.5948 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 6082.7 | uss_MB: 5062.6 | child_rss_MB: 6185.4 | allocated_MB: 448.0 | reserved_MB: 29420.0 | inactive_split_MB: 876.0 +rank1 [2026-10-28 18:32:55,586] INFO : Step 8800/50000 rank1 | Train Loss: 1.5247 | Train Loss Unweighted: 0.8421 | Train Loss Components: Auxiliary: 0.2808 | Cardinality: 0.1524 | Classification: 0.1581 | Classification_binary: 0.2323 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2217 | Regression_eta: 0.0029 | Regression_pt: 0.2242 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.2254 | Regression_cos_phi: 0.1403 | Regression_energy: 0.1118 | Regression_eta: 0.1184 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0892 | LR: 9.33e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.9916s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7382 | max_loss: 0.2414 | max_backward: 1.7741 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 6078.3 | uss_MB: 5061.0 | child_rss_MB: 6166.0 | allocated_MB: 447.9 | reserved_MB: 29420.0 | inactive_split_MB: 1260.3 +rank1 [2026-10-28 18:37:36,896] INFO : Step 8900/50000 rank1 | Train Loss: 1.7540 | Train Loss Unweighted: 0.9436 | Train Loss Components: Auxiliary: 0.3227 | Cardinality: 0.2072 | Classification: 0.1816 | Classification_binary: 0.2735 | HitGrouping: 0.1904 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2393 | Regression_eta: 0.0031 | Regression_pt: 0.2426 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1023 | Classification_binary: 0.2676 | Regression_cos_phi: 0.1892 | Regression_energy: 0.1518 | Regression_eta: 0.0711 | Regression_pt: 0.1675 | Regression_sin_phi: 0.1313 | LR: 9.31e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4475s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8077 | max_loss: 0.2089 | max_backward: 1.9976 | max_optimizer_step: 0.2211 | max_tensorboard_logging: 0.5747 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 6080.9 | uss_MB: 5000.4 | child_rss_MB: 6188.7 | allocated_MB: 451.8 | reserved_MB: 29420.0 | inactive_split_MB: 1133.8 +rank1 [2026-10-28 18:42:20,745] INFO : Step 9000/50000 rank1 | Train Loss: 1.7807 | Train Loss Unweighted: 0.9795 | Train Loss Components: Auxiliary: 0.3326 | Cardinality: 0.1942 | Classification: 0.1600 | Classification_binary: 0.3157 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2461 | Regression_eta: 0.0033 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0911 | Classification_binary: 0.3016 | Regression_cos_phi: 0.1547 | Regression_energy: 0.1294 | Regression_eta: 0.0903 | Regression_pt: 0.1425 | Regression_sin_phi: 0.1609 | LR: 9.30e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8566s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.8696 | max_loss: 0.1559 | max_backward: 1.8245 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.9520 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 6064.6 | uss_MB: 5051.1 | child_rss_MB: 6187.5 | allocated_MB: 448.9 | reserved_MB: 29420.0 | inactive_split_MB: 1051.6 +rank1 [2026-10-28 18:42:21,624] INFO : Running validation on rank1 +rank1 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 18:43:19,658] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:43:20,246] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 18:43:47,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 18:43:48,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 18:43:49,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 18:43:50,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 18:44:00,274] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 18:44:00,792] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:44:01,307] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 18:44:17,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 18:44:18,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 18:44:19,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 18:44:20,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 18:44:30,748] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 18:44:31,056] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 18:44:31,910] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 18:44:47,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 18:44:48,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 18:44:49,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 18:44:50,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 18:45:00,416] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 18:50:14,978] INFO : Step 9100/50000 rank1 | Train Loss: 1.6002 | Train Loss Unweighted: 0.8685 | Train Loss Components: Auxiliary: 0.3013 | Cardinality: 0.1975 | Classification: 0.1658 | Classification_binary: 0.2349 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2304 | Regression_eta: 0.0027 | Regression_pt: 0.2318 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0888 | Classification_binary: 0.2222 | Regression_cos_phi: 0.1344 | Regression_energy: 0.1147 | Regression_eta: 0.0892 | Regression_pt: 0.1235 | Regression_sin_phi: 0.1220 | LR: 9.28e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5185s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7981 | max_loss: 0.1460 | max_backward: 1.6475 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.9178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7065.6 | uss_MB: 6052.1 | child_rss_MB: 6167.4 | allocated_MB: 449.7 | reserved_MB: 25788.0 | inactive_split_MB: 957.3 +rank1 [2026-10-28 18:55:02,311] INFO : Step 9200/50000 rank1 | Train Loss: 1.6859 | Train Loss Unweighted: 0.8867 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.1906 | Classification: 0.1447 | Classification_binary: 0.2542 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2389 | Regression_eta: 0.0033 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1009 | Classification_binary: 0.2454 | Regression_cos_phi: 0.1771 | Regression_energy: 0.1284 | Regression_eta: 0.0940 | Regression_pt: 0.1415 | Regression_sin_phi: 0.1180 | LR: 9.27e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3384s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7865 | max_loss: 0.1441 | max_backward: 1.8922 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6128.5 | uss_MB: 5123.6 | child_rss_MB: 6161.7 | allocated_MB: 450.8 | reserved_MB: 25788.0 | inactive_split_MB: 1040.1 +rank1 [2026-10-28 18:59:49,908] INFO : Step 9300/50000 rank1 | Train Loss: 1.8980 | Train Loss Unweighted: 0.9669 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.3104 | Classification: 0.1729 | Classification_binary: 0.3073 | HitGrouping: 0.1936 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2397 | Regression_eta: 0.0030 | Regression_pt: 0.2407 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1093 | Classification_binary: 0.3132 | Regression_cos_phi: 0.1255 | Regression_energy: 0.1247 | Regression_eta: 0.0627 | Regression_pt: 0.1344 | Regression_sin_phi: 0.1132 | LR: 9.25e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.1840s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0143 | max_forward: 0.7819 | max_loss: 0.1515 | max_backward: 1.8209 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6071.1 | uss_MB: 5121.9 | child_rss_MB: 5951.4 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1031.1 +rank1 [2026-10-28 19:04:46,837] INFO : Step 9400/50000 rank1 | Train Loss: 1.5878 | Train Loss Unweighted: 0.8563 | Train Loss Components: Auxiliary: 0.2880 | Cardinality: 0.1552 | Classification: 0.1580 | Classification_binary: 0.2625 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2141 | Regression_eta: 0.0030 | Regression_pt: 0.2153 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0874 | Classification_binary: 0.2529 | Regression_cos_phi: 0.1147 | Regression_energy: 0.1024 | Regression_eta: 0.0634 | Regression_pt: 0.1134 | Regression_sin_phi: 0.1376 | LR: 9.24e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0180s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7342 | max_loss: 0.2471 | max_backward: 1.7320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.4671 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6067.3 | uss_MB: 5118.7 | child_rss_MB: 5945.7 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1121.1 +rank1 [2026-10-28 19:09:47,342] INFO : Step 9500/50000 rank1 | Train Loss: 1.5519 | Train Loss Unweighted: 0.8141 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1719 | Classification: 0.1393 | Classification_binary: 0.2430 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2129 | Regression_eta: 0.0030 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.2678 | Regression_cos_phi: 0.1376 | Regression_energy: 0.1141 | Regression_eta: 0.0931 | Regression_pt: 0.1217 | Regression_sin_phi: 0.1390 | LR: 9.22e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1295s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0076 | max_forward: 0.7261 | max_loss: 0.1376 | max_backward: 1.8267 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6069.1 | uss_MB: 5122.8 | child_rss_MB: 5952.7 | allocated_MB: 446.7 | reserved_MB: 27752.0 | inactive_split_MB: 1128.4 +rank1 [2026-10-28 19:14:55,455] INFO : Step 9600/50000 rank1 | Train Loss: 1.7379 | Train Loss Unweighted: 0.9171 | Train Loss Components: Auxiliary: 0.3221 | Cardinality: 0.2199 | Classification: 0.1679 | Classification_binary: 0.2886 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2245 | Regression_eta: 0.0032 | Regression_pt: 0.2297 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1008 | Classification_binary: 0.2857 | Regression_cos_phi: 0.1525 | Regression_energy: 0.1274 | Regression_eta: 0.0830 | Regression_pt: 0.1392 | Regression_sin_phi: 0.1505 | LR: 9.21e-04 | DataLoad Time: 0.0157s | Model Forward Time: 7.7320s | Phase Times: max_pre_step_barrier: 0.0081 | max_batch_to_device: 0.0183 | max_forward: 3.1116 | max_loss: 0.7194 | max_backward: 4.4829 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6068.7 | uss_MB: 5120.4 | child_rss_MB: 5924.4 | allocated_MB: 449.4 | reserved_MB: 29732.0 | inactive_split_MB: 1322.6 +rank1 [2026-10-28 19:20:02,940] INFO : Step 9700/50000 rank1 | Train Loss: 1.8683 | Train Loss Unweighted: 0.9602 | Train Loss Components: Auxiliary: 0.3449 | Cardinality: 0.2461 | Classification: 0.1512 | Classification_binary: 0.3271 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2364 | Regression_eta: 0.0037 | Regression_pt: 0.2383 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0885 | Classification_binary: 0.3623 | Regression_cos_phi: 0.1636 | Regression_energy: 0.1191 | Regression_eta: 0.1343 | Regression_pt: 0.1348 | Regression_sin_phi: 0.1693 | LR: 9.19e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.0318s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0069 | max_forward: 0.8299 | max_loss: 0.7776 | max_backward: 2.5571 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5726 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6067.5 | uss_MB: 5123.7 | child_rss_MB: 5943.1 | allocated_MB: 450.3 | reserved_MB: 29734.0 | inactive_split_MB: 1128.1 +rank1 [2026-10-28 19:25:00,572] INFO : Step 9800/50000 rank1 | Train Loss: 1.6119 | Train Loss Unweighted: 0.8200 | Train Loss Components: Auxiliary: 0.3036 | Cardinality: 0.2266 | Classification: 0.1371 | Classification_binary: 0.2410 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2176 | Regression_eta: 0.0027 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2260 | Regression_cos_phi: 0.0810 | Regression_energy: 0.1190 | Regression_eta: 0.0520 | Regression_pt: 0.1297 | Regression_sin_phi: 0.1546 | LR: 9.17e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5939s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7430 | max_loss: 0.2467 | max_backward: 1.8961 | max_optimizer_step: 0.2008 | max_tensorboard_logging: 0.9056 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6064.8 | uss_MB: 5122.0 | child_rss_MB: 5925.9 | allocated_MB: 446.9 | reserved_MB: 29734.0 | inactive_split_MB: 905.1 +rank1 [2026-10-28 19:29:40,210] INFO : Step 9900/50000 rank1 | Train Loss: 1.5408 | Train Loss Unweighted: 0.8391 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1631 | Classification: 0.1532 | Classification_binary: 0.2416 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2178 | Regression_eta: 0.0029 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0924 | Classification_binary: 0.2867 | Regression_cos_phi: 0.1068 | Regression_energy: 0.1152 | Regression_eta: 0.0926 | Regression_pt: 0.1256 | Regression_sin_phi: 0.1253 | LR: 9.16e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6033s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7611 | max_loss: 0.1425 | max_backward: 1.8994 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6066.1 | uss_MB: 5122.8 | child_rss_MB: 5945.5 | allocated_MB: 448.3 | reserved_MB: 29734.0 | inactive_split_MB: 1119.7 +rank1 [2026-10-28 19:34:23,378] INFO : Step 10000/50000 rank1 | Train Loss: 1.7147 | Train Loss Unweighted: 0.9225 | Train Loss Components: Auxiliary: 0.3261 | Cardinality: 0.1969 | Classification: 0.1590 | Classification_binary: 0.2912 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2305 | Regression_eta: 0.0029 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.2423 | Regression_cos_phi: 0.1275 | Regression_energy: 0.0963 | Regression_eta: 0.0577 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0929 | LR: 9.14e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8783s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1140 | max_forward: 0.8951 | max_loss: 0.1695 | max_backward: 1.8837 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 1.0339 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 6055.8 | uss_MB: 5112.2 | child_rss_MB: 5922.2 | allocated_MB: 449.4 | reserved_MB: 29734.0 | inactive_split_MB: 1320.6 +rank1 [2026-10-28 19:34:24,225] INFO : Running validation on rank1 +rank1 [2026-10-28 19:34:24,225] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 19:34:24,225] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 19:35:21,693] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:35:22,180] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 19:35:48,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 19:35:49,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 19:35:50,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 19:35:51,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 19:36:01,987] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 19:36:02,455] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:36:02,961] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 19:36:18,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 19:36:20,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 19:36:21,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 19:36:22,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 19:36:32,161] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 19:36:36,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 19:36:36,977] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 19:36:52,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 19:36:53,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 19:36:54,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 19:36:55,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 19:37:05,029] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-28 19:42:23,379] INFO : Step 10100/50000 rank1 | Train Loss: 1.6697 | Train Loss Unweighted: 0.8627 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.2287 | Classification: 0.1283 | Classification_binary: 0.2543 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2359 | Regression_eta: 0.0032 | Regression_pt: 0.2378 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0948 | Classification_binary: 0.2417 | Regression_cos_phi: 0.1355 | Regression_energy: 0.1108 | Regression_eta: 0.0909 | Regression_pt: 0.1208 | Regression_sin_phi: 0.1477 | LR: 9.12e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.7399s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7652 | max_loss: 0.1496 | max_backward: 1.8791 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 0.9326 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 7065.4 | uss_MB: 6119.6 | child_rss_MB: 5927.2 | allocated_MB: 448.1 | reserved_MB: 26360.0 | inactive_split_MB: 1080.0 +rank1 [2026-10-28 19:47:10,586] INFO : Step 10200/50000 rank1 | Train Loss: 1.3968 | Train Loss Unweighted: 0.7619 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1483 | Classification: 0.1293 | Classification_binary: 0.2070 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2090 | Regression_eta: 0.0026 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2056 | Regression_cos_phi: 0.0966 | Regression_energy: 0.0997 | Regression_eta: 0.0635 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0913 | LR: 9.11e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.4837s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0330 | max_forward: 0.8960 | max_loss: 0.5080 | max_backward: 1.9040 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.6816 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 6137.8 | uss_MB: 5194.1 | child_rss_MB: 5912.5 | allocated_MB: 450.3 | reserved_MB: 27130.0 | inactive_split_MB: 1088.5 +rank1 [2026-10-28 19:51:56,122] INFO : Step 10300/50000 rank1 | Train Loss: 1.6039 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.2943 | Cardinality: 0.1656 | Classification: 0.1487 | Classification_binary: 0.2679 | HitGrouping: 0.1881 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2132 | Regression_eta: 0.0030 | Regression_pt: 0.2158 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2815 | Regression_cos_phi: 0.0878 | Regression_energy: 0.1063 | Regression_eta: 0.0580 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0898 | LR: 9.09e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7923s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8152 | max_loss: 0.2478 | max_backward: 2.0483 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6143.7 | uss_MB: 5198.5 | child_rss_MB: 5930.4 | allocated_MB: 448.7 | reserved_MB: 28404.0 | inactive_split_MB: 991.9 +rank1 [2026-10-28 19:56:39,451] INFO : Step 10400/50000 rank1 | Train Loss: 1.7188 | Train Loss Unweighted: 0.8798 | Train Loss Components: Auxiliary: 0.3209 | Cardinality: 0.2578 | Classification: 0.1544 | Classification_binary: 0.2910 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2139 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1106 | Classification_binary: 0.2707 | Regression_cos_phi: 0.1661 | Regression_energy: 0.1202 | Regression_eta: 0.0795 | Regression_pt: 0.1280 | Regression_sin_phi: 0.1131 | LR: 9.07e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6747s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7821 | max_loss: 0.2448 | max_backward: 1.8782 | max_optimizer_step: 0.2932 | max_tensorboard_logging: 0.8061 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 6134.3 | uss_MB: 5193.2 | child_rss_MB: 5938.4 | allocated_MB: 448.9 | reserved_MB: 28404.0 | inactive_split_MB: 983.3 +rank1 [2026-10-28 20:01:29,563] INFO : Step 10500/50000 rank1 | Train Loss: 1.4652 | Train Loss Unweighted: 0.7986 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1697 | Classification: 0.1280 | Classification_binary: 0.2159 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2232 | Regression_eta: 0.0026 | Regression_pt: 0.2263 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1182 | Classification_binary: 0.2214 | Regression_cos_phi: 0.0837 | Regression_energy: 0.1271 | Regression_eta: 0.0828 | Regression_pt: 0.1379 | Regression_sin_phi: 0.1084 | LR: 9.06e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.2132s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.7113 | max_loss: 0.1455 | max_backward: 1.6308 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.7760 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6134.1 | uss_MB: 5192.0 | child_rss_MB: 5916.0 | allocated_MB: 444.7 | reserved_MB: 28404.0 | inactive_split_MB: 1307.3 +rank1 [2026-10-28 20:06:14,520] INFO : Step 10600/50000 rank1 | Train Loss: 1.5657 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2802 | Cardinality: 0.1297 | Classification: 0.1617 | Classification_binary: 0.2471 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2084 | Regression_eta: 0.0037 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1077 | Classification_binary: 0.2597 | Regression_cos_phi: 0.1323 | Regression_energy: 0.1116 | Regression_eta: 0.0933 | Regression_pt: 0.1225 | Regression_sin_phi: 0.1528 | LR: 9.04e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.9408s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0173 | max_forward: 0.7875 | max_loss: 0.2211 | max_backward: 1.9019 | max_optimizer_step: 0.2814 | max_tensorboard_logging: 0.8518 | max_tensorboard_flush: 0.0072 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6133.2 | uss_MB: 5193.0 | child_rss_MB: 5934.7 | allocated_MB: 449.8 | reserved_MB: 28404.0 | inactive_split_MB: 1130.2 +rank1 [2026-10-28 20:10:59,504] INFO : Step 10700/50000 rank1 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8676 | Train Loss Components: Auxiliary: 0.3003 | Cardinality: 0.1749 | Classification: 0.1453 | Classification_binary: 0.2956 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2085 | Regression_eta: 0.0033 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0858 | Classification_binary: 0.3178 | Regression_cos_phi: 0.1212 | Regression_energy: 0.1129 | Regression_eta: 0.1317 | Regression_pt: 0.1232 | Regression_sin_phi: 0.1351 | LR: 9.02e-04 | DataLoad Time: 0.0114s | Model Forward Time: 5.1708s | Phase Times: max_pre_step_barrier: 0.0305 | max_batch_to_device: 0.0154 | max_forward: 2.0246 | max_loss: 0.2864 | max_backward: 2.7424 | max_optimizer_step: 0.1980 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6136.9 | uss_MB: 5195.8 | child_rss_MB: 5906.4 | allocated_MB: 446.2 | reserved_MB: 29148.0 | inactive_split_MB: 1134.5 +rank1 [2026-10-28 20:15:41,682] INFO : Step 10800/50000 rank1 | Train Loss: 1.5849 | Train Loss Unweighted: 0.7936 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1942 | Classification: 0.1383 | Classification_binary: 0.2480 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2007 | Regression_eta: 0.0032 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0970 | Classification_binary: 0.2284 | Regression_cos_phi: 0.1772 | Regression_energy: 0.1146 | Regression_eta: 0.0819 | Regression_pt: 0.1228 | Regression_sin_phi: 0.1232 | LR: 9.00e-04 | DataLoad Time: 0.0175s | Model Forward Time: 8.2570s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2408 | max_backward: 1.7977 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 1.7470 | max_tensorboard_flush: 0.0028 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 6126.4 | uss_MB: 4363.9 | child_rss_MB: 5910.3 | allocated_MB: 448.6 | reserved_MB: 30628.0 | inactive_split_MB: 1292.6 +rank1 [2026-10-28 20:20:32,832] INFO : Step 10900/50000 rank1 | Train Loss: 1.3552 | Train Loss Unweighted: 0.7247 | Train Loss Components: Auxiliary: 0.2531 | Cardinality: 0.1449 | Classification: 0.1275 | Classification_binary: 0.2074 | HitGrouping: 0.1874 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1917 | Regression_eta: 0.0024 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0875 | Classification_binary: 0.2217 | Regression_cos_phi: 0.1057 | Regression_energy: 0.0810 | Regression_eta: 0.0625 | Regression_pt: 0.0896 | Regression_sin_phi: 0.1019 | LR: 8.99e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4552s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7076 | max_loss: 0.2425 | max_backward: 1.6536 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6131.8 | uss_MB: 5191.3 | child_rss_MB: 5932.5 | allocated_MB: 446.4 | reserved_MB: 30628.0 | inactive_split_MB: 984.9 +rank1 [2026-10-28 20:25:36,607] INFO : Step 11000/50000 rank1 | Train Loss: 1.5999 | Train Loss Unweighted: 0.8322 | Train Loss Components: Auxiliary: 0.2959 | Cardinality: 0.1737 | Classification: 0.1477 | Classification_binary: 0.2877 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0017 | Regression_energy: 0.1943 | Regression_eta: 0.0030 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0971 | Classification_binary: 0.2938 | Regression_cos_phi: 0.1303 | Regression_energy: 0.1056 | Regression_eta: 0.0697 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0842 | LR: 8.97e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9316s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8260 | max_loss: 0.1503 | max_backward: 1.9603 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6124.9 | uss_MB: 5185.7 | child_rss_MB: 5918.3 | allocated_MB: 451.8 | reserved_MB: 30628.0 | inactive_split_MB: 996.6 +rank1 [2026-10-28 20:25:37,478] INFO : Running validation on rank1 +rank1 [2026-10-28 20:25:37,478] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 20:26:32,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:26:33,361] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 20:27:00,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 20:27:01,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 20:27:02,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 20:27:03,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 20:27:13,462] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 20:27:14,319] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:27:14,833] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 20:27:31,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 20:27:32,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 20:27:33,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 20:27:35,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 20:27:45,154] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-28 20:27:46,110] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 20:27:46,649] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 20:28:02,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 20:28:03,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 20:28:04,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 20:28:05,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 20:28:15,085] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 20:33:29,190] INFO : Step 11100/50000 rank1 | Train Loss: 1.6214 | Train Loss Unweighted: 0.8328 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.2186 | Classification: 0.1350 | Classification_binary: 0.2802 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2042 | Regression_eta: 0.0030 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0847 | Classification_binary: 0.2626 | Regression_cos_phi: 0.1319 | Regression_energy: 0.0917 | Regression_eta: 0.0900 | Regression_pt: 0.1019 | Regression_sin_phi: 0.1258 | LR: 8.95e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8831s | Phase Times: max_pre_step_barrier: 0.0088 | max_batch_to_device: 0.0068 | max_forward: 0.8488 | max_loss: 0.1548 | max_backward: 1.7645 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 1.1046 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7069.9 | uss_MB: 6125.7 | child_rss_MB: 5940.1 | allocated_MB: 448.1 | reserved_MB: 24340.0 | inactive_split_MB: 1081.9 +rank1 [2026-10-28 20:38:18,706] INFO : Step 11200/50000 rank1 | Train Loss: 1.7050 | Train Loss Unweighted: 0.8927 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2640 | Classification: 0.1534 | Classification_binary: 0.2523 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2404 | Regression_eta: 0.0030 | Regression_pt: 0.2410 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0776 | Classification_binary: 0.2120 | Regression_cos_phi: 0.1150 | Regression_energy: 0.1215 | Regression_eta: 0.0786 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0589 | LR: 8.93e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.8971s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8707 | max_loss: 0.2001 | max_backward: 2.2037 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.7912 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 6163.4 | uss_MB: 5218.5 | child_rss_MB: 5949.2 | allocated_MB: 447.8 | reserved_MB: 26392.0 | inactive_split_MB: 938.7 +rank1 [2026-10-28 20:43:01,723] INFO : Step 11300/50000 rank1 | Train Loss: 1.3885 | Train Loss Unweighted: 0.7511 | Train Loss Components: Auxiliary: 0.2594 | Cardinality: 0.1357 | Classification: 0.1316 | Classification_binary: 0.2357 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1874 | Regression_eta: 0.0024 | Regression_pt: 0.1915 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1054 | Classification_binary: 0.2711 | Regression_cos_phi: 0.1272 | Regression_energy: 0.1057 | Regression_eta: 0.0784 | Regression_pt: 0.1167 | Regression_sin_phi: 0.1139 | LR: 8.91e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4298s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8053 | max_loss: 0.1469 | max_backward: 2.0062 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.4618 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1345 | Memory: rss_MB: 6165.0 | uss_MB: 5222.0 | child_rss_MB: 5948.8 | allocated_MB: 450.6 | reserved_MB: 26400.0 | inactive_split_MB: 1092.9 +rank1 [2026-10-28 20:48:05,522] INFO : Step 11400/50000 rank1 | Train Loss: 1.5865 | Train Loss Unweighted: 0.8323 | Train Loss Components: Auxiliary: 0.2981 | Cardinality: 0.1866 | Classification: 0.1338 | Classification_binary: 0.2899 | HitGrouping: 0.1840 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1999 | Regression_eta: 0.0028 | Regression_pt: 0.2030 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0804 | Classification_binary: 0.3080 | Regression_cos_phi: 0.1251 | Regression_energy: 0.1179 | Regression_eta: 0.0784 | Regression_pt: 0.1285 | Regression_sin_phi: 0.1172 | LR: 8.89e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.7675s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0769 | max_forward: 1.3800 | max_loss: 0.6715 | max_backward: 3.0339 | max_optimizer_step: 0.2660 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6163.4 | uss_MB: 5219.6 | child_rss_MB: 5946.4 | allocated_MB: 448.1 | reserved_MB: 27144.0 | inactive_split_MB: 1081.9 +rank1 [2026-10-28 20:52:57,975] INFO : Step 11500/50000 rank1 | Train Loss: 1.4825 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2779 | Cardinality: 0.1999 | Classification: 0.1273 | Classification_binary: 0.2463 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1956 | Regression_eta: 0.0026 | Regression_pt: 0.1972 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0879 | Classification_binary: 0.2131 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0891 | Regression_eta: 0.0595 | Regression_pt: 0.0968 | Regression_sin_phi: 0.1189 | LR: 8.88e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0562s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.7822 | max_loss: 0.1471 | max_backward: 1.7308 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 1.4583 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6164.2 | uss_MB: 5219.4 | child_rss_MB: 5944.4 | allocated_MB: 450.8 | reserved_MB: 27144.0 | inactive_split_MB: 1342.3 +rank1 [2026-10-28 20:57:41,224] INFO : Step 11600/50000 rank1 | Train Loss: 1.4219 | Train Loss Unweighted: 0.7444 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1679 | Classification: 0.1415 | Classification_binary: 0.2106 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1945 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.2194 | Regression_cos_phi: 0.1151 | Regression_energy: 0.0867 | Regression_eta: 0.0553 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0955 | LR: 8.86e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7014s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8362 | max_loss: 0.1428 | max_backward: 2.0109 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6156.9 | uss_MB: 5220.2 | child_rss_MB: 5947.3 | allocated_MB: 448.6 | reserved_MB: 27146.0 | inactive_split_MB: 1315.4 +rank1 [2026-10-28 21:02:35,117] INFO : Step 11700/50000 rank1 | Train Loss: 1.5203 | Train Loss Unweighted: 0.8017 | Train Loss Components: Auxiliary: 0.2866 | Cardinality: 0.1889 | Classification: 0.1302 | Classification_binary: 0.2577 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2024 | Regression_eta: 0.0026 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0868 | Classification_binary: 0.2637 | Regression_cos_phi: 0.1358 | Regression_energy: 0.1092 | Regression_eta: 0.0700 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0983 | LR: 8.84e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.3172s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 1.0014 | max_loss: 0.1975 | max_backward: 2.5347 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0688 | Memory: rss_MB: 6157.0 | uss_MB: 5144.6 | child_rss_MB: 5922.6 | allocated_MB: 448.7 | reserved_MB: 27266.0 | inactive_split_MB: 1090.9 +rank1 [2026-10-28 21:07:42,591] INFO : Step 11800/50000 rank1 | Train Loss: 1.5591 | Train Loss Unweighted: 0.7907 | Train Loss Components: Auxiliary: 0.2873 | Cardinality: 0.2051 | Classification: 0.1346 | Classification_binary: 0.2655 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1918 | Regression_eta: 0.0030 | Regression_pt: 0.1929 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.2321 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0976 | Regression_eta: 0.0789 | Regression_pt: 0.1057 | Regression_sin_phi: 0.0848 | LR: 8.82e-04 | DataLoad Time: 0.0202s | Model Forward Time: 7.4843s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.0966 | max_forward: 1.7907 | max_loss: 0.6496 | max_backward: 2.7103 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 2.3587 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6136.9 | uss_MB: 5209.7 | child_rss_MB: 5946.4 | allocated_MB: 450.4 | reserved_MB: 27266.0 | inactive_split_MB: 1187.2 +rank1 [2026-10-28 21:12:51,258] INFO : Step 11900/50000 rank1 | Train Loss: 1.3283 | Train Loss Unweighted: 0.7113 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1476 | Classification: 0.1193 | Classification_binary: 0.2094 | HitGrouping: 0.1824 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0021 | Regression_pt: 0.1901 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1930 | Regression_cos_phi: 0.1061 | Regression_energy: 0.0944 | Regression_eta: 0.0490 | Regression_pt: 0.1005 | Regression_sin_phi: 0.1226 | LR: 8.80e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.1707s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0785 | max_forward: 0.7917 | max_loss: 0.1465 | max_backward: 1.5595 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6137.9 | uss_MB: 5216.4 | child_rss_MB: 5947.1 | allocated_MB: 443.9 | reserved_MB: 27266.0 | inactive_split_MB: 955.3 +rank1 [2026-10-28 21:17:48,735] INFO : Step 12000/50000 rank1 | Train Loss: 1.4074 | Train Loss Unweighted: 0.7415 | Train Loss Components: Auxiliary: 0.2683 | Cardinality: 0.1719 | Classification: 0.1235 | Classification_binary: 0.2283 | HitGrouping: 0.1812 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1909 | Regression_eta: 0.0022 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0987 | Regression_eta: 0.0539 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0888 | LR: 8.78e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2514s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0303 | max_forward: 0.7589 | max_loss: 0.1459 | max_backward: 1.9073 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6136.6 | uss_MB: 5219.4 | child_rss_MB: 5952.0 | allocated_MB: 447.6 | reserved_MB: 27266.0 | inactive_split_MB: 1310.4 +rank1 [2026-10-28 21:17:49,718] INFO : Running validation on rank1 +rank1 [2026-10-28 21:17:49,718] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 21:17:49,718] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 21:18:44,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:18:45,601] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 21:19:11,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 21:19:12,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 21:19:13,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 21:19:14,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 21:19:24,782] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 21:19:25,274] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:19:25,828] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 21:19:41,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 21:19:42,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 21:19:43,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 21:19:44,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 21:19:54,537] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 21:19:55,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 21:19:56,382] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 21:20:12,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 21:20:13,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 21:20:13,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 21:20:14,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 21:20:24,814] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 21:25:33,195] INFO : Step 12100/50000 rank1 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8521 | Train Loss Components: Auxiliary: 0.3093 | Cardinality: 0.2088 | Classification: 0.1337 | Classification_binary: 0.2971 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0029 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0826 | Classification_binary: 0.2928 | Regression_cos_phi: 0.1152 | Regression_energy: 0.0968 | Regression_eta: 0.0627 | Regression_pt: 0.1032 | Regression_sin_phi: 0.1221 | LR: 8.76e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9389s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8139 | max_loss: 0.1586 | max_backward: 1.9999 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9439 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7027.3 | uss_MB: 6104.0 | child_rss_MB: 5925.8 | allocated_MB: 450.8 | reserved_MB: 28008.0 | inactive_split_MB: 1125.6 +rank1 [2026-10-28 21:30:25,390] INFO : Step 12200/50000 rank1 | Train Loss: 1.4655 | Train Loss Unweighted: 0.7431 | Train Loss Components: Auxiliary: 0.2757 | Cardinality: 0.1845 | Classification: 0.1184 | Classification_binary: 0.2450 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1862 | Regression_eta: 0.0026 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.2226 | Regression_cos_phi: 0.0949 | Regression_energy: 0.0905 | Regression_eta: 0.0595 | Regression_pt: 0.0976 | Regression_sin_phi: 0.0973 | LR: 8.74e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4985s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0364 | max_forward: 0.7684 | max_loss: 0.1438 | max_backward: 1.8240 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.7243 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0697 | Memory: rss_MB: 6148.5 | uss_MB: 5233.4 | child_rss_MB: 5949.1 | allocated_MB: 445.9 | reserved_MB: 28008.0 | inactive_split_MB: 995.3 +rank1 [2026-10-28 21:35:16,328] INFO : Step 12300/50000 rank1 | Train Loss: 1.3845 | Train Loss Unweighted: 0.7270 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1523 | Classification: 0.1341 | Classification_binary: 0.2109 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1869 | Regression_eta: 0.0026 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0878 | Classification_binary: 0.2363 | Regression_cos_phi: 0.1713 | Regression_energy: 0.1132 | Regression_eta: 0.1004 | Regression_pt: 0.1253 | Regression_sin_phi: 0.1190 | LR: 8.72e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5873s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7264 | max_loss: 0.1976 | max_backward: 1.8534 | max_optimizer_step: 0.2993 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6147.3 | uss_MB: 5233.4 | child_rss_MB: 5944.1 | allocated_MB: 448.1 | reserved_MB: 29988.0 | inactive_split_MB: 1298.2 +rank1 [2026-10-28 21:40:15,076] INFO : Step 12400/50000 rank1 | Train Loss: 1.4006 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.2654 | Cardinality: 0.1602 | Classification: 0.1310 | Classification_binary: 0.2363 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2255 | Regression_cos_phi: 0.1082 | Regression_energy: 0.0792 | Regression_eta: 0.0734 | Regression_pt: 0.0869 | Regression_sin_phi: 0.1069 | LR: 8.70e-04 | DataLoad Time: 0.0156s | Model Forward Time: 7.1286s | Phase Times: max_pre_step_barrier: 0.0107 | max_batch_to_device: 0.0207 | max_forward: 1.2284 | max_loss: 0.2740 | max_backward: 2.6428 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8714 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 6147.7 | uss_MB: 5022.1 | child_rss_MB: 5936.9 | allocated_MB: 448.8 | reserved_MB: 29988.0 | inactive_split_MB: 1332.2 +rank1 [2026-10-28 21:45:07,595] INFO : Step 12500/50000 rank1 | Train Loss: 1.4931 | Train Loss Unweighted: 0.7691 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1768 | Classification: 0.1197 | Classification_binary: 0.2587 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1910 | Regression_eta: 0.0024 | Regression_pt: 0.1944 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2519 | Regression_cos_phi: 0.1286 | Regression_energy: 0.0922 | Regression_eta: 0.0614 | Regression_pt: 0.1016 | Regression_sin_phi: 0.1137 | LR: 8.68e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.7417s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0333 | max_forward: 0.8806 | max_loss: 0.1518 | max_backward: 2.1554 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.5983 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6144.5 | uss_MB: 5232.3 | child_rss_MB: 5946.8 | allocated_MB: 452.8 | reserved_MB: 29990.0 | inactive_split_MB: 1046.7 +rank1 [2026-10-28 21:50:03,650] INFO : Step 12600/50000 rank1 | Train Loss: 1.3547 | Train Loss Unweighted: 0.7028 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1629 | Classification: 0.1033 | Classification_binary: 0.2112 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1912 | Regression_eta: 0.0024 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1979 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0999 | Regression_eta: 0.0655 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0981 | LR: 8.66e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5163s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.7639 | max_loss: 0.2516 | max_backward: 1.8947 | max_optimizer_step: 0.2049 | max_tensorboard_logging: 0.7350 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6141.1 | uss_MB: 5233.0 | child_rss_MB: 5920.5 | allocated_MB: 446.9 | reserved_MB: 29990.0 | inactive_split_MB: 1034.9 +rank1 [2026-10-28 21:54:54,644] INFO : Step 12700/50000 rank1 | Train Loss: 1.3557 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1573 | Classification: 0.1262 | Classification_binary: 0.2132 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1863 | Regression_eta: 0.0022 | Regression_pt: 0.1887 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.2011 | Regression_cos_phi: 0.1120 | Regression_energy: 0.1140 | Regression_eta: 0.0836 | Regression_pt: 0.1255 | Regression_sin_phi: 0.0877 | LR: 8.64e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.1254s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7499 | max_loss: 0.2453 | max_backward: 3.4334 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6139.6 | uss_MB: 5232.1 | child_rss_MB: 5948.4 | allocated_MB: 450.2 | reserved_MB: 29990.0 | inactive_split_MB: 1245.6 +rank1 [2026-10-28 21:59:39,371] INFO : Step 12800/50000 rank1 | Train Loss: 1.5298 | Train Loss Unweighted: 0.7754 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1939 | Classification: 0.1237 | Classification_binary: 0.2821 | HitGrouping: 0.1822 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1810 | Regression_eta: 0.0027 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2576 | Regression_cos_phi: 0.0955 | Regression_energy: 0.0898 | Regression_eta: 0.0465 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0685 | LR: 8.62e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.2926s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.5299 | max_backward: 2.3185 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6133.4 | uss_MB: 4813.1 | child_rss_MB: 5927.2 | allocated_MB: 449.9 | reserved_MB: 32006.0 | inactive_split_MB: 889.3 +rank1 [2026-10-28 22:04:39,756] INFO : Step 12900/50000 rank1 | Train Loss: 1.3302 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1576 | Classification: 0.1054 | Classification_binary: 0.2343 | HitGrouping: 0.1811 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1758 | Regression_eta: 0.0020 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1960 | Regression_cos_phi: 0.0684 | Regression_energy: 0.1029 | Regression_eta: 0.0553 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0989 | LR: 8.60e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.6276s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0137 | max_forward: 0.9836 | max_loss: 0.2293 | max_backward: 2.2888 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.9302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6139.6 | uss_MB: 5231.4 | child_rss_MB: 5950.5 | allocated_MB: 445.9 | reserved_MB: 32006.0 | inactive_split_MB: 1247.6 +rank1 [2026-10-28 22:09:25,338] INFO : Step 13000/50000 rank1 | Train Loss: 1.3124 | Train Loss Unweighted: 0.7073 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1406 | Classification: 0.1147 | Classification_binary: 0.1973 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1944 | Regression_eta: 0.0023 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.2017 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0876 | Regression_eta: 0.0557 | Regression_pt: 0.0980 | Regression_sin_phi: 0.1236 | LR: 8.58e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2800s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0065 | max_forward: 0.7321 | max_loss: 0.1616 | max_backward: 1.7374 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6140.2 | uss_MB: 5231.6 | child_rss_MB: 5949.6 | allocated_MB: 444.7 | reserved_MB: 32006.0 | inactive_split_MB: 1300.2 +rank1 [2026-10-28 22:09:26,329] INFO : Running validation on rank1 +rank1 [2026-10-28 22:09:26,329] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 22:09:26,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 22:10:21,901] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:10:22,476] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 22:10:48,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 22:10:49,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 22:10:50,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 22:10:51,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 22:11:01,814] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 22:11:02,351] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:11:02,875] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 22:11:19,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 22:11:20,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 22:11:21,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 22:11:22,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 22:11:32,459] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 22:11:32,690] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 22:11:33,356] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 22:11:49,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 22:11:49,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 22:11:50,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 22:11:51,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 22:12:01,762] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 22:17:08,486] INFO : Step 13100/50000 rank1 | Train Loss: 1.6165 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3082 | Cardinality: 0.2635 | Classification: 0.1364 | Classification_binary: 0.2628 | HitGrouping: 0.1835 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0026 | Regression_pt: 0.1957 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0854 | Classification_binary: 0.2344 | Regression_cos_phi: 0.1020 | Regression_energy: 0.1052 | Regression_eta: 0.0502 | Regression_pt: 0.1143 | Regression_sin_phi: 0.1053 | LR: 8.56e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2705s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0479 | max_forward: 0.8994 | max_loss: 0.2807 | max_backward: 2.1108 | max_optimizer_step: 0.2760 | max_tensorboard_logging: 0.9690 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 7057.8 | uss_MB: 6143.5 | child_rss_MB: 5948.1 | allocated_MB: 446.5 | reserved_MB: 23120.0 | inactive_split_MB: 920.0 +rank1 [2026-10-28 22:22:01,650] INFO : Step 13200/50000 rank1 | Train Loss: 1.4928 | Train Loss Unweighted: 0.7469 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.2211 | Classification: 0.1278 | Classification_binary: 0.2505 | HitGrouping: 0.1746 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1808 | Regression_eta: 0.0025 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2172 | Regression_cos_phi: 0.1095 | Regression_energy: 0.1123 | Regression_eta: 0.0552 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0768 | LR: 8.54e-04 | DataLoad Time: 0.0194s | Model Forward Time: 6.5986s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.7755 | max_loss: 0.2864 | max_backward: 3.4589 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6222.1 | uss_MB: 5074.0 | child_rss_MB: 5943.3 | allocated_MB: 448.3 | reserved_MB: 25156.0 | inactive_split_MB: 978.0 +rank1 [2026-10-28 22:26:52,974] INFO : Step 13300/50000 rank1 | Train Loss: 1.3251 | Train Loss Unweighted: 0.6741 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1607 | Classification: 0.1132 | Classification_binary: 0.1927 | HitGrouping: 0.1773 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1805 | Regression_eta: 0.0024 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0908 | Regression_energy: 0.1021 | Regression_eta: 0.0497 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0989 | LR: 8.52e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3794s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8118 | max_loss: 0.1467 | max_backward: 1.5503 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.9039 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6221.5 | uss_MB: 5307.8 | child_rss_MB: 5952.8 | allocated_MB: 446.0 | reserved_MB: 26736.0 | inactive_split_MB: 965.8 +rank1 [2026-10-28 22:31:44,119] INFO : Step 13400/50000 rank1 | Train Loss: 1.2635 | Train Loss Unweighted: 0.6770 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1436 | Classification: 0.1317 | Classification_binary: 0.2016 | HitGrouping: 0.1790 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1692 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0970 | Regression_eta: 0.0588 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0764 | LR: 8.50e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7363s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8396 | max_loss: 0.1412 | max_backward: 2.0130 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.8047 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6225.0 | uss_MB: 5310.9 | child_rss_MB: 5925.4 | allocated_MB: 445.8 | reserved_MB: 26736.0 | inactive_split_MB: 1014.4 +rank1 [2026-10-28 22:36:27,627] INFO : Step 13500/50000 rank1 | Train Loss: 1.3935 | Train Loss Unweighted: 0.7401 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1713 | Classification: 0.1245 | Classification_binary: 0.2541 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0021 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.2705 | Regression_cos_phi: 0.0936 | Regression_energy: 0.0945 | Regression_eta: 0.0646 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0882 | LR: 8.48e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5764s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0859 | max_forward: 0.8721 | max_loss: 0.1498 | max_backward: 1.7581 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.7855 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6221.6 | uss_MB: 5308.9 | child_rss_MB: 5951.2 | allocated_MB: 449.1 | reserved_MB: 26736.0 | inactive_split_MB: 922.5 +rank1 [2026-10-28 22:41:19,303] INFO : Step 13600/50000 rank1 | Train Loss: 1.3997 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1870 | Classification: 0.1206 | Classification_binary: 0.2286 | HitGrouping: 0.1765 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1702 | Regression_eta: 0.0026 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.2042 | Regression_cos_phi: 0.0966 | Regression_energy: 0.0994 | Regression_eta: 0.0590 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0871 | LR: 8.45e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2267s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7703 | max_loss: 0.2484 | max_backward: 1.6943 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 6210.7 | uss_MB: 5299.9 | child_rss_MB: 5953.4 | allocated_MB: 450.3 | reserved_MB: 26736.0 | inactive_split_MB: 922.0 +rank1 [2026-10-28 22:46:12,326] INFO : Step 13700/50000 rank1 | Train Loss: 1.2648 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2420 | Cardinality: 0.1315 | Classification: 0.1136 | Classification_binary: 0.1964 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1793 | Regression_eta: 0.0020 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1775 | Regression_cos_phi: 0.0993 | Regression_energy: 0.1172 | Regression_eta: 0.0507 | Regression_pt: 0.1282 | Regression_sin_phi: 0.0854 | LR: 8.43e-04 | DataLoad Time: 0.0105s | Model Forward Time: 6.3000s | Phase Times: max_pre_step_barrier: 0.0406 | max_batch_to_device: 0.0119 | max_forward: 1.2281 | max_loss: 0.2480 | max_backward: 3.6472 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.6460 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0767 | Memory: rss_MB: 6216.0 | uss_MB: 5307.4 | child_rss_MB: 5948.4 | allocated_MB: 446.9 | reserved_MB: 26742.0 | inactive_split_MB: 1058.7 +rank1 [2026-10-28 22:51:09,234] INFO : Step 13800/50000 rank1 | Train Loss: 1.3469 | Train Loss Unweighted: 0.7266 | Train Loss Components: Auxiliary: 0.2604 | Cardinality: 0.1436 | Classification: 0.1184 | Classification_binary: 0.2382 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1812 | Regression_eta: 0.0021 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.2643 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0989 | Regression_eta: 0.0741 | Regression_pt: 0.1062 | Regression_sin_phi: 0.1106 | LR: 8.41e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.8536s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7995 | max_loss: 0.1458 | max_backward: 3.4735 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6221.0 | uss_MB: 5310.1 | child_rss_MB: 5919.7 | allocated_MB: 446.4 | reserved_MB: 27082.0 | inactive_split_MB: 1370.3 +rank1 [2026-10-28 22:56:04,348] INFO : Step 13900/50000 rank1 | Train Loss: 1.4882 | Train Loss Unweighted: 0.7682 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1631 | Classification: 0.1199 | Classification_binary: 0.2624 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1900 | Regression_eta: 0.0029 | Regression_pt: 0.1902 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.2163 | Regression_cos_phi: 0.0810 | Regression_energy: 0.1024 | Regression_eta: 0.0732 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0708 | LR: 8.39e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4519s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.7750 | max_loss: 0.1563 | max_backward: 1.5716 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.8868 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6217.9 | uss_MB: 5307.2 | child_rss_MB: 5946.5 | allocated_MB: 449.9 | reserved_MB: 27316.0 | inactive_split_MB: 918.0 +rank1 [2026-10-28 23:00:54,751] INFO : Step 14000/50000 rank1 | Train Loss: 1.2239 | Train Loss Unweighted: 0.6384 | Train Loss Components: Auxiliary: 0.2362 | Cardinality: 0.1573 | Classification: 0.0898 | Classification_binary: 0.1853 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0021 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0755 | Classification_binary: 0.1675 | Regression_cos_phi: 0.0931 | Regression_energy: 0.0931 | Regression_eta: 0.0510 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0886 | LR: 8.37e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5039s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7611 | max_loss: 0.1442 | max_backward: 1.7887 | max_optimizer_step: 0.2976 | max_tensorboard_logging: 0.7418 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6219.2 | uss_MB: 5309.9 | child_rss_MB: 5948.9 | allocated_MB: 446.8 | reserved_MB: 29264.0 | inactive_split_MB: 1150.9 +rank1 [2026-10-28 23:00:55,744] INFO : Running validation on rank1 +rank1 [2026-10-28 23:00:55,744] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 23:01:50,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:01:51,151] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 23:02:17,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 23:02:19,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 23:02:21,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 23:02:22,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 23:02:32,441] INFO : Time taken to make predictions on device 1 is: 0.28 min +rank1 [2026-10-28 23:02:33,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:02:33,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 23:02:50,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 23:02:51,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 23:02:52,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 23:02:53,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 23:03:03,286] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 23:03:03,406] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:03:03,862] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 23:03:19,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 23:03:20,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 23:03:21,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 23:03:22,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 23:03:32,138] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 23:08:22,314] INFO : Step 14100/50000 rank1 | Train Loss: 1.3472 | Train Loss Unweighted: 0.6964 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1511 | Classification: 0.1164 | Classification_binary: 0.2090 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1819 | Regression_eta: 0.0023 | Regression_pt: 0.1843 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2018 | Regression_cos_phi: 0.1214 | Regression_energy: 0.0949 | Regression_eta: 0.0504 | Regression_pt: 0.1039 | Regression_sin_phi: 0.0722 | LR: 8.35e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.2190s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8885 | max_loss: 0.1430 | max_backward: 2.1784 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 1.1214 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7149.4 | uss_MB: 6238.2 | child_rss_MB: 5960.3 | allocated_MB: 451.8 | reserved_MB: 25710.0 | inactive_split_MB: 1103.7 +rank1 [2026-10-28 23:12:52,862] INFO : Step 14200/50000 rank1 | Train Loss: 1.4358 | Train Loss Unweighted: 0.7354 | Train Loss Components: Auxiliary: 0.2744 | Cardinality: 0.1685 | Classification: 0.1248 | Classification_binary: 0.2623 | HitGrouping: 0.1814 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1709 | Regression_eta: 0.0023 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.2400 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0820 | Regression_eta: 0.0543 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0883 | LR: 8.32e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.3443s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0875 | max_forward: 0.8579 | max_loss: 0.1577 | max_backward: 1.8303 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6333.7 | uss_MB: 5422.0 | child_rss_MB: 5960.5 | allocated_MB: 449.2 | reserved_MB: 27638.0 | inactive_split_MB: 1319.6 +rank1 [2026-10-28 23:17:26,477] INFO : Step 14300/50000 rank1 | Train Loss: 1.2642 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2452 | Cardinality: 0.1652 | Classification: 0.1069 | Classification_binary: 0.2108 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0018 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0981 | Regression_eta: 0.0394 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0628 | LR: 8.30e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3458s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7599 | max_loss: 0.1477 | max_backward: 1.7369 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6331.9 | uss_MB: 5419.4 | child_rss_MB: 5959.7 | allocated_MB: 446.3 | reserved_MB: 29582.0 | inactive_split_MB: 949.7 +rank1 [2026-10-28 23:22:05,870] INFO : Step 14400/50000 rank1 | Train Loss: 1.2457 | Train Loss Unweighted: 0.6253 | Train Loss Components: Auxiliary: 0.2300 | Cardinality: 0.1408 | Classification: 0.1072 | Classification_binary: 0.1847 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1629 | Regression_eta: 0.0022 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2152 | Regression_cos_phi: 0.0902 | Regression_energy: 0.0939 | Regression_eta: 0.0612 | Regression_pt: 0.1052 | Regression_sin_phi: 0.1637 | LR: 8.28e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.9708s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.9667 | max_loss: 0.2306 | max_backward: 2.7434 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6331.8 | uss_MB: 4969.8 | child_rss_MB: 5956.2 | allocated_MB: 450.7 | reserved_MB: 29584.0 | inactive_split_MB: 1252.8 +rank1 [2026-10-28 23:26:50,539] INFO : Step 14500/50000 rank1 | Train Loss: 1.4093 | Train Loss Unweighted: 0.7194 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1803 | Classification: 0.1211 | Classification_binary: 0.2406 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0869 | Classification_binary: 0.2661 | Regression_cos_phi: 0.0867 | Regression_energy: 0.1020 | Regression_eta: 0.0489 | Regression_pt: 0.1109 | Regression_sin_phi: 0.1033 | LR: 8.26e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8494s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8204 | max_loss: 0.1455 | max_backward: 2.0838 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7592 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1066 | Memory: rss_MB: 6329.0 | uss_MB: 5423.5 | child_rss_MB: 5932.6 | allocated_MB: 449.1 | reserved_MB: 29584.0 | inactive_split_MB: 1312.1 +rank1 [2026-10-28 23:31:26,129] INFO : Step 14600/50000 rank1 | Train Loss: 1.4466 | Train Loss Unweighted: 0.7628 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1607 | Classification: 0.1257 | Classification_binary: 0.2761 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1767 | Regression_eta: 0.0023 | Regression_pt: 0.1796 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.2485 | Regression_cos_phi: 0.0916 | Regression_energy: 0.1081 | Regression_eta: 0.0508 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0871 | LR: 8.24e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8562s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0622 | max_forward: 0.8658 | max_loss: 0.1496 | max_backward: 1.9094 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.9894 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6320.2 | uss_MB: 5414.6 | child_rss_MB: 5962.0 | allocated_MB: 450.2 | reserved_MB: 29584.0 | inactive_split_MB: 939.8 +rank1 [2026-10-28 23:35:59,714] INFO : Step 14700/50000 rank1 | Train Loss: 1.2298 | Train Loss Unweighted: 0.6544 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1321 | Classification: 0.0995 | Classification_binary: 0.1882 | HitGrouping: 0.1788 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1813 | Regression_eta: 0.0021 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.1715 | Regression_cos_phi: 0.1362 | Regression_energy: 0.1037 | Regression_eta: 0.0530 | Regression_pt: 0.1121 | Regression_sin_phi: 0.1140 | LR: 8.21e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.9552s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8361 | max_loss: 0.2462 | max_backward: 1.9700 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.8314 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6323.9 | uss_MB: 5421.4 | child_rss_MB: 5964.3 | allocated_MB: 445.8 | reserved_MB: 31548.0 | inactive_split_MB: 1127.1 +rank1 [2026-10-28 23:40:34,505] INFO : Step 14800/50000 rank1 | Train Loss: 1.2682 | Train Loss Unweighted: 0.6574 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1516 | Classification: 0.1063 | Classification_binary: 0.2120 | HitGrouping: 0.1767 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1664 | Regression_eta: 0.0020 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0868 | Regression_eta: 0.0406 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0616 | LR: 8.19e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.6940s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7530 | max_loss: 0.1476 | max_backward: 1.8229 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6323.4 | uss_MB: 5420.5 | child_rss_MB: 5957.9 | allocated_MB: 449.4 | reserved_MB: 33554.0 | inactive_split_MB: 1332.6 +rank1 [2026-10-28 23:45:11,571] INFO : Step 14900/50000 rank1 | Train Loss: 1.2753 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2409 | Cardinality: 0.1305 | Classification: 0.1126 | Classification_binary: 0.2387 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1497 | Regression_eta: 0.0022 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.2395 | Regression_cos_phi: 0.1127 | Regression_energy: 0.0814 | Regression_eta: 0.0629 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0812 | LR: 8.17e-04 | DataLoad Time: 0.0141s | Model Forward Time: 4.8762s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.7955 | max_loss: 0.1633 | max_backward: 1.8376 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.9623 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1206 | Memory: rss_MB: 6322.3 | uss_MB: 5420.8 | child_rss_MB: 5961.8 | allocated_MB: 449.7 | reserved_MB: 33554.0 | inactive_split_MB: 1310.1 +rank1 [2026-10-28 23:49:45,307] INFO : Step 15000/50000 rank1 | Train Loss: 1.2701 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1516 | Classification: 0.1012 | Classification_binary: 0.2121 | HitGrouping: 0.1755 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.2051 | Regression_cos_phi: 0.1058 | Regression_energy: 0.0858 | Regression_eta: 0.0633 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0818 | LR: 8.15e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1026s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7505 | max_loss: 0.2449 | max_backward: 1.7335 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.4870 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6318.2 | uss_MB: 5419.0 | child_rss_MB: 5953.6 | allocated_MB: 447.3 | reserved_MB: 33554.0 | inactive_split_MB: 1038.0 +rank1 [2026-10-28 23:49:46,285] INFO : Running validation on rank1 +rank1 [2026-10-28 23:49:46,285] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-28 23:49:46,285] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-28 23:50:37,476] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:50:38,005] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-28 23:51:04,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-28 23:51:05,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-28 23:51:06,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-28 23:51:07,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-28 23:51:17,723] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 23:51:18,491] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:51:18,969] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-28 23:51:35,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-28 23:51:36,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-28 23:51:37,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-28 23:51:38,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-28 23:51:48,030] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-28 23:51:48,549] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-28 23:51:49,025] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-28 23:52:05,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-28 23:52:05,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-28 23:52:06,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-28 23:52:07,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-28 23:52:17,626] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-28 23:57:18,881] INFO : Step 15100/50000 rank1 | Train Loss: 1.2388 | Train Loss Unweighted: 0.6590 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1366 | Classification: 0.1178 | Classification_binary: 0.1841 | HitGrouping: 0.1810 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1743 | Regression_eta: 0.0018 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.2052 | Regression_cos_phi: 0.0934 | Regression_energy: 0.0912 | Regression_eta: 0.0443 | Regression_pt: 0.1000 | Regression_sin_phi: 0.0890 | LR: 8.12e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.8281s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.8010 | max_loss: 0.1660 | max_backward: 1.9832 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.8565 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7422.7 | uss_MB: 6518.8 | child_rss_MB: 5938.1 | allocated_MB: 446.2 | reserved_MB: 25230.0 | inactive_split_MB: 1087.3 +rank1 [2026-10-29 00:02:05,558] INFO : Step 15200/50000 rank1 | Train Loss: 1.3362 | Train Loss Unweighted: 0.6611 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1596 | Classification: 0.1086 | Classification_binary: 0.2256 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1600 | Regression_eta: 0.0022 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2551 | Regression_cos_phi: 0.1271 | Regression_energy: 0.0898 | Regression_eta: 0.0655 | Regression_pt: 0.0984 | Regression_sin_phi: 0.1022 | LR: 8.10e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6167s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1077 | max_forward: 0.8572 | max_loss: 0.1505 | max_backward: 1.8951 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7089 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6402.3 | uss_MB: 5500.4 | child_rss_MB: 5939.7 | allocated_MB: 449.9 | reserved_MB: 25448.0 | inactive_split_MB: 1183.4 +rank1 [2026-10-29 00:06:54,749] INFO : Step 15300/50000 rank1 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6783 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1400 | Classification: 0.1062 | Classification_binary: 0.2357 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1646 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2490 | Regression_cos_phi: 0.0812 | Regression_energy: 0.1010 | Regression_eta: 0.0644 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0893 | LR: 8.08e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7484s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7962 | max_loss: 0.2606 | max_backward: 1.9831 | max_optimizer_step: 0.2491 | max_tensorboard_logging: 0.7840 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6402.1 | uss_MB: 5500.3 | child_rss_MB: 5940.3 | allocated_MB: 446.7 | reserved_MB: 27282.0 | inactive_split_MB: 1189.4 +rank1 [2026-10-29 00:11:34,440] INFO : Step 15400/50000 rank1 | Train Loss: 1.2329 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1583 | Classification: 0.1127 | Classification_binary: 0.1891 | HitGrouping: 0.1750 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1648 | Regression_eta: 0.0018 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0681 | Regression_eta: 0.0406 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0811 | LR: 8.05e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2509s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.7153 | max_loss: 0.1508 | max_backward: 1.6501 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.5965 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6395.9 | uss_MB: 5491.2 | child_rss_MB: 5941.0 | allocated_MB: 444.7 | reserved_MB: 27520.0 | inactive_split_MB: 1102.9 +rank1 [2026-10-29 00:16:23,331] INFO : Step 15500/50000 rank1 | Train Loss: 1.2944 | Train Loss Unweighted: 0.6657 | Train Loss Components: Auxiliary: 0.2480 | Cardinality: 0.1645 | Classification: 0.1174 | Classification_binary: 0.2036 | HitGrouping: 0.1739 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1691 | Regression_eta: 0.0021 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1781 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0872 | Regression_eta: 0.0396 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0708 | LR: 8.03e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.2292s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.9970 | max_loss: 0.1740 | max_backward: 2.3916 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.8759 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 6406.1 | uss_MB: 5472.3 | child_rss_MB: 5938.5 | allocated_MB: 449.9 | reserved_MB: 27520.0 | inactive_split_MB: 1174.5 +rank1 [2026-10-29 00:21:07,253] INFO : Step 15600/50000 rank1 | Train Loss: 1.4613 | Train Loss Unweighted: 0.7617 | Train Loss Components: Auxiliary: 0.2872 | Cardinality: 0.1801 | Classification: 0.1276 | Classification_binary: 0.2641 | HitGrouping: 0.1785 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1795 | Regression_eta: 0.0025 | Regression_pt: 0.1856 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.2557 | Regression_cos_phi: 0.0988 | Regression_energy: 0.0895 | Regression_eta: 0.0634 | Regression_pt: 0.1015 | Regression_sin_phi: 0.1082 | LR: 8.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.7229s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.9893 | max_loss: 0.1680 | max_backward: 2.1731 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.0419 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6396.7 | uss_MB: 5034.1 | child_rss_MB: 5938.2 | allocated_MB: 448.1 | reserved_MB: 27520.0 | inactive_split_MB: 1080.9 +rank1 [2026-10-29 00:25:54,170] INFO : Step 15700/50000 rank1 | Train Loss: 1.3299 | Train Loss Unweighted: 0.6547 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1880 | Classification: 0.1192 | Classification_binary: 0.2203 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1547 | Regression_eta: 0.0020 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0870 | Classification_binary: 0.2413 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0750 | Regression_eta: 0.0496 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0964 | LR: 7.98e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.0327s | Phase Times: max_pre_step_barrier: 0.0061 | max_batch_to_device: 0.0068 | max_forward: 0.9761 | max_loss: 0.2543 | max_backward: 2.0225 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8016 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.1020 | Memory: rss_MB: 6402.6 | uss_MB: 5496.7 | child_rss_MB: 5941.3 | allocated_MB: 448.3 | reserved_MB: 27520.0 | inactive_split_MB: 1390.5 +rank1 [2026-10-29 00:30:33,526] INFO : Step 15800/50000 rank1 | Train Loss: 1.2079 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1572 | Classification: 0.0909 | Classification_binary: 0.1814 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1704 | Regression_eta: 0.0019 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.2110 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0793 | Regression_eta: 0.0522 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0898 | LR: 7.96e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3435s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1066 | max_forward: 0.8154 | max_loss: 0.1440 | max_backward: 1.6020 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.7740 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6405.3 | uss_MB: 5497.2 | child_rss_MB: 5941.3 | allocated_MB: 447.1 | reserved_MB: 27520.0 | inactive_split_MB: 1038.3 +rank1 [2026-10-29 00:35:19,265] INFO : Step 15900/50000 rank1 | Train Loss: 1.4055 | Train Loss Unweighted: 0.7313 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1808 | Classification: 0.1304 | Classification_binary: 0.2426 | HitGrouping: 0.1763 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1752 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1016 | Classification_binary: 0.2696 | Regression_cos_phi: 0.1184 | Regression_energy: 0.0864 | Regression_eta: 0.0869 | Regression_pt: 0.0931 | Regression_sin_phi: 0.1338 | LR: 7.94e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7738s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 1.0653 | max_loss: 0.1515 | max_backward: 2.0024 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.6957 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6402.5 | uss_MB: 5500.0 | child_rss_MB: 5940.1 | allocated_MB: 447.4 | reserved_MB: 27520.0 | inactive_split_MB: 1172.5 +rank1 [2026-10-29 00:40:00,683] INFO : Step 16000/50000 rank1 | Train Loss: 1.3107 | Train Loss Unweighted: 0.6580 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1760 | Classification: 0.1019 | Classification_binary: 0.2391 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1548 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2209 | Regression_cos_phi: 0.0926 | Regression_energy: 0.0739 | Regression_eta: 0.0552 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0717 | LR: 7.91e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7633s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.1053 | max_forward: 1.1703 | max_loss: 0.1516 | max_backward: 1.8974 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6403.0 | uss_MB: 5502.7 | child_rss_MB: 5941.0 | allocated_MB: 447.2 | reserved_MB: 27520.0 | inactive_split_MB: 1195.6 +rank1 [2026-10-29 00:40:01,695] INFO : Running validation on rank1 +rank1 [2026-10-29 00:40:01,695] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 00:40:01,695] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 00:40:58,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:40:58,625] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 00:41:23,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 00:41:24,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 00:41:25,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 00:41:26,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 00:41:36,605] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 00:41:36,990] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:41:37,530] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 00:41:54,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 00:41:55,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 00:41:56,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 00:41:57,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 00:42:07,278] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 00:42:07,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 00:42:07,840] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 00:42:23,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 00:42:24,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 00:42:24,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 00:42:25,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 00:42:35,871] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 00:47:34,083] INFO : Step 16100/50000 rank1 | Train Loss: 1.2037 | Train Loss Unweighted: 0.6025 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1467 | Classification: 0.1080 | Classification_binary: 0.1878 | HitGrouping: 0.1747 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1512 | Regression_eta: 0.0018 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.1694 | Regression_cos_phi: 0.1027 | Regression_energy: 0.0955 | Regression_eta: 0.0472 | Regression_pt: 0.1032 | Regression_sin_phi: 0.1276 | LR: 7.89e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0454s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7665 | max_loss: 0.1405 | max_backward: 1.4602 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7547.3 | uss_MB: 6643.5 | child_rss_MB: 5941.5 | allocated_MB: 446.1 | reserved_MB: 27680.0 | inactive_split_MB: 1038.2 +rank1 [2026-10-29 00:52:15,156] INFO : Step 16200/50000 rank1 | Train Loss: 1.2083 | Train Loss Unweighted: 0.6192 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1499 | Classification: 0.1107 | Classification_binary: 0.2017 | HitGrouping: 0.1711 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1506 | Regression_eta: 0.0018 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.2163 | Regression_cos_phi: 0.0976 | Regression_energy: 0.0882 | Regression_eta: 0.0456 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0675 | LR: 7.86e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9521s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0473 | max_forward: 0.8343 | max_loss: 0.2530 | max_backward: 2.1136 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6458.0 | uss_MB: 5555.6 | child_rss_MB: 5961.3 | allocated_MB: 446.7 | reserved_MB: 27682.0 | inactive_split_MB: 1157.2 +rank1 [2026-10-29 00:56:51,718] INFO : Step 16300/50000 rank1 | Train Loss: 1.2745 | Train Loss Unweighted: 0.6291 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1604 | Classification: 0.1062 | Classification_binary: 0.2417 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1373 | Regression_eta: 0.0022 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2687 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0798 | Regression_eta: 0.0712 | Regression_pt: 0.0868 | Regression_sin_phi: 0.1010 | LR: 7.84e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.8113s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.8059 | max_loss: 0.2506 | max_backward: 2.1763 | max_optimizer_step: 0.5213 | max_tensorboard_logging: 0.5923 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6459.9 | uss_MB: 5555.3 | child_rss_MB: 5936.4 | allocated_MB: 450.9 | reserved_MB: 27682.0 | inactive_split_MB: 950.8 +rank1 [2026-10-29 01:01:26,580] INFO : Step 16400/50000 rank1 | Train Loss: 1.2654 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2380 | Cardinality: 0.1554 | Classification: 0.1095 | Classification_binary: 0.2018 | HitGrouping: 0.1699 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1561 | Regression_eta: 0.0023 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1867 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0759 | Regression_eta: 0.0587 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0942 | LR: 7.81e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.2778s | Phase Times: max_pre_step_barrier: 0.0533 | max_batch_to_device: 0.0064 | max_forward: 0.7459 | max_loss: 0.1486 | max_backward: 1.5352 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.6677 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6452.2 | uss_MB: 5545.1 | child_rss_MB: 5962.3 | allocated_MB: 446.4 | reserved_MB: 27682.0 | inactive_split_MB: 949.1 +rank1 [2026-10-29 01:06:05,049] INFO : Step 16500/50000 rank1 | Train Loss: 1.1728 | Train Loss Unweighted: 0.5918 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1444 | Classification: 0.0927 | Classification_binary: 0.1686 | HitGrouping: 0.1723 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0021 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0939 | Regression_eta: 0.0739 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0955 | LR: 7.79e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4904s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8043 | max_loss: 0.1772 | max_backward: 2.0250 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6458.1 | uss_MB: 5554.8 | child_rss_MB: 5927.7 | allocated_MB: 448.6 | reserved_MB: 29698.0 | inactive_split_MB: 1167.6 +rank1 [2026-10-29 01:10:53,703] INFO : Step 16600/50000 rank1 | Train Loss: 1.2920 | Train Loss Unweighted: 0.6427 | Train Loss Components: Auxiliary: 0.2455 | Cardinality: 0.1794 | Classification: 0.1148 | Classification_binary: 0.2098 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1561 | Regression_eta: 0.0021 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.2308 | Regression_cos_phi: 0.1167 | Regression_energy: 0.0896 | Regression_eta: 0.0606 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0829 | LR: 7.77e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2557s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0377 | max_forward: 1.0830 | max_loss: 0.1480 | max_backward: 1.7919 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6457.2 | uss_MB: 5552.9 | child_rss_MB: 5951.6 | allocated_MB: 449.3 | reserved_MB: 29698.0 | inactive_split_MB: 1014.7 +rank1 [2026-10-29 01:15:45,847] INFO : Step 16700/50000 rank1 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6325 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1731 | Classification: 0.1072 | Classification_binary: 0.2252 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1469 | Regression_eta: 0.0019 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2294 | Regression_cos_phi: 0.1089 | Regression_energy: 0.0752 | Regression_eta: 0.0516 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0939 | LR: 7.74e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4538s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0722 | max_loss: 0.1506 | max_backward: 2.0365 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5067 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6456.8 | uss_MB: 5555.5 | child_rss_MB: 5918.8 | allocated_MB: 448.3 | reserved_MB: 29698.0 | inactive_split_MB: 893.3 +rank1 [2026-10-29 01:20:20,822] INFO : Step 16800/50000 rank1 | Train Loss: 1.1947 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1542 | Classification: 0.1036 | Classification_binary: 0.1800 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0023 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0695 | Regression_eta: 0.0539 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0715 | LR: 7.72e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2742s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0166 | max_forward: 0.8224 | max_loss: 0.1466 | max_backward: 1.6608 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6428.0 | uss_MB: 5552.9 | child_rss_MB: 5812.3 | allocated_MB: 446.4 | reserved_MB: 29698.0 | inactive_split_MB: 1161.6 +rank1 [2026-10-29 01:25:06,863] INFO : Step 16900/50000 rank1 | Train Loss: 1.2025 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1406 | Classification: 0.1120 | Classification_binary: 0.1884 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1446 | Regression_eta: 0.0023 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.1776 | Regression_cos_phi: 0.1065 | Regression_energy: 0.0678 | Regression_eta: 0.0490 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0762 | LR: 7.69e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3232s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7445 | max_loss: 0.1453 | max_backward: 1.8211 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6418.8 | uss_MB: 5553.4 | child_rss_MB: 5783.4 | allocated_MB: 448.9 | reserved_MB: 29698.0 | inactive_split_MB: 901.3 +rank1 [2026-10-29 01:29:52,082] INFO : Step 17000/50000 rank1 | Train Loss: 1.2049 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2349 | Cardinality: 0.1467 | Classification: 0.1018 | Classification_binary: 0.2228 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.2203 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0714 | Regression_eta: 0.0502 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0531 | LR: 7.67e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1428s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0141 | max_forward: 1.3980 | max_loss: 0.5129 | max_backward: 2.5664 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.5255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1218 | Memory: rss_MB: 6382.7 | uss_MB: 5556.2 | child_rss_MB: 5576.1 | allocated_MB: 448.2 | reserved_MB: 29698.0 | inactive_split_MB: 955.8 +rank1 [2026-10-29 01:29:53,073] INFO : Running validation on rank1 +rank1 [2026-10-29 01:29:53,074] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 01:29:53,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 01:30:46,067] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:30:46,579] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 01:31:13,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 01:31:15,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 01:31:16,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 01:31:17,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 01:31:27,375] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 01:31:28,036] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:31:28,590] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 01:31:44,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 01:31:45,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 01:31:46,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 01:31:47,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 01:31:57,202] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 01:31:58,252] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 01:31:58,743] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 01:32:14,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 01:32:15,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 01:32:16,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 01:32:17,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 01:32:27,383] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 01:37:36,198] INFO : Step 17100/50000 rank1 | Train Loss: 1.2386 | Train Loss Unweighted: 0.5919 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1931 | Classification: 0.0995 | Classification_binary: 0.1969 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0895 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0954 | Regression_energy: 0.0895 | Regression_eta: 0.0609 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0726 | LR: 7.64e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.3596s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0067 | max_forward: 1.0304 | max_loss: 0.1869 | max_backward: 2.3467 | max_optimizer_step: 0.3175 | max_tensorboard_logging: 0.7405 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 7631.0 | uss_MB: 6798.9 | child_rss_MB: 5579.4 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 1373.9 +rank1 [2026-10-29 01:42:27,134] INFO : Step 17200/50000 rank1 | Train Loss: 1.1810 | Train Loss Unweighted: 0.5844 | Train Loss Components: Auxiliary: 0.2270 | Cardinality: 0.1495 | Classification: 0.1085 | Classification_binary: 0.1683 | HitGrouping: 0.1731 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0022 | Regression_pt: 0.1515 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0948 | Regression_energy: 0.0805 | Regression_eta: 0.0503 | Regression_pt: 0.0884 | Regression_sin_phi: 0.1153 | LR: 7.62e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.1058s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7638 | max_loss: 0.2424 | max_backward: 1.9393 | max_optimizer_step: 0.2921 | max_tensorboard_logging: 2.1267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6444.8 | uss_MB: 5633.5 | child_rss_MB: 5577.8 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 988.1 +rank1 [2026-10-29 01:47:11,312] INFO : Step 17300/50000 rank1 | Train Loss: 1.2271 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2397 | Cardinality: 0.1372 | Classification: 0.1102 | Classification_binary: 0.2127 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1512 | Regression_eta: 0.0022 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1980 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0779 | Regression_eta: 0.0494 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0851 | LR: 7.59e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.1265s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0243 | max_forward: 1.4273 | max_loss: 0.6053 | max_backward: 3.4810 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6426.5 | uss_MB: 5175.0 | child_rss_MB: 5586.3 | allocated_MB: 449.6 | reserved_MB: 26752.0 | inactive_split_MB: 1120.1 +rank1 [2026-10-29 01:52:04,203] INFO : Step 17400/50000 rank1 | Train Loss: 1.2297 | Train Loss Unweighted: 0.6013 | Train Loss Components: Auxiliary: 0.2404 | Cardinality: 0.1709 | Classification: 0.1017 | Classification_binary: 0.2232 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1354 | Regression_eta: 0.0019 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.2203 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0652 | Regression_eta: 0.0450 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0765 | LR: 7.57e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.6386s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0185 | max_forward: 1.0000 | max_loss: 0.1504 | max_backward: 1.9673 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6331 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6409.7 | uss_MB: 5631.7 | child_rss_MB: 5583.3 | allocated_MB: 447.1 | reserved_MB: 28732.0 | inactive_split_MB: 1125.6 +rank1 [2026-10-29 01:56:48,790] INFO : Step 17500/50000 rank1 | Train Loss: 1.2152 | Train Loss Unweighted: 0.6044 | Train Loss Components: Auxiliary: 0.2290 | Cardinality: 0.1485 | Classification: 0.1023 | Classification_binary: 0.1789 | HitGrouping: 0.1757 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1590 | Regression_eta: 0.0026 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1454 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0873 | Regression_eta: 0.0390 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0538 | LR: 7.54e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2340s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0167 | max_forward: 0.7711 | max_loss: 0.1449 | max_backward: 1.8153 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.4947 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6388.7 | uss_MB: 5638.8 | child_rss_MB: 5515.3 | allocated_MB: 450.1 | reserved_MB: 28732.0 | inactive_split_MB: 1375.8 +rank1 [2026-10-29 02:01:39,792] INFO : Step 17600/50000 rank1 | Train Loss: 1.1560 | Train Loss Unweighted: 0.6042 | Train Loss Components: Auxiliary: 0.2286 | Cardinality: 0.1294 | Classification: 0.1057 | Classification_binary: 0.1993 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1471 | Regression_eta: 0.0018 | Regression_pt: 0.1484 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1795 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0917 | Regression_eta: 0.0390 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0604 | LR: 7.52e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.8160s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.8868 | max_loss: 0.1705 | max_backward: 2.0183 | max_optimizer_step: 0.2066 | max_tensorboard_logging: 0.7222 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1349 | Memory: rss_MB: 6384.6 | uss_MB: 5639.3 | child_rss_MB: 5506.2 | allocated_MB: 448.3 | reserved_MB: 28732.0 | inactive_split_MB: 1319.7 +rank1 [2026-10-29 02:06:28,891] INFO : Step 17700/50000 rank1 | Train Loss: 1.7848 | Train Loss Unweighted: 0.7433 | Train Loss Components: Auxiliary: 0.3415 | Cardinality: 0.4306 | Classification: 0.1200 | Classification_binary: 0.2341 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1920 | Regression_eta: 0.0037 | Regression_pt: 0.1910 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0885 | Classification_binary: 0.2604 | Regression_cos_phi: 0.1389 | Regression_energy: 0.1035 | Regression_eta: 0.1062 | Regression_pt: 0.1111 | Regression_sin_phi: 0.1223 | LR: 7.49e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5772s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8788 | max_loss: 0.4738 | max_backward: 1.9859 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.5305 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6359.7 | uss_MB: 5640.4 | child_rss_MB: 5400.4 | allocated_MB: 450.1 | reserved_MB: 28738.0 | inactive_split_MB: 1125.6 +rank1 [2026-10-29 02:11:14,773] INFO : Step 17800/50000 rank1 | Train Loss: 1.1828 | Train Loss Unweighted: 0.6014 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1456 | Classification: 0.0982 | Classification_binary: 0.1996 | HitGrouping: 0.1749 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1503 | Regression_eta: 0.0019 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1741 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0799 | Regression_eta: 0.0604 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0669 | LR: 7.47e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.1932s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.3167 | max_backward: 2.2751 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 6343.6 | uss_MB: 5261.8 | child_rss_MB: 5331.5 | allocated_MB: 444.9 | reserved_MB: 28738.0 | inactive_split_MB: 1155.9 +rank1 [2026-10-29 02:16:10,311] INFO : Step 17900/50000 rank1 | Train Loss: 1.0765 | Train Loss Unweighted: 0.5646 | Train Loss Components: Auxiliary: 0.2108 | Cardinality: 0.1097 | Classification: 0.0932 | Classification_binary: 0.1585 | HitGrouping: 0.1686 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1536 | Regression_eta: 0.0016 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0793 | Regression_eta: 0.0471 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0571 | LR: 7.44e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2966s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7593 | max_loss: 0.4918 | max_backward: 1.7987 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.6052 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6290.9 | uss_MB: 5640.2 | child_rss_MB: 5109.0 | allocated_MB: 447.1 | reserved_MB: 28738.0 | inactive_split_MB: 1154.4 +rank1 [2026-10-29 02:20:55,875] INFO : Step 18000/50000 rank1 | Train Loss: 1.2227 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2303 | Cardinality: 0.1450 | Classification: 0.1088 | Classification_binary: 0.2099 | HitGrouping: 0.1650 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1447 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2479 | Regression_cos_phi: 0.1192 | Regression_energy: 0.0896 | Regression_eta: 0.0659 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0877 | LR: 7.42e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7013s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9293 | max_loss: 0.1567 | max_backward: 2.2476 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6283.0 | uss_MB: 5642.9 | child_rss_MB: 5063.5 | allocated_MB: 449.4 | reserved_MB: 28738.0 | inactive_split_MB: 1324.2 +rank1 [2026-10-29 02:20:56,966] INFO : Running validation on rank1 +rank1 [2026-10-29 02:20:56,966] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 02:20:56,966] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 02:21:57,510] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:21:58,045] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 02:22:25,247] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 02:22:26,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 02:22:27,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 02:22:28,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 02:22:38,284] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 02:22:38,913] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:22:39,418] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 02:22:55,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 02:22:56,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 02:22:57,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 02:22:58,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 02:23:08,166] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 02:23:08,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 02:23:09,544] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 02:23:25,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 02:23:26,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 02:23:27,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 02:23:28,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 02:23:38,171] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 02:28:43,001] INFO : Step 18100/50000 rank1 | Train Loss: 1.2922 | Train Loss Unweighted: 0.6330 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1763 | Classification: 0.1061 | Classification_binary: 0.2257 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1467 | Regression_eta: 0.0022 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1938 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0838 | Regression_eta: 0.0443 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0998 | LR: 7.39e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.0664s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.9150 | max_loss: 0.1573 | max_backward: 2.1915 | max_optimizer_step: 0.2256 | max_tensorboard_logging: 0.8426 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7551.6 | uss_MB: 6907.4 | child_rss_MB: 5063.5 | allocated_MB: 448.8 | reserved_MB: 25396.0 | inactive_split_MB: 1192.4 +rank1 [2026-10-29 02:33:21,588] INFO : Step 18200/50000 rank1 | Train Loss: 1.1403 | Train Loss Unweighted: 0.5693 | Train Loss Components: Auxiliary: 0.2140 | Cardinality: 0.1362 | Classification: 0.1099 | Classification_binary: 0.1658 | HitGrouping: 0.1698 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1444 | Regression_eta: 0.0021 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.1813 | Regression_cos_phi: 0.1529 | Regression_energy: 0.0859 | Regression_eta: 0.0784 | Regression_pt: 0.0930 | Regression_sin_phi: 0.0867 | LR: 7.36e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.5286s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0059 | max_forward: 0.7843 | max_loss: 0.1427 | max_backward: 1.4948 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 1.0509 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6418.0 | uss_MB: 5773.9 | child_rss_MB: 5038.4 | allocated_MB: 446.0 | reserved_MB: 27340.0 | inactive_split_MB: 922.1 +rank1 [2026-10-29 02:37:53,622] INFO : Step 18300/50000 rank1 | Train Loss: 1.1938 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2339 | Cardinality: 0.1526 | Classification: 0.1133 | Classification_binary: 0.1878 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1493 | Regression_eta: 0.0019 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1892 | Regression_cos_phi: 0.0966 | Regression_energy: 0.0902 | Regression_eta: 0.0549 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0668 | LR: 7.34e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9198s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.9058 | max_loss: 0.1527 | max_backward: 2.1689 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6422.1 | uss_MB: 5784.1 | child_rss_MB: 5032.3 | allocated_MB: 448.4 | reserved_MB: 27340.0 | inactive_split_MB: 1185.8 +rank1 [2026-10-29 02:42:24,375] INFO : Step 18400/50000 rank1 | Train Loss: 1.1959 | Train Loss Unweighted: 0.6109 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1130 | Classification: 0.1042 | Classification_binary: 0.2213 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1400 | Regression_eta: 0.0022 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.2090 | Regression_cos_phi: 0.1095 | Regression_energy: 0.0669 | Regression_eta: 0.0538 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0604 | LR: 7.31e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.7071s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8434 | max_loss: 0.1474 | max_backward: 1.9441 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6423.0 | uss_MB: 5783.9 | child_rss_MB: 5008.1 | allocated_MB: 448.8 | reserved_MB: 29306.0 | inactive_split_MB: 1091.4 +rank1 [2026-10-29 02:47:17,190] INFO : Step 18500/50000 rank1 | Train Loss: 1.2719 | Train Loss Unweighted: 0.6036 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1990 | Classification: 0.1087 | Classification_binary: 0.1793 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1538 | Regression_eta: 0.0024 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.1920 | Regression_cos_phi: 0.1040 | Regression_energy: 0.0700 | Regression_eta: 0.0551 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0739 | LR: 7.29e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.4217s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 1.6245 | max_loss: 0.3527 | max_backward: 3.4112 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.7932 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 6423.2 | uss_MB: 5781.7 | child_rss_MB: 5035.9 | allocated_MB: 447.9 | reserved_MB: 29306.0 | inactive_split_MB: 1193.4 +rank1 [2026-10-29 02:52:01,281] INFO : Step 18600/50000 rank1 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5476 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1351 | Classification: 0.0870 | Classification_binary: 0.1697 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1433 | Regression_eta: 0.0017 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.2117 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0726 | Regression_eta: 0.0511 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0749 | LR: 7.26e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.2400s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0527 | max_forward: 0.7639 | max_loss: 0.1373 | max_backward: 1.5273 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.7424 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6420.8 | uss_MB: 5779.1 | child_rss_MB: 5038.0 | allocated_MB: 445.6 | reserved_MB: 29306.0 | inactive_split_MB: 1194.4 +rank1 [2026-10-29 02:56:35,338] INFO : Step 18700/50000 rank1 | Train Loss: 1.2376 | Train Loss Unweighted: 0.5906 | Train Loss Components: Auxiliary: 0.2405 | Cardinality: 0.1841 | Classification: 0.1039 | Classification_binary: 0.2012 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1400 | Regression_eta: 0.0020 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.2321 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0733 | Regression_eta: 0.0633 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0853 | LR: 7.24e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7565s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8903 | max_loss: 0.1469 | max_backward: 2.1433 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4822 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1128 | Memory: rss_MB: 6419.7 | uss_MB: 5784.3 | child_rss_MB: 5013.1 | allocated_MB: 449.2 | reserved_MB: 31324.0 | inactive_split_MB: 1093.4 +rank1 [2026-10-29 03:01:09,006] INFO : Step 18800/50000 rank1 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1585 | Classification: 0.0943 | Classification_binary: 0.2038 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1342 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1850 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0592 | Regression_eta: 0.0461 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0673 | LR: 7.21e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4217s | Phase Times: max_pre_step_barrier: 0.1142 | max_batch_to_device: 0.0068 | max_forward: 0.7808 | max_loss: 0.2502 | max_backward: 1.8229 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0996 | Memory: rss_MB: 6421.2 | uss_MB: 5781.7 | child_rss_MB: 5036.2 | allocated_MB: 450.4 | reserved_MB: 31336.0 | inactive_split_MB: 1098.4 +rank1 [2026-10-29 03:05:45,768] INFO : Step 18900/50000 rank1 | Train Loss: 1.0747 | Train Loss Unweighted: 0.5385 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.1306 | Classification: 0.0887 | Classification_binary: 0.1613 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1414 | Regression_eta: 0.0017 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0698 | Regression_eta: 0.0906 | Regression_pt: 0.0802 | Regression_sin_phi: 0.0640 | LR: 7.18e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.3926s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7400 | max_loss: 0.1656 | max_backward: 1.8003 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.7596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6421.5 | uss_MB: 5781.2 | child_rss_MB: 5039.7 | allocated_MB: 443.5 | reserved_MB: 31336.0 | inactive_split_MB: 928.5 +rank1 [2026-10-29 03:10:20,956] INFO : Step 19000/50000 rank1 | Train Loss: 1.1726 | Train Loss Unweighted: 0.5966 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1418 | Classification: 0.1102 | Classification_binary: 0.1870 | HitGrouping: 0.1705 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1463 | Regression_eta: 0.0019 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1960 | Regression_cos_phi: 0.1006 | Regression_energy: 0.0853 | Regression_eta: 0.0517 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0637 | LR: 7.16e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8916s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7637 | max_loss: 0.5203 | max_backward: 1.8720 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 1.1055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6422.5 | uss_MB: 5783.8 | child_rss_MB: 5041.8 | allocated_MB: 446.6 | reserved_MB: 31336.0 | inactive_split_MB: 1097.4 +rank1 [2026-10-29 03:10:21,982] INFO : Running validation on rank1 +rank1 [2026-10-29 03:10:21,982] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 03:10:21,982] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 03:11:18,711] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:11:19,247] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 03:11:45,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 03:11:46,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 03:11:48,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 03:11:50,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 03:12:00,775] INFO : Time taken to make predictions on device 1 is: 0.28 min +rank1 [2026-10-29 03:12:01,383] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:12:01,927] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 03:12:17,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 03:12:18,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 03:12:19,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 03:12:20,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 03:12:30,858] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 03:12:31,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 03:12:31,972] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 03:12:47,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 03:12:48,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 03:12:49,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 03:12:50,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 03:13:00,512] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 03:17:58,240] INFO : Step 19100/50000 rank1 | Train Loss: 1.2150 | Train Loss Unweighted: 0.6107 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1439 | Classification: 0.1037 | Classification_binary: 0.2230 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1395 | Regression_eta: 0.0020 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1966 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0675 | Regression_eta: 0.0387 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0886 | LR: 7.13e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7830s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8630 | max_loss: 0.1532 | max_backward: 1.9622 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.7474 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7697.6 | uss_MB: 7053.9 | child_rss_MB: 5049.6 | allocated_MB: 449.4 | reserved_MB: 23972.0 | inactive_split_MB: 1371.8 +rank1 [2026-10-29 03:22:35,410] INFO : Step 19200/50000 rank1 | Train Loss: 1.1075 | Train Loss Unweighted: 0.5303 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1382 | Classification: 0.0958 | Classification_binary: 0.1758 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1270 | Regression_eta: 0.0019 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0669 | Regression_eta: 0.0390 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0561 | LR: 7.10e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.0044s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8287 | max_loss: 0.1556 | max_backward: 1.8127 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6504.0 | uss_MB: 5863.3 | child_rss_MB: 5048.7 | allocated_MB: 445.3 | reserved_MB: 27640.0 | inactive_split_MB: 940.7 +rank1 [2026-10-29 03:27:09,996] INFO : Step 19300/50000 rank1 | Train Loss: 1.0792 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2111 | Cardinality: 0.1254 | Classification: 0.0911 | Classification_binary: 0.1561 | HitGrouping: 0.1662 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1321 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0804 | Regression_eta: 0.0337 | Regression_pt: 0.0863 | Regression_sin_phi: 0.1028 | LR: 7.08e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5131s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8891 | max_loss: 0.1513 | max_backward: 1.7637 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6508.5 | uss_MB: 5869.2 | child_rss_MB: 5044.9 | allocated_MB: 445.1 | reserved_MB: 27640.0 | inactive_split_MB: 1366.9 +rank1 [2026-10-29 03:31:44,571] INFO : Step 19400/50000 rank1 | Train Loss: 1.2411 | Train Loss Unweighted: 0.6183 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1555 | Classification: 0.1055 | Classification_binary: 0.2145 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1463 | Regression_eta: 0.0020 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.2540 | Regression_cos_phi: 0.1206 | Regression_energy: 0.0824 | Regression_eta: 0.0732 | Regression_pt: 0.0900 | Regression_sin_phi: 0.0979 | LR: 7.05e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.6745s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0092 | max_forward: 0.8079 | max_loss: 0.1550 | max_backward: 1.7710 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 2.8282 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6511.0 | uss_MB: 5872.2 | child_rss_MB: 5045.1 | allocated_MB: 452.5 | reserved_MB: 28010.0 | inactive_split_MB: 1195.9 +rank1 [2026-10-29 03:36:21,472] INFO : Step 19500/50000 rank1 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6008 | Train Loss Components: Auxiliary: 0.2448 | Cardinality: 0.1836 | Classification: 0.1087 | Classification_binary: 0.2123 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1374 | Regression_eta: 0.0023 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2049 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0671 | Regression_eta: 0.1000 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0721 | LR: 7.02e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6189s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7880 | max_loss: 0.1506 | max_backward: 1.7760 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.7620 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6510.7 | uss_MB: 5870.2 | child_rss_MB: 5041.2 | allocated_MB: 449.4 | reserved_MB: 29680.0 | inactive_split_MB: 1368.8 +rank1 [2026-10-29 03:40:54,623] INFO : Step 19600/50000 rank1 | Train Loss: 1.1451 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1464 | Classification: 0.0933 | Classification_binary: 0.1736 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1533 | Regression_eta: 0.0018 | Regression_pt: 0.1557 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.1626 | Regression_cos_phi: 0.1145 | Regression_energy: 0.0778 | Regression_eta: 0.0583 | Regression_pt: 0.0847 | Regression_sin_phi: 0.1095 | LR: 7.00e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.3528s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8082 | max_loss: 0.1437 | max_backward: 1.6649 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.6567 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6510.9 | uss_MB: 5869.4 | child_rss_MB: 5045.0 | allocated_MB: 444.5 | reserved_MB: 29680.0 | inactive_split_MB: 1016.8 +rank1 [2026-10-29 03:45:30,310] INFO : Step 19700/50000 rank1 | Train Loss: 1.1424 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2236 | Cardinality: 0.1385 | Classification: 0.1093 | Classification_binary: 0.1797 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1379 | Regression_eta: 0.0020 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1887 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0681 | Regression_eta: 0.0515 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0765 | LR: 6.97e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.4892s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7612 | max_loss: 0.2297 | max_backward: 1.9668 | max_optimizer_step: 0.5229 | max_tensorboard_logging: 0.5756 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6508.5 | uss_MB: 5869.4 | child_rss_MB: 5047.4 | allocated_MB: 449.1 | reserved_MB: 29680.0 | inactive_split_MB: 1100.9 +rank1 [2026-10-29 03:50:08,779] INFO : Step 19800/50000 rank1 | Train Loss: 1.2806 | Train Loss Unweighted: 0.6390 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1432 | Classification: 0.1110 | Classification_binary: 0.2282 | HitGrouping: 0.1668 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0030 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2531 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0705 | Regression_eta: 0.0834 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0783 | LR: 6.94e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5054s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8034 | max_loss: 0.1812 | max_backward: 1.9502 | max_optimizer_step: 0.2119 | max_tensorboard_logging: 0.5857 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6510.7 | uss_MB: 5873.6 | child_rss_MB: 5045.7 | allocated_MB: 452.3 | reserved_MB: 29680.0 | inactive_split_MB: 1110.9 +rank1 [2026-10-29 03:54:46,663] INFO : Step 19900/50000 rank1 | Train Loss: 1.0692 | Train Loss Unweighted: 0.5407 | Train Loss Components: Auxiliary: 0.2077 | Cardinality: 0.1171 | Classification: 0.0922 | Classification_binary: 0.1812 | HitGrouping: 0.1610 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1630 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0700 | Regression_eta: 0.0502 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0870 | LR: 6.92e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6132s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7533 | max_loss: 0.2412 | max_backward: 2.0720 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5848 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6505.8 | uss_MB: 5870.0 | child_rss_MB: 5057.5 | allocated_MB: 444.7 | reserved_MB: 29682.0 | inactive_split_MB: 1000.0 +rank1 [2026-10-29 03:59:25,167] INFO : Step 20000/50000 rank1 | Train Loss: 1.0304 | Train Loss Unweighted: 0.5275 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1060 | Classification: 0.0940 | Classification_binary: 0.1604 | HitGrouping: 0.1656 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1344 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1312 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0500 | Regression_eta: 0.0479 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0601 | LR: 6.89e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.8360s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.8721 | max_loss: 0.1507 | max_backward: 2.1861 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6505.7 | uss_MB: 5870.7 | child_rss_MB: 5049.5 | allocated_MB: 445.9 | reserved_MB: 30966.0 | inactive_split_MB: 1179.2 +rank1 [2026-10-29 03:59:26,238] INFO : Running validation on rank1 +rank1 [2026-10-29 03:59:26,239] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 03:59:26,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 04:00:19,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:00:19,534] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 04:00:44,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 04:00:46,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 04:00:47,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 04:00:48,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 04:00:58,398] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 04:00:59,044] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:00:59,501] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 04:01:15,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 04:01:16,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 04:01:17,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 04:01:18,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 04:01:28,102] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:01:28,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:01:29,374] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 04:01:44,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 04:01:45,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 04:01:46,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 04:01:47,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 04:01:57,509] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 04:06:53,362] INFO : Step 20100/50000 rank1 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2437 | Cardinality: 0.1586 | Classification: 0.1055 | Classification_binary: 0.2238 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1360 | Regression_eta: 0.0021 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.2286 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0755 | Regression_eta: 0.0612 | Regression_pt: 0.0864 | Regression_sin_phi: 0.1030 | LR: 6.86e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.6040s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0372 | max_forward: 0.8293 | max_loss: 0.1584 | max_backward: 1.7738 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.7139 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7674.6 | uss_MB: 7034.7 | child_rss_MB: 5034.6 | allocated_MB: 448.4 | reserved_MB: 29328.0 | inactive_split_MB: 969.2 +rank1 [2026-10-29 04:11:25,835] INFO : Step 20200/50000 rank1 | Train Loss: 1.3163 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.1695 | Classification: 0.1000 | Classification_binary: 0.2250 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1433 | Regression_eta: 0.0029 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.2040 | Regression_cos_phi: 0.1138 | Regression_energy: 0.0690 | Regression_eta: 0.0626 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0791 | LR: 6.84e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6409s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8823 | max_loss: 0.1595 | max_backward: 2.1030 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5136 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7674.6 | uss_MB: 7036.8 | child_rss_MB: 5027.8 | allocated_MB: 449.6 | reserved_MB: 29340.0 | inactive_split_MB: 938.7 +rank1 [2026-10-29 04:16:03,707] INFO : Step 20300/50000 rank1 | Train Loss: 1.1420 | Train Loss Unweighted: 0.5734 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1487 | Classification: 0.1087 | Classification_binary: 0.1713 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1438 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.1557 | Regression_cos_phi: 0.0982 | Regression_energy: 0.0736 | Regression_eta: 0.0541 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0729 | LR: 6.81e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9958s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.9650 | max_loss: 0.1904 | max_backward: 2.0801 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.8679 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6555.4 | uss_MB: 5913.2 | child_rss_MB: 5021.5 | allocated_MB: 444.8 | reserved_MB: 29344.0 | inactive_split_MB: 986.2 +rank1 [2026-10-29 04:20:44,976] INFO : Step 20400/50000 rank1 | Train Loss: 1.1926 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1642 | Classification: 0.1091 | Classification_binary: 0.1922 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1329 | Regression_eta: 0.0018 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0829 | Classification_binary: 0.1875 | Regression_cos_phi: 0.1441 | Regression_energy: 0.0573 | Regression_eta: 0.0712 | Regression_pt: 0.0634 | Regression_sin_phi: 0.1158 | LR: 6.78e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.2524s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0167 | max_forward: 1.2639 | max_loss: 0.1873 | max_backward: 2.4261 | max_optimizer_step: 0.3105 | max_tensorboard_logging: 0.7562 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6556.4 | uss_MB: 5916.7 | child_rss_MB: 5031.4 | allocated_MB: 447.7 | reserved_MB: 29344.0 | inactive_split_MB: 985.2 +rank1 [2026-10-29 04:25:27,116] INFO : Step 20500/50000 rank1 | Train Loss: 1.2110 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1591 | Classification: 0.1022 | Classification_binary: 0.2117 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1333 | Regression_eta: 0.0017 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0754 | Classification_binary: 0.2313 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0820 | Regression_eta: 0.0654 | Regression_pt: 0.0898 | Regression_sin_phi: 0.1325 | LR: 6.76e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5477s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7705 | max_loss: 0.1525 | max_backward: 1.8060 | max_optimizer_step: 0.2622 | max_tensorboard_logging: 0.7595 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6553.9 | uss_MB: 5915.2 | child_rss_MB: 5013.6 | allocated_MB: 450.6 | reserved_MB: 29344.0 | inactive_split_MB: 1123.4 +rank1 [2026-10-29 04:30:07,865] INFO : Step 20600/50000 rank1 | Train Loss: 1.1161 | Train Loss Unweighted: 0.5564 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1436 | Classification: 0.0978 | Classification_binary: 0.1795 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1793 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0644 | Regression_eta: 0.0372 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0495 | LR: 6.73e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4374s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8358 | max_loss: 0.5148 | max_backward: 1.8393 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6556.9 | uss_MB: 5916.8 | child_rss_MB: 5024.7 | allocated_MB: 445.4 | reserved_MB: 31380.0 | inactive_split_MB: 917.0 +rank1 [2026-10-29 04:34:51,728] INFO : Step 20700/50000 rank1 | Train Loss: 1.0740 | Train Loss Unweighted: 0.5459 | Train Loss Components: Auxiliary: 0.2086 | Cardinality: 0.1252 | Classification: 0.0962 | Classification_binary: 0.1560 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.1641 | Regression_cos_phi: 0.1263 | Regression_energy: 0.0674 | Regression_eta: 0.0513 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0989 | LR: 6.70e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0700s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7111 | max_loss: 0.2396 | max_backward: 1.6829 | max_optimizer_step: 0.2121 | max_tensorboard_logging: 0.4996 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6553.0 | uss_MB: 5912.9 | child_rss_MB: 5017.8 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1185.7 +rank1 [2026-10-29 04:39:29,326] INFO : Step 20800/50000 rank1 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5426 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1263 | Classification: 0.0965 | Classification_binary: 0.1999 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1207 | Regression_eta: 0.0017 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.2307 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0641 | Regression_eta: 0.0588 | Regression_pt: 0.0710 | Regression_sin_phi: 0.1117 | LR: 6.67e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7621s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.9079 | max_loss: 0.1550 | max_backward: 2.1083 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.5897 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6553.9 | uss_MB: 5918.9 | child_rss_MB: 5012.5 | allocated_MB: 448.2 | reserved_MB: 31380.0 | inactive_split_MB: 1157.8 +rank1 [2026-10-29 04:44:09,835] INFO : Step 20900/50000 rank1 | Train Loss: 1.2188 | Train Loss Unweighted: 0.5970 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1525 | Classification: 0.1062 | Classification_binary: 0.2022 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0022 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1709 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0892 | Regression_eta: 0.0371 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0857 | LR: 6.65e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7883s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0138 | max_forward: 0.8922 | max_loss: 0.1535 | max_backward: 2.0483 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6550.7 | uss_MB: 5916.4 | child_rss_MB: 5027.7 | allocated_MB: 447.8 | reserved_MB: 31380.0 | inactive_split_MB: 1126.3 +rank1 [2026-10-29 04:48:48,418] INFO : Step 21000/50000 rank1 | Train Loss: 1.0503 | Train Loss Unweighted: 0.5338 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1267 | Classification: 0.0900 | Classification_binary: 0.1573 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1405 | Regression_eta: 0.0016 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0569 | Regression_eta: 0.0480 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0617 | LR: 6.62e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8536s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7596 | max_loss: 0.2536 | max_backward: 1.8713 | max_optimizer_step: 0.3177 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6551.7 | uss_MB: 5915.9 | child_rss_MB: 5032.5 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1148.0 +rank1 [2026-10-29 04:48:49,284] INFO : Running validation on rank1 +rank1 [2026-10-29 04:48:49,284] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 04:48:49,284] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 04:49:44,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:49:44,825] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 04:50:11,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 04:50:12,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 04:50:13,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 04:50:14,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 04:50:24,631] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:50:25,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:50:25,767] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 04:50:41,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 04:50:42,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 04:50:43,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 04:50:44,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 04:50:54,658] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 04:50:55,260] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 04:50:55,905] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 04:51:11,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 04:51:12,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 04:51:13,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 04:51:14,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 04:51:24,078] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 04:56:25,805] INFO : Step 21100/50000 rank1 | Train Loss: 1.1646 | Train Loss Unweighted: 0.5785 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1401 | Classification: 0.1095 | Classification_binary: 0.1927 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1344 | Regression_eta: 0.0022 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.1659 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0745 | Regression_eta: 0.0446 | Regression_pt: 0.0820 | Regression_sin_phi: 0.0672 | LR: 6.59e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.4042s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.9258 | max_loss: 0.1506 | max_backward: 2.4347 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.8145 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7847.0 | uss_MB: 5890.3 | child_rss_MB: 5018.3 | allocated_MB: 448.7 | reserved_MB: 23438.0 | inactive_split_MB: 911.7 +rank1 [2026-10-29 04:56:59,983] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank1 [2026-10-29 04:56:59,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-29 05:01:22,612] INFO : Step 21200/50000 rank1 | Train Loss: 1.1437 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1413 | Classification: 0.0985 | Classification_binary: 0.1768 | HitGrouping: 0.1669 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0020 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1701 | Regression_cos_phi: 0.1050 | Regression_energy: 0.0913 | Regression_eta: 0.0688 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0851 | LR: 6.56e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5640s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7993 | max_loss: 0.1450 | max_backward: 1.7427 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7842.9 | uss_MB: 7208.8 | child_rss_MB: 5012.9 | allocated_MB: 446.9 | reserved_MB: 23590.0 | inactive_split_MB: 1086.9 +rank1 [2026-10-29 05:06:02,214] INFO : Step 21300/50000 rank1 | Train Loss: 1.1586 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1648 | Classification: 0.1118 | Classification_binary: 0.1760 | HitGrouping: 0.1681 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1392 | Regression_eta: 0.0021 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0712 | Regression_eta: 0.0367 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0730 | LR: 6.54e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.2672s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0558 | max_forward: 0.9498 | max_loss: 0.1699 | max_backward: 2.4288 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.7788 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1270 | Memory: rss_MB: 6605.4 | uss_MB: 5978.0 | child_rss_MB: 5012.8 | allocated_MB: 447.4 | reserved_MB: 25440.0 | inactive_split_MB: 1003.9 +rank1 [2026-10-29 05:10:41,727] INFO : Step 21400/50000 rank1 | Train Loss: 1.1843 | Train Loss Unweighted: 0.5823 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1516 | Classification: 0.1038 | Classification_binary: 0.2065 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1335 | Regression_eta: 0.0020 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2341 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0576 | Regression_eta: 0.0587 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0823 | LR: 6.51e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.0992s | Phase Times: max_pre_step_barrier: 0.0085 | max_batch_to_device: 0.0640 | max_forward: 0.9920 | max_loss: 0.5036 | max_backward: 2.0711 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.8200 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6606.3 | uss_MB: 5978.8 | child_rss_MB: 5018.5 | allocated_MB: 448.9 | reserved_MB: 25440.0 | inactive_split_MB: 994.4 +rank1 [2026-10-29 05:15:15,087] INFO : Step 21500/50000 rank1 | Train Loss: 1.1484 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1478 | Classification: 0.0953 | Classification_binary: 0.2062 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1299 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2150 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0822 | Regression_eta: 0.0673 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0831 | LR: 6.48e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.9376s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8371 | max_loss: 0.1537 | max_backward: 2.1150 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.8633 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6603.8 | uss_MB: 5979.6 | child_rss_MB: 5017.0 | allocated_MB: 447.2 | reserved_MB: 27056.0 | inactive_split_MB: 1468.9 +rank1 [2026-10-29 05:19:51,738] INFO : Step 21600/50000 rank1 | Train Loss: 1.0171 | Train Loss Unweighted: 0.5098 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1167 | Classification: 0.0973 | Classification_binary: 0.1559 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1265 | Regression_eta: 0.0014 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0788 | Regression_eta: 0.0396 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0851 | LR: 6.45e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5912s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0062 | max_forward: 0.7285 | max_loss: 0.1940 | max_backward: 1.8148 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.8819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6602.2 | uss_MB: 5978.2 | child_rss_MB: 5014.0 | allocated_MB: 445.7 | reserved_MB: 27056.0 | inactive_split_MB: 1196.5 +rank1 [2026-10-29 05:24:30,477] INFO : Step 21700/50000 rank1 | Train Loss: 1.1623 | Train Loss Unweighted: 0.5535 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1498 | Classification: 0.1025 | Classification_binary: 0.1819 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.2081 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0711 | Regression_eta: 0.0666 | Regression_pt: 0.0797 | Regression_sin_phi: 0.1065 | LR: 6.43e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4951s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7947 | max_loss: 0.1379 | max_backward: 1.9808 | max_optimizer_step: 0.2157 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6600.2 | uss_MB: 5978.2 | child_rss_MB: 5010.8 | allocated_MB: 448.2 | reserved_MB: 27056.0 | inactive_split_MB: 1045.2 +rank1 [2026-10-29 05:29:08,470] INFO : Step 21800/50000 rank1 | Train Loss: 1.0898 | Train Loss Unweighted: 0.5518 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1233 | Classification: 0.0977 | Classification_binary: 0.1967 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1269 | Regression_eta: 0.0015 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.2048 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0636 | Regression_eta: 0.0477 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0710 | LR: 6.40e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.7217s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8952 | max_loss: 0.1765 | max_backward: 2.1628 | max_optimizer_step: 0.2138 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6602.0 | uss_MB: 5872.4 | child_rss_MB: 5012.0 | allocated_MB: 448.4 | reserved_MB: 27056.0 | inactive_split_MB: 1001.4 +rank1 [2026-10-29 05:33:46,455] INFO : Step 21900/50000 rank1 | Train Loss: 1.0699 | Train Loss Unweighted: 0.5306 | Train Loss Components: Auxiliary: 0.2121 | Cardinality: 0.1436 | Classification: 0.0991 | Classification_binary: 0.1704 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1276 | Regression_eta: 0.0015 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0733 | Regression_eta: 0.0459 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0517 | LR: 6.37e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4511s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7813 | max_loss: 0.1495 | max_backward: 1.8703 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 6600.0 | uss_MB: 5976.9 | child_rss_MB: 5013.0 | allocated_MB: 447.4 | reserved_MB: 27056.0 | inactive_split_MB: 1002.4 +rank1 [2026-10-29 05:38:23,933] INFO : Step 22000/50000 rank1 | Train Loss: 1.0659 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2100 | Cardinality: 0.1294 | Classification: 0.0955 | Classification_binary: 0.1521 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1360 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1827 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0614 | Regression_eta: 0.0500 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0904 | LR: 6.34e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4497s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7855 | max_loss: 0.1383 | max_backward: 1.9648 | max_optimizer_step: 0.2132 | max_tensorboard_logging: 0.5884 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6605.4 | uss_MB: 5979.9 | child_rss_MB: 5015.2 | allocated_MB: 449.1 | reserved_MB: 27056.0 | inactive_split_MB: 1180.9 +rank1 [2026-10-29 05:38:24,834] INFO : Running validation on rank1 +rank1 [2026-10-29 05:38:24,834] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 05:38:24,834] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 05:39:16,672] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:39:17,216] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 05:39:42,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 05:39:43,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 05:39:45,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 05:39:46,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 05:39:56,046] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 05:39:56,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:39:57,074] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 05:40:12,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 05:40:14,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 05:40:15,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 05:40:16,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 05:40:26,576] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 05:40:27,478] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 05:40:27,967] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 05:40:43,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 05:40:44,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 05:40:45,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 05:40:46,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 05:40:56,375] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 05:45:56,687] INFO : Step 22100/50000 rank1 | Train Loss: 1.1235 | Train Loss Unweighted: 0.5530 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1528 | Classification: 0.0997 | Classification_binary: 0.1905 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1304 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1796 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0695 | Regression_eta: 0.0247 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0441 | LR: 6.32e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5239s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8135 | max_loss: 0.1488 | max_backward: 1.7877 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5979 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7900.1 | uss_MB: 7271.0 | child_rss_MB: 5019.5 | allocated_MB: 451.8 | reserved_MB: 29136.0 | inactive_split_MB: 1136.2 +rank1 [2026-10-29 05:50:45,343] INFO : Step 22200/50000 rank1 | Train Loss: 1.1922 | Train Loss Unweighted: 0.5684 | Train Loss Components: Auxiliary: 0.2361 | Cardinality: 0.1699 | Classification: 0.0906 | Classification_binary: 0.2085 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.2206 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0637 | Regression_eta: 0.0478 | Regression_pt: 0.0684 | Regression_sin_phi: 0.1026 | LR: 6.29e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5702s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8616 | max_loss: 0.1545 | max_backward: 1.6969 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 0.7120 | max_tensorboard_flush: 0.0150 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7899.1 | uss_MB: 7269.8 | child_rss_MB: 5025.9 | allocated_MB: 448.1 | reserved_MB: 29136.0 | inactive_split_MB: 1371.0 +rank1 [2026-10-29 05:55:21,940] INFO : Step 22300/50000 rank1 | Train Loss: 1.0221 | Train Loss Unweighted: 0.5006 | Train Loss Components: Auxiliary: 0.1973 | Cardinality: 0.1169 | Classification: 0.0876 | Classification_binary: 0.1576 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1252 | Regression_eta: 0.0021 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0713 | Regression_eta: 0.0405 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0568 | LR: 6.26e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5029s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7903 | max_loss: 0.1446 | max_backward: 1.7078 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 0.8083 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6690.1 | uss_MB: 6062.0 | child_rss_MB: 5023.7 | allocated_MB: 446.4 | reserved_MB: 29136.0 | inactive_split_MB: 1159.3 +rank1 [2026-10-29 06:00:08,368] INFO : Step 22400/50000 rank1 | Train Loss: 1.1170 | Train Loss Unweighted: 0.5609 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1391 | Classification: 0.1065 | Classification_binary: 0.1807 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1348 | Regression_eta: 0.0019 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.1968 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0676 | Regression_eta: 0.0516 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0660 | LR: 6.23e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9535s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7658 | max_loss: 0.2479 | max_backward: 1.9808 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9928 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6691.0 | uss_MB: 6065.0 | child_rss_MB: 5027.0 | allocated_MB: 447.1 | reserved_MB: 31152.0 | inactive_split_MB: 925.7 +rank1 [2026-10-29 06:04:41,559] INFO : Step 22500/50000 rank1 | Train Loss: 1.1245 | Train Loss Unweighted: 0.5482 | Train Loss Components: Auxiliary: 0.2231 | Cardinality: 0.1463 | Classification: 0.0856 | Classification_binary: 0.2083 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1244 | Regression_eta: 0.0016 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.2141 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0671 | Regression_eta: 0.0453 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0777 | LR: 6.20e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0089s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 1.1916 | max_loss: 0.1613 | max_backward: 2.1225 | max_optimizer_step: 0.2176 | max_tensorboard_logging: 0.7664 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6685.2 | uss_MB: 6063.6 | child_rss_MB: 5003.5 | allocated_MB: 447.4 | reserved_MB: 31152.0 | inactive_split_MB: 959.7 +rank1 [2026-10-29 06:09:13,550] INFO : Step 22600/50000 rank1 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5177 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1260 | Classification: 0.0928 | Classification_binary: 0.1789 | HitGrouping: 0.1619 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1208 | Regression_eta: 0.0019 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0518 | Regression_eta: 0.0869 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0632 | LR: 6.18e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.8015s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0651 | max_forward: 1.1064 | max_loss: 0.1531 | max_backward: 1.9236 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.8515 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6676.3 | uss_MB: 6063.0 | child_rss_MB: 5024.2 | allocated_MB: 448.3 | reserved_MB: 31154.0 | inactive_split_MB: 961.5 +rank1 [2026-10-29 06:13:51,719] INFO : Step 22700/50000 rank1 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5090 | Train Loss Components: Auxiliary: 0.1959 | Cardinality: 0.1101 | Classification: 0.0877 | Classification_binary: 0.1479 | HitGrouping: 0.1634 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1339 | Regression_eta: 0.0015 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1415 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0707 | Regression_eta: 0.0365 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0504 | LR: 6.15e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.4596s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 1.0743 | max_loss: 0.1383 | max_backward: 1.7235 | max_optimizer_step: 0.2204 | max_tensorboard_logging: 0.5170 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 6672.9 | uss_MB: 6061.8 | child_rss_MB: 5030.8 | allocated_MB: 446.5 | reserved_MB: 31154.0 | inactive_split_MB: 972.7 +rank1 [2026-10-29 06:18:32,627] INFO : Step 22800/50000 rank1 | Train Loss: 1.0730 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2215 | Cardinality: 0.1332 | Classification: 0.1095 | Classification_binary: 0.1782 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1289 | Regression_eta: 0.0014 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0890 | Classification_binary: 0.2094 | Regression_cos_phi: 0.0865 | Regression_energy: 0.0662 | Regression_eta: 0.0408 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0807 | LR: 6.12e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.7941s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.2500 | max_backward: 2.0107 | max_optimizer_step: 0.2761 | max_tensorboard_logging: 0.6168 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 6673.9 | uss_MB: 6062.6 | child_rss_MB: 5029.6 | allocated_MB: 447.2 | reserved_MB: 31154.0 | inactive_split_MB: 922.1 +rank1 [2026-10-29 06:23:13,203] INFO : Step 22900/50000 rank1 | Train Loss: 1.1830 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2389 | Cardinality: 0.1788 | Classification: 0.0948 | Classification_binary: 0.2023 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1393 | Regression_eta: 0.0018 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0864 | Regression_eta: 0.0368 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0546 | LR: 6.09e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.9920s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0073 | max_forward: 0.7992 | max_loss: 0.1478 | max_backward: 1.9051 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.9858 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 6664.2 | uss_MB: 6055.9 | child_rss_MB: 5028.8 | allocated_MB: 448.8 | reserved_MB: 31154.0 | inactive_split_MB: 1109.2 +rank1 [2026-10-29 06:27:51,864] INFO : Step 23000/50000 rank1 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5014 | Train Loss Components: Auxiliary: 0.1951 | Cardinality: 0.1184 | Classification: 0.0781 | Classification_binary: 0.1455 | HitGrouping: 0.1653 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1219 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0784 | Regression_eta: 0.0504 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0541 | LR: 6.06e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6101s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7528 | max_loss: 0.2292 | max_backward: 1.7818 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.8884 | max_tensorboard_flush: 0.0060 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6672.0 | uss_MB: 6063.2 | child_rss_MB: 5028.5 | allocated_MB: 443.9 | reserved_MB: 31154.0 | inactive_split_MB: 985.1 +rank1 [2026-10-29 06:27:52,750] INFO : Running validation on rank1 +rank1 [2026-10-29 06:27:52,750] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 06:27:52,750] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 06:28:45,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:28:45,726] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 06:29:12,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 06:29:13,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 06:29:14,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 06:29:15,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 06:29:25,269] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 06:29:26,162] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:29:26,631] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 06:29:42,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 06:29:43,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 06:29:44,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 06:29:45,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 06:29:55,299] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 06:29:56,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 06:29:56,645] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 06:30:12,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 06:30:13,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 06:30:14,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 06:30:14,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 06:30:24,942] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 06:35:29,455] INFO : Step 23100/50000 rank1 | Train Loss: 1.1240 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1320 | Classification: 0.1122 | Classification_binary: 0.1745 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1274 | Regression_eta: 0.0021 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0848 | Classification_binary: 0.1852 | Regression_cos_phi: 0.1252 | Regression_energy: 0.0592 | Regression_eta: 0.0938 | Regression_pt: 0.0675 | Regression_sin_phi: 0.1314 | LR: 6.04e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8781s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7914 | max_loss: 0.1430 | max_backward: 1.7255 | max_optimizer_step: 0.3295 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0695 | Memory: rss_MB: 7843.1 | uss_MB: 7233.5 | child_rss_MB: 5030.9 | allocated_MB: 449.4 | reserved_MB: 25680.0 | inactive_split_MB: 1046.0 +rank1 [2026-10-29 06:40:11,955] INFO : Step 23200/50000 rank1 | Train Loss: 1.1736 | Train Loss Unweighted: 0.5745 | Train Loss Components: Auxiliary: 0.2341 | Cardinality: 0.1586 | Classification: 0.1057 | Classification_binary: 0.2033 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1304 | Regression_eta: 0.0018 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.2128 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0617 | Regression_eta: 0.0563 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0908 | LR: 6.01e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.1112s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9381 | max_loss: 0.1692 | max_backward: 2.2545 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6161 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 7846.1 | uss_MB: 5861.1 | child_rss_MB: 5025.2 | allocated_MB: 447.8 | reserved_MB: 25680.0 | inactive_split_MB: 1169.7 +rank1 [2026-10-29 06:44:52,827] INFO : Step 23300/50000 rank1 | Train Loss: 1.0647 | Train Loss Unweighted: 0.5262 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1268 | Classification: 0.0958 | Classification_binary: 0.1665 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1301 | Regression_eta: 0.0017 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0610 | Regression_eta: 0.0557 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0611 | LR: 5.98e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6088s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8613 | max_loss: 0.1501 | max_backward: 2.0035 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 0.6274 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6729.0 | uss_MB: 6118.5 | child_rss_MB: 5031.7 | allocated_MB: 447.3 | reserved_MB: 29702.0 | inactive_split_MB: 1252.4 +rank1 [2026-10-29 06:49:32,134] INFO : Step 23400/50000 rank1 | Train Loss: 0.9916 | Train Loss Unweighted: 0.5023 | Train Loss Components: Auxiliary: 0.1957 | Cardinality: 0.1164 | Classification: 0.1006 | Classification_binary: 0.1374 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0837 | Regression_energy: 0.0819 | Regression_eta: 0.0590 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0869 | LR: 5.95e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.3963s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7229 | max_loss: 0.2412 | max_backward: 1.6798 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.4992 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1172 | Memory: rss_MB: 6727.8 | uss_MB: 6117.0 | child_rss_MB: 5031.9 | allocated_MB: 445.7 | reserved_MB: 29702.0 | inactive_split_MB: 1158.3 +rank1 [2026-10-29 06:54:42,355] INFO : Step 23500/50000 rank1 | Train Loss: 1.0349 | Train Loss Unweighted: 0.5234 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1083 | Classification: 0.0851 | Classification_binary: 0.1870 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1235 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1723 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0591 | Regression_eta: 0.0377 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0680 | LR: 5.92e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7755s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7924 | max_loss: 0.2020 | max_backward: 1.9787 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.7987 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6728.4 | uss_MB: 6111.2 | child_rss_MB: 5026.6 | allocated_MB: 446.9 | reserved_MB: 31774.0 | inactive_split_MB: 1042.0 +rank1 [2026-10-29 06:59:38,279] INFO : Step 23600/50000 rank1 | Train Loss: 1.1326 | Train Loss Unweighted: 0.5418 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1458 | Classification: 0.0933 | Classification_binary: 0.1900 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1267 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0911 | Regression_energy: 0.0651 | Regression_eta: 0.0614 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0980 | LR: 5.90e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9832s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0063 | max_forward: 0.7946 | max_loss: 0.1855 | max_backward: 1.9114 | max_optimizer_step: 0.2664 | max_tensorboard_logging: 0.9547 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6730.1 | uss_MB: 6120.3 | child_rss_MB: 5028.3 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1333.8 +rank1 [2026-10-29 07:04:20,062] INFO : Step 23700/50000 rank1 | Train Loss: 1.0167 | Train Loss Unweighted: 0.4968 | Train Loss Components: Auxiliary: 0.1932 | Cardinality: 0.1158 | Classification: 0.0853 | Classification_binary: 0.1475 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1285 | Regression_eta: 0.0017 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0664 | Regression_eta: 0.0419 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0412 | LR: 5.87e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.7139s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0221 | max_forward: 0.7952 | max_loss: 0.1461 | max_backward: 1.8308 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7718 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6725.6 | uss_MB: 6118.0 | child_rss_MB: 5028.2 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1160.4 +rank1 [2026-10-29 07:08:57,915] INFO : Step 23800/50000 rank1 | Train Loss: 1.0651 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1325 | Classification: 0.0886 | Classification_binary: 0.1616 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1318 | Regression_eta: 0.0015 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1699 | Regression_cos_phi: 0.1108 | Regression_energy: 0.0820 | Regression_eta: 0.0412 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0740 | LR: 5.84e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6711s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8562 | max_loss: 0.1426 | max_backward: 1.8700 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6730.2 | uss_MB: 6120.2 | child_rss_MB: 5026.7 | allocated_MB: 449.4 | reserved_MB: 31776.0 | inactive_split_MB: 1046.7 +rank1 [2026-10-29 07:13:41,572] INFO : Step 23900/50000 rank1 | Train Loss: 1.1515 | Train Loss Unweighted: 0.5718 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1646 | Classification: 0.0962 | Classification_binary: 0.2086 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2347 | Regression_cos_phi: 0.1034 | Regression_energy: 0.0853 | Regression_eta: 0.0584 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0735 | LR: 5.81e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.8383s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.8887 | max_loss: 0.2019 | max_backward: 2.1185 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6224 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6714.4 | uss_MB: 6108.3 | child_rss_MB: 5032.9 | allocated_MB: 447.1 | reserved_MB: 31776.0 | inactive_split_MB: 1168.8 +rank1 [2026-10-29 07:18:20,709] INFO : Step 24000/50000 rank1 | Train Loss: 1.0174 | Train Loss Unweighted: 0.5187 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1133 | Classification: 0.0873 | Classification_binary: 0.1674 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0014 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1354 | Regression_cos_phi: 0.0397 | Regression_energy: 0.0561 | Regression_eta: 0.0257 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0509 | LR: 5.78e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.6299s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 1.3237 | max_loss: 0.2403 | max_backward: 1.8319 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5068 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6721.2 | uss_MB: 6114.9 | child_rss_MB: 5029.4 | allocated_MB: 446.4 | reserved_MB: 31776.0 | inactive_split_MB: 957.1 +rank1 [2026-10-29 07:18:21,623] INFO : Running validation on rank1 +rank1 [2026-10-29 07:18:21,623] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 07:19:13,460] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:19:13,969] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 07:19:40,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 07:19:42,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 07:19:43,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 07:19:44,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 07:19:54,341] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 07:19:55,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:19:55,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 07:20:11,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 07:20:12,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 07:20:13,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 07:20:14,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 07:20:24,618] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 07:20:25,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 07:20:25,608] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 07:20:41,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 07:20:42,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 07:20:43,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 07:20:43,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 07:20:53,934] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 07:26:06,615] INFO : Step 24100/50000 rank1 | Train Loss: 1.0669 | Train Loss Unweighted: 0.5331 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1324 | Classification: 0.0928 | Classification_binary: 0.1578 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1390 | Regression_eta: 0.0017 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.1880 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0793 | Regression_eta: 0.0424 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0765 | LR: 5.75e-04 | DataLoad Time: 0.0182s | Model Forward Time: 7.5186s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.7836 | max_loss: 0.1442 | max_backward: 1.8747 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 3.6030 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0626 | Memory: rss_MB: 7857.5 | uss_MB: 7243.1 | child_rss_MB: 5051.0 | allocated_MB: 448.1 | reserved_MB: 24490.0 | inactive_split_MB: 1198.5 +rank1 [2026-10-29 07:30:59,926] INFO : Step 24200/50000 rank1 | Train Loss: 1.1691 | Train Loss Unweighted: 0.5645 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1546 | Classification: 0.1031 | Classification_binary: 0.1915 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1325 | Regression_eta: 0.0019 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1796 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0623 | Regression_eta: 0.0347 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0654 | LR: 5.73e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.2340s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.9207 | max_loss: 0.1533 | max_backward: 2.2288 | max_optimizer_step: 0.2436 | max_tensorboard_logging: 0.8522 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7858.2 | uss_MB: 7247.2 | child_rss_MB: 5039.0 | allocated_MB: 450.2 | reserved_MB: 26490.0 | inactive_split_MB: 976.9 +rank1 [2026-10-29 07:35:42,203] INFO : Step 24300/50000 rank1 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5336 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1380 | Classification: 0.0927 | Classification_binary: 0.1863 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1247 | Regression_eta: 0.0020 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1892 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0730 | Regression_eta: 0.0510 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0396 | LR: 5.70e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6721s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8648 | max_loss: 0.1481 | max_backward: 1.7152 | max_optimizer_step: 0.2781 | max_tensorboard_logging: 0.7831 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6810.0 | uss_MB: 6199.1 | child_rss_MB: 5043.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 983.3 +rank1 [2026-10-29 07:40:19,152] INFO : Step 24400/50000 rank1 | Train Loss: 0.9707 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0879 | Classification_binary: 0.1454 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0014 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0638 | Regression_eta: 0.0485 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0493 | LR: 5.67e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.8582s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7628 | max_loss: 0.1940 | max_backward: 2.4253 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.9481 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6811.6 | uss_MB: 5803.5 | child_rss_MB: 5042.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 1129.3 +rank1 [2026-10-29 07:45:00,050] INFO : Step 24500/50000 rank1 | Train Loss: 1.0937 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1319 | Classification: 0.0941 | Classification_binary: 0.1676 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1334 | Regression_eta: 0.0016 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1630 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0694 | Regression_eta: 0.0359 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0632 | LR: 5.64e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4286s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7565 | max_loss: 0.5105 | max_backward: 1.8057 | max_optimizer_step: 0.2534 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6807.0 | uss_MB: 6198.6 | child_rss_MB: 5045.1 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 945.2 +rank1 [2026-10-29 07:49:44,236] INFO : Step 24600/50000 rank1 | Train Loss: 1.0924 | Train Loss Unweighted: 0.5511 | Train Loss Components: Auxiliary: 0.2237 | Cardinality: 0.1233 | Classification: 0.0963 | Classification_binary: 0.1991 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1255 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1934 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0574 | Regression_eta: 0.0451 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0568 | LR: 5.61e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6483s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.9060 | max_loss: 0.1523 | max_backward: 2.0894 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.5102 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6811.1 | uss_MB: 6200.6 | child_rss_MB: 5046.3 | allocated_MB: 449.5 | reserved_MB: 26492.0 | inactive_split_MB: 943.2 +rank1 [2026-10-29 07:54:29,278] INFO : Step 24700/50000 rank1 | Train Loss: 0.9972 | Train Loss Unweighted: 0.4984 | Train Loss Components: Auxiliary: 0.1950 | Cardinality: 0.1160 | Classification: 0.0782 | Classification_binary: 0.1692 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1109 | Regression_cos_phi: 0.0441 | Regression_energy: 0.0692 | Regression_eta: 0.0372 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0519 | LR: 5.58e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5116s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0060 | max_forward: 0.7847 | max_loss: 0.1519 | max_backward: 1.7159 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.8325 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6813.0 | uss_MB: 6199.7 | child_rss_MB: 5050.6 | allocated_MB: 447.9 | reserved_MB: 27708.0 | inactive_split_MB: 1169.6 +rank1 [2026-10-29 07:59:09,067] INFO : Step 24800/50000 rank1 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4722 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1202 | Classification: 0.0808 | Classification_binary: 0.1484 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0016 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1830 | Regression_cos_phi: 0.0966 | Regression_energy: 0.0712 | Regression_eta: 0.0685 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0994 | LR: 5.56e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7182s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0127 | max_forward: 0.7903 | max_loss: 0.1427 | max_backward: 1.7798 | max_optimizer_step: 0.2260 | max_tensorboard_logging: 0.8943 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6813.0 | uss_MB: 6199.9 | child_rss_MB: 5051.5 | allocated_MB: 447.6 | reserved_MB: 28508.0 | inactive_split_MB: 989.1 +rank1 [2026-10-29 08:03:50,794] INFO : Step 24900/50000 rank1 | Train Loss: 1.0773 | Train Loss Unweighted: 0.5300 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1506 | Classification: 0.0964 | Classification_binary: 0.1776 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1257 | Regression_eta: 0.0016 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.2047 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0637 | Regression_eta: 0.0557 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0986 | LR: 5.53e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.3168s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8100 | max_loss: 0.1483 | max_backward: 1.9423 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 1.0834 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6802.2 | uss_MB: 5593.7 | child_rss_MB: 5050.0 | allocated_MB: 449.1 | reserved_MB: 28508.0 | inactive_split_MB: 993.6 +rank1 [2026-10-29 08:08:37,678] INFO : Step 25000/50000 rank1 | Train Loss: 1.1315 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1671 | Classification: 0.0915 | Classification_binary: 0.1899 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1201 | Regression_eta: 0.0020 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1818 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0474 | Regression_eta: 0.0322 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0556 | LR: 5.50e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7055s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8152 | max_loss: 0.2476 | max_backward: 1.7290 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.8750 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6807.0 | uss_MB: 6195.6 | child_rss_MB: 5052.2 | allocated_MB: 448.5 | reserved_MB: 28508.0 | inactive_split_MB: 953.9 +rank1 [2026-10-29 08:08:38,675] INFO : Running validation on rank1 +rank1 [2026-10-29 08:08:38,675] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 08:08:38,676] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 08:09:30,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:09:31,356] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 08:09:58,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 08:09:59,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 08:10:00,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 08:10:01,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 08:10:11,110] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 08:10:11,661] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:10:12,184] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 08:10:28,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 08:10:29,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 08:10:30,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 08:10:31,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 08:10:41,258] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 08:10:42,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 08:10:42,694] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 08:10:58,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 08:10:59,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 08:11:00,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 08:11:01,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 08:11:11,026] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 08:16:19,774] INFO : Step 25100/50000 rank1 | Train Loss: 1.0264 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1288 | Classification: 0.0850 | Classification_binary: 0.1497 | HitGrouping: 0.1664 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1378 | Regression_eta: 0.0016 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0902 | Regression_eta: 0.0440 | Regression_pt: 0.0956 | Regression_sin_phi: 0.1066 | LR: 5.47e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.5424s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0059 | max_forward: 0.7878 | max_loss: 0.1437 | max_backward: 2.2383 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9692 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7903.8 | uss_MB: 7286.7 | child_rss_MB: 5042.9 | allocated_MB: 446.8 | reserved_MB: 24062.0 | inactive_split_MB: 1159.3 +rank1 [2026-10-29 08:21:20,699] INFO : Step 25200/50000 rank1 | Train Loss: 1.0775 | Train Loss Unweighted: 0.5434 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1397 | Classification: 0.0996 | Classification_binary: 0.1706 | HitGrouping: 0.1584 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1340 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1968 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0683 | Regression_eta: 0.0456 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0643 | LR: 5.44e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2317s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8831 | max_loss: 0.1456 | max_backward: 2.1320 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7903.3 | uss_MB: 7287.2 | child_rss_MB: 5042.2 | allocated_MB: 448.0 | reserved_MB: 26042.0 | inactive_split_MB: 1093.3 +rank1 [2026-10-29 08:26:17,218] INFO : Step 25300/50000 rank1 | Train Loss: 1.0761 | Train Loss Unweighted: 0.5109 | Train Loss Components: Auxiliary: 0.2132 | Cardinality: 0.1361 | Classification: 0.0961 | Classification_binary: 0.1821 | HitGrouping: 0.1562 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1138 | Regression_eta: 0.0018 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1849 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0614 | Regression_eta: 0.0464 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0823 | LR: 5.42e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.1304s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.9122 | max_loss: 0.1821 | max_backward: 2.1499 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.9506 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6897.2 | uss_MB: 5222.8 | child_rss_MB: 5036.8 | allocated_MB: 446.7 | reserved_MB: 28062.0 | inactive_split_MB: 1098.3 +rank1 [2026-10-29 08:31:10,694] INFO : Step 25400/50000 rank1 | Train Loss: 1.0222 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1238 | Classification: 0.0928 | Classification_binary: 0.1653 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1221 | Regression_eta: 0.0015 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0630 | Regression_eta: 0.0453 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0603 | LR: 5.39e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.7077s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1954 | max_backward: 1.8415 | max_optimizer_step: 0.3277 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6896.2 | uss_MB: 6278.8 | child_rss_MB: 5052.2 | allocated_MB: 449.3 | reserved_MB: 28062.0 | inactive_split_MB: 945.6 +rank1 [2026-10-29 08:36:13,578] INFO : Step 25500/50000 rank1 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4596 | Train Loss Components: Auxiliary: 0.1817 | Cardinality: 0.1001 | Classification: 0.0953 | Classification_binary: 0.1334 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0016 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0734 | Classification_binary: 0.0962 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0421 | Regression_eta: 0.0243 | Regression_pt: 0.0452 | Regression_sin_phi: 0.0296 | LR: 5.36e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7847s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7884 | max_loss: 0.1475 | max_backward: 2.0272 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.7862 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0483 | Memory: rss_MB: 6898.2 | uss_MB: 6283.2 | child_rss_MB: 5050.7 | allocated_MB: 450.7 | reserved_MB: 28062.0 | inactive_split_MB: 1166.9 +rank1 [2026-10-29 08:41:08,951] INFO : Step 25600/50000 rank1 | Train Loss: 1.0724 | Train Loss Unweighted: 0.5330 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1190 | Classification: 0.0985 | Classification_binary: 0.1843 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1231 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1896 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0586 | Regression_eta: 0.0555 | Regression_pt: 0.0630 | Regression_sin_phi: 0.1087 | LR: 5.33e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.1586s | Phase Times: max_pre_step_barrier: 0.0194 | max_batch_to_device: 0.0083 | max_forward: 1.0622 | max_loss: 0.3311 | max_backward: 2.5999 | max_optimizer_step: 0.3019 | max_tensorboard_logging: 0.8123 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6895.2 | uss_MB: 5585.2 | child_rss_MB: 5023.7 | allocated_MB: 450.4 | reserved_MB: 28064.0 | inactive_split_MB: 1163.5 +rank1 [2026-10-29 08:46:09,676] INFO : Step 25700/50000 rank1 | Train Loss: 1.0595 | Train Loss Unweighted: 0.5120 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.1394 | Classification: 0.0920 | Classification_binary: 0.1853 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0575 | Regression_eta: 0.0353 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0439 | LR: 5.30e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.8084s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.8320 | max_loss: 0.2520 | max_backward: 2.6962 | max_optimizer_step: 0.2699 | max_tensorboard_logging: 0.7057 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6885.0 | uss_MB: 6272.6 | child_rss_MB: 5051.6 | allocated_MB: 448.6 | reserved_MB: 29942.0 | inactive_split_MB: 1165.2 +rank1 [2026-10-29 08:51:40,159] INFO : Step 25800/50000 rank1 | Train Loss: 0.9592 | Train Loss Unweighted: 0.4733 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1038 | Classification: 0.0787 | Classification_binary: 0.1531 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1182 | Regression_eta: 0.0016 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0516 | Regression_eta: 0.0402 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0698 | LR: 5.27e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4824s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7549 | max_loss: 0.1480 | max_backward: 1.5296 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 0.8796 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1361 | Memory: rss_MB: 6889.1 | uss_MB: 6277.5 | child_rss_MB: 5023.6 | allocated_MB: 445.9 | reserved_MB: 30100.0 | inactive_split_MB: 961.8 +rank1 [2026-10-29 08:56:50,110] INFO : Step 25900/50000 rank1 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4613 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1142 | Classification: 0.0768 | Classification_binary: 0.1501 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1145 | Regression_eta: 0.0015 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0868 | Regression_energy: 0.0618 | Regression_eta: 0.0511 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0813 | LR: 5.25e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8320s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0124 | max_forward: 0.7901 | max_loss: 0.1430 | max_backward: 1.9362 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8933 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 6880.5 | uss_MB: 6281.6 | child_rss_MB: 5008.5 | allocated_MB: 447.7 | reserved_MB: 30100.0 | inactive_split_MB: 1167.6 +rank1 [2026-10-29 09:01:47,521] INFO : Step 26000/50000 rank1 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5169 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1347 | Classification: 0.0896 | Classification_binary: 0.1938 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0015 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1925 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0564 | Regression_eta: 0.0443 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0848 | LR: 5.22e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.4961s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8125 | max_loss: 0.2612 | max_backward: 1.8771 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.8972 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6845.2 | uss_MB: 5700.0 | child_rss_MB: 4882.0 | allocated_MB: 450.3 | reserved_MB: 30108.0 | inactive_split_MB: 1161.3 +rank1 [2026-10-29 09:01:48,413] INFO : Running validation on rank1 +rank1 [2026-10-29 09:01:48,414] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 09:01:48,414] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 09:02:42,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:02:42,920] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 09:03:09,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 09:03:10,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 09:03:11,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 09:03:12,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 09:03:22,350] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 09:03:22,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:03:23,247] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 09:03:39,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 09:03:40,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 09:03:41,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 09:03:42,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 09:03:52,070] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 09:03:52,917] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:03:53,429] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 09:04:08,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 09:04:09,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 09:04:10,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 09:04:11,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 09:04:21,585] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 09:09:37,593] INFO : Step 26100/50000 rank1 | Train Loss: 0.9519 | Train Loss Unweighted: 0.4738 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1091 | Classification: 0.0871 | Classification_binary: 0.1547 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1147 | Regression_eta: 0.0013 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0717 | Regression_eta: 0.0391 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0510 | LR: 5.19e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2096s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0321 | max_forward: 0.8143 | max_loss: 0.1540 | max_backward: 1.7669 | max_optimizer_step: 0.2382 | max_tensorboard_logging: 1.1185 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 8060.0 | uss_MB: 7433.5 | child_rss_MB: 4850.5 | allocated_MB: 445.1 | reserved_MB: 25850.0 | inactive_split_MB: 1376.9 +rank1 [2026-10-29 09:14:36,072] INFO : Step 26200/50000 rank1 | Train Loss: 0.9758 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.0970 | Classification: 0.0858 | Classification_binary: 0.1501 | HitGrouping: 0.1665 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.1795 | Regression_cos_phi: 0.0758 | Regression_energy: 0.1020 | Regression_eta: 0.0502 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0697 | LR: 5.16e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6742s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7728 | max_loss: 0.1384 | max_backward: 1.8195 | max_optimizer_step: 0.3036 | max_tensorboard_logging: 0.7010 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0602 | Memory: rss_MB: 8060.1 | uss_MB: 7488.7 | child_rss_MB: 4848.5 | allocated_MB: 447.4 | reserved_MB: 27830.0 | inactive_split_MB: 1117.7 +rank1 [2026-10-29 09:19:51,625] INFO : Step 26300/50000 rank1 | Train Loss: 1.0568 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2146 | Cardinality: 0.1416 | Classification: 0.0935 | Classification_binary: 0.1772 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1670 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0680 | Regression_eta: 0.0390 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0719 | LR: 5.13e-04 | DataLoad Time: 0.0175s | Model Forward Time: 6.1523s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.8205 | max_loss: 0.1808 | max_backward: 2.8872 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 1.1024 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6927.6 | uss_MB: 6354.4 | child_rss_MB: 4842.0 | allocated_MB: 450.8 | reserved_MB: 29830.0 | inactive_split_MB: 1114.5 +rank1 [2026-10-29 09:24:55,631] INFO : Step 26400/50000 rank1 | Train Loss: 1.0536 | Train Loss Unweighted: 0.5239 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1323 | Classification: 0.0937 | Classification_binary: 0.1845 | HitGrouping: 0.1559 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1209 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1717 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0753 | Regression_eta: 0.0362 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0440 | LR: 5.10e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.5429s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8639 | max_loss: 0.1569 | max_backward: 2.4710 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 1.0049 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6924.7 | uss_MB: 6353.2 | child_rss_MB: 4853.6 | allocated_MB: 447.6 | reserved_MB: 29832.0 | inactive_split_MB: 1186.4 +rank1 [2026-10-29 09:29:59,513] INFO : Step 26500/50000 rank1 | Train Loss: 0.9812 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1299 | Classification: 0.0807 | Classification_binary: 0.1400 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1217 | Regression_eta: 0.0017 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1156 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0803 | Regression_eta: 0.0485 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0578 | LR: 5.08e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8057s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7436 | max_loss: 0.1579 | max_backward: 1.8417 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.9265 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6921.4 | uss_MB: 6354.1 | child_rss_MB: 4851.7 | allocated_MB: 444.4 | reserved_MB: 29832.0 | inactive_split_MB: 1229.5 +rank1 [2026-10-29 09:34:58,506] INFO : Step 26600/50000 rank1 | Train Loss: 1.0108 | Train Loss Unweighted: 0.5066 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.1182 | Classification: 0.0957 | Classification_binary: 0.1588 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1242 | Regression_eta: 0.0014 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1801 | Regression_cos_phi: 0.0857 | Regression_energy: 0.0662 | Regression_eta: 0.0523 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0833 | LR: 5.05e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6961s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7561 | max_loss: 0.1430 | max_backward: 1.8427 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6921.4 | uss_MB: 6356.0 | child_rss_MB: 4827.0 | allocated_MB: 447.9 | reserved_MB: 29832.0 | inactive_split_MB: 1037.2 +rank1 [2026-10-29 09:40:07,140] INFO : Step 26700/50000 rank1 | Train Loss: 1.0878 | Train Loss Unweighted: 0.5266 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1269 | Classification: 0.0956 | Classification_binary: 0.1961 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1144 | Regression_eta: 0.0018 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2322 | Regression_cos_phi: 0.0923 | Regression_energy: 0.0635 | Regression_eta: 0.0624 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0696 | LR: 5.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1893s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.9788 | max_loss: 0.1540 | max_backward: 1.9407 | max_optimizer_step: 0.2295 | max_tensorboard_logging: 1.0076 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6911.8 | uss_MB: 6344.9 | child_rss_MB: 4859.9 | allocated_MB: 450.1 | reserved_MB: 29832.0 | inactive_split_MB: 1190.4 +rank1 [2026-10-29 09:48:41,512] INFO : Step 26800/50000 rank1 | Train Loss: 1.0077 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1102 | Classification: 0.0787 | Classification_binary: 0.1553 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1359 | Regression_eta: 0.0015 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0822 | Regression_eta: 0.0479 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0718 | LR: 4.99e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.8150s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.7774 | max_loss: 0.2523 | max_backward: 1.8741 | max_optimizer_step: 0.2873 | max_tensorboard_logging: 0.8206 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6909.2 | uss_MB: 6351.0 | child_rss_MB: 4816.3 | allocated_MB: 444.7 | reserved_MB: 29832.0 | inactive_split_MB: 1121.3 +rank1 [2026-10-29 09:53:40,047] INFO : Step 26900/50000 rank1 | Train Loss: 0.9524 | Train Loss Unweighted: 0.4793 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1099 | Classification: 0.0833 | Classification_binary: 0.1305 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1313 | Regression_eta: 0.0016 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1223 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0676 | Regression_eta: 0.0409 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0595 | LR: 4.96e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.4173s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7292 | max_loss: 0.1373 | max_backward: 1.7442 | max_optimizer_step: 0.3152 | max_tensorboard_logging: 0.6764 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 6905.2 | uss_MB: 5520.6 | child_rss_MB: 4788.1 | allocated_MB: 446.9 | reserved_MB: 29832.0 | inactive_split_MB: 1016.7 +rank1 [2026-10-29 09:58:37,161] INFO : Step 27000/50000 rank1 | Train Loss: 1.0285 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1243 | Classification: 0.0862 | Classification_binary: 0.1888 | HitGrouping: 0.1539 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0016 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2248 | Regression_cos_phi: 0.1151 | Regression_energy: 0.0698 | Regression_eta: 0.0557 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0771 | LR: 4.94e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.1610s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.9134 | max_loss: 0.1497 | max_backward: 2.1803 | max_optimizer_step: 0.2254 | max_tensorboard_logging: 0.9490 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6904.1 | uss_MB: 6354.5 | child_rss_MB: 4772.3 | allocated_MB: 449.7 | reserved_MB: 29832.0 | inactive_split_MB: 1024.3 +rank1 [2026-10-29 09:58:38,107] INFO : Running validation on rank1 +rank1 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 09:59:32,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 09:59:33,283] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 10:00:00,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 10:00:01,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 10:00:02,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 10:00:03,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 10:00:13,367] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 10:00:14,008] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:00:14,538] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 10:00:30,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 10:00:32,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 10:00:33,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 10:00:34,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 10:00:44,992] INFO : Time taken to make predictions on device 1 is: 0.28 min +rank1 [2026-10-29 10:00:46,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:00:46,867] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 10:01:02,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 10:01:03,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 10:01:04,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 10:01:05,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 10:01:15,440] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 10:06:37,478] INFO : Step 27100/50000 rank1 | Train Loss: 1.0614 | Train Loss Unweighted: 0.4989 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1449 | Classification: 0.0752 | Classification_binary: 0.1877 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1920 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0553 | Regression_eta: 0.0299 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0763 | LR: 4.91e-04 | DataLoad Time: 0.0208s | Model Forward Time: 7.1178s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0966 | max_forward: 2.1724 | max_loss: 0.6705 | max_backward: 3.1723 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 1.5888 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 8047.0 | uss_MB: 7496.7 | child_rss_MB: 4778.7 | allocated_MB: 447.0 | reserved_MB: 27292.0 | inactive_split_MB: 983.4 +rank1 [2026-10-29 10:11:19,567] INFO : Step 27200/50000 rank1 | Train Loss: 0.9045 | Train Loss Unweighted: 0.4389 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1028 | Classification: 0.0770 | Classification_binary: 0.1287 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1145 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0647 | Regression_eta: 0.0346 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0446 | LR: 4.88e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7903s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7725 | max_loss: 0.5032 | max_backward: 1.6788 | max_optimizer_step: 0.2815 | max_tensorboard_logging: 0.9686 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8047.0 | uss_MB: 7493.7 | child_rss_MB: 4786.7 | allocated_MB: 445.7 | reserved_MB: 29236.0 | inactive_split_MB: 1232.8 +rank1 [2026-10-29 10:15:58,638] INFO : Step 27300/50000 rank1 | Train Loss: 1.0726 | Train Loss Unweighted: 0.5095 | Train Loss Components: Auxiliary: 0.2138 | Cardinality: 0.1337 | Classification: 0.1081 | Classification_binary: 0.1640 | HitGrouping: 0.1596 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1159 | Regression_eta: 0.0019 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1446 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0474 | Regression_eta: 0.0340 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0742 | LR: 4.85e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1433s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8896 | max_loss: 0.1460 | max_backward: 2.0153 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6974.0 | uss_MB: 6424.6 | child_rss_MB: 4784.6 | allocated_MB: 450.9 | reserved_MB: 29330.0 | inactive_split_MB: 1103.1 +rank1 [2026-10-29 10:20:31,815] INFO : Step 27400/50000 rank1 | Train Loss: 1.0758 | Train Loss Unweighted: 0.5282 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1344 | Classification: 0.0930 | Classification_binary: 0.1899 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1207 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0645 | Regression_eta: 0.0342 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0548 | LR: 4.82e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.0228s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8446 | max_loss: 0.1488 | max_backward: 2.0519 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.8725 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6977.0 | uss_MB: 5627.2 | child_rss_MB: 4777.8 | allocated_MB: 448.4 | reserved_MB: 31310.0 | inactive_split_MB: 895.7 +rank1 [2026-10-29 10:25:07,332] INFO : Step 27500/50000 rank1 | Train Loss: 0.9697 | Train Loss Unweighted: 0.4574 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1111 | Classification: 0.0831 | Classification_binary: 0.1473 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1114 | Regression_eta: 0.0018 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0616 | Regression_eta: 0.0565 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0760 | LR: 4.80e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4624s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.7904 | max_loss: 0.1563 | max_backward: 1.6291 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.7457 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6966.1 | uss_MB: 4710.9 | child_rss_MB: 4769.8 | allocated_MB: 447.3 | reserved_MB: 31310.0 | inactive_split_MB: 986.8 +rank1 [2026-10-29 10:30:03,960] INFO : Step 27600/50000 rank1 | Train Loss: 0.9616 | Train Loss Unweighted: 0.4618 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1092 | Classification: 0.0865 | Classification_binary: 0.1284 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0019 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0981 | Regression_energy: 0.0822 | Regression_eta: 0.0593 | Regression_pt: 0.0891 | Regression_sin_phi: 0.0881 | LR: 4.77e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5715s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0380 | max_forward: 0.7674 | max_loss: 0.1463 | max_backward: 1.9164 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.5145 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6974.1 | uss_MB: 6422.9 | child_rss_MB: 4778.4 | allocated_MB: 446.7 | reserved_MB: 31310.0 | inactive_split_MB: 1182.0 +rank1 [2026-10-29 10:34:46,993] INFO : Step 27700/50000 rank1 | Train Loss: 1.0693 | Train Loss Unweighted: 0.5190 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1414 | Classification: 0.0939 | Classification_binary: 0.1783 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0015 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1832 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0606 | Regression_eta: 0.0405 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0673 | LR: 4.74e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.6753s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.8151 | max_loss: 0.1750 | max_backward: 2.0398 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.5213 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0659 | Memory: rss_MB: 6974.1 | uss_MB: 6424.8 | child_rss_MB: 4756.0 | allocated_MB: 448.4 | reserved_MB: 33312.0 | inactive_split_MB: 986.3 +rank1 [2026-10-29 10:39:29,303] INFO : Step 27800/50000 rank1 | Train Loss: 1.0875 | Train Loss Unweighted: 0.5294 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1226 | Classification: 0.0924 | Classification_binary: 0.1842 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1237 | Regression_eta: 0.0019 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.1997 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0701 | Regression_eta: 0.0580 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0720 | LR: 4.71e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.8085s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0152 | max_forward: 0.8463 | max_loss: 0.1744 | max_backward: 1.7379 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.9397 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 6971.2 | uss_MB: 6421.2 | child_rss_MB: 4773.7 | allocated_MB: 452.2 | reserved_MB: 33312.0 | inactive_split_MB: 1320.1 +rank1 [2026-10-29 10:44:11,937] INFO : Step 27900/50000 rank1 | Train Loss: 0.9483 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0988 | Classification: 0.0859 | Classification_binary: 0.1439 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1244 | Regression_eta: 0.0014 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1317 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0563 | Regression_eta: 0.0327 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0555 | LR: 4.68e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.4737s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.7626 | max_loss: 0.1370 | max_backward: 1.8424 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6972.2 | uss_MB: 6423.3 | child_rss_MB: 4773.1 | allocated_MB: 447.1 | reserved_MB: 33312.0 | inactive_split_MB: 1112.6 +rank1 [2026-10-29 10:48:55,971] INFO : Step 28000/50000 rank1 | Train Loss: 0.9840 | Train Loss Unweighted: 0.4736 | Train Loss Components: Auxiliary: 0.1949 | Cardinality: 0.1114 | Classification: 0.0927 | Classification_binary: 0.1564 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1099 | Regression_eta: 0.0015 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0610 | Regression_eta: 0.0571 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0687 | LR: 4.66e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9423s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8192 | max_loss: 0.2552 | max_backward: 2.1299 | max_optimizer_step: 0.2920 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 6969.4 | uss_MB: 6424.2 | child_rss_MB: 4776.8 | allocated_MB: 448.2 | reserved_MB: 33312.0 | inactive_split_MB: 1178.0 +rank1 [2026-10-29 10:48:56,835] INFO : Running validation on rank1 +rank1 [2026-10-29 10:48:56,835] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 10:48:56,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 10:49:51,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:49:52,003] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 10:50:18,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 10:50:19,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 10:50:20,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 10:50:21,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 10:50:31,733] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 10:50:32,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:50:33,003] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 10:50:49,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 10:50:50,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 10:50:50,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 10:50:51,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 10:51:01,974] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 10:51:02,905] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 10:51:03,444] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 10:51:19,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 10:51:20,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 10:51:20,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 10:51:21,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 10:51:31,750] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 10:56:32,744] INFO : Step 28100/50000 rank1 | Train Loss: 1.0915 | Train Loss Unweighted: 0.5085 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1649 | Classification: 0.0833 | Classification_binary: 0.1834 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1187 | Regression_eta: 0.0017 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0611 | Regression_eta: 0.0435 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0435 | LR: 4.63e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8665s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8571 | max_loss: 0.1603 | max_backward: 1.9112 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 0.7670 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0838 | Memory: rss_MB: 8288.3 | uss_MB: 7733.7 | child_rss_MB: 4773.7 | allocated_MB: 448.9 | reserved_MB: 25946.0 | inactive_split_MB: 1313.1 +rank1 [2026-10-29 11:01:18,173] INFO : Step 28200/50000 rank1 | Train Loss: 0.9241 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1144 | Classification: 0.0770 | Classification_binary: 0.1441 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1163 | Regression_eta: 0.0014 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1302 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0572 | Regression_eta: 0.0336 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0616 | LR: 4.60e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8980s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0751 | max_forward: 0.8691 | max_loss: 0.5236 | max_backward: 1.8483 | max_optimizer_step: 0.2480 | max_tensorboard_logging: 0.8546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 8289.0 | uss_MB: 7733.3 | child_rss_MB: 4770.2 | allocated_MB: 445.0 | reserved_MB: 25948.0 | inactive_split_MB: 976.4 +rank1 [2026-10-29 11:07:40,703] INFO : Step 28300/50000 rank1 | Train Loss: 0.9513 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1028 | Classification: 0.0882 | Classification_binary: 0.1368 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1147 | Regression_eta: 0.0016 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0619 | Regression_eta: 0.0660 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0772 | LR: 4.57e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9469s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.7916 | max_loss: 0.1449 | max_backward: 2.0850 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.8583 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7055.4 | uss_MB: 6496.0 | child_rss_MB: 4774.9 | allocated_MB: 447.2 | reserved_MB: 25952.0 | inactive_split_MB: 1165.3 +rank1 [2026-10-29 11:12:27,615] INFO : Step 28400/50000 rank1 | Train Loss: 1.0604 | Train Loss Unweighted: 0.4982 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1382 | Classification: 0.0847 | Classification_binary: 0.1862 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1111 | Regression_eta: 0.0018 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1900 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0521 | Regression_eta: 0.0447 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0899 | LR: 4.55e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.0308s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9139 | max_loss: 0.1474 | max_backward: 2.1323 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.6846 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0517 | Memory: rss_MB: 7055.5 | uss_MB: 6496.2 | child_rss_MB: 4770.0 | allocated_MB: 448.2 | reserved_MB: 26968.0 | inactive_split_MB: 835.9 +rank1 [2026-10-29 11:17:11,503] INFO : Step 28500/50000 rank1 | Train Loss: 1.0282 | Train Loss Unweighted: 0.5005 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1237 | Classification: 0.0911 | Classification_binary: 0.1756 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0013 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0543 | Regression_eta: 0.0501 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0600 | LR: 4.52e-04 | DataLoad Time: 0.0216s | Model Forward Time: 9.4579s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0886 | max_forward: 0.9758 | max_loss: 0.5123 | max_backward: 2.0811 | max_optimizer_step: 0.2797 | max_tensorboard_logging: 1.5644 | max_tensorboard_flush: 0.0047 | max_post_step_barrier: 0.0889 | Memory: rss_MB: 7042.2 | uss_MB: 4349.9 | child_rss_MB: 4774.7 | allocated_MB: 452.0 | reserved_MB: 27968.0 | inactive_split_MB: 923.2 +rank1 [2026-10-29 11:21:56,254] INFO : Step 28600/50000 rank1 | Train Loss: 0.9753 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1202 | Classification: 0.0804 | Classification_binary: 0.1446 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1132 | Regression_eta: 0.0017 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0669 | Regression_eta: 0.0361 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0730 | LR: 4.49e-04 | DataLoad Time: 0.0236s | Model Forward Time: 4.9420s | Phase Times: max_pre_step_barrier: 0.1127 | max_batch_to_device: 0.0146 | max_forward: 0.7731 | max_loss: 0.2404 | max_backward: 1.6352 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 1.0006 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7048.8 | uss_MB: 5469.7 | child_rss_MB: 4776.7 | allocated_MB: 446.8 | reserved_MB: 30056.0 | inactive_split_MB: 896.4 +rank1 [2026-10-29 11:26:34,428] INFO : Step 28700/50000 rank1 | Train Loss: 1.0558 | Train Loss Unweighted: 0.5173 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1082 | Classification: 0.1065 | Classification_binary: 0.1612 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1217 | Regression_eta: 0.0024 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0988 | Regression_energy: 0.0772 | Regression_eta: 0.1098 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0741 | LR: 4.46e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.9585s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0686 | max_forward: 0.8423 | max_loss: 0.1475 | max_backward: 2.0170 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.8830 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 7053.8 | uss_MB: 6496.4 | child_rss_MB: 4774.2 | allocated_MB: 447.9 | reserved_MB: 30056.0 | inactive_split_MB: 925.2 +rank1 [2026-10-29 11:31:18,243] INFO : Step 28800/50000 rank1 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5167 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1474 | Classification: 0.0969 | Classification_binary: 0.1939 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1722 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0701 | Regression_eta: 0.0283 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0638 | LR: 4.44e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.8750s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0075 | max_forward: 0.8427 | max_loss: 0.2661 | max_backward: 2.8464 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7050.1 | uss_MB: 4315.7 | child_rss_MB: 4755.7 | allocated_MB: 448.2 | reserved_MB: 30058.0 | inactive_split_MB: 976.8 +rank1 [2026-10-29 11:35:53,801] INFO : Step 28900/50000 rank1 | Train Loss: 0.9389 | Train Loss Unweighted: 0.4547 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.1078 | Classification: 0.0733 | Classification_binary: 0.1521 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1507 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0656 | Regression_eta: 0.0409 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0561 | LR: 4.41e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.5687s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0062 | max_forward: 0.7499 | max_loss: 0.2426 | max_backward: 1.8556 | max_optimizer_step: 0.3135 | max_tensorboard_logging: 0.5831 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7048.4 | uss_MB: 6494.0 | child_rss_MB: 4774.7 | allocated_MB: 444.3 | reserved_MB: 30058.0 | inactive_split_MB: 1133.2 +rank1 [2026-10-29 11:40:53,319] INFO : Step 29000/50000 rank1 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4535 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0939 | Classification: 0.0855 | Classification_binary: 0.1308 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0015 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0634 | Regression_eta: 0.0428 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0550 | LR: 4.38e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.8471s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0062 | max_forward: 0.9270 | max_loss: 0.2073 | max_backward: 2.1896 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6044 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7047.7 | uss_MB: 6496.6 | child_rss_MB: 4759.3 | allocated_MB: 450.2 | reserved_MB: 30060.0 | inactive_split_MB: 1402.9 +rank1 [2026-10-29 11:40:54,335] INFO : Running validation on rank1 +rank1 [2026-10-29 11:40:54,335] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 11:41:46,666] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:41:47,173] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 11:42:16,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 11:42:17,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 11:42:18,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 11:42:19,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 11:42:29,982] INFO : Time taken to make predictions on device 1 is: 0.28 min +rank1 [2026-10-29 11:42:31,032] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:42:31,573] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 11:42:47,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 11:42:48,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 11:42:49,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 11:42:50,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 11:43:00,846] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 11:43:00,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 11:43:01,399] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 11:43:16,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 11:43:17,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 11:43:18,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 11:43:19,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 11:43:29,450] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 11:48:35,108] INFO : Step 29100/50000 rank1 | Train Loss: 1.0559 | Train Loss Unweighted: 0.5040 | Train Loss Components: Auxiliary: 0.2142 | Cardinality: 0.1307 | Classification: 0.0939 | Classification_binary: 0.1851 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1098 | Regression_eta: 0.0015 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1962 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0595 | Regression_eta: 0.0371 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0718 | LR: 4.35e-04 | DataLoad Time: 0.0151s | Model Forward Time: 7.6741s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.6779 | max_loss: 0.3815 | max_backward: 4.2661 | max_optimizer_step: 0.3470 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8253.0 | uss_MB: 6395.3 | child_rss_MB: 4770.6 | allocated_MB: 448.8 | reserved_MB: 24188.0 | inactive_split_MB: 1162.1 +rank1 [2026-10-29 11:53:34,049] INFO : Step 29200/50000 rank1 | Train Loss: 0.9909 | Train Loss Unweighted: 0.4801 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1204 | Classification: 0.0923 | Classification_binary: 0.1746 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1054 | Regression_eta: 0.0014 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1286 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0602 | Regression_eta: 0.0339 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0524 | LR: 4.33e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7768s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8354 | max_loss: 0.1558 | max_backward: 2.1201 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0677 | Memory: rss_MB: 8251.2 | uss_MB: 7692.4 | child_rss_MB: 4763.1 | allocated_MB: 447.9 | reserved_MB: 24190.0 | inactive_split_MB: 1166.5 +rank1 [2026-10-29 11:58:27,981] INFO : Step 29300/50000 rank1 | Train Loss: 0.9337 | Train Loss Unweighted: 0.4462 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1121 | Classification: 0.0773 | Classification_binary: 0.1359 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1146 | Regression_eta: 0.0015 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1328 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0531 | Regression_eta: 0.0448 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0463 | LR: 4.30e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9131s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0277 | max_forward: 0.8212 | max_loss: 0.1457 | max_backward: 1.8414 | max_optimizer_step: 0.2550 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7135.1 | uss_MB: 5997.0 | child_rss_MB: 4761.5 | allocated_MB: 444.9 | reserved_MB: 24190.0 | inactive_split_MB: 1038.6 +rank1 [2026-10-29 12:03:07,469] INFO : Step 29400/50000 rank1 | Train Loss: 1.0186 | Train Loss Unweighted: 0.4675 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1372 | Classification: 0.0948 | Classification_binary: 0.1561 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0020 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0604 | Regression_eta: 0.0399 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0500 | LR: 4.27e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8009s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8495 | max_loss: 0.1434 | max_backward: 1.9359 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7145.4 | uss_MB: 6588.5 | child_rss_MB: 4762.3 | allocated_MB: 447.9 | reserved_MB: 26208.0 | inactive_split_MB: 957.3 +rank1 [2026-10-29 12:08:16,204] INFO : Step 29500/50000 rank1 | Train Loss: 1.0909 | Train Loss Unweighted: 0.5362 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.1071 | Classification_binary: 0.1896 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1176 | Regression_eta: 0.0019 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1979 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0512 | Regression_eta: 0.0449 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0593 | LR: 4.24e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9673s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0876 | max_forward: 0.8925 | max_loss: 0.1527 | max_backward: 1.9621 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.8162 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7137.1 | uss_MB: 6580.0 | child_rss_MB: 4760.9 | allocated_MB: 448.9 | reserved_MB: 26214.0 | inactive_split_MB: 1166.5 +rank1 [2026-10-29 12:13:12,652] INFO : Step 29600/50000 rank1 | Train Loss: 0.9608 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.1935 | Cardinality: 0.1093 | Classification: 0.0850 | Classification_binary: 0.1542 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1387 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0555 | Regression_eta: 0.0389 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0551 | LR: 4.22e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8672s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7848 | max_loss: 0.1495 | max_backward: 1.8042 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.0178 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7115.2 | uss_MB: 6585.9 | child_rss_MB: 4743.4 | allocated_MB: 445.5 | reserved_MB: 26214.0 | inactive_split_MB: 1171.9 +rank1 [2026-10-29 12:18:07,441] INFO : Step 29700/50000 rank1 | Train Loss: 0.9511 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1260 | Classification: 0.0829 | Classification_binary: 0.1357 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1183 | Regression_eta: 0.0014 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0661 | Regression_eta: 0.0362 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0428 | LR: 4.19e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.2342s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7531 | max_loss: 0.2308 | max_backward: 2.2313 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.9708 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7117.3 | uss_MB: 6587.7 | child_rss_MB: 4741.9 | allocated_MB: 446.6 | reserved_MB: 26214.0 | inactive_split_MB: 1160.6 +rank1 [2026-10-29 12:23:08,926] INFO : Step 29800/50000 rank1 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4915 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1104 | Classification: 0.0864 | Classification_binary: 0.1759 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1836 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0637 | Regression_eta: 0.0370 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0788 | LR: 4.16e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.4226s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8210 | max_loss: 0.1528 | max_backward: 2.4753 | max_optimizer_step: 0.3576 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0038 | max_post_step_barrier: 0.1798 | Memory: rss_MB: 7115.5 | uss_MB: 5536.1 | child_rss_MB: 4741.9 | allocated_MB: 450.3 | reserved_MB: 26214.0 | inactive_split_MB: 938.8 +rank1 [2026-10-29 12:28:10,143] INFO : Step 29900/50000 rank1 | Train Loss: 0.9604 | Train Loss Unweighted: 0.4559 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1243 | Classification: 0.0825 | Classification_binary: 0.1616 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1040 | Regression_eta: 0.0015 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0595 | Regression_eta: 0.0291 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0618 | LR: 4.14e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.9497s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1142 | max_forward: 0.9271 | max_loss: 0.1531 | max_backward: 1.8698 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.8620 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7091.0 | uss_MB: 6437.5 | child_rss_MB: 4745.0 | allocated_MB: 450.0 | reserved_MB: 26214.0 | inactive_split_MB: 1166.0 +rank1 [2026-10-29 12:32:58,888] INFO : Step 30000/50000 rank1 | Train Loss: 0.9450 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.1132 | Classification: 0.0847 | Classification_binary: 0.1394 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0708 | Regression_eta: 0.0455 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0591 | LR: 4.11e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.1879s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 1.0327 | max_loss: 0.1419 | max_backward: 1.6503 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.4878 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7039.0 | uss_MB: 6586.5 | child_rss_MB: 4745.4 | allocated_MB: 443.8 | reserved_MB: 26214.0 | inactive_split_MB: 1152.2 +rank1 [2026-10-29 12:47:12,789] INFO : Running validation on rank1 +rank1 [2026-10-29 12:47:12,790] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 12:47:12,790] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 14:48:59,961] INFO : Initializing model with seed=12345; process seed=12346 +rank1 [2026-10-29 14:48:59,976] INFO : configured dtype=torch.bfloat16 for autocast +rank1 [2026-10-29 14:49:20,929] INFO : Instantiating model +rank1 [2026-10-29 14:49:20,929] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank1 [2026-10-29 14:49:20,930] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank1 [2026-10-29 14:49:20,948] INFO : Input encoding initialization took 0.02s +rank1 [2026-10-29 14:49:20,948] INFO : nn0 parameters: 171776 +rank1 [2026-10-29 14:49:20,948] INFO : Initializing shared backbone layers, num_convs=6 +rank1 [2026-10-29 14:49:20,948] INFO : Initializing backbone layer 0 +rank1 [2026-10-29 14:49:20,983] INFO : Initializing backbone layer 1 +rank1 [2026-10-29 14:49:20,992] INFO : Initializing backbone layer 2 +rank1 [2026-10-29 14:49:21,000] INFO : Initializing backbone layer 3 +rank1 [2026-10-29 14:49:21,009] INFO : Initializing backbone layer 4 +rank1 [2026-10-29 14:49:21,017] INFO : Initializing backbone layer 5 +rank1 [2026-10-29 14:49:21,025] INFO : Backbone initialization took 0.08s +rank1 [2026-10-29 14:49:21,026] INFO : backbone parameters: 7901376 +rank1 [2026-10-29 14:49:21,026] INFO : Initializing output DNNs +rank1 [2026-10-29 14:49:21,062] INFO : Output DNNs initialization took 0.04s +rank1 [2026-10-29 14:49:21,062] INFO : backbone_mode=BackboneMode.SHARED +rank1 [2026-10-29 14:49:21,062] INFO : jagged_attention enabled=False +rank1 [2026-10-29 14:49:21,062] INFO : flash_attn_varlen enabled=False +rank1 [2026-10-29 14:49:21,063] INFO : task_queries enabled=False +rank1 [2026-10-29 14:49:21,063] INFO : classification_norm parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : regression_norm parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : classification_query parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : regression_query parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : classification_readout parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : regression_readout parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : output_mode=set +rank1 [2026-10-29 14:49:21,063] INFO : set_decoder parameters: 4713812 +rank1 [2026-10-29 14:49:21,063] INFO : nn_binary_particle parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : nn_pid parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : nn_pt parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : nn_eta parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : nn_sin_phi parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : nn_cos_phi parameters: 0 +rank1 [2026-10-29 14:49:21,063] INFO : nn_energy parameters: 0 +rank1 [2026-10-29 14:49:21,064] INFO : Total MLPF parameters: 12786964 +rank1 [2026-10-29 14:49:21,064] INFO : MLPF __init__ done +rank1 [2026-10-29 14:49:21,064] INFO : Instantiated model +rank1 [2026-10-29 14:49:21,064] INFO : Moving model to device rank=1 +rank1 [2026-10-29 14:49:21,085] INFO : Moved model to device rank=1 +rank1 [2026-10-29 14:49:21,087] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank1 [2026-10-29 14:49:21,560] INFO : Restoring training from step 30001 +rank1 [2026-10-29 14:49:21,568] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth +rank1 [2026-10-29 14:49:21,569] INFO : Configured model for SyncBatchNorm rank=1 +rank1 [2026-10-29 14:49:22,134] INFO : Configured model for DistributedDataParallel rank=1 +rank1 [2026-10-29 14:49:22,287] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank1 [2026-10-29 14:49:22,287] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank1 [2026-10-29 14:49:22,289] INFO : Trainable parameters: 12786964 +rank1 [2026-10-29 14:49:22,289] INFO : Non-trainable parameters: 23268 +rank1 [2026-10-29 14:49:22,289] INFO : Total parameters: 12810232 +rank1 [2026-10-29 14:49:22,292] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank1 [2026-10-29 14:49:22,292] INFO : Getting dataloaders +rank1 [2026-10-29 14:49:22,292] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:49:28,710] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:49:29,734] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:49:30,959] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank1 [2026-10-29 14:49:30,960] INFO : Creating WeightedMixIterator with 1 data loaders. +rank1 [2026-10-29 14:49:30,960] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:49:31,230] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:49:31,481] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 14:49:31,731] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank1 [2026-10-29 14:49:31,732] INFO : Creating InterleavedIterator with 1 data loaders. +rank1 [2026-10-29 14:49:45,623] INFO : Got dataloaders +rank1 [2026-10-29 14:49:45,623] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [8888] of epochs [1] +rank1 [2026-10-29 14:49:45,623] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank1 [2026-10-29 14:49:45,623] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank1 [2026-10-29 14:49:45,623] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank1 [2026-10-29 14:49:47,294] INFO : loader split=train rank=1 len=21112 +rank1 [2026-10-29 14:49:47,295] INFO : loader split=valid rank=1 len=12 +rank1 [2026-10-29 14:49:47,295] INFO : Starting training from step 30001 to 50000 on rank 1 of 8 +rank1 [2026-10-29 14:49:47,295] INFO : Creating train_iterator +rank1 [2026-10-29 14:49:47,295] INFO : ResumableSampler physical:train loading state: start_index=142208 +rank1 [2026-10-29 14:50:01,468] INFO : Created train_iterator +rank1 [2026-10-29 14:54:42,003] INFO : Step 30100/50000 rank1 | Train Loss: 0.9854 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.2024 | Cardinality: 0.1295 | Classification: 0.0965 | Classification_binary: 0.1561 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0799 | Regression_energy: 0.0659 | Regression_eta: 0.0362 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0661 | LR: 4.08e-04 | DataLoad Time: 0.0183s | Model Forward Time: 3.6354s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7011 | max_loss: 0.1473 | max_backward: 1.5273 | max_optimizer_step: 0.1574 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0061 | Memory: rss_MB: 4775.9 | uss_MB: 3472.5 | child_rss_MB: 6376.2 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 850.1 +rank1 [2026-10-29 14:59:12,794] INFO : Step 30200/50000 rank1 | Train Loss: 1.0165 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2044 | Cardinality: 0.1520 | Classification: 0.0856 | Classification_binary: 0.1718 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1527 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0465 | Regression_eta: 0.0287 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0561 | LR: 4.06e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9884s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.7915 | max_loss: 0.2568 | max_backward: 1.8926 | max_optimizer_step: 0.1551 | max_tensorboard_logging: 0.4604 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4776.3 | uss_MB: 3476.7 | child_rss_MB: 6367.6 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 842.0 +rank1 [2026-10-29 15:03:42,966] INFO : Step 30300/50000 rank1 | Train Loss: 0.9183 | Train Loss Unweighted: 0.4449 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1232 | Classification: 0.0820 | Classification_binary: 0.1459 | HitGrouping: 0.1516 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0012 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1249 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0555 | Regression_eta: 0.0391 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0445 | LR: 4.03e-04 | DataLoad Time: 0.0175s | Model Forward Time: 3.8570s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7525 | max_loss: 0.2476 | max_backward: 1.8262 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.4574 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0039 | Memory: rss_MB: 4773.5 | uss_MB: 3475.1 | child_rss_MB: 6385.6 | allocated_MB: 495.8 | reserved_MB: 28760.0 | inactive_split_MB: 848.2 +rank1 [2026-10-29 15:08:13,476] INFO : Step 30400/50000 rank1 | Train Loss: 0.9035 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1137 | Classification: 0.0848 | Classification_binary: 0.1358 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0616 | Regression_eta: 0.0501 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0598 | LR: 4.00e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9728s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7769 | max_loss: 0.1398 | max_backward: 1.9002 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.4925 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4770.8 | uss_MB: 3475.5 | child_rss_MB: 6366.2 | allocated_MB: 498.8 | reserved_MB: 28760.0 | inactive_split_MB: 878.9 +rank1 [2026-10-29 15:12:45,608] INFO : Step 30500/50000 rank1 | Train Loss: 1.0399 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1239 | Classification: 0.0901 | Classification_binary: 0.1841 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0018 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0584 | Regression_eta: 0.0471 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0819 | LR: 3.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3864s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9688 | max_loss: 0.1627 | max_backward: 2.3054 | max_optimizer_step: 0.1555 | max_tensorboard_logging: 0.4908 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4768.4 | uss_MB: 3475.6 | child_rss_MB: 6405.5 | allocated_MB: 501.7 | reserved_MB: 28760.0 | inactive_split_MB: 944.3 +rank1 [2026-10-29 15:17:13,866] INFO : Step 30600/50000 rank1 | Train Loss: 0.9987 | Train Loss Unweighted: 0.4959 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1112 | Classification: 0.0932 | Classification_binary: 0.1720 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1137 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0585 | Regression_eta: 0.0423 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0599 | LR: 3.95e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2234s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7736 | max_loss: 0.1566 | max_backward: 1.7922 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.6994 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4770.8 | uss_MB: 3476.1 | child_rss_MB: 6399.5 | allocated_MB: 500.1 | reserved_MB: 28760.0 | inactive_split_MB: 845.9 +rank1 [2026-10-29 15:21:45,752] INFO : Step 30700/50000 rank1 | Train Loss: 0.9490 | Train Loss Unweighted: 0.4575 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1237 | Classification: 0.0844 | Classification_binary: 0.1399 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0014 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1189 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0494 | Regression_eta: 0.0413 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0602 | LR: 3.92e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1851s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7215 | max_loss: 0.5560 | max_backward: 2.0906 | max_optimizer_step: 0.1553 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4768.7 | uss_MB: 3474.9 | child_rss_MB: 6379.2 | allocated_MB: 493.8 | reserved_MB: 28760.0 | inactive_split_MB: 827.5 +rank1 [2026-10-29 15:26:16,185] INFO : Step 30800/50000 rank1 | Train Loss: 1.0320 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1331 | Classification: 0.0916 | Classification_binary: 0.1632 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1173 | Regression_eta: 0.0015 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.1753 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0635 | Regression_eta: 0.0574 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0724 | LR: 3.90e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6909s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1016 | max_forward: 0.8041 | max_loss: 0.1462 | max_backward: 1.9351 | max_optimizer_step: 0.1552 | max_tensorboard_logging: 1.1007 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4769.5 | uss_MB: 3475.6 | child_rss_MB: 6407.8 | allocated_MB: 497.9 | reserved_MB: 28760.0 | inactive_split_MB: 842.0 +rank1 [2026-10-29 15:30:49,188] INFO : Step 30900/50000 rank1 | Train Loss: 1.0226 | Train Loss Unweighted: 0.4952 | Train Loss Components: Auxiliary: 0.2139 | Cardinality: 0.1339 | Classification: 0.0875 | Classification_binary: 0.1859 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1092 | Regression_eta: 0.0013 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.2090 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0631 | Regression_eta: 0.0393 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0706 | LR: 3.87e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4019s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7610 | max_loss: 0.2067 | max_backward: 1.7028 | max_optimizer_step: 0.1599 | max_tensorboard_logging: 1.0327 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4771.1 | uss_MB: 3476.3 | child_rss_MB: 6385.8 | allocated_MB: 497.2 | reserved_MB: 28760.0 | inactive_split_MB: 838.0 +rank1 [2026-10-29 15:35:26,546] INFO : Step 31000/50000 rank1 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4630 | Train Loss Components: Auxiliary: 0.2006 | Cardinality: 0.1217 | Classification: 0.0821 | Classification_binary: 0.1592 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1085 | Regression_eta: 0.0018 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1407 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0586 | Regression_eta: 0.0568 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0628 | LR: 3.84e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4978s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7780 | max_loss: 0.2475 | max_backward: 1.8803 | max_optimizer_step: 0.1588 | max_tensorboard_logging: 0.9947 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4772.1 | uss_MB: 3477.0 | child_rss_MB: 6402.3 | allocated_MB: 498.2 | reserved_MB: 28760.0 | inactive_split_MB: 820.4 +rank1 [2026-10-29 15:35:27,413] INFO : Running validation on rank1 +rank1 [2026-10-29 15:35:27,413] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 15:35:27,413] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 15:36:19,715] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:36:20,266] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 15:36:45,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 15:36:47,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 15:36:48,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 15:36:49,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 15:36:59,081] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-29 15:36:59,566] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:37:00,103] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 15:37:15,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 15:37:16,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 15:37:17,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 15:37:18,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 15:37:28,801] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 15:37:29,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 15:37:29,838] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 15:37:44,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 15:37:45,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 15:37:46,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 15:37:47,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 15:37:57,549] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 15:42:50,283] INFO : Step 31100/50000 rank1 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4264 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1054 | Classification: 0.0853 | Classification_binary: 0.1288 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0595 | Regression_eta: 0.0623 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0782 | LR: 3.82e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.5224s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7353 | max_loss: 0.1410 | max_backward: 1.7959 | max_optimizer_step: 0.2754 | max_tensorboard_logging: 0.9599 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 6463.6 | uss_MB: 5188.4 | child_rss_MB: 6394.5 | allocated_MB: 495.8 | reserved_MB: 27508.0 | inactive_split_MB: 851.2 +rank1 [2026-10-29 15:47:34,509] INFO : Step 31200/50000 rank1 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4709 | Train Loss Components: Auxiliary: 0.2045 | Cardinality: 0.1286 | Classification: 0.0862 | Classification_binary: 0.1646 | HitGrouping: 0.1524 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1077 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0640 | Regression_eta: 0.0346 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0419 | LR: 3.79e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3223s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8441 | max_loss: 0.1469 | max_backward: 2.0813 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 6462.4 | uss_MB: 5189.4 | child_rss_MB: 6422.3 | allocated_MB: 496.9 | reserved_MB: 27508.0 | inactive_split_MB: 935.6 +rank1 [2026-10-29 15:52:19,964] INFO : Step 31300/50000 rank1 | Train Loss: 1.0631 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1394 | Classification: 0.0917 | Classification_binary: 0.1731 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.2028 | Regression_cos_phi: 0.0972 | Regression_energy: 0.0730 | Regression_eta: 0.0624 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0962 | LR: 3.76e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2499s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7950 | max_loss: 0.1940 | max_backward: 1.8810 | max_optimizer_step: 0.2519 | max_tensorboard_logging: 0.5719 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5602.7 | uss_MB: 4330.1 | child_rss_MB: 6421.1 | allocated_MB: 498.6 | reserved_MB: 29580.0 | inactive_split_MB: 773.5 +rank1 [2026-10-29 15:56:52,523] INFO : Step 31400/50000 rank1 | Train Loss: 0.9019 | Train Loss Unweighted: 0.4258 | Train Loss Components: Auxiliary: 0.1823 | Cardinality: 0.1101 | Classification: 0.0746 | Classification_binary: 0.1337 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1070 | Regression_eta: 0.0014 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1240 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0684 | Regression_eta: 0.0351 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0679 | LR: 3.74e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2769s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8478 | max_loss: 0.2460 | max_backward: 2.0403 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.4770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5595.4 | uss_MB: 4331.0 | child_rss_MB: 6422.8 | allocated_MB: 495.2 | reserved_MB: 29580.0 | inactive_split_MB: 986.6 +rank1 [2026-10-29 16:01:34,249] INFO : Step 31500/50000 rank1 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4434 | Train Loss Components: Auxiliary: 0.1867 | Cardinality: 0.1036 | Classification: 0.0834 | Classification_binary: 0.1437 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1064 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0554 | Regression_eta: 0.0467 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0539 | LR: 3.71e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.7506s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7930 | max_loss: 0.1458 | max_backward: 1.7855 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 2.2800 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5572.8 | uss_MB: 4310.0 | child_rss_MB: 6426.4 | allocated_MB: 496.8 | reserved_MB: 29580.0 | inactive_split_MB: 850.8 +rank1 [2026-10-29 16:06:05,474] INFO : Step 31600/50000 rank1 | Train Loss: 1.0288 | Train Loss Unweighted: 0.4730 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1317 | Classification: 0.0906 | Classification_binary: 0.1743 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1012 | Regression_eta: 0.0017 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1865 | Regression_cos_phi: 0.1040 | Regression_energy: 0.0550 | Regression_eta: 0.0512 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0791 | LR: 3.69e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1938s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1021 | max_forward: 0.8041 | max_loss: 0.5000 | max_backward: 1.8881 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0133 | Memory: rss_MB: 5567.6 | uss_MB: 4330.2 | child_rss_MB: 6421.3 | allocated_MB: 494.5 | reserved_MB: 29580.0 | inactive_split_MB: 850.5 +rank1 [2026-10-29 16:10:48,123] INFO : Step 31700/50000 rank1 | Train Loss: 0.9206 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1106 | Classification: 0.0729 | Classification_binary: 0.1465 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0573 | Regression_eta: 0.0389 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0560 | LR: 3.66e-04 | DataLoad Time: 0.0185s | Model Forward Time: 3.7506s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0062 | max_forward: 0.7097 | max_loss: 0.1806 | max_backward: 1.6117 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0100 | Memory: rss_MB: 5567.0 | uss_MB: 4328.6 | child_rss_MB: 6424.1 | allocated_MB: 495.3 | reserved_MB: 29580.0 | inactive_split_MB: 969.6 +rank1 [2026-10-29 16:15:56,265] INFO : Step 31800/50000 rank1 | Train Loss: 0.9090 | Train Loss Unweighted: 0.4310 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1140 | Classification: 0.0748 | Classification_binary: 0.1332 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1094 | Regression_eta: 0.0013 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0674 | Regression_eta: 0.0354 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0603 | LR: 3.64e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8747s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7102 | max_loss: 0.1436 | max_backward: 1.6093 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5562.0 | uss_MB: 4326.9 | child_rss_MB: 6424.4 | allocated_MB: 499.4 | reserved_MB: 29584.0 | inactive_split_MB: 1068.5 +rank1 [2026-10-29 16:20:39,712] INFO : Step 31900/50000 rank1 | Train Loss: 0.9919 | Train Loss Unweighted: 0.4645 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1269 | Classification: 0.0911 | Classification_binary: 0.1745 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0975 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1670 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0431 | Regression_eta: 0.0451 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0683 | LR: 3.61e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3407s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.2538 | max_backward: 1.9768 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.5597 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5568.8 | uss_MB: 4333.1 | child_rss_MB: 6433.0 | allocated_MB: 496.2 | reserved_MB: 29584.0 | inactive_split_MB: 829.8 +rank1 [2026-10-29 16:25:21,506] INFO : Step 32000/50000 rank1 | Train Loss: 0.9662 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1172 | Classification: 0.0861 | Classification_binary: 0.1650 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1010 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0564 | Regression_eta: 0.0331 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0469 | LR: 3.58e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2272s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7801 | max_loss: 0.1534 | max_backward: 1.8397 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5561.3 | uss_MB: 4329.3 | child_rss_MB: 6422.4 | allocated_MB: 497.7 | reserved_MB: 29584.0 | inactive_split_MB: 927.6 +rank1 [2026-10-29 16:25:22,456] INFO : Running validation on rank1 +rank1 [2026-10-29 16:25:22,456] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 16:25:22,456] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 16:26:16,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:26:16,710] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 16:26:42,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 16:26:43,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 16:26:44,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 16:26:45,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 16:26:55,857] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 16:26:56,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:26:57,009] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 16:27:12,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 16:27:13,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 16:27:14,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 16:27:15,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 16:27:25,722] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 16:27:26,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 16:27:27,382] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 16:27:43,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 16:27:44,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 16:27:44,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 16:27:45,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 16:27:55,819] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 16:32:59,848] INFO : Step 32100/50000 rank1 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4572 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1125 | Classification: 0.0829 | Classification_binary: 0.1389 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1159 | Regression_eta: 0.0016 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0666 | Regression_eta: 0.0581 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0753 | LR: 3.56e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8027s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0431 | max_forward: 0.7694 | max_loss: 0.1379 | max_backward: 1.5662 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 1.3675 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6985.8 | uss_MB: 5757.8 | child_rss_MB: 6447.8 | allocated_MB: 499.0 | reserved_MB: 25862.0 | inactive_split_MB: 892.0 +rank1 [2026-10-29 16:37:33,176] INFO : Step 32200/50000 rank1 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1193 | Classification: 0.0958 | Classification_binary: 0.1508 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1092 | Regression_eta: 0.0016 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1310 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0501 | Regression_eta: 0.0359 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0777 | LR: 3.53e-04 | DataLoad Time: 0.0145s | Model Forward Time: 5.1406s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8207 | max_loss: 0.1504 | max_backward: 2.0613 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.9148 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6989.2 | uss_MB: 5763.2 | child_rss_MB: 6457.2 | allocated_MB: 499.4 | reserved_MB: 25864.0 | inactive_split_MB: 778.6 +rank1 [2026-10-29 16:42:04,507] INFO : Step 32300/50000 rank1 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4909 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1295 | Classification: 0.0875 | Classification_binary: 0.1834 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0017 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1963 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0596 | Regression_eta: 0.0700 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0911 | LR: 3.51e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7857s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8382 | max_loss: 0.2412 | max_backward: 2.0637 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.9540 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5664.5 | uss_MB: 4450.0 | child_rss_MB: 6462.2 | allocated_MB: 499.0 | reserved_MB: 27716.0 | inactive_split_MB: 775.0 +rank1 [2026-10-29 16:46:37,461] INFO : Step 32400/50000 rank1 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1089 | Classification: 0.0860 | Classification_binary: 0.1494 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1059 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1657 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0599 | Regression_eta: 0.0361 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0553 | LR: 3.48e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5426s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1137 | max_forward: 0.8338 | max_loss: 0.1463 | max_backward: 1.8285 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 0.8970 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5664.1 | uss_MB: 4449.3 | child_rss_MB: 6459.6 | allocated_MB: 497.5 | reserved_MB: 27716.0 | inactive_split_MB: 964.6 +rank1 [2026-10-29 16:51:11,723] INFO : Step 32500/50000 rank1 | Train Loss: 0.9126 | Train Loss Unweighted: 0.4568 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.0989 | Classification: 0.0980 | Classification_binary: 0.1302 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1128 | Regression_eta: 0.0017 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0877 | Regression_eta: 0.0396 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0622 | LR: 3.46e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.0965s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7619 | max_loss: 0.1497 | max_backward: 1.5269 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.7665 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5643.1 | uss_MB: 4429.2 | child_rss_MB: 6454.9 | allocated_MB: 495.6 | reserved_MB: 27718.0 | inactive_split_MB: 1084.2 +rank1 [2026-10-29 16:55:48,603] INFO : Step 32600/50000 rank1 | Train Loss: 0.9821 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2025 | Cardinality: 0.1270 | Classification: 0.0917 | Classification_binary: 0.1687 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1000 | Regression_eta: 0.0014 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1697 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0478 | Regression_eta: 0.0263 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0528 | LR: 3.43e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2680s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1505 | max_backward: 1.8823 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 5658.5 | uss_MB: 4448.5 | child_rss_MB: 6453.0 | allocated_MB: 496.5 | reserved_MB: 27844.0 | inactive_split_MB: 835.5 +rank1 [2026-10-29 17:00:16,962] INFO : Step 32700/50000 rank1 | Train Loss: 1.0218 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2103 | Cardinality: 0.1308 | Classification: 0.0908 | Classification_binary: 0.1761 | HitGrouping: 0.1579 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0655 | Regression_eta: 0.0466 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0632 | LR: 3.41e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2033s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7654 | max_loss: 0.2523 | max_backward: 1.8639 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4904 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.1101 | Memory: rss_MB: 5657.8 | uss_MB: 4448.3 | child_rss_MB: 6459.1 | allocated_MB: 495.8 | reserved_MB: 27844.0 | inactive_split_MB: 1177.8 +rank1 [2026-10-29 17:04:50,647] INFO : Step 32800/50000 rank1 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4798 | Train Loss Components: Auxiliary: 0.1977 | Cardinality: 0.1139 | Classification: 0.0957 | Classification_binary: 0.1480 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1163 | Regression_eta: 0.0019 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0641 | Classification_binary: 0.1516 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0658 | Regression_eta: 0.0535 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0716 | LR: 3.38e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0489s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7437 | max_loss: 0.1443 | max_backward: 1.7603 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.5989 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5618.6 | uss_MB: 4447.1 | child_rss_MB: 6328.3 | allocated_MB: 497.6 | reserved_MB: 27846.0 | inactive_split_MB: 1089.8 +rank1 [2026-10-29 17:09:25,012] INFO : Step 32900/50000 rank1 | Train Loss: 1.0585 | Train Loss Unweighted: 0.4917 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1472 | Classification: 0.0991 | Classification_binary: 0.1566 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0020 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1888 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0550 | Regression_eta: 0.0515 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0829 | LR: 3.36e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2355s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0141 | max_forward: 0.7853 | max_loss: 0.1442 | max_backward: 1.9716 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.5450 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5622.7 | uss_MB: 4451.0 | child_rss_MB: 6346.8 | allocated_MB: 498.5 | reserved_MB: 27846.0 | inactive_split_MB: 844.8 +rank1 [2026-10-29 17:13:57,544] INFO : Step 33000/50000 rank1 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4896 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1109 | Classification: 0.0902 | Classification_binary: 0.1757 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1096 | Regression_eta: 0.0015 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0604 | Regression_eta: 0.0632 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0676 | LR: 3.33e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1388s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0372 | max_forward: 0.7798 | max_loss: 0.1504 | max_backward: 1.8342 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5614.6 | uss_MB: 4448.7 | child_rss_MB: 6348.5 | allocated_MB: 497.7 | reserved_MB: 27846.0 | inactive_split_MB: 857.6 +rank1 [2026-10-29 17:13:58,424] INFO : Running validation on rank1 +rank1 [2026-10-29 17:13:58,424] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 17:13:58,424] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 17:14:49,526] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:14:50,068] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 17:15:17,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 17:15:18,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 17:15:19,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 17:15:20,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 17:15:30,370] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 17:15:31,022] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:15:31,849] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 17:15:48,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 17:15:49,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 17:15:49,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 17:15:50,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 17:16:00,999] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 17:16:01,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 17:16:01,833] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 17:16:17,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 17:16:18,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 17:16:18,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 17:16:19,744] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 17:16:29,753] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 17:21:40,219] INFO : Step 33100/50000 rank1 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4666 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1297 | Classification: 0.0842 | Classification_binary: 0.1537 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1125 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1322 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0501 | Regression_eta: 0.0507 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0682 | LR: 3.31e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1420s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7607 | max_loss: 0.1464 | max_backward: 1.8352 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6888.0 | uss_MB: 5749.0 | child_rss_MB: 6329.7 | allocated_MB: 496.1 | reserved_MB: 24572.0 | inactive_split_MB: 941.9 +rank1 [2026-10-29 17:26:33,026] INFO : Step 33200/50000 rank1 | Train Loss: 0.9345 | Train Loss Unweighted: 0.4417 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.1121 | Classification: 0.0825 | Classification_binary: 0.1333 | HitGrouping: 0.1549 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0017 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0615 | Regression_eta: 0.0398 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0613 | LR: 3.28e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2181s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0447 | max_forward: 0.7619 | max_loss: 0.4589 | max_backward: 1.9181 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.4922 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6887.2 | uss_MB: 5750.0 | child_rss_MB: 6329.3 | allocated_MB: 497.6 | reserved_MB: 24578.0 | inactive_split_MB: 835.2 +rank1 [2026-10-29 17:31:31,564] INFO : Step 33300/50000 rank1 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1270 | Classification: 0.0780 | Classification_binary: 0.1518 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0985 | Regression_eta: 0.0012 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0591 | Regression_eta: 0.0279 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0567 | LR: 3.26e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.6839s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0334 | max_forward: 0.7851 | max_loss: 0.1434 | max_backward: 2.0284 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.7975 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5654.9 | uss_MB: 4170.1 | child_rss_MB: 6324.3 | allocated_MB: 496.9 | reserved_MB: 24578.0 | inactive_split_MB: 903.1 +rank1 [2026-10-29 17:36:22,673] INFO : Step 33400/50000 rank1 | Train Loss: 0.9537 | Train Loss Unweighted: 0.4634 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1239 | Classification: 0.0872 | Classification_binary: 0.1563 | HitGrouping: 0.1500 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0013 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0485 | Regression_eta: 0.0289 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0329 | LR: 3.23e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2710s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7951 | max_loss: 0.2522 | max_backward: 1.9349 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0134 | Memory: rss_MB: 5664.7 | uss_MB: 4533.0 | child_rss_MB: 6325.6 | allocated_MB: 500.7 | reserved_MB: 26578.0 | inactive_split_MB: 843.2 +rank1 [2026-10-29 17:41:05,305] INFO : Step 33500/50000 rank1 | Train Loss: 0.9039 | Train Loss Unweighted: 0.4494 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.1057 | Classification: 0.0795 | Classification_binary: 0.1278 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0011 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1198 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0620 | Regression_eta: 0.0266 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0342 | LR: 3.21e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1841s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0177 | max_forward: 0.7696 | max_loss: 0.1421 | max_backward: 1.9122 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 0.5562 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5654.7 | uss_MB: 4519.8 | child_rss_MB: 6324.9 | allocated_MB: 494.9 | reserved_MB: 26578.0 | inactive_split_MB: 839.3 +rank1 [2026-10-29 17:45:56,712] INFO : Step 33600/50000 rank1 | Train Loss: 0.9179 | Train Loss Unweighted: 0.4415 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.0997 | Classification: 0.0812 | Classification_binary: 0.1440 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0574 | Regression_eta: 0.0557 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0814 | LR: 3.19e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4051s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7783 | max_loss: 0.2418 | max_backward: 2.0392 | max_optimizer_step: 0.2787 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5667.7 | uss_MB: 4535.9 | child_rss_MB: 6324.1 | allocated_MB: 497.9 | reserved_MB: 28594.0 | inactive_split_MB: 1127.0 +rank1 [2026-10-29 17:50:43,455] INFO : Step 33700/50000 rank1 | Train Loss: 1.0564 | Train Loss Unweighted: 0.4962 | Train Loss Components: Auxiliary: 0.2235 | Cardinality: 0.1517 | Classification: 0.0917 | Classification_binary: 0.1786 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1956 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0813 | Regression_eta: 0.0786 | Regression_pt: 0.0883 | Regression_sin_phi: 0.1027 | LR: 3.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1465s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0121 | max_forward: 0.7981 | max_loss: 0.1554 | max_backward: 1.9054 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5667.6 | uss_MB: 4535.1 | child_rss_MB: 6300.3 | allocated_MB: 501.8 | reserved_MB: 28594.0 | inactive_split_MB: 1103.2 +rank1 [2026-10-29 17:55:24,637] INFO : Step 33800/50000 rank1 | Train Loss: 0.9305 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1894 | Cardinality: 0.1080 | Classification: 0.0751 | Classification_binary: 0.1369 | HitGrouping: 0.1527 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1201 | Regression_eta: 0.0014 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1349 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0848 | Regression_eta: 0.0430 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0716 | LR: 3.14e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1048s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8033 | max_loss: 0.1441 | max_backward: 1.7197 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0176 | Memory: rss_MB: 5662.9 | uss_MB: 4531.8 | child_rss_MB: 6320.7 | allocated_MB: 496.9 | reserved_MB: 28594.0 | inactive_split_MB: 909.5 +rank1 [2026-10-29 18:00:01,587] INFO : Step 33900/50000 rank1 | Train Loss: 0.9360 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.1942 | Cardinality: 0.1035 | Classification: 0.0805 | Classification_binary: 0.1361 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1124 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1538 | Regression_cos_phi: 0.0799 | Regression_energy: 0.0724 | Regression_eta: 0.0449 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0469 | LR: 3.11e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2980s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7557 | max_loss: 0.2439 | max_backward: 1.8923 | max_optimizer_step: 0.2623 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5665.6 | uss_MB: 4533.9 | child_rss_MB: 6306.4 | allocated_MB: 496.5 | reserved_MB: 28598.0 | inactive_split_MB: 941.5 +rank1 [2026-10-29 18:04:39,053] INFO : Step 34000/50000 rank1 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0930 | Classification: 0.0875 | Classification_binary: 0.1606 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1761 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0470 | Regression_eta: 0.0304 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0714 | LR: 3.09e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3499s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0943 | max_forward: 0.8289 | max_loss: 0.4616 | max_backward: 2.0219 | max_optimizer_step: 0.1767 | max_tensorboard_logging: 0.4773 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5666.0 | uss_MB: 4534.2 | child_rss_MB: 6326.4 | allocated_MB: 496.0 | reserved_MB: 28598.0 | inactive_split_MB: 1123.0 +rank1 [2026-10-29 18:04:39,968] INFO : Running validation on rank1 +rank1 [2026-10-29 18:04:39,968] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 18:04:39,968] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 18:05:30,627] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:05:31,101] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 18:05:55,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 18:05:56,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 18:05:57,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 18:05:58,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 18:06:08,262] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 18:06:08,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:06:09,214] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 18:06:25,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 18:06:26,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 18:06:26,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 18:06:27,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 18:06:37,949] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 18:06:39,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:06:39,487] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 18:06:54,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 18:06:55,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 18:06:56,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 18:06:57,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 18:07:07,434] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 18:12:12,107] INFO : Step 34100/50000 rank1 | Train Loss: 1.0240 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2153 | Cardinality: 0.1202 | Classification: 0.0979 | Classification_binary: 0.1668 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1210 | Regression_eta: 0.0013 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0827 | Regression_eta: 0.0337 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0785 | LR: 3.06e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5011s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0078 | max_forward: 0.9085 | max_loss: 0.1531 | max_backward: 2.1727 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.4680 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6900.4 | uss_MB: 5761.6 | child_rss_MB: 6326.4 | allocated_MB: 496.4 | reserved_MB: 24560.0 | inactive_split_MB: 1172.2 +rank1 [2026-10-29 18:17:11,666] INFO : Step 34200/50000 rank1 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4225 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.1030 | Classification: 0.0694 | Classification_binary: 0.1279 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0016 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.1369 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0507 | Regression_eta: 0.0970 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0506 | LR: 3.04e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.8724s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 1.1401 | max_loss: 1.0887 | max_backward: 3.3565 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5402 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6897.3 | uss_MB: 5759.3 | child_rss_MB: 6328.4 | allocated_MB: 493.1 | reserved_MB: 27668.0 | inactive_split_MB: 842.0 +rank1 [2026-10-29 18:21:53,344] INFO : Step 34300/50000 rank1 | Train Loss: 0.8999 | Train Loss Unweighted: 0.4291 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.1046 | Classification: 0.0782 | Classification_binary: 0.1421 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0015 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1570 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0519 | Regression_eta: 0.0433 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0719 | LR: 3.02e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5622s | Phase Times: max_pre_step_barrier: 0.1089 | max_batch_to_device: 0.0070 | max_forward: 0.7738 | max_loss: 0.4754 | max_backward: 1.8888 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5721.4 | uss_MB: 4584.0 | child_rss_MB: 6315.8 | allocated_MB: 498.9 | reserved_MB: 29668.0 | inactive_split_MB: 777.1 +rank1 [2026-10-29 18:26:42,331] INFO : Step 34400/50000 rank1 | Train Loss: 0.9382 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2026 | Cardinality: 0.1181 | Classification: 0.0846 | Classification_binary: 0.1681 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0545 | Regression_eta: 0.0407 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0560 | LR: 2.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6568s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.8411 | max_loss: 0.2739 | max_backward: 2.1385 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5734.9 | uss_MB: 4598.3 | child_rss_MB: 6315.2 | allocated_MB: 496.8 | reserved_MB: 29668.0 | inactive_split_MB: 858.8 +rank1 [2026-10-29 18:31:40,823] INFO : Step 34500/50000 rank1 | Train Loss: 0.9189 | Train Loss Unweighted: 0.4456 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1063 | Classification: 0.0767 | Classification_binary: 0.1421 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0645 | Regression_eta: 0.0441 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0569 | LR: 2.97e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.2130s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.7430 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5727.7 | uss_MB: 4595.8 | child_rss_MB: 6313.8 | allocated_MB: 494.3 | reserved_MB: 29668.0 | inactive_split_MB: 926.0 +rank1 [2026-10-29 18:36:27,182] INFO : Step 34600/50000 rank1 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4340 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1046 | Classification: 0.0794 | Classification_binary: 0.1340 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0016 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1247 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0478 | Regression_eta: 0.0347 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0581 | LR: 2.95e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.6952s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0170 | max_forward: 0.9895 | max_loss: 0.4622 | max_backward: 2.2541 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5731.7 | uss_MB: 4597.0 | child_rss_MB: 6311.8 | allocated_MB: 498.6 | reserved_MB: 29672.0 | inactive_split_MB: 765.1 +rank1 [2026-10-29 18:40:58,825] INFO : Step 34700/50000 rank1 | Train Loss: 0.9586 | Train Loss Unweighted: 0.4539 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1185 | Classification: 0.0862 | Classification_binary: 0.1624 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0501 | Regression_eta: 0.0488 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0811 | LR: 2.92e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3657s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7862 | max_loss: 0.2628 | max_backward: 1.9231 | max_optimizer_step: 0.2860 | max_tensorboard_logging: 0.4599 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5730.7 | uss_MB: 4596.4 | child_rss_MB: 6303.0 | allocated_MB: 499.0 | reserved_MB: 29672.0 | inactive_split_MB: 1074.3 +rank1 [2026-10-29 18:45:38,651] INFO : Step 34800/50000 rank1 | Train Loss: 0.9346 | Train Loss Unweighted: 0.4373 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1276 | Classification: 0.0867 | Classification_binary: 0.1551 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0959 | Regression_eta: 0.0013 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1422 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0510 | Regression_eta: 0.0554 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0654 | LR: 2.90e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2157s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0770 | max_forward: 0.8404 | max_loss: 0.1556 | max_backward: 1.8035 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.5351 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5727.0 | uss_MB: 4594.2 | child_rss_MB: 6312.4 | allocated_MB: 497.4 | reserved_MB: 29672.0 | inactive_split_MB: 1113.6 +rank1 [2026-10-29 18:50:17,935] INFO : Step 34900/50000 rank1 | Train Loss: 0.8719 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1154 | Classification: 0.0681 | Classification_binary: 0.1220 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1093 | Regression_eta: 0.0013 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1165 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0663 | Regression_eta: 0.0306 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0340 | LR: 2.88e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.1473s | Phase Times: max_pre_step_barrier: 0.0080 | max_batch_to_device: 0.0693 | max_forward: 1.2127 | max_loss: 0.2834 | max_backward: 3.0996 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.6125 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 5724.6 | uss_MB: 4576.6 | child_rss_MB: 6292.2 | allocated_MB: 495.3 | reserved_MB: 29672.0 | inactive_split_MB: 864.7 +rank1 [2026-10-29 18:55:19,194] INFO : Step 35000/50000 rank1 | Train Loss: 0.9157 | Train Loss Unweighted: 0.4220 | Train Loss Components: Auxiliary: 0.1897 | Cardinality: 0.1230 | Classification: 0.0771 | Classification_binary: 0.1455 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0980 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0548 | Regression_eta: 0.0586 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0655 | LR: 2.85e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5145s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.8243 | max_loss: 0.4759 | max_backward: 2.1181 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5728.2 | uss_MB: 4598.3 | child_rss_MB: 6307.6 | allocated_MB: 498.3 | reserved_MB: 29672.0 | inactive_split_MB: 766.0 +rank1 [2026-10-29 18:55:20,126] INFO : Running validation on rank1 +rank1 [2026-10-29 18:55:20,127] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 18:55:20,127] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 18:56:11,604] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:56:12,120] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 18:56:36,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 18:56:37,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 18:56:38,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 18:56:39,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 18:56:49,413] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 18:56:49,875] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:56:50,324] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 18:57:06,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 18:57:08,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 18:57:09,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 18:57:10,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 18:57:20,467] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 18:57:20,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 18:57:20,904] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 18:57:36,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 18:57:37,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 18:57:38,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 18:57:39,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 18:57:49,090] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 19:02:56,670] INFO : Step 35100/50000 rank1 | Train Loss: 0.9622 | Train Loss Unweighted: 0.4512 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1150 | Classification: 0.0857 | Classification_binary: 0.1724 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0948 | Regression_eta: 0.0014 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.1866 | Regression_cos_phi: 0.0940 | Regression_energy: 0.0576 | Regression_eta: 0.0589 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0841 | LR: 2.83e-04 | DataLoad Time: 0.0132s | Model Forward Time: 4.8792s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1018 | max_forward: 0.8921 | max_loss: 0.1518 | max_backward: 2.0328 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8633 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5815.2 | uss_MB: 4686.8 | child_rss_MB: 6305.0 | allocated_MB: 499.4 | reserved_MB: 23366.0 | inactive_split_MB: 1243.8 +rank1 [2026-10-29 19:07:43,278] INFO : Step 35200/50000 rank1 | Train Loss: 0.9331 | Train Loss Unweighted: 0.4385 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.1234 | Classification: 0.0814 | Classification_binary: 0.1441 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0580 | Regression_eta: 0.0328 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0617 | LR: 2.81e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1474s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0577 | max_forward: 0.7693 | max_loss: 0.1423 | max_backward: 1.7209 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5822.3 | uss_MB: 4693.9 | child_rss_MB: 6304.7 | allocated_MB: 494.4 | reserved_MB: 25346.0 | inactive_split_MB: 1054.2 +rank1 [2026-10-29 19:12:30,668] INFO : Step 35300/50000 rank1 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4075 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1105 | Classification: 0.0819 | Classification_binary: 0.1245 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0989 | Regression_eta: 0.0014 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1251 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0456 | Regression_eta: 0.0234 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0670 | LR: 2.79e-04 | DataLoad Time: 0.0177s | Model Forward Time: 6.8656s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 1.2856 | max_loss: 0.4079 | max_backward: 4.4969 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5749 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5814.5 | uss_MB: 4685.2 | child_rss_MB: 6307.8 | allocated_MB: 498.3 | reserved_MB: 25346.0 | inactive_split_MB: 1140.5 +rank1 [2026-10-29 19:17:14,808] INFO : Step 35400/50000 rank1 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4888 | Train Loss Components: Auxiliary: 0.2130 | Cardinality: 0.1296 | Classification: 0.0908 | Classification_binary: 0.1718 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0014 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0803 | Regression_eta: 0.0373 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0469 | LR: 2.76e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.3446s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8312 | max_loss: 0.1490 | max_backward: 2.0287 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.4708 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5820.8 | uss_MB: 4696.7 | child_rss_MB: 6308.5 | allocated_MB: 497.8 | reserved_MB: 25346.0 | inactive_split_MB: 1132.0 +rank1 [2026-10-29 19:21:58,249] INFO : Step 35500/50000 rank1 | Train Loss: 0.9444 | Train Loss Unweighted: 0.4557 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1201 | Classification: 0.0924 | Classification_binary: 0.1605 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0438 | Regression_eta: 0.0337 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0534 | LR: 2.74e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3615s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7670 | max_loss: 0.2318 | max_backward: 1.8405 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5820.9 | uss_MB: 4694.9 | child_rss_MB: 6312.3 | allocated_MB: 497.5 | reserved_MB: 26942.0 | inactive_split_MB: 1130.5 +rank1 [2026-10-29 19:26:41,939] INFO : Step 35600/50000 rank1 | Train Loss: 0.9280 | Train Loss Unweighted: 0.4451 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1095 | Classification: 0.0861 | Classification_binary: 0.1334 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0565 | Regression_eta: 0.0396 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0424 | LR: 2.72e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.7398s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7460 | max_loss: 0.1440 | max_backward: 1.8206 | max_optimizer_step: 0.2911 | max_tensorboard_logging: 0.6036 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 5820.5 | uss_MB: 3919.7 | child_rss_MB: 6316.0 | allocated_MB: 496.1 | reserved_MB: 26942.0 | inactive_split_MB: 907.6 +rank1 [2026-10-29 19:31:29,665] INFO : Step 35700/50000 rank1 | Train Loss: 0.9116 | Train Loss Unweighted: 0.4301 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1148 | Classification: 0.0829 | Classification_binary: 0.1432 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0998 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1343 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0524 | Regression_eta: 0.0287 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0488 | LR: 2.70e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6316s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8312 | max_loss: 0.2568 | max_backward: 2.2622 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5823.2 | uss_MB: 4699.5 | child_rss_MB: 6323.6 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1241.0 +rank1 [2026-10-29 19:36:11,896] INFO : Step 35800/50000 rank1 | Train Loss: 1.0095 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1252 | Classification: 0.0857 | Classification_binary: 0.1786 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1066 | Regression_eta: 0.0014 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1845 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0672 | Regression_eta: 0.0333 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0361 | LR: 2.68e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1747s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7644 | max_loss: 0.5090 | max_backward: 1.8855 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4861 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5816.8 | uss_MB: 4694.8 | child_rss_MB: 6319.7 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1129.0 +rank1 [2026-10-29 19:40:48,761] INFO : Step 35900/50000 rank1 | Train Loss: 0.9596 | Train Loss Unweighted: 0.4571 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1173 | Classification: 0.0913 | Classification_binary: 0.1468 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0014 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0563 | Regression_eta: 0.0343 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0714 | LR: 2.65e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.0314s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7217 | max_loss: 0.1443 | max_backward: 1.6894 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.4586 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5816.9 | uss_MB: 4695.3 | child_rss_MB: 6319.9 | allocated_MB: 495.4 | reserved_MB: 29024.0 | inactive_split_MB: 934.6 +rank1 [2026-10-29 19:45:19,092] INFO : Step 36000/50000 rank1 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0912 | Classification: 0.0938 | Classification_binary: 0.1314 | HitGrouping: 0.1587 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1135 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1102 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0499 | Regression_eta: 0.0366 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0517 | LR: 2.63e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3012s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0073 | max_forward: 0.7630 | max_loss: 0.4769 | max_backward: 1.8869 | max_optimizer_step: 0.2587 | max_tensorboard_logging: 0.5842 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5815.2 | uss_MB: 4695.9 | child_rss_MB: 6324.3 | allocated_MB: 495.0 | reserved_MB: 29024.0 | inactive_split_MB: 937.5 +rank1 [2026-10-29 19:45:19,981] INFO : Running validation on rank1 +rank1 [2026-10-29 19:45:19,981] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 19:45:19,981] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 19:46:10,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:46:11,093] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 19:46:35,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 19:46:36,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 19:46:37,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 19:46:38,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 19:46:48,903] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 19:46:49,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:46:49,932] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 19:47:05,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 19:47:06,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 19:47:07,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 19:47:08,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 19:47:18,731] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 19:47:19,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 19:47:19,885] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 19:47:35,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 19:47:36,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 19:47:37,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 19:47:38,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 19:47:48,094] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 19:53:12,007] INFO : Step 36100/50000 rank1 | Train Loss: 0.9292 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1100 | Classification: 0.0906 | Classification_binary: 0.1490 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0554 | Regression_eta: 0.0383 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0459 | LR: 2.61e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.7853s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7768 | max_loss: 0.2455 | max_backward: 1.8077 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 1.0653 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5880.7 | uss_MB: 4756.1 | child_rss_MB: 6293.3 | allocated_MB: 499.2 | reserved_MB: 26012.0 | inactive_split_MB: 1080.2 +rank1 [2026-10-29 19:58:21,470] INFO : Step 36200/50000 rank1 | Train Loss: 0.9030 | Train Loss Unweighted: 0.4308 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1163 | Classification: 0.0762 | Classification_binary: 0.1596 | HitGrouping: 0.1506 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1435 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0490 | Regression_eta: 0.0225 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0332 | LR: 2.59e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5373s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0777 | max_forward: 0.8125 | max_loss: 0.1541 | max_backward: 1.8492 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 5884.8 | uss_MB: 4767.2 | child_rss_MB: 6290.6 | allocated_MB: 494.4 | reserved_MB: 26016.0 | inactive_split_MB: 1377.4 +rank1 [2026-10-29 20:03:20,657] INFO : Step 36300/50000 rank1 | Train Loss: 0.9576 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1236 | Classification: 0.0876 | Classification_binary: 0.1347 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1100 | Regression_eta: 0.0017 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1108 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0603 | Regression_eta: 0.0784 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0933 | LR: 2.57e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.4513s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.8065 | max_loss: 0.1484 | max_backward: 1.9315 | max_optimizer_step: 0.1930 | max_tensorboard_logging: 0.6801 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5878.7 | uss_MB: 4768.0 | child_rss_MB: 6273.7 | allocated_MB: 496.0 | reserved_MB: 28008.0 | inactive_split_MB: 1156.5 +rank1 [2026-10-29 20:08:10,866] INFO : Step 36400/50000 rank1 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4147 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.0999 | Classification: 0.0796 | Classification_binary: 0.1372 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0609 | Regression_eta: 0.0374 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0505 | LR: 2.55e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.8728s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0192 | max_forward: 1.1829 | max_loss: 0.3790 | max_backward: 3.0917 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 5880.4 | uss_MB: 4768.9 | child_rss_MB: 6270.8 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 845.2 +rank1 [2026-10-29 20:12:41,929] INFO : Step 36500/50000 rank1 | Train Loss: 0.9762 | Train Loss Unweighted: 0.4543 | Train Loss Components: Auxiliary: 0.2088 | Cardinality: 0.1239 | Classification: 0.0805 | Classification_binary: 0.1682 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1934 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0541 | Regression_eta: 0.0412 | Regression_pt: 0.0590 | Regression_sin_phi: 0.1039 | LR: 2.52e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2749s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7782 | max_loss: 0.1554 | max_backward: 1.9378 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.5103 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5877.6 | uss_MB: 4769.9 | child_rss_MB: 6273.0 | allocated_MB: 499.0 | reserved_MB: 30008.0 | inactive_split_MB: 913.8 +rank1 [2026-10-29 20:17:47,229] INFO : Step 36600/50000 rank1 | Train Loss: 0.8699 | Train Loss Unweighted: 0.4190 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.1138 | Classification: 0.0734 | Classification_binary: 0.1317 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0010 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1024 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0497 | Regression_eta: 0.0179 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0452 | LR: 2.50e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.2962s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7561 | max_loss: 0.1454 | max_backward: 1.9017 | max_optimizer_step: 0.5115 | max_tensorboard_logging: 0.5672 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5872.6 | uss_MB: 4766.8 | child_rss_MB: 6277.6 | allocated_MB: 494.4 | reserved_MB: 30008.0 | inactive_split_MB: 856.0 +rank1 [2026-10-29 20:22:56,452] INFO : Step 36700/50000 rank1 | Train Loss: 0.9065 | Train Loss Unweighted: 0.4327 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.1102 | Classification: 0.0849 | Classification_binary: 0.1262 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1087 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0594 | Regression_eta: 0.0372 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0419 | LR: 2.48e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.0805s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.7169 | max_loss: 0.1507 | max_backward: 1.7104 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5871.4 | uss_MB: 4766.4 | child_rss_MB: 6270.7 | allocated_MB: 496.5 | reserved_MB: 30008.0 | inactive_split_MB: 1154.6 +rank1 [2026-10-29 20:28:04,050] INFO : Step 36800/50000 rank1 | Train Loss: 0.9068 | Train Loss Unweighted: 0.4227 | Train Loss Components: Auxiliary: 0.1925 | Cardinality: 0.1100 | Classification: 0.0739 | Classification_binary: 0.1571 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0945 | Regression_eta: 0.0013 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0370 | Regression_energy: 0.0480 | Regression_eta: 0.0374 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0353 | LR: 2.46e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4683s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8487 | max_loss: 0.1525 | max_backward: 2.1121 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5877.1 | uss_MB: 4770.1 | child_rss_MB: 6274.6 | allocated_MB: 499.3 | reserved_MB: 30008.0 | inactive_split_MB: 939.6 +rank1 [2026-10-29 20:33:09,546] INFO : Step 36900/50000 rank1 | Train Loss: 0.8934 | Train Loss Unweighted: 0.4126 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1194 | Classification: 0.0721 | Classification_binary: 0.1507 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0013 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0526 | Regression_eta: 0.0295 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0697 | LR: 2.44e-04 | DataLoad Time: 0.0125s | Model Forward Time: 5.5742s | Phase Times: max_pre_step_barrier: 0.0139 | max_batch_to_device: 0.0068 | max_forward: 1.0975 | max_loss: 0.2241 | max_backward: 2.7278 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.6216 | max_tensorboard_flush: 0.0085 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5871.7 | uss_MB: 4750.7 | child_rss_MB: 6260.8 | allocated_MB: 498.9 | reserved_MB: 30008.0 | inactive_split_MB: 771.1 +rank1 [2026-10-29 20:38:05,960] INFO : Step 37000/50000 rank1 | Train Loss: 0.9040 | Train Loss Unweighted: 0.4182 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1192 | Classification: 0.0807 | Classification_binary: 0.1318 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1282 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0642 | Regression_eta: 0.0307 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0479 | LR: 2.42e-04 | DataLoad Time: 0.0162s | Model Forward Time: 3.8504s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7055 | max_loss: 0.1505 | max_backward: 1.5664 | max_optimizer_step: 0.1900 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5866.9 | uss_MB: 4763.2 | child_rss_MB: 6273.0 | allocated_MB: 493.9 | reserved_MB: 30008.0 | inactive_split_MB: 1169.8 +rank1 [2026-10-29 20:38:06,931] INFO : Running validation on rank1 +rank1 [2026-10-29 20:38:06,931] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 20:38:06,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 20:39:05,977] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:39:06,530] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 20:39:31,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 20:39:32,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 20:39:33,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 20:39:34,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 20:39:44,625] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 20:39:45,042] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:39:45,534] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 20:40:01,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 20:40:02,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 20:40:03,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 20:40:05,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 20:40:15,054] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 20:40:16,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 20:40:16,609] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 20:40:32,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 20:40:33,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 20:40:34,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 20:40:35,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 20:40:45,441] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 20:45:59,178] INFO : Step 37100/50000 rank1 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4614 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1255 | Classification: 0.0999 | Classification_binary: 0.1492 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1041 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0621 | Regression_eta: 0.0378 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0530 | LR: 2.40e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8451s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0061 | max_forward: 0.8553 | max_loss: 0.1461 | max_backward: 2.1723 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5959.4 | uss_MB: 4852.0 | child_rss_MB: 6311.9 | allocated_MB: 499.3 | reserved_MB: 25668.0 | inactive_split_MB: 852.4 +rank1 [2026-10-29 20:50:39,578] INFO : Step 37200/50000 rank1 | Train Loss: 0.9288 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.1928 | Cardinality: 0.1108 | Classification: 0.0836 | Classification_binary: 0.1643 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0015 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0517 | Regression_eta: 0.0613 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0572 | LR: 2.38e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.6406s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8460 | max_loss: 0.2603 | max_backward: 2.0959 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5966.0 | uss_MB: 4861.5 | child_rss_MB: 6312.1 | allocated_MB: 497.9 | reserved_MB: 25674.0 | inactive_split_MB: 846.0 +rank1 [2026-10-29 20:55:11,587] INFO : Step 37300/50000 rank1 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1275 | Classification: 0.0781 | Classification_binary: 0.1424 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0982 | Regression_eta: 0.0013 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1192 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0618 | Regression_eta: 0.0282 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0447 | LR: 2.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0203s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7653 | max_loss: 0.1472 | max_backward: 1.7105 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.4778 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0692 | Memory: rss_MB: 5962.2 | uss_MB: 4859.4 | child_rss_MB: 6302.2 | allocated_MB: 497.3 | reserved_MB: 27616.0 | inactive_split_MB: 974.4 +rank1 [2026-10-29 21:00:09,947] INFO : Step 37400/50000 rank1 | Train Loss: 0.8836 | Train Loss Unweighted: 0.4228 | Train Loss Components: Auxiliary: 0.1825 | Cardinality: 0.0983 | Classification: 0.0851 | Classification_binary: 0.1289 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1025 | Regression_eta: 0.0015 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0490 | Regression_eta: 0.0492 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0431 | LR: 2.34e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5380s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7611 | max_loss: 0.1956 | max_backward: 1.9276 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7505 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5960.0 | uss_MB: 4860.1 | child_rss_MB: 6301.8 | allocated_MB: 495.0 | reserved_MB: 27690.0 | inactive_split_MB: 915.5 +rank1 [2026-10-29 21:05:01,763] INFO : Step 37500/50000 rank1 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1260 | Classification: 0.0804 | Classification_binary: 0.1577 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1005 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1795 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0432 | Regression_eta: 0.0561 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0749 | LR: 2.32e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4455s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.9030 | max_loss: 0.1494 | max_backward: 1.9871 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5938.4 | uss_MB: 4859.4 | child_rss_MB: 6224.4 | allocated_MB: 496.1 | reserved_MB: 27690.0 | inactive_split_MB: 923.5 +rank1 [2026-10-29 21:09:54,197] INFO : Step 37600/50000 rank1 | Train Loss: 0.9091 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.0995 | Classification: 0.0683 | Classification_binary: 0.1534 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0957 | Regression_eta: 0.0015 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1531 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0513 | Regression_eta: 0.0374 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0537 | LR: 2.30e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3493s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0070 | max_forward: 0.8280 | max_loss: 0.1484 | max_backward: 1.8328 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5933.4 | uss_MB: 4856.7 | child_rss_MB: 6230.2 | allocated_MB: 497.1 | reserved_MB: 27690.0 | inactive_split_MB: 927.5 +rank1 [2026-10-29 21:14:45,920] INFO : Step 37700/50000 rank1 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3995 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.0864 | Classification: 0.0813 | Classification_binary: 0.1231 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0012 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0705 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0559 | Regression_eta: 0.0352 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0332 | LR: 2.28e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4208s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7582 | max_loss: 0.2486 | max_backward: 1.9197 | max_optimizer_step: 0.3005 | max_tensorboard_logging: 0.5767 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5802.8 | uss_MB: 4857.6 | child_rss_MB: 5767.0 | allocated_MB: 495.2 | reserved_MB: 29766.0 | inactive_split_MB: 925.5 +rank1 [2026-10-29 21:19:50,095] INFO : Step 37800/50000 rank1 | Train Loss: 0.8717 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1057 | Classification: 0.0822 | Classification_binary: 0.1363 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0014 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1297 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0399 | Regression_eta: 0.0372 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0581 | LR: 2.26e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.6249s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 1.2105 | max_loss: 0.2449 | max_backward: 2.0574 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 5800.5 | uss_MB: 4856.8 | child_rss_MB: 5763.9 | allocated_MB: 498.3 | reserved_MB: 29766.0 | inactive_split_MB: 930.0 +rank1 [2026-10-29 21:24:49,255] INFO : Step 37900/50000 rank1 | Train Loss: 0.9843 | Train Loss Unweighted: 0.4684 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1353 | Classification: 0.0912 | Classification_binary: 0.1705 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0014 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0557 | Regression_eta: 0.0301 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0465 | LR: 2.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.2102s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0297 | max_forward: 0.8081 | max_loss: 0.1547 | max_backward: 2.6708 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5096 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5782.9 | uss_MB: 4859.7 | child_rss_MB: 5732.7 | allocated_MB: 495.6 | reserved_MB: 29766.0 | inactive_split_MB: 912.0 +rank1 [2026-10-29 21:29:44,506] INFO : Step 38000/50000 rank1 | Train Loss: 0.8556 | Train Loss Unweighted: 0.4002 | Train Loss Components: Auxiliary: 0.1833 | Cardinality: 0.1090 | Classification: 0.0730 | Classification_binary: 0.1293 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0972 | Regression_eta: 0.0011 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0518 | Regression_eta: 0.0325 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0469 | LR: 2.22e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1653s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7330 | max_loss: 0.2284 | max_backward: 1.8082 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5778.8 | uss_MB: 4857.4 | child_rss_MB: 5727.0 | allocated_MB: 494.5 | reserved_MB: 29766.0 | inactive_split_MB: 847.3 +rank1 [2026-10-29 21:29:45,440] INFO : Running validation on rank1 +rank1 [2026-10-29 21:29:45,440] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 21:29:45,440] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 21:30:41,619] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:30:42,096] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 21:31:09,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 21:31:10,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 21:31:11,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 21:31:12,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 21:31:22,721] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 21:31:23,194] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:31:23,630] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 21:31:39,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 21:31:40,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 21:31:41,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 21:31:42,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 21:31:52,496] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 21:31:52,734] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 21:31:53,234] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 21:32:08,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 21:32:09,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 21:32:10,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 21:32:11,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 21:32:21,527] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-29 21:37:39,070] INFO : Step 38100/50000 rank1 | Train Loss: 0.9020 | Train Loss Unweighted: 0.4337 | Train Loss Components: Auxiliary: 0.1907 | Cardinality: 0.1231 | Classification: 0.0888 | Classification_binary: 0.1339 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0014 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1428 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0592 | Regression_eta: 0.0343 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0581 | LR: 2.20e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8730s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0963 | max_forward: 0.8445 | max_loss: 0.1406 | max_backward: 1.9197 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.9697 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7028.7 | uss_MB: 6098.3 | child_rss_MB: 5734.5 | allocated_MB: 496.0 | reserved_MB: 27516.0 | inactive_split_MB: 970.5 +rank1 [2026-10-29 21:42:37,687] INFO : Step 38200/50000 rank1 | Train Loss: 0.9174 | Train Loss Unweighted: 0.4351 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.0971 | Classification: 0.0888 | Classification_binary: 0.1526 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0948 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0974 | Regression_energy: 0.0393 | Regression_eta: 0.0344 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0541 | LR: 2.18e-04 | DataLoad Time: 0.0178s | Model Forward Time: 6.4459s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.8480 | max_loss: 0.2651 | max_backward: 3.1905 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.9151 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5881.2 | uss_MB: 4342.7 | child_rss_MB: 5733.1 | allocated_MB: 499.7 | reserved_MB: 27516.0 | inactive_split_MB: 848.5 +rank1 [2026-10-29 21:47:30,433] INFO : Step 38300/50000 rank1 | Train Loss: 0.9342 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1998 | Cardinality: 0.1099 | Classification: 0.0869 | Classification_binary: 0.1470 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0013 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0726 | Regression_eta: 0.0489 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0461 | LR: 2.16e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.6859s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0664 | max_forward: 0.8187 | max_loss: 0.1464 | max_backward: 1.8154 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.9517 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5875.2 | uss_MB: 4949.5 | child_rss_MB: 5725.9 | allocated_MB: 495.5 | reserved_MB: 27516.0 | inactive_split_MB: 1085.5 +rank1 [2026-10-29 21:52:14,428] INFO : Step 38400/50000 rank1 | Train Loss: 0.8281 | Train Loss Unweighted: 0.4036 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0915 | Classification: 0.0723 | Classification_binary: 0.1160 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1069 | Regression_eta: 0.0012 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.0995 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0536 | Regression_eta: 0.0221 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0417 | LR: 2.14e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.7608s | Phase Times: max_pre_step_barrier: 0.0868 | max_batch_to_device: 0.0062 | max_forward: 0.9937 | max_loss: 0.2500 | max_backward: 1.9690 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5872.2 | uss_MB: 4950.8 | child_rss_MB: 5730.9 | allocated_MB: 496.1 | reserved_MB: 27516.0 | inactive_split_MB: 972.6 +rank1 [2026-10-29 21:57:09,317] INFO : Step 38500/50000 rank1 | Train Loss: 0.9386 | Train Loss Unweighted: 0.4484 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1248 | Classification: 0.1044 | Classification_binary: 0.1389 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1433 | Regression_cos_phi: 0.0664 | Regression_energy: 0.0621 | Regression_eta: 0.0327 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0376 | LR: 2.12e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2041s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.7200 | max_loss: 0.1386 | max_backward: 1.5323 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5868.3 | uss_MB: 4952.0 | child_rss_MB: 5707.9 | allocated_MB: 496.0 | reserved_MB: 27520.0 | inactive_split_MB: 776.0 +rank1 [2026-10-29 22:01:43,605] INFO : Step 38600/50000 rank1 | Train Loss: 0.9227 | Train Loss Unweighted: 0.4436 | Train Loss Components: Auxiliary: 0.1984 | Cardinality: 0.1056 | Classification: 0.0788 | Classification_binary: 0.1660 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0979 | Regression_eta: 0.0013 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0475 | Regression_eta: 0.0297 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0379 | LR: 2.11e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.4283s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8785 | max_loss: 0.1473 | max_backward: 2.0276 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5861.9 | uss_MB: 4951.4 | child_rss_MB: 5677.1 | allocated_MB: 499.0 | reserved_MB: 29592.0 | inactive_split_MB: 827.5 +rank1 [2026-10-29 22:06:19,438] INFO : Step 38700/50000 rank1 | Train Loss: 0.8153 | Train Loss Unweighted: 0.3795 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0947 | Classification: 0.0722 | Classification_binary: 0.1256 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0893 | Regression_eta: 0.0013 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0567 | Regression_eta: 0.0310 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0612 | LR: 2.09e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2095s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7251 | max_loss: 0.2433 | max_backward: 1.8091 | max_optimizer_step: 0.1965 | max_tensorboard_logging: 0.4744 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5858.3 | uss_MB: 4950.6 | child_rss_MB: 5664.7 | allocated_MB: 493.1 | reserved_MB: 29592.0 | inactive_split_MB: 1080.0 +rank1 [2026-10-29 22:11:06,083] INFO : Step 38800/50000 rank1 | Train Loss: 0.8386 | Train Loss Unweighted: 0.4077 | Train Loss Components: Auxiliary: 0.1776 | Cardinality: 0.0905 | Classification: 0.0849 | Classification_binary: 0.1308 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0012 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0546 | Regression_eta: 0.0370 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0848 | LR: 2.07e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1589s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.7346 | max_loss: 0.1426 | max_backward: 1.8320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5857.6 | uss_MB: 4949.0 | child_rss_MB: 5646.6 | allocated_MB: 497.9 | reserved_MB: 29592.0 | inactive_split_MB: 1086.4 +rank1 [2026-10-29 22:15:54,682] INFO : Step 38900/50000 rank1 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1916 | Cardinality: 0.1035 | Classification: 0.0839 | Classification_binary: 0.1479 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0014 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1745 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0475 | Regression_eta: 0.0516 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0956 | LR: 2.05e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.8773s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0144 | max_forward: 2.1610 | max_loss: 0.2966 | max_backward: 2.8203 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4876 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 5858.5 | uss_MB: 4951.7 | child_rss_MB: 5647.2 | allocated_MB: 500.0 | reserved_MB: 29592.0 | inactive_split_MB: 849.6 +rank1 [2026-10-29 22:20:36,723] INFO : Step 39000/50000 rank1 | Train Loss: 0.8505 | Train Loss Unweighted: 0.4196 | Train Loss Components: Auxiliary: 0.1813 | Cardinality: 0.0941 | Classification: 0.0808 | Classification_binary: 0.1368 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0993 | Regression_eta: 0.0015 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1526 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0536 | Regression_eta: 0.0385 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0632 | LR: 2.03e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2849s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7392 | max_loss: 0.2445 | max_backward: 1.8584 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5844.6 | uss_MB: 4939.8 | child_rss_MB: 5638.4 | allocated_MB: 496.8 | reserved_MB: 29592.0 | inactive_split_MB: 1086.0 +rank1 [2026-10-29 22:20:37,629] INFO : Running validation on rank1 +rank1 [2026-10-29 22:20:37,630] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 22:20:37,630] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 22:21:31,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:21:31,758] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 22:21:59,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 22:22:00,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 22:22:01,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 22:22:02,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 22:22:12,144] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 22:22:12,679] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:22:13,173] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 22:22:29,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 22:22:30,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 22:22:31,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 22:22:32,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 22:22:42,771] INFO : Time taken to make predictions on device 1 is: 0.26 min +rank1 [2026-10-29 22:22:43,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 22:22:43,799] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 22:22:59,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 22:23:00,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 22:23:00,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 22:23:01,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 22:23:11,862] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 22:28:30,320] INFO : Step 39100/50000 rank1 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1110 | Classification: 0.0839 | Classification_binary: 0.1198 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1058 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0596 | Regression_eta: 0.0213 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0513 | LR: 2.01e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5498s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7800 | max_loss: 0.2457 | max_backward: 1.9719 | max_optimizer_step: 0.3248 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 7010.9 | uss_MB: 6103.0 | child_rss_MB: 5608.2 | allocated_MB: 497.5 | reserved_MB: 24374.0 | inactive_split_MB: 923.6 +rank1 [2026-10-29 22:33:29,806] INFO : Step 39200/50000 rank1 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4242 | Train Loss Components: Auxiliary: 0.1863 | Cardinality: 0.1210 | Classification: 0.0912 | Classification_binary: 0.1382 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0535 | Regression_eta: 0.0273 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0553 | LR: 2.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2574s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0324 | max_forward: 0.7817 | max_loss: 0.1452 | max_backward: 1.9286 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5940.0 | uss_MB: 5036.4 | child_rss_MB: 5609.8 | allocated_MB: 497.9 | reserved_MB: 26338.0 | inactive_split_MB: 912.4 +rank1 [2026-10-29 22:38:28,991] INFO : Step 39300/50000 rank1 | Train Loss: 0.9298 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1080 | Classification: 0.0882 | Classification_binary: 0.1594 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0951 | Regression_eta: 0.0013 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1750 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0496 | Regression_eta: 0.0410 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0656 | LR: 1.98e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1925s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7922 | max_loss: 0.2572 | max_backward: 1.8558 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 5927.1 | uss_MB: 5036.1 | child_rss_MB: 5613.3 | allocated_MB: 499.4 | reserved_MB: 26338.0 | inactive_split_MB: 1408.0 +rank1 [2026-10-29 22:43:21,471] INFO : Step 39400/50000 rank1 | Train Loss: 0.8421 | Train Loss Unweighted: 0.3964 | Train Loss Components: Auxiliary: 0.1755 | Cardinality: 0.1006 | Classification: 0.0679 | Classification_binary: 0.1346 | HitGrouping: 0.1534 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1262 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0558 | Regression_eta: 0.0238 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0478 | LR: 1.96e-04 | DataLoad Time: 0.0195s | Model Forward Time: 6.2994s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1084 | max_forward: 1.1740 | max_loss: 0.3046 | max_backward: 2.6778 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8613 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5928.1 | uss_MB: 4351.2 | child_rss_MB: 5609.1 | allocated_MB: 495.0 | reserved_MB: 26338.0 | inactive_split_MB: 1095.6 +rank1 [2026-10-29 22:48:17,291] INFO : Step 39500/50000 rank1 | Train Loss: 0.8539 | Train Loss Unweighted: 0.4179 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0879 | Classification: 0.0889 | Classification_binary: 0.1243 | HitGrouping: 0.1565 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1052 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0525 | Regression_eta: 0.0246 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0331 | LR: 1.94e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6165s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7373 | max_loss: 0.1406 | max_backward: 1.8719 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.8502 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5922.7 | uss_MB: 5037.5 | child_rss_MB: 5598.5 | allocated_MB: 492.8 | reserved_MB: 28318.0 | inactive_split_MB: 1078.3 +rank1 [2026-10-29 22:53:41,208] INFO : Step 39600/50000 rank1 | Train Loss: 0.9371 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1161 | Classification: 0.0813 | Classification_binary: 0.1596 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1039 | Regression_eta: 0.0013 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0396 | Regression_energy: 0.0550 | Regression_eta: 0.0424 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0413 | LR: 1.93e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.3258s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0173 | max_forward: 1.4879 | max_loss: 0.2520 | max_backward: 1.9863 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.8346 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5921.1 | uss_MB: 5038.4 | child_rss_MB: 5602.9 | allocated_MB: 496.0 | reserved_MB: 28318.0 | inactive_split_MB: 858.4 +rank1 [2026-10-29 22:58:48,059] INFO : Step 39700/50000 rank1 | Train Loss: 0.9204 | Train Loss Unweighted: 0.4427 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1101 | Classification: 0.0841 | Classification_binary: 0.1567 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0997 | Regression_eta: 0.0013 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0469 | Regression_eta: 0.0256 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0421 | LR: 1.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2756s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.7571 | max_loss: 0.4946 | max_backward: 1.9147 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 5909.3 | uss_MB: 5037.4 | child_rss_MB: 5602.2 | allocated_MB: 497.0 | reserved_MB: 30324.0 | inactive_split_MB: 1175.5 +rank1 [2026-10-29 23:04:38,179] INFO : Step 39800/50000 rank1 | Train Loss: 0.8155 | Train Loss Unweighted: 0.3780 | Train Loss Components: Auxiliary: 0.1674 | Cardinality: 0.0963 | Classification: 0.0643 | Classification_binary: 0.1275 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0428 | Regression_eta: 0.0314 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0502 | LR: 1.89e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0220s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7197 | max_loss: 0.1489 | max_backward: 1.6690 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.4760 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1068 | Memory: rss_MB: 5897.7 | uss_MB: 5034.7 | child_rss_MB: 5594.3 | allocated_MB: 494.4 | reserved_MB: 30324.0 | inactive_split_MB: 897.3 +rank1 [2026-10-29 23:09:43,446] INFO : Step 39900/50000 rank1 | Train Loss: 0.8844 | Train Loss Unweighted: 0.3998 | Train Loss Components: Auxiliary: 0.1871 | Cardinality: 0.1077 | Classification: 0.0758 | Classification_binary: 0.1443 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0883 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1367 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0527 | Regression_eta: 0.0197 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0388 | LR: 1.88e-04 | DataLoad Time: 0.0116s | Model Forward Time: 4.9916s | Phase Times: max_pre_step_barrier: 0.0268 | max_batch_to_device: 0.0727 | max_forward: 1.7016 | max_loss: 0.4968 | max_backward: 2.0829 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.4776 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5898.5 | uss_MB: 5037.5 | child_rss_MB: 5601.6 | allocated_MB: 496.1 | reserved_MB: 30324.0 | inactive_split_MB: 1077.6 +rank1 [2026-10-29 23:14:38,166] INFO : Step 40000/50000 rank1 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1170 | Classification: 0.0871 | Classification_binary: 0.1550 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0489 | Regression_eta: 0.0363 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0655 | LR: 1.86e-04 | DataLoad Time: 0.0130s | Model Forward Time: 5.4271s | Phase Times: max_pre_step_barrier: 0.1084 | max_batch_to_device: 0.0070 | max_forward: 1.1725 | max_loss: 0.2555 | max_backward: 2.4297 | max_optimizer_step: 0.2969 | max_tensorboard_logging: 0.6818 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5889.3 | uss_MB: 5028.2 | child_rss_MB: 5597.1 | allocated_MB: 501.7 | reserved_MB: 30324.0 | inactive_split_MB: 1368.0 +rank1 [2026-10-29 23:14:39,117] INFO : Running validation on rank1 +rank1 [2026-10-29 23:14:39,118] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-29 23:14:39,118] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-29 23:15:31,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:15:32,369] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-29 23:16:03,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-29 23:16:04,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-29 23:16:06,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-29 23:16:09,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-29 23:16:19,054] INFO : Time taken to make predictions on device 1 is: 0.30 min +rank1 [2026-10-29 23:16:28,294] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:16:28,834] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-29 23:16:45,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-29 23:16:46,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-29 23:16:47,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-29 23:16:48,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-29 23:16:58,281] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-29 23:16:58,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-29 23:16:59,027] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-29 23:17:14,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-29 23:17:15,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-29 23:17:16,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-29 23:17:17,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-29 23:17:27,717] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-29 23:22:41,683] INFO : Step 40100/50000 rank1 | Train Loss: 0.8115 | Train Loss Unweighted: 0.3794 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.1028 | Classification: 0.0708 | Classification_binary: 0.1239 | HitGrouping: 0.1460 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0909 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.0993 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0403 | Regression_eta: 0.0348 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0441 | LR: 1.84e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.0759s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.1076 | max_forward: 0.8426 | max_loss: 0.1437 | max_backward: 1.8847 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 7273.7 | uss_MB: 5477.4 | child_rss_MB: 5617.3 | allocated_MB: 493.6 | reserved_MB: 28004.0 | inactive_split_MB: 975.8 +rank1 [2026-10-29 23:27:53,247] INFO : Step 40200/50000 rank1 | Train Loss: 0.8264 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1679 | Cardinality: 0.0984 | Classification: 0.0703 | Classification_binary: 0.1210 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0913 | Regression_eta: 0.0013 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0474 | Regression_eta: 0.0471 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0691 | LR: 1.83e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8936s | Phase Times: max_pre_step_barrier: 0.0270 | max_batch_to_device: 0.0062 | max_forward: 0.8324 | max_loss: 0.1802 | max_backward: 2.0569 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0152 | max_post_step_barrier: 0.0867 | Memory: rss_MB: 5955.5 | uss_MB: 5094.1 | child_rss_MB: 5625.1 | allocated_MB: 497.1 | reserved_MB: 28008.0 | inactive_split_MB: 935.5 +rank1 [2026-10-29 23:33:22,808] INFO : Step 40300/50000 rank1 | Train Loss: 0.9351 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1181 | Classification: 0.0959 | Classification_binary: 0.1652 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0014 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1645 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0419 | Regression_eta: 0.0350 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0588 | LR: 1.81e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.3105s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8191 | max_loss: 0.1494 | max_backward: 1.9817 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 0.9693 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 5955.1 | uss_MB: 5096.4 | child_rss_MB: 5610.7 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 927.5 +rank1 [2026-10-29 23:38:38,678] INFO : Step 40400/50000 rank1 | Train Loss: 0.9379 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1359 | Classification: 0.0771 | Classification_binary: 0.1554 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0944 | Regression_eta: 0.0014 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0375 | Regression_energy: 0.0575 | Regression_eta: 0.0310 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0685 | LR: 1.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6738s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0500 | max_forward: 0.8111 | max_loss: 0.5067 | max_backward: 1.8144 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5951.4 | uss_MB: 5092.9 | child_rss_MB: 5616.0 | allocated_MB: 497.0 | reserved_MB: 28008.0 | inactive_split_MB: 931.6 +rank1 [2026-10-29 23:43:43,800] INFO : Step 40500/50000 rank1 | Train Loss: 0.8053 | Train Loss Unweighted: 0.3782 | Train Loss Components: Auxiliary: 0.1649 | Cardinality: 0.0788 | Classification: 0.0749 | Classification_binary: 0.1246 | HitGrouping: 0.1502 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1070 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0356 | Regression_eta: 0.0271 | Regression_pt: 0.0392 | Regression_sin_phi: 0.0301 | LR: 1.78e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3982s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.8420 | max_loss: 0.1414 | max_backward: 1.5396 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.9215 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5948.4 | uss_MB: 5092.0 | child_rss_MB: 5597.4 | allocated_MB: 494.5 | reserved_MB: 28012.0 | inactive_split_MB: 979.3 +rank1 [2026-10-29 23:48:43,695] INFO : Step 40600/50000 rank1 | Train Loss: 0.8980 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1108 | Classification: 0.0752 | Classification_binary: 0.1417 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0955 | Regression_eta: 0.0015 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0524 | Regression_eta: 0.0460 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0830 | LR: 1.76e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3179s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7855 | max_loss: 0.1488 | max_backward: 1.8742 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 5948.2 | uss_MB: 5094.3 | child_rss_MB: 5593.1 | allocated_MB: 499.7 | reserved_MB: 28012.0 | inactive_split_MB: 1102.0 +rank1 [2026-10-29 23:53:39,581] INFO : Step 40700/50000 rank1 | Train Loss: 0.9220 | Train Loss Unweighted: 0.4304 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1148 | Classification: 0.0767 | Classification_binary: 0.1645 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1591 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0523 | Regression_eta: 0.0458 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0731 | LR: 1.75e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5463s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.8091 | max_loss: 0.2475 | max_backward: 2.0011 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.5972 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5949.6 | uss_MB: 5096.1 | child_rss_MB: 5593.6 | allocated_MB: 498.9 | reserved_MB: 28012.0 | inactive_split_MB: 776.0 +rank1 [2026-10-30 00:01:00,050] INFO : Step 40800/50000 rank1 | Train Loss: 0.8642 | Train Loss Unweighted: 0.4082 | Train Loss Components: Auxiliary: 0.1815 | Cardinality: 0.1013 | Classification: 0.0834 | Classification_binary: 0.1260 | HitGrouping: 0.1507 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1055 | Regression_cos_phi: 0.0253 | Regression_energy: 0.0522 | Regression_eta: 0.0304 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0414 | LR: 1.73e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.1146s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7033 | max_loss: 0.1373 | max_backward: 1.5042 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1068 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5935.0 | uss_MB: 4327.0 | child_rss_MB: 5586.6 | allocated_MB: 493.1 | reserved_MB: 28012.0 | inactive_split_MB: 897.3 +rank1 [2026-10-30 00:06:14,592] INFO : Step 40900/50000 rank1 | Train Loss: 0.8749 | Train Loss Unweighted: 0.4038 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1070 | Classification: 0.0854 | Classification_binary: 0.1300 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0923 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1562 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0562 | Regression_eta: 0.0466 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0648 | LR: 1.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5722s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7515 | max_loss: 0.2142 | max_backward: 1.7950 | max_optimizer_step: 0.4960 | max_tensorboard_logging: 0.9322 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5926.9 | uss_MB: 5094.2 | child_rss_MB: 5563.3 | allocated_MB: 497.9 | reserved_MB: 30012.0 | inactive_split_MB: 893.0 +rank1 [2026-10-30 00:11:07,452] INFO : Step 41000/50000 rank1 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.0956 | Classification: 0.0893 | Classification_binary: 0.1497 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0014 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1373 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0411 | Regression_eta: 0.0435 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0408 | LR: 1.70e-04 | DataLoad Time: 0.0108s | Model Forward Time: 5.1015s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0071 | max_forward: 1.1963 | max_loss: 0.1519 | max_backward: 2.0474 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.0118 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 5918.6 | uss_MB: 5085.3 | child_rss_MB: 5547.4 | allocated_MB: 498.7 | reserved_MB: 30012.0 | inactive_split_MB: 1108.0 +rank1 [2026-10-30 00:11:08,373] INFO : Running validation on rank1 +rank1 [2026-10-30 00:11:08,373] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 00:11:08,373] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 00:11:59,246] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:12:00,028] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 00:12:25,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 00:12:26,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 00:12:27,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 00:12:28,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 00:12:38,994] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 00:12:39,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:12:40,281] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 00:12:56,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 00:12:57,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 00:12:58,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 00:12:59,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 00:13:09,605] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 00:13:09,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 00:13:10,648] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 00:13:26,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 00:13:27,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 00:13:28,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 00:13:29,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 00:13:39,534] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 00:19:15,373] INFO : Step 41100/50000 rank1 | Train Loss: 0.9430 | Train Loss Unweighted: 0.4319 | Train Loss Components: Auxiliary: 0.1995 | Cardinality: 0.1330 | Classification: 0.0721 | Classification_binary: 0.1579 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0013 | Regression_pt: 0.0998 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0648 | Regression_eta: 0.0382 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0609 | LR: 1.69e-04 | DataLoad Time: 0.0153s | Model Forward Time: 6.3658s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8221 | max_loss: 0.4800 | max_backward: 2.2615 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7242.0 | uss_MB: 5759.2 | child_rss_MB: 5549.6 | allocated_MB: 497.6 | reserved_MB: 23400.0 | inactive_split_MB: 991.8 +rank1 [2026-10-30 00:24:27,079] INFO : Step 41200/50000 rank1 | Train Loss: 0.8544 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1097 | Classification: 0.0734 | Classification_binary: 0.1131 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1015 | Regression_eta: 0.0015 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0707 | Regression_eta: 0.0259 | Regression_pt: 0.0781 | Regression_sin_phi: 0.0537 | LR: 1.67e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2544s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7030 | max_loss: 0.2408 | max_backward: 1.7750 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5316 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6039.9 | uss_MB: 5208.8 | child_rss_MB: 5554.6 | allocated_MB: 496.1 | reserved_MB: 27308.0 | inactive_split_MB: 984.0 +rank1 [2026-10-30 00:29:20,952] INFO : Step 41300/50000 rank1 | Train Loss: 0.8221 | Train Loss Unweighted: 0.3852 | Train Loss Components: Auxiliary: 0.1744 | Cardinality: 0.0906 | Classification: 0.0754 | Classification_binary: 0.1369 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0849 | Regression_eta: 0.0012 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1272 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0390 | Regression_eta: 0.0242 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0426 | LR: 1.66e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4704s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8083 | max_loss: 0.1794 | max_backward: 1.9993 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.5906 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6040.0 | uss_MB: 5213.3 | child_rss_MB: 5551.2 | allocated_MB: 497.0 | reserved_MB: 27308.0 | inactive_split_MB: 1229.0 +rank1 [2026-10-30 00:34:25,620] INFO : Step 41400/50000 rank1 | Train Loss: 0.9216 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1171 | Classification: 0.0901 | Classification_binary: 0.1523 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1010 | Regression_eta: 0.0011 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0528 | Regression_eta: 0.0347 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0411 | LR: 1.64e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6363s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0065 | max_forward: 0.7831 | max_loss: 0.4937 | max_backward: 1.9271 | max_optimizer_step: 0.3071 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6033.3 | uss_MB: 5208.0 | child_rss_MB: 5553.7 | allocated_MB: 498.4 | reserved_MB: 29288.0 | inactive_split_MB: 1180.4 +rank1 [2026-10-30 00:39:29,959] INFO : Step 41500/50000 rank1 | Train Loss: 0.8584 | Train Loss Unweighted: 0.4111 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1030 | Classification: 0.0769 | Classification_binary: 0.1210 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0359 | Regression_energy: 0.0509 | Regression_eta: 0.0342 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0586 | LR: 1.63e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.0420s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.1908 | max_loss: 0.3017 | max_backward: 3.1310 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 6009.7 | uss_MB: 5150.0 | child_rss_MB: 5552.7 | allocated_MB: 494.5 | reserved_MB: 29288.0 | inactive_split_MB: 976.9 +rank1 [2026-10-30 00:44:19,403] INFO : Step 41600/50000 rank1 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4026 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1032 | Classification: 0.0762 | Classification_binary: 0.1246 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0992 | Regression_eta: 0.0013 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0580 | Regression_eta: 0.0334 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0542 | LR: 1.61e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9946s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0065 | max_forward: 0.7279 | max_loss: 0.2339 | max_backward: 2.5338 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.4866 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1404 | Memory: rss_MB: 6001.6 | uss_MB: 5206.5 | child_rss_MB: 5546.1 | allocated_MB: 496.4 | reserved_MB: 29288.0 | inactive_split_MB: 761.6 +rank1 [2026-10-30 00:49:10,876] INFO : Step 41700/50000 rank1 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1941 | Cardinality: 0.1112 | Classification: 0.0767 | Classification_binary: 0.1493 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0988 | Regression_eta: 0.0013 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0442 | Regression_eta: 0.0457 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0749 | LR: 1.60e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5071s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7998 | max_loss: 0.1827 | max_backward: 1.9774 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6004.5 | uss_MB: 5210.8 | child_rss_MB: 5548.7 | allocated_MB: 497.0 | reserved_MB: 29288.0 | inactive_split_MB: 1220.0 +rank1 [2026-10-30 00:53:58,971] INFO : Step 41800/50000 rank1 | Train Loss: 0.9125 | Train Loss Unweighted: 0.4285 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.1148 | Classification: 0.0784 | Classification_binary: 0.1537 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1581 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0513 | Regression_eta: 0.0370 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0442 | LR: 1.58e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8252s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7952 | max_loss: 0.1582 | max_backward: 1.8899 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9392 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1143 | Memory: rss_MB: 5984.7 | uss_MB: 5200.7 | child_rss_MB: 5542.4 | allocated_MB: 499.3 | reserved_MB: 31288.0 | inactive_split_MB: 932.9 +rank1 [2026-10-30 00:59:00,784] INFO : Step 41900/50000 rank1 | Train Loss: 0.8303 | Train Loss Unweighted: 0.3927 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1132 | Classification: 0.0782 | Classification_binary: 0.1167 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0010 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1094 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0568 | Regression_eta: 0.0347 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0479 | LR: 1.57e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1519s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7413 | max_loss: 0.1970 | max_backward: 1.7614 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5208 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5992.5 | uss_MB: 5207.2 | child_rss_MB: 5522.2 | allocated_MB: 497.1 | reserved_MB: 31288.0 | inactive_split_MB: 1188.0 +rank1 [2026-10-30 01:03:51,615] INFO : Step 42000/50000 rank1 | Train Loss: 0.8724 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1847 | Cardinality: 0.0893 | Classification: 0.0874 | Classification_binary: 0.1377 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0958 | Regression_eta: 0.0012 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1266 | Regression_cos_phi: 0.0566 | Regression_energy: 0.0615 | Regression_eta: 0.0319 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0586 | LR: 1.56e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7570s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1147 | max_forward: 0.8308 | max_loss: 0.1392 | max_backward: 1.8336 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5996.6 | uss_MB: 5209.4 | child_rss_MB: 5540.2 | allocated_MB: 496.3 | reserved_MB: 31288.0 | inactive_split_MB: 930.5 +rank1 [2026-10-30 01:03:52,596] INFO : Running validation on rank1 +rank1 [2026-10-30 01:03:52,596] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 01:03:52,597] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 01:04:46,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:04:46,670] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 01:05:13,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 01:05:14,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 01:05:15,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 01:05:16,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 01:05:26,997] INFO : Time taken to make predictions on device 1 is: 0.27 min +rank1 [2026-10-30 01:05:27,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:05:27,884] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 01:05:43,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 01:05:44,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 01:05:45,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 01:05:46,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 01:05:56,799] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 01:05:57,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:05:57,664] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 01:06:13,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 01:06:14,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 01:06:15,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 01:06:16,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 01:06:26,752] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 01:11:42,850] INFO : Step 42100/50000 rank1 | Train Loss: 0.9390 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1200 | Classification: 0.0884 | Classification_binary: 0.1619 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0926 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0520 | Regression_eta: 0.0282 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0467 | LR: 1.54e-04 | DataLoad Time: 0.0178s | Model Forward Time: 6.0853s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0976 | max_forward: 1.0189 | max_loss: 0.3859 | max_backward: 2.5922 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 1.0368 | max_tensorboard_flush: 0.0149 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7501.8 | uss_MB: 4866.0 | child_rss_MB: 5544.9 | allocated_MB: 497.0 | reserved_MB: 25850.0 | inactive_split_MB: 892.8 +rank1 [2026-10-30 01:16:53,955] INFO : Step 42200/50000 rank1 | Train Loss: 0.9057 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1144 | Classification: 0.0879 | Classification_binary: 0.1381 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1320 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0504 | Regression_eta: 0.0429 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0664 | LR: 1.53e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.1526s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7457 | max_loss: 0.1999 | max_backward: 1.6824 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.5916 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6096.0 | uss_MB: 5306.7 | child_rss_MB: 5537.7 | allocated_MB: 497.3 | reserved_MB: 27866.0 | inactive_split_MB: 959.8 +rank1 [2026-10-30 01:18:06,040] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank1 [2026-10-30 01:18:06,040] INFO : ResumableSampler physical:train loading state: start_index=0 +rank1 [2026-10-30 01:21:57,450] INFO : Step 42300/50000 rank1 | Train Loss: 0.9052 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1933 | Cardinality: 0.1127 | Classification: 0.0787 | Classification_binary: 0.1569 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0868 | Regression_eta: 0.0013 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1633 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0394 | Regression_eta: 0.0345 | Regression_pt: 0.0426 | Regression_sin_phi: 0.0495 | LR: 1.52e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.1175s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.9764 | max_loss: 0.1904 | max_backward: 2.7671 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.9997 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6118.1 | uss_MB: 5325.9 | child_rss_MB: 5544.6 | allocated_MB: 499.1 | reserved_MB: 27872.0 | inactive_split_MB: 1033.6 +rank1 [2026-10-30 01:26:55,612] INFO : Step 42400/50000 rank1 | Train Loss: 0.9358 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.1976 | Cardinality: 0.1220 | Classification: 0.0815 | Classification_binary: 0.1533 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1394 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0626 | Regression_eta: 0.0317 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0474 | LR: 1.50e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8872s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8701 | max_loss: 0.1465 | max_backward: 1.9416 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.8192 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6099.3 | uss_MB: 5308.3 | child_rss_MB: 5546.3 | allocated_MB: 499.8 | reserved_MB: 29908.0 | inactive_split_MB: 939.2 +rank1 [2026-10-30 01:31:49,012] INFO : Step 42500/50000 rank1 | Train Loss: 0.8126 | Train Loss Unweighted: 0.3791 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0975 | Classification: 0.0765 | Classification_binary: 0.1172 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0918 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1133 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0455 | Regression_eta: 0.0289 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0561 | LR: 1.49e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5925s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0858 | max_forward: 0.8156 | max_loss: 0.1407 | max_backward: 1.8570 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7997 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6101.2 | uss_MB: 5308.6 | child_rss_MB: 5538.8 | allocated_MB: 497.5 | reserved_MB: 29908.0 | inactive_split_MB: 943.4 +rank1 [2026-10-30 01:36:44,535] INFO : Step 42600/50000 rank1 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0772 | Classification_binary: 0.1347 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0012 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0514 | Regression_eta: 0.0357 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0505 | LR: 1.48e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0562s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7519 | max_loss: 0.1393 | max_backward: 1.8021 | max_optimizer_step: 0.3023 | max_tensorboard_logging: 0.7973 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6101.5 | uss_MB: 4501.2 | child_rss_MB: 5544.2 | allocated_MB: 497.0 | reserved_MB: 29908.0 | inactive_split_MB: 877.2 +rank1 [2026-10-30 01:41:48,353] INFO : Step 42700/50000 rank1 | Train Loss: 0.9129 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1178 | Classification: 0.0751 | Classification_binary: 0.1632 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0858 | Regression_eta: 0.0014 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0371 | Regression_eta: 0.0191 | Regression_pt: 0.0407 | Regression_sin_phi: 0.0358 | LR: 1.47e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6152s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 1.0869 | max_loss: 0.1547 | max_backward: 1.6854 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7949 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6101.6 | uss_MB: 5308.9 | child_rss_MB: 5545.8 | allocated_MB: 497.3 | reserved_MB: 29908.0 | inactive_split_MB: 953.4 +rank1 [2026-10-30 01:46:39,116] INFO : Step 42800/50000 rank1 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1179 | Classification: 0.0767 | Classification_binary: 0.1362 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0013 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1239 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0476 | Regression_eta: 0.0386 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0680 | LR: 1.45e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3529s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8438 | max_loss: 0.1478 | max_backward: 1.6710 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.6821 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6089.4 | uss_MB: 5295.4 | child_rss_MB: 5546.6 | allocated_MB: 499.0 | reserved_MB: 30126.0 | inactive_split_MB: 875.7 +rank1 [2026-10-30 01:51:40,828] INFO : Step 42900/50000 rank1 | Train Loss: 0.7857 | Train Loss Unweighted: 0.3758 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0951 | Classification: 0.0820 | Classification_binary: 0.1113 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0898 | Regression_eta: 0.0010 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1195 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0579 | Regression_eta: 0.0254 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0307 | LR: 1.44e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9237s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8792 | max_loss: 0.1461 | max_backward: 1.9928 | max_optimizer_step: 0.1903 | max_tensorboard_logging: 0.9695 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6098.2 | uss_MB: 5308.5 | child_rss_MB: 5544.6 | allocated_MB: 496.5 | reserved_MB: 30126.0 | inactive_split_MB: 951.8 +rank1 [2026-10-30 01:56:37,111] INFO : Step 43000/50000 rank1 | Train Loss: 0.8899 | Train Loss Unweighted: 0.4207 | Train Loss Components: Auxiliary: 0.1911 | Cardinality: 0.1035 | Classification: 0.0847 | Classification_binary: 0.1529 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0422 | Regression_eta: 0.0258 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0595 | LR: 1.43e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7379s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1951 | max_backward: 1.9694 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.8418 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6086.1 | uss_MB: 5311.0 | child_rss_MB: 5508.6 | allocated_MB: 495.5 | reserved_MB: 30126.0 | inactive_split_MB: 1120.4 +rank1 [2026-10-30 01:56:38,136] INFO : Running validation on rank1 +rank1 [2026-10-30 01:56:38,136] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 01:56:38,136] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 01:57:33,182] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:57:33,675] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 01:57:59,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 01:58:00,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 01:58:01,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 01:58:02,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 01:58:12,628] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 01:58:13,037] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:58:13,511] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 01:58:29,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 01:58:30,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 01:58:31,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 01:58:32,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 01:58:42,837] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 01:58:43,304] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 01:58:43,756] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 01:58:59,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 01:59:00,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 01:59:00,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 01:59:01,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 01:59:11,813] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-30 02:04:24,631] INFO : Step 43100/50000 rank1 | Train Loss: 0.8965 | Train Loss Unweighted: 0.4137 | Train Loss Components: Auxiliary: 0.1901 | Cardinality: 0.1074 | Classification: 0.0705 | Classification_binary: 0.1472 | HitGrouping: 0.1435 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1350 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0406 | Regression_eta: 0.0448 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0697 | LR: 1.42e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2206s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.2396 | max_backward: 1.9954 | max_optimizer_step: 0.3209 | max_tensorboard_logging: 1.0803 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7631.1 | uss_MB: 6853.7 | child_rss_MB: 5519.2 | allocated_MB: 496.0 | reserved_MB: 26194.0 | inactive_split_MB: 843.4 +rank1 [2026-10-30 02:09:06,731] INFO : Step 43200/50000 rank1 | Train Loss: 0.8164 | Train Loss Unweighted: 0.3744 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0957 | Classification: 0.0763 | Classification_binary: 0.1197 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1184 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0590 | Regression_eta: 0.0426 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0606 | LR: 1.40e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4243s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7823 | max_loss: 0.4822 | max_backward: 1.6766 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6123.7 | uss_MB: 5349.6 | child_rss_MB: 5511.0 | allocated_MB: 495.8 | reserved_MB: 26686.0 | inactive_split_MB: 848.9 +rank1 [2026-10-30 02:13:48,743] INFO : Step 43300/50000 rank1 | Train Loss: 0.9151 | Train Loss Unweighted: 0.4260 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1202 | Classification: 0.0906 | Classification_binary: 0.1467 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0013 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0467 | Regression_eta: 0.0299 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0307 | LR: 1.39e-04 | DataLoad Time: 0.0123s | Model Forward Time: 4.7347s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0155 | max_forward: 0.8938 | max_loss: 0.1464 | max_backward: 2.1010 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.5477 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1243 | Memory: rss_MB: 6097.4 | uss_MB: 5352.9 | child_rss_MB: 5427.7 | allocated_MB: 497.9 | reserved_MB: 28278.0 | inactive_split_MB: 1309.2 +rank1 [2026-10-30 02:18:30,112] INFO : Step 43400/50000 rank1 | Train Loss: 0.9723 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2082 | Cardinality: 0.1222 | Classification: 0.0880 | Classification_binary: 0.1684 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0015 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0499 | Regression_eta: 0.0328 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0571 | LR: 1.38e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4346s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.5043 | max_backward: 2.0661 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.4824 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6077.9 | uss_MB: 5350.3 | child_rss_MB: 5335.1 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 851.9 +rank1 [2026-10-30 02:23:12,892] INFO : Step 43500/50000 rank1 | Train Loss: 0.8402 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.0947 | Classification: 0.0776 | Classification_binary: 0.1278 | HitGrouping: 0.1501 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0999 | Regression_eta: 0.0012 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1244 | Regression_cos_phi: 0.0346 | Regression_energy: 0.0532 | Regression_eta: 0.0322 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0262 | LR: 1.37e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8627s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8086 | max_loss: 0.1489 | max_backward: 2.0887 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.7978 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1094 | Memory: rss_MB: 6077.6 | uss_MB: 5352.8 | child_rss_MB: 5340.6 | allocated_MB: 498.6 | reserved_MB: 28278.0 | inactive_split_MB: 909.7 +rank1 [2026-10-30 02:28:09,613] INFO : Step 43600/50000 rank1 | Train Loss: 0.7985 | Train Loss Unweighted: 0.3653 | Train Loss Components: Auxiliary: 0.1621 | Cardinality: 0.0887 | Classification: 0.0728 | Classification_binary: 0.1184 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0851 | Regression_eta: 0.0013 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1141 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0446 | Regression_eta: 0.0539 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0499 | LR: 1.36e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.2523s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0170 | max_forward: 1.2777 | max_loss: 0.2064 | max_backward: 1.9835 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 1.1084 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6044.0 | uss_MB: 4957.3 | child_rss_MB: 5260.0 | allocated_MB: 493.5 | reserved_MB: 28278.0 | inactive_split_MB: 1313.2 +rank1 [2026-10-30 02:33:07,295] INFO : Step 43700/50000 rank1 | Train Loss: 0.8599 | Train Loss Unweighted: 0.3884 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1089 | Classification: 0.0797 | Classification_binary: 0.1434 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0812 | Regression_eta: 0.0013 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1650 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0436 | Regression_eta: 0.0375 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0808 | LR: 1.35e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6048s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7902 | max_loss: 0.1752 | max_backward: 1.8774 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8044 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6046.0 | uss_MB: 5350.4 | child_rss_MB: 5241.5 | allocated_MB: 498.0 | reserved_MB: 28278.0 | inactive_split_MB: 1046.0 +rank1 [2026-10-30 02:37:45,726] INFO : Step 43800/50000 rank1 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1875 | Cardinality: 0.1049 | Classification: 0.0814 | Classification_binary: 0.1423 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0907 | Regression_eta: 0.0012 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1323 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0512 | Regression_eta: 0.0335 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0541 | LR: 1.34e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5751s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7940 | max_loss: 0.4805 | max_backward: 1.8159 | max_optimizer_step: 0.2941 | max_tensorboard_logging: 0.7850 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6040.0 | uss_MB: 5349.3 | child_rss_MB: 5218.8 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 839.4 +rank1 [2026-10-30 02:42:41,736] INFO : Step 43900/50000 rank1 | Train Loss: 0.7987 | Train Loss Unweighted: 0.3885 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.0999 | Classification: 0.0753 | Classification_binary: 0.1166 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0968 | Regression_eta: 0.0010 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1314 | Regression_cos_phi: 0.0361 | Regression_energy: 0.0536 | Regression_eta: 0.0276 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0479 | LR: 1.33e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5962s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0846 | max_forward: 0.8891 | max_loss: 0.1768 | max_backward: 1.7357 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7516 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6030.6 | uss_MB: 5347.7 | child_rss_MB: 5179.6 | allocated_MB: 494.5 | reserved_MB: 28278.0 | inactive_split_MB: 1129.5 +rank1 [2026-10-30 02:47:33,826] INFO : Step 44000/50000 rank1 | Train Loss: 0.8263 | Train Loss Unweighted: 0.3926 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.1027 | Classification: 0.0840 | Classification_binary: 0.1318 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0868 | Regression_eta: 0.0011 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0409 | Regression_eta: 0.0238 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0399 | LR: 1.32e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4530s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8816 | max_loss: 0.1426 | max_backward: 1.9616 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.4846 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6020.6 | uss_MB: 5352.2 | child_rss_MB: 5134.9 | allocated_MB: 496.3 | reserved_MB: 28762.0 | inactive_split_MB: 1307.2 +rank1 [2026-10-30 02:47:34,860] INFO : Running validation on rank1 +rank1 [2026-10-30 02:47:34,860] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 02:47:34,860] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 02:48:28,154] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 02:48:28,670] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 02:48:55,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 02:48:56,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 02:48:57,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 02:48:59,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 02:49:09,022] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 02:49:09,617] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 02:49:10,104] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 02:49:26,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 02:49:27,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 02:49:28,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 02:49:29,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 02:49:39,201] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 02:49:39,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 02:49:40,185] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 02:49:55,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 02:49:56,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 02:49:57,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 02:49:58,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 02:50:08,329] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-30 02:55:20,174] INFO : Step 44100/50000 rank1 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1986 | Cardinality: 0.1334 | Classification: 0.0764 | Classification_binary: 0.1516 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0320 | Regression_energy: 0.0641 | Regression_eta: 0.0456 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0402 | LR: 1.31e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.1227s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0079 | max_forward: 0.8221 | max_loss: 0.1579 | max_backward: 2.0192 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 1.0889 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7539.1 | uss_MB: 6869.4 | child_rss_MB: 5111.3 | allocated_MB: 498.6 | reserved_MB: 25920.0 | inactive_split_MB: 1073.8 +rank1 [2026-10-30 03:00:12,362] INFO : Step 44200/50000 rank1 | Train Loss: 0.9018 | Train Loss Unweighted: 0.4273 | Train Loss Components: Auxiliary: 0.1909 | Cardinality: 0.1109 | Classification: 0.0847 | Classification_binary: 0.1354 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0012 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0503 | Regression_eta: 0.0439 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0816 | LR: 1.30e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3582s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.1013 | max_forward: 0.8608 | max_loss: 0.1485 | max_backward: 1.6010 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 6119.2 | uss_MB: 5452.1 | child_rss_MB: 5083.8 | allocated_MB: 496.1 | reserved_MB: 27956.0 | inactive_split_MB: 1248.1 +rank1 [2026-10-30 03:04:47,145] INFO : Step 44300/50000 rank1 | Train Loss: 0.8239 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1722 | Cardinality: 0.1007 | Classification: 0.0789 | Classification_binary: 0.1118 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1099 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0531 | Regression_eta: 0.0212 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0405 | LR: 1.29e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3183s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8120 | max_loss: 0.1377 | max_backward: 1.4700 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9055 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6110.2 | uss_MB: 5452.8 | child_rss_MB: 5031.6 | allocated_MB: 492.8 | reserved_MB: 27956.0 | inactive_split_MB: 1035.8 +rank1 [2026-10-30 03:09:18,055] INFO : Step 44400/50000 rank1 | Train Loss: 0.8683 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.0849 | Classification: 0.0836 | Classification_binary: 0.1500 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0531 | Regression_eta: 0.0308 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0465 | LR: 1.28e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.9271s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8907 | max_loss: 0.1476 | max_backward: 2.0623 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6106.5 | uss_MB: 5454.6 | child_rss_MB: 5002.7 | allocated_MB: 499.5 | reserved_MB: 27956.0 | inactive_split_MB: 926.0 +rank1 [2026-10-30 03:14:01,913] INFO : Step 44500/50000 rank1 | Train Loss: 0.8672 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.0989 | Classification: 0.0738 | Classification_binary: 0.1455 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0534 | Regression_eta: 0.0257 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0391 | LR: 1.27e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2935s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0597 | max_forward: 0.8669 | max_loss: 0.1547 | max_backward: 1.7327 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.5628 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6105.7 | uss_MB: 5452.5 | child_rss_MB: 5004.4 | allocated_MB: 498.3 | reserved_MB: 27956.0 | inactive_split_MB: 860.5 +rank1 [2026-10-30 03:18:59,362] INFO : Step 44600/50000 rank1 | Train Loss: 0.7900 | Train Loss Unweighted: 0.3743 | Train Loss Components: Auxiliary: 0.1688 | Cardinality: 0.0956 | Classification: 0.0724 | Classification_binary: 0.1157 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0919 | Regression_eta: 0.0010 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0394 | Regression_eta: 0.0329 | Regression_pt: 0.0421 | Regression_sin_phi: 0.0496 | LR: 1.26e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5453s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0745 | max_forward: 0.7797 | max_loss: 0.1409 | max_backward: 1.9334 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6022 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6098.6 | uss_MB: 5447.3 | child_rss_MB: 4994.6 | allocated_MB: 494.5 | reserved_MB: 27958.0 | inactive_split_MB: 927.6 +rank1 [2026-10-30 03:23:44,789] INFO : Step 44700/50000 rank1 | Train Loss: 0.8669 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1878 | Cardinality: 0.1160 | Classification: 0.0768 | Classification_binary: 0.1380 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0863 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0363 | Regression_energy: 0.0432 | Regression_eta: 0.0323 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0533 | LR: 1.25e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4959s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8629 | max_loss: 0.1449 | max_backward: 2.0281 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.5242 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6099.2 | uss_MB: 5434.3 | child_rss_MB: 5003.0 | allocated_MB: 498.4 | reserved_MB: 27958.0 | inactive_split_MB: 781.6 +rank1 [2026-10-30 03:28:42,239] INFO : Step 44800/50000 rank1 | Train Loss: 0.8585 | Train Loss Unweighted: 0.4003 | Train Loss Components: Auxiliary: 0.1869 | Cardinality: 0.1024 | Classification: 0.0769 | Classification_binary: 0.1550 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0823 | Regression_eta: 0.0011 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0370 | Regression_eta: 0.0261 | Regression_pt: 0.0400 | Regression_sin_phi: 0.0507 | LR: 1.24e-04 | DataLoad Time: 0.0140s | Model Forward Time: 4.8067s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0072 | max_forward: 0.7883 | max_loss: 0.5033 | max_backward: 1.9727 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6090.2 | uss_MB: 5455.2 | child_rss_MB: 4956.4 | allocated_MB: 496.5 | reserved_MB: 27958.0 | inactive_split_MB: 778.0 +rank1 [2026-10-30 03:33:12,493] INFO : Step 44900/50000 rank1 | Train Loss: 0.8358 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.1024 | Classification: 0.0797 | Classification_binary: 0.1294 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0859 | Regression_eta: 0.0011 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0289 | Classification_binary: 0.1259 | Regression_cos_phi: 0.0300 | Regression_energy: 0.0529 | Regression_eta: 0.0203 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0537 | LR: 1.23e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3690s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7360 | max_loss: 0.1482 | max_backward: 1.6886 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7825 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6084.1 | uss_MB: 5452.9 | child_rss_MB: 4956.9 | allocated_MB: 494.3 | reserved_MB: 27958.0 | inactive_split_MB: 922.0 +rank1 [2026-10-30 03:37:54,705] INFO : Step 45000/50000 rank1 | Train Loss: 0.8678 | Train Loss Unweighted: 0.3957 | Train Loss Components: Auxiliary: 0.1790 | Cardinality: 0.1036 | Classification: 0.0801 | Classification_binary: 0.1237 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0941 | Regression_eta: 0.0013 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0567 | Regression_eta: 0.0290 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0432 | LR: 1.22e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.5700s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7189 | max_loss: 0.2183 | max_backward: 1.8172 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.8812 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6058.9 | uss_MB: 5454.9 | child_rss_MB: 4844.5 | allocated_MB: 495.0 | reserved_MB: 27960.0 | inactive_split_MB: 978.5 +rank1 [2026-10-30 03:37:55,713] INFO : Running validation on rank1 +rank1 [2026-10-30 03:37:55,713] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 03:37:55,713] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 03:38:46,292] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 03:38:46,807] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 03:39:14,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 03:39:15,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 03:39:17,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 03:39:19,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 03:39:29,350] INFO : Time taken to make predictions on device 1 is: 0.28 min +rank1 [2026-10-30 03:39:29,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 03:39:30,322] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 03:39:46,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 03:39:47,263] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 03:39:48,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 03:39:49,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 03:39:59,222] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 03:40:00,144] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 03:40:00,673] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 03:40:16,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 03:40:17,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 03:40:18,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 03:40:19,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 03:40:29,236] INFO : Time taken to make predictions on device 1 is: 0.24 min +rank1 [2026-10-30 03:45:39,355] INFO : Step 45100/50000 rank1 | Train Loss: 0.8712 | Train Loss Unweighted: 0.4129 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0997 | Classification: 0.0837 | Classification_binary: 0.1445 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0910 | Regression_eta: 0.0012 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0470 | Regression_eta: 0.0483 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0407 | LR: 1.21e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.1919s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8448 | max_loss: 0.1523 | max_backward: 2.1445 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 1.0003 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7489.1 | uss_MB: 6883.8 | child_rss_MB: 4866.9 | allocated_MB: 499.4 | reserved_MB: 24746.0 | inactive_split_MB: 784.5 +rank1 [2026-10-30 03:50:46,386] INFO : Step 45200/50000 rank1 | Train Loss: 0.9130 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1058 | Classification: 0.0776 | Classification_binary: 0.1464 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0963 | Regression_eta: 0.0015 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0567 | Regression_eta: 0.0437 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0727 | LR: 1.20e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.5422s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1574 | max_backward: 1.6440 | max_optimizer_step: 0.2111 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1112 | Memory: rss_MB: 6132.8 | uss_MB: 4597.7 | child_rss_MB: 4866.1 | allocated_MB: 499.7 | reserved_MB: 26730.0 | inactive_split_MB: 1073.5 +rank1 [2026-10-30 03:55:44,003] INFO : Step 45300/50000 rank1 | Train Loss: 0.8299 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.1126 | Classification: 0.0744 | Classification_binary: 0.1207 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.0991 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0538 | Regression_eta: 0.0228 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0418 | LR: 1.19e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.1830s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0827 | max_forward: 0.7966 | max_loss: 0.1432 | max_backward: 1.5141 | max_optimizer_step: 0.2646 | max_tensorboard_logging: 0.6448 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6128.3 | uss_MB: 5538.9 | child_rss_MB: 4864.4 | allocated_MB: 495.4 | reserved_MB: 27170.0 | inactive_split_MB: 881.8 +rank1 [2026-10-30 04:00:47,341] INFO : Step 45400/50000 rank1 | Train Loss: 0.8255 | Train Loss Unweighted: 0.4011 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.0984 | Classification: 0.0907 | Classification_binary: 0.1338 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0870 | Regression_eta: 0.0011 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1229 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0419 | Regression_eta: 0.0215 | Regression_pt: 0.0451 | Regression_sin_phi: 0.0342 | LR: 1.19e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.3468s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7846 | max_loss: 0.1934 | max_backward: 1.9857 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 0.7880 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6124.9 | uss_MB: 4519.9 | child_rss_MB: 4862.5 | allocated_MB: 499.2 | reserved_MB: 27170.0 | inactive_split_MB: 989.3 +rank1 [2026-10-30 04:05:44,853] INFO : Step 45500/50000 rank1 | Train Loss: 0.8945 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1090 | Classification: 0.0825 | Classification_binary: 0.1567 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0012 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1457 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0451 | Regression_eta: 0.0266 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0429 | LR: 1.18e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4582s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8686 | max_loss: 0.1508 | max_backward: 1.9581 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.4909 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6130.5 | uss_MB: 5543.1 | child_rss_MB: 4864.9 | allocated_MB: 498.4 | reserved_MB: 28766.0 | inactive_split_MB: 883.2 +rank1 [2026-10-30 04:10:39,744] INFO : Step 45600/50000 rank1 | Train Loss: 0.8704 | Train Loss Unweighted: 0.4121 | Train Loss Components: Auxiliary: 0.1840 | Cardinality: 0.1126 | Classification: 0.0802 | Classification_binary: 0.1324 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0983 | Regression_eta: 0.0010 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1040 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0377 | Regression_eta: 0.0244 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0360 | LR: 1.17e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8859s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0163 | max_forward: 0.8161 | max_loss: 0.2729 | max_backward: 2.1846 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6122.7 | uss_MB: 5488.3 | child_rss_MB: 4871.5 | allocated_MB: 495.6 | reserved_MB: 28766.0 | inactive_split_MB: 1080.4 +rank1 [2026-10-30 04:15:37,195] INFO : Step 45700/50000 rank1 | Train Loss: 0.8006 | Train Loss Unweighted: 0.3784 | Train Loss Components: Auxiliary: 0.1681 | Cardinality: 0.0873 | Classification: 0.0773 | Classification_binary: 0.1111 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0012 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.0904 | Regression_cos_phi: 0.0341 | Regression_energy: 0.0332 | Regression_eta: 0.0284 | Regression_pt: 0.0367 | Regression_sin_phi: 0.0392 | LR: 1.16e-04 | DataLoad Time: 0.0107s | Model Forward Time: 4.8459s | Phase Times: max_pre_step_barrier: 0.0327 | max_batch_to_device: 0.0149 | max_forward: 1.1858 | max_loss: 0.1429 | max_backward: 1.6968 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1366 | Memory: rss_MB: 6126.5 | uss_MB: 5540.3 | child_rss_MB: 4873.0 | allocated_MB: 496.6 | reserved_MB: 28766.0 | inactive_split_MB: 903.7 +rank1 [2026-10-30 04:20:28,800] INFO : Step 45800/50000 rank1 | Train Loss: 0.8566 | Train Loss Unweighted: 0.3982 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1097 | Classification: 0.0794 | Classification_binary: 0.1398 | HitGrouping: 0.1406 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0393 | Regression_eta: 0.0249 | Regression_pt: 0.0433 | Regression_sin_phi: 0.0457 | LR: 1.16e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1211s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.9805 | max_loss: 0.5234 | max_backward: 2.3872 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.8153 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6131.1 | uss_MB: 5547.8 | child_rss_MB: 4874.9 | allocated_MB: 501.7 | reserved_MB: 28766.0 | inactive_split_MB: 1064.3 +rank1 [2026-10-30 04:25:16,811] INFO : Step 45900/50000 rank1 | Train Loss: 0.8754 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1042 | Classification: 0.0696 | Classification_binary: 0.1484 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0659 | Regression_eta: 0.0479 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0541 | LR: 1.15e-04 | DataLoad Time: 0.0134s | Model Forward Time: 5.8512s | Phase Times: max_pre_step_barrier: 0.0308 | max_batch_to_device: 0.0265 | max_forward: 1.7626 | max_loss: 0.1678 | max_backward: 2.6283 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 6124.4 | uss_MB: 5544.6 | child_rss_MB: 4869.8 | allocated_MB: 496.0 | reserved_MB: 29170.0 | inactive_split_MB: 983.4 +rank1 [2026-10-30 04:29:51,934] INFO : Step 46000/50000 rank1 | Train Loss: 0.7531 | Train Loss Unweighted: 0.3556 | Train Loss Components: Auxiliary: 0.1535 | Cardinality: 0.0870 | Classification: 0.0690 | Classification_binary: 0.1026 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0903 | Regression_eta: 0.0011 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1029 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0553 | Regression_eta: 0.0322 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0354 | LR: 1.14e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3131s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0329 | max_forward: 0.7497 | max_loss: 0.1389 | max_backward: 1.9140 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5104 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6122.4 | uss_MB: 5543.2 | child_rss_MB: 4869.8 | allocated_MB: 492.4 | reserved_MB: 29170.0 | inactive_split_MB: 973.8 +rank1 [2026-10-30 04:29:53,062] INFO : Running validation on rank1 +rank1 [2026-10-30 04:29:53,062] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 04:29:53,062] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 04:30:44,349] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 04:30:44,927] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 04:31:11,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 04:31:12,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 04:31:13,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 04:31:14,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 04:31:24,533] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 04:31:25,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 04:31:25,932] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 04:31:41,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 04:31:42,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 04:31:43,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 04:31:44,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 04:31:54,541] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 04:31:54,978] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 04:31:55,463] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 04:32:10,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 04:32:11,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 04:32:12,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 04:32:13,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 04:32:23,560] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 04:37:27,739] INFO : Step 46100/50000 rank1 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3769 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.0999 | Classification: 0.0743 | Classification_binary: 0.1253 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0010 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0411 | Regression_eta: 0.0364 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0435 | LR: 1.13e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.8733s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.9293 | max_loss: 0.1452 | max_backward: 2.3008 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.4968 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1219 | Memory: rss_MB: 7472.5 | uss_MB: 6889.9 | child_rss_MB: 4882.8 | allocated_MB: 498.3 | reserved_MB: 23298.0 | inactive_split_MB: 776.1 +rank1 [2026-10-30 04:42:19,615] INFO : Step 46200/50000 rank1 | Train Loss: 0.8661 | Train Loss Unweighted: 0.3994 | Train Loss Components: Auxiliary: 0.1947 | Cardinality: 0.1164 | Classification: 0.0793 | Classification_binary: 0.1511 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0833 | Regression_eta: 0.0011 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0447 | Regression_eta: 0.0389 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0750 | LR: 1.13e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.7281s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8070 | max_loss: 0.2423 | max_backward: 1.9860 | max_optimizer_step: 0.2482 | max_tensorboard_logging: 0.7539 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6199.1 | uss_MB: 5617.8 | child_rss_MB: 4859.8 | allocated_MB: 499.3 | reserved_MB: 25288.0 | inactive_split_MB: 1325.5 +rank1 [2026-10-30 04:46:59,454] INFO : Step 46300/50000 rank1 | Train Loss: 0.8634 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0916 | Classification: 0.0803 | Classification_binary: 0.1320 | HitGrouping: 0.1536 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1031 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.1275 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0727 | Regression_eta: 0.0306 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0608 | LR: 1.12e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5167s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7891 | max_loss: 0.1470 | max_backward: 1.7900 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.7709 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6201.3 | uss_MB: 5618.4 | child_rss_MB: 4887.4 | allocated_MB: 498.4 | reserved_MB: 25290.0 | inactive_split_MB: 977.6 +rank1 [2026-10-30 04:51:36,900] INFO : Step 46400/50000 rank1 | Train Loss: 0.8499 | Train Loss Unweighted: 0.4102 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.0956 | Classification: 0.0755 | Classification_binary: 0.1172 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1274 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0574 | Regression_eta: 0.0278 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0532 | LR: 1.11e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9772s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7847 | max_loss: 0.2487 | max_backward: 1.9181 | max_optimizer_step: 0.3148 | max_tensorboard_logging: 0.9408 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6188.6 | uss_MB: 5608.9 | child_rss_MB: 4878.3 | allocated_MB: 501.6 | reserved_MB: 25290.0 | inactive_split_MB: 992.4 +rank1 [2026-10-30 04:56:10,902] INFO : Step 46500/50000 rank1 | Train Loss: 0.8687 | Train Loss Unweighted: 0.4055 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1002 | Classification: 0.0822 | Classification_binary: 0.1448 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0872 | Regression_eta: 0.0013 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1578 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0468 | Regression_eta: 0.0246 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0646 | LR: 1.11e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5417s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1839 | max_backward: 1.9629 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6193.9 | uss_MB: 5618.8 | child_rss_MB: 4880.0 | allocated_MB: 498.8 | reserved_MB: 29306.0 | inactive_split_MB: 924.4 +rank1 [2026-10-30 05:00:44,088] INFO : Step 46600/50000 rank1 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4049 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1082 | Classification: 0.0757 | Classification_binary: 0.1508 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0450 | Regression_eta: 0.0297 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0529 | LR: 1.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2106s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7939 | max_loss: 0.2001 | max_backward: 1.6859 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6188.5 | uss_MB: 5615.4 | child_rss_MB: 4846.0 | allocated_MB: 496.0 | reserved_MB: 29306.0 | inactive_split_MB: 901.7 +rank1 [2026-10-30 05:05:18,399] INFO : Step 46700/50000 rank1 | Train Loss: 0.8417 | Train Loss Unweighted: 0.3855 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0982 | Classification: 0.0807 | Classification_binary: 0.1184 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0916 | Regression_eta: 0.0015 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1022 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0399 | Regression_eta: 0.0894 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0366 | LR: 1.10e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7639s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8551 | max_loss: 0.5124 | max_backward: 1.9004 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.7762 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1375 | Memory: rss_MB: 6185.4 | uss_MB: 5616.8 | child_rss_MB: 4868.3 | allocated_MB: 495.0 | reserved_MB: 29308.0 | inactive_split_MB: 1000.4 +rank1 [2026-10-30 05:09:48,389] INFO : Step 46800/50000 rank1 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3911 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0920 | Classification: 0.0920 | Classification_binary: 0.1296 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0828 | Regression_eta: 0.0013 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0418 | Regression_eta: 0.0491 | Regression_pt: 0.0451 | Regression_sin_phi: 0.0491 | LR: 1.09e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4619s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7934 | max_loss: 0.1440 | max_backward: 1.7630 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.7219 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6183.1 | uss_MB: 5617.4 | child_rss_MB: 4871.4 | allocated_MB: 495.6 | reserved_MB: 29308.0 | inactive_split_MB: 1332.0 +rank1 [2026-10-30 05:14:19,737] INFO : Step 46900/50000 rank1 | Train Loss: 0.8496 | Train Loss Unweighted: 0.4142 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0976 | Classification: 0.0776 | Classification_binary: 0.1448 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0946 | Regression_eta: 0.0010 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1326 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0395 | Regression_eta: 0.0315 | Regression_pt: 0.0429 | Regression_sin_phi: 0.0432 | LR: 1.09e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4511s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.9007 | max_loss: 0.1576 | max_backward: 1.8470 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6183.1 | uss_MB: 5616.1 | child_rss_MB: 4872.4 | allocated_MB: 495.4 | reserved_MB: 29310.0 | inactive_split_MB: 850.5 +rank1 [2026-10-30 05:18:49,738] INFO : Step 47000/50000 rank1 | Train Loss: 0.7980 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0982 | Classification: 0.0774 | Classification_binary: 0.1196 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1014 | Regression_cos_phi: 0.0419 | Regression_energy: 0.0391 | Regression_eta: 0.0265 | Regression_pt: 0.0425 | Regression_sin_phi: 0.0275 | LR: 1.08e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8521s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 1.2405 | max_loss: 0.1502 | max_backward: 2.1273 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.7105 | max_tensorboard_flush: 0.0082 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6184.1 | uss_MB: 5617.4 | child_rss_MB: 4877.5 | allocated_MB: 494.9 | reserved_MB: 29310.0 | inactive_split_MB: 852.6 +rank1 [2026-10-30 05:18:50,807] INFO : Running validation on rank1 +rank1 [2026-10-30 05:18:50,807] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 05:19:41,789] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 05:19:42,336] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 05:20:08,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 05:20:09,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 05:20:10,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 05:20:11,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 05:20:21,145] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 05:20:21,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 05:20:22,014] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 05:20:38,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 05:20:39,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 05:20:40,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 05:20:41,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 05:20:51,013] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 05:20:51,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 05:20:52,199] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 05:21:07,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 05:21:08,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 05:21:09,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 05:21:10,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 05:21:20,130] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 05:26:12,382] INFO : Step 47100/50000 rank1 | Train Loss: 0.8046 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0851 | Classification: 0.0882 | Classification_binary: 0.1121 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0936 | Regression_eta: 0.0010 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.1182 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0431 | Regression_eta: 0.0458 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0490 | LR: 1.07e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5742s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8250 | max_loss: 0.1453 | max_backward: 1.6267 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8108 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 7449.0 | uss_MB: 6878.6 | child_rss_MB: 4837.4 | allocated_MB: 494.1 | reserved_MB: 25416.0 | inactive_split_MB: 894.5 +rank1 [2026-10-30 05:30:44,526] INFO : Step 47200/50000 rank1 | Train Loss: 0.8884 | Train Loss Unweighted: 0.4191 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1138 | Classification: 0.0793 | Classification_binary: 0.1489 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0010 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0289 | Regression_energy: 0.0512 | Regression_eta: 0.0299 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0450 | LR: 1.07e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9165s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7960 | max_loss: 0.1482 | max_backward: 1.9261 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.9435 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0661 | Memory: rss_MB: 6249.9 | uss_MB: 5682.5 | child_rss_MB: 4852.8 | allocated_MB: 496.8 | reserved_MB: 26534.0 | inactive_split_MB: 847.6 +rank1 [2026-10-30 05:35:15,255] INFO : Step 47300/50000 rank1 | Train Loss: 0.8547 | Train Loss Unweighted: 0.3920 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1049 | Classification: 0.0798 | Classification_binary: 0.1368 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0012 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1321 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0406 | Regression_eta: 0.0309 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0512 | LR: 1.06e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5893s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7649 | max_loss: 0.2313 | max_backward: 1.8030 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8438 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6253.9 | uss_MB: 5688.5 | child_rss_MB: 4855.9 | allocated_MB: 496.9 | reserved_MB: 28534.0 | inactive_split_MB: 933.0 +rank1 [2026-10-30 05:39:47,015] INFO : Step 47400/50000 rank1 | Train Loss: 0.7606 | Train Loss Unweighted: 0.3755 | Train Loss Components: Auxiliary: 0.1640 | Cardinality: 0.0704 | Classification: 0.0737 | Classification_binary: 0.1072 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0960 | Regression_eta: 0.0010 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1134 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0482 | Regression_eta: 0.0325 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0587 | LR: 1.06e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5314s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0275 | max_forward: 0.8465 | max_loss: 0.1482 | max_backward: 1.7103 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.8388 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 6241.7 | uss_MB: 5678.8 | child_rss_MB: 4860.9 | allocated_MB: 496.4 | reserved_MB: 28534.0 | inactive_split_MB: 913.7 +rank1 [2026-10-30 05:44:18,078] INFO : Step 47500/50000 rank1 | Train Loss: 0.8134 | Train Loss Unweighted: 0.3681 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1057 | Classification: 0.0739 | Classification_binary: 0.1303 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0805 | Regression_eta: 0.0011 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0374 | Regression_energy: 0.0438 | Regression_eta: 0.0338 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0584 | LR: 1.06e-04 | DataLoad Time: 0.0137s | Model Forward Time: 4.7818s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1751 | max_backward: 1.9857 | max_optimizer_step: 0.3263 | max_tensorboard_logging: 0.7363 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6249.7 | uss_MB: 5690.9 | child_rss_MB: 4845.9 | allocated_MB: 498.3 | reserved_MB: 28534.0 | inactive_split_MB: 851.5 +rank1 [2026-10-30 05:48:48,525] INFO : Step 47600/50000 rank1 | Train Loss: 0.8663 | Train Loss Unweighted: 0.4076 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1090 | Classification: 0.0820 | Classification_binary: 0.1491 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0865 | Regression_eta: 0.0011 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0404 | Regression_energy: 0.0344 | Regression_eta: 0.0353 | Regression_pt: 0.0374 | Regression_sin_phi: 0.0381 | LR: 1.05e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6239s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.9385 | max_loss: 0.1530 | max_backward: 2.1678 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6249.7 | uss_MB: 5691.7 | child_rss_MB: 4853.8 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 843.5 +rank1 [2026-10-30 05:53:19,839] INFO : Step 47700/50000 rank1 | Train Loss: 0.8615 | Train Loss Unweighted: 0.4088 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0954 | Classification: 0.0764 | Classification_binary: 0.1291 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1002 | Regression_eta: 0.0011 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1239 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0581 | Regression_eta: 0.0283 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0636 | LR: 1.05e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2179s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7406 | max_loss: 0.2391 | max_backward: 1.7059 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1444 | Memory: rss_MB: 6248.7 | uss_MB: 5687.5 | child_rss_MB: 4851.8 | allocated_MB: 493.9 | reserved_MB: 28534.0 | inactive_split_MB: 895.3 +rank1 [2026-10-30 05:57:52,843] INFO : Step 47800/50000 rank1 | Train Loss: 0.8267 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.0942 | Classification: 0.0882 | Classification_binary: 0.1186 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0870 | Regression_eta: 0.0012 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1102 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0548 | Regression_eta: 0.0307 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0362 | LR: 1.04e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.0874s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.7869 | max_loss: 0.5030 | max_backward: 2.0522 | max_optimizer_step: 0.2267 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6249.7 | uss_MB: 5692.3 | child_rss_MB: 4857.3 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 916.9 +rank1 [2026-10-30 06:02:25,107] INFO : Step 47900/50000 rank1 | Train Loss: 0.8423 | Train Loss Unweighted: 0.4000 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.0845 | Classification: 0.0794 | Classification_binary: 0.1416 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1817 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0489 | Regression_eta: 0.0771 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0779 | LR: 1.04e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.5098s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7994 | max_loss: 0.1511 | max_backward: 1.9354 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.6685 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6247.4 | uss_MB: 5692.2 | child_rss_MB: 4860.2 | allocated_MB: 496.8 | reserved_MB: 30642.0 | inactive_split_MB: 779.5 +rank1 [2026-10-30 06:06:56,645] INFO : Step 48000/50000 rank1 | Train Loss: 0.8641 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0972 | Classification: 0.0768 | Classification_binary: 0.1396 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0890 | Regression_eta: 0.0015 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1377 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0391 | Regression_eta: 0.0275 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0430 | LR: 1.04e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.4090s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0057 | max_forward: 0.8017 | max_loss: 0.1550 | max_backward: 1.6990 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.7498 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6247.1 | uss_MB: 5690.2 | child_rss_MB: 4855.9 | allocated_MB: 495.1 | reserved_MB: 30642.0 | inactive_split_MB: 896.2 +rank1 [2026-10-30 06:06:57,653] INFO : Running validation on rank1 +rank1 [2026-10-30 06:06:57,653] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 06:06:57,653] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 06:07:49,054] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:07:49,561] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 06:08:14,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 06:08:15,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 06:08:16,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 06:08:17,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 06:08:27,118] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 06:08:28,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:08:28,619] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 06:08:44,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 06:08:45,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 06:08:46,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 06:08:47,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 06:08:57,690] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 06:08:58,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:08:58,715] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 06:09:14,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 06:09:15,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 06:09:16,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 06:09:16,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 06:09:26,938] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 06:14:16,585] INFO : Step 48100/50000 rank1 | Train Loss: 0.8226 | Train Loss Unweighted: 0.3895 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.1043 | Classification: 0.0717 | Classification_binary: 0.1116 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1023 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1078 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0572 | Regression_eta: 0.0228 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0438 | LR: 1.03e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4982s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7477 | max_loss: 0.1426 | max_backward: 1.7972 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.6718 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1303 | Memory: rss_MB: 7752.6 | uss_MB: 7191.3 | child_rss_MB: 4880.7 | allocated_MB: 494.1 | reserved_MB: 23078.0 | inactive_split_MB: 1096.3 +rank1 [2026-10-30 06:18:47,187] INFO : Step 48200/50000 rank1 | Train Loss: 0.8500 | Train Loss Unweighted: 0.3835 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1027 | Classification: 0.0771 | Classification_binary: 0.1374 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0827 | Regression_eta: 0.0012 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0533 | Regression_eta: 0.0361 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0517 | LR: 1.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2133s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8132 | max_loss: 0.2545 | max_backward: 2.1167 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1241 | Memory: rss_MB: 6310.4 | uss_MB: 5751.6 | child_rss_MB: 4867.3 | allocated_MB: 496.6 | reserved_MB: 25094.0 | inactive_split_MB: 913.4 +rank1 [2026-10-30 06:23:18,412] INFO : Step 48300/50000 rank1 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4303 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1089 | Classification: 0.0832 | Classification_binary: 0.1567 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0013 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1417 | Regression_cos_phi: 0.0349 | Regression_energy: 0.0419 | Regression_eta: 0.0261 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0301 | LR: 1.03e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4482s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8079 | max_loss: 0.1526 | max_backward: 1.9733 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6317.4 | uss_MB: 5760.9 | child_rss_MB: 4872.1 | allocated_MB: 498.5 | reserved_MB: 25098.0 | inactive_split_MB: 771.0 +rank1 [2026-10-30 06:27:49,028] INFO : Step 48400/50000 rank1 | Train Loss: 0.7961 | Train Loss Unweighted: 0.3733 | Train Loss Components: Auxiliary: 0.1682 | Cardinality: 0.0877 | Classification: 0.0685 | Classification_binary: 0.1191 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0012 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1133 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0631 | Regression_eta: 0.0303 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0467 | LR: 1.02e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2394s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7916 | max_loss: 0.1471 | max_backward: 1.6943 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5940 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6314.4 | uss_MB: 5759.4 | child_rss_MB: 4842.4 | allocated_MB: 496.3 | reserved_MB: 25098.0 | inactive_split_MB: 1087.5 +rank1 [2026-10-30 06:32:21,028] INFO : Step 48500/50000 rank1 | Train Loss: 0.8668 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0816 | Classification: 0.0832 | Classification_binary: 0.1257 | HitGrouping: 0.1542 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1167 | Regression_cos_phi: 0.0344 | Regression_energy: 0.0472 | Regression_eta: 0.0347 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0331 | LR: 1.02e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3830s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7587 | max_loss: 0.1606 | max_backward: 1.7573 | max_optimizer_step: 0.3024 | max_tensorboard_logging: 0.5851 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6309.3 | uss_MB: 5759.8 | child_rss_MB: 4851.4 | allocated_MB: 498.9 | reserved_MB: 25098.0 | inactive_split_MB: 1192.4 +rank1 [2026-10-30 06:36:52,987] INFO : Step 48600/50000 rank1 | Train Loss: 0.8434 | Train Loss Unweighted: 0.4010 | Train Loss Components: Auxiliary: 0.1850 | Cardinality: 0.0964 | Classification: 0.0791 | Classification_binary: 0.1425 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0404 | Regression_energy: 0.0518 | Regression_eta: 0.0428 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0548 | LR: 1.02e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4845s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7760 | max_loss: 0.1505 | max_backward: 1.8991 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0654 | Memory: rss_MB: 6311.3 | uss_MB: 5762.6 | child_rss_MB: 4856.0 | allocated_MB: 497.0 | reserved_MB: 26848.0 | inactive_split_MB: 777.5 +rank1 [2026-10-30 06:41:29,683] INFO : Step 48700/50000 rank1 | Train Loss: 0.9079 | Train Loss Unweighted: 0.4266 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1195 | Classification: 0.0920 | Classification_binary: 0.1456 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0413 | Regression_eta: 0.0259 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0518 | LR: 1.02e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5443s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7797 | max_loss: 0.1578 | max_backward: 1.7107 | max_optimizer_step: 0.5280 | max_tensorboard_logging: 0.7650 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6308.9 | uss_MB: 5759.1 | child_rss_MB: 4857.7 | allocated_MB: 496.9 | reserved_MB: 26848.0 | inactive_split_MB: 891.8 +rank1 [2026-10-30 06:46:02,598] INFO : Step 48800/50000 rank1 | Train Loss: 0.8011 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1685 | Cardinality: 0.0863 | Classification: 0.0837 | Classification_binary: 0.1116 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0965 | Regression_eta: 0.0012 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1298 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0458 | Regression_eta: 0.0357 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0548 | LR: 1.01e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5016s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7374 | max_loss: 0.2432 | max_backward: 1.8459 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.7427 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6306.9 | uss_MB: 5759.1 | child_rss_MB: 4857.0 | allocated_MB: 493.9 | reserved_MB: 26848.0 | inactive_split_MB: 991.0 +rank1 [2026-10-30 06:50:45,084] INFO : Step 48900/50000 rank1 | Train Loss: 0.8551 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.1109 | Classification: 0.0760 | Classification_binary: 0.1310 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1534 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0448 | Regression_eta: 0.0343 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0371 | LR: 1.01e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5424s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8768 | max_loss: 0.1470 | max_backward: 2.0816 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6306.9 | uss_MB: 5761.7 | child_rss_MB: 4858.9 | allocated_MB: 499.5 | reserved_MB: 26852.0 | inactive_split_MB: 1398.0 +rank1 [2026-10-30 06:55:20,574] INFO : Step 49000/50000 rank1 | Train Loss: 0.9005 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.2010 | Cardinality: 0.1248 | Classification: 0.0800 | Classification_binary: 0.1499 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0939 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0608 | Regression_eta: 0.0390 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0502 | LR: 1.01e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9020s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0439 | max_forward: 0.8365 | max_loss: 0.1536 | max_backward: 1.9893 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8890 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6309.3 | uss_MB: 5762.6 | child_rss_MB: 4865.2 | allocated_MB: 497.5 | reserved_MB: 26852.0 | inactive_split_MB: 943.6 +rank1 [2026-10-30 06:55:21,668] INFO : Running validation on rank1 +rank1 [2026-10-30 06:55:21,668] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 06:55:21,669] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 06:56:13,351] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:56:13,863] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 06:56:39,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 06:56:40,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 06:56:41,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 06:56:42,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 06:56:52,223] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 06:56:52,549] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:56:53,030] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 06:57:09,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 06:57:10,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 06:57:11,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 06:57:12,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 06:57:22,117] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 06:57:22,632] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 06:57:23,150] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 06:57:38,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 06:57:39,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 06:57:40,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 06:57:41,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 06:57:51,105] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 07:02:45,630] INFO : Step 49100/50000 rank1 | Train Loss: 0.8219 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.0880 | Classification: 0.0841 | Classification_binary: 0.1303 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0855 | Regression_eta: 0.0012 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1067 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0446 | Regression_eta: 0.0221 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0348 | LR: 1.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3713s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7738 | max_loss: 0.1431 | max_backward: 1.6939 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6276 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 7720.3 | uss_MB: 7170.2 | child_rss_MB: 4829.0 | allocated_MB: 495.9 | reserved_MB: 24730.0 | inactive_split_MB: 936.0 +rank1 [2026-10-30 07:07:18,677] INFO : Step 49200/50000 rank1 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3843 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0856 | Classification: 0.0829 | Classification_binary: 0.1188 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0892 | Regression_eta: 0.0011 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0603 | Regression_eta: 0.0290 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0377 | LR: 1.01e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.7742s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7590 | max_loss: 0.5032 | max_backward: 1.9826 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.7926 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6365.4 | uss_MB: 5817.0 | child_rss_MB: 4830.4 | allocated_MB: 494.5 | reserved_MB: 26752.0 | inactive_split_MB: 1117.5 +rank1 [2026-10-30 07:11:50,861] INFO : Step 49300/50000 rank1 | Train Loss: 0.9397 | Train Loss Unweighted: 0.4282 | Train Loss Components: Auxiliary: 0.2065 | Cardinality: 0.1351 | Classification: 0.0889 | Classification_binary: 0.1552 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0901 | Regression_eta: 0.0012 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0514 | Regression_eta: 0.0300 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0511 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4914s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0070 | max_forward: 0.7807 | max_loss: 0.1520 | max_backward: 1.8619 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6076 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6376.6 | uss_MB: 5827.0 | child_rss_MB: 4814.1 | allocated_MB: 496.4 | reserved_MB: 26752.0 | inactive_split_MB: 1120.6 +rank1 [2026-10-30 07:16:22,251] INFO : Step 49400/50000 rank1 | Train Loss: 0.8870 | Train Loss Unweighted: 0.4213 | Train Loss Components: Auxiliary: 0.1937 | Cardinality: 0.1005 | Classification: 0.0815 | Classification_binary: 0.1465 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1102 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0473 | Regression_eta: 0.0299 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0523 | LR: 1.00e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2706s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8046 | max_loss: 0.1540 | max_backward: 1.7244 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.6030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6373.7 | uss_MB: 5825.4 | child_rss_MB: 4813.8 | allocated_MB: 498.5 | reserved_MB: 28804.0 | inactive_split_MB: 886.2 +rank1 [2026-10-30 07:20:55,284] INFO : Step 49500/50000 rank1 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3949 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0991 | Classification: 0.0800 | Classification_binary: 0.1101 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0013 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1057 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0348 | Regression_eta: 0.0484 | Regression_pt: 0.0397 | Regression_sin_phi: 0.0431 | LR: 1.00e-04 | DataLoad Time: 0.0139s | Model Forward Time: 5.2265s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0074 | max_forward: 0.9924 | max_loss: 0.1865 | max_backward: 2.5853 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.6095 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0429 | Memory: rss_MB: 6377.1 | uss_MB: 5695.1 | child_rss_MB: 4830.9 | allocated_MB: 495.9 | reserved_MB: 28804.0 | inactive_split_MB: 849.6 +rank1 [2026-10-30 07:25:48,133] INFO : Step 49600/50000 rank1 | Train Loss: 0.9075 | Train Loss Unweighted: 0.4421 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1021 | Classification: 0.0939 | Classification_binary: 0.1401 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0540 | Regression_eta: 0.0380 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0561 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4402s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7852 | max_loss: 0.1445 | max_backward: 2.0967 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4754 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6378.7 | uss_MB: 5829.2 | child_rss_MB: 4808.2 | allocated_MB: 497.8 | reserved_MB: 28804.0 | inactive_split_MB: 845.2 +rank1 [2026-10-30 07:30:32,223] INFO : Step 49700/50000 rank1 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4307 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1074 | Classification: 0.0931 | Classification_binary: 0.1611 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0865 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1573 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0401 | Regression_eta: 0.0366 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0639 | LR: 1.00e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6420s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.8169 | max_loss: 0.1522 | max_backward: 2.0553 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.4937 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1402 | Memory: rss_MB: 6379.2 | uss_MB: 5828.6 | child_rss_MB: 4829.3 | allocated_MB: 499.3 | reserved_MB: 30718.0 | inactive_split_MB: 838.8 +rank1 [2026-10-30 07:35:17,062] INFO : Step 49800/50000 rank1 | Train Loss: 0.8342 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.1036 | Classification: 0.0769 | Classification_binary: 0.1292 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1254 | Regression_cos_phi: 0.0419 | Regression_energy: 0.0466 | Regression_eta: 0.0244 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0510 | LR: 1.00e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.1028s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0329 | max_forward: 0.7490 | max_loss: 0.1889 | max_backward: 2.2249 | max_optimizer_step: 0.2452 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6374.9 | uss_MB: 5812.4 | child_rss_MB: 4828.7 | allocated_MB: 494.4 | reserved_MB: 30718.0 | inactive_split_MB: 1134.4 +rank1 [2026-10-30 07:39:58,302] INFO : Step 49900/50000 rank1 | Train Loss: 0.8436 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.0947 | Classification: 0.0943 | Classification_binary: 0.1179 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0982 | Regression_eta: 0.0012 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0473 | Regression_eta: 0.0654 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0475 | LR: 1.00e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.4159s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7496 | max_loss: 0.5142 | max_backward: 1.9365 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6373.9 | uss_MB: 5825.7 | child_rss_MB: 4831.8 | allocated_MB: 494.8 | reserved_MB: 30718.0 | inactive_split_MB: 1090.4 +rank1 [2026-10-30 07:44:45,887] INFO : Step 50000/50000 rank1 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4040 | Train Loss Components: Auxiliary: 0.1921 | Cardinality: 0.1113 | Classification: 0.0818 | Classification_binary: 0.1436 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0872 | Regression_eta: 0.0014 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0383 | Regression_eta: 0.0344 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0369 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.6364s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 1.3997 | max_loss: 0.5678 | max_backward: 3.0655 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6373.9 | uss_MB: 5825.4 | child_rss_MB: 4827.6 | allocated_MB: 497.9 | reserved_MB: 30720.0 | inactive_split_MB: 906.1 +rank1 [2026-10-30 07:44:46,919] INFO : Running validation on rank1 +rank1 [2026-10-30 07:44:46,919] INFO : ResumableSampler physical:valid resetting +rank1 [2026-10-30 07:44:46,919] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank1 [2026-10-30 07:45:38,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 07:45:39,246] INFO : Running predictions on cld_edm_ttbar_hits +rank1 [2026-10-30 07:46:04,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_1_0.parquet +rank1 [2026-10-30 07:46:05,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_1_1.parquet +rank1 [2026-10-30 07:46:06,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_1_2.parquet +rank1 [2026-10-30 07:46:07,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_1_3.parquet +rank1 [2026-10-30 07:46:17,094] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 07:46:17,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 07:46:17,995] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank1 [2026-10-30 07:46:33,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_1_0.parquet +rank1 [2026-10-30 07:46:34,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_1_1.parquet +rank1 [2026-10-30 07:46:35,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_1_2.parquet +rank1 [2026-10-30 07:46:36,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_1_3.parquet +rank1 [2026-10-30 07:46:46,674] INFO : Time taken to make predictions on device 1 is: 0.25 min +rank1 [2026-10-30 07:46:47,615] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank1 [2026-10-30 07:46:48,153] INFO : Running predictions on cld_edm_qq_hits +rank1 [2026-10-30 07:47:03,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_1_0.parquet +rank1 [2026-10-30 07:47:04,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_1_1.parquet +rank1 [2026-10-30 07:47:05,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_1_2.parquet +rank1 [2026-10-30 07:47:05,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_1_3.parquet +rank1 [2026-10-30 07:47:16,003] INFO : Time taken to make predictions on device 1 is: 0.23 min +rank1 [2026-10-30 07:47:17,237] INFO : Training completed. Total time on device 1: 1017.499min diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.2 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.2 new file mode 100644 index 0000000000000000000000000000000000000000..f3dbb317aba7c9ffbd1d67c593533bbb09c3a070 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.2 @@ -0,0 +1,2631 @@ +rank2 [2026-10-28 11:12:32,376] INFO : Initializing model with seed=12345; process seed=12347 +rank2 [2026-10-28 11:12:32,377] INFO : configured dtype=torch.bfloat16 for autocast +rank2 [2026-10-28 11:12:54,158] INFO : Instantiating model +rank2 [2026-10-28 11:12:54,158] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank2 [2026-10-28 11:12:54,158] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank2 [2026-10-28 11:12:54,176] INFO : Input encoding initialization took 0.02s +rank2 [2026-10-28 11:12:54,176] INFO : nn0 parameters: 171776 +rank2 [2026-10-28 11:12:54,176] INFO : Initializing shared backbone layers, num_convs=6 +rank2 [2026-10-28 11:12:54,176] INFO : Initializing backbone layer 0 +rank2 [2026-10-28 11:12:54,212] INFO : Initializing backbone layer 1 +rank2 [2026-10-28 11:12:54,221] INFO : Initializing backbone layer 2 +rank2 [2026-10-28 11:12:54,230] INFO : Initializing backbone layer 3 +rank2 [2026-10-28 11:12:54,238] INFO : Initializing backbone layer 4 +rank2 [2026-10-28 11:12:54,247] INFO : Initializing backbone layer 5 +rank2 [2026-10-28 11:12:54,255] INFO : Backbone initialization took 0.08s +rank2 [2026-10-28 11:12:54,256] INFO : backbone parameters: 7901376 +rank2 [2026-10-28 11:12:54,256] INFO : Initializing output DNNs +rank2 [2026-10-28 11:12:54,292] INFO : Output DNNs initialization took 0.04s +rank2 [2026-10-28 11:12:54,292] INFO : backbone_mode=BackboneMode.SHARED +rank2 [2026-10-28 11:12:54,292] INFO : jagged_attention enabled=False +rank2 [2026-10-28 11:12:54,292] INFO : flash_attn_varlen enabled=False +rank2 [2026-10-28 11:12:54,292] INFO : task_queries enabled=False +rank2 [2026-10-28 11:12:54,292] INFO : classification_norm parameters: 0 +rank2 [2026-10-28 11:12:54,292] INFO : regression_norm parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : classification_query parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : regression_query parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : classification_readout parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : regression_readout parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : output_mode=set +rank2 [2026-10-28 11:12:54,293] INFO : set_decoder parameters: 4713812 +rank2 [2026-10-28 11:12:54,293] INFO : nn_binary_particle parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : nn_pid parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : nn_pt parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : nn_eta parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : nn_sin_phi parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : nn_cos_phi parameters: 0 +rank2 [2026-10-28 11:12:54,293] INFO : nn_energy parameters: 0 +rank2 [2026-10-28 11:12:54,294] INFO : Total MLPF parameters: 12786964 +rank2 [2026-10-28 11:12:54,294] INFO : MLPF __init__ done +rank2 [2026-10-28 11:12:54,294] INFO : Instantiated model +rank2 [2026-10-28 11:12:54,294] INFO : Moving model to device rank=2 +rank2 [2026-10-28 11:12:54,334] INFO : Moved model to device rank=2 +rank2 [2026-10-28 11:12:54,337] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank2 [2026-10-28 11:12:54,338] INFO : Configured model for SyncBatchNorm rank=2 +rank2 [2026-10-28 11:12:54,648] INFO : Configured model for DistributedDataParallel rank=2 +rank2 [2026-10-28 11:12:54,809] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank2 [2026-10-28 11:12:54,809] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank2 [2026-10-28 11:12:54,810] INFO : Trainable parameters: 12786964 +rank2 [2026-10-28 11:12:54,810] INFO : Non-trainable parameters: 23268 +rank2 [2026-10-28 11:12:54,810] INFO : Total parameters: 12810232 +rank2 [2026-10-28 11:12:54,813] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank2 [2026-10-28 11:12:54,813] INFO : Getting dataloaders +rank2 [2026-10-28 11:12:54,813] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:03,744] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:04,813] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,147] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-28 11:13:06,148] INFO : Creating WeightedMixIterator with 1 data loaders. +rank2 [2026-10-28 11:13:06,148] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,419] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,694] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:13:06,965] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-28 11:13:06,966] INFO : Creating InterleavedIterator with 1 data loaders. +rank2 [2026-10-28 11:13:21,330] INFO : Got dataloaders +rank2 [2026-10-28 11:13:21,330] INFO : loader split=train rank=2 len=21112 +rank2 [2026-10-28 11:13:21,330] INFO : loader split=valid rank=2 len=12 +rank2 [2026-10-28 11:13:21,330] INFO : Starting training from step 1 to 50000 on rank 2 of 8 +rank2 [2026-10-28 11:13:21,331] INFO : Creating train_iterator +rank2 [2026-10-28 11:13:21,331] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-28 11:13:35,670] INFO : Created train_iterator +rank2 [2026-10-28 11:18:11,207] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5408100029081107, 'Classification': 0.8863177470117808, 'Regression_pt': 0.9386234817653895, 'Regression_eta': 0.019238746587652712, 'Regression_sin_phi': 0.006576384851359762, 'Regression_cos_phi': 0.006405904629500583, 'Regression_energy': 1.0145655398070812}, weights={'Classification_binary': 1.0, 'Classification': 0.6101762056350708, 'Regression_pt': 0.5761735439300537, 'Regression_eta': 28.110458374023438, 'Regression_sin_phi': 82.23515319824219, 'Regression_cos_phi': 84.42367553710938, 'Regression_energy': 0.5330458879470825} +rank2 [2026-10-28 11:18:13,718] INFO : Step 100/50000 rank2 | Train Loss: 4.2631 | Train Loss Unweighted: 2.7366 | Train Loss Components: Auxiliary: 0.7941 | Cardinality: 0.3942 | Classification: 0.6312 | Classification_binary: 0.4549 | HitGrouping: 0.3381 | Regression_cos_phi: 0.0047 | Regression_energy: 0.8263 | Regression_eta: 0.0122 | Regression_pt: 0.8026 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.6979 | Classification_binary: 0.4294 | Regression_cos_phi: 0.0043 | Regression_energy: 0.9218 | Regression_eta: 0.0100 | Regression_pt: 0.9047 | Regression_sin_phi: 0.0046 | LR: 1.00e-03 | DataLoad Time: 0.0164s | Model Forward Time: 3.5371s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6509 | max_loss: 0.1961 | max_backward: 1.4166 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4769.1 | uss_MB: 3475.5 | child_rss_MB: 6372.1 | allocated_MB: 446.7 | reserved_MB: 24804.0 | inactive_split_MB: 1063.6 +rank2 [2026-10-28 11:22:42,537] INFO : Step 200/50000 rank2 | Train Loss: 4.2029 | Train Loss Unweighted: 2.5447 | Train Loss Components: Auxiliary: 0.8074 | Cardinality: 0.4500 | Classification: 0.5373 | Classification_binary: 0.5135 | HitGrouping: 0.3101 | Regression_cos_phi: 0.0042 | Regression_energy: 0.7508 | Regression_eta: 0.0093 | Regression_pt: 0.7252 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.3453 | Classification_binary: 0.5244 | Regression_cos_phi: 0.4656 | Regression_energy: 0.4429 | Regression_eta: 0.2750 | Regression_pt: 0.4579 | Regression_sin_phi: 0.4262 | LR: 1.00e-03 | DataLoad Time: 0.0153s | Model Forward Time: 4.6623s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9214 | max_loss: 0.1502 | max_backward: 2.1220 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 4772.0 | uss_MB: 3480.3 | child_rss_MB: 6374.7 | allocated_MB: 449.0 | reserved_MB: 26252.0 | inactive_split_MB: 1054.4 +rank2 [2026-10-28 11:27:10,390] INFO : Step 300/50000 rank2 | Train Loss: 3.5114 | Train Loss Unweighted: 2.0975 | Train Loss Components: Auxiliary: 0.6791 | Cardinality: 0.3510 | Classification: 0.4201 | Classification_binary: 0.4975 | HitGrouping: 0.2993 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5752 | Regression_eta: 0.0073 | Regression_pt: 0.5905 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2689 | Classification_binary: 0.4954 | Regression_cos_phi: 0.3019 | Regression_energy: 0.2834 | Regression_eta: 0.2335 | Regression_pt: 0.3084 | Regression_sin_phi: 0.3451 | LR: 1.00e-03 | DataLoad Time: 0.0158s | Model Forward Time: 4.2603s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.7369 | max_loss: 0.1526 | max_backward: 1.7330 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.9030 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 4773.0 | uss_MB: 3484.3 | child_rss_MB: 6364.6 | allocated_MB: 446.1 | reserved_MB: 26252.0 | inactive_split_MB: 1068.0 +rank2 [2026-10-28 11:31:37,117] INFO : Step 400/50000 rank2 | Train Loss: 3.3615 | Train Loss Unweighted: 1.9456 | Train Loss Components: Auxiliary: 0.6310 | Cardinality: 0.3555 | Classification: 0.4058 | Classification_binary: 0.4436 | HitGrouping: 0.3006 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5363 | Regression_eta: 0.0080 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2557 | Classification_binary: 0.4080 | Regression_cos_phi: 0.2562 | Regression_energy: 0.2736 | Regression_eta: 0.1535 | Regression_pt: 0.3052 | Regression_sin_phi: 0.1855 | LR: 1.00e-03 | DataLoad Time: 0.0173s | Model Forward Time: 4.5022s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1151 | max_forward: 0.7642 | max_loss: 0.1384 | max_backward: 1.7906 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9359 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0880 | Memory: rss_MB: 4771.8 | uss_MB: 3484.8 | child_rss_MB: 6366.3 | allocated_MB: 446.8 | reserved_MB: 26252.0 | inactive_split_MB: 1064.0 +rank2 [2026-10-28 11:36:08,659] INFO : Step 500/50000 rank2 | Train Loss: 3.8257 | Train Loss Unweighted: 2.0630 | Train Loss Components: Auxiliary: 0.7010 | Cardinality: 0.4740 | Classification: 0.4380 | Classification_binary: 0.4817 | HitGrouping: 0.2961 | Regression_cos_phi: 0.0044 | Regression_energy: 0.5611 | Regression_eta: 0.0087 | Regression_pt: 0.5645 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2622 | Classification_binary: 0.5259 | Regression_cos_phi: 0.4185 | Regression_energy: 0.2957 | Regression_eta: 0.2337 | Regression_pt: 0.3157 | Regression_sin_phi: 0.4563 | LR: 1.00e-03 | DataLoad Time: 0.0156s | Model Forward Time: 5.1259s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0077 | max_forward: 1.2433 | max_loss: 0.1536 | max_backward: 2.4885 | max_optimizer_step: 0.1562 | max_tensorboard_logging: 1.0374 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0034 | Memory: rss_MB: 4768.9 | uss_MB: 3486.1 | child_rss_MB: 6361.5 | allocated_MB: 449.6 | reserved_MB: 26256.0 | inactive_split_MB: 929.2 +rank2 [2026-10-28 11:40:42,093] INFO : Step 600/50000 rank2 | Train Loss: 3.4702 | Train Loss Unweighted: 1.9733 | Train Loss Components: Auxiliary: 0.6509 | Cardinality: 0.3585 | Classification: 0.3825 | Classification_binary: 0.5061 | HitGrouping: 0.2915 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5321 | Regression_eta: 0.0074 | Regression_pt: 0.5377 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2536 | Classification_binary: 0.5125 | Regression_cos_phi: 0.3653 | Regression_energy: 0.2917 | Regression_eta: 0.2072 | Regression_pt: 0.3177 | Regression_sin_phi: 0.3978 | LR: 1.00e-03 | DataLoad Time: 0.0147s | Model Forward Time: 4.5586s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8098 | max_loss: 0.1544 | max_backward: 1.9541 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 0.9377 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 4772.7 | uss_MB: 3486.1 | child_rss_MB: 6366.7 | allocated_MB: 449.5 | reserved_MB: 27396.0 | inactive_split_MB: 1022.5 +rank2 [2026-10-28 11:45:16,604] INFO : Step 700/50000 rank2 | Train Loss: 2.9693 | Train Loss Unweighted: 1.8072 | Train Loss Components: Auxiliary: 0.5907 | Cardinality: 0.2768 | Classification: 0.3175 | Classification_binary: 0.3969 | HitGrouping: 0.2848 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5321 | Regression_eta: 0.0061 | Regression_pt: 0.5492 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1962 | Classification_binary: 0.3853 | Regression_cos_phi: 0.1796 | Regression_energy: 0.2625 | Regression_eta: 0.1826 | Regression_pt: 0.2850 | Regression_sin_phi: 0.2586 | LR: 1.00e-03 | DataLoad Time: 0.0208s | Model Forward Time: 3.9463s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7910 | max_loss: 0.2436 | max_backward: 1.8672 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.8 | uss_MB: 3484.3 | child_rss_MB: 6369.5 | allocated_MB: 446.1 | reserved_MB: 27396.0 | inactive_split_MB: 1066.2 +rank2 [2026-10-28 11:49:48,175] INFO : Step 800/50000 rank2 | Train Loss: 3.1527 | Train Loss Unweighted: 1.8424 | Train Loss Components: Auxiliary: 0.6214 | Cardinality: 0.3518 | Classification: 0.3758 | Classification_binary: 0.3967 | HitGrouping: 0.2962 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5290 | Regression_eta: 0.0063 | Regression_pt: 0.5287 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2099 | Classification_binary: 0.4081 | Regression_cos_phi: 0.2405 | Regression_energy: 0.2320 | Regression_eta: 0.1733 | Regression_pt: 0.2484 | Regression_sin_phi: 0.2071 | LR: 9.99e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6446s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0093 | max_forward: 0.8507 | max_loss: 0.1493 | max_backward: 1.9706 | max_optimizer_step: 0.1582 | max_tensorboard_logging: 1.0840 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0072 | Memory: rss_MB: 4754.0 | uss_MB: 3484.3 | child_rss_MB: 6293.3 | allocated_MB: 450.9 | reserved_MB: 27396.0 | inactive_split_MB: 920.9 +rank2 [2026-10-28 11:54:30,570] INFO : Step 900/50000 rank2 | Train Loss: 3.2552 | Train Loss Unweighted: 1.9216 | Train Loss Components: Auxiliary: 0.6208 | Cardinality: 0.2901 | Classification: 0.3818 | Classification_binary: 0.4734 | HitGrouping: 0.2881 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5239 | Regression_eta: 0.0075 | Regression_pt: 0.5284 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2316 | Classification_binary: 0.4568 | Regression_cos_phi: 0.2348 | Regression_energy: 0.2637 | Regression_eta: 0.2513 | Regression_pt: 0.2954 | Regression_sin_phi: 0.2577 | LR: 9.99e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6735s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8050 | max_loss: 0.1486 | max_backward: 1.9075 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.9729 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4751.7 | uss_MB: 3485.4 | child_rss_MB: 6307.4 | allocated_MB: 449.7 | reserved_MB: 27406.0 | inactive_split_MB: 927.3 +rank2 [2026-10-28 11:59:03,649] INFO : Step 1000/50000 rank2 | Train Loss: 2.8964 | Train Loss Unweighted: 1.7179 | Train Loss Components: Auxiliary: 0.5694 | Cardinality: 0.2509 | Classification: 0.3299 | Classification_binary: 0.4129 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4804 | Regression_eta: 0.0063 | Regression_pt: 0.4827 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1974 | Classification_binary: 0.4118 | Regression_cos_phi: 0.2467 | Regression_energy: 0.2653 | Regression_eta: 0.1501 | Regression_pt: 0.2859 | Regression_sin_phi: 0.1987 | LR: 9.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4511s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7367 | max_loss: 0.2419 | max_backward: 1.8137 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 1.0408 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 4746.9 | uss_MB: 3485.7 | child_rss_MB: 6281.8 | allocated_MB: 449.1 | reserved_MB: 27406.0 | inactive_split_MB: 959.2 +rank2 [2026-10-28 11:59:04,553] INFO : Running validation on rank2 +rank2 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 11:59:57,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 11:59:57,578] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 12:00:26,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 12:00:27,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 12:00:28,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 12:00:30,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 12:00:40,029] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-28 12:00:40,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:00:41,018] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 12:00:56,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 12:00:57,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 12:00:58,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 12:00:59,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 12:01:09,913] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 12:01:10,611] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:01:11,132] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 12:01:26,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 12:01:27,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 12:01:28,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 12:01:29,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 12:01:39,273] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 12:06:47,466] INFO : Step 1100/50000 rank2 | Train Loss: 2.7351 | Train Loss Unweighted: 1.6531 | Train Loss Components: Auxiliary: 0.5524 | Cardinality: 0.2470 | Classification: 0.3238 | Classification_binary: 0.3516 | HitGrouping: 0.2744 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4831 | Regression_eta: 0.0054 | Regression_pt: 0.4842 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2058 | Classification_binary: 0.3688 | Regression_cos_phi: 0.2139 | Regression_energy: 0.2363 | Regression_eta: 0.1618 | Regression_pt: 0.2597 | Regression_sin_phi: 0.2366 | LR: 9.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7307s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7345 | max_loss: 0.2408 | max_backward: 1.8906 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 1.1618 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6205.6 | uss_MB: 4949.9 | child_rss_MB: 6308.4 | allocated_MB: 448.2 | reserved_MB: 24590.0 | inactive_split_MB: 1051.2 +rank2 [2026-10-28 12:11:28,615] INFO : Step 1200/50000 rank2 | Train Loss: 2.9931 | Train Loss Unweighted: 1.7517 | Train Loss Components: Auxiliary: 0.5886 | Cardinality: 0.2847 | Classification: 0.3460 | Classification_binary: 0.4151 | HitGrouping: 0.2795 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4874 | Regression_eta: 0.0065 | Regression_pt: 0.4908 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1753 | Classification_binary: 0.3986 | Regression_cos_phi: 0.1889 | Regression_energy: 0.2467 | Regression_eta: 0.1725 | Regression_pt: 0.2689 | Regression_sin_phi: 0.2757 | LR: 9.99e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.9382s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0061 | max_forward: 0.9910 | max_loss: 0.1662 | max_backward: 2.3256 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.8355 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 6205.6 | uss_MB: 4943.2 | child_rss_MB: 6333.1 | allocated_MB: 449.6 | reserved_MB: 24590.0 | inactive_split_MB: 1155.8 +rank2 [2026-10-28 12:16:08,306] INFO : Step 1300/50000 rank2 | Train Loss: 3.0304 | Train Loss Unweighted: 1.7607 | Train Loss Components: Auxiliary: 0.5900 | Cardinality: 0.2909 | Classification: 0.3397 | Classification_binary: 0.4400 | HitGrouping: 0.2683 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4807 | Regression_eta: 0.0062 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2046 | Classification_binary: 0.3984 | Regression_cos_phi: 0.2414 | Regression_energy: 0.2621 | Regression_eta: 0.1505 | Regression_pt: 0.2897 | Regression_sin_phi: 0.2093 | LR: 9.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3292s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8206 | max_loss: 0.1504 | max_backward: 1.9515 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5688 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5595.4 | uss_MB: 4349.7 | child_rss_MB: 6286.2 | allocated_MB: 448.6 | reserved_MB: 25778.0 | inactive_split_MB: 1050.0 +rank2 [2026-10-28 12:20:48,119] INFO : Step 1400/50000 rank2 | Train Loss: 2.5732 | Train Loss Unweighted: 1.5137 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2349 | Classification: 0.2906 | Classification_binary: 0.3573 | HitGrouping: 0.2641 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4252 | Regression_eta: 0.0055 | Regression_pt: 0.4303 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1570 | Classification_binary: 0.3171 | Regression_cos_phi: 0.1508 | Regression_energy: 0.2161 | Regression_eta: 0.1272 | Regression_pt: 0.2309 | Regression_sin_phi: 0.2161 | LR: 9.98e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.1764s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.8166 | max_loss: 0.1962 | max_backward: 1.8388 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4964 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5588.2 | uss_MB: 3765.0 | child_rss_MB: 6275.0 | allocated_MB: 454.5 | reserved_MB: 28272.0 | inactive_split_MB: 1143.7 +rank2 [2026-10-28 12:25:37,857] INFO : Step 1500/50000 rank2 | Train Loss: 2.8564 | Train Loss Unweighted: 1.5902 | Train Loss Components: Auxiliary: 0.5576 | Cardinality: 0.3788 | Classification: 0.3167 | Classification_binary: 0.3534 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4517 | Regression_eta: 0.0055 | Regression_pt: 0.4576 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1963 | Classification_binary: 0.3921 | Regression_cos_phi: 0.2245 | Regression_energy: 0.2418 | Regression_eta: 0.1445 | Regression_pt: 0.2646 | Regression_sin_phi: 0.2064 | LR: 9.98e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6165s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.0272 | max_loss: 0.2459 | max_backward: 1.8682 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.0883 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0105 | Memory: rss_MB: 5566.9 | uss_MB: 4329.3 | child_rss_MB: 6244.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 989.3 +rank2 [2026-10-28 12:30:20,922] INFO : Step 1600/50000 rank2 | Train Loss: 2.8293 | Train Loss Unweighted: 1.6754 | Train Loss Components: Auxiliary: 0.5537 | Cardinality: 0.2456 | Classification: 0.3224 | Classification_binary: 0.4168 | HitGrouping: 0.2726 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4572 | Regression_eta: 0.0058 | Regression_pt: 0.4676 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1971 | Classification_binary: 0.4045 | Regression_cos_phi: 0.2396 | Regression_energy: 0.2333 | Regression_eta: 0.1613 | Regression_pt: 0.2569 | Regression_sin_phi: 0.2280 | LR: 9.98e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5701s | Phase Times: max_pre_step_barrier: 0.1094 | max_batch_to_device: 0.0066 | max_forward: 0.7720 | max_loss: 0.1510 | max_backward: 1.7462 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5578.6 | uss_MB: 4348.9 | child_rss_MB: 6226.8 | allocated_MB: 448.1 | reserved_MB: 28272.0 | inactive_split_MB: 989.9 +rank2 [2026-10-28 12:35:04,110] INFO : Step 1700/50000 rank2 | Train Loss: 2.7295 | Train Loss Unweighted: 1.5885 | Train Loss Components: Auxiliary: 0.5261 | Cardinality: 0.2505 | Classification: 0.2942 | Classification_binary: 0.3852 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4488 | Regression_eta: 0.0059 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1768 | Classification_binary: 0.4106 | Regression_cos_phi: 0.2646 | Regression_energy: 0.2232 | Regression_eta: 0.1614 | Regression_pt: 0.2410 | Regression_sin_phi: 0.2485 | LR: 9.97e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.9347s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 1.0304 | max_loss: 0.4846 | max_backward: 1.9957 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0278 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5571.8 | uss_MB: 4349.5 | child_rss_MB: 6226.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 999.4 +rank2 [2026-10-28 12:39:46,515] INFO : Step 1800/50000 rank2 | Train Loss: 2.5541 | Train Loss Unweighted: 1.4989 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2504 | Classification: 0.2799 | Classification_binary: 0.3357 | HitGrouping: 0.2658 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4350 | Regression_eta: 0.0052 | Regression_pt: 0.4384 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1620 | Classification_binary: 0.3256 | Regression_cos_phi: 0.1996 | Regression_energy: 0.2077 | Regression_eta: 0.1488 | Regression_pt: 0.2273 | Regression_sin_phi: 0.1493 | LR: 9.97e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3594s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7316 | max_loss: 0.1436 | max_backward: 1.6570 | max_optimizer_step: 0.2625 | max_tensorboard_logging: 0.9220 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5563.3 | uss_MB: 4348.3 | child_rss_MB: 6170.2 | allocated_MB: 444.7 | reserved_MB: 28272.0 | inactive_split_MB: 991.9 +rank2 [2026-10-28 12:44:29,073] INFO : Step 1900/50000 rank2 | Train Loss: 2.7201 | Train Loss Unweighted: 1.5780 | Train Loss Components: Auxiliary: 0.5257 | Cardinality: 0.2710 | Classification: 0.3152 | Classification_binary: 0.3657 | HitGrouping: 0.2653 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4371 | Regression_eta: 0.0058 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1927 | Classification_binary: 0.3929 | Regression_cos_phi: 0.2110 | Regression_energy: 0.2265 | Regression_eta: 0.2037 | Regression_pt: 0.2535 | Regression_sin_phi: 0.2511 | LR: 9.97e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4044s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.7723 | max_loss: 0.1448 | max_backward: 2.0316 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5561.2 | uss_MB: 4335.4 | child_rss_MB: 6153.7 | allocated_MB: 448.9 | reserved_MB: 28272.0 | inactive_split_MB: 1166.2 +rank2 [2026-10-28 12:49:11,971] INFO : Step 2000/50000 rank2 | Train Loss: 2.7893 | Train Loss Unweighted: 1.5772 | Train Loss Components: Auxiliary: 0.5342 | Cardinality: 0.3149 | Classification: 0.3056 | Classification_binary: 0.4233 | HitGrouping: 0.2623 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4169 | Regression_eta: 0.0058 | Regression_pt: 0.4204 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1869 | Classification_binary: 0.4064 | Regression_cos_phi: 0.2221 | Regression_energy: 0.2103 | Regression_eta: 0.1572 | Regression_pt: 0.2310 | Regression_sin_phi: 0.2138 | LR: 9.96e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.5208s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0070 | max_forward: 1.0424 | max_loss: 0.2160 | max_backward: 2.3184 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.9849 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0132 | Memory: rss_MB: 5555.4 | uss_MB: 4316.1 | child_rss_MB: 6148.5 | allocated_MB: 451.3 | reserved_MB: 28272.0 | inactive_split_MB: 992.4 +rank2 [2026-10-28 12:49:12,926] INFO : Running validation on rank2 +rank2 [2026-10-28 12:49:12,926] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 12:49:12,926] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 12:50:07,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:50:08,359] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 12:50:34,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 12:50:35,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 12:50:36,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 12:50:37,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 12:50:47,425] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 12:50:48,335] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:50:48,902] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 12:51:04,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 12:51:05,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 12:51:06,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 12:51:07,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 12:51:17,948] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 12:51:18,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 12:51:18,823] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 12:51:35,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 12:51:35,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 12:51:36,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 12:51:37,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 12:51:47,617] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 12:57:01,909] INFO : Step 2100/50000 rank2 | Train Loss: 2.3990 | Train Loss Unweighted: 1.4049 | Train Loss Components: Auxiliary: 0.4671 | Cardinality: 0.2199 | Classification: 0.2491 | Classification_binary: 0.3399 | HitGrouping: 0.2605 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4015 | Regression_eta: 0.0046 | Regression_pt: 0.4053 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1473 | Classification_binary: 0.3086 | Regression_cos_phi: 0.1881 | Regression_energy: 0.2097 | Regression_eta: 0.1177 | Regression_pt: 0.2312 | Regression_sin_phi: 0.2735 | LR: 9.96e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8144s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7238 | max_loss: 0.1490 | max_backward: 1.8098 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.2800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6608.5 | uss_MB: 5401.4 | child_rss_MB: 6187.1 | allocated_MB: 444.8 | reserved_MB: 23654.0 | inactive_split_MB: 964.6 +rank2 [2026-10-28 13:01:41,080] INFO : Step 2200/50000 rank2 | Train Loss: 2.6105 | Train Loss Unweighted: 1.4907 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2489 | Classification: 0.2976 | Classification_binary: 0.3552 | HitGrouping: 0.2519 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4108 | Regression_eta: 0.0054 | Regression_pt: 0.4161 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1900 | Classification_binary: 0.3754 | Regression_cos_phi: 0.2528 | Regression_energy: 0.2111 | Regression_eta: 0.1514 | Regression_pt: 0.2296 | Regression_sin_phi: 0.2113 | LR: 9.96e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.9431s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 1.0006 | max_loss: 0.1446 | max_backward: 1.7671 | max_optimizer_step: 0.2923 | max_tensorboard_logging: 1.2733 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6609.6 | uss_MB: 5403.3 | child_rss_MB: 6178.9 | allocated_MB: 445.7 | reserved_MB: 24780.0 | inactive_split_MB: 1132.5 +rank2 [2026-10-28 13:06:12,757] INFO : Step 2300/50000 rank2 | Train Loss: 2.7462 | Train Loss Unweighted: 1.5643 | Train Loss Components: Auxiliary: 0.5175 | Cardinality: 0.2724 | Classification: 0.3064 | Classification_binary: 0.4106 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4145 | Regression_eta: 0.0056 | Regression_pt: 0.4213 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1653 | Classification_binary: 0.4121 | Regression_cos_phi: 0.2364 | Regression_energy: 0.2013 | Regression_eta: 0.1699 | Regression_pt: 0.2239 | Regression_sin_phi: 0.2136 | LR: 9.95e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.7174s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0353 | max_forward: 0.7930 | max_loss: 0.1572 | max_backward: 1.9535 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0140 | Memory: rss_MB: 5702.4 | uss_MB: 4498.0 | child_rss_MB: 6178.9 | allocated_MB: 451.2 | reserved_MB: 26626.0 | inactive_split_MB: 1036.8 +rank2 [2026-10-28 13:10:49,217] INFO : Step 2400/50000 rank2 | Train Loss: 2.5721 | Train Loss Unweighted: 1.4496 | Train Loss Components: Auxiliary: 0.4832 | Cardinality: 0.2667 | Classification: 0.2677 | Classification_binary: 0.3686 | HitGrouping: 0.2521 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3975 | Regression_eta: 0.0053 | Regression_pt: 0.4051 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1561 | Classification_binary: 0.3696 | Regression_cos_phi: 0.1696 | Regression_energy: 0.2140 | Regression_eta: 0.1278 | Regression_pt: 0.2376 | Regression_sin_phi: 0.2001 | LR: 9.95e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.0751s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7367 | max_loss: 0.1564 | max_backward: 1.6902 | max_optimizer_step: 0.2413 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5703.1 | uss_MB: 4496.2 | child_rss_MB: 6167.5 | allocated_MB: 448.4 | reserved_MB: 26626.0 | inactive_split_MB: 1138.1 +rank2 [2026-10-28 13:15:33,529] INFO : Step 2500/50000 rank2 | Train Loss: 2.3709 | Train Loss Unweighted: 1.3478 | Train Loss Components: Auxiliary: 0.4634 | Cardinality: 0.2742 | Classification: 0.2528 | Classification_binary: 0.3100 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3861 | Regression_eta: 0.0042 | Regression_pt: 0.3903 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1255 | Classification_binary: 0.3008 | Regression_cos_phi: 0.1387 | Regression_energy: 0.2014 | Regression_eta: 0.1195 | Regression_pt: 0.2176 | Regression_sin_phi: 0.1411 | LR: 9.94e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3397s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7598 | max_loss: 0.2442 | max_backward: 1.9617 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5783 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5689.7 | uss_MB: 4483.6 | child_rss_MB: 6173.2 | allocated_MB: 447.2 | reserved_MB: 27052.0 | inactive_split_MB: 1262.0 +rank2 [2026-10-28 13:20:26,044] INFO : Step 2600/50000 rank2 | Train Loss: 2.6216 | Train Loss Unweighted: 1.5111 | Train Loss Components: Auxiliary: 0.4988 | Cardinality: 0.2823 | Classification: 0.2863 | Classification_binary: 0.3723 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4174 | Regression_eta: 0.0052 | Regression_pt: 0.4247 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1918 | Classification_binary: 0.3659 | Regression_cos_phi: 0.2208 | Regression_energy: 0.2174 | Regression_eta: 0.1228 | Regression_pt: 0.2374 | Regression_sin_phi: 0.1812 | LR: 9.94e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.2886s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8125 | max_loss: 0.1494 | max_backward: 1.9790 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5698.8 | uss_MB: 4500.2 | child_rss_MB: 6171.1 | allocated_MB: 448.2 | reserved_MB: 28734.0 | inactive_split_MB: 1300.2 +rank2 [2026-10-28 13:25:15,883] INFO : Step 2700/50000 rank2 | Train Loss: 2.5735 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4801 | Cardinality: 0.2457 | Classification: 0.2692 | Classification_binary: 0.4021 | HitGrouping: 0.2514 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3948 | Regression_eta: 0.0060 | Regression_pt: 0.3979 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1636 | Classification_binary: 0.4247 | Regression_cos_phi: 0.1875 | Regression_energy: 0.2206 | Regression_eta: 0.1694 | Regression_pt: 0.2402 | Regression_sin_phi: 0.2462 | LR: 9.94e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3198s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0070 | max_forward: 0.8435 | max_loss: 0.1556 | max_backward: 1.9409 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5695.7 | uss_MB: 4500.5 | child_rss_MB: 6182.0 | allocated_MB: 447.4 | reserved_MB: 28734.0 | inactive_split_MB: 980.7 +rank2 [2026-10-28 13:29:49,852] INFO : Step 2800/50000 rank2 | Train Loss: 2.3517 | Train Loss Unweighted: 1.3315 | Train Loss Components: Auxiliary: 0.4396 | Cardinality: 0.2218 | Classification: 0.2269 | Classification_binary: 0.3206 | HitGrouping: 0.2464 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3829 | Regression_eta: 0.0052 | Regression_pt: 0.3909 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1015 | Classification_binary: 0.3076 | Regression_cos_phi: 0.1327 | Regression_energy: 0.2268 | Regression_eta: 0.0954 | Regression_pt: 0.2474 | Regression_sin_phi: 0.1860 | LR: 9.93e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3269s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0062 | max_forward: 0.8898 | max_loss: 0.1952 | max_backward: 1.8882 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0163 | Memory: rss_MB: 5687.7 | uss_MB: 4493.8 | child_rss_MB: 6179.6 | allocated_MB: 445.9 | reserved_MB: 28734.0 | inactive_split_MB: 1208.5 +rank2 [2026-10-28 13:34:34,297] INFO : Step 2900/50000 rank2 | Train Loss: 2.4505 | Train Loss Unweighted: 1.3674 | Train Loss Components: Auxiliary: 0.4603 | Cardinality: 0.2746 | Classification: 0.2611 | Classification_binary: 0.3388 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3750 | Regression_eta: 0.0049 | Regression_pt: 0.3827 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1144 | Classification_binary: 0.3169 | Regression_cos_phi: 0.1873 | Regression_energy: 0.2019 | Regression_eta: 0.1025 | Regression_pt: 0.2221 | Regression_sin_phi: 0.1425 | LR: 9.93e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.3932s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7484 | max_loss: 0.6089 | max_backward: 2.2353 | max_optimizer_step: 0.1775 | max_tensorboard_logging: 2.4313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0155 | Memory: rss_MB: 5691.8 | uss_MB: 4497.3 | child_rss_MB: 6175.9 | allocated_MB: 446.7 | reserved_MB: 28734.0 | inactive_split_MB: 1133.2 +rank2 [2026-10-28 13:39:22,258] INFO : Step 3000/50000 rank2 | Train Loss: 2.6179 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4760 | Cardinality: 0.2600 | Classification: 0.3170 | Classification_binary: 0.3907 | HitGrouping: 0.2509 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3743 | Regression_eta: 0.0054 | Regression_pt: 0.3819 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1948 | Classification_binary: 0.4148 | Regression_cos_phi: 0.2159 | Regression_energy: 0.2056 | Regression_eta: 0.1598 | Regression_pt: 0.2270 | Regression_sin_phi: 0.2150 | LR: 9.92e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8199s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0842 | max_forward: 0.8203 | max_loss: 0.1511 | max_backward: 1.9714 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.0307 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5691.4 | uss_MB: 4499.3 | child_rss_MB: 6154.8 | allocated_MB: 451.4 | reserved_MB: 28740.0 | inactive_split_MB: 1256.0 +rank2 [2026-10-28 13:39:23,128] INFO : Running validation on rank2 +rank2 [2026-10-28 13:39:23,129] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 13:39:23,129] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 13:40:15,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:40:16,491] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 13:40:43,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 13:40:44,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 13:40:46,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 13:40:47,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 13:40:57,037] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:40:57,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:40:58,514] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 13:41:14,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 13:41:15,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 13:41:16,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 13:41:16,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 13:41:27,006] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 13:41:28,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 13:41:28,597] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 13:41:44,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 13:41:45,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 13:41:46,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 13:41:47,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 13:41:57,115] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 13:46:51,408] INFO : Step 3100/50000 rank2 | Train Loss: 2.3456 | Train Loss Unweighted: 1.3820 | Train Loss Components: Auxiliary: 0.4397 | Cardinality: 0.2017 | Classification: 0.2549 | Classification_binary: 0.3432 | HitGrouping: 0.2399 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3852 | Regression_eta: 0.0046 | Regression_pt: 0.3892 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1423 | Classification_binary: 0.3413 | Regression_cos_phi: 0.1930 | Regression_energy: 0.2024 | Regression_eta: 0.1585 | Regression_pt: 0.2197 | Regression_sin_phi: 0.2235 | LR: 9.91e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4913s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0425 | max_forward: 0.7824 | max_loss: 0.1439 | max_backward: 1.7183 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.8666 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6905.9 | uss_MB: 5708.9 | child_rss_MB: 6176.0 | allocated_MB: 447.6 | reserved_MB: 24468.0 | inactive_split_MB: 1269.5 +rank2 [2026-10-28 13:51:36,981] INFO : Step 3200/50000 rank2 | Train Loss: 2.2453 | Train Loss Unweighted: 1.2729 | Train Loss Components: Auxiliary: 0.4171 | Cardinality: 0.2143 | Classification: 0.2366 | Classification_binary: 0.3116 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3569 | Regression_eta: 0.0047 | Regression_pt: 0.3584 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1278 | Classification_binary: 0.2989 | Regression_cos_phi: 0.2045 | Regression_energy: 0.1969 | Regression_eta: 0.1518 | Regression_pt: 0.2131 | Regression_sin_phi: 0.2295 | LR: 9.91e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6651s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7985 | max_loss: 0.1379 | max_backward: 1.7895 | max_optimizer_step: 0.2027 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6908.5 | uss_MB: 5710.9 | child_rss_MB: 6167.1 | allocated_MB: 448.6 | reserved_MB: 26448.0 | inactive_split_MB: 1391.4 +rank2 [2026-10-28 13:56:16,619] INFO : Step 3300/50000 rank2 | Train Loss: 2.5474 | Train Loss Unweighted: 1.4472 | Train Loss Components: Auxiliary: 0.4779 | Cardinality: 0.2732 | Classification: 0.2824 | Classification_binary: 0.3640 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3911 | Regression_eta: 0.0054 | Regression_pt: 0.3992 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1670 | Classification_binary: 0.3885 | Regression_cos_phi: 0.2024 | Regression_energy: 0.2062 | Regression_eta: 0.1545 | Regression_pt: 0.2282 | Regression_sin_phi: 0.2249 | LR: 9.90e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6668s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8323 | max_loss: 0.1479 | max_backward: 2.0543 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.7834 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5785.1 | uss_MB: 4587.8 | child_rss_MB: 6143.0 | allocated_MB: 448.7 | reserved_MB: 26460.0 | inactive_split_MB: 1087.4 +rank2 [2026-10-28 14:00:49,248] INFO : Step 3400/50000 rank2 | Train Loss: 2.5743 | Train Loss Unweighted: 1.4358 | Train Loss Components: Auxiliary: 0.4720 | Cardinality: 0.2664 | Classification: 0.2699 | Classification_binary: 0.3915 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3784 | Regression_eta: 0.0053 | Regression_pt: 0.3851 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1820 | Classification_binary: 0.4200 | Regression_cos_phi: 0.2931 | Regression_energy: 0.1844 | Regression_eta: 0.1715 | Regression_pt: 0.2042 | Regression_sin_phi: 0.2933 | LR: 9.90e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2614s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7827 | max_loss: 0.1630 | max_backward: 1.9191 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5606 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5794.1 | uss_MB: 4597.7 | child_rss_MB: 6161.5 | allocated_MB: 450.4 | reserved_MB: 28138.0 | inactive_split_MB: 1261.0 +rank2 [2026-10-28 14:05:23,603] INFO : Step 3500/50000 rank2 | Train Loss: 2.1916 | Train Loss Unweighted: 1.2418 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2204 | Classification: 0.2212 | Classification_binary: 0.3041 | HitGrouping: 0.2426 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3517 | Regression_eta: 0.0047 | Regression_pt: 0.3558 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1354 | Classification_binary: 0.2799 | Regression_cos_phi: 0.1756 | Regression_energy: 0.1897 | Regression_eta: 0.1468 | Regression_pt: 0.2088 | Regression_sin_phi: 0.1410 | LR: 9.89e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2264s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7972 | max_loss: 0.1441 | max_backward: 1.7398 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6845 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5780.0 | uss_MB: 4584.6 | child_rss_MB: 6167.3 | allocated_MB: 443.7 | reserved_MB: 28532.0 | inactive_split_MB: 1176.3 +rank2 [2026-10-28 14:10:06,692] INFO : Step 3600/50000 rank2 | Train Loss: 2.3195 | Train Loss Unweighted: 1.3024 | Train Loss Components: Auxiliary: 0.4248 | Cardinality: 0.2221 | Classification: 0.2518 | Classification_binary: 0.3172 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3610 | Regression_eta: 0.0052 | Regression_pt: 0.3623 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1613 | Classification_binary: 0.3345 | Regression_cos_phi: 0.2256 | Regression_energy: 0.1860 | Regression_eta: 0.1671 | Regression_pt: 0.1997 | Regression_sin_phi: 0.2409 | LR: 9.89e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3074s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8456 | max_loss: 0.1388 | max_backward: 1.9466 | max_optimizer_step: 0.1811 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5787.8 | uss_MB: 4598.8 | child_rss_MB: 6163.2 | allocated_MB: 449.1 | reserved_MB: 28532.0 | inactive_split_MB: 1277.8 +rank2 [2026-10-28 14:14:47,798] INFO : Step 3700/50000 rank2 | Train Loss: 2.5861 | Train Loss Unweighted: 1.4427 | Train Loss Components: Auxiliary: 0.4700 | Cardinality: 0.2678 | Classification: 0.2978 | Classification_binary: 0.3844 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3719 | Regression_eta: 0.0058 | Regression_pt: 0.3773 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1648 | Classification_binary: 0.3740 | Regression_cos_phi: 0.2176 | Regression_energy: 0.2195 | Regression_eta: 0.1411 | Regression_pt: 0.2366 | Regression_sin_phi: 0.2310 | LR: 9.88e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5012s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.8001 | max_loss: 0.1519 | max_backward: 1.8239 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.7916 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0181 | Memory: rss_MB: 5775.1 | uss_MB: 4600.5 | child_rss_MB: 6165.4 | allocated_MB: 450.6 | reserved_MB: 30118.0 | inactive_split_MB: 1097.6 +rank2 [2026-10-28 14:19:24,598] INFO : Step 3800/50000 rank2 | Train Loss: 2.3269 | Train Loss Unweighted: 1.3016 | Train Loss Components: Auxiliary: 0.4225 | Cardinality: 0.2364 | Classification: 0.2400 | Classification_binary: 0.3389 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3534 | Regression_eta: 0.0054 | Regression_pt: 0.3591 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1230 | Classification_binary: 0.3385 | Regression_cos_phi: 0.1952 | Regression_energy: 0.1983 | Regression_eta: 0.1351 | Regression_pt: 0.2162 | Regression_sin_phi: 0.1800 | LR: 9.87e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.4698s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7555 | max_loss: 0.1601 | max_backward: 1.7612 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.9394 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5772.0 | uss_MB: 4595.7 | child_rss_MB: 6158.5 | allocated_MB: 447.4 | reserved_MB: 30118.0 | inactive_split_MB: 1355.2 +rank2 [2026-10-28 14:24:05,269] INFO : Step 3900/50000 rank2 | Train Loss: 2.0276 | Train Loss Unweighted: 1.1874 | Train Loss Components: Auxiliary: 0.3861 | Cardinality: 0.1825 | Classification: 0.2116 | Classification_binary: 0.2707 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3485 | Regression_eta: 0.0038 | Regression_pt: 0.3489 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1410 | Classification_binary: 0.2860 | Regression_cos_phi: 0.1394 | Regression_energy: 0.1864 | Regression_eta: 0.1029 | Regression_pt: 0.2004 | Regression_sin_phi: 0.1903 | LR: 9.87e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.4480s | Phase Times: max_pre_step_barrier: 0.0095 | max_batch_to_device: 0.0140 | max_forward: 1.1007 | max_loss: 0.1867 | max_backward: 2.0952 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 0.5091 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5772.2 | uss_MB: 4598.5 | child_rss_MB: 6165.7 | allocated_MB: 448.6 | reserved_MB: 30118.0 | inactive_split_MB: 1119.4 +rank2 [2026-10-28 14:28:52,324] INFO : Step 4000/50000 rank2 | Train Loss: 2.3921 | Train Loss Unweighted: 1.3683 | Train Loss Components: Auxiliary: 0.4366 | Cardinality: 0.2254 | Classification: 0.2720 | Classification_binary: 0.3524 | HitGrouping: 0.2405 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3647 | Regression_eta: 0.0048 | Regression_pt: 0.3692 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1617 | Classification_binary: 0.4063 | Regression_cos_phi: 0.2370 | Regression_energy: 0.1810 | Regression_eta: 0.1700 | Regression_pt: 0.1967 | Regression_sin_phi: 0.2422 | LR: 9.86e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8914s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.8532 | max_loss: 0.1476 | max_backward: 2.4362 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.6138 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5773.5 | uss_MB: 4601.5 | child_rss_MB: 6164.9 | allocated_MB: 449.7 | reserved_MB: 30118.0 | inactive_split_MB: 899.8 +rank2 [2026-10-28 14:28:53,307] INFO : Running validation on rank2 +rank2 [2026-10-28 14:28:53,308] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 14:28:53,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 14:29:46,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:29:46,699] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 14:30:12,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 14:30:14,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 14:30:15,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 14:30:16,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 14:30:26,118] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 14:30:26,980] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:30:27,512] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 14:30:43,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 14:30:44,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 14:30:45,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 14:30:46,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 14:30:56,189] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 14:30:57,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 14:30:57,799] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 14:31:13,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 14:31:14,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 14:31:14,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 14:31:15,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 14:31:25,794] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 14:36:30,159] INFO : Step 4100/50000 rank2 | Train Loss: 2.5062 | Train Loss Unweighted: 1.3582 | Train Loss Components: Auxiliary: 0.4559 | Cardinality: 0.3208 | Classification: 0.2565 | Classification_binary: 0.3823 | HitGrouping: 0.2386 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3543 | Regression_eta: 0.0046 | Regression_pt: 0.3553 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1646 | Classification_binary: 0.3455 | Regression_cos_phi: 0.2149 | Regression_energy: 0.1843 | Regression_eta: 0.1196 | Regression_pt: 0.1949 | Regression_sin_phi: 0.1801 | LR: 9.85e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.2052s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0069 | max_forward: 0.8943 | max_loss: 0.2518 | max_backward: 1.9865 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 1.1537 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6976.0 | uss_MB: 5798.7 | child_rss_MB: 6190.7 | allocated_MB: 449.4 | reserved_MB: 25148.0 | inactive_split_MB: 1028.6 +rank2 [2026-10-28 14:41:18,155] INFO : Step 4200/50000 rank2 | Train Loss: 2.0961 | Train Loss Unweighted: 1.2232 | Train Loss Components: Auxiliary: 0.3927 | Cardinality: 0.2101 | Classification: 0.2242 | Classification_binary: 0.2902 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3497 | Regression_eta: 0.0042 | Regression_pt: 0.3510 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1460 | Classification_binary: 0.2498 | Regression_cos_phi: 0.1717 | Regression_energy: 0.1776 | Regression_eta: 0.0975 | Regression_pt: 0.1902 | Regression_sin_phi: 0.1399 | LR: 9.84e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.6538s | Phase Times: max_pre_step_barrier: 0.1131 | max_batch_to_device: 0.0199 | max_forward: 1.5911 | max_loss: 0.3393 | max_backward: 3.0707 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5197 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6973.9 | uss_MB: 5797.7 | child_rss_MB: 6186.7 | allocated_MB: 446.0 | reserved_MB: 25296.0 | inactive_split_MB: 922.1 +rank2 [2026-10-28 14:46:18,702] INFO : Step 4300/50000 rank2 | Train Loss: 2.2792 | Train Loss Unweighted: 1.3065 | Train Loss Components: Auxiliary: 0.4224 | Cardinality: 0.2158 | Classification: 0.2339 | Classification_binary: 0.3250 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3669 | Regression_eta: 0.0044 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1495 | Classification_binary: 0.3340 | Regression_cos_phi: 0.2099 | Regression_energy: 0.1941 | Regression_eta: 0.1276 | Regression_pt: 0.2174 | Regression_sin_phi: 0.1881 | LR: 9.84e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5471s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0938 | max_forward: 0.8451 | max_loss: 0.1396 | max_backward: 1.9669 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.6639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5862.7 | uss_MB: 4685.4 | child_rss_MB: 6195.4 | allocated_MB: 448.4 | reserved_MB: 28580.0 | inactive_split_MB: 1346.0 +rank2 [2026-10-28 14:50:58,886] INFO : Step 4400/50000 rank2 | Train Loss: 2.4955 | Train Loss Unweighted: 1.3385 | Train Loss Components: Auxiliary: 0.4510 | Cardinality: 0.3020 | Classification: 0.2394 | Classification_binary: 0.3842 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3482 | Regression_eta: 0.0051 | Regression_pt: 0.3563 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1226 | Classification_binary: 0.3744 | Regression_cos_phi: 0.2604 | Regression_energy: 0.1779 | Regression_eta: 0.1248 | Regression_pt: 0.1979 | Regression_sin_phi: 0.2102 | LR: 9.83e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2045s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0083 | max_forward: 0.7889 | max_loss: 0.1505 | max_backward: 1.8572 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4868 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5870.1 | uss_MB: 4697.1 | child_rss_MB: 6175.1 | allocated_MB: 449.1 | reserved_MB: 28582.0 | inactive_split_MB: 1105.9 +rank2 [2026-10-28 14:55:40,224] INFO : Step 4500/50000 rank2 | Train Loss: 2.3283 | Train Loss Unweighted: 1.2750 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.3015 | Classification: 0.2318 | Classification_binary: 0.3495 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3409 | Regression_eta: 0.0044 | Regression_pt: 0.3441 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1343 | Classification_binary: 0.3339 | Regression_cos_phi: 0.1592 | Regression_energy: 0.1829 | Regression_eta: 0.0991 | Regression_pt: 0.2025 | Regression_sin_phi: 0.1719 | LR: 9.82e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0180s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7326 | max_loss: 0.2428 | max_backward: 1.7841 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5868.2 | uss_MB: 4696.0 | child_rss_MB: 6178.6 | allocated_MB: 447.8 | reserved_MB: 28582.0 | inactive_split_MB: 1258.0 +rank2 [2026-10-28 15:00:16,448] INFO : Step 4600/50000 rank2 | Train Loss: 2.0353 | Train Loss Unweighted: 1.1777 | Train Loss Components: Auxiliary: 0.3804 | Cardinality: 0.1920 | Classification: 0.2053 | Classification_binary: 0.2642 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3490 | Regression_eta: 0.0039 | Regression_pt: 0.3512 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1430 | Classification_binary: 0.2645 | Regression_cos_phi: 0.1346 | Regression_energy: 0.1826 | Regression_eta: 0.1026 | Regression_pt: 0.1968 | Regression_sin_phi: 0.1727 | LR: 9.81e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.0845s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0402 | max_forward: 0.7143 | max_loss: 0.1412 | max_backward: 1.6726 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 5861.4 | uss_MB: 4694.6 | child_rss_MB: 6186.8 | allocated_MB: 446.4 | reserved_MB: 28582.0 | inactive_split_MB: 981.4 +rank2 [2026-10-28 15:05:12,090] INFO : Step 4700/50000 rank2 | Train Loss: 2.2426 | Train Loss Unweighted: 1.2576 | Train Loss Components: Auxiliary: 0.4073 | Cardinality: 0.2222 | Classification: 0.2485 | Classification_binary: 0.3333 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3307 | Regression_eta: 0.0049 | Regression_pt: 0.3356 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1578 | Classification_binary: 0.3674 | Regression_cos_phi: 0.1923 | Regression_energy: 0.1733 | Regression_eta: 0.2208 | Regression_pt: 0.1905 | Regression_sin_phi: 0.2395 | LR: 9.81e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6196s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.1471 | max_backward: 2.0627 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.7062 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0769 | Memory: rss_MB: 5860.5 | uss_MB: 4699.2 | child_rss_MB: 6169.2 | allocated_MB: 447.7 | reserved_MB: 28652.0 | inactive_split_MB: 1164.9 +rank2 [2026-10-28 15:09:59,906] INFO : Step 4800/50000 rank2 | Train Loss: 2.2697 | Train Loss Unweighted: 1.2637 | Train Loss Components: Auxiliary: 0.4132 | Cardinality: 0.2493 | Classification: 0.2262 | Classification_binary: 0.3667 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3287 | Regression_eta: 0.0044 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1555 | Classification_binary: 0.3936 | Regression_cos_phi: 0.2210 | Regression_energy: 0.1843 | Regression_eta: 0.1375 | Regression_pt: 0.2007 | Regression_sin_phi: 0.2518 | LR: 9.80e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0687s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0065 | max_forward: 0.7626 | max_loss: 0.1571 | max_backward: 1.6465 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5855.3 | uss_MB: 4693.4 | child_rss_MB: 6188.9 | allocated_MB: 450.7 | reserved_MB: 28652.0 | inactive_split_MB: 1088.6 +rank2 [2026-10-28 15:14:51,789] INFO : Step 4900/50000 rank2 | Train Loss: 2.0529 | Train Loss Unweighted: 1.1598 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.1942 | Classification: 0.2269 | Classification_binary: 0.2874 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3172 | Regression_eta: 0.0041 | Regression_pt: 0.3199 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1338 | Classification_binary: 0.2496 | Regression_cos_phi: 0.1552 | Regression_energy: 0.1728 | Regression_eta: 0.0945 | Regression_pt: 0.1857 | Regression_sin_phi: 0.1229 | LR: 9.79e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3249s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7293 | max_loss: 0.1974 | max_backward: 1.8633 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5858.6 | uss_MB: 4697.0 | child_rss_MB: 6189.0 | allocated_MB: 447.4 | reserved_MB: 28652.0 | inactive_split_MB: 1266.1 +rank2 [2026-10-28 15:19:59,464] INFO : Step 5000/50000 rank2 | Train Loss: 2.0827 | Train Loss Unweighted: 1.1872 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2154 | Classification: 0.2462 | Classification_binary: 0.2941 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3169 | Regression_eta: 0.0042 | Regression_pt: 0.3219 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1743 | Classification_binary: 0.3057 | Regression_cos_phi: 0.2029 | Regression_energy: 0.1693 | Regression_eta: 0.1341 | Regression_pt: 0.1868 | Regression_sin_phi: 0.2182 | LR: 9.78e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3261s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.8382 | max_loss: 0.1450 | max_backward: 1.9472 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0203 | Memory: rss_MB: 5858.9 | uss_MB: 4697.5 | child_rss_MB: 6177.0 | allocated_MB: 446.9 | reserved_MB: 29316.0 | inactive_split_MB: 980.8 +rank2 [2026-10-28 15:20:00,329] INFO : Running validation on rank2 +rank2 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 15:20:55,978] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:20:56,481] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 15:21:22,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 15:21:23,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 15:21:24,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 15:21:25,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 15:21:35,992] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 15:21:36,445] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:21:36,966] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 15:21:53,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 15:21:54,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 15:21:55,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 15:21:56,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 15:22:06,507] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 15:22:06,553] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 15:22:07,050] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 15:22:22,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 15:22:24,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 15:22:24,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 15:22:26,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 15:22:36,169] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 15:28:09,086] INFO : Step 5100/50000 rank2 | Train Loss: 2.4098 | Train Loss Unweighted: 1.3457 | Train Loss Components: Auxiliary: 0.4318 | Cardinality: 0.2397 | Classification: 0.2711 | Classification_binary: 0.3798 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3391 | Regression_eta: 0.0051 | Regression_pt: 0.3453 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1543 | Classification_binary: 0.3851 | Regression_cos_phi: 0.2671 | Regression_energy: 0.1734 | Regression_eta: 0.1408 | Regression_pt: 0.1904 | Regression_sin_phi: 0.2424 | LR: 9.77e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.8401s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1511 | max_backward: 1.7914 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 1.0544 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5949.7 | uss_MB: 4786.0 | child_rss_MB: 6159.3 | allocated_MB: 447.6 | reserved_MB: 25650.0 | inactive_split_MB: 1122.4 +rank2 [2026-10-28 15:32:58,661] INFO : Step 5200/50000 rank2 | Train Loss: 2.0929 | Train Loss Unweighted: 1.1974 | Train Loss Components: Auxiliary: 0.3875 | Cardinality: 0.2007 | Classification: 0.2193 | Classification_binary: 0.3278 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3194 | Regression_eta: 0.0041 | Regression_pt: 0.3227 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1224 | Classification_binary: 0.3228 | Regression_cos_phi: 0.2390 | Regression_energy: 0.1497 | Regression_eta: 0.1032 | Regression_pt: 0.1656 | Regression_sin_phi: 0.1599 | LR: 9.76e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4918s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7458 | max_loss: 0.2423 | max_backward: 1.7123 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9648 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5957.0 | uss_MB: 4796.7 | child_rss_MB: 6179.0 | allocated_MB: 450.7 | reserved_MB: 28708.0 | inactive_split_MB: 1139.3 +rank2 [2026-10-28 15:37:52,596] INFO : Step 5300/50000 rank2 | Train Loss: 1.9905 | Train Loss Unweighted: 1.1352 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.2083 | Classification: 0.2239 | Classification_binary: 0.2818 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0037 | Regression_pt: 0.3135 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1510 | Classification_binary: 0.2724 | Regression_cos_phi: 0.1733 | Regression_energy: 0.1629 | Regression_eta: 0.1162 | Regression_pt: 0.1807 | Regression_sin_phi: 0.1765 | LR: 9.75e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1036s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 1.2453 | max_loss: 0.2326 | max_backward: 2.1392 | max_optimizer_step: 0.1883 | max_tensorboard_logging: 1.0947 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5945.0 | uss_MB: 4784.6 | child_rss_MB: 6156.3 | allocated_MB: 445.9 | reserved_MB: 30688.0 | inactive_split_MB: 925.3 +rank2 [2026-10-28 15:42:39,663] INFO : Step 5400/50000 rank2 | Train Loss: 2.2932 | Train Loss Unweighted: 1.2957 | Train Loss Components: Auxiliary: 0.4189 | Cardinality: 0.2411 | Classification: 0.2689 | Classification_binary: 0.3420 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3351 | Regression_eta: 0.0045 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1671 | Classification_binary: 0.3248 | Regression_cos_phi: 0.1860 | Regression_energy: 0.1698 | Regression_eta: 0.1177 | Regression_pt: 0.1857 | Regression_sin_phi: 0.1971 | LR: 9.74e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5701s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7688 | max_loss: 0.1697 | max_backward: 1.8189 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8641 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5955.2 | uss_MB: 4796.8 | child_rss_MB: 6163.2 | allocated_MB: 450.8 | reserved_MB: 30688.0 | inactive_split_MB: 915.1 +rank2 [2026-10-28 15:47:42,229] INFO : Step 5500/50000 rank2 | Train Loss: 2.1734 | Train Loss Unweighted: 1.2295 | Train Loss Components: Auxiliary: 0.3970 | Cardinality: 0.2269 | Classification: 0.2218 | Classification_binary: 0.3571 | HitGrouping: 0.2200 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3188 | Regression_eta: 0.0041 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1165 | Classification_binary: 0.3472 | Regression_cos_phi: 0.1463 | Regression_energy: 0.1674 | Regression_eta: 0.1081 | Regression_pt: 0.1822 | Regression_sin_phi: 0.1665 | LR: 9.73e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.9285s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0153 | max_forward: 1.1913 | max_loss: 0.6135 | max_backward: 3.0599 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.6982 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5952.5 | uss_MB: 4797.4 | child_rss_MB: 6170.1 | allocated_MB: 446.9 | reserved_MB: 32804.0 | inactive_split_MB: 913.2 +rank2 [2026-10-28 15:52:30,288] INFO : Step 5600/50000 rank2 | Train Loss: 2.0307 | Train Loss Unweighted: 1.1169 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.1966 | Classification: 0.2085 | Classification_binary: 0.3031 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2975 | Regression_eta: 0.0047 | Regression_pt: 0.2988 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1267 | Classification_binary: 0.3061 | Regression_cos_phi: 0.1662 | Regression_energy: 0.1458 | Regression_eta: 0.1682 | Regression_pt: 0.1583 | Regression_sin_phi: 0.1792 | LR: 9.72e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.7953s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7794 | max_loss: 0.1497 | max_backward: 1.9410 | max_optimizer_step: 0.5154 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0061 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5920.9 | uss_MB: 4797.0 | child_rss_MB: 6174.4 | allocated_MB: 445.4 | reserved_MB: 32804.0 | inactive_split_MB: 1164.2 +rank2 [2026-10-28 15:57:16,862] INFO : Step 5700/50000 rank2 | Train Loss: 2.2064 | Train Loss Unweighted: 1.2224 | Train Loss Components: Auxiliary: 0.4023 | Cardinality: 0.2320 | Classification: 0.2526 | Classification_binary: 0.3174 | HitGrouping: 0.2343 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3173 | Regression_eta: 0.0045 | Regression_pt: 0.3259 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1454 | Classification_binary: 0.2989 | Regression_cos_phi: 0.1523 | Regression_energy: 0.1634 | Regression_eta: 0.1118 | Regression_pt: 0.1800 | Regression_sin_phi: 0.1448 | LR: 9.71e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.6681s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7746 | max_loss: 0.4928 | max_backward: 1.9590 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8719 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 5893.1 | uss_MB: 3755.9 | child_rss_MB: 6168.2 | allocated_MB: 448.4 | reserved_MB: 32804.0 | inactive_split_MB: 913.0 +rank2 [2026-10-28 16:02:01,174] INFO : Step 5800/50000 rank2 | Train Loss: 2.1438 | Train Loss Unweighted: 1.2123 | Train Loss Components: Auxiliary: 0.3898 | Cardinality: 0.2335 | Classification: 0.2404 | Classification_binary: 0.3575 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3006 | Regression_eta: 0.0042 | Regression_pt: 0.3054 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1526 | Classification_binary: 0.3552 | Regression_cos_phi: 0.2022 | Regression_energy: 0.1526 | Regression_eta: 0.1215 | Regression_pt: 0.1683 | Regression_sin_phi: 0.1665 | LR: 9.70e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.5871s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8028 | max_loss: 0.1540 | max_backward: 1.8645 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.8379 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5883.5 | uss_MB: 4797.1 | child_rss_MB: 6169.4 | allocated_MB: 448.3 | reserved_MB: 32804.0 | inactive_split_MB: 916.5 +rank2 [2026-10-28 16:06:46,106] INFO : Step 5900/50000 rank2 | Train Loss: 2.2094 | Train Loss Unweighted: 1.1453 | Train Loss Components: Auxiliary: 0.3936 | Cardinality: 0.2891 | Classification: 0.2198 | Classification_binary: 0.3337 | HitGrouping: 0.2238 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2890 | Regression_eta: 0.0046 | Regression_pt: 0.2935 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1658 | Classification_binary: 0.3747 | Regression_cos_phi: 0.1998 | Regression_energy: 0.1761 | Regression_eta: 0.1811 | Regression_pt: 0.1886 | Regression_sin_phi: 0.2155 | LR: 9.69e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.5862s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0065 | max_forward: 1.0922 | max_loss: 0.1477 | max_backward: 1.8468 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8251 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0991 | Memory: rss_MB: 5881.4 | uss_MB: 4798.1 | child_rss_MB: 6148.2 | allocated_MB: 446.5 | reserved_MB: 32804.0 | inactive_split_MB: 927.1 +rank2 [2026-10-28 16:11:31,999] INFO : Step 6000/50000 rank2 | Train Loss: 1.8458 | Train Loss Unweighted: 1.0363 | Train Loss Components: Auxiliary: 0.3398 | Cardinality: 0.1796 | Classification: 0.1819 | Classification_binary: 0.2710 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2865 | Regression_eta: 0.0032 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1288 | Classification_binary: 0.2537 | Regression_cos_phi: 0.1221 | Regression_energy: 0.1482 | Regression_eta: 0.0848 | Regression_pt: 0.1611 | Regression_sin_phi: 0.1448 | LR: 9.68e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5085s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7380 | max_loss: 0.2483 | max_backward: 1.8094 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.9243 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5850.9 | uss_MB: 4795.2 | child_rss_MB: 6173.4 | allocated_MB: 448.1 | reserved_MB: 32804.0 | inactive_split_MB: 1232.3 +rank2 [2026-10-28 16:11:32,848] INFO : Running validation on rank2 +rank2 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 16:12:26,406] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:12:26,927] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 16:12:54,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 16:12:55,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 16:12:56,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 16:12:57,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 16:13:07,546] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-28 16:13:07,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:13:08,323] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 16:13:24,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 16:13:25,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 16:13:26,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 16:13:27,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 16:13:37,748] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 16:13:38,169] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 16:13:38,687] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 16:13:54,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 16:13:55,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 16:13:56,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 16:13:57,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 16:14:07,186] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 16:19:13,933] INFO : Step 6100/50000 rank2 | Train Loss: 2.0818 | Train Loss Unweighted: 1.1297 | Train Loss Components: Auxiliary: 0.3734 | Cardinality: 0.2287 | Classification: 0.2079 | Classification_binary: 0.3249 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2929 | Regression_eta: 0.0044 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1157 | Classification_binary: 0.3195 | Regression_cos_phi: 0.1947 | Regression_energy: 0.1566 | Regression_eta: 0.1189 | Regression_pt: 0.1701 | Regression_sin_phi: 0.1316 | LR: 9.67e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.5154s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8130 | max_loss: 0.1524 | max_backward: 3.1921 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 1.4719 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5941.1 | uss_MB: 4883.2 | child_rss_MB: 6189.9 | allocated_MB: 448.4 | reserved_MB: 24434.0 | inactive_split_MB: 1287.8 +rank2 [2026-10-28 16:23:58,090] INFO : Step 6200/50000 rank2 | Train Loss: 2.3716 | Train Loss Unweighted: 1.2065 | Train Loss Components: Auxiliary: 0.4163 | Cardinality: 0.3927 | Classification: 0.2343 | Classification_binary: 0.3679 | HitGrouping: 0.2165 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2973 | Regression_eta: 0.0043 | Regression_pt: 0.2981 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1331 | Classification_binary: 0.3858 | Regression_cos_phi: 0.1624 | Regression_energy: 0.1547 | Regression_eta: 0.1100 | Regression_pt: 0.1659 | Regression_sin_phi: 0.1804 | LR: 9.66e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.5605s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8556 | max_loss: 0.1996 | max_backward: 1.9593 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.5823 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5946.6 | uss_MB: 3767.3 | child_rss_MB: 6181.8 | allocated_MB: 447.9 | reserved_MB: 27020.0 | inactive_split_MB: 1185.4 +rank2 [2026-10-28 16:28:38,491] INFO : Step 6300/50000 rank2 | Train Loss: 1.9346 | Train Loss Unweighted: 1.0816 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.2071 | Classification: 0.2053 | Classification_binary: 0.2781 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2941 | Regression_eta: 0.0037 | Regression_pt: 0.2966 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0964 | Classification_binary: 0.2707 | Regression_cos_phi: 0.1535 | Regression_energy: 0.1734 | Regression_eta: 0.0886 | Regression_pt: 0.1923 | Regression_sin_phi: 0.1177 | LR: 9.65e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5944s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0068 | max_forward: 0.7412 | max_loss: 0.1444 | max_backward: 1.7431 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.9045 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5941.6 | uss_MB: 4896.5 | child_rss_MB: 6164.9 | allocated_MB: 444.6 | reserved_MB: 27022.0 | inactive_split_MB: 1185.4 +rank2 [2026-10-28 16:33:23,144] INFO : Step 6400/50000 rank2 | Train Loss: 2.0123 | Train Loss Unweighted: 1.0992 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2082 | Classification: 0.2291 | Classification_binary: 0.2885 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2842 | Regression_eta: 0.0040 | Regression_pt: 0.2891 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1760 | Classification_binary: 0.3277 | Regression_cos_phi: 0.2173 | Regression_energy: 0.1635 | Regression_eta: 0.1507 | Regression_pt: 0.1811 | Regression_sin_phi: 0.1775 | LR: 9.64e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.7577s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0069 | max_forward: 1.1063 | max_loss: 0.2471 | max_backward: 1.9949 | max_optimizer_step: 0.2726 | max_tensorboard_logging: 0.8233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5936.1 | uss_MB: 4896.3 | child_rss_MB: 6176.5 | allocated_MB: 451.6 | reserved_MB: 27022.0 | inactive_split_MB: 1113.0 +rank2 [2026-10-28 16:37:57,183] INFO : Step 6500/50000 rank2 | Train Loss: 2.1791 | Train Loss Unweighted: 1.2220 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2499 | Classification: 0.2271 | Classification_binary: 0.3508 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3177 | Regression_eta: 0.0042 | Regression_pt: 0.3181 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1202 | Classification_binary: 0.3629 | Regression_cos_phi: 0.1853 | Regression_energy: 0.1911 | Regression_eta: 0.1117 | Regression_pt: 0.2059 | Regression_sin_phi: 0.1828 | LR: 9.63e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9962s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0548 | max_forward: 0.8069 | max_loss: 0.1518 | max_backward: 2.0692 | max_optimizer_step: 0.1860 | max_tensorboard_logging: 0.9414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5926.6 | uss_MB: 4895.5 | child_rss_MB: 6160.0 | allocated_MB: 449.6 | reserved_MB: 28336.0 | inactive_split_MB: 1199.2 +rank2 [2026-10-28 16:42:33,043] INFO : Step 6600/50000 rank2 | Train Loss: 1.9690 | Train Loss Unweighted: 1.0745 | Train Loss Components: Auxiliary: 0.3537 | Cardinality: 0.2132 | Classification: 0.1982 | Classification_binary: 0.3095 | HitGrouping: 0.2136 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2771 | Regression_eta: 0.0040 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1177 | Classification_binary: 0.2836 | Regression_cos_phi: 0.1305 | Regression_energy: 0.1353 | Regression_eta: 0.0951 | Regression_pt: 0.1497 | Regression_sin_phi: 0.1102 | LR: 9.62e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.8250s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7382 | max_loss: 0.1467 | max_backward: 1.8335 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 1.1664 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5923.1 | uss_MB: 4893.3 | child_rss_MB: 6174.8 | allocated_MB: 447.4 | reserved_MB: 28340.0 | inactive_split_MB: 1012.8 +rank2 [2026-10-28 16:47:06,903] INFO : Step 6700/50000 rank2 | Train Loss: 1.9470 | Train Loss Unweighted: 1.0781 | Train Loss Components: Auxiliary: 0.3616 | Cardinality: 0.2363 | Classification: 0.1923 | Classification_binary: 0.2661 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3028 | Regression_eta: 0.0035 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1096 | Classification_binary: 0.2449 | Regression_cos_phi: 0.1261 | Regression_energy: 0.1462 | Regression_eta: 0.0915 | Regression_pt: 0.1608 | Regression_sin_phi: 0.1496 | LR: 9.61e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6588s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7926 | max_loss: 0.2421 | max_backward: 2.0577 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.7621 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5925.5 | uss_MB: 4898.4 | child_rss_MB: 6176.2 | allocated_MB: 450.1 | reserved_MB: 29806.0 | inactive_split_MB: 1002.0 +rank2 [2026-10-28 16:51:50,914] INFO : Step 6800/50000 rank2 | Train Loss: 2.1609 | Train Loss Unweighted: 1.1507 | Train Loss Components: Auxiliary: 0.3771 | Cardinality: 0.2180 | Classification: 0.2315 | Classification_binary: 0.3225 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0025 | Regression_energy: 0.2916 | Regression_eta: 0.0048 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1257 | Classification_binary: 0.3435 | Regression_cos_phi: 0.2803 | Regression_energy: 0.1612 | Regression_eta: 0.1364 | Regression_pt: 0.1773 | Regression_sin_phi: 0.2314 | LR: 9.60e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.1916s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0070 | max_forward: 0.9144 | max_loss: 0.4866 | max_backward: 2.1484 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 1.2438 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5920.8 | uss_MB: 4896.0 | child_rss_MB: 6174.1 | allocated_MB: 451.3 | reserved_MB: 29806.0 | inactive_split_MB: 1098.9 +rank2 [2026-10-28 16:56:32,288] INFO : Step 6900/50000 rank2 | Train Loss: 1.9936 | Train Loss Unweighted: 1.1069 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.1991 | Classification: 0.2085 | Classification_binary: 0.3385 | HitGrouping: 0.2125 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2742 | Regression_eta: 0.0041 | Regression_pt: 0.2777 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1066 | Classification_binary: 0.3316 | Regression_cos_phi: 0.1333 | Regression_energy: 0.1432 | Regression_eta: 0.0911 | Regression_pt: 0.1572 | Regression_sin_phi: 0.1367 | LR: 9.58e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6486s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7764 | max_loss: 0.2225 | max_backward: 1.6637 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1184 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5918.2 | uss_MB: 4895.4 | child_rss_MB: 6170.7 | allocated_MB: 450.8 | reserved_MB: 29806.0 | inactive_split_MB: 1193.4 +rank2 [2026-10-28 17:01:16,519] INFO : Step 7000/50000 rank2 | Train Loss: 1.9256 | Train Loss Unweighted: 1.0803 | Train Loss Components: Auxiliary: 0.3608 | Cardinality: 0.2036 | Classification: 0.1946 | Classification_binary: 0.2738 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2998 | Regression_eta: 0.0039 | Regression_pt: 0.3044 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1309 | Classification_binary: 0.2794 | Regression_cos_phi: 0.1946 | Regression_energy: 0.1320 | Regression_eta: 0.1427 | Regression_pt: 0.1427 | Regression_sin_phi: 0.1563 | LR: 9.57e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.7628s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7896 | max_loss: 0.1557 | max_backward: 1.7415 | max_optimizer_step: 0.2946 | max_tensorboard_logging: 1.0959 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 5913.7 | uss_MB: 4893.6 | child_rss_MB: 6162.0 | allocated_MB: 450.3 | reserved_MB: 29806.0 | inactive_split_MB: 1283.4 +rank2 [2026-10-28 17:01:17,465] INFO : Running validation on rank2 +rank2 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 17:02:11,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:02:12,718] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 17:02:38,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 17:02:39,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 17:02:40,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 17:02:41,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 17:02:51,970] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 17:02:52,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:02:53,109] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 17:03:08,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 17:03:09,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 17:03:10,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 17:03:11,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 17:03:21,808] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 17:03:22,841] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:03:23,343] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 17:03:39,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 17:03:39,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 17:03:40,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 17:03:41,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 17:03:51,751] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 17:08:58,935] INFO : Step 7100/50000 rank2 | Train Loss: 1.8251 | Train Loss Unweighted: 1.0448 | Train Loss Components: Auxiliary: 0.3403 | Cardinality: 0.1731 | Classification: 0.2040 | Classification_binary: 0.2675 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2809 | Regression_eta: 0.0035 | Regression_pt: 0.2853 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1541 | Classification_binary: 0.2977 | Regression_cos_phi: 0.1567 | Regression_energy: 0.1658 | Regression_eta: 0.1242 | Regression_pt: 0.1821 | Regression_sin_phi: 0.1915 | LR: 9.56e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.2545s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0622 | max_forward: 0.8282 | max_loss: 0.1540 | max_backward: 1.9352 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.3584 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6015.2 | uss_MB: 4988.2 | child_rss_MB: 6155.9 | allocated_MB: 449.9 | reserved_MB: 22498.0 | inactive_split_MB: 1193.4 +rank2 [2026-10-28 17:13:31,724] INFO : Step 7200/50000 rank2 | Train Loss: 2.1146 | Train Loss Unweighted: 1.1506 | Train Loss Components: Auxiliary: 0.3887 | Cardinality: 0.2411 | Classification: 0.2243 | Classification_binary: 0.3469 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2844 | Regression_eta: 0.0044 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1358 | Classification_binary: 0.3621 | Regression_cos_phi: 0.1882 | Regression_energy: 0.1509 | Regression_eta: 0.1242 | Regression_pt: 0.1642 | Regression_sin_phi: 0.1862 | LR: 9.55e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5735s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0369 | max_forward: 0.7969 | max_loss: 0.1517 | max_backward: 1.9380 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6030.2 | uss_MB: 5003.5 | child_rss_MB: 6152.6 | allocated_MB: 448.6 | reserved_MB: 22906.0 | inactive_split_MB: 1087.6 +rank2 [2026-10-28 17:18:11,974] INFO : Step 7300/50000 rank2 | Train Loss: 1.8657 | Train Loss Unweighted: 1.0129 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2046 | Classification: 0.1759 | Classification_binary: 0.2802 | HitGrouping: 0.2112 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2746 | Regression_eta: 0.0037 | Regression_pt: 0.2747 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0927 | Classification_binary: 0.2631 | Regression_cos_phi: 0.1594 | Regression_energy: 0.1626 | Regression_eta: 0.0920 | Regression_pt: 0.1755 | Regression_sin_phi: 0.1577 | LR: 9.53e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4772s | Phase Times: max_pre_step_barrier: 0.0082 | max_batch_to_device: 0.0150 | max_forward: 0.7713 | max_loss: 0.1507 | max_backward: 1.8303 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.6113 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6029.9 | uss_MB: 5003.9 | child_rss_MB: 6151.8 | allocated_MB: 449.9 | reserved_MB: 25044.0 | inactive_split_MB: 918.5 +rank2 [2026-10-28 17:22:47,474] INFO : Step 7400/50000 rank2 | Train Loss: 1.8326 | Train Loss Unweighted: 1.0006 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2227 | Classification: 0.1795 | Classification_binary: 0.2563 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2767 | Regression_eta: 0.0034 | Regression_pt: 0.2813 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1092 | Classification_binary: 0.2595 | Regression_cos_phi: 0.1758 | Regression_energy: 0.1500 | Regression_eta: 0.1171 | Regression_pt: 0.1647 | Regression_sin_phi: 0.1728 | LR: 9.52e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.2038s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0684 | max_forward: 0.7762 | max_loss: 0.1384 | max_backward: 1.7881 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6026.9 | uss_MB: 5000.5 | child_rss_MB: 6151.8 | allocated_MB: 447.4 | reserved_MB: 25044.0 | inactive_split_MB: 1152.4 +rank2 [2026-10-28 17:27:28,847] INFO : Step 7500/50000 rank2 | Train Loss: 2.1479 | Train Loss Unweighted: 1.1510 | Train Loss Components: Auxiliary: 0.3847 | Cardinality: 0.2329 | Classification: 0.2181 | Classification_binary: 0.3323 | HitGrouping: 0.2143 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2947 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1263 | Classification_binary: 0.3442 | Regression_cos_phi: 0.2277 | Regression_energy: 0.1758 | Regression_eta: 0.1294 | Regression_pt: 0.1928 | Regression_sin_phi: 0.2165 | LR: 9.51e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2509s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7726 | max_loss: 0.2518 | max_backward: 1.8458 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1040 | Memory: rss_MB: 6026.8 | uss_MB: 5002.5 | child_rss_MB: 6153.2 | allocated_MB: 452.3 | reserved_MB: 26846.0 | inactive_split_MB: 1117.1 +rank2 [2026-10-28 17:32:29,655] INFO : Step 7600/50000 rank2 | Train Loss: 1.9984 | Train Loss Unweighted: 1.0873 | Train Loss Components: Auxiliary: 0.3667 | Cardinality: 0.2088 | Classification: 0.2053 | Classification_binary: 0.3405 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2647 | Regression_eta: 0.0038 | Regression_pt: 0.2689 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1055 | Classification_binary: 0.3209 | Regression_cos_phi: 0.1795 | Regression_energy: 0.1347 | Regression_eta: 0.0774 | Regression_pt: 0.1461 | Regression_sin_phi: 0.1637 | LR: 9.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5774s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7834 | max_loss: 0.1641 | max_backward: 1.8407 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.8255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6029.1 | uss_MB: 5003.6 | child_rss_MB: 6150.1 | allocated_MB: 448.9 | reserved_MB: 26846.0 | inactive_split_MB: 1232.3 +rank2 [2026-10-28 17:37:19,852] INFO : Step 7700/50000 rank2 | Train Loss: 1.7524 | Train Loss Unweighted: 0.9761 | Train Loss Components: Auxiliary: 0.3200 | Cardinality: 0.1783 | Classification: 0.1756 | Classification_binary: 0.2521 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2705 | Regression_eta: 0.0035 | Regression_pt: 0.2709 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0920 | Classification_binary: 0.2494 | Regression_cos_phi: 0.1609 | Regression_energy: 0.1464 | Regression_eta: 0.1155 | Regression_pt: 0.1602 | Regression_sin_phi: 0.1310 | LR: 9.48e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4844s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7659 | max_loss: 0.1551 | max_backward: 1.8103 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6027.2 | uss_MB: 5004.4 | child_rss_MB: 6154.6 | allocated_MB: 448.8 | reserved_MB: 26846.0 | inactive_split_MB: 1042.2 +rank2 [2026-10-28 17:42:10,706] INFO : Step 7800/50000 rank2 | Train Loss: 1.8039 | Train Loss Unweighted: 1.0226 | Train Loss Components: Auxiliary: 0.3378 | Cardinality: 0.1686 | Classification: 0.1822 | Classification_binary: 0.2556 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2863 | Regression_eta: 0.0034 | Regression_pt: 0.2914 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1238 | Classification_binary: 0.2747 | Regression_cos_phi: 0.2361 | Regression_energy: 0.1497 | Regression_eta: 0.1427 | Regression_pt: 0.1647 | Regression_sin_phi: 0.1856 | LR: 9.47e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.6872s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.8680 | max_loss: 0.1424 | max_backward: 1.9949 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.8163 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 6024.0 | uss_MB: 5003.5 | child_rss_MB: 6156.5 | allocated_MB: 448.6 | reserved_MB: 26846.0 | inactive_split_MB: 1396.9 +rank2 [2026-10-28 17:47:05,102] INFO : Step 7900/50000 rank2 | Train Loss: 2.0641 | Train Loss Unweighted: 1.0960 | Train Loss Components: Auxiliary: 0.3855 | Cardinality: 0.2401 | Classification: 0.2031 | Classification_binary: 0.3367 | HitGrouping: 0.2085 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2721 | Regression_eta: 0.0041 | Regression_pt: 0.2759 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1362 | Classification_binary: 0.3387 | Regression_cos_phi: 0.2235 | Regression_energy: 0.1598 | Regression_eta: 0.1151 | Regression_pt: 0.1743 | Regression_sin_phi: 0.2365 | LR: 9.46e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6365s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7971 | max_loss: 0.2430 | max_backward: 1.9131 | max_optimizer_step: 0.2681 | max_tensorboard_logging: 0.7971 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6025.3 | uss_MB: 5004.3 | child_rss_MB: 6161.5 | allocated_MB: 451.8 | reserved_MB: 27044.0 | inactive_split_MB: 827.4 +rank2 [2026-10-28 17:51:56,121] INFO : Step 8000/50000 rank2 | Train Loss: 1.8391 | Train Loss Unweighted: 0.9813 | Train Loss Components: Auxiliary: 0.3355 | Cardinality: 0.2153 | Classification: 0.1625 | Classification_binary: 0.2896 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2605 | Regression_eta: 0.0035 | Regression_pt: 0.2616 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0985 | Classification_binary: 0.2712 | Regression_cos_phi: 0.1704 | Regression_energy: 0.1488 | Regression_eta: 0.0862 | Regression_pt: 0.1634 | Regression_sin_phi: 0.1479 | LR: 9.44e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.5542s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 1.2188 | max_loss: 0.4570 | max_backward: 3.9408 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.5566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6013.9 | uss_MB: 4999.4 | child_rss_MB: 6155.0 | allocated_MB: 448.4 | reserved_MB: 27044.0 | inactive_split_MB: 1156.9 +rank2 [2026-10-28 17:51:56,979] INFO : Running validation on rank2 +rank2 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 17:52:50,095] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:52:50,644] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 17:53:16,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 17:53:17,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 17:53:18,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 17:53:19,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 17:53:29,943] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 17:53:30,251] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:53:30,807] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 17:53:46,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 17:53:47,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 17:53:48,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 17:53:49,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 17:53:59,998] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 17:54:01,631] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 17:54:02,177] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 17:54:17,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 17:54:18,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 17:54:19,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 17:54:20,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 17:54:30,071] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 17:59:43,744] INFO : Step 8100/50000 rank2 | Train Loss: 1.7101 | Train Loss Unweighted: 0.9276 | Train Loss Components: Auxiliary: 0.3166 | Cardinality: 0.1787 | Classification: 0.1664 | Classification_binary: 0.2381 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2564 | Regression_eta: 0.0032 | Regression_pt: 0.2599 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1163 | Classification_binary: 0.2302 | Regression_cos_phi: 0.1691 | Regression_energy: 0.1497 | Regression_eta: 0.1134 | Regression_pt: 0.1645 | Regression_sin_phi: 0.2392 | LR: 9.43e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6984s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7628 | max_loss: 0.1353 | max_backward: 2.0123 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.8300 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6970.2 | uss_MB: 5951.6 | child_rss_MB: 6190.8 | allocated_MB: 447.2 | reserved_MB: 25306.0 | inactive_split_MB: 1261.3 +rank2 [2026-10-28 18:04:56,111] INFO : Step 8200/50000 rank2 | Train Loss: 1.8745 | Train Loss Unweighted: 1.0293 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.1946 | Classification: 0.1986 | Classification_binary: 0.2966 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2609 | Regression_eta: 0.0035 | Regression_pt: 0.2657 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1173 | Classification_binary: 0.3761 | Regression_cos_phi: 0.1800 | Regression_energy: 0.1351 | Regression_eta: 0.1234 | Regression_pt: 0.1499 | Regression_sin_phi: 0.1990 | LR: 9.42e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3077s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7790 | max_loss: 0.1535 | max_backward: 1.9181 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.5352 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6080.6 | uss_MB: 5064.4 | child_rss_MB: 6184.5 | allocated_MB: 448.2 | reserved_MB: 25360.0 | inactive_split_MB: 1159.7 +rank2 [2026-10-28 18:09:37,298] INFO : Step 8300/50000 rank2 | Train Loss: 1.8727 | Train Loss Unweighted: 1.0119 | Train Loss Components: Auxiliary: 0.3401 | Cardinality: 0.1867 | Classification: 0.1728 | Classification_binary: 0.3272 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2513 | Regression_eta: 0.0033 | Regression_pt: 0.2533 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1150 | Classification_binary: 0.3445 | Regression_cos_phi: 0.1976 | Regression_energy: 0.1492 | Regression_eta: 0.1066 | Regression_pt: 0.1597 | Regression_sin_phi: 0.1936 | LR: 9.40e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5443s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0436 | max_forward: 1.1303 | max_loss: 0.1503 | max_backward: 2.0213 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6082.4 | uss_MB: 5067.6 | child_rss_MB: 6182.5 | allocated_MB: 449.6 | reserved_MB: 27362.0 | inactive_split_MB: 1355.2 +rank2 [2026-10-28 18:14:10,521] INFO : Step 8400/50000 rank2 | Train Loss: 1.6705 | Train Loss Unweighted: 0.9205 | Train Loss Components: Auxiliary: 0.3113 | Cardinality: 0.1938 | Classification: 0.1656 | Classification_binary: 0.2512 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2485 | Regression_eta: 0.0028 | Regression_pt: 0.2494 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1137 | Classification_binary: 0.2188 | Regression_cos_phi: 0.1308 | Regression_energy: 0.1431 | Regression_eta: 0.0787 | Regression_pt: 0.1549 | Regression_sin_phi: 0.1263 | LR: 9.39e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4495s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8079 | max_loss: 0.1514 | max_backward: 1.6942 | max_optimizer_step: 0.2670 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6074.8 | uss_MB: 5060.4 | child_rss_MB: 6188.9 | allocated_MB: 446.4 | reserved_MB: 27366.0 | inactive_split_MB: 1386.9 +rank2 [2026-10-28 18:18:45,093] INFO : Step 8500/50000 rank2 | Train Loss: 1.8447 | Train Loss Unweighted: 0.9684 | Train Loss Components: Auxiliary: 0.3299 | Cardinality: 0.2197 | Classification: 0.1758 | Classification_binary: 0.2823 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2478 | Regression_eta: 0.0040 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1043 | Classification_binary: 0.2930 | Regression_cos_phi: 0.1846 | Regression_energy: 0.1347 | Regression_eta: 0.1688 | Regression_pt: 0.1495 | Regression_sin_phi: 0.1618 | LR: 9.37e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1699s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0066 | max_forward: 0.7185 | max_loss: 0.1465 | max_backward: 1.6783 | max_optimizer_step: 0.2239 | max_tensorboard_logging: 0.6269 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 6077.8 | uss_MB: 5064.0 | child_rss_MB: 6185.3 | allocated_MB: 448.4 | reserved_MB: 27368.0 | inactive_split_MB: 1257.8 +rank2 [2026-10-28 18:23:30,153] INFO : Step 8600/50000 rank2 | Train Loss: 1.9431 | Train Loss Unweighted: 1.0237 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.2483 | Classification: 0.1814 | Classification_binary: 0.3328 | HitGrouping: 0.2024 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2495 | Regression_eta: 0.0034 | Regression_pt: 0.2527 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1226 | Classification_binary: 0.3323 | Regression_cos_phi: 0.1342 | Regression_energy: 0.1477 | Regression_eta: 0.0917 | Regression_pt: 0.1624 | Regression_sin_phi: 0.1946 | LR: 9.36e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5558s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1011 | max_forward: 0.9340 | max_loss: 0.1508 | max_backward: 1.7704 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.7686 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 6082.0 | uss_MB: 5062.1 | child_rss_MB: 6191.5 | allocated_MB: 448.9 | reserved_MB: 27368.0 | inactive_split_MB: 1268.3 +rank2 [2026-10-28 18:28:08,445] INFO : Step 8700/50000 rank2 | Train Loss: 1.9739 | Train Loss Unweighted: 1.0707 | Train Loss Components: Auxiliary: 0.3725 | Cardinality: 0.2322 | Classification: 0.1854 | Classification_binary: 0.2934 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2900 | Regression_eta: 0.0037 | Regression_pt: 0.2944 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1102 | Classification_binary: 0.2885 | Regression_cos_phi: 0.1732 | Regression_energy: 0.1723 | Regression_eta: 0.1052 | Regression_pt: 0.1891 | Regression_sin_phi: 0.1660 | LR: 9.34e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.1634s | Phase Times: max_pre_step_barrier: 0.0890 | max_batch_to_device: 0.0067 | max_forward: 0.7726 | max_loss: 0.1574 | max_backward: 1.6332 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 2.5948 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 6082.7 | uss_MB: 5062.6 | child_rss_MB: 6185.4 | allocated_MB: 448.0 | reserved_MB: 29420.0 | inactive_split_MB: 876.0 +rank2 [2026-10-28 18:32:55,586] INFO : Step 8800/50000 rank2 | Train Loss: 1.5247 | Train Loss Unweighted: 0.8421 | Train Loss Components: Auxiliary: 0.2808 | Cardinality: 0.1524 | Classification: 0.1581 | Classification_binary: 0.2323 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2217 | Regression_eta: 0.0029 | Regression_pt: 0.2242 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0965 | Classification_binary: 0.1992 | Regression_cos_phi: 0.1225 | Regression_energy: 0.1118 | Regression_eta: 0.0631 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0997 | LR: 9.33e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9910s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7382 | max_loss: 0.2414 | max_backward: 1.7741 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 6078.3 | uss_MB: 5061.0 | child_rss_MB: 6166.0 | allocated_MB: 447.9 | reserved_MB: 29420.0 | inactive_split_MB: 1260.3 +rank2 [2026-10-28 18:37:36,891] INFO : Step 8900/50000 rank2 | Train Loss: 1.7540 | Train Loss Unweighted: 0.9436 | Train Loss Components: Auxiliary: 0.3227 | Cardinality: 0.2072 | Classification: 0.1816 | Classification_binary: 0.2735 | HitGrouping: 0.1904 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2393 | Regression_eta: 0.0031 | Regression_pt: 0.2426 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0992 | Classification_binary: 0.2971 | Regression_cos_phi: 0.1469 | Regression_energy: 0.1239 | Regression_eta: 0.1043 | Regression_pt: 0.1345 | Regression_sin_phi: 0.1539 | LR: 9.31e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4484s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8077 | max_loss: 0.2089 | max_backward: 1.9976 | max_optimizer_step: 0.2211 | max_tensorboard_logging: 0.5747 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 6080.9 | uss_MB: 5000.4 | child_rss_MB: 6188.7 | allocated_MB: 451.8 | reserved_MB: 29420.0 | inactive_split_MB: 1133.8 +rank2 [2026-10-28 18:42:20,744] INFO : Step 9000/50000 rank2 | Train Loss: 1.7807 | Train Loss Unweighted: 0.9795 | Train Loss Components: Auxiliary: 0.3326 | Cardinality: 0.1942 | Classification: 0.1600 | Classification_binary: 0.3157 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2461 | Regression_eta: 0.0033 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0813 | Classification_binary: 0.2778 | Regression_cos_phi: 0.1137 | Regression_energy: 0.1047 | Regression_eta: 0.0685 | Regression_pt: 0.1157 | Regression_sin_phi: 0.1273 | LR: 9.30e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8566s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.8696 | max_loss: 0.1559 | max_backward: 1.8245 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.9520 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 6064.6 | uss_MB: 5051.1 | child_rss_MB: 6187.5 | allocated_MB: 448.9 | reserved_MB: 29420.0 | inactive_split_MB: 1051.6 +rank2 [2026-10-28 18:42:21,625] INFO : Running validation on rank2 +rank2 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 18:43:19,656] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:43:20,246] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 18:43:47,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 18:43:48,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 18:43:49,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 18:43:50,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 18:44:00,140] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 18:44:00,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:44:01,305] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 18:44:17,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 18:44:18,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 18:44:19,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 18:44:20,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 18:44:30,429] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 18:44:31,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 18:44:31,605] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 18:44:47,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 18:44:48,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 18:44:49,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 18:44:50,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 18:45:00,102] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 18:50:14,978] INFO : Step 9100/50000 rank2 | Train Loss: 1.6002 | Train Loss Unweighted: 0.8685 | Train Loss Components: Auxiliary: 0.3013 | Cardinality: 0.1975 | Classification: 0.1658 | Classification_binary: 0.2349 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2304 | Regression_eta: 0.0027 | Regression_pt: 0.2318 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1107 | Classification_binary: 0.2627 | Regression_cos_phi: 0.1082 | Regression_energy: 0.1350 | Regression_eta: 0.0953 | Regression_pt: 0.1485 | Regression_sin_phi: 0.1513 | LR: 9.28e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5176s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7981 | max_loss: 0.1460 | max_backward: 1.6475 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.9178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7065.6 | uss_MB: 6052.1 | child_rss_MB: 6167.4 | allocated_MB: 449.7 | reserved_MB: 25788.0 | inactive_split_MB: 957.3 +rank2 [2026-10-28 18:55:02,312] INFO : Step 9200/50000 rank2 | Train Loss: 1.6859 | Train Loss Unweighted: 0.8867 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.1906 | Classification: 0.1447 | Classification_binary: 0.2542 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2389 | Regression_eta: 0.0033 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2506 | Regression_cos_phi: 0.1531 | Regression_energy: 0.1165 | Regression_eta: 0.0694 | Regression_pt: 0.1269 | Regression_sin_phi: 0.1263 | LR: 9.27e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3388s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7865 | max_loss: 0.1441 | max_backward: 1.8922 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6128.5 | uss_MB: 5123.6 | child_rss_MB: 6161.7 | allocated_MB: 450.8 | reserved_MB: 25788.0 | inactive_split_MB: 1040.1 +rank2 [2026-10-28 18:59:49,908] INFO : Step 9300/50000 rank2 | Train Loss: 1.8980 | Train Loss Unweighted: 0.9669 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.3104 | Classification: 0.1729 | Classification_binary: 0.3073 | HitGrouping: 0.1936 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2397 | Regression_eta: 0.0030 | Regression_pt: 0.2407 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0964 | Classification_binary: 0.2722 | Regression_cos_phi: 0.1336 | Regression_energy: 0.1297 | Regression_eta: 0.0748 | Regression_pt: 0.1402 | Regression_sin_phi: 0.1427 | LR: 9.25e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.1849s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0143 | max_forward: 0.7819 | max_loss: 0.1515 | max_backward: 1.8209 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6071.1 | uss_MB: 5121.9 | child_rss_MB: 5951.4 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1031.1 +rank2 [2026-10-28 19:04:46,837] INFO : Step 9400/50000 rank2 | Train Loss: 1.5878 | Train Loss Unweighted: 0.8563 | Train Loss Components: Auxiliary: 0.2880 | Cardinality: 0.1552 | Classification: 0.1580 | Classification_binary: 0.2625 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2141 | Regression_eta: 0.0030 | Regression_pt: 0.2153 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1044 | Classification_binary: 0.2603 | Regression_cos_phi: 0.1396 | Regression_energy: 0.1098 | Regression_eta: 0.0818 | Regression_pt: 0.1210 | Regression_sin_phi: 0.1153 | LR: 9.24e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0177s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7342 | max_loss: 0.2471 | max_backward: 1.7320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.4671 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6067.3 | uss_MB: 5118.7 | child_rss_MB: 5945.7 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1121.1 +rank2 [2026-10-28 19:09:47,342] INFO : Step 9500/50000 rank2 | Train Loss: 1.5519 | Train Loss Unweighted: 0.8141 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1719 | Classification: 0.1393 | Classification_binary: 0.2430 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2129 | Regression_eta: 0.0030 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0918 | Classification_binary: 0.2531 | Regression_cos_phi: 0.1039 | Regression_energy: 0.1264 | Regression_eta: 0.0578 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0779 | LR: 9.22e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1287s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0076 | max_forward: 0.7261 | max_loss: 0.1376 | max_backward: 1.8267 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6069.1 | uss_MB: 5122.8 | child_rss_MB: 5952.7 | allocated_MB: 446.7 | reserved_MB: 27752.0 | inactive_split_MB: 1128.4 +rank2 [2026-10-28 19:14:55,462] INFO : Step 9600/50000 rank2 | Train Loss: 1.7379 | Train Loss Unweighted: 0.9171 | Train Loss Components: Auxiliary: 0.3221 | Cardinality: 0.2199 | Classification: 0.1679 | Classification_binary: 0.2886 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2245 | Regression_eta: 0.0032 | Regression_pt: 0.2297 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1146 | Classification_binary: 0.3280 | Regression_cos_phi: 0.1555 | Regression_energy: 0.1318 | Regression_eta: 0.1253 | Regression_pt: 0.1451 | Regression_sin_phi: 0.1687 | LR: 9.21e-04 | DataLoad Time: 0.0180s | Model Forward Time: 7.7263s | Phase Times: max_pre_step_barrier: 0.0081 | max_batch_to_device: 0.0183 | max_forward: 3.1116 | max_loss: 0.7194 | max_backward: 4.4829 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6068.7 | uss_MB: 5120.4 | child_rss_MB: 5924.4 | allocated_MB: 449.4 | reserved_MB: 29732.0 | inactive_split_MB: 1322.6 +rank2 [2026-10-28 19:20:02,940] INFO : Step 9700/50000 rank2 | Train Loss: 1.8683 | Train Loss Unweighted: 0.9602 | Train Loss Components: Auxiliary: 0.3449 | Cardinality: 0.2461 | Classification: 0.1512 | Classification_binary: 0.3271 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2364 | Regression_eta: 0.0037 | Regression_pt: 0.2383 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.3015 | Regression_cos_phi: 0.1376 | Regression_energy: 0.1239 | Regression_eta: 0.0929 | Regression_pt: 0.1357 | Regression_sin_phi: 0.1267 | LR: 9.19e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0318s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0069 | max_forward: 0.8299 | max_loss: 0.7776 | max_backward: 2.5571 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5726 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6067.5 | uss_MB: 5123.7 | child_rss_MB: 5943.1 | allocated_MB: 450.3 | reserved_MB: 29734.0 | inactive_split_MB: 1128.1 +rank2 [2026-10-28 19:25:00,573] INFO : Step 9800/50000 rank2 | Train Loss: 1.6119 | Train Loss Unweighted: 0.8200 | Train Loss Components: Auxiliary: 0.3036 | Cardinality: 0.2266 | Classification: 0.1371 | Classification_binary: 0.2410 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2176 | Regression_eta: 0.0027 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.2230 | Regression_cos_phi: 0.1370 | Regression_energy: 0.1223 | Regression_eta: 0.0866 | Regression_pt: 0.1306 | Regression_sin_phi: 0.1508 | LR: 9.17e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5938s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7430 | max_loss: 0.2467 | max_backward: 1.8961 | max_optimizer_step: 0.2008 | max_tensorboard_logging: 0.9056 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6064.8 | uss_MB: 5122.0 | child_rss_MB: 5925.9 | allocated_MB: 446.9 | reserved_MB: 29734.0 | inactive_split_MB: 905.1 +rank2 [2026-10-28 19:29:40,210] INFO : Step 9900/50000 rank2 | Train Loss: 1.5408 | Train Loss Unweighted: 0.8391 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1631 | Classification: 0.1532 | Classification_binary: 0.2416 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2178 | Regression_eta: 0.0029 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.2436 | Regression_cos_phi: 0.0950 | Regression_energy: 0.1092 | Regression_eta: 0.0648 | Regression_pt: 0.1219 | Regression_sin_phi: 0.1044 | LR: 9.16e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6031s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7611 | max_loss: 0.1425 | max_backward: 1.8994 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6066.1 | uss_MB: 5122.8 | child_rss_MB: 5945.5 | allocated_MB: 448.3 | reserved_MB: 29734.0 | inactive_split_MB: 1119.7 +rank2 [2026-10-28 19:34:23,378] INFO : Step 10000/50000 rank2 | Train Loss: 1.7147 | Train Loss Unweighted: 0.9225 | Train Loss Components: Auxiliary: 0.3261 | Cardinality: 0.1969 | Classification: 0.1590 | Classification_binary: 0.2912 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2305 | Regression_eta: 0.0029 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1098 | Classification_binary: 0.3146 | Regression_cos_phi: 0.1506 | Regression_energy: 0.1354 | Regression_eta: 0.0819 | Regression_pt: 0.1506 | Regression_sin_phi: 0.1446 | LR: 9.14e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8804s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1140 | max_forward: 0.8951 | max_loss: 0.1695 | max_backward: 1.8837 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 1.0339 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 6055.8 | uss_MB: 5112.2 | child_rss_MB: 5922.2 | allocated_MB: 449.4 | reserved_MB: 29734.0 | inactive_split_MB: 1320.6 +rank2 [2026-10-28 19:34:24,225] INFO : Running validation on rank2 +rank2 [2026-10-28 19:34:24,225] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 19:34:24,225] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 19:35:21,692] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:35:22,179] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 19:35:48,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 19:35:49,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 19:35:50,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 19:35:51,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 19:36:01,785] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 19:36:02,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:36:02,955] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 19:36:19,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 19:36:20,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 19:36:21,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 19:36:22,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 19:36:32,453] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 19:36:36,471] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 19:36:36,973] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 19:36:52,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 19:36:53,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 19:36:54,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 19:36:55,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 19:37:05,462] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 19:42:23,380] INFO : Step 10100/50000 rank2 | Train Loss: 1.6697 | Train Loss Unweighted: 0.8627 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.2287 | Classification: 0.1283 | Classification_binary: 0.2543 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2359 | Regression_eta: 0.0032 | Regression_pt: 0.2378 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.2545 | Regression_cos_phi: 0.1225 | Regression_energy: 0.1649 | Regression_eta: 0.0908 | Regression_pt: 0.1768 | Regression_sin_phi: 0.1322 | LR: 9.12e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7386s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7652 | max_loss: 0.1496 | max_backward: 1.8791 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 0.9326 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 7065.4 | uss_MB: 6119.6 | child_rss_MB: 5927.2 | allocated_MB: 448.1 | reserved_MB: 26360.0 | inactive_split_MB: 1080.0 +rank2 [2026-10-28 19:47:10,586] INFO : Step 10200/50000 rank2 | Train Loss: 1.3968 | Train Loss Unweighted: 0.7619 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1483 | Classification: 0.1293 | Classification_binary: 0.2070 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2090 | Regression_eta: 0.0026 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0907 | Classification_binary: 0.2411 | Regression_cos_phi: 0.1314 | Regression_energy: 0.1104 | Regression_eta: 0.1146 | Regression_pt: 0.1232 | Regression_sin_phi: 0.1348 | LR: 9.11e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.4814s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0330 | max_forward: 0.8960 | max_loss: 0.5080 | max_backward: 1.9040 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.6816 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 6137.8 | uss_MB: 5194.1 | child_rss_MB: 5912.5 | allocated_MB: 450.3 | reserved_MB: 27130.0 | inactive_split_MB: 1088.5 +rank2 [2026-10-28 19:51:56,122] INFO : Step 10300/50000 rank2 | Train Loss: 1.6039 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.2943 | Cardinality: 0.1656 | Classification: 0.1487 | Classification_binary: 0.2679 | HitGrouping: 0.1881 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2132 | Regression_eta: 0.0030 | Regression_pt: 0.2158 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0907 | Classification_binary: 0.2746 | Regression_cos_phi: 0.1418 | Regression_energy: 0.1096 | Regression_eta: 0.0876 | Regression_pt: 0.1221 | Regression_sin_phi: 0.1538 | LR: 9.09e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7926s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8152 | max_loss: 0.2478 | max_backward: 2.0483 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6143.7 | uss_MB: 5198.5 | child_rss_MB: 5930.4 | allocated_MB: 448.7 | reserved_MB: 28404.0 | inactive_split_MB: 991.9 +rank2 [2026-10-28 19:56:39,452] INFO : Step 10400/50000 rank2 | Train Loss: 1.7188 | Train Loss Unweighted: 0.8798 | Train Loss Components: Auxiliary: 0.3209 | Cardinality: 0.2578 | Classification: 0.1544 | Classification_binary: 0.2910 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2139 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0982 | Classification_binary: 0.2941 | Regression_cos_phi: 0.1296 | Regression_energy: 0.1095 | Regression_eta: 0.0767 | Regression_pt: 0.1200 | Regression_sin_phi: 0.1226 | LR: 9.07e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6756s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7821 | max_loss: 0.2448 | max_backward: 1.8782 | max_optimizer_step: 0.2932 | max_tensorboard_logging: 0.8061 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 6134.3 | uss_MB: 5193.2 | child_rss_MB: 5938.4 | allocated_MB: 448.9 | reserved_MB: 28404.0 | inactive_split_MB: 983.3 +rank2 [2026-10-28 20:01:29,564] INFO : Step 10500/50000 rank2 | Train Loss: 1.4652 | Train Loss Unweighted: 0.7986 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1697 | Classification: 0.1280 | Classification_binary: 0.2159 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2232 | Regression_eta: 0.0026 | Regression_pt: 0.2263 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.1894 | Regression_cos_phi: 0.1268 | Regression_energy: 0.1121 | Regression_eta: 0.0428 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0736 | LR: 9.06e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.2143s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.7113 | max_loss: 0.1455 | max_backward: 1.6308 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.7760 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6134.1 | uss_MB: 5192.0 | child_rss_MB: 5916.0 | allocated_MB: 444.7 | reserved_MB: 28404.0 | inactive_split_MB: 1307.3 +rank2 [2026-10-28 20:06:14,521] INFO : Step 10600/50000 rank2 | Train Loss: 1.5657 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2802 | Cardinality: 0.1297 | Classification: 0.1617 | Classification_binary: 0.2471 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2084 | Regression_eta: 0.0037 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1045 | Classification_binary: 0.2904 | Regression_cos_phi: 0.2048 | Regression_energy: 0.1219 | Regression_eta: 0.1454 | Regression_pt: 0.1310 | Regression_sin_phi: 0.1466 | LR: 9.04e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9415s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0173 | max_forward: 0.7875 | max_loss: 0.2211 | max_backward: 1.9019 | max_optimizer_step: 0.2814 | max_tensorboard_logging: 0.8518 | max_tensorboard_flush: 0.0072 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6133.2 | uss_MB: 5193.0 | child_rss_MB: 5934.7 | allocated_MB: 449.8 | reserved_MB: 28404.0 | inactive_split_MB: 1130.2 +rank2 [2026-10-28 20:10:59,504] INFO : Step 10700/50000 rank2 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8676 | Train Loss Components: Auxiliary: 0.3003 | Cardinality: 0.1749 | Classification: 0.1453 | Classification_binary: 0.2956 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2085 | Regression_eta: 0.0033 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1193 | Classification_binary: 0.3264 | Regression_cos_phi: 0.1561 | Regression_energy: 0.1174 | Regression_eta: 0.1157 | Regression_pt: 0.1300 | Regression_sin_phi: 0.1790 | LR: 9.02e-04 | DataLoad Time: 0.0120s | Model Forward Time: 5.1695s | Phase Times: max_pre_step_barrier: 0.0305 | max_batch_to_device: 0.0154 | max_forward: 2.0246 | max_loss: 0.2864 | max_backward: 2.7424 | max_optimizer_step: 0.1980 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6136.9 | uss_MB: 5195.8 | child_rss_MB: 5906.4 | allocated_MB: 446.2 | reserved_MB: 29148.0 | inactive_split_MB: 1134.5 +rank2 [2026-10-28 20:15:41,623] INFO : Step 10800/50000 rank2 | Train Loss: 1.5849 | Train Loss Unweighted: 0.7936 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1942 | Classification: 0.1383 | Classification_binary: 0.2480 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2007 | Regression_eta: 0.0032 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2790 | Regression_cos_phi: 0.1409 | Regression_energy: 0.1101 | Regression_eta: 0.0836 | Regression_pt: 0.1212 | Regression_sin_phi: 0.1156 | LR: 9.00e-04 | DataLoad Time: 0.0183s | Model Forward Time: 8.1970s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2408 | max_backward: 1.7977 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 1.7470 | max_tensorboard_flush: 0.0028 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 6126.4 | uss_MB: 4363.9 | child_rss_MB: 5910.3 | allocated_MB: 448.6 | reserved_MB: 30628.0 | inactive_split_MB: 1292.6 +rank2 [2026-10-28 20:20:32,833] INFO : Step 10900/50000 rank2 | Train Loss: 1.3552 | Train Loss Unweighted: 0.7247 | Train Loss Components: Auxiliary: 0.2531 | Cardinality: 0.1449 | Classification: 0.1275 | Classification_binary: 0.2074 | HitGrouping: 0.1874 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1917 | Regression_eta: 0.0024 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0794 | Classification_binary: 0.2370 | Regression_cos_phi: 0.1075 | Regression_energy: 0.1109 | Regression_eta: 0.0658 | Regression_pt: 0.1177 | Regression_sin_phi: 0.1042 | LR: 8.99e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4555s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7076 | max_loss: 0.2425 | max_backward: 1.6536 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6131.8 | uss_MB: 5191.3 | child_rss_MB: 5932.5 | allocated_MB: 446.4 | reserved_MB: 30628.0 | inactive_split_MB: 984.9 +rank2 [2026-10-28 20:25:36,608] INFO : Step 11000/50000 rank2 | Train Loss: 1.5999 | Train Loss Unweighted: 0.8322 | Train Loss Components: Auxiliary: 0.2959 | Cardinality: 0.1737 | Classification: 0.1477 | Classification_binary: 0.2877 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0017 | Regression_energy: 0.1943 | Regression_eta: 0.0030 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0944 | Classification_binary: 0.2929 | Regression_cos_phi: 0.1481 | Regression_energy: 0.1071 | Regression_eta: 0.0847 | Regression_pt: 0.1162 | Regression_sin_phi: 0.1106 | LR: 8.97e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9322s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8260 | max_loss: 0.1503 | max_backward: 1.9603 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6124.9 | uss_MB: 5185.7 | child_rss_MB: 5918.3 | allocated_MB: 451.8 | reserved_MB: 30628.0 | inactive_split_MB: 996.6 +rank2 [2026-10-28 20:25:37,479] INFO : Running validation on rank2 +rank2 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 20:26:32,846] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:26:33,360] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 20:27:00,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 20:27:01,327] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 20:27:02,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 20:27:03,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 20:27:13,405] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 20:27:14,319] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:27:14,832] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 20:27:31,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 20:27:32,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 20:27:34,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 20:27:35,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 20:27:45,254] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-28 20:27:46,111] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 20:27:46,649] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 20:28:02,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 20:28:02,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 20:28:03,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 20:28:04,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 20:28:14,538] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 20:33:29,190] INFO : Step 11100/50000 rank2 | Train Loss: 1.6214 | Train Loss Unweighted: 0.8328 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.2186 | Classification: 0.1350 | Classification_binary: 0.2802 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2042 | Regression_eta: 0.0030 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0832 | Classification_binary: 0.2704 | Regression_cos_phi: 0.1196 | Regression_energy: 0.1170 | Regression_eta: 0.0897 | Regression_pt: 0.1268 | Regression_sin_phi: 0.1220 | LR: 8.95e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8815s | Phase Times: max_pre_step_barrier: 0.0088 | max_batch_to_device: 0.0068 | max_forward: 0.8488 | max_loss: 0.1548 | max_backward: 1.7645 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 1.1046 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7069.9 | uss_MB: 6125.7 | child_rss_MB: 5940.1 | allocated_MB: 448.1 | reserved_MB: 24340.0 | inactive_split_MB: 1081.9 +rank2 [2026-10-28 20:38:18,707] INFO : Step 11200/50000 rank2 | Train Loss: 1.7050 | Train Loss Unweighted: 0.8927 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2640 | Classification: 0.1534 | Classification_binary: 0.2523 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2404 | Regression_eta: 0.0030 | Regression_pt: 0.2410 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1032 | Classification_binary: 0.2499 | Regression_cos_phi: 0.1030 | Regression_energy: 0.1252 | Regression_eta: 0.0820 | Regression_pt: 0.1347 | Regression_sin_phi: 0.1280 | LR: 8.93e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.8972s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8707 | max_loss: 0.2001 | max_backward: 2.2037 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.7912 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 6163.4 | uss_MB: 5218.5 | child_rss_MB: 5949.2 | allocated_MB: 447.8 | reserved_MB: 26392.0 | inactive_split_MB: 938.7 +rank2 [2026-10-28 20:43:01,724] INFO : Step 11300/50000 rank2 | Train Loss: 1.3885 | Train Loss Unweighted: 0.7511 | Train Loss Components: Auxiliary: 0.2594 | Cardinality: 0.1357 | Classification: 0.1316 | Classification_binary: 0.2357 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1874 | Regression_eta: 0.0024 | Regression_pt: 0.1915 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.2622 | Regression_cos_phi: 0.1169 | Regression_energy: 0.1123 | Regression_eta: 0.0672 | Regression_pt: 0.1240 | Regression_sin_phi: 0.1197 | LR: 8.91e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4300s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8053 | max_loss: 0.1469 | max_backward: 2.0062 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.4618 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1345 | Memory: rss_MB: 6165.0 | uss_MB: 5222.0 | child_rss_MB: 5948.8 | allocated_MB: 450.6 | reserved_MB: 26400.0 | inactive_split_MB: 1092.9 +rank2 [2026-10-28 20:48:05,522] INFO : Step 11400/50000 rank2 | Train Loss: 1.5865 | Train Loss Unweighted: 0.8323 | Train Loss Components: Auxiliary: 0.2981 | Cardinality: 0.1866 | Classification: 0.1338 | Classification_binary: 0.2899 | HitGrouping: 0.1840 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1999 | Regression_eta: 0.0028 | Regression_pt: 0.2030 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1018 | Classification_binary: 0.3086 | Regression_cos_phi: 0.1374 | Regression_energy: 0.0946 | Regression_eta: 0.0862 | Regression_pt: 0.1068 | Regression_sin_phi: 0.1254 | LR: 8.89e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.7654s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0769 | max_forward: 1.3800 | max_loss: 0.6715 | max_backward: 3.0339 | max_optimizer_step: 0.2660 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6163.4 | uss_MB: 5219.6 | child_rss_MB: 5946.4 | allocated_MB: 448.1 | reserved_MB: 27144.0 | inactive_split_MB: 1081.9 +rank2 [2026-10-28 20:52:57,975] INFO : Step 11500/50000 rank2 | Train Loss: 1.4825 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2779 | Cardinality: 0.1999 | Classification: 0.1273 | Classification_binary: 0.2463 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1956 | Regression_eta: 0.0026 | Regression_pt: 0.1972 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.2257 | Regression_cos_phi: 0.1020 | Regression_energy: 0.0948 | Regression_eta: 0.0724 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0848 | LR: 8.88e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0556s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.7822 | max_loss: 0.1471 | max_backward: 1.7308 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 1.4583 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6164.2 | uss_MB: 5219.4 | child_rss_MB: 5944.4 | allocated_MB: 450.8 | reserved_MB: 27144.0 | inactive_split_MB: 1342.3 +rank2 [2026-10-28 20:57:41,223] INFO : Step 11600/50000 rank2 | Train Loss: 1.4219 | Train Loss Unweighted: 0.7444 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1679 | Classification: 0.1415 | Classification_binary: 0.2106 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1945 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0922 | Classification_binary: 0.1961 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0981 | Regression_eta: 0.0521 | Regression_pt: 0.1063 | Regression_sin_phi: 0.1013 | LR: 8.86e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7008s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8362 | max_loss: 0.1428 | max_backward: 2.0109 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6156.9 | uss_MB: 5220.2 | child_rss_MB: 5947.3 | allocated_MB: 448.6 | reserved_MB: 27146.0 | inactive_split_MB: 1315.4 +rank2 [2026-10-28 21:02:35,116] INFO : Step 11700/50000 rank2 | Train Loss: 1.5203 | Train Loss Unweighted: 0.8017 | Train Loss Components: Auxiliary: 0.2866 | Cardinality: 0.1889 | Classification: 0.1302 | Classification_binary: 0.2577 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2024 | Regression_eta: 0.0026 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0977 | Classification_binary: 0.2801 | Regression_cos_phi: 0.1208 | Regression_energy: 0.1253 | Regression_eta: 0.0639 | Regression_pt: 0.1395 | Regression_sin_phi: 0.1363 | LR: 8.84e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.3171s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 1.0014 | max_loss: 0.1975 | max_backward: 2.5347 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0688 | Memory: rss_MB: 6157.0 | uss_MB: 5144.6 | child_rss_MB: 5922.6 | allocated_MB: 448.7 | reserved_MB: 27266.0 | inactive_split_MB: 1090.9 +rank2 [2026-10-28 21:07:42,591] INFO : Step 11800/50000 rank2 | Train Loss: 1.5591 | Train Loss Unweighted: 0.7907 | Train Loss Components: Auxiliary: 0.2873 | Cardinality: 0.2051 | Classification: 0.1346 | Classification_binary: 0.2655 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1918 | Regression_eta: 0.0030 | Regression_pt: 0.1929 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2846 | Regression_cos_phi: 0.1160 | Regression_energy: 0.1098 | Regression_eta: 0.0690 | Regression_pt: 0.1195 | Regression_sin_phi: 0.1001 | LR: 8.82e-04 | DataLoad Time: 0.0197s | Model Forward Time: 7.4842s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.0966 | max_forward: 1.7907 | max_loss: 0.6496 | max_backward: 2.7103 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 2.3587 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6136.9 | uss_MB: 5209.7 | child_rss_MB: 5946.4 | allocated_MB: 450.4 | reserved_MB: 27266.0 | inactive_split_MB: 1187.2 +rank2 [2026-10-28 21:12:51,257] INFO : Step 11900/50000 rank2 | Train Loss: 1.3283 | Train Loss Unweighted: 0.7113 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1476 | Classification: 0.1193 | Classification_binary: 0.2094 | HitGrouping: 0.1824 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0021 | Regression_pt: 0.1901 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0491 | Regression_energy: 0.1089 | Regression_eta: 0.0461 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0941 | LR: 8.80e-04 | DataLoad Time: 0.0228s | Model Forward Time: 4.1677s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0785 | max_forward: 0.7917 | max_loss: 0.1465 | max_backward: 1.5595 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6137.9 | uss_MB: 5216.4 | child_rss_MB: 5947.1 | allocated_MB: 443.9 | reserved_MB: 27266.0 | inactive_split_MB: 955.3 +rank2 [2026-10-28 21:17:48,734] INFO : Step 12000/50000 rank2 | Train Loss: 1.4074 | Train Loss Unweighted: 0.7415 | Train Loss Components: Auxiliary: 0.2683 | Cardinality: 0.1719 | Classification: 0.1235 | Classification_binary: 0.2283 | HitGrouping: 0.1812 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1909 | Regression_eta: 0.0022 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.2398 | Regression_cos_phi: 0.1295 | Regression_energy: 0.0973 | Regression_eta: 0.0692 | Regression_pt: 0.1075 | Regression_sin_phi: 0.1154 | LR: 8.78e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2507s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0303 | max_forward: 0.7589 | max_loss: 0.1459 | max_backward: 1.9073 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6136.6 | uss_MB: 5219.4 | child_rss_MB: 5952.0 | allocated_MB: 447.6 | reserved_MB: 27266.0 | inactive_split_MB: 1310.4 +rank2 [2026-10-28 21:17:49,718] INFO : Running validation on rank2 +rank2 [2026-10-28 21:17:49,718] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 21:17:49,718] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 21:18:44,922] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:18:45,601] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 21:19:11,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 21:19:12,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 21:19:13,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 21:19:14,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 21:19:24,630] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 21:19:25,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:19:25,829] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 21:19:41,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 21:19:42,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 21:19:43,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 21:19:44,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 21:19:54,721] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 21:19:55,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 21:19:56,386] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 21:20:12,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 21:20:13,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 21:20:13,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 21:20:14,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 21:20:24,742] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 21:25:33,195] INFO : Step 12100/50000 rank2 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8521 | Train Loss Components: Auxiliary: 0.3093 | Cardinality: 0.2088 | Classification: 0.1337 | Classification_binary: 0.2971 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0029 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.2764 | Regression_cos_phi: 0.1422 | Regression_energy: 0.0976 | Regression_eta: 0.0886 | Regression_pt: 0.1043 | Regression_sin_phi: 0.1507 | LR: 8.76e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9396s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8139 | max_loss: 0.1586 | max_backward: 1.9999 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9439 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7027.3 | uss_MB: 6104.0 | child_rss_MB: 5925.8 | allocated_MB: 450.8 | reserved_MB: 28008.0 | inactive_split_MB: 1125.6 +rank2 [2026-10-28 21:30:25,391] INFO : Step 12200/50000 rank2 | Train Loss: 1.4655 | Train Loss Unweighted: 0.7431 | Train Loss Components: Auxiliary: 0.2757 | Cardinality: 0.1845 | Classification: 0.1184 | Classification_binary: 0.2450 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1862 | Regression_eta: 0.0026 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0910 | Classification_binary: 0.2500 | Regression_cos_phi: 0.0871 | Regression_energy: 0.1173 | Regression_eta: 0.0728 | Regression_pt: 0.1281 | Regression_sin_phi: 0.1627 | LR: 8.74e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4986s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0364 | max_forward: 0.7684 | max_loss: 0.1438 | max_backward: 1.8240 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.7243 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0697 | Memory: rss_MB: 6148.5 | uss_MB: 5233.4 | child_rss_MB: 5949.1 | allocated_MB: 445.9 | reserved_MB: 28008.0 | inactive_split_MB: 995.3 +rank2 [2026-10-28 21:35:16,327] INFO : Step 12300/50000 rank2 | Train Loss: 1.3845 | Train Loss Unweighted: 0.7270 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1523 | Classification: 0.1341 | Classification_binary: 0.2109 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1869 | Regression_eta: 0.0026 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.2048 | Regression_cos_phi: 0.0884 | Regression_energy: 0.1101 | Regression_eta: 0.0545 | Regression_pt: 0.1211 | Regression_sin_phi: 0.0835 | LR: 8.72e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5868s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7264 | max_loss: 0.1976 | max_backward: 1.8534 | max_optimizer_step: 0.2993 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6147.3 | uss_MB: 5233.4 | child_rss_MB: 5944.1 | allocated_MB: 448.1 | reserved_MB: 29988.0 | inactive_split_MB: 1298.2 +rank2 [2026-10-28 21:40:15,077] INFO : Step 12400/50000 rank2 | Train Loss: 1.4006 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.2654 | Cardinality: 0.1602 | Classification: 0.1310 | Classification_binary: 0.2363 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2673 | Regression_cos_phi: 0.1160 | Regression_energy: 0.0925 | Regression_eta: 0.0667 | Regression_pt: 0.1014 | Regression_sin_phi: 0.1418 | LR: 8.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 7.1210s | Phase Times: max_pre_step_barrier: 0.0107 | max_batch_to_device: 0.0207 | max_forward: 1.2284 | max_loss: 0.2740 | max_backward: 2.6428 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8714 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 6147.7 | uss_MB: 5022.1 | child_rss_MB: 5936.9 | allocated_MB: 448.8 | reserved_MB: 29988.0 | inactive_split_MB: 1332.2 +rank2 [2026-10-28 21:45:07,596] INFO : Step 12500/50000 rank2 | Train Loss: 1.4931 | Train Loss Unweighted: 0.7691 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1768 | Classification: 0.1197 | Classification_binary: 0.2587 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1910 | Regression_eta: 0.0024 | Regression_pt: 0.1944 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0863 | Classification_binary: 0.2542 | Regression_cos_phi: 0.1415 | Regression_energy: 0.0920 | Regression_eta: 0.0647 | Regression_pt: 0.1026 | Regression_sin_phi: 0.1282 | LR: 8.68e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7428s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0333 | max_forward: 0.8806 | max_loss: 0.1518 | max_backward: 2.1554 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.5983 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6144.5 | uss_MB: 5232.3 | child_rss_MB: 5946.8 | allocated_MB: 452.8 | reserved_MB: 29990.0 | inactive_split_MB: 1046.7 +rank2 [2026-10-28 21:50:03,651] INFO : Step 12600/50000 rank2 | Train Loss: 1.3547 | Train Loss Unweighted: 0.7028 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1629 | Classification: 0.1033 | Classification_binary: 0.2112 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1912 | Regression_eta: 0.0024 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.1862 | Regression_cos_phi: 0.0908 | Regression_energy: 0.1034 | Regression_eta: 0.0630 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0939 | LR: 8.66e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5167s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.7639 | max_loss: 0.2516 | max_backward: 1.8947 | max_optimizer_step: 0.2049 | max_tensorboard_logging: 0.7350 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6141.1 | uss_MB: 5233.0 | child_rss_MB: 5920.5 | allocated_MB: 446.9 | reserved_MB: 29990.0 | inactive_split_MB: 1034.9 +rank2 [2026-10-28 21:54:54,644] INFO : Step 12700/50000 rank2 | Train Loss: 1.3557 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1573 | Classification: 0.1262 | Classification_binary: 0.2132 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1863 | Regression_eta: 0.0022 | Regression_pt: 0.1887 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1954 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0875 | Regression_eta: 0.0498 | Regression_pt: 0.0952 | Regression_sin_phi: 0.1049 | LR: 8.64e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.1258s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7499 | max_loss: 0.2453 | max_backward: 3.4334 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6139.6 | uss_MB: 5232.1 | child_rss_MB: 5948.4 | allocated_MB: 450.2 | reserved_MB: 29990.0 | inactive_split_MB: 1245.6 +rank2 [2026-10-28 21:59:39,371] INFO : Step 12800/50000 rank2 | Train Loss: 1.5298 | Train Loss Unweighted: 0.7754 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1939 | Classification: 0.1237 | Classification_binary: 0.2821 | HitGrouping: 0.1822 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1810 | Regression_eta: 0.0027 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0761 | Classification_binary: 0.2948 | Regression_cos_phi: 0.1369 | Regression_energy: 0.0936 | Regression_eta: 0.0788 | Regression_pt: 0.1031 | Regression_sin_phi: 0.1194 | LR: 8.62e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.2923s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.5299 | max_backward: 2.3185 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6133.4 | uss_MB: 4813.1 | child_rss_MB: 5927.2 | allocated_MB: 449.9 | reserved_MB: 32006.0 | inactive_split_MB: 889.3 +rank2 [2026-10-28 22:04:39,764] INFO : Step 12900/50000 rank2 | Train Loss: 1.3302 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1576 | Classification: 0.1054 | Classification_binary: 0.2343 | HitGrouping: 0.1811 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1758 | Regression_eta: 0.0020 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2366 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0845 | Regression_eta: 0.0599 | Regression_pt: 0.0936 | Regression_sin_phi: 0.1167 | LR: 8.60e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.6341s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0137 | max_forward: 0.9836 | max_loss: 0.2293 | max_backward: 2.2888 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.9302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6139.6 | uss_MB: 5231.4 | child_rss_MB: 5950.5 | allocated_MB: 445.9 | reserved_MB: 32006.0 | inactive_split_MB: 1247.6 +rank2 [2026-10-28 22:09:25,338] INFO : Step 13000/50000 rank2 | Train Loss: 1.3124 | Train Loss Unweighted: 0.7073 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1406 | Classification: 0.1147 | Classification_binary: 0.1973 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1944 | Regression_eta: 0.0023 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.2187 | Regression_cos_phi: 0.1009 | Regression_energy: 0.1246 | Regression_eta: 0.0879 | Regression_pt: 0.1327 | Regression_sin_phi: 0.1040 | LR: 8.58e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2805s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0065 | max_forward: 0.7321 | max_loss: 0.1616 | max_backward: 1.7374 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6140.2 | uss_MB: 5231.6 | child_rss_MB: 5949.6 | allocated_MB: 444.7 | reserved_MB: 32006.0 | inactive_split_MB: 1300.2 +rank2 [2026-10-28 22:09:26,329] INFO : Running validation on rank2 +rank2 [2026-10-28 22:09:26,329] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 22:09:26,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 22:10:21,901] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:10:22,474] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 22:10:48,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 22:10:49,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 22:10:50,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 22:10:51,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 22:11:01,865] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 22:11:02,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:11:02,875] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 22:11:18,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 22:11:19,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 22:11:20,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 22:11:21,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 22:11:31,937] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 22:11:32,690] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 22:11:33,218] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 22:11:48,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 22:11:49,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 22:11:50,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 22:11:51,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 22:12:01,642] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 22:17:08,486] INFO : Step 13100/50000 rank2 | Train Loss: 1.6165 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3082 | Cardinality: 0.2635 | Classification: 0.1364 | Classification_binary: 0.2628 | HitGrouping: 0.1835 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0026 | Regression_pt: 0.1957 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.2631 | Regression_cos_phi: 0.1241 | Regression_energy: 0.0980 | Regression_eta: 0.0720 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0647 | LR: 8.56e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2710s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0479 | max_forward: 0.8994 | max_loss: 0.2807 | max_backward: 2.1108 | max_optimizer_step: 0.2760 | max_tensorboard_logging: 0.9690 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 7057.8 | uss_MB: 6143.5 | child_rss_MB: 5948.1 | allocated_MB: 446.5 | reserved_MB: 23120.0 | inactive_split_MB: 920.0 +rank2 [2026-10-28 22:22:01,651] INFO : Step 13200/50000 rank2 | Train Loss: 1.4928 | Train Loss Unweighted: 0.7469 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.2211 | Classification: 0.1278 | Classification_binary: 0.2505 | HitGrouping: 0.1746 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1808 | Regression_eta: 0.0025 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2431 | Regression_cos_phi: 0.0867 | Regression_energy: 0.0945 | Regression_eta: 0.0582 | Regression_pt: 0.1044 | Regression_sin_phi: 0.1189 | LR: 8.54e-04 | DataLoad Time: 0.0176s | Model Forward Time: 6.6010s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.7755 | max_loss: 0.2864 | max_backward: 3.4589 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6222.1 | uss_MB: 5074.0 | child_rss_MB: 5943.3 | allocated_MB: 448.3 | reserved_MB: 25156.0 | inactive_split_MB: 978.0 +rank2 [2026-10-28 22:26:52,973] INFO : Step 13300/50000 rank2 | Train Loss: 1.3251 | Train Loss Unweighted: 0.6741 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1607 | Classification: 0.1132 | Classification_binary: 0.1927 | HitGrouping: 0.1773 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1805 | Regression_eta: 0.0024 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0874 | Regression_eta: 0.0477 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0850 | LR: 8.52e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3787s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8118 | max_loss: 0.1467 | max_backward: 1.5503 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.9039 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6221.5 | uss_MB: 5307.8 | child_rss_MB: 5952.8 | allocated_MB: 446.0 | reserved_MB: 26736.0 | inactive_split_MB: 965.8 +rank2 [2026-10-28 22:31:44,118] INFO : Step 13400/50000 rank2 | Train Loss: 1.2635 | Train Loss Unweighted: 0.6770 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1436 | Classification: 0.1317 | Classification_binary: 0.2016 | HitGrouping: 0.1790 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1692 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2395 | Regression_cos_phi: 0.1026 | Regression_energy: 0.0879 | Regression_eta: 0.0584 | Regression_pt: 0.0953 | Regression_sin_phi: 0.1194 | LR: 8.50e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7361s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8396 | max_loss: 0.1412 | max_backward: 2.0130 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.8047 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6225.0 | uss_MB: 5310.9 | child_rss_MB: 5925.4 | allocated_MB: 445.8 | reserved_MB: 26736.0 | inactive_split_MB: 1014.4 +rank2 [2026-10-28 22:36:27,627] INFO : Step 13500/50000 rank2 | Train Loss: 1.3935 | Train Loss Unweighted: 0.7401 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1713 | Classification: 0.1245 | Classification_binary: 0.2541 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0021 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0812 | Classification_binary: 0.2650 | Regression_cos_phi: 0.1106 | Regression_energy: 0.0994 | Regression_eta: 0.0693 | Regression_pt: 0.1106 | Regression_sin_phi: 0.1365 | LR: 8.48e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5757s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0859 | max_forward: 0.8721 | max_loss: 0.1498 | max_backward: 1.7581 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.7855 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6221.6 | uss_MB: 5308.9 | child_rss_MB: 5951.2 | allocated_MB: 449.1 | reserved_MB: 26736.0 | inactive_split_MB: 922.5 +rank2 [2026-10-28 22:41:19,302] INFO : Step 13600/50000 rank2 | Train Loss: 1.3997 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1870 | Classification: 0.1206 | Classification_binary: 0.2286 | HitGrouping: 0.1765 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1702 | Regression_eta: 0.0026 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.2399 | Regression_cos_phi: 0.1391 | Regression_energy: 0.0780 | Regression_eta: 0.0747 | Regression_pt: 0.0864 | Regression_sin_phi: 0.1425 | LR: 8.45e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.2262s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7703 | max_loss: 0.2484 | max_backward: 1.6943 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 6210.7 | uss_MB: 5299.9 | child_rss_MB: 5953.4 | allocated_MB: 450.3 | reserved_MB: 26736.0 | inactive_split_MB: 922.0 +rank2 [2026-10-28 22:46:12,327] INFO : Step 13700/50000 rank2 | Train Loss: 1.2648 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2420 | Cardinality: 0.1315 | Classification: 0.1136 | Classification_binary: 0.1964 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1793 | Regression_eta: 0.0020 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0777 | Classification_binary: 0.2183 | Regression_cos_phi: 0.1030 | Regression_energy: 0.0889 | Regression_eta: 0.0579 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0843 | LR: 8.43e-04 | DataLoad Time: 0.0137s | Model Forward Time: 6.2843s | Phase Times: max_pre_step_barrier: 0.0406 | max_batch_to_device: 0.0119 | max_forward: 1.2281 | max_loss: 0.2480 | max_backward: 3.6472 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.6460 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0767 | Memory: rss_MB: 6216.0 | uss_MB: 5307.4 | child_rss_MB: 5948.4 | allocated_MB: 446.9 | reserved_MB: 26742.0 | inactive_split_MB: 1058.7 +rank2 [2026-10-28 22:51:09,234] INFO : Step 13800/50000 rank2 | Train Loss: 1.3469 | Train Loss Unweighted: 0.7266 | Train Loss Components: Auxiliary: 0.2604 | Cardinality: 0.1436 | Classification: 0.1184 | Classification_binary: 0.2382 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1812 | Regression_eta: 0.0021 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.2603 | Regression_cos_phi: 0.1050 | Regression_energy: 0.0955 | Regression_eta: 0.0788 | Regression_pt: 0.1055 | Regression_sin_phi: 0.1309 | LR: 8.41e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.8548s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7995 | max_loss: 0.1458 | max_backward: 3.4735 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6221.0 | uss_MB: 5310.1 | child_rss_MB: 5919.7 | allocated_MB: 446.4 | reserved_MB: 27082.0 | inactive_split_MB: 1370.3 +rank2 [2026-10-28 22:56:04,350] INFO : Step 13900/50000 rank2 | Train Loss: 1.4882 | Train Loss Unweighted: 0.7682 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1631 | Classification: 0.1199 | Classification_binary: 0.2624 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1900 | Regression_eta: 0.0029 | Regression_pt: 0.1902 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2824 | Regression_cos_phi: 0.1034 | Regression_energy: 0.1059 | Regression_eta: 0.0638 | Regression_pt: 0.1163 | Regression_sin_phi: 0.1034 | LR: 8.39e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4549s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.7750 | max_loss: 0.1563 | max_backward: 1.5716 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.8868 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6217.9 | uss_MB: 5307.2 | child_rss_MB: 5946.5 | allocated_MB: 449.9 | reserved_MB: 27316.0 | inactive_split_MB: 918.0 +rank2 [2026-10-28 23:00:54,751] INFO : Step 14000/50000 rank2 | Train Loss: 1.2239 | Train Loss Unweighted: 0.6384 | Train Loss Components: Auxiliary: 0.2362 | Cardinality: 0.1573 | Classification: 0.0898 | Classification_binary: 0.1853 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0021 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0818 | Regression_eta: 0.0555 | Regression_pt: 0.0908 | Regression_sin_phi: 0.1125 | LR: 8.37e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5032s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7611 | max_loss: 0.1442 | max_backward: 1.7887 | max_optimizer_step: 0.2976 | max_tensorboard_logging: 0.7418 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6219.2 | uss_MB: 5309.9 | child_rss_MB: 5948.9 | allocated_MB: 446.8 | reserved_MB: 29264.0 | inactive_split_MB: 1150.9 +rank2 [2026-10-28 23:00:55,745] INFO : Running validation on rank2 +rank2 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 23:01:50,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:01:51,151] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 23:02:17,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 23:02:19,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 23:02:20,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 23:02:22,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 23:02:32,170] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-28 23:02:33,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:02:33,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 23:02:49,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 23:02:50,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 23:02:51,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 23:02:52,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 23:03:02,904] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 23:03:03,407] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:03:03,862] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 23:03:19,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 23:03:20,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 23:03:21,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 23:03:21,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 23:03:31,842] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-28 23:08:22,315] INFO : Step 14100/50000 rank2 | Train Loss: 1.3472 | Train Loss Unweighted: 0.6964 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1511 | Classification: 0.1164 | Classification_binary: 0.2090 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1819 | Regression_eta: 0.0023 | Regression_pt: 0.1843 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.2216 | Regression_cos_phi: 0.1024 | Regression_energy: 0.1021 | Regression_eta: 0.0571 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0948 | LR: 8.35e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2193s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8885 | max_loss: 0.1430 | max_backward: 2.1784 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 1.1214 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7149.4 | uss_MB: 6238.2 | child_rss_MB: 5960.3 | allocated_MB: 451.8 | reserved_MB: 25710.0 | inactive_split_MB: 1103.7 +rank2 [2026-10-28 23:12:52,862] INFO : Step 14200/50000 rank2 | Train Loss: 1.4358 | Train Loss Unweighted: 0.7354 | Train Loss Components: Auxiliary: 0.2744 | Cardinality: 0.1685 | Classification: 0.1248 | Classification_binary: 0.2623 | HitGrouping: 0.1814 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1709 | Regression_eta: 0.0023 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.2588 | Regression_cos_phi: 0.1304 | Regression_energy: 0.0907 | Regression_eta: 0.0620 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0943 | LR: 8.32e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.3445s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0875 | max_forward: 0.8579 | max_loss: 0.1577 | max_backward: 1.8303 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6333.7 | uss_MB: 5422.0 | child_rss_MB: 5960.5 | allocated_MB: 449.2 | reserved_MB: 27638.0 | inactive_split_MB: 1319.6 +rank2 [2026-10-28 23:17:26,478] INFO : Step 14300/50000 rank2 | Train Loss: 1.2642 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2452 | Cardinality: 0.1652 | Classification: 0.1069 | Classification_binary: 0.2108 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0018 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0869 | Classification_binary: 0.2305 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0857 | Regression_eta: 0.0452 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0655 | LR: 8.30e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3474s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7599 | max_loss: 0.1477 | max_backward: 1.7369 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6331.9 | uss_MB: 5419.4 | child_rss_MB: 5959.7 | allocated_MB: 446.3 | reserved_MB: 29582.0 | inactive_split_MB: 949.7 +rank2 [2026-10-28 23:22:05,870] INFO : Step 14400/50000 rank2 | Train Loss: 1.2457 | Train Loss Unweighted: 0.6253 | Train Loss Components: Auxiliary: 0.2300 | Cardinality: 0.1408 | Classification: 0.1072 | Classification_binary: 0.1847 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1629 | Regression_eta: 0.0022 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.1864 | Regression_cos_phi: 0.1108 | Regression_energy: 0.0813 | Regression_eta: 0.0569 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0831 | LR: 8.28e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.9706s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.9667 | max_loss: 0.2306 | max_backward: 2.7434 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6331.8 | uss_MB: 4969.8 | child_rss_MB: 5956.2 | allocated_MB: 450.7 | reserved_MB: 29584.0 | inactive_split_MB: 1252.8 +rank2 [2026-10-28 23:26:50,539] INFO : Step 14500/50000 rank2 | Train Loss: 1.4093 | Train Loss Unweighted: 0.7194 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1803 | Classification: 0.1211 | Classification_binary: 0.2406 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2155 | Regression_cos_phi: 0.1061 | Regression_energy: 0.0963 | Regression_eta: 0.1467 | Regression_pt: 0.1059 | Regression_sin_phi: 0.1170 | LR: 8.26e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.8494s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8204 | max_loss: 0.1455 | max_backward: 2.0838 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7592 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1066 | Memory: rss_MB: 6329.0 | uss_MB: 5423.5 | child_rss_MB: 5932.6 | allocated_MB: 449.1 | reserved_MB: 29584.0 | inactive_split_MB: 1312.1 +rank2 [2026-10-28 23:31:26,129] INFO : Step 14600/50000 rank2 | Train Loss: 1.4466 | Train Loss Unweighted: 0.7628 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1607 | Classification: 0.1257 | Classification_binary: 0.2761 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1767 | Regression_eta: 0.0023 | Regression_pt: 0.1796 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.2732 | Regression_cos_phi: 0.0946 | Regression_energy: 0.0976 | Regression_eta: 0.0479 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0648 | LR: 8.24e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.8580s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0622 | max_forward: 0.8658 | max_loss: 0.1496 | max_backward: 1.9094 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.9894 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6320.2 | uss_MB: 5414.6 | child_rss_MB: 5962.0 | allocated_MB: 450.2 | reserved_MB: 29584.0 | inactive_split_MB: 939.8 +rank2 [2026-10-28 23:35:59,707] INFO : Step 14700/50000 rank2 | Train Loss: 1.2298 | Train Loss Unweighted: 0.6544 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1321 | Classification: 0.0995 | Classification_binary: 0.1882 | HitGrouping: 0.1788 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1813 | Regression_eta: 0.0021 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1879 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0982 | Regression_eta: 0.0476 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0905 | LR: 8.21e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.9555s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8361 | max_loss: 0.2462 | max_backward: 1.9700 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.8314 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6323.9 | uss_MB: 5421.4 | child_rss_MB: 5964.3 | allocated_MB: 445.8 | reserved_MB: 31548.0 | inactive_split_MB: 1127.1 +rank2 [2026-10-28 23:40:34,506] INFO : Step 14800/50000 rank2 | Train Loss: 1.2682 | Train Loss Unweighted: 0.6574 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1516 | Classification: 0.1063 | Classification_binary: 0.2120 | HitGrouping: 0.1767 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1664 | Regression_eta: 0.0020 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.2116 | Regression_cos_phi: 0.0940 | Regression_energy: 0.0963 | Regression_eta: 0.0716 | Regression_pt: 0.1041 | Regression_sin_phi: 0.0887 | LR: 8.19e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.6957s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7530 | max_loss: 0.1476 | max_backward: 1.8229 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6323.4 | uss_MB: 5420.5 | child_rss_MB: 5957.9 | allocated_MB: 449.4 | reserved_MB: 33554.0 | inactive_split_MB: 1332.6 +rank2 [2026-10-28 23:45:11,571] INFO : Step 14900/50000 rank2 | Train Loss: 1.2753 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2409 | Cardinality: 0.1305 | Classification: 0.1126 | Classification_binary: 0.2387 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1497 | Regression_eta: 0.0022 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2226 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0874 | Regression_eta: 0.0517 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0968 | LR: 8.17e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8754s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.7955 | max_loss: 0.1633 | max_backward: 1.8376 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.9623 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1206 | Memory: rss_MB: 6322.3 | uss_MB: 5420.8 | child_rss_MB: 5961.8 | allocated_MB: 449.7 | reserved_MB: 33554.0 | inactive_split_MB: 1310.1 +rank2 [2026-10-28 23:49:45,306] INFO : Step 15000/50000 rank2 | Train Loss: 1.2701 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1516 | Classification: 0.1012 | Classification_binary: 0.2121 | HitGrouping: 0.1755 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0829 | Classification_binary: 0.2171 | Regression_cos_phi: 0.0797 | Regression_energy: 0.0777 | Regression_eta: 0.0480 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0750 | LR: 8.15e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.1012s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7505 | max_loss: 0.2449 | max_backward: 1.7335 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.4870 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6318.2 | uss_MB: 5419.0 | child_rss_MB: 5953.6 | allocated_MB: 447.3 | reserved_MB: 33554.0 | inactive_split_MB: 1038.0 +rank2 [2026-10-28 23:49:46,286] INFO : Running validation on rank2 +rank2 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-28 23:50:37,476] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:50:38,005] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-28 23:51:04,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-28 23:51:05,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-28 23:51:06,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-28 23:51:07,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-28 23:51:17,665] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 23:51:18,491] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:51:18,970] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-28 23:51:35,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-28 23:51:36,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-28 23:51:37,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-28 23:51:38,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-28 23:51:48,017] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-28 23:51:48,549] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-28 23:51:49,027] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-28 23:52:04,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-28 23:52:05,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-28 23:52:06,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-28 23:52:07,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-28 23:52:17,358] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-28 23:57:18,881] INFO : Step 15100/50000 rank2 | Train Loss: 1.2388 | Train Loss Unweighted: 0.6590 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1366 | Classification: 0.1178 | Classification_binary: 0.1841 | HitGrouping: 0.1810 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1743 | Regression_eta: 0.0018 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0779 | Regression_eta: 0.0541 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0729 | LR: 8.12e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.8275s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.8010 | max_loss: 0.1660 | max_backward: 1.9832 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.8565 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7422.7 | uss_MB: 6518.8 | child_rss_MB: 5938.1 | allocated_MB: 446.2 | reserved_MB: 25230.0 | inactive_split_MB: 1087.3 +rank2 [2026-10-29 00:02:05,559] INFO : Step 15200/50000 rank2 | Train Loss: 1.3362 | Train Loss Unweighted: 0.6611 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1596 | Classification: 0.1086 | Classification_binary: 0.2256 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1600 | Regression_eta: 0.0022 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.2391 | Regression_cos_phi: 0.0908 | Regression_energy: 0.1043 | Regression_eta: 0.0621 | Regression_pt: 0.1144 | Regression_sin_phi: 0.1076 | LR: 8.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6164s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1077 | max_forward: 0.8572 | max_loss: 0.1505 | max_backward: 1.8951 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7089 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6402.3 | uss_MB: 5500.4 | child_rss_MB: 5939.7 | allocated_MB: 449.9 | reserved_MB: 25448.0 | inactive_split_MB: 1183.4 +rank2 [2026-10-29 00:06:54,749] INFO : Step 15300/50000 rank2 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6783 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1400 | Classification: 0.1062 | Classification_binary: 0.2357 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1646 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.2354 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0850 | Regression_eta: 0.0505 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0859 | LR: 8.08e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7473s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7962 | max_loss: 0.2606 | max_backward: 1.9831 | max_optimizer_step: 0.2491 | max_tensorboard_logging: 0.7840 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6402.1 | uss_MB: 5500.3 | child_rss_MB: 5940.3 | allocated_MB: 446.7 | reserved_MB: 27282.0 | inactive_split_MB: 1189.4 +rank2 [2026-10-29 00:11:34,440] INFO : Step 15400/50000 rank2 | Train Loss: 1.2329 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1583 | Classification: 0.1127 | Classification_binary: 0.1891 | HitGrouping: 0.1750 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1648 | Regression_eta: 0.0018 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0919 | Classification_binary: 0.2008 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0835 | Regression_eta: 0.0603 | Regression_pt: 0.0905 | Regression_sin_phi: 0.1006 | LR: 8.05e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2514s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.7153 | max_loss: 0.1508 | max_backward: 1.6501 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.5965 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6395.9 | uss_MB: 5491.2 | child_rss_MB: 5941.0 | allocated_MB: 444.7 | reserved_MB: 27520.0 | inactive_split_MB: 1102.9 +rank2 [2026-10-29 00:16:23,332] INFO : Step 15500/50000 rank2 | Train Loss: 1.2944 | Train Loss Unweighted: 0.6657 | Train Loss Components: Auxiliary: 0.2480 | Cardinality: 0.1645 | Classification: 0.1174 | Classification_binary: 0.2036 | HitGrouping: 0.1739 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1691 | Regression_eta: 0.0021 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0718 | Classification_binary: 0.2031 | Regression_cos_phi: 0.1049 | Regression_energy: 0.1021 | Regression_eta: 0.0776 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0897 | LR: 8.03e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.2294s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.9970 | max_loss: 0.1740 | max_backward: 2.3916 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.8759 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 6406.1 | uss_MB: 5472.3 | child_rss_MB: 5938.5 | allocated_MB: 449.9 | reserved_MB: 27520.0 | inactive_split_MB: 1174.5 +rank2 [2026-10-29 00:21:07,252] INFO : Step 15600/50000 rank2 | Train Loss: 1.4613 | Train Loss Unweighted: 0.7617 | Train Loss Components: Auxiliary: 0.2872 | Cardinality: 0.1801 | Classification: 0.1276 | Classification_binary: 0.2641 | HitGrouping: 0.1785 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1795 | Regression_eta: 0.0025 | Regression_pt: 0.1856 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2509 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0933 | Regression_eta: 0.0641 | Regression_pt: 0.1040 | Regression_sin_phi: 0.1151 | LR: 8.01e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.7233s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.9893 | max_loss: 0.1680 | max_backward: 2.1731 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.0419 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6396.7 | uss_MB: 5034.1 | child_rss_MB: 5938.2 | allocated_MB: 448.1 | reserved_MB: 27520.0 | inactive_split_MB: 1080.9 +rank2 [2026-10-29 00:25:54,170] INFO : Step 15700/50000 rank2 | Train Loss: 1.3299 | Train Loss Unweighted: 0.6547 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1880 | Classification: 0.1192 | Classification_binary: 0.2203 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1547 | Regression_eta: 0.0020 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1847 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0815 | Regression_eta: 0.0297 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0705 | LR: 7.98e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.0325s | Phase Times: max_pre_step_barrier: 0.0061 | max_batch_to_device: 0.0068 | max_forward: 0.9761 | max_loss: 0.2543 | max_backward: 2.0225 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8016 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.1020 | Memory: rss_MB: 6402.6 | uss_MB: 5496.7 | child_rss_MB: 5941.3 | allocated_MB: 448.3 | reserved_MB: 27520.0 | inactive_split_MB: 1390.5 +rank2 [2026-10-29 00:30:33,527] INFO : Step 15800/50000 rank2 | Train Loss: 1.2079 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1572 | Classification: 0.0909 | Classification_binary: 0.1814 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1704 | Regression_eta: 0.0019 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1775 | Regression_cos_phi: 0.0717 | Regression_energy: 0.1002 | Regression_eta: 0.0572 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0898 | LR: 7.96e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3439s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1066 | max_forward: 0.8154 | max_loss: 0.1440 | max_backward: 1.6020 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.7740 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6405.3 | uss_MB: 5497.2 | child_rss_MB: 5941.3 | allocated_MB: 447.1 | reserved_MB: 27520.0 | inactive_split_MB: 1038.3 +rank2 [2026-10-29 00:35:19,265] INFO : Step 15900/50000 rank2 | Train Loss: 1.4055 | Train Loss Unweighted: 0.7313 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1808 | Classification: 0.1304 | Classification_binary: 0.2426 | HitGrouping: 0.1763 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1752 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.2614 | Regression_cos_phi: 0.1239 | Regression_energy: 0.0945 | Regression_eta: 0.0559 | Regression_pt: 0.1051 | Regression_sin_phi: 0.1076 | LR: 7.94e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.7742s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 1.0653 | max_loss: 0.1515 | max_backward: 2.0024 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.6957 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6402.5 | uss_MB: 5500.0 | child_rss_MB: 5940.1 | allocated_MB: 447.4 | reserved_MB: 27520.0 | inactive_split_MB: 1172.5 +rank2 [2026-10-29 00:40:00,683] INFO : Step 16000/50000 rank2 | Train Loss: 1.3107 | Train Loss Unweighted: 0.6580 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1760 | Classification: 0.1019 | Classification_binary: 0.2391 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1548 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2714 | Regression_cos_phi: 0.1109 | Regression_energy: 0.0721 | Regression_eta: 0.0759 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0891 | LR: 7.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7629s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.1053 | max_forward: 1.1703 | max_loss: 0.1516 | max_backward: 1.8974 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6403.0 | uss_MB: 5502.7 | child_rss_MB: 5941.0 | allocated_MB: 447.2 | reserved_MB: 27520.0 | inactive_split_MB: 1195.6 +rank2 [2026-10-29 00:40:01,696] INFO : Running validation on rank2 +rank2 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 00:40:58,072] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:40:58,626] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 00:41:23,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 00:41:24,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 00:41:25,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 00:41:26,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 00:41:36,547] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 00:41:36,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:41:37,530] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 00:41:53,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 00:41:54,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 00:41:55,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 00:41:56,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 00:42:06,743] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 00:42:07,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 00:42:07,841] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 00:42:23,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 00:42:24,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 00:42:24,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 00:42:25,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 00:42:35,726] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 00:47:34,084] INFO : Step 16100/50000 rank2 | Train Loss: 1.2037 | Train Loss Unweighted: 0.6025 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1467 | Classification: 0.1080 | Classification_binary: 0.1878 | HitGrouping: 0.1747 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1512 | Regression_eta: 0.0018 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1830 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0759 | Regression_eta: 0.0603 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0941 | LR: 7.89e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.0460s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7665 | max_loss: 0.1405 | max_backward: 1.4602 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7547.3 | uss_MB: 6643.5 | child_rss_MB: 5941.5 | allocated_MB: 446.1 | reserved_MB: 27680.0 | inactive_split_MB: 1038.2 +rank2 [2026-10-29 00:52:15,156] INFO : Step 16200/50000 rank2 | Train Loss: 1.2083 | Train Loss Unweighted: 0.6192 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1499 | Classification: 0.1107 | Classification_binary: 0.2017 | HitGrouping: 0.1711 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1506 | Regression_eta: 0.0018 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1889 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0654 | Regression_eta: 0.0437 | Regression_pt: 0.0736 | Regression_sin_phi: 0.0821 | LR: 7.86e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.9538s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0473 | max_forward: 0.8343 | max_loss: 0.2530 | max_backward: 2.1136 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6458.0 | uss_MB: 5555.6 | child_rss_MB: 5961.3 | allocated_MB: 446.7 | reserved_MB: 27682.0 | inactive_split_MB: 1157.2 +rank2 [2026-10-29 00:56:51,719] INFO : Step 16300/50000 rank2 | Train Loss: 1.2745 | Train Loss Unweighted: 0.6291 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1604 | Classification: 0.1062 | Classification_binary: 0.2417 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1373 | Regression_eta: 0.0022 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.2486 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0653 | Regression_eta: 0.0598 | Regression_pt: 0.0703 | Regression_sin_phi: 0.1092 | LR: 7.84e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.8117s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.8059 | max_loss: 0.2506 | max_backward: 2.1763 | max_optimizer_step: 0.5213 | max_tensorboard_logging: 0.5923 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6459.9 | uss_MB: 5555.3 | child_rss_MB: 5936.4 | allocated_MB: 450.9 | reserved_MB: 27682.0 | inactive_split_MB: 950.8 +rank2 [2026-10-29 01:01:26,580] INFO : Step 16400/50000 rank2 | Train Loss: 1.2654 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2380 | Cardinality: 0.1554 | Classification: 0.1095 | Classification_binary: 0.2018 | HitGrouping: 0.1699 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1561 | Regression_eta: 0.0023 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.1764 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0880 | Regression_eta: 0.0636 | Regression_pt: 0.0979 | Regression_sin_phi: 0.0754 | LR: 7.81e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.2784s | Phase Times: max_pre_step_barrier: 0.0533 | max_batch_to_device: 0.0064 | max_forward: 0.7459 | max_loss: 0.1486 | max_backward: 1.5352 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.6677 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6452.2 | uss_MB: 5545.1 | child_rss_MB: 5962.3 | allocated_MB: 446.4 | reserved_MB: 27682.0 | inactive_split_MB: 949.1 +rank2 [2026-10-29 01:06:05,049] INFO : Step 16500/50000 rank2 | Train Loss: 1.1728 | Train Loss Unweighted: 0.5918 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1444 | Classification: 0.0927 | Classification_binary: 0.1686 | HitGrouping: 0.1723 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0021 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.1698 | Regression_cos_phi: 0.1120 | Regression_energy: 0.0855 | Regression_eta: 0.0627 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0955 | LR: 7.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4907s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8043 | max_loss: 0.1772 | max_backward: 2.0250 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6458.1 | uss_MB: 5554.8 | child_rss_MB: 5927.7 | allocated_MB: 448.6 | reserved_MB: 29698.0 | inactive_split_MB: 1167.6 +rank2 [2026-10-29 01:10:53,702] INFO : Step 16600/50000 rank2 | Train Loss: 1.2920 | Train Loss Unweighted: 0.6427 | Train Loss Components: Auxiliary: 0.2455 | Cardinality: 0.1794 | Classification: 0.1148 | Classification_binary: 0.2098 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1561 | Regression_eta: 0.0021 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2035 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0893 | Regression_eta: 0.0534 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0773 | LR: 7.77e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.2553s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0377 | max_forward: 1.0830 | max_loss: 0.1480 | max_backward: 1.7919 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6457.2 | uss_MB: 5552.9 | child_rss_MB: 5951.6 | allocated_MB: 449.3 | reserved_MB: 29698.0 | inactive_split_MB: 1014.7 +rank2 [2026-10-29 01:15:45,846] INFO : Step 16700/50000 rank2 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6325 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1731 | Classification: 0.1072 | Classification_binary: 0.2252 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1469 | Regression_eta: 0.0019 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.1943 | Regression_cos_phi: 0.1139 | Regression_energy: 0.0898 | Regression_eta: 0.0449 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0652 | LR: 7.74e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4534s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0722 | max_loss: 0.1506 | max_backward: 2.0365 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5067 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6456.8 | uss_MB: 5555.5 | child_rss_MB: 5918.8 | allocated_MB: 448.3 | reserved_MB: 29698.0 | inactive_split_MB: 893.3 +rank2 [2026-10-29 01:20:20,822] INFO : Step 16800/50000 rank2 | Train Loss: 1.1947 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1542 | Classification: 0.1036 | Classification_binary: 0.1800 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0023 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1996 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0768 | Regression_eta: 0.0507 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0615 | LR: 7.72e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.2743s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0166 | max_forward: 0.8224 | max_loss: 0.1466 | max_backward: 1.6608 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6428.0 | uss_MB: 5552.9 | child_rss_MB: 5812.3 | allocated_MB: 446.4 | reserved_MB: 29698.0 | inactive_split_MB: 1161.6 +rank2 [2026-10-29 01:25:06,863] INFO : Step 16900/50000 rank2 | Train Loss: 1.2025 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1406 | Classification: 0.1120 | Classification_binary: 0.1884 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1446 | Regression_eta: 0.0023 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0895 | Classification_binary: 0.2128 | Regression_cos_phi: 0.1128 | Regression_energy: 0.0846 | Regression_eta: 0.0940 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0959 | LR: 7.69e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.3244s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7445 | max_loss: 0.1453 | max_backward: 1.8211 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6418.8 | uss_MB: 5553.4 | child_rss_MB: 5783.4 | allocated_MB: 448.9 | reserved_MB: 29698.0 | inactive_split_MB: 901.3 +rank2 [2026-10-29 01:29:52,083] INFO : Step 17000/50000 rank2 | Train Loss: 1.2049 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2349 | Cardinality: 0.1467 | Classification: 0.1018 | Classification_binary: 0.2228 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.2065 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0962 | Regression_eta: 0.0418 | Regression_pt: 0.1042 | Regression_sin_phi: 0.0679 | LR: 7.67e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.1422s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0141 | max_forward: 1.3980 | max_loss: 0.5129 | max_backward: 2.5664 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.5255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1218 | Memory: rss_MB: 6382.7 | uss_MB: 5556.2 | child_rss_MB: 5576.1 | allocated_MB: 448.2 | reserved_MB: 29698.0 | inactive_split_MB: 955.8 +rank2 [2026-10-29 01:29:53,074] INFO : Running validation on rank2 +rank2 [2026-10-29 01:29:53,074] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 01:29:53,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 01:30:46,067] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:30:46,579] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 01:31:13,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 01:31:14,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 01:31:16,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 01:31:17,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 01:31:27,202] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 01:31:28,037] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:31:28,589] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 01:31:44,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 01:31:45,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 01:31:46,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 01:31:47,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 01:31:57,428] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 01:31:58,252] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 01:31:58,743] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 01:32:14,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 01:32:15,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 01:32:16,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 01:32:17,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 01:32:27,087] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 01:37:36,198] INFO : Step 17100/50000 rank2 | Train Loss: 1.2386 | Train Loss Unweighted: 0.5919 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1931 | Classification: 0.0995 | Classification_binary: 0.1969 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1795 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0711 | Regression_eta: 0.0507 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0679 | LR: 7.64e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.3605s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0067 | max_forward: 1.0304 | max_loss: 0.1869 | max_backward: 2.3467 | max_optimizer_step: 0.3175 | max_tensorboard_logging: 0.7405 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 7631.0 | uss_MB: 6798.9 | child_rss_MB: 5579.4 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 1373.9 +rank2 [2026-10-29 01:42:27,134] INFO : Step 17200/50000 rank2 | Train Loss: 1.1810 | Train Loss Unweighted: 0.5844 | Train Loss Components: Auxiliary: 0.2270 | Cardinality: 0.1495 | Classification: 0.1085 | Classification_binary: 0.1683 | HitGrouping: 0.1731 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0022 | Regression_pt: 0.1515 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0957 | Regression_energy: 0.0875 | Regression_eta: 0.0793 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0767 | LR: 7.62e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.1057s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7638 | max_loss: 0.2424 | max_backward: 1.9393 | max_optimizer_step: 0.2921 | max_tensorboard_logging: 2.1267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6444.8 | uss_MB: 5633.5 | child_rss_MB: 5577.8 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 988.1 +rank2 [2026-10-29 01:47:11,318] INFO : Step 17300/50000 rank2 | Train Loss: 1.2271 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2397 | Cardinality: 0.1372 | Classification: 0.1102 | Classification_binary: 0.2127 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1512 | Regression_eta: 0.0022 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.2161 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0943 | Regression_eta: 0.0520 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0951 | LR: 7.59e-04 | DataLoad Time: 0.0196s | Model Forward Time: 6.1250s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0243 | max_forward: 1.4273 | max_loss: 0.6053 | max_backward: 3.4810 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6426.5 | uss_MB: 5175.0 | child_rss_MB: 5586.3 | allocated_MB: 449.6 | reserved_MB: 26752.0 | inactive_split_MB: 1120.1 +rank2 [2026-10-29 01:52:04,203] INFO : Step 17400/50000 rank2 | Train Loss: 1.2297 | Train Loss Unweighted: 0.6013 | Train Loss Components: Auxiliary: 0.2404 | Cardinality: 0.1709 | Classification: 0.1017 | Classification_binary: 0.2232 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1354 | Regression_eta: 0.0019 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.2052 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0698 | Regression_eta: 0.0448 | Regression_pt: 0.0769 | Regression_sin_phi: 0.1117 | LR: 7.57e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6385s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0185 | max_forward: 1.0000 | max_loss: 0.1504 | max_backward: 1.9673 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6331 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6409.7 | uss_MB: 5631.7 | child_rss_MB: 5583.3 | allocated_MB: 447.1 | reserved_MB: 28732.0 | inactive_split_MB: 1125.6 +rank2 [2026-10-29 01:56:48,791] INFO : Step 17500/50000 rank2 | Train Loss: 1.2152 | Train Loss Unweighted: 0.6044 | Train Loss Components: Auxiliary: 0.2290 | Cardinality: 0.1485 | Classification: 0.1023 | Classification_binary: 0.1789 | HitGrouping: 0.1757 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1590 | Regression_eta: 0.0026 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2081 | Regression_cos_phi: 0.1377 | Regression_energy: 0.0794 | Regression_eta: 0.2321 | Regression_pt: 0.0878 | Regression_sin_phi: 0.1363 | LR: 7.54e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.2349s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0167 | max_forward: 0.7711 | max_loss: 0.1449 | max_backward: 1.8153 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.4947 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6388.7 | uss_MB: 5638.8 | child_rss_MB: 5515.3 | allocated_MB: 450.1 | reserved_MB: 28732.0 | inactive_split_MB: 1375.8 +rank2 [2026-10-29 02:01:39,792] INFO : Step 17600/50000 rank2 | Train Loss: 1.1560 | Train Loss Unweighted: 0.6042 | Train Loss Components: Auxiliary: 0.2286 | Cardinality: 0.1294 | Classification: 0.1057 | Classification_binary: 0.1993 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1471 | Regression_eta: 0.0018 | Regression_pt: 0.1484 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2054 | Regression_cos_phi: 0.1152 | Regression_energy: 0.0922 | Regression_eta: 0.0528 | Regression_pt: 0.1013 | Regression_sin_phi: 0.1083 | LR: 7.52e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.8143s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.8868 | max_loss: 0.1705 | max_backward: 2.0183 | max_optimizer_step: 0.2066 | max_tensorboard_logging: 0.7222 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1349 | Memory: rss_MB: 6384.6 | uss_MB: 5639.3 | child_rss_MB: 5506.2 | allocated_MB: 448.3 | reserved_MB: 28732.0 | inactive_split_MB: 1319.7 +rank2 [2026-10-29 02:06:28,890] INFO : Step 17700/50000 rank2 | Train Loss: 1.7848 | Train Loss Unweighted: 0.7433 | Train Loss Components: Auxiliary: 0.3415 | Cardinality: 0.4306 | Classification: 0.1200 | Classification_binary: 0.2341 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1920 | Regression_eta: 0.0037 | Regression_pt: 0.1910 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0825 | Classification_binary: 0.2526 | Regression_cos_phi: 0.1754 | Regression_energy: 0.0956 | Regression_eta: 0.1123 | Regression_pt: 0.1043 | Regression_sin_phi: 0.1082 | LR: 7.49e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5773s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8788 | max_loss: 0.4738 | max_backward: 1.9859 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.5305 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6359.7 | uss_MB: 5640.4 | child_rss_MB: 5400.4 | allocated_MB: 450.1 | reserved_MB: 28738.0 | inactive_split_MB: 1125.6 +rank2 [2026-10-29 02:11:14,774] INFO : Step 17800/50000 rank2 | Train Loss: 1.1828 | Train Loss Unweighted: 0.6014 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1456 | Classification: 0.0982 | Classification_binary: 0.1996 | HitGrouping: 0.1749 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1503 | Regression_eta: 0.0019 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.1949 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0993 | Regression_eta: 0.0508 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0689 | LR: 7.47e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.1937s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.3167 | max_backward: 2.2751 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 6343.6 | uss_MB: 5261.8 | child_rss_MB: 5331.5 | allocated_MB: 444.9 | reserved_MB: 28738.0 | inactive_split_MB: 1155.9 +rank2 [2026-10-29 02:16:10,312] INFO : Step 17900/50000 rank2 | Train Loss: 1.0765 | Train Loss Unweighted: 0.5646 | Train Loss Components: Auxiliary: 0.2108 | Cardinality: 0.1097 | Classification: 0.0932 | Classification_binary: 0.1585 | HitGrouping: 0.1686 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1536 | Regression_eta: 0.0016 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1568 | Regression_cos_phi: 0.1354 | Regression_energy: 0.0991 | Regression_eta: 0.0652 | Regression_pt: 0.1083 | Regression_sin_phi: 0.1038 | LR: 7.44e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2963s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7593 | max_loss: 0.4918 | max_backward: 1.7987 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.6052 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6290.9 | uss_MB: 5640.2 | child_rss_MB: 5109.0 | allocated_MB: 447.1 | reserved_MB: 28738.0 | inactive_split_MB: 1154.4 +rank2 [2026-10-29 02:20:55,876] INFO : Step 18000/50000 rank2 | Train Loss: 1.2227 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2303 | Cardinality: 0.1450 | Classification: 0.1088 | Classification_binary: 0.2099 | HitGrouping: 0.1650 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1447 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2074 | Regression_cos_phi: 0.0835 | Regression_energy: 0.0777 | Regression_eta: 0.0534 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0944 | LR: 7.42e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7014s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9293 | max_loss: 0.1567 | max_backward: 2.2476 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6283.0 | uss_MB: 5642.9 | child_rss_MB: 5063.5 | allocated_MB: 449.4 | reserved_MB: 28738.0 | inactive_split_MB: 1324.2 +rank2 [2026-10-29 02:20:56,966] INFO : Running validation on rank2 +rank2 [2026-10-29 02:20:56,966] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 02:20:56,967] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 02:21:57,508] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:21:58,045] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 02:22:25,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 02:22:26,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 02:22:27,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 02:22:28,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 02:22:38,314] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 02:22:38,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:22:39,417] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 02:22:55,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 02:22:56,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 02:22:57,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 02:22:58,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 02:23:08,512] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 02:23:08,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 02:23:09,543] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 02:23:25,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 02:23:26,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 02:23:27,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 02:23:27,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 02:23:37,973] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 02:28:43,002] INFO : Step 18100/50000 rank2 | Train Loss: 1.2922 | Train Loss Unweighted: 0.6330 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1763 | Classification: 0.1061 | Classification_binary: 0.2257 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1467 | Regression_eta: 0.0022 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.2162 | Regression_cos_phi: 0.1096 | Regression_energy: 0.0828 | Regression_eta: 0.0596 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0716 | LR: 7.39e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0670s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.9150 | max_loss: 0.1573 | max_backward: 2.1915 | max_optimizer_step: 0.2256 | max_tensorboard_logging: 0.8426 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7551.6 | uss_MB: 6907.4 | child_rss_MB: 5063.5 | allocated_MB: 448.8 | reserved_MB: 25396.0 | inactive_split_MB: 1192.4 +rank2 [2026-10-29 02:33:21,588] INFO : Step 18200/50000 rank2 | Train Loss: 1.1403 | Train Loss Unweighted: 0.5693 | Train Loss Components: Auxiliary: 0.2140 | Cardinality: 0.1362 | Classification: 0.1099 | Classification_binary: 0.1658 | HitGrouping: 0.1698 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1444 | Regression_eta: 0.0021 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1449 | Regression_cos_phi: 0.1174 | Regression_energy: 0.0669 | Regression_eta: 0.0459 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0679 | LR: 7.36e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.5276s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0059 | max_forward: 0.7843 | max_loss: 0.1427 | max_backward: 1.4948 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 1.0509 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6418.0 | uss_MB: 5773.9 | child_rss_MB: 5038.4 | allocated_MB: 446.0 | reserved_MB: 27340.0 | inactive_split_MB: 922.1 +rank2 [2026-10-29 02:37:53,622] INFO : Step 18300/50000 rank2 | Train Loss: 1.1938 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2339 | Cardinality: 0.1526 | Classification: 0.1133 | Classification_binary: 0.1878 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1493 | Regression_eta: 0.0019 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0857 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0719 | Regression_eta: 0.0518 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0853 | LR: 7.34e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9192s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.9058 | max_loss: 0.1527 | max_backward: 2.1689 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6422.1 | uss_MB: 5784.1 | child_rss_MB: 5032.3 | allocated_MB: 448.4 | reserved_MB: 27340.0 | inactive_split_MB: 1185.8 +rank2 [2026-10-29 02:42:24,375] INFO : Step 18400/50000 rank2 | Train Loss: 1.1959 | Train Loss Unweighted: 0.6109 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1130 | Classification: 0.1042 | Classification_binary: 0.2213 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1400 | Regression_eta: 0.0022 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.2359 | Regression_cos_phi: 0.1063 | Regression_energy: 0.0901 | Regression_eta: 0.0576 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0924 | LR: 7.31e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7070s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8434 | max_loss: 0.1474 | max_backward: 1.9441 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6423.0 | uss_MB: 5783.9 | child_rss_MB: 5008.1 | allocated_MB: 448.8 | reserved_MB: 29306.0 | inactive_split_MB: 1091.4 +rank2 [2026-10-29 02:47:17,191] INFO : Step 18500/50000 rank2 | Train Loss: 1.2719 | Train Loss Unweighted: 0.6036 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1990 | Classification: 0.1087 | Classification_binary: 0.1793 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1538 | Regression_eta: 0.0024 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0765 | Regression_eta: 0.0588 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0754 | LR: 7.29e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.4219s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 1.6245 | max_loss: 0.3527 | max_backward: 3.4112 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.7932 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 6423.2 | uss_MB: 5781.7 | child_rss_MB: 5035.9 | allocated_MB: 447.9 | reserved_MB: 29306.0 | inactive_split_MB: 1193.4 +rank2 [2026-10-29 02:52:01,281] INFO : Step 18600/50000 rank2 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5476 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1351 | Classification: 0.0870 | Classification_binary: 0.1697 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1433 | Regression_eta: 0.0017 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2070 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0745 | Regression_eta: 0.0407 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0782 | LR: 7.26e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2403s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0527 | max_forward: 0.7639 | max_loss: 0.1373 | max_backward: 1.5273 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.7424 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6420.8 | uss_MB: 5779.1 | child_rss_MB: 5038.0 | allocated_MB: 445.6 | reserved_MB: 29306.0 | inactive_split_MB: 1194.4 +rank2 [2026-10-29 02:56:35,338] INFO : Step 18700/50000 rank2 | Train Loss: 1.2376 | Train Loss Unweighted: 0.5906 | Train Loss Components: Auxiliary: 0.2405 | Cardinality: 0.1841 | Classification: 0.1039 | Classification_binary: 0.2012 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1400 | Regression_eta: 0.0020 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2013 | Regression_cos_phi: 0.0984 | Regression_energy: 0.0869 | Regression_eta: 0.0845 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0898 | LR: 7.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7553s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8903 | max_loss: 0.1469 | max_backward: 2.1433 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4822 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1128 | Memory: rss_MB: 6419.7 | uss_MB: 5784.3 | child_rss_MB: 5013.1 | allocated_MB: 449.2 | reserved_MB: 31324.0 | inactive_split_MB: 1093.4 +rank2 [2026-10-29 03:01:09,007] INFO : Step 18800/50000 rank2 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1585 | Classification: 0.0943 | Classification_binary: 0.2038 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1342 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2263 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0645 | Regression_eta: 0.0413 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0678 | LR: 7.21e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.4236s | Phase Times: max_pre_step_barrier: 0.1142 | max_batch_to_device: 0.0068 | max_forward: 0.7808 | max_loss: 0.2502 | max_backward: 1.8229 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0996 | Memory: rss_MB: 6421.2 | uss_MB: 5781.7 | child_rss_MB: 5036.2 | allocated_MB: 450.4 | reserved_MB: 31336.0 | inactive_split_MB: 1098.4 +rank2 [2026-10-29 03:05:45,769] INFO : Step 18900/50000 rank2 | Train Loss: 1.0747 | Train Loss Unweighted: 0.5385 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.1306 | Classification: 0.0887 | Classification_binary: 0.1613 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1414 | Regression_eta: 0.0017 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0654 | Regression_eta: 0.0328 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0581 | LR: 7.18e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3944s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7400 | max_loss: 0.1656 | max_backward: 1.8003 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.7596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6421.5 | uss_MB: 5781.2 | child_rss_MB: 5039.7 | allocated_MB: 443.5 | reserved_MB: 31336.0 | inactive_split_MB: 928.5 +rank2 [2026-10-29 03:10:20,957] INFO : Step 19000/50000 rank2 | Train Loss: 1.1726 | Train Loss Unweighted: 0.5966 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1418 | Classification: 0.1102 | Classification_binary: 0.1870 | HitGrouping: 0.1705 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1463 | Regression_eta: 0.0019 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0869 | Regression_energy: 0.0807 | Regression_eta: 0.0729 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0998 | LR: 7.16e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8908s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7637 | max_loss: 0.5203 | max_backward: 1.8720 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 1.1055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6422.5 | uss_MB: 5783.8 | child_rss_MB: 5041.8 | allocated_MB: 446.6 | reserved_MB: 31336.0 | inactive_split_MB: 1097.4 +rank2 [2026-10-29 03:10:21,982] INFO : Running validation on rank2 +rank2 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 03:11:18,710] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:11:19,248] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 03:11:45,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 03:11:46,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 03:11:48,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 03:11:50,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 03:12:00,038] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 03:12:01,384] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:12:02,266] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 03:12:18,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 03:12:19,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 03:12:20,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 03:12:21,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 03:12:31,277] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 03:12:31,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 03:12:31,972] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 03:12:47,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 03:12:48,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 03:12:49,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 03:12:50,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 03:13:00,270] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 03:17:58,240] INFO : Step 19100/50000 rank2 | Train Loss: 1.2150 | Train Loss Unweighted: 0.6107 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1439 | Classification: 0.1037 | Classification_binary: 0.2230 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1395 | Regression_eta: 0.0020 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.2052 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0639 | Regression_eta: 0.0352 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0761 | LR: 7.13e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7836s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8630 | max_loss: 0.1532 | max_backward: 1.9622 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.7474 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7697.6 | uss_MB: 7053.9 | child_rss_MB: 5049.6 | allocated_MB: 449.4 | reserved_MB: 23972.0 | inactive_split_MB: 1371.8 +rank2 [2026-10-29 03:22:35,411] INFO : Step 19200/50000 rank2 | Train Loss: 1.1075 | Train Loss Unweighted: 0.5303 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1382 | Classification: 0.0958 | Classification_binary: 0.1758 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1270 | Regression_eta: 0.0019 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0592 | Regression_eta: 0.0580 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0651 | LR: 7.10e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.0037s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8287 | max_loss: 0.1556 | max_backward: 1.8127 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6504.0 | uss_MB: 5863.3 | child_rss_MB: 5048.7 | allocated_MB: 445.3 | reserved_MB: 27640.0 | inactive_split_MB: 940.7 +rank2 [2026-10-29 03:27:09,997] INFO : Step 19300/50000 rank2 | Train Loss: 1.0792 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2111 | Cardinality: 0.1254 | Classification: 0.0911 | Classification_binary: 0.1561 | HitGrouping: 0.1662 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1993 | Regression_cos_phi: 0.1316 | Regression_energy: 0.0808 | Regression_eta: 0.0603 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0826 | LR: 7.08e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.5138s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8891 | max_loss: 0.1513 | max_backward: 1.7637 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6508.5 | uss_MB: 5869.2 | child_rss_MB: 5044.9 | allocated_MB: 445.1 | reserved_MB: 27640.0 | inactive_split_MB: 1366.9 +rank2 [2026-10-29 03:31:44,570] INFO : Step 19400/50000 rank2 | Train Loss: 1.2411 | Train Loss Unweighted: 0.6183 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1555 | Classification: 0.1055 | Classification_binary: 0.2145 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1463 | Regression_eta: 0.0020 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2283 | Regression_cos_phi: 0.1216 | Regression_energy: 0.0924 | Regression_eta: 0.0613 | Regression_pt: 0.1007 | Regression_sin_phi: 0.1235 | LR: 7.05e-04 | DataLoad Time: 0.0157s | Model Forward Time: 6.6751s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0092 | max_forward: 0.8079 | max_loss: 0.1550 | max_backward: 1.7710 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 2.8282 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6511.0 | uss_MB: 5872.2 | child_rss_MB: 5045.1 | allocated_MB: 452.5 | reserved_MB: 28010.0 | inactive_split_MB: 1195.9 +rank2 [2026-10-29 03:36:21,473] INFO : Step 19500/50000 rank2 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6008 | Train Loss Components: Auxiliary: 0.2448 | Cardinality: 0.1836 | Classification: 0.1087 | Classification_binary: 0.2123 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1374 | Regression_eta: 0.0023 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2034 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0704 | Regression_eta: 0.0561 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0824 | LR: 7.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6205s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7880 | max_loss: 0.1506 | max_backward: 1.7760 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.7620 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6510.7 | uss_MB: 5870.2 | child_rss_MB: 5041.2 | allocated_MB: 449.4 | reserved_MB: 29680.0 | inactive_split_MB: 1368.8 +rank2 [2026-10-29 03:40:54,623] INFO : Step 19600/50000 rank2 | Train Loss: 1.1451 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1464 | Classification: 0.0933 | Classification_binary: 0.1736 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1533 | Regression_eta: 0.0018 | Regression_pt: 0.1557 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0849 | Regression_eta: 0.0377 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0491 | LR: 7.00e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.3535s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8082 | max_loss: 0.1437 | max_backward: 1.6649 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.6567 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6510.9 | uss_MB: 5869.4 | child_rss_MB: 5045.0 | allocated_MB: 444.5 | reserved_MB: 29680.0 | inactive_split_MB: 1016.8 +rank2 [2026-10-29 03:45:30,310] INFO : Step 19700/50000 rank2 | Train Loss: 1.1424 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2236 | Cardinality: 0.1385 | Classification: 0.1093 | Classification_binary: 0.1797 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1379 | Regression_eta: 0.0020 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0666 | Regression_eta: 0.0657 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0958 | LR: 6.97e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4865s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7612 | max_loss: 0.2297 | max_backward: 1.9668 | max_optimizer_step: 0.5229 | max_tensorboard_logging: 0.5756 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6508.5 | uss_MB: 5869.4 | child_rss_MB: 5047.4 | allocated_MB: 449.1 | reserved_MB: 29680.0 | inactive_split_MB: 1100.9 +rank2 [2026-10-29 03:50:08,780] INFO : Step 19800/50000 rank2 | Train Loss: 1.2806 | Train Loss Unweighted: 0.6390 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1432 | Classification: 0.1110 | Classification_binary: 0.2282 | HitGrouping: 0.1668 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0030 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.2376 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0732 | Regression_eta: 0.0871 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0755 | LR: 6.94e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5062s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8034 | max_loss: 0.1812 | max_backward: 1.9502 | max_optimizer_step: 0.2119 | max_tensorboard_logging: 0.5857 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6510.7 | uss_MB: 5873.6 | child_rss_MB: 5045.7 | allocated_MB: 452.3 | reserved_MB: 29680.0 | inactive_split_MB: 1110.9 +rank2 [2026-10-29 03:54:46,663] INFO : Step 19900/50000 rank2 | Train Loss: 1.0692 | Train Loss Unweighted: 0.5407 | Train Loss Components: Auxiliary: 0.2077 | Cardinality: 0.1171 | Classification: 0.0922 | Classification_binary: 0.1812 | HitGrouping: 0.1610 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1981 | Regression_cos_phi: 0.0864 | Regression_energy: 0.0723 | Regression_eta: 0.0371 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0634 | LR: 6.92e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6140s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7533 | max_loss: 0.2412 | max_backward: 2.0720 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5848 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6505.8 | uss_MB: 5870.0 | child_rss_MB: 5057.5 | allocated_MB: 444.7 | reserved_MB: 29682.0 | inactive_split_MB: 1000.0 +rank2 [2026-10-29 03:59:25,168] INFO : Step 20000/50000 rank2 | Train Loss: 1.0304 | Train Loss Unweighted: 0.5275 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1060 | Classification: 0.0940 | Classification_binary: 0.1604 | HitGrouping: 0.1656 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1344 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1750 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0813 | Regression_eta: 0.0443 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0476 | LR: 6.89e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.8365s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.8721 | max_loss: 0.1507 | max_backward: 2.1861 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6505.7 | uss_MB: 5870.7 | child_rss_MB: 5049.5 | allocated_MB: 445.9 | reserved_MB: 30966.0 | inactive_split_MB: 1179.2 +rank2 [2026-10-29 03:59:26,239] INFO : Running validation on rank2 +rank2 [2026-10-29 03:59:26,239] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 03:59:26,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 04:00:19,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:00:19,532] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 04:00:44,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 04:00:45,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 04:00:47,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 04:00:48,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 04:00:58,269] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 04:00:59,045] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:00:59,501] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 04:01:15,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 04:01:16,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 04:01:17,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 04:01:17,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 04:01:27,970] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 04:01:28,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:01:29,372] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 04:01:45,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 04:01:46,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 04:01:47,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 04:01:48,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 04:01:58,182] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 04:06:53,362] INFO : Step 20100/50000 rank2 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2437 | Cardinality: 0.1586 | Classification: 0.1055 | Classification_binary: 0.2238 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1360 | Regression_eta: 0.0021 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.2074 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0787 | Regression_eta: 0.0480 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0831 | LR: 6.86e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6018s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0372 | max_forward: 0.8293 | max_loss: 0.1584 | max_backward: 1.7738 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.7139 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7674.6 | uss_MB: 7034.7 | child_rss_MB: 5034.6 | allocated_MB: 448.4 | reserved_MB: 29328.0 | inactive_split_MB: 969.2 +rank2 [2026-10-29 04:11:25,835] INFO : Step 20200/50000 rank2 | Train Loss: 1.3163 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.1695 | Classification: 0.1000 | Classification_binary: 0.2250 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1433 | Regression_eta: 0.0029 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.2607 | Regression_cos_phi: 0.1496 | Regression_energy: 0.0834 | Regression_eta: 0.1390 | Regression_pt: 0.0925 | Regression_sin_phi: 0.1095 | LR: 6.84e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.6429s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8823 | max_loss: 0.1595 | max_backward: 2.1030 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5136 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7674.6 | uss_MB: 7036.8 | child_rss_MB: 5027.8 | allocated_MB: 449.6 | reserved_MB: 29340.0 | inactive_split_MB: 938.7 +rank2 [2026-10-29 04:16:03,709] INFO : Step 20300/50000 rank2 | Train Loss: 1.1420 | Train Loss Unweighted: 0.5734 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1487 | Classification: 0.1087 | Classification_binary: 0.1713 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1438 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0881 | Regression_energy: 0.0789 | Regression_eta: 0.0509 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0850 | LR: 6.81e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.9973s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.9650 | max_loss: 0.1904 | max_backward: 2.0801 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.8679 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6555.4 | uss_MB: 5913.2 | child_rss_MB: 5021.5 | allocated_MB: 444.8 | reserved_MB: 29344.0 | inactive_split_MB: 986.2 +rank2 [2026-10-29 04:20:44,976] INFO : Step 20400/50000 rank2 | Train Loss: 1.1926 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1642 | Classification: 0.1091 | Classification_binary: 0.1922 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1329 | Regression_eta: 0.0018 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.2187 | Regression_cos_phi: 0.0982 | Regression_energy: 0.0689 | Regression_eta: 0.0674 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0920 | LR: 6.78e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.2517s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0167 | max_forward: 1.2639 | max_loss: 0.1873 | max_backward: 2.4261 | max_optimizer_step: 0.3105 | max_tensorboard_logging: 0.7562 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6556.4 | uss_MB: 5916.7 | child_rss_MB: 5031.4 | allocated_MB: 447.7 | reserved_MB: 29344.0 | inactive_split_MB: 985.2 +rank2 [2026-10-29 04:25:27,116] INFO : Step 20500/50000 rank2 | Train Loss: 1.2110 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1591 | Classification: 0.1022 | Classification_binary: 0.2117 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1333 | Regression_eta: 0.0017 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.2213 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0628 | Regression_eta: 0.0442 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0748 | LR: 6.76e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.5495s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7705 | max_loss: 0.1525 | max_backward: 1.8060 | max_optimizer_step: 0.2622 | max_tensorboard_logging: 0.7595 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6553.9 | uss_MB: 5915.2 | child_rss_MB: 5013.6 | allocated_MB: 450.6 | reserved_MB: 29344.0 | inactive_split_MB: 1123.4 +rank2 [2026-10-29 04:30:07,865] INFO : Step 20600/50000 rank2 | Train Loss: 1.1161 | Train Loss Unweighted: 0.5564 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1436 | Classification: 0.0978 | Classification_binary: 0.1795 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1685 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0846 | Regression_eta: 0.0409 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0647 | LR: 6.73e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4365s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8358 | max_loss: 0.5148 | max_backward: 1.8393 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6556.9 | uss_MB: 5916.8 | child_rss_MB: 5024.7 | allocated_MB: 445.4 | reserved_MB: 31380.0 | inactive_split_MB: 917.0 +rank2 [2026-10-29 04:34:51,728] INFO : Step 20700/50000 rank2 | Train Loss: 1.0740 | Train Loss Unweighted: 0.5459 | Train Loss Components: Auxiliary: 0.2086 | Cardinality: 0.1252 | Classification: 0.0962 | Classification_binary: 0.1560 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1452 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0735 | Regression_eta: 0.0667 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0967 | LR: 6.70e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0699s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7111 | max_loss: 0.2396 | max_backward: 1.6829 | max_optimizer_step: 0.2121 | max_tensorboard_logging: 0.4996 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6553.0 | uss_MB: 5912.9 | child_rss_MB: 5017.8 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1185.7 +rank2 [2026-10-29 04:39:29,327] INFO : Step 20800/50000 rank2 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5426 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1263 | Classification: 0.0965 | Classification_binary: 0.1999 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1207 | Regression_eta: 0.0017 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1839 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0739 | Regression_eta: 0.0439 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0665 | LR: 6.67e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7621s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.9079 | max_loss: 0.1550 | max_backward: 2.1083 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.5897 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6553.9 | uss_MB: 5918.9 | child_rss_MB: 5012.5 | allocated_MB: 448.2 | reserved_MB: 31380.0 | inactive_split_MB: 1157.8 +rank2 [2026-10-29 04:44:09,835] INFO : Step 20900/50000 rank2 | Train Loss: 1.2188 | Train Loss Unweighted: 0.5970 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1525 | Classification: 0.1062 | Classification_binary: 0.2022 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0022 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0822 | Classification_binary: 0.2115 | Regression_cos_phi: 0.1150 | Regression_energy: 0.0747 | Regression_eta: 0.0649 | Regression_pt: 0.0831 | Regression_sin_phi: 0.1209 | LR: 6.65e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7885s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0138 | max_forward: 0.8922 | max_loss: 0.1535 | max_backward: 2.0483 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6550.7 | uss_MB: 5916.4 | child_rss_MB: 5027.7 | allocated_MB: 447.8 | reserved_MB: 31380.0 | inactive_split_MB: 1126.3 +rank2 [2026-10-29 04:48:48,418] INFO : Step 21000/50000 rank2 | Train Loss: 1.0503 | Train Loss Unweighted: 0.5338 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1267 | Classification: 0.0900 | Classification_binary: 0.1573 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1405 | Regression_eta: 0.0016 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1555 | Regression_cos_phi: 0.0777 | Regression_energy: 0.0657 | Regression_eta: 0.0486 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0521 | LR: 6.62e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8545s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7596 | max_loss: 0.2536 | max_backward: 1.8713 | max_optimizer_step: 0.3177 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6551.7 | uss_MB: 5915.9 | child_rss_MB: 5032.5 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1148.0 +rank2 [2026-10-29 04:48:49,285] INFO : Running validation on rank2 +rank2 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 04:49:44,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:49:44,825] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 04:50:11,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 04:50:12,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 04:50:13,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 04:50:14,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 04:50:24,504] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 04:50:25,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:50:25,767] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 04:50:42,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 04:50:43,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 04:50:44,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 04:50:45,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 04:50:55,167] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 04:50:55,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 04:50:55,860] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 04:51:11,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 04:51:12,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 04:51:13,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 04:51:14,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 04:51:24,168] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 04:56:25,806] INFO : Step 21100/50000 rank2 | Train Loss: 1.1646 | Train Loss Unweighted: 0.5785 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1401 | Classification: 0.1095 | Classification_binary: 0.1927 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1344 | Regression_eta: 0.0022 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.1973 | Regression_cos_phi: 0.1314 | Regression_energy: 0.0672 | Regression_eta: 0.0736 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0768 | LR: 6.59e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.4044s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.9258 | max_loss: 0.1506 | max_backward: 2.4347 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.8145 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7847.0 | uss_MB: 5890.3 | child_rss_MB: 5018.3 | allocated_MB: 448.7 | reserved_MB: 23438.0 | inactive_split_MB: 911.7 +rank2 [2026-10-29 04:56:59,983] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank2 [2026-10-29 04:56:59,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-29 05:01:22,613] INFO : Step 21200/50000 rank2 | Train Loss: 1.1437 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1413 | Classification: 0.0985 | Classification_binary: 0.1768 | HitGrouping: 0.1669 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0020 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0609 | Regression_eta: 0.0402 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0639 | LR: 6.56e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5648s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7993 | max_loss: 0.1450 | max_backward: 1.7427 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7842.9 | uss_MB: 7208.8 | child_rss_MB: 5012.9 | allocated_MB: 446.9 | reserved_MB: 23590.0 | inactive_split_MB: 1086.9 +rank2 [2026-10-29 05:06:02,215] INFO : Step 21300/50000 rank2 | Train Loss: 1.1586 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1648 | Classification: 0.1118 | Classification_binary: 0.1760 | HitGrouping: 0.1681 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1392 | Regression_eta: 0.0021 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0808 | Regression_eta: 0.1367 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0591 | LR: 6.54e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.2687s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0558 | max_forward: 0.9498 | max_loss: 0.1699 | max_backward: 2.4288 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.7788 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1270 | Memory: rss_MB: 6605.4 | uss_MB: 5978.0 | child_rss_MB: 5012.8 | allocated_MB: 447.4 | reserved_MB: 25440.0 | inactive_split_MB: 1003.9 +rank2 [2026-10-29 05:10:41,727] INFO : Step 21400/50000 rank2 | Train Loss: 1.1843 | Train Loss Unweighted: 0.5823 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1516 | Classification: 0.1038 | Classification_binary: 0.2065 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1335 | Regression_eta: 0.0020 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.2097 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0721 | Regression_eta: 0.0678 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0683 | LR: 6.51e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.0973s | Phase Times: max_pre_step_barrier: 0.0085 | max_batch_to_device: 0.0640 | max_forward: 0.9920 | max_loss: 0.5036 | max_backward: 2.0711 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.8200 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6606.3 | uss_MB: 5978.8 | child_rss_MB: 5018.5 | allocated_MB: 448.9 | reserved_MB: 25440.0 | inactive_split_MB: 994.4 +rank2 [2026-10-29 05:15:15,087] INFO : Step 21500/50000 rank2 | Train Loss: 1.1484 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1478 | Classification: 0.0953 | Classification_binary: 0.2062 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1299 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2107 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0642 | Regression_eta: 0.0551 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0610 | LR: 6.48e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.9372s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8371 | max_loss: 0.1537 | max_backward: 2.1150 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.8633 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6603.8 | uss_MB: 5979.6 | child_rss_MB: 5017.0 | allocated_MB: 447.2 | reserved_MB: 27056.0 | inactive_split_MB: 1468.9 +rank2 [2026-10-29 05:19:51,738] INFO : Step 21600/50000 rank2 | Train Loss: 1.0171 | Train Loss Unweighted: 0.5098 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1167 | Classification: 0.0973 | Classification_binary: 0.1559 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1265 | Regression_eta: 0.0014 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0668 | Regression_eta: 0.0470 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0601 | LR: 6.45e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5918s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0062 | max_forward: 0.7285 | max_loss: 0.1940 | max_backward: 1.8148 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.8819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6602.2 | uss_MB: 5978.2 | child_rss_MB: 5014.0 | allocated_MB: 445.7 | reserved_MB: 27056.0 | inactive_split_MB: 1196.5 +rank2 [2026-10-29 05:24:30,476] INFO : Step 21700/50000 rank2 | Train Loss: 1.1623 | Train Loss Unweighted: 0.5535 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1498 | Classification: 0.1025 | Classification_binary: 0.1819 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1893 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0697 | Regression_eta: 0.0707 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0798 | LR: 6.43e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4947s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7947 | max_loss: 0.1379 | max_backward: 1.9808 | max_optimizer_step: 0.2157 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6600.2 | uss_MB: 5978.2 | child_rss_MB: 5010.8 | allocated_MB: 448.2 | reserved_MB: 27056.0 | inactive_split_MB: 1045.2 +rank2 [2026-10-29 05:29:08,471] INFO : Step 21800/50000 rank2 | Train Loss: 1.0898 | Train Loss Unweighted: 0.5518 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1233 | Classification: 0.0977 | Classification_binary: 0.1967 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1269 | Regression_eta: 0.0015 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1877 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0630 | Regression_eta: 0.0379 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0900 | LR: 6.40e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.7218s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8952 | max_loss: 0.1765 | max_backward: 2.1628 | max_optimizer_step: 0.2138 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6602.0 | uss_MB: 5872.4 | child_rss_MB: 5012.0 | allocated_MB: 448.4 | reserved_MB: 27056.0 | inactive_split_MB: 1001.4 +rank2 [2026-10-29 05:33:46,455] INFO : Step 21900/50000 rank2 | Train Loss: 1.0699 | Train Loss Unweighted: 0.5306 | Train Loss Components: Auxiliary: 0.2121 | Cardinality: 0.1436 | Classification: 0.0991 | Classification_binary: 0.1704 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1276 | Regression_eta: 0.0015 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1409 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0755 | Regression_eta: 0.0379 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0472 | LR: 6.37e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4514s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7813 | max_loss: 0.1495 | max_backward: 1.8703 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 6600.0 | uss_MB: 5976.9 | child_rss_MB: 5013.0 | allocated_MB: 447.4 | reserved_MB: 27056.0 | inactive_split_MB: 1002.4 +rank2 [2026-10-29 05:38:23,934] INFO : Step 22000/50000 rank2 | Train Loss: 1.0659 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2100 | Cardinality: 0.1294 | Classification: 0.0955 | Classification_binary: 0.1521 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1360 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0717 | Regression_eta: 0.0410 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0658 | LR: 6.34e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4503s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7855 | max_loss: 0.1383 | max_backward: 1.9648 | max_optimizer_step: 0.2132 | max_tensorboard_logging: 0.5884 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6605.4 | uss_MB: 5979.9 | child_rss_MB: 5015.2 | allocated_MB: 449.1 | reserved_MB: 27056.0 | inactive_split_MB: 1180.9 +rank2 [2026-10-29 05:38:24,834] INFO : Running validation on rank2 +rank2 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 05:39:16,672] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:39:17,215] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 05:39:42,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 05:39:43,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 05:39:44,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 05:39:45,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 05:39:55,890] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 05:39:56,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:39:57,075] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 05:40:13,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 05:40:14,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 05:40:15,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 05:40:16,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 05:40:26,810] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 05:40:27,479] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 05:40:27,967] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 05:40:43,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 05:40:44,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 05:40:45,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 05:40:46,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 05:40:56,322] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 05:45:56,688] INFO : Step 22100/50000 rank2 | Train Loss: 1.1235 | Train Loss Unweighted: 0.5530 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1528 | Classification: 0.0997 | Classification_binary: 0.1905 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1304 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.1900 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0725 | Regression_eta: 0.0506 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0868 | LR: 6.32e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5235s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8135 | max_loss: 0.1488 | max_backward: 1.7877 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5979 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7900.1 | uss_MB: 7271.0 | child_rss_MB: 5019.5 | allocated_MB: 451.8 | reserved_MB: 29136.0 | inactive_split_MB: 1136.2 +rank2 [2026-10-29 05:50:45,342] INFO : Step 22200/50000 rank2 | Train Loss: 1.1922 | Train Loss Unweighted: 0.5684 | Train Loss Components: Auxiliary: 0.2361 | Cardinality: 0.1699 | Classification: 0.0906 | Classification_binary: 0.2085 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1994 | Regression_cos_phi: 0.1078 | Regression_energy: 0.0800 | Regression_eta: 0.0603 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0984 | LR: 6.29e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5700s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8616 | max_loss: 0.1545 | max_backward: 1.6969 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 0.7120 | max_tensorboard_flush: 0.0150 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7899.1 | uss_MB: 7269.8 | child_rss_MB: 5025.9 | allocated_MB: 448.1 | reserved_MB: 29136.0 | inactive_split_MB: 1371.0 +rank2 [2026-10-29 05:55:21,940] INFO : Step 22300/50000 rank2 | Train Loss: 1.0221 | Train Loss Unweighted: 0.5006 | Train Loss Components: Auxiliary: 0.1973 | Cardinality: 0.1169 | Classification: 0.0876 | Classification_binary: 0.1576 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1252 | Regression_eta: 0.0021 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1644 | Regression_cos_phi: 0.0926 | Regression_energy: 0.0747 | Regression_eta: 0.0524 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0554 | LR: 6.26e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5034s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7903 | max_loss: 0.1446 | max_backward: 1.7078 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 0.8083 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6690.1 | uss_MB: 6062.0 | child_rss_MB: 5023.7 | allocated_MB: 446.4 | reserved_MB: 29136.0 | inactive_split_MB: 1159.3 +rank2 [2026-10-29 06:00:08,368] INFO : Step 22400/50000 rank2 | Train Loss: 1.1170 | Train Loss Unweighted: 0.5609 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1391 | Classification: 0.1065 | Classification_binary: 0.1807 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1348 | Regression_eta: 0.0019 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.1799 | Regression_cos_phi: 0.0329 | Regression_energy: 0.0683 | Regression_eta: 0.0458 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0991 | LR: 6.23e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9535s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7658 | max_loss: 0.2479 | max_backward: 1.9808 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9928 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6691.0 | uss_MB: 6065.0 | child_rss_MB: 5027.0 | allocated_MB: 447.1 | reserved_MB: 31152.0 | inactive_split_MB: 925.7 +rank2 [2026-10-29 06:04:41,560] INFO : Step 22500/50000 rank2 | Train Loss: 1.1245 | Train Loss Unweighted: 0.5482 | Train Loss Components: Auxiliary: 0.2231 | Cardinality: 0.1463 | Classification: 0.0856 | Classification_binary: 0.2083 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1244 | Regression_eta: 0.0016 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.2282 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0745 | Regression_eta: 0.0455 | Regression_pt: 0.0815 | Regression_sin_phi: 0.1002 | LR: 6.20e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.0098s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 1.1916 | max_loss: 0.1613 | max_backward: 2.1225 | max_optimizer_step: 0.2176 | max_tensorboard_logging: 0.7664 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6685.2 | uss_MB: 6063.6 | child_rss_MB: 5003.5 | allocated_MB: 447.4 | reserved_MB: 31152.0 | inactive_split_MB: 959.7 +rank2 [2026-10-29 06:09:13,551] INFO : Step 22600/50000 rank2 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5177 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1260 | Classification: 0.0928 | Classification_binary: 0.1789 | HitGrouping: 0.1619 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1208 | Regression_eta: 0.0019 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1649 | Regression_cos_phi: 0.0954 | Regression_energy: 0.0761 | Regression_eta: 0.0537 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0779 | LR: 6.18e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.8019s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0651 | max_forward: 1.1064 | max_loss: 0.1531 | max_backward: 1.9236 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.8515 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6676.3 | uss_MB: 6063.0 | child_rss_MB: 5024.2 | allocated_MB: 448.3 | reserved_MB: 31154.0 | inactive_split_MB: 961.5 +rank2 [2026-10-29 06:13:51,719] INFO : Step 22700/50000 rank2 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5090 | Train Loss Components: Auxiliary: 0.1959 | Cardinality: 0.1101 | Classification: 0.0877 | Classification_binary: 0.1479 | HitGrouping: 0.1634 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1339 | Regression_eta: 0.0015 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1649 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0754 | Regression_eta: 0.0364 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0648 | LR: 6.15e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.4585s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 1.0743 | max_loss: 0.1383 | max_backward: 1.7235 | max_optimizer_step: 0.2204 | max_tensorboard_logging: 0.5170 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 6672.9 | uss_MB: 6061.8 | child_rss_MB: 5030.8 | allocated_MB: 446.5 | reserved_MB: 31154.0 | inactive_split_MB: 972.7 +rank2 [2026-10-29 06:18:32,626] INFO : Step 22800/50000 rank2 | Train Loss: 1.0730 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2215 | Cardinality: 0.1332 | Classification: 0.1095 | Classification_binary: 0.1782 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1289 | Regression_eta: 0.0014 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0786 | Regression_eta: 0.0313 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0785 | LR: 6.12e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.7958s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.2500 | max_backward: 2.0107 | max_optimizer_step: 0.2761 | max_tensorboard_logging: 0.6168 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 6673.9 | uss_MB: 6062.6 | child_rss_MB: 5029.6 | allocated_MB: 447.2 | reserved_MB: 31154.0 | inactive_split_MB: 922.1 +rank2 [2026-10-29 06:23:13,203] INFO : Step 22900/50000 rank2 | Train Loss: 1.1830 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2389 | Cardinality: 0.1788 | Classification: 0.0948 | Classification_binary: 0.2023 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1393 | Regression_eta: 0.0018 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.2139 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0695 | Regression_eta: 0.0547 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0620 | LR: 6.09e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.9931s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0073 | max_forward: 0.7992 | max_loss: 0.1478 | max_backward: 1.9051 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.9858 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 6664.2 | uss_MB: 6055.9 | child_rss_MB: 5028.8 | allocated_MB: 448.8 | reserved_MB: 31154.0 | inactive_split_MB: 1109.2 +rank2 [2026-10-29 06:27:51,864] INFO : Step 23000/50000 rank2 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5014 | Train Loss Components: Auxiliary: 0.1951 | Cardinality: 0.1184 | Classification: 0.0781 | Classification_binary: 0.1455 | HitGrouping: 0.1653 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1221 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0792 | Regression_eta: 0.0560 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0532 | LR: 6.06e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6096s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7528 | max_loss: 0.2292 | max_backward: 1.7818 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.8884 | max_tensorboard_flush: 0.0060 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6672.0 | uss_MB: 6063.2 | child_rss_MB: 5028.5 | allocated_MB: 443.9 | reserved_MB: 31154.0 | inactive_split_MB: 985.1 +rank2 [2026-10-29 06:27:52,750] INFO : Running validation on rank2 +rank2 [2026-10-29 06:27:52,750] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 06:27:52,751] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 06:28:45,237] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:28:45,725] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 06:29:12,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 06:29:13,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 06:29:14,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 06:29:15,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 06:29:25,254] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 06:29:26,162] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:29:26,632] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 06:29:42,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 06:29:43,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 06:29:44,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 06:29:45,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 06:29:55,275] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 06:29:56,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 06:29:56,645] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 06:30:12,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 06:30:13,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 06:30:14,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 06:30:15,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 06:30:25,051] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 06:35:29,456] INFO : Step 23100/50000 rank2 | Train Loss: 1.1240 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1320 | Classification: 0.1122 | Classification_binary: 0.1745 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1274 | Regression_eta: 0.0021 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1838 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0612 | Regression_eta: 0.0429 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0513 | LR: 6.04e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8791s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7914 | max_loss: 0.1430 | max_backward: 1.7255 | max_optimizer_step: 0.3295 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0695 | Memory: rss_MB: 7843.1 | uss_MB: 7233.5 | child_rss_MB: 5030.9 | allocated_MB: 449.4 | reserved_MB: 25680.0 | inactive_split_MB: 1046.0 +rank2 [2026-10-29 06:40:11,956] INFO : Step 23200/50000 rank2 | Train Loss: 1.1736 | Train Loss Unweighted: 0.5745 | Train Loss Components: Auxiliary: 0.2341 | Cardinality: 0.1586 | Classification: 0.1057 | Classification_binary: 0.2033 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1304 | Regression_eta: 0.0018 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.2330 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0741 | Regression_eta: 0.0433 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0762 | LR: 6.01e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.1140s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9381 | max_loss: 0.1692 | max_backward: 2.2545 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6161 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 7846.1 | uss_MB: 5861.1 | child_rss_MB: 5025.2 | allocated_MB: 447.8 | reserved_MB: 25680.0 | inactive_split_MB: 1169.7 +rank2 [2026-10-29 06:44:52,828] INFO : Step 23300/50000 rank2 | Train Loss: 1.0647 | Train Loss Unweighted: 0.5262 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1268 | Classification: 0.0958 | Classification_binary: 0.1665 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1301 | Regression_eta: 0.0017 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1732 | Regression_cos_phi: 0.1224 | Regression_energy: 0.0799 | Regression_eta: 0.0767 | Regression_pt: 0.0843 | Regression_sin_phi: 0.1149 | LR: 5.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6094s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8613 | max_loss: 0.1501 | max_backward: 2.0035 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 0.6274 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6729.0 | uss_MB: 6118.5 | child_rss_MB: 5031.7 | allocated_MB: 447.3 | reserved_MB: 29702.0 | inactive_split_MB: 1252.4 +rank2 [2026-10-29 06:49:32,124] INFO : Step 23400/50000 rank2 | Train Loss: 0.9916 | Train Loss Unweighted: 0.5023 | Train Loss Components: Auxiliary: 0.1957 | Cardinality: 0.1164 | Classification: 0.1006 | Classification_binary: 0.1374 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1413 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0619 | Regression_eta: 0.0371 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0527 | LR: 5.95e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3944s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7229 | max_loss: 0.2412 | max_backward: 1.6798 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.4992 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1172 | Memory: rss_MB: 6727.8 | uss_MB: 6117.0 | child_rss_MB: 5031.9 | allocated_MB: 445.7 | reserved_MB: 29702.0 | inactive_split_MB: 1158.3 +rank2 [2026-10-29 06:54:42,356] INFO : Step 23500/50000 rank2 | Train Loss: 1.0349 | Train Loss Unweighted: 0.5234 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1083 | Classification: 0.0851 | Classification_binary: 0.1870 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1235 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1941 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0695 | Regression_eta: 0.0334 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0708 | LR: 5.92e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7761s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7924 | max_loss: 0.2020 | max_backward: 1.9787 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.7987 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6728.4 | uss_MB: 6111.2 | child_rss_MB: 5026.6 | allocated_MB: 446.9 | reserved_MB: 31774.0 | inactive_split_MB: 1042.0 +rank2 [2026-10-29 06:59:38,280] INFO : Step 23600/50000 rank2 | Train Loss: 1.1326 | Train Loss Unweighted: 0.5418 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1458 | Classification: 0.0933 | Classification_binary: 0.1900 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1267 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1780 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0779 | Regression_eta: 0.0529 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0867 | LR: 5.90e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9843s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0063 | max_forward: 0.7946 | max_loss: 0.1855 | max_backward: 1.9114 | max_optimizer_step: 0.2664 | max_tensorboard_logging: 0.9547 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6730.1 | uss_MB: 6120.3 | child_rss_MB: 5028.3 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1333.8 +rank2 [2026-10-29 07:04:20,063] INFO : Step 23700/50000 rank2 | Train Loss: 1.0167 | Train Loss Unweighted: 0.4968 | Train Loss Components: Auxiliary: 0.1932 | Cardinality: 0.1158 | Classification: 0.0853 | Classification_binary: 0.1475 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1285 | Regression_eta: 0.0017 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1485 | Regression_cos_phi: 0.1127 | Regression_energy: 0.0859 | Regression_eta: 0.0631 | Regression_pt: 0.0948 | Regression_sin_phi: 0.1478 | LR: 5.87e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.7147s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0221 | max_forward: 0.7952 | max_loss: 0.1461 | max_backward: 1.8308 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7718 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6725.6 | uss_MB: 6118.0 | child_rss_MB: 5028.2 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1160.4 +rank2 [2026-10-29 07:08:57,915] INFO : Step 23800/50000 rank2 | Train Loss: 1.0651 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1325 | Classification: 0.0886 | Classification_binary: 0.1616 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1318 | Regression_eta: 0.0015 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1432 | Regression_cos_phi: 0.1016 | Regression_energy: 0.0643 | Regression_eta: 0.0468 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0754 | LR: 5.84e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.6728s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8562 | max_loss: 0.1426 | max_backward: 1.8700 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6730.2 | uss_MB: 6120.2 | child_rss_MB: 5026.7 | allocated_MB: 449.4 | reserved_MB: 31776.0 | inactive_split_MB: 1046.7 +rank2 [2026-10-29 07:13:41,573] INFO : Step 23900/50000 rank2 | Train Loss: 1.1515 | Train Loss Unweighted: 0.5718 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1646 | Classification: 0.0962 | Classification_binary: 0.2086 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.2170 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0730 | Regression_eta: 0.0422 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0596 | LR: 5.81e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8375s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.8887 | max_loss: 0.2019 | max_backward: 2.1185 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6224 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6714.4 | uss_MB: 6108.3 | child_rss_MB: 5032.9 | allocated_MB: 447.1 | reserved_MB: 31776.0 | inactive_split_MB: 1168.8 +rank2 [2026-10-29 07:18:20,710] INFO : Step 24000/50000 rank2 | Train Loss: 1.0174 | Train Loss Unweighted: 0.5187 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1133 | Classification: 0.0873 | Classification_binary: 0.1674 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0014 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1597 | Regression_cos_phi: 0.0970 | Regression_energy: 0.0625 | Regression_eta: 0.0400 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0597 | LR: 5.78e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.6297s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 1.3237 | max_loss: 0.2403 | max_backward: 1.8319 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5068 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6721.2 | uss_MB: 6114.9 | child_rss_MB: 5029.4 | allocated_MB: 446.4 | reserved_MB: 31776.0 | inactive_split_MB: 957.1 +rank2 [2026-10-29 07:18:21,624] INFO : Running validation on rank2 +rank2 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 07:19:13,460] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:19:13,969] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 07:19:41,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 07:19:42,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 07:19:43,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 07:19:44,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 07:19:54,520] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 07:19:55,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:19:55,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 07:20:11,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 07:20:12,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 07:20:13,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 07:20:14,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 07:20:24,909] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 07:20:25,040] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 07:20:25,605] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 07:20:41,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 07:20:42,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 07:20:42,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 07:20:43,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 07:20:53,755] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 07:26:06,615] INFO : Step 24100/50000 rank2 | Train Loss: 1.0669 | Train Loss Unweighted: 0.5331 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1324 | Classification: 0.0928 | Classification_binary: 0.1578 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1390 | Regression_eta: 0.0017 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0665 | Regression_eta: 0.0505 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0878 | LR: 5.75e-04 | DataLoad Time: 0.0174s | Model Forward Time: 7.5194s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.7836 | max_loss: 0.1442 | max_backward: 1.8747 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 3.6030 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0626 | Memory: rss_MB: 7857.5 | uss_MB: 7243.1 | child_rss_MB: 5051.0 | allocated_MB: 448.1 | reserved_MB: 24490.0 | inactive_split_MB: 1198.5 +rank2 [2026-10-29 07:30:59,926] INFO : Step 24200/50000 rank2 | Train Loss: 1.1691 | Train Loss Unweighted: 0.5645 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1546 | Classification: 0.1031 | Classification_binary: 0.1915 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1325 | Regression_eta: 0.0019 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.2000 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0653 | Regression_eta: 0.0523 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0854 | LR: 5.73e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2342s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.9207 | max_loss: 0.1533 | max_backward: 2.2288 | max_optimizer_step: 0.2436 | max_tensorboard_logging: 0.8522 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7858.2 | uss_MB: 7247.2 | child_rss_MB: 5039.0 | allocated_MB: 450.2 | reserved_MB: 26490.0 | inactive_split_MB: 976.9 +rank2 [2026-10-29 07:35:42,204] INFO : Step 24300/50000 rank2 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5336 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1380 | Classification: 0.0927 | Classification_binary: 0.1863 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1247 | Regression_eta: 0.0020 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0561 | Regression_eta: 0.0580 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0748 | LR: 5.70e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6723s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8648 | max_loss: 0.1481 | max_backward: 1.7152 | max_optimizer_step: 0.2781 | max_tensorboard_logging: 0.7831 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6810.0 | uss_MB: 6199.1 | child_rss_MB: 5043.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 983.3 +rank2 [2026-10-29 07:40:19,144] INFO : Step 24400/50000 rank2 | Train Loss: 0.9707 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0879 | Classification_binary: 0.1454 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0014 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0549 | Regression_eta: 0.0418 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0494 | LR: 5.67e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.8488s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7628 | max_loss: 0.1940 | max_backward: 2.4253 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.9481 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6811.6 | uss_MB: 5803.5 | child_rss_MB: 5042.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 1129.3 +rank2 [2026-10-29 07:45:00,050] INFO : Step 24500/50000 rank2 | Train Loss: 1.0937 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1319 | Classification: 0.0941 | Classification_binary: 0.1676 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1334 | Regression_eta: 0.0016 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1667 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0714 | Regression_eta: 0.0427 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0436 | LR: 5.64e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4288s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7565 | max_loss: 0.5105 | max_backward: 1.8057 | max_optimizer_step: 0.2534 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6807.0 | uss_MB: 6198.6 | child_rss_MB: 5045.1 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 945.2 +rank2 [2026-10-29 07:49:44,236] INFO : Step 24600/50000 rank2 | Train Loss: 1.0924 | Train Loss Unweighted: 0.5511 | Train Loss Components: Auxiliary: 0.2237 | Cardinality: 0.1233 | Classification: 0.0963 | Classification_binary: 0.1991 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1255 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.2040 | Regression_cos_phi: 0.0991 | Regression_energy: 0.0767 | Regression_eta: 0.0467 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0850 | LR: 5.61e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6473s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.9060 | max_loss: 0.1523 | max_backward: 2.0894 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.5102 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6811.1 | uss_MB: 6200.6 | child_rss_MB: 5046.3 | allocated_MB: 449.5 | reserved_MB: 26492.0 | inactive_split_MB: 943.2 +rank2 [2026-10-29 07:54:29,278] INFO : Step 24700/50000 rank2 | Train Loss: 0.9972 | Train Loss Unweighted: 0.4984 | Train Loss Components: Auxiliary: 0.1950 | Cardinality: 0.1160 | Classification: 0.0782 | Classification_binary: 0.1692 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1874 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0665 | Regression_eta: 0.0379 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0563 | LR: 5.58e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5125s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0060 | max_forward: 0.7847 | max_loss: 0.1519 | max_backward: 1.7159 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.8325 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6813.0 | uss_MB: 6199.7 | child_rss_MB: 5050.6 | allocated_MB: 447.9 | reserved_MB: 27708.0 | inactive_split_MB: 1169.6 +rank2 [2026-10-29 07:59:09,068] INFO : Step 24800/50000 rank2 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4722 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1202 | Classification: 0.0808 | Classification_binary: 0.1484 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0016 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0530 | Regression_eta: 0.0330 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0569 | LR: 5.56e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7181s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0127 | max_forward: 0.7903 | max_loss: 0.1427 | max_backward: 1.7798 | max_optimizer_step: 0.2260 | max_tensorboard_logging: 0.8943 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6813.0 | uss_MB: 6199.9 | child_rss_MB: 5051.5 | allocated_MB: 447.6 | reserved_MB: 28508.0 | inactive_split_MB: 989.1 +rank2 [2026-10-29 08:03:50,794] INFO : Step 24900/50000 rank2 | Train Loss: 1.0773 | Train Loss Unweighted: 0.5300 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1506 | Classification: 0.0964 | Classification_binary: 0.1776 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1257 | Regression_eta: 0.0016 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0635 | Regression_eta: 0.0479 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0621 | LR: 5.53e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.3166s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8100 | max_loss: 0.1483 | max_backward: 1.9423 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 1.0834 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6802.2 | uss_MB: 5593.7 | child_rss_MB: 5050.0 | allocated_MB: 449.1 | reserved_MB: 28508.0 | inactive_split_MB: 993.6 +rank2 [2026-10-29 08:08:37,678] INFO : Step 25000/50000 rank2 | Train Loss: 1.1315 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1671 | Classification: 0.0915 | Classification_binary: 0.1899 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1201 | Regression_eta: 0.0020 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1943 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0663 | Regression_eta: 0.0464 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0860 | LR: 5.50e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.7047s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8152 | max_loss: 0.2476 | max_backward: 1.7290 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.8750 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6807.0 | uss_MB: 6195.6 | child_rss_MB: 5052.2 | allocated_MB: 448.5 | reserved_MB: 28508.0 | inactive_split_MB: 953.9 +rank2 [2026-10-29 08:08:38,676] INFO : Running validation on rank2 +rank2 [2026-10-29 08:08:38,676] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 08:08:38,676] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 08:09:30,804] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:09:31,356] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 08:09:57,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 08:09:58,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 08:09:59,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 08:10:01,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 08:10:11,034] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 08:10:11,661] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:10:12,185] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 08:10:28,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 08:10:29,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 08:10:30,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 08:10:31,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 08:10:41,023] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 08:10:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 08:10:42,694] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 08:10:58,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 08:10:59,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 08:11:00,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 08:11:00,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 08:11:10,881] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 08:16:19,774] INFO : Step 25100/50000 rank2 | Train Loss: 1.0264 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1288 | Classification: 0.0850 | Classification_binary: 0.1497 | HitGrouping: 0.1664 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1378 | Regression_eta: 0.0016 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0772 | Regression_eta: 0.0510 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0766 | LR: 5.47e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.5421s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0059 | max_forward: 0.7878 | max_loss: 0.1437 | max_backward: 2.2383 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9692 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7903.8 | uss_MB: 7286.7 | child_rss_MB: 5042.9 | allocated_MB: 446.8 | reserved_MB: 24062.0 | inactive_split_MB: 1159.3 +rank2 [2026-10-29 08:21:20,699] INFO : Step 25200/50000 rank2 | Train Loss: 1.0775 | Train Loss Unweighted: 0.5434 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1397 | Classification: 0.0996 | Classification_binary: 0.1706 | HitGrouping: 0.1584 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1340 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1973 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0704 | Regression_eta: 0.0502 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0699 | LR: 5.44e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2316s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8831 | max_loss: 0.1456 | max_backward: 2.1320 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7903.3 | uss_MB: 7287.2 | child_rss_MB: 5042.2 | allocated_MB: 448.0 | reserved_MB: 26042.0 | inactive_split_MB: 1093.3 +rank2 [2026-10-29 08:26:17,211] INFO : Step 25300/50000 rank2 | Train Loss: 1.0761 | Train Loss Unweighted: 0.5109 | Train Loss Components: Auxiliary: 0.2132 | Cardinality: 0.1361 | Classification: 0.0961 | Classification_binary: 0.1821 | HitGrouping: 0.1562 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1138 | Regression_eta: 0.0018 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.2175 | Regression_cos_phi: 0.1134 | Regression_energy: 0.0739 | Regression_eta: 0.0739 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0865 | LR: 5.42e-04 | DataLoad Time: 0.0191s | Model Forward Time: 6.1310s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.9122 | max_loss: 0.1821 | max_backward: 2.1499 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.9506 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6897.2 | uss_MB: 5222.8 | child_rss_MB: 5036.8 | allocated_MB: 446.7 | reserved_MB: 28062.0 | inactive_split_MB: 1098.3 +rank2 [2026-10-29 08:31:10,694] INFO : Step 25400/50000 rank2 | Train Loss: 1.0222 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1238 | Classification: 0.0928 | Classification_binary: 0.1653 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1221 | Regression_eta: 0.0015 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1645 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0599 | Regression_eta: 0.0397 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0661 | LR: 5.39e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.7079s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1954 | max_backward: 1.8415 | max_optimizer_step: 0.3277 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6896.2 | uss_MB: 6278.8 | child_rss_MB: 5052.2 | allocated_MB: 449.3 | reserved_MB: 28062.0 | inactive_split_MB: 945.6 +rank2 [2026-10-29 08:36:13,579] INFO : Step 25500/50000 rank2 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4596 | Train Loss Components: Auxiliary: 0.1817 | Cardinality: 0.1001 | Classification: 0.0953 | Classification_binary: 0.1334 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0016 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0705 | Regression_eta: 0.0656 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0600 | LR: 5.36e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.7856s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7884 | max_loss: 0.1475 | max_backward: 2.0272 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.7862 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0483 | Memory: rss_MB: 6898.2 | uss_MB: 6283.2 | child_rss_MB: 5050.7 | allocated_MB: 450.7 | reserved_MB: 28062.0 | inactive_split_MB: 1166.9 +rank2 [2026-10-29 08:41:08,951] INFO : Step 25600/50000 rank2 | Train Loss: 1.0724 | Train Loss Unweighted: 0.5330 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1190 | Classification: 0.0985 | Classification_binary: 0.1843 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1231 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.1667 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0685 | Regression_eta: 0.0463 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0967 | LR: 5.33e-04 | DataLoad Time: 0.0187s | Model Forward Time: 6.1579s | Phase Times: max_pre_step_barrier: 0.0194 | max_batch_to_device: 0.0083 | max_forward: 1.0622 | max_loss: 0.3311 | max_backward: 2.5999 | max_optimizer_step: 0.3019 | max_tensorboard_logging: 0.8123 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6895.2 | uss_MB: 5585.2 | child_rss_MB: 5023.7 | allocated_MB: 450.4 | reserved_MB: 28064.0 | inactive_split_MB: 1163.5 +rank2 [2026-10-29 08:46:09,676] INFO : Step 25700/50000 rank2 | Train Loss: 1.0595 | Train Loss Unweighted: 0.5120 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.1394 | Classification: 0.0920 | Classification_binary: 0.1853 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1747 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0675 | Regression_eta: 0.0431 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0847 | LR: 5.30e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.8078s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.8320 | max_loss: 0.2520 | max_backward: 2.6962 | max_optimizer_step: 0.2699 | max_tensorboard_logging: 0.7057 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6885.0 | uss_MB: 6272.6 | child_rss_MB: 5051.6 | allocated_MB: 448.6 | reserved_MB: 29942.0 | inactive_split_MB: 1165.2 +rank2 [2026-10-29 08:51:40,159] INFO : Step 25800/50000 rank2 | Train Loss: 0.9592 | Train Loss Unweighted: 0.4733 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1038 | Classification: 0.0787 | Classification_binary: 0.1531 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1182 | Regression_eta: 0.0016 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0574 | Regression_energy: 0.0569 | Regression_eta: 0.0395 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0490 | LR: 5.27e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4828s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7549 | max_loss: 0.1480 | max_backward: 1.5296 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 0.8796 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1361 | Memory: rss_MB: 6889.1 | uss_MB: 6277.5 | child_rss_MB: 5023.6 | allocated_MB: 445.9 | reserved_MB: 30100.0 | inactive_split_MB: 961.8 +rank2 [2026-10-29 08:56:50,110] INFO : Step 25900/50000 rank2 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4613 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1142 | Classification: 0.0768 | Classification_binary: 0.1501 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1145 | Regression_eta: 0.0015 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0993 | Regression_energy: 0.0673 | Regression_eta: 0.0490 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0852 | LR: 5.25e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8318s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0124 | max_forward: 0.7901 | max_loss: 0.1430 | max_backward: 1.9362 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8933 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 6880.5 | uss_MB: 6281.6 | child_rss_MB: 5008.5 | allocated_MB: 447.7 | reserved_MB: 30100.0 | inactive_split_MB: 1167.6 +rank2 [2026-10-29 09:01:47,509] INFO : Step 26000/50000 rank2 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5169 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1347 | Classification: 0.0896 | Classification_binary: 0.1938 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0015 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1733 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0834 | Regression_eta: 0.0447 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0628 | LR: 5.22e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.4847s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8125 | max_loss: 0.2612 | max_backward: 1.8771 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.8972 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6845.2 | uss_MB: 5700.0 | child_rss_MB: 4882.0 | allocated_MB: 450.3 | reserved_MB: 30108.0 | inactive_split_MB: 1161.3 +rank2 [2026-10-29 09:01:48,414] INFO : Running validation on rank2 +rank2 [2026-10-29 09:01:48,415] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 09:01:48,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 09:02:42,435] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:02:42,920] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 09:03:09,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 09:03:10,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 09:03:11,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 09:03:12,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 09:03:22,287] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 09:03:22,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:03:23,247] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 09:03:39,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 09:03:40,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 09:03:41,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 09:03:42,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 09:03:52,522] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 09:03:52,918] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:03:53,429] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 09:04:08,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 09:04:09,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 09:04:10,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 09:04:11,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 09:04:21,430] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 09:09:37,593] INFO : Step 26100/50000 rank2 | Train Loss: 0.9519 | Train Loss Unweighted: 0.4738 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1091 | Classification: 0.0871 | Classification_binary: 0.1547 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1147 | Regression_eta: 0.0013 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1636 | Regression_cos_phi: 0.0763 | Regression_energy: 0.0513 | Regression_eta: 0.0396 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0587 | LR: 5.19e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.2081s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0321 | max_forward: 0.8143 | max_loss: 0.1540 | max_backward: 1.7669 | max_optimizer_step: 0.2382 | max_tensorboard_logging: 1.1185 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 8060.0 | uss_MB: 7433.5 | child_rss_MB: 4850.5 | allocated_MB: 445.1 | reserved_MB: 25850.0 | inactive_split_MB: 1376.9 +rank2 [2026-10-29 09:14:36,073] INFO : Step 26200/50000 rank2 | Train Loss: 0.9758 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.0970 | Classification: 0.0858 | Classification_binary: 0.1501 | HitGrouping: 0.1665 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0545 | Regression_eta: 0.0503 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0695 | LR: 5.16e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6740s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7728 | max_loss: 0.1384 | max_backward: 1.8195 | max_optimizer_step: 0.3036 | max_tensorboard_logging: 0.7010 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0602 | Memory: rss_MB: 8060.1 | uss_MB: 7488.7 | child_rss_MB: 4848.5 | allocated_MB: 447.4 | reserved_MB: 27830.0 | inactive_split_MB: 1117.7 +rank2 [2026-10-29 09:19:51,625] INFO : Step 26300/50000 rank2 | Train Loss: 1.0568 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2146 | Cardinality: 0.1416 | Classification: 0.0935 | Classification_binary: 0.1772 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0796 | Classification_binary: 0.1806 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0674 | Regression_eta: 0.0524 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0908 | LR: 5.13e-04 | DataLoad Time: 0.0191s | Model Forward Time: 6.1506s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.8205 | max_loss: 0.1808 | max_backward: 2.8872 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 1.1024 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6927.6 | uss_MB: 6354.4 | child_rss_MB: 4842.0 | allocated_MB: 450.8 | reserved_MB: 29830.0 | inactive_split_MB: 1114.5 +rank2 [2026-10-29 09:24:55,631] INFO : Step 26400/50000 rank2 | Train Loss: 1.0536 | Train Loss Unweighted: 0.5239 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1323 | Classification: 0.0937 | Classification_binary: 0.1845 | HitGrouping: 0.1559 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1209 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1923 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0590 | Regression_eta: 0.0546 | Regression_pt: 0.0642 | Regression_sin_phi: 0.1186 | LR: 5.10e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.5421s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8639 | max_loss: 0.1569 | max_backward: 2.4710 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 1.0049 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6924.7 | uss_MB: 6353.2 | child_rss_MB: 4853.6 | allocated_MB: 447.6 | reserved_MB: 29832.0 | inactive_split_MB: 1186.4 +rank2 [2026-10-29 09:29:59,513] INFO : Step 26500/50000 rank2 | Train Loss: 0.9812 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1299 | Classification: 0.0807 | Classification_binary: 0.1400 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1217 | Regression_eta: 0.0017 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0643 | Regression_eta: 0.0499 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0586 | LR: 5.08e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8047s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7436 | max_loss: 0.1579 | max_backward: 1.8417 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.9265 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6921.4 | uss_MB: 6354.1 | child_rss_MB: 4851.7 | allocated_MB: 444.4 | reserved_MB: 29832.0 | inactive_split_MB: 1229.5 +rank2 [2026-10-29 09:34:58,506] INFO : Step 26600/50000 rank2 | Train Loss: 1.0108 | Train Loss Unweighted: 0.5066 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.1182 | Classification: 0.0957 | Classification_binary: 0.1588 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1242 | Regression_eta: 0.0014 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1375 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0649 | Regression_eta: 0.0341 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0645 | LR: 5.05e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6965s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7561 | max_loss: 0.1430 | max_backward: 1.8427 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6921.4 | uss_MB: 6356.0 | child_rss_MB: 4827.0 | allocated_MB: 447.9 | reserved_MB: 29832.0 | inactive_split_MB: 1037.2 +rank2 [2026-10-29 09:40:07,140] INFO : Step 26700/50000 rank2 | Train Loss: 1.0878 | Train Loss Unweighted: 0.5266 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1269 | Classification: 0.0956 | Classification_binary: 0.1961 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1144 | Regression_eta: 0.0018 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1933 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0612 | Regression_eta: 0.0587 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0674 | LR: 5.02e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.1878s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.9788 | max_loss: 0.1540 | max_backward: 1.9407 | max_optimizer_step: 0.2295 | max_tensorboard_logging: 1.0076 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6911.8 | uss_MB: 6344.9 | child_rss_MB: 4859.9 | allocated_MB: 450.1 | reserved_MB: 29832.0 | inactive_split_MB: 1190.4 +rank2 [2026-10-29 09:48:41,511] INFO : Step 26800/50000 rank2 | Train Loss: 1.0077 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1102 | Classification: 0.0787 | Classification_binary: 0.1553 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1359 | Regression_eta: 0.0015 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0334 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0647 | Regression_eta: 0.0308 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0571 | LR: 4.99e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.8126s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.7774 | max_loss: 0.2523 | max_backward: 1.8741 | max_optimizer_step: 0.2873 | max_tensorboard_logging: 0.8206 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6909.2 | uss_MB: 6351.0 | child_rss_MB: 4816.3 | allocated_MB: 444.7 | reserved_MB: 29832.0 | inactive_split_MB: 1121.3 +rank2 [2026-10-29 09:53:40,041] INFO : Step 26900/50000 rank2 | Train Loss: 0.9524 | Train Loss Unweighted: 0.4793 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1099 | Classification: 0.0833 | Classification_binary: 0.1305 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1313 | Regression_eta: 0.0016 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1252 | Regression_cos_phi: 0.0810 | Regression_energy: 0.0688 | Regression_eta: 0.0386 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0702 | LR: 4.96e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.4179s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7292 | max_loss: 0.1373 | max_backward: 1.7442 | max_optimizer_step: 0.3152 | max_tensorboard_logging: 0.6764 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 6905.2 | uss_MB: 5520.6 | child_rss_MB: 4788.1 | allocated_MB: 446.9 | reserved_MB: 29832.0 | inactive_split_MB: 1016.7 +rank2 [2026-10-29 09:58:37,162] INFO : Step 27000/50000 rank2 | Train Loss: 1.0285 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1243 | Classification: 0.0862 | Classification_binary: 0.1888 | HitGrouping: 0.1539 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0016 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1927 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0554 | Regression_eta: 0.0362 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0686 | LR: 4.94e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.1624s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.9134 | max_loss: 0.1497 | max_backward: 2.1803 | max_optimizer_step: 0.2254 | max_tensorboard_logging: 0.9490 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6904.1 | uss_MB: 6354.5 | child_rss_MB: 4772.3 | allocated_MB: 449.7 | reserved_MB: 29832.0 | inactive_split_MB: 1024.3 +rank2 [2026-10-29 09:58:38,108] INFO : Running validation on rank2 +rank2 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 09:59:32,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 09:59:33,284] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 10:00:00,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 10:00:01,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 10:00:02,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 10:00:03,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 10:00:13,362] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 10:00:14,009] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:00:14,538] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 10:00:30,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 10:00:33,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 10:00:34,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 10:00:35,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 10:00:45,274] INFO : Time taken to make predictions on device 2 is: 0.28 min +rank2 [2026-10-29 10:00:46,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:00:46,863] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 10:01:02,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 10:01:03,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 10:01:04,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 10:01:05,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 10:01:15,113] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 10:06:37,478] INFO : Step 27100/50000 rank2 | Train Loss: 1.0614 | Train Loss Unweighted: 0.4989 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1449 | Classification: 0.0752 | Classification_binary: 0.1877 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.1451 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0556 | Regression_eta: 0.0395 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0569 | LR: 4.91e-04 | DataLoad Time: 0.0209s | Model Forward Time: 7.1175s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0966 | max_forward: 2.1724 | max_loss: 0.6705 | max_backward: 3.1723 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 1.5888 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 8047.0 | uss_MB: 7496.7 | child_rss_MB: 4778.7 | allocated_MB: 447.0 | reserved_MB: 27292.0 | inactive_split_MB: 983.4 +rank2 [2026-10-29 10:11:19,567] INFO : Step 27200/50000 rank2 | Train Loss: 0.9045 | Train Loss Unweighted: 0.4389 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1028 | Classification: 0.0770 | Classification_binary: 0.1287 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1440 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0721 | Regression_eta: 0.0304 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0553 | LR: 4.88e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.7891s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7725 | max_loss: 0.5032 | max_backward: 1.6788 | max_optimizer_step: 0.2815 | max_tensorboard_logging: 0.9686 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8047.0 | uss_MB: 7493.7 | child_rss_MB: 4786.7 | allocated_MB: 445.7 | reserved_MB: 29236.0 | inactive_split_MB: 1232.8 +rank2 [2026-10-29 10:15:58,638] INFO : Step 27300/50000 rank2 | Train Loss: 1.0726 | Train Loss Unweighted: 0.5095 | Train Loss Components: Auxiliary: 0.2138 | Cardinality: 0.1337 | Classification: 0.1081 | Classification_binary: 0.1640 | HitGrouping: 0.1596 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1159 | Regression_eta: 0.0019 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1761 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0665 | Regression_eta: 0.0521 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0625 | LR: 4.85e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1442s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8896 | max_loss: 0.1460 | max_backward: 2.0153 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6974.0 | uss_MB: 6424.6 | child_rss_MB: 4784.6 | allocated_MB: 450.9 | reserved_MB: 29330.0 | inactive_split_MB: 1103.1 +rank2 [2026-10-29 10:20:31,814] INFO : Step 27400/50000 rank2 | Train Loss: 1.0758 | Train Loss Unweighted: 0.5282 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1344 | Classification: 0.0930 | Classification_binary: 0.1899 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1207 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.2102 | Regression_cos_phi: 0.0768 | Regression_energy: 0.0560 | Regression_eta: 0.0429 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0619 | LR: 4.82e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.0218s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8446 | max_loss: 0.1488 | max_backward: 2.0519 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.8725 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6977.0 | uss_MB: 5627.2 | child_rss_MB: 4777.8 | allocated_MB: 448.4 | reserved_MB: 31310.0 | inactive_split_MB: 895.7 +rank2 [2026-10-29 10:25:07,332] INFO : Step 27500/50000 rank2 | Train Loss: 0.9697 | Train Loss Unweighted: 0.4574 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1111 | Classification: 0.0831 | Classification_binary: 0.1473 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1114 | Regression_eta: 0.0018 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0610 | Regression_eta: 0.0555 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0550 | LR: 4.80e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4609s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.7904 | max_loss: 0.1563 | max_backward: 1.6291 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.7457 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6966.1 | uss_MB: 4710.9 | child_rss_MB: 4769.8 | allocated_MB: 447.3 | reserved_MB: 31310.0 | inactive_split_MB: 986.8 +rank2 [2026-10-29 10:30:03,959] INFO : Step 27600/50000 rank2 | Train Loss: 0.9616 | Train Loss Unweighted: 0.4618 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1092 | Classification: 0.0865 | Classification_binary: 0.1284 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0019 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1157 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0609 | Regression_eta: 0.0335 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0512 | LR: 4.77e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5704s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0380 | max_forward: 0.7674 | max_loss: 0.1463 | max_backward: 1.9164 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.5145 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6974.1 | uss_MB: 6422.9 | child_rss_MB: 4778.4 | allocated_MB: 446.7 | reserved_MB: 31310.0 | inactive_split_MB: 1182.0 +rank2 [2026-10-29 10:34:46,993] INFO : Step 27700/50000 rank2 | Train Loss: 1.0693 | Train Loss Unweighted: 0.5190 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1414 | Classification: 0.0939 | Classification_binary: 0.1783 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0015 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1712 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0630 | Regression_eta: 0.0321 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0620 | LR: 4.74e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.6750s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.8151 | max_loss: 0.1750 | max_backward: 2.0398 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.5213 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0659 | Memory: rss_MB: 6974.1 | uss_MB: 6424.8 | child_rss_MB: 4756.0 | allocated_MB: 448.4 | reserved_MB: 33312.0 | inactive_split_MB: 986.3 +rank2 [2026-10-29 10:39:29,303] INFO : Step 27800/50000 rank2 | Train Loss: 1.0875 | Train Loss Unweighted: 0.5294 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1226 | Classification: 0.0924 | Classification_binary: 0.1842 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1237 | Regression_eta: 0.0019 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1769 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0676 | Regression_eta: 0.0465 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0876 | LR: 4.71e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.8113s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0152 | max_forward: 0.8463 | max_loss: 0.1744 | max_backward: 1.7379 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.9397 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 6971.2 | uss_MB: 6421.2 | child_rss_MB: 4773.7 | allocated_MB: 452.2 | reserved_MB: 33312.0 | inactive_split_MB: 1320.1 +rank2 [2026-10-29 10:44:11,938] INFO : Step 27900/50000 rank2 | Train Loss: 0.9483 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0988 | Classification: 0.0859 | Classification_binary: 0.1439 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1244 | Regression_eta: 0.0014 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1675 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0728 | Regression_eta: 0.0442 | Regression_pt: 0.0793 | Regression_sin_phi: 0.0636 | LR: 4.68e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.4739s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.7626 | max_loss: 0.1370 | max_backward: 1.8424 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6972.2 | uss_MB: 6423.3 | child_rss_MB: 4773.1 | allocated_MB: 447.1 | reserved_MB: 33312.0 | inactive_split_MB: 1112.6 +rank2 [2026-10-29 10:48:55,972] INFO : Step 28000/50000 rank2 | Train Loss: 0.9840 | Train Loss Unweighted: 0.4736 | Train Loss Components: Auxiliary: 0.1949 | Cardinality: 0.1114 | Classification: 0.0927 | Classification_binary: 0.1564 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1099 | Regression_eta: 0.0015 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0542 | Regression_eta: 0.0450 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0610 | LR: 4.66e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9441s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8192 | max_loss: 0.2552 | max_backward: 2.1299 | max_optimizer_step: 0.2920 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 6969.4 | uss_MB: 6424.2 | child_rss_MB: 4776.8 | allocated_MB: 448.2 | reserved_MB: 33312.0 | inactive_split_MB: 1178.0 +rank2 [2026-10-29 10:48:56,835] INFO : Running validation on rank2 +rank2 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 10:49:51,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:49:52,003] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 10:50:18,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 10:50:19,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 10:50:20,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 10:50:21,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 10:50:31,790] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 10:50:32,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:50:32,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 10:50:48,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 10:50:49,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 10:50:50,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 10:50:51,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 10:51:01,911] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 10:51:02,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 10:51:03,448] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 10:51:19,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 10:51:19,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 10:51:20,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 10:51:21,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 10:51:31,575] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 10:56:32,746] INFO : Step 28100/50000 rank2 | Train Loss: 1.0915 | Train Loss Unweighted: 0.5085 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1649 | Classification: 0.0833 | Classification_binary: 0.1834 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1187 | Regression_eta: 0.0017 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1581 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0615 | Regression_eta: 0.0407 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0670 | LR: 4.63e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8680s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8571 | max_loss: 0.1603 | max_backward: 1.9112 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 0.7670 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0838 | Memory: rss_MB: 8288.3 | uss_MB: 7733.7 | child_rss_MB: 4773.7 | allocated_MB: 448.9 | reserved_MB: 25946.0 | inactive_split_MB: 1313.1 +rank2 [2026-10-29 11:01:18,173] INFO : Step 28200/50000 rank2 | Train Loss: 0.9241 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1144 | Classification: 0.0770 | Classification_binary: 0.1441 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1163 | Regression_eta: 0.0014 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0571 | Regression_eta: 0.0395 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0690 | LR: 4.60e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8981s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0751 | max_forward: 0.8691 | max_loss: 0.5236 | max_backward: 1.8483 | max_optimizer_step: 0.2480 | max_tensorboard_logging: 0.8546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 8289.0 | uss_MB: 7733.3 | child_rss_MB: 4770.2 | allocated_MB: 445.0 | reserved_MB: 25948.0 | inactive_split_MB: 976.4 +rank2 [2026-10-29 11:07:40,703] INFO : Step 28300/50000 rank2 | Train Loss: 0.9513 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1028 | Classification: 0.0882 | Classification_binary: 0.1368 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1147 | Regression_eta: 0.0016 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0513 | Regression_eta: 0.0345 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0533 | LR: 4.57e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9474s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.7916 | max_loss: 0.1449 | max_backward: 2.0850 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.8583 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7055.4 | uss_MB: 6496.0 | child_rss_MB: 4774.9 | allocated_MB: 447.2 | reserved_MB: 25952.0 | inactive_split_MB: 1165.3 +rank2 [2026-10-29 11:12:27,615] INFO : Step 28400/50000 rank2 | Train Loss: 1.0604 | Train Loss Unweighted: 0.4982 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1382 | Classification: 0.0847 | Classification_binary: 0.1862 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1111 | Regression_eta: 0.0018 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2008 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0648 | Regression_eta: 0.0408 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0525 | LR: 4.55e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.0298s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9139 | max_loss: 0.1474 | max_backward: 2.1323 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.6846 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0517 | Memory: rss_MB: 7055.5 | uss_MB: 6496.2 | child_rss_MB: 4770.0 | allocated_MB: 448.2 | reserved_MB: 26968.0 | inactive_split_MB: 835.9 +rank2 [2026-10-29 11:17:11,503] INFO : Step 28500/50000 rank2 | Train Loss: 1.0282 | Train Loss Unweighted: 0.5005 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1237 | Classification: 0.0911 | Classification_binary: 0.1756 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0013 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1700 | Regression_cos_phi: 0.0898 | Regression_energy: 0.0619 | Regression_eta: 0.0425 | Regression_pt: 0.0689 | Regression_sin_phi: 0.1033 | LR: 4.52e-04 | DataLoad Time: 0.0213s | Model Forward Time: 9.4584s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0886 | max_forward: 0.9758 | max_loss: 0.5123 | max_backward: 2.0811 | max_optimizer_step: 0.2797 | max_tensorboard_logging: 1.5644 | max_tensorboard_flush: 0.0047 | max_post_step_barrier: 0.0889 | Memory: rss_MB: 7042.2 | uss_MB: 4349.9 | child_rss_MB: 4774.7 | allocated_MB: 452.0 | reserved_MB: 27968.0 | inactive_split_MB: 923.2 +rank2 [2026-10-29 11:21:56,249] INFO : Step 28600/50000 rank2 | Train Loss: 0.9753 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1202 | Classification: 0.0804 | Classification_binary: 0.1446 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1132 | Regression_eta: 0.0017 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1516 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0545 | Regression_eta: 0.0359 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0588 | LR: 4.49e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.9443s | Phase Times: max_pre_step_barrier: 0.1127 | max_batch_to_device: 0.0146 | max_forward: 0.7731 | max_loss: 0.2404 | max_backward: 1.6352 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 1.0006 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7048.8 | uss_MB: 5469.7 | child_rss_MB: 4776.7 | allocated_MB: 446.8 | reserved_MB: 30056.0 | inactive_split_MB: 896.4 +rank2 [2026-10-29 11:26:34,428] INFO : Step 28700/50000 rank2 | Train Loss: 1.0558 | Train Loss Unweighted: 0.5173 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1082 | Classification: 0.1065 | Classification_binary: 0.1612 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1217 | Regression_eta: 0.0024 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0446 | Regression_eta: 0.0458 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0468 | LR: 4.46e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.9570s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0686 | max_forward: 0.8423 | max_loss: 0.1475 | max_backward: 2.0170 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.8830 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 7053.8 | uss_MB: 6496.4 | child_rss_MB: 4774.2 | allocated_MB: 447.9 | reserved_MB: 30056.0 | inactive_split_MB: 925.2 +rank2 [2026-10-29 11:31:18,244] INFO : Step 28800/50000 rank2 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5167 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1474 | Classification: 0.0969 | Classification_binary: 0.1939 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1862 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0513 | Regression_eta: 0.0438 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0641 | LR: 4.44e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.8762s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0075 | max_forward: 0.8427 | max_loss: 0.2661 | max_backward: 2.8464 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7050.1 | uss_MB: 4315.7 | child_rss_MB: 4755.7 | allocated_MB: 448.2 | reserved_MB: 30058.0 | inactive_split_MB: 976.8 +rank2 [2026-10-29 11:35:53,801] INFO : Step 28900/50000 rank2 | Train Loss: 0.9389 | Train Loss Unweighted: 0.4547 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.1078 | Classification: 0.0733 | Classification_binary: 0.1521 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0643 | Regression_eta: 0.0363 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0600 | LR: 4.41e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.5668s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0062 | max_forward: 0.7499 | max_loss: 0.2426 | max_backward: 1.8556 | max_optimizer_step: 0.3135 | max_tensorboard_logging: 0.5831 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7048.4 | uss_MB: 6494.0 | child_rss_MB: 4774.7 | allocated_MB: 444.3 | reserved_MB: 30058.0 | inactive_split_MB: 1133.2 +rank2 [2026-10-29 11:40:53,318] INFO : Step 29000/50000 rank2 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4535 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0939 | Classification: 0.0855 | Classification_binary: 0.1308 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0015 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0759 | Regression_energy: 0.0602 | Regression_eta: 0.0555 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0719 | LR: 4.38e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.8473s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0062 | max_forward: 0.9270 | max_loss: 0.2073 | max_backward: 2.1896 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6044 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7047.7 | uss_MB: 6496.6 | child_rss_MB: 4759.3 | allocated_MB: 450.2 | reserved_MB: 30060.0 | inactive_split_MB: 1402.9 +rank2 [2026-10-29 11:40:54,336] INFO : Running validation on rank2 +rank2 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 11:41:46,665] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:41:47,174] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 11:42:15,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 11:42:18,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 11:42:19,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 11:42:20,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 11:42:30,116] INFO : Time taken to make predictions on device 2 is: 0.29 min +rank2 [2026-10-29 11:42:31,032] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:42:31,573] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 11:42:47,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 11:42:48,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 11:42:49,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 11:42:50,719] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 11:43:00,731] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 11:43:00,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 11:43:01,398] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 11:43:17,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 11:43:17,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 11:43:18,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 11:43:19,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 11:43:29,522] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 11:48:35,107] INFO : Step 29100/50000 rank2 | Train Loss: 1.0559 | Train Loss Unweighted: 0.5040 | Train Loss Components: Auxiliary: 0.2142 | Cardinality: 0.1307 | Classification: 0.0939 | Classification_binary: 0.1851 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1098 | Regression_eta: 0.0015 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0586 | Regression_eta: 0.0369 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0635 | LR: 4.35e-04 | DataLoad Time: 0.0145s | Model Forward Time: 7.6739s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.6779 | max_loss: 0.3815 | max_backward: 4.2661 | max_optimizer_step: 0.3470 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8253.0 | uss_MB: 6395.3 | child_rss_MB: 4770.6 | allocated_MB: 448.8 | reserved_MB: 24188.0 | inactive_split_MB: 1162.1 +rank2 [2026-10-29 11:53:34,049] INFO : Step 29200/50000 rank2 | Train Loss: 0.9909 | Train Loss Unweighted: 0.4801 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1204 | Classification: 0.0923 | Classification_binary: 0.1746 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1054 | Regression_eta: 0.0014 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0556 | Regression_eta: 0.0336 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0430 | LR: 4.33e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7780s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8354 | max_loss: 0.1558 | max_backward: 2.1201 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0677 | Memory: rss_MB: 8251.2 | uss_MB: 7692.4 | child_rss_MB: 4763.1 | allocated_MB: 447.9 | reserved_MB: 24190.0 | inactive_split_MB: 1166.5 +rank2 [2026-10-29 11:58:27,981] INFO : Step 29300/50000 rank2 | Train Loss: 0.9337 | Train Loss Unweighted: 0.4462 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1121 | Classification: 0.0773 | Classification_binary: 0.1359 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1146 | Regression_eta: 0.0015 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1174 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0808 | Regression_eta: 0.0428 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0770 | LR: 4.30e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.9127s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0277 | max_forward: 0.8212 | max_loss: 0.1457 | max_backward: 1.8414 | max_optimizer_step: 0.2550 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7135.1 | uss_MB: 5997.0 | child_rss_MB: 4761.5 | allocated_MB: 444.9 | reserved_MB: 24190.0 | inactive_split_MB: 1038.6 +rank2 [2026-10-29 12:03:07,470] INFO : Step 29400/50000 rank2 | Train Loss: 1.0186 | Train Loss Unweighted: 0.4675 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1372 | Classification: 0.0948 | Classification_binary: 0.1561 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0020 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0479 | Regression_eta: 0.0340 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0494 | LR: 4.27e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8012s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8495 | max_loss: 0.1434 | max_backward: 1.9359 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7145.4 | uss_MB: 6588.5 | child_rss_MB: 4762.3 | allocated_MB: 447.9 | reserved_MB: 26208.0 | inactive_split_MB: 957.3 +rank2 [2026-10-29 12:08:16,204] INFO : Step 29500/50000 rank2 | Train Loss: 1.0909 | Train Loss Unweighted: 0.5362 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.1071 | Classification_binary: 0.1896 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1176 | Regression_eta: 0.0019 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1886 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0817 | Regression_eta: 0.0574 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0786 | LR: 4.24e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.9676s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0876 | max_forward: 0.8925 | max_loss: 0.1527 | max_backward: 1.9621 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.8162 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7137.1 | uss_MB: 6580.0 | child_rss_MB: 4760.9 | allocated_MB: 448.9 | reserved_MB: 26214.0 | inactive_split_MB: 1166.5 +rank2 [2026-10-29 12:13:12,651] INFO : Step 29600/50000 rank2 | Train Loss: 0.9608 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.1935 | Cardinality: 0.1093 | Classification: 0.0850 | Classification_binary: 0.1542 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0988 | Regression_energy: 0.0778 | Regression_eta: 0.0481 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0513 | LR: 4.22e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.8681s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7848 | max_loss: 0.1495 | max_backward: 1.8042 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.0178 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7115.2 | uss_MB: 6585.9 | child_rss_MB: 4743.4 | allocated_MB: 445.5 | reserved_MB: 26214.0 | inactive_split_MB: 1171.9 +rank2 [2026-10-29 12:18:07,441] INFO : Step 29700/50000 rank2 | Train Loss: 0.9511 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1260 | Classification: 0.0829 | Classification_binary: 0.1357 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1183 | Regression_eta: 0.0014 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0367 | Regression_energy: 0.0548 | Regression_eta: 0.0411 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0487 | LR: 4.19e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.2344s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7531 | max_loss: 0.2308 | max_backward: 2.2313 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.9708 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7117.3 | uss_MB: 6587.7 | child_rss_MB: 4741.9 | allocated_MB: 446.6 | reserved_MB: 26214.0 | inactive_split_MB: 1160.6 +rank2 [2026-10-29 12:23:08,925] INFO : Step 29800/50000 rank2 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4915 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1104 | Classification: 0.0864 | Classification_binary: 0.1759 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1597 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0595 | Regression_eta: 0.0375 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0555 | LR: 4.16e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.4237s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8210 | max_loss: 0.1528 | max_backward: 2.4753 | max_optimizer_step: 0.3576 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0038 | max_post_step_barrier: 0.1798 | Memory: rss_MB: 7115.5 | uss_MB: 5536.1 | child_rss_MB: 4741.9 | allocated_MB: 450.3 | reserved_MB: 26214.0 | inactive_split_MB: 938.8 +rank2 [2026-10-29 12:28:10,147] INFO : Step 29900/50000 rank2 | Train Loss: 0.9604 | Train Loss Unweighted: 0.4559 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1243 | Classification: 0.0825 | Classification_binary: 0.1616 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1040 | Regression_eta: 0.0015 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1615 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0532 | Regression_eta: 0.0477 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0423 | LR: 4.14e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9476s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1142 | max_forward: 0.9271 | max_loss: 0.1531 | max_backward: 1.8698 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.8620 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7091.0 | uss_MB: 6437.5 | child_rss_MB: 4745.0 | allocated_MB: 450.0 | reserved_MB: 26214.0 | inactive_split_MB: 1166.0 +rank2 [2026-10-29 12:32:58,888] INFO : Step 30000/50000 rank2 | Train Loss: 0.9450 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.1132 | Classification: 0.0847 | Classification_binary: 0.1394 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0571 | Regression_eta: 0.0714 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0492 | LR: 4.11e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1883s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 1.0327 | max_loss: 0.1419 | max_backward: 1.6503 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.4878 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7039.0 | uss_MB: 6586.5 | child_rss_MB: 4745.4 | allocated_MB: 443.8 | reserved_MB: 26214.0 | inactive_split_MB: 1152.2 +rank2 [2026-10-29 12:47:12,790] INFO : Running validation on rank2 +rank2 [2026-10-29 12:47:12,790] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 12:47:12,790] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 14:49:00,129] INFO : Initializing model with seed=12345; process seed=12347 +rank2 [2026-10-29 14:49:00,144] INFO : configured dtype=torch.bfloat16 for autocast +rank2 [2026-10-29 14:49:20,929] INFO : Instantiating model +rank2 [2026-10-29 14:49:20,930] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank2 [2026-10-29 14:49:20,930] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank2 [2026-10-29 14:49:20,948] INFO : Input encoding initialization took 0.02s +rank2 [2026-10-29 14:49:20,949] INFO : nn0 parameters: 171776 +rank2 [2026-10-29 14:49:20,949] INFO : Initializing shared backbone layers, num_convs=6 +rank2 [2026-10-29 14:49:20,949] INFO : Initializing backbone layer 0 +rank2 [2026-10-29 14:49:20,983] INFO : Initializing backbone layer 1 +rank2 [2026-10-29 14:49:20,992] INFO : Initializing backbone layer 2 +rank2 [2026-10-29 14:49:21,000] INFO : Initializing backbone layer 3 +rank2 [2026-10-29 14:49:21,009] INFO : Initializing backbone layer 4 +rank2 [2026-10-29 14:49:21,017] INFO : Initializing backbone layer 5 +rank2 [2026-10-29 14:49:21,026] INFO : Backbone initialization took 0.08s +rank2 [2026-10-29 14:49:21,026] INFO : backbone parameters: 7901376 +rank2 [2026-10-29 14:49:21,026] INFO : Initializing output DNNs +rank2 [2026-10-29 14:49:21,062] INFO : Output DNNs initialization took 0.04s +rank2 [2026-10-29 14:49:21,063] INFO : backbone_mode=BackboneMode.SHARED +rank2 [2026-10-29 14:49:21,063] INFO : jagged_attention enabled=False +rank2 [2026-10-29 14:49:21,063] INFO : flash_attn_varlen enabled=False +rank2 [2026-10-29 14:49:21,063] INFO : task_queries enabled=False +rank2 [2026-10-29 14:49:21,063] INFO : classification_norm parameters: 0 +rank2 [2026-10-29 14:49:21,063] INFO : regression_norm parameters: 0 +rank2 [2026-10-29 14:49:21,063] INFO : classification_query parameters: 0 +rank2 [2026-10-29 14:49:21,063] INFO : regression_query parameters: 0 +rank2 [2026-10-29 14:49:21,063] INFO : classification_readout parameters: 0 +rank2 [2026-10-29 14:49:21,063] INFO : regression_readout parameters: 0 +rank2 [2026-10-29 14:49:21,063] INFO : output_mode=set +rank2 [2026-10-29 14:49:21,063] INFO : set_decoder parameters: 4713812 +rank2 [2026-10-29 14:49:21,063] INFO : nn_binary_particle parameters: 0 +rank2 [2026-10-29 14:49:21,063] INFO : nn_pid parameters: 0 +rank2 [2026-10-29 14:49:21,063] INFO : nn_pt parameters: 0 +rank2 [2026-10-29 14:49:21,064] INFO : nn_eta parameters: 0 +rank2 [2026-10-29 14:49:21,064] INFO : nn_sin_phi parameters: 0 +rank2 [2026-10-29 14:49:21,064] INFO : nn_cos_phi parameters: 0 +rank2 [2026-10-29 14:49:21,064] INFO : nn_energy parameters: 0 +rank2 [2026-10-29 14:49:21,064] INFO : Total MLPF parameters: 12786964 +rank2 [2026-10-29 14:49:21,064] INFO : MLPF __init__ done +rank2 [2026-10-29 14:49:21,065] INFO : Instantiated model +rank2 [2026-10-29 14:49:21,065] INFO : Moving model to device rank=2 +rank2 [2026-10-29 14:49:21,087] INFO : Moved model to device rank=2 +rank2 [2026-10-29 14:49:21,088] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank2 [2026-10-29 14:49:21,620] INFO : Restoring training from step 30001 +rank2 [2026-10-29 14:49:21,634] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth +rank2 [2026-10-29 14:49:21,635] INFO : Configured model for SyncBatchNorm rank=2 +rank2 [2026-10-29 14:49:22,135] INFO : Configured model for DistributedDataParallel rank=2 +rank2 [2026-10-29 14:49:22,289] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank2 [2026-10-29 14:49:22,289] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank2 [2026-10-29 14:49:22,291] INFO : Trainable parameters: 12786964 +rank2 [2026-10-29 14:49:22,291] INFO : Non-trainable parameters: 23268 +rank2 [2026-10-29 14:49:22,291] INFO : Total parameters: 12810232 +rank2 [2026-10-29 14:49:22,295] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank2 [2026-10-29 14:49:22,295] INFO : Getting dataloaders +rank2 [2026-10-29 14:49:22,295] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:49:28,715] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:49:29,733] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:49:30,959] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank2 [2026-10-29 14:49:30,959] INFO : Creating WeightedMixIterator with 1 data loaders. +rank2 [2026-10-29 14:49:30,959] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:49:31,247] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:49:31,520] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 14:49:31,780] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank2 [2026-10-29 14:49:31,780] INFO : Creating InterleavedIterator with 1 data loaders. +rank2 [2026-10-29 14:49:45,155] INFO : Got dataloaders +rank2 [2026-10-29 14:49:45,155] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [8888] of epochs [1] +rank2 [2026-10-29 14:49:45,155] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank2 [2026-10-29 14:49:45,155] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank2 [2026-10-29 14:49:45,155] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank2 [2026-10-29 14:49:47,178] INFO : loader split=train rank=2 len=21112 +rank2 [2026-10-29 14:49:47,179] INFO : loader split=valid rank=2 len=12 +rank2 [2026-10-29 14:49:47,179] INFO : Starting training from step 30001 to 50000 on rank 2 of 8 +rank2 [2026-10-29 14:49:47,179] INFO : Creating train_iterator +rank2 [2026-10-29 14:49:47,179] INFO : ResumableSampler physical:train loading state: start_index=142208 +rank2 [2026-10-29 14:50:01,246] INFO : Created train_iterator +rank2 [2026-10-29 14:54:42,004] INFO : Step 30100/50000 rank2 | Train Loss: 0.9854 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.2024 | Cardinality: 0.1295 | Classification: 0.0965 | Classification_binary: 0.1561 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0583 | Regression_eta: 0.0413 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0656 | LR: 4.08e-04 | DataLoad Time: 0.0180s | Model Forward Time: 3.6361s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7011 | max_loss: 0.1473 | max_backward: 1.5273 | max_optimizer_step: 0.1574 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0061 | Memory: rss_MB: 4775.9 | uss_MB: 3472.5 | child_rss_MB: 6376.2 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 850.1 +rank2 [2026-10-29 14:59:12,794] INFO : Step 30200/50000 rank2 | Train Loss: 1.0165 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2044 | Cardinality: 0.1520 | Classification: 0.0856 | Classification_binary: 0.1718 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1807 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0478 | Regression_eta: 0.0414 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0607 | LR: 4.06e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9883s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.7915 | max_loss: 0.2568 | max_backward: 1.8926 | max_optimizer_step: 0.1551 | max_tensorboard_logging: 0.4604 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4776.3 | uss_MB: 3476.7 | child_rss_MB: 6367.6 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 842.0 +rank2 [2026-10-29 15:03:42,967] INFO : Step 30300/50000 rank2 | Train Loss: 0.9183 | Train Loss Unweighted: 0.4449 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1232 | Classification: 0.0820 | Classification_binary: 0.1459 | HitGrouping: 0.1516 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0012 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0512 | Regression_eta: 0.0320 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0595 | LR: 4.03e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.8585s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7525 | max_loss: 0.2476 | max_backward: 1.8262 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.4574 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0039 | Memory: rss_MB: 4773.5 | uss_MB: 3475.1 | child_rss_MB: 6385.6 | allocated_MB: 495.8 | reserved_MB: 28760.0 | inactive_split_MB: 848.2 +rank2 [2026-10-29 15:08:13,475] INFO : Step 30400/50000 rank2 | Train Loss: 0.9035 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1137 | Classification: 0.0848 | Classification_binary: 0.1358 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0920 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0571 | Regression_eta: 0.0540 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0786 | LR: 4.00e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.9719s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7769 | max_loss: 0.1398 | max_backward: 1.9002 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.4925 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4770.8 | uss_MB: 3475.5 | child_rss_MB: 6366.2 | allocated_MB: 498.8 | reserved_MB: 28760.0 | inactive_split_MB: 878.9 +rank2 [2026-10-29 15:12:45,607] INFO : Step 30500/50000 rank2 | Train Loss: 1.0399 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1239 | Classification: 0.0901 | Classification_binary: 0.1841 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0018 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1848 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0492 | Regression_eta: 0.0475 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0546 | LR: 3.98e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.3863s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9688 | max_loss: 0.1627 | max_backward: 2.3054 | max_optimizer_step: 0.1555 | max_tensorboard_logging: 0.4908 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4768.4 | uss_MB: 3475.6 | child_rss_MB: 6405.5 | allocated_MB: 501.7 | reserved_MB: 28760.0 | inactive_split_MB: 944.3 +rank2 [2026-10-29 15:17:13,865] INFO : Step 30600/50000 rank2 | Train Loss: 0.9987 | Train Loss Unweighted: 0.4959 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1112 | Classification: 0.0932 | Classification_binary: 0.1720 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1137 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0557 | Regression_eta: 0.0374 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0688 | LR: 3.95e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2221s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7736 | max_loss: 0.1566 | max_backward: 1.7922 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.6994 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4770.8 | uss_MB: 3476.1 | child_rss_MB: 6399.5 | allocated_MB: 500.1 | reserved_MB: 28760.0 | inactive_split_MB: 845.9 +rank2 [2026-10-29 15:21:45,751] INFO : Step 30700/50000 rank2 | Train Loss: 0.9490 | Train Loss Unweighted: 0.4575 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1237 | Classification: 0.0844 | Classification_binary: 0.1399 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0014 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1306 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0646 | Regression_eta: 0.0294 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0556 | LR: 3.92e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1844s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7215 | max_loss: 0.5560 | max_backward: 2.0906 | max_optimizer_step: 0.1553 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4768.7 | uss_MB: 3474.9 | child_rss_MB: 6379.2 | allocated_MB: 493.8 | reserved_MB: 28760.0 | inactive_split_MB: 827.5 +rank2 [2026-10-29 15:26:16,186] INFO : Step 30800/50000 rank2 | Train Loss: 1.0320 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1331 | Classification: 0.0916 | Classification_binary: 0.1632 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1173 | Regression_eta: 0.0015 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0516 | Regression_eta: 0.0390 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0591 | LR: 3.90e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6914s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1016 | max_forward: 0.8041 | max_loss: 0.1462 | max_backward: 1.9351 | max_optimizer_step: 0.1552 | max_tensorboard_logging: 1.1007 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4769.5 | uss_MB: 3475.6 | child_rss_MB: 6407.8 | allocated_MB: 497.9 | reserved_MB: 28760.0 | inactive_split_MB: 842.0 +rank2 [2026-10-29 15:30:49,188] INFO : Step 30900/50000 rank2 | Train Loss: 1.0226 | Train Loss Unweighted: 0.4952 | Train Loss Components: Auxiliary: 0.2139 | Cardinality: 0.1339 | Classification: 0.0875 | Classification_binary: 0.1859 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1092 | Regression_eta: 0.0013 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.2041 | Regression_cos_phi: 0.0945 | Regression_energy: 0.0576 | Regression_eta: 0.0499 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0696 | LR: 3.87e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4017s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7610 | max_loss: 0.2067 | max_backward: 1.7028 | max_optimizer_step: 0.1599 | max_tensorboard_logging: 1.0327 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4771.1 | uss_MB: 3476.3 | child_rss_MB: 6385.8 | allocated_MB: 497.2 | reserved_MB: 28760.0 | inactive_split_MB: 838.0 +rank2 [2026-10-29 15:35:26,546] INFO : Step 31000/50000 rank2 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4630 | Train Loss Components: Auxiliary: 0.2006 | Cardinality: 0.1217 | Classification: 0.0821 | Classification_binary: 0.1592 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1085 | Regression_eta: 0.0018 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1541 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0638 | Regression_eta: 0.0514 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0789 | LR: 3.84e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4981s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7780 | max_loss: 0.2475 | max_backward: 1.8803 | max_optimizer_step: 0.1588 | max_tensorboard_logging: 0.9947 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4772.1 | uss_MB: 3477.0 | child_rss_MB: 6402.3 | allocated_MB: 498.2 | reserved_MB: 28760.0 | inactive_split_MB: 820.4 +rank2 [2026-10-29 15:35:27,414] INFO : Running validation on rank2 +rank2 [2026-10-29 15:35:27,414] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 15:35:27,414] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 15:36:19,714] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:36:20,267] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 15:36:45,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 15:36:46,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 15:36:47,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 15:36:49,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 15:36:59,016] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 15:36:59,566] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:37:00,105] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 15:37:15,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 15:37:16,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 15:37:17,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 15:37:18,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 15:37:28,832] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 15:37:29,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 15:37:29,838] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 15:37:45,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 15:37:46,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 15:37:47,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 15:37:48,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 15:37:58,012] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 15:42:50,283] INFO : Step 31100/50000 rank2 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4264 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1054 | Classification: 0.0853 | Classification_binary: 0.1288 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0587 | Regression_eta: 0.0416 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0600 | LR: 3.82e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5236s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7353 | max_loss: 0.1410 | max_backward: 1.7959 | max_optimizer_step: 0.2754 | max_tensorboard_logging: 0.9599 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 6463.6 | uss_MB: 5188.4 | child_rss_MB: 6394.5 | allocated_MB: 495.8 | reserved_MB: 27508.0 | inactive_split_MB: 851.2 +rank2 [2026-10-29 15:47:34,510] INFO : Step 31200/50000 rank2 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4709 | Train Loss Components: Auxiliary: 0.2045 | Cardinality: 0.1286 | Classification: 0.0862 | Classification_binary: 0.1646 | HitGrouping: 0.1524 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1077 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1821 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0563 | Regression_eta: 0.0442 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0571 | LR: 3.79e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3225s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8441 | max_loss: 0.1469 | max_backward: 2.0813 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 6462.4 | uss_MB: 5189.4 | child_rss_MB: 6422.3 | allocated_MB: 496.9 | reserved_MB: 27508.0 | inactive_split_MB: 935.6 +rank2 [2026-10-29 15:52:19,963] INFO : Step 31300/50000 rank2 | Train Loss: 1.0631 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1394 | Classification: 0.0917 | Classification_binary: 0.1731 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1881 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0569 | Regression_eta: 0.0455 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0655 | LR: 3.76e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2492s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7950 | max_loss: 0.1940 | max_backward: 1.8810 | max_optimizer_step: 0.2519 | max_tensorboard_logging: 0.5719 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5602.7 | uss_MB: 4330.1 | child_rss_MB: 6421.1 | allocated_MB: 498.6 | reserved_MB: 29580.0 | inactive_split_MB: 773.5 +rank2 [2026-10-29 15:56:52,523] INFO : Step 31400/50000 rank2 | Train Loss: 0.9019 | Train Loss Unweighted: 0.4258 | Train Loss Components: Auxiliary: 0.1823 | Cardinality: 0.1101 | Classification: 0.0746 | Classification_binary: 0.1337 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1070 | Regression_eta: 0.0014 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1134 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0503 | Regression_eta: 0.0557 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0599 | LR: 3.74e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2770s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8478 | max_loss: 0.2460 | max_backward: 2.0403 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.4770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5595.4 | uss_MB: 4331.0 | child_rss_MB: 6422.8 | allocated_MB: 495.2 | reserved_MB: 29580.0 | inactive_split_MB: 986.6 +rank2 [2026-10-29 16:01:34,249] INFO : Step 31500/50000 rank2 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4434 | Train Loss Components: Auxiliary: 0.1867 | Cardinality: 0.1036 | Classification: 0.0834 | Classification_binary: 0.1437 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1064 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0722 | Regression_energy: 0.0588 | Regression_eta: 0.0366 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0559 | LR: 3.71e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.7508s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7930 | max_loss: 0.1458 | max_backward: 1.7855 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 2.2800 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5572.8 | uss_MB: 4310.0 | child_rss_MB: 6426.4 | allocated_MB: 496.8 | reserved_MB: 29580.0 | inactive_split_MB: 850.8 +rank2 [2026-10-29 16:06:05,476] INFO : Step 31600/50000 rank2 | Train Loss: 1.0288 | Train Loss Unweighted: 0.4730 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1317 | Classification: 0.0906 | Classification_binary: 0.1743 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1012 | Regression_eta: 0.0017 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1737 | Regression_cos_phi: 0.1011 | Regression_energy: 0.0569 | Regression_eta: 0.0448 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0534 | LR: 3.69e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1946s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1021 | max_forward: 0.8041 | max_loss: 0.5000 | max_backward: 1.8881 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0133 | Memory: rss_MB: 5567.6 | uss_MB: 4330.2 | child_rss_MB: 6421.3 | allocated_MB: 494.5 | reserved_MB: 29580.0 | inactive_split_MB: 850.5 +rank2 [2026-10-29 16:10:48,123] INFO : Step 31700/50000 rank2 | Train Loss: 0.9206 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1106 | Classification: 0.0729 | Classification_binary: 0.1465 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0303 | Classification_binary: 0.1407 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0662 | Regression_eta: 0.0324 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0461 | LR: 3.66e-04 | DataLoad Time: 0.0179s | Model Forward Time: 3.7510s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0062 | max_forward: 0.7097 | max_loss: 0.1806 | max_backward: 1.6117 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0100 | Memory: rss_MB: 5567.0 | uss_MB: 4328.6 | child_rss_MB: 6424.1 | allocated_MB: 495.3 | reserved_MB: 29580.0 | inactive_split_MB: 969.6 +rank2 [2026-10-29 16:15:56,265] INFO : Step 31800/50000 rank2 | Train Loss: 0.9090 | Train Loss Unweighted: 0.4310 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1140 | Classification: 0.0748 | Classification_binary: 0.1332 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1094 | Regression_eta: 0.0013 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1132 | Regression_cos_phi: 0.0406 | Regression_energy: 0.0492 | Regression_eta: 0.0315 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0570 | LR: 3.64e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.8746s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7102 | max_loss: 0.1436 | max_backward: 1.6093 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5562.0 | uss_MB: 4326.9 | child_rss_MB: 6424.4 | allocated_MB: 499.4 | reserved_MB: 29584.0 | inactive_split_MB: 1068.5 +rank2 [2026-10-29 16:20:39,711] INFO : Step 31900/50000 rank2 | Train Loss: 0.9919 | Train Loss Unweighted: 0.4645 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1269 | Classification: 0.0911 | Classification_binary: 0.1745 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0975 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1712 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0526 | Regression_eta: 0.0574 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0535 | LR: 3.61e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3408s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.2538 | max_backward: 1.9768 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.5597 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5568.8 | uss_MB: 4333.1 | child_rss_MB: 6433.0 | allocated_MB: 496.2 | reserved_MB: 29584.0 | inactive_split_MB: 829.8 +rank2 [2026-10-29 16:25:21,506] INFO : Step 32000/50000 rank2 | Train Loss: 0.9662 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1172 | Classification: 0.0861 | Classification_binary: 0.1650 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1010 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0529 | Regression_eta: 0.0401 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0835 | LR: 3.58e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2274s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7801 | max_loss: 0.1534 | max_backward: 1.8397 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5561.3 | uss_MB: 4329.3 | child_rss_MB: 6422.4 | allocated_MB: 497.7 | reserved_MB: 29584.0 | inactive_split_MB: 927.6 +rank2 [2026-10-29 16:25:22,457] INFO : Running validation on rank2 +rank2 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 16:26:16,177] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:26:16,710] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 16:26:42,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 16:26:43,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 16:26:44,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 16:26:45,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 16:26:55,977] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 16:26:56,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:26:57,009] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 16:27:13,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 16:27:14,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 16:27:15,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 16:27:15,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 16:27:25,964] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 16:27:26,913] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 16:27:27,382] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 16:27:43,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 16:27:44,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 16:27:44,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 16:27:45,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 16:27:55,777] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 16:32:59,848] INFO : Step 32100/50000 rank2 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4572 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1125 | Classification: 0.0829 | Classification_binary: 0.1389 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1159 | Regression_eta: 0.0016 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1269 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0577 | Regression_eta: 0.0540 | Regression_pt: 0.0624 | Regression_sin_phi: 0.1023 | LR: 3.56e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8028s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0431 | max_forward: 0.7694 | max_loss: 0.1379 | max_backward: 1.5662 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 1.3675 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6985.8 | uss_MB: 5757.8 | child_rss_MB: 6447.8 | allocated_MB: 499.0 | reserved_MB: 25862.0 | inactive_split_MB: 892.0 +rank2 [2026-10-29 16:37:33,176] INFO : Step 32200/50000 rank2 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1193 | Classification: 0.0958 | Classification_binary: 0.1508 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1092 | Regression_eta: 0.0016 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0496 | Regression_eta: 0.0420 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0589 | LR: 3.53e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.1399s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8207 | max_loss: 0.1504 | max_backward: 2.0613 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.9148 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6989.2 | uss_MB: 5763.2 | child_rss_MB: 6457.2 | allocated_MB: 499.4 | reserved_MB: 25864.0 | inactive_split_MB: 778.6 +rank2 [2026-10-29 16:42:04,508] INFO : Step 32300/50000 rank2 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4909 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1295 | Classification: 0.0875 | Classification_binary: 0.1834 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0017 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1955 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0583 | Regression_eta: 0.0388 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0628 | LR: 3.51e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7852s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8382 | max_loss: 0.2412 | max_backward: 2.0637 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.9540 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5664.5 | uss_MB: 4450.0 | child_rss_MB: 6462.2 | allocated_MB: 499.0 | reserved_MB: 27716.0 | inactive_split_MB: 775.0 +rank2 [2026-10-29 16:46:37,461] INFO : Step 32400/50000 rank2 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1089 | Classification: 0.0860 | Classification_binary: 0.1494 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1059 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.1349 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0674 | Regression_eta: 0.0441 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0679 | LR: 3.48e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5412s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1137 | max_forward: 0.8338 | max_loss: 0.1463 | max_backward: 1.8285 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 0.8970 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5664.1 | uss_MB: 4449.3 | child_rss_MB: 6459.6 | allocated_MB: 497.5 | reserved_MB: 27716.0 | inactive_split_MB: 964.6 +rank2 [2026-10-29 16:51:11,722] INFO : Step 32500/50000 rank2 | Train Loss: 0.9126 | Train Loss Unweighted: 0.4568 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.0989 | Classification: 0.0980 | Classification_binary: 0.1302 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1128 | Regression_eta: 0.0017 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.0946 | Regression_cos_phi: 0.0278 | Regression_energy: 0.0602 | Regression_eta: 0.0225 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0377 | LR: 3.46e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0955s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7619 | max_loss: 0.1497 | max_backward: 1.5269 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.7665 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5643.1 | uss_MB: 4429.2 | child_rss_MB: 6454.9 | allocated_MB: 495.6 | reserved_MB: 27718.0 | inactive_split_MB: 1084.2 +rank2 [2026-10-29 16:55:48,603] INFO : Step 32600/50000 rank2 | Train Loss: 0.9821 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2025 | Cardinality: 0.1270 | Classification: 0.0917 | Classification_binary: 0.1687 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1000 | Regression_eta: 0.0014 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1848 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0554 | Regression_eta: 0.0409 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0839 | LR: 3.43e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2681s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1505 | max_backward: 1.8823 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 5658.5 | uss_MB: 4448.5 | child_rss_MB: 6453.0 | allocated_MB: 496.5 | reserved_MB: 27844.0 | inactive_split_MB: 835.5 +rank2 [2026-10-29 17:00:16,963] INFO : Step 32700/50000 rank2 | Train Loss: 1.0218 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2103 | Cardinality: 0.1308 | Classification: 0.0908 | Classification_binary: 0.1761 | HitGrouping: 0.1579 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0585 | Regression_eta: 0.0397 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0517 | LR: 3.41e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2028s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7654 | max_loss: 0.2523 | max_backward: 1.8639 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4904 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.1101 | Memory: rss_MB: 5657.8 | uss_MB: 4448.3 | child_rss_MB: 6459.1 | allocated_MB: 495.8 | reserved_MB: 27844.0 | inactive_split_MB: 1177.8 +rank2 [2026-10-29 17:04:50,646] INFO : Step 32800/50000 rank2 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4798 | Train Loss Components: Auxiliary: 0.1977 | Cardinality: 0.1139 | Classification: 0.0957 | Classification_binary: 0.1480 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1163 | Regression_eta: 0.0019 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1397 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0610 | Regression_eta: 0.0406 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0587 | LR: 3.38e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.0495s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7437 | max_loss: 0.1443 | max_backward: 1.7603 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.5989 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5618.6 | uss_MB: 4447.1 | child_rss_MB: 6328.3 | allocated_MB: 497.6 | reserved_MB: 27846.0 | inactive_split_MB: 1089.8 +rank2 [2026-10-29 17:09:25,011] INFO : Step 32900/50000 rank2 | Train Loss: 1.0585 | Train Loss Unweighted: 0.4917 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1472 | Classification: 0.0991 | Classification_binary: 0.1566 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0020 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1493 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0631 | Regression_eta: 0.0450 | Regression_pt: 0.0684 | Regression_sin_phi: 0.0611 | LR: 3.36e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2356s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0141 | max_forward: 0.7853 | max_loss: 0.1442 | max_backward: 1.9716 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.5450 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5622.7 | uss_MB: 4451.0 | child_rss_MB: 6346.8 | allocated_MB: 498.5 | reserved_MB: 27846.0 | inactive_split_MB: 844.8 +rank2 [2026-10-29 17:13:57,543] INFO : Step 33000/50000 rank2 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4896 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1109 | Classification: 0.0902 | Classification_binary: 0.1757 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1096 | Regression_eta: 0.0015 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0624 | Regression_eta: 0.0305 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0464 | LR: 3.33e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1381s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0372 | max_forward: 0.7798 | max_loss: 0.1504 | max_backward: 1.8342 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5614.6 | uss_MB: 4448.7 | child_rss_MB: 6348.5 | allocated_MB: 497.7 | reserved_MB: 27846.0 | inactive_split_MB: 857.6 +rank2 [2026-10-29 17:13:58,425] INFO : Running validation on rank2 +rank2 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 17:14:49,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:14:50,069] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 17:15:17,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 17:15:18,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 17:15:19,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 17:15:20,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 17:15:30,321] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 17:15:31,022] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:15:31,507] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 17:15:47,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 17:15:48,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 17:15:49,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 17:15:50,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 17:16:00,274] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 17:16:01,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 17:16:01,836] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 17:16:17,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 17:16:18,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 17:16:19,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 17:16:19,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 17:16:29,890] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 17:21:40,219] INFO : Step 33100/50000 rank2 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4666 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1297 | Classification: 0.0842 | Classification_binary: 0.1537 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1125 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1574 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0675 | Regression_eta: 0.0404 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0884 | LR: 3.31e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1417s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7607 | max_loss: 0.1464 | max_backward: 1.8352 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6888.0 | uss_MB: 5749.0 | child_rss_MB: 6329.7 | allocated_MB: 496.1 | reserved_MB: 24572.0 | inactive_split_MB: 941.9 +rank2 [2026-10-29 17:26:33,026] INFO : Step 33200/50000 rank2 | Train Loss: 0.9345 | Train Loss Unweighted: 0.4417 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.1121 | Classification: 0.0825 | Classification_binary: 0.1333 | HitGrouping: 0.1549 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0017 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1314 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0488 | Regression_eta: 0.0388 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0607 | LR: 3.28e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.2167s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0447 | max_forward: 0.7619 | max_loss: 0.4589 | max_backward: 1.9181 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.4922 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6887.2 | uss_MB: 5750.0 | child_rss_MB: 6329.3 | allocated_MB: 497.6 | reserved_MB: 24578.0 | inactive_split_MB: 835.2 +rank2 [2026-10-29 17:31:31,554] INFO : Step 33300/50000 rank2 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1270 | Classification: 0.0780 | Classification_binary: 0.1518 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0985 | Regression_eta: 0.0012 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1557 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0566 | Regression_eta: 0.0387 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0656 | LR: 3.26e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.6822s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0334 | max_forward: 0.7851 | max_loss: 0.1434 | max_backward: 2.0284 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.7975 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5654.9 | uss_MB: 4170.1 | child_rss_MB: 6324.3 | allocated_MB: 496.9 | reserved_MB: 24578.0 | inactive_split_MB: 903.1 +rank2 [2026-10-29 17:36:22,673] INFO : Step 33400/50000 rank2 | Train Loss: 0.9537 | Train Loss Unweighted: 0.4634 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1239 | Classification: 0.0872 | Classification_binary: 0.1563 | HitGrouping: 0.1500 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0013 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1309 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0570 | Regression_eta: 0.0273 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0514 | LR: 3.23e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2713s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7951 | max_loss: 0.2522 | max_backward: 1.9349 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0134 | Memory: rss_MB: 5664.7 | uss_MB: 4533.0 | child_rss_MB: 6325.6 | allocated_MB: 500.7 | reserved_MB: 26578.0 | inactive_split_MB: 843.2 +rank2 [2026-10-29 17:41:05,304] INFO : Step 33500/50000 rank2 | Train Loss: 0.9039 | Train Loss Unweighted: 0.4494 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.1057 | Classification: 0.0795 | Classification_binary: 0.1278 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0011 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1077 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0745 | Regression_eta: 0.0408 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0557 | LR: 3.21e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1840s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0177 | max_forward: 0.7696 | max_loss: 0.1421 | max_backward: 1.9122 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 0.5562 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5654.7 | uss_MB: 4519.8 | child_rss_MB: 6324.9 | allocated_MB: 494.9 | reserved_MB: 26578.0 | inactive_split_MB: 839.3 +rank2 [2026-10-29 17:45:56,711] INFO : Step 33600/50000 rank2 | Train Loss: 0.9179 | Train Loss Unweighted: 0.4415 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.0997 | Classification: 0.0812 | Classification_binary: 0.1440 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1377 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0482 | Regression_eta: 0.0233 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0533 | LR: 3.19e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4036s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7783 | max_loss: 0.2418 | max_backward: 2.0392 | max_optimizer_step: 0.2787 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5667.7 | uss_MB: 4535.9 | child_rss_MB: 6324.1 | allocated_MB: 497.9 | reserved_MB: 28594.0 | inactive_split_MB: 1127.0 +rank2 [2026-10-29 17:50:43,456] INFO : Step 33700/50000 rank2 | Train Loss: 1.0564 | Train Loss Unweighted: 0.4962 | Train Loss Components: Auxiliary: 0.2235 | Cardinality: 0.1517 | Classification: 0.0917 | Classification_binary: 0.1786 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0638 | Regression_eta: 0.0493 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0566 | LR: 3.16e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.1479s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0121 | max_forward: 0.7981 | max_loss: 0.1554 | max_backward: 1.9054 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5667.6 | uss_MB: 4535.1 | child_rss_MB: 6300.3 | allocated_MB: 501.8 | reserved_MB: 28594.0 | inactive_split_MB: 1103.2 +rank2 [2026-10-29 17:55:24,637] INFO : Step 33800/50000 rank2 | Train Loss: 0.9305 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1894 | Cardinality: 0.1080 | Classification: 0.0751 | Classification_binary: 0.1369 | HitGrouping: 0.1527 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1201 | Regression_eta: 0.0014 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1302 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0697 | Regression_eta: 0.0380 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0649 | LR: 3.14e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1057s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8033 | max_loss: 0.1441 | max_backward: 1.7197 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0176 | Memory: rss_MB: 5662.9 | uss_MB: 4531.8 | child_rss_MB: 6320.7 | allocated_MB: 496.9 | reserved_MB: 28594.0 | inactive_split_MB: 909.5 +rank2 [2026-10-29 18:00:01,587] INFO : Step 33900/50000 rank2 | Train Loss: 0.9360 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.1942 | Cardinality: 0.1035 | Classification: 0.0805 | Classification_binary: 0.1361 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1124 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0578 | Regression_eta: 0.0501 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0544 | LR: 3.11e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2979s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7557 | max_loss: 0.2439 | max_backward: 1.8923 | max_optimizer_step: 0.2623 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5665.6 | uss_MB: 4533.9 | child_rss_MB: 6306.4 | allocated_MB: 496.5 | reserved_MB: 28598.0 | inactive_split_MB: 941.5 +rank2 [2026-10-29 18:04:39,054] INFO : Step 34000/50000 rank2 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0930 | Classification: 0.0875 | Classification_binary: 0.1606 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1779 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0509 | Regression_eta: 0.0290 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0520 | LR: 3.09e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3502s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0943 | max_forward: 0.8289 | max_loss: 0.4616 | max_backward: 2.0219 | max_optimizer_step: 0.1767 | max_tensorboard_logging: 0.4773 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5666.0 | uss_MB: 4534.2 | child_rss_MB: 6326.4 | allocated_MB: 496.0 | reserved_MB: 28598.0 | inactive_split_MB: 1123.0 +rank2 [2026-10-29 18:04:39,968] INFO : Running validation on rank2 +rank2 [2026-10-29 18:04:39,969] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 18:04:39,969] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 18:05:30,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:05:31,101] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 18:05:54,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 18:05:55,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 18:05:56,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 18:05:57,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 18:06:07,861] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 18:06:08,708] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:06:09,217] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 18:06:24,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 18:06:25,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 18:06:26,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 18:06:27,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 18:06:37,942] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 18:06:39,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:06:39,483] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 18:06:54,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 18:06:55,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 18:06:56,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 18:06:57,320] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 18:07:07,331] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 18:12:12,108] INFO : Step 34100/50000 rank2 | Train Loss: 1.0240 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2153 | Cardinality: 0.1202 | Classification: 0.0979 | Classification_binary: 0.1668 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1210 | Regression_eta: 0.0013 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1738 | Regression_cos_phi: 0.1032 | Regression_energy: 0.0652 | Regression_eta: 0.0407 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0665 | LR: 3.06e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5017s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0078 | max_forward: 0.9085 | max_loss: 0.1531 | max_backward: 2.1727 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.4680 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6900.4 | uss_MB: 5761.6 | child_rss_MB: 6326.4 | allocated_MB: 496.4 | reserved_MB: 24560.0 | inactive_split_MB: 1172.2 +rank2 [2026-10-29 18:17:11,667] INFO : Step 34200/50000 rank2 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4225 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.1030 | Classification: 0.0694 | Classification_binary: 0.1279 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0016 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1190 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0752 | Regression_eta: 0.0305 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0316 | LR: 3.04e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.8742s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 1.1401 | max_loss: 1.0887 | max_backward: 3.3565 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5402 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6897.3 | uss_MB: 5759.3 | child_rss_MB: 6328.4 | allocated_MB: 493.1 | reserved_MB: 27668.0 | inactive_split_MB: 842.0 +rank2 [2026-10-29 18:21:53,344] INFO : Step 34300/50000 rank2 | Train Loss: 0.8999 | Train Loss Unweighted: 0.4291 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.1046 | Classification: 0.0782 | Classification_binary: 0.1421 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0015 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1310 | Regression_cos_phi: 0.0392 | Regression_energy: 0.0509 | Regression_eta: 0.0403 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0466 | LR: 3.02e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.5633s | Phase Times: max_pre_step_barrier: 0.1089 | max_batch_to_device: 0.0070 | max_forward: 0.7738 | max_loss: 0.4754 | max_backward: 1.8888 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5721.4 | uss_MB: 4584.0 | child_rss_MB: 6315.8 | allocated_MB: 498.9 | reserved_MB: 29668.0 | inactive_split_MB: 777.1 +rank2 [2026-10-29 18:26:42,331] INFO : Step 34400/50000 rank2 | Train Loss: 0.9382 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2026 | Cardinality: 0.1181 | Classification: 0.0846 | Classification_binary: 0.1681 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0444 | Regression_eta: 0.0256 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0416 | LR: 2.99e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6580s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.8411 | max_loss: 0.2739 | max_backward: 2.1385 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5734.9 | uss_MB: 4598.3 | child_rss_MB: 6315.2 | allocated_MB: 496.8 | reserved_MB: 29668.0 | inactive_split_MB: 858.8 +rank2 [2026-10-29 18:31:40,822] INFO : Step 34500/50000 rank2 | Train Loss: 0.9189 | Train Loss Unweighted: 0.4456 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1063 | Classification: 0.0767 | Classification_binary: 0.1421 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1352 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0603 | Regression_eta: 0.0347 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0534 | LR: 2.97e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2127s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.7430 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5727.7 | uss_MB: 4595.8 | child_rss_MB: 6313.8 | allocated_MB: 494.3 | reserved_MB: 29668.0 | inactive_split_MB: 926.0 +rank2 [2026-10-29 18:36:27,181] INFO : Step 34600/50000 rank2 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4340 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1046 | Classification: 0.0794 | Classification_binary: 0.1340 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0016 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0553 | Regression_eta: 0.0399 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0912 | LR: 2.95e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.6955s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0170 | max_forward: 0.9895 | max_loss: 0.4622 | max_backward: 2.2541 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5731.7 | uss_MB: 4597.0 | child_rss_MB: 6311.8 | allocated_MB: 498.6 | reserved_MB: 29672.0 | inactive_split_MB: 765.1 +rank2 [2026-10-29 18:40:58,826] INFO : Step 34700/50000 rank2 | Train Loss: 0.9586 | Train Loss Unweighted: 0.4539 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1185 | Classification: 0.0862 | Classification_binary: 0.1624 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1721 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0415 | Regression_eta: 0.0350 | Regression_pt: 0.0439 | Regression_sin_phi: 0.0845 | LR: 2.92e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3671s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7862 | max_loss: 0.2628 | max_backward: 1.9231 | max_optimizer_step: 0.2860 | max_tensorboard_logging: 0.4599 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5730.7 | uss_MB: 4596.4 | child_rss_MB: 6303.0 | allocated_MB: 499.0 | reserved_MB: 29672.0 | inactive_split_MB: 1074.3 +rank2 [2026-10-29 18:45:38,652] INFO : Step 34800/50000 rank2 | Train Loss: 0.9346 | Train Loss Unweighted: 0.4373 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1276 | Classification: 0.0867 | Classification_binary: 0.1551 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0959 | Regression_eta: 0.0013 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1740 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0606 | Regression_eta: 0.0408 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0652 | LR: 2.90e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2173s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0770 | max_forward: 0.8404 | max_loss: 0.1556 | max_backward: 1.8035 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.5351 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5727.0 | uss_MB: 4594.2 | child_rss_MB: 6312.4 | allocated_MB: 497.4 | reserved_MB: 29672.0 | inactive_split_MB: 1113.6 +rank2 [2026-10-29 18:50:17,936] INFO : Step 34900/50000 rank2 | Train Loss: 0.8719 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1154 | Classification: 0.0681 | Classification_binary: 0.1220 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1093 | Regression_eta: 0.0013 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1139 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0574 | Regression_eta: 0.0379 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0590 | LR: 2.88e-04 | DataLoad Time: 0.0198s | Model Forward Time: 6.1447s | Phase Times: max_pre_step_barrier: 0.0080 | max_batch_to_device: 0.0693 | max_forward: 1.2127 | max_loss: 0.2834 | max_backward: 3.0996 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.6125 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 5724.6 | uss_MB: 4576.6 | child_rss_MB: 6292.2 | allocated_MB: 495.3 | reserved_MB: 29672.0 | inactive_split_MB: 864.7 +rank2 [2026-10-29 18:55:19,194] INFO : Step 35000/50000 rank2 | Train Loss: 0.9157 | Train Loss Unweighted: 0.4220 | Train Loss Components: Auxiliary: 0.1897 | Cardinality: 0.1230 | Classification: 0.0771 | Classification_binary: 0.1455 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0980 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1429 | Regression_cos_phi: 0.0328 | Regression_energy: 0.0471 | Regression_eta: 0.0433 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0385 | LR: 2.85e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5129s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.8243 | max_loss: 0.4759 | max_backward: 2.1181 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5728.2 | uss_MB: 4598.3 | child_rss_MB: 6307.6 | allocated_MB: 498.3 | reserved_MB: 29672.0 | inactive_split_MB: 766.0 +rank2 [2026-10-29 18:55:20,127] INFO : Running validation on rank2 +rank2 [2026-10-29 18:55:20,127] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 18:55:20,127] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 18:56:11,603] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:56:12,116] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 18:56:36,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 18:56:37,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 18:56:38,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 18:56:39,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 18:56:49,687] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 18:56:49,876] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:56:50,324] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 18:57:07,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 18:57:08,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 18:57:09,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 18:57:10,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 18:57:20,427] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-29 18:57:20,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 18:57:20,900] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 18:57:36,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 18:57:36,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 18:57:37,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 18:57:38,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 18:57:48,585] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 19:02:56,670] INFO : Step 35100/50000 rank2 | Train Loss: 0.9622 | Train Loss Unweighted: 0.4512 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1150 | Classification: 0.0857 | Classification_binary: 0.1724 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0948 | Regression_eta: 0.0014 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1937 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0477 | Regression_eta: 0.0380 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0587 | LR: 2.83e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8770s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1018 | max_forward: 0.8921 | max_loss: 0.1518 | max_backward: 2.0328 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8633 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5815.2 | uss_MB: 4686.8 | child_rss_MB: 6305.0 | allocated_MB: 499.4 | reserved_MB: 23366.0 | inactive_split_MB: 1243.8 +rank2 [2026-10-29 19:07:43,277] INFO : Step 35200/50000 rank2 | Train Loss: 0.9331 | Train Loss Unweighted: 0.4385 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.1234 | Classification: 0.0814 | Classification_binary: 0.1441 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1296 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0513 | Regression_eta: 0.0382 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0679 | LR: 2.81e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1466s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0577 | max_forward: 0.7693 | max_loss: 0.1423 | max_backward: 1.7209 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5822.3 | uss_MB: 4693.9 | child_rss_MB: 6304.7 | allocated_MB: 494.4 | reserved_MB: 25346.0 | inactive_split_MB: 1054.2 +rank2 [2026-10-29 19:12:30,668] INFO : Step 35300/50000 rank2 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4075 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1105 | Classification: 0.0819 | Classification_binary: 0.1245 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0989 | Regression_eta: 0.0014 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1339 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0450 | Regression_eta: 0.0296 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0366 | LR: 2.79e-04 | DataLoad Time: 0.0183s | Model Forward Time: 6.8651s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 1.2856 | max_loss: 0.4079 | max_backward: 4.4969 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5749 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5814.5 | uss_MB: 4685.2 | child_rss_MB: 6307.8 | allocated_MB: 498.3 | reserved_MB: 25346.0 | inactive_split_MB: 1140.5 +rank2 [2026-10-29 19:17:14,809] INFO : Step 35400/50000 rank2 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4888 | Train Loss Components: Auxiliary: 0.2130 | Cardinality: 0.1296 | Classification: 0.0908 | Classification_binary: 0.1718 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0014 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2157 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0561 | Regression_eta: 0.0429 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0708 | LR: 2.76e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3447s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8312 | max_loss: 0.1490 | max_backward: 2.0287 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.4708 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5820.8 | uss_MB: 4696.7 | child_rss_MB: 6308.5 | allocated_MB: 497.8 | reserved_MB: 25346.0 | inactive_split_MB: 1132.0 +rank2 [2026-10-29 19:21:58,249] INFO : Step 35500/50000 rank2 | Train Loss: 0.9444 | Train Loss Unweighted: 0.4557 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1201 | Classification: 0.0924 | Classification_binary: 0.1605 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0498 | Regression_eta: 0.0383 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0454 | LR: 2.74e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.3607s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7670 | max_loss: 0.2318 | max_backward: 1.8405 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5820.9 | uss_MB: 4694.9 | child_rss_MB: 6312.3 | allocated_MB: 497.5 | reserved_MB: 26942.0 | inactive_split_MB: 1130.5 +rank2 [2026-10-29 19:26:41,938] INFO : Step 35600/50000 rank2 | Train Loss: 0.9280 | Train Loss Unweighted: 0.4451 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1095 | Classification: 0.0861 | Classification_binary: 0.1334 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.1521 | Regression_cos_phi: 0.0798 | Regression_energy: 0.0625 | Regression_eta: 0.0376 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0419 | LR: 2.72e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.7388s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7460 | max_loss: 0.1440 | max_backward: 1.8206 | max_optimizer_step: 0.2911 | max_tensorboard_logging: 0.6036 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 5820.5 | uss_MB: 3919.7 | child_rss_MB: 6316.0 | allocated_MB: 496.1 | reserved_MB: 26942.0 | inactive_split_MB: 907.6 +rank2 [2026-10-29 19:31:29,665] INFO : Step 35700/50000 rank2 | Train Loss: 0.9116 | Train Loss Unweighted: 0.4301 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1148 | Classification: 0.0829 | Classification_binary: 0.1432 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0998 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0380 | Regression_eta: 0.0292 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0400 | LR: 2.70e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6320s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8312 | max_loss: 0.2568 | max_backward: 2.2622 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5823.2 | uss_MB: 4699.5 | child_rss_MB: 6323.6 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1241.0 +rank2 [2026-10-29 19:36:11,896] INFO : Step 35800/50000 rank2 | Train Loss: 1.0095 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1252 | Classification: 0.0857 | Classification_binary: 0.1786 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1066 | Regression_eta: 0.0014 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0550 | Regression_eta: 0.0342 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0467 | LR: 2.68e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1744s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7644 | max_loss: 0.5090 | max_backward: 1.8855 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4861 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5816.8 | uss_MB: 4694.8 | child_rss_MB: 6319.7 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1129.0 +rank2 [2026-10-29 19:40:48,762] INFO : Step 35900/50000 rank2 | Train Loss: 0.9596 | Train Loss Unweighted: 0.4571 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1173 | Classification: 0.0913 | Classification_binary: 0.1468 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0014 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1541 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0670 | Regression_eta: 0.0541 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0808 | LR: 2.65e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.0315s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7217 | max_loss: 0.1443 | max_backward: 1.6894 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.4586 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5816.9 | uss_MB: 4695.3 | child_rss_MB: 6319.9 | allocated_MB: 495.4 | reserved_MB: 29024.0 | inactive_split_MB: 934.6 +rank2 [2026-10-29 19:45:19,091] INFO : Step 36000/50000 rank2 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0912 | Classification: 0.0938 | Classification_binary: 0.1314 | HitGrouping: 0.1587 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1135 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1053 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0577 | Regression_eta: 0.0329 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0580 | LR: 2.63e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2993s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0073 | max_forward: 0.7630 | max_loss: 0.4769 | max_backward: 1.8869 | max_optimizer_step: 0.2587 | max_tensorboard_logging: 0.5842 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5815.2 | uss_MB: 4695.9 | child_rss_MB: 6324.3 | allocated_MB: 495.0 | reserved_MB: 29024.0 | inactive_split_MB: 937.5 +rank2 [2026-10-29 19:45:19,981] INFO : Running validation on rank2 +rank2 [2026-10-29 19:45:19,981] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 19:45:19,981] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 19:46:10,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:46:11,093] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 19:46:35,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 19:46:36,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 19:46:37,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 19:46:38,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 19:46:48,965] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 19:46:49,454] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:46:49,932] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 19:47:05,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 19:47:06,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 19:47:07,696] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 19:47:08,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 19:47:18,676] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 19:47:19,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 19:47:19,885] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 19:47:35,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 19:47:36,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 19:47:37,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 19:47:38,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 19:47:48,165] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 19:53:12,007] INFO : Step 36100/50000 rank2 | Train Loss: 0.9292 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1100 | Classification: 0.0906 | Classification_binary: 0.1490 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1309 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0621 | Regression_eta: 0.0365 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0367 | LR: 2.61e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7842s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7768 | max_loss: 0.2455 | max_backward: 1.8077 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 1.0653 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5880.7 | uss_MB: 4756.1 | child_rss_MB: 6293.3 | allocated_MB: 499.2 | reserved_MB: 26012.0 | inactive_split_MB: 1080.2 +rank2 [2026-10-29 19:58:21,470] INFO : Step 36200/50000 rank2 | Train Loss: 0.9030 | Train Loss Unweighted: 0.4308 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1163 | Classification: 0.0762 | Classification_binary: 0.1596 | HitGrouping: 0.1506 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0473 | Regression_eta: 0.0282 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0565 | LR: 2.59e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5363s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0777 | max_forward: 0.8125 | max_loss: 0.1541 | max_backward: 1.8492 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 5884.8 | uss_MB: 4767.2 | child_rss_MB: 6290.6 | allocated_MB: 494.4 | reserved_MB: 26016.0 | inactive_split_MB: 1377.4 +rank2 [2026-10-29 20:03:20,658] INFO : Step 36300/50000 rank2 | Train Loss: 0.9576 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1236 | Classification: 0.0876 | Classification_binary: 0.1347 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1100 | Regression_eta: 0.0017 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0542 | Regression_eta: 0.0355 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0786 | LR: 2.57e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.4520s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.8065 | max_loss: 0.1484 | max_backward: 1.9315 | max_optimizer_step: 0.1930 | max_tensorboard_logging: 0.6801 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5878.7 | uss_MB: 4768.0 | child_rss_MB: 6273.7 | allocated_MB: 496.0 | reserved_MB: 28008.0 | inactive_split_MB: 1156.5 +rank2 [2026-10-29 20:08:10,865] INFO : Step 36400/50000 rank2 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4147 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.0999 | Classification: 0.0796 | Classification_binary: 0.1372 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0562 | Regression_eta: 0.0275 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0666 | LR: 2.55e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.8724s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0192 | max_forward: 1.1829 | max_loss: 0.3790 | max_backward: 3.0917 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 5880.4 | uss_MB: 4768.9 | child_rss_MB: 6270.8 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 845.2 +rank2 [2026-10-29 20:12:41,929] INFO : Step 36500/50000 rank2 | Train Loss: 0.9762 | Train Loss Unweighted: 0.4543 | Train Loss Components: Auxiliary: 0.2088 | Cardinality: 0.1239 | Classification: 0.0805 | Classification_binary: 0.1682 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0514 | Regression_eta: 0.0400 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0607 | LR: 2.52e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2747s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7782 | max_loss: 0.1554 | max_backward: 1.9378 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.5103 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5877.6 | uss_MB: 4769.9 | child_rss_MB: 6273.0 | allocated_MB: 499.0 | reserved_MB: 30008.0 | inactive_split_MB: 913.8 +rank2 [2026-10-29 20:17:47,229] INFO : Step 36600/50000 rank2 | Train Loss: 0.8699 | Train Loss Unweighted: 0.4190 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.1138 | Classification: 0.0734 | Classification_binary: 0.1317 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0010 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1368 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0560 | Regression_eta: 0.0420 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0563 | LR: 2.50e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2969s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7561 | max_loss: 0.1454 | max_backward: 1.9017 | max_optimizer_step: 0.5115 | max_tensorboard_logging: 0.5672 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5872.6 | uss_MB: 4766.8 | child_rss_MB: 6277.6 | allocated_MB: 494.4 | reserved_MB: 30008.0 | inactive_split_MB: 856.0 +rank2 [2026-10-29 20:22:56,451] INFO : Step 36700/50000 rank2 | Train Loss: 0.9065 | Train Loss Unweighted: 0.4327 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.1102 | Classification: 0.0849 | Classification_binary: 0.1262 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1087 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1082 | Regression_cos_phi: 0.0372 | Regression_energy: 0.0634 | Regression_eta: 0.0374 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0556 | LR: 2.48e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.0799s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.7169 | max_loss: 0.1507 | max_backward: 1.7104 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5871.4 | uss_MB: 4766.4 | child_rss_MB: 6270.7 | allocated_MB: 496.5 | reserved_MB: 30008.0 | inactive_split_MB: 1154.6 +rank2 [2026-10-29 20:28:04,049] INFO : Step 36800/50000 rank2 | Train Loss: 0.9068 | Train Loss Unweighted: 0.4227 | Train Loss Components: Auxiliary: 0.1925 | Cardinality: 0.1100 | Classification: 0.0739 | Classification_binary: 0.1571 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0945 | Regression_eta: 0.0013 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0767 | Regression_energy: 0.0429 | Regression_eta: 0.0330 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0659 | LR: 2.46e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4672s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8487 | max_loss: 0.1525 | max_backward: 2.1121 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5877.1 | uss_MB: 4770.1 | child_rss_MB: 6274.6 | allocated_MB: 499.3 | reserved_MB: 30008.0 | inactive_split_MB: 939.6 +rank2 [2026-10-29 20:33:09,546] INFO : Step 36900/50000 rank2 | Train Loss: 0.8934 | Train Loss Unweighted: 0.4126 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1194 | Classification: 0.0721 | Classification_binary: 0.1507 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0013 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1729 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0504 | Regression_eta: 0.0317 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0627 | LR: 2.44e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.5633s | Phase Times: max_pre_step_barrier: 0.0139 | max_batch_to_device: 0.0068 | max_forward: 1.0975 | max_loss: 0.2241 | max_backward: 2.7278 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.6216 | max_tensorboard_flush: 0.0085 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5871.7 | uss_MB: 4750.7 | child_rss_MB: 6260.8 | allocated_MB: 498.9 | reserved_MB: 30008.0 | inactive_split_MB: 771.1 +rank2 [2026-10-29 20:38:05,960] INFO : Step 37000/50000 rank2 | Train Loss: 0.9040 | Train Loss Unweighted: 0.4182 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1192 | Classification: 0.0807 | Classification_binary: 0.1318 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1725 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0534 | Regression_eta: 0.0390 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0447 | LR: 2.42e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8497s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7055 | max_loss: 0.1505 | max_backward: 1.5664 | max_optimizer_step: 0.1900 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5866.9 | uss_MB: 4763.2 | child_rss_MB: 6273.0 | allocated_MB: 493.9 | reserved_MB: 30008.0 | inactive_split_MB: 1169.8 +rank2 [2026-10-29 20:38:06,931] INFO : Running validation on rank2 +rank2 [2026-10-29 20:38:06,932] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 20:38:06,932] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 20:39:05,979] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:39:06,530] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 20:39:31,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 20:39:32,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 20:39:33,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 20:39:34,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 20:39:44,514] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 20:39:45,043] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:39:45,534] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 20:40:01,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 20:40:02,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 20:40:03,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 20:40:04,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 20:40:15,001] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 20:40:16,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 20:40:16,607] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 20:40:32,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 20:40:32,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 20:40:34,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 20:40:35,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 20:40:45,111] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 20:45:59,178] INFO : Step 37100/50000 rank2 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4614 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1255 | Classification: 0.0999 | Classification_binary: 0.1492 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1041 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1724 | Regression_cos_phi: 0.0390 | Regression_energy: 0.0440 | Regression_eta: 0.0341 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0568 | LR: 2.40e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8452s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0061 | max_forward: 0.8553 | max_loss: 0.1461 | max_backward: 2.1723 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5959.4 | uss_MB: 4852.0 | child_rss_MB: 6311.9 | allocated_MB: 499.3 | reserved_MB: 25668.0 | inactive_split_MB: 852.4 +rank2 [2026-10-29 20:50:39,578] INFO : Step 37200/50000 rank2 | Train Loss: 0.9288 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.1928 | Cardinality: 0.1108 | Classification: 0.0836 | Classification_binary: 0.1643 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0015 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1493 | Regression_cos_phi: 0.0360 | Regression_energy: 0.0475 | Regression_eta: 0.0339 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0445 | LR: 2.38e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6398s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8460 | max_loss: 0.2603 | max_backward: 2.0959 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5966.0 | uss_MB: 4861.5 | child_rss_MB: 6312.1 | allocated_MB: 497.9 | reserved_MB: 25674.0 | inactive_split_MB: 846.0 +rank2 [2026-10-29 20:55:11,586] INFO : Step 37300/50000 rank2 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1275 | Classification: 0.0781 | Classification_binary: 0.1424 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0982 | Regression_eta: 0.0013 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0647 | Regression_eta: 0.0564 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0484 | LR: 2.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0201s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7653 | max_loss: 0.1472 | max_backward: 1.7105 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.4778 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0692 | Memory: rss_MB: 5962.2 | uss_MB: 4859.4 | child_rss_MB: 6302.2 | allocated_MB: 497.3 | reserved_MB: 27616.0 | inactive_split_MB: 974.4 +rank2 [2026-10-29 21:00:09,948] INFO : Step 37400/50000 rank2 | Train Loss: 0.8836 | Train Loss Unweighted: 0.4228 | Train Loss Components: Auxiliary: 0.1825 | Cardinality: 0.0983 | Classification: 0.0851 | Classification_binary: 0.1289 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1025 | Regression_eta: 0.0015 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0547 | Regression_eta: 0.0695 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0606 | LR: 2.34e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5389s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7611 | max_loss: 0.1956 | max_backward: 1.9276 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7505 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5960.0 | uss_MB: 4860.1 | child_rss_MB: 6301.8 | allocated_MB: 495.0 | reserved_MB: 27690.0 | inactive_split_MB: 915.5 +rank2 [2026-10-29 21:05:01,763] INFO : Step 37500/50000 rank2 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1260 | Classification: 0.0804 | Classification_binary: 0.1577 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1005 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0584 | Regression_eta: 0.0381 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0624 | LR: 2.32e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4459s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.9030 | max_loss: 0.1494 | max_backward: 1.9871 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5938.4 | uss_MB: 4859.4 | child_rss_MB: 6224.4 | allocated_MB: 496.1 | reserved_MB: 27690.0 | inactive_split_MB: 923.5 +rank2 [2026-10-29 21:09:54,197] INFO : Step 37600/50000 rank2 | Train Loss: 0.9091 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.0995 | Classification: 0.0683 | Classification_binary: 0.1534 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0957 | Regression_eta: 0.0015 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0529 | Regression_eta: 0.0403 | Regression_pt: 0.0570 | Regression_sin_phi: 0.0464 | LR: 2.30e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3495s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0070 | max_forward: 0.8280 | max_loss: 0.1484 | max_backward: 1.8328 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5933.4 | uss_MB: 4856.7 | child_rss_MB: 6230.2 | allocated_MB: 497.1 | reserved_MB: 27690.0 | inactive_split_MB: 927.5 +rank2 [2026-10-29 21:14:45,920] INFO : Step 37700/50000 rank2 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3995 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.0864 | Classification: 0.0813 | Classification_binary: 0.1231 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0012 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1173 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0448 | Regression_eta: 0.0309 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0440 | LR: 2.28e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4212s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7582 | max_loss: 0.2486 | max_backward: 1.9197 | max_optimizer_step: 0.3005 | max_tensorboard_logging: 0.5767 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5802.8 | uss_MB: 4857.6 | child_rss_MB: 5767.0 | allocated_MB: 495.2 | reserved_MB: 29766.0 | inactive_split_MB: 925.5 +rank2 [2026-10-29 21:19:50,096] INFO : Step 37800/50000 rank2 | Train Loss: 0.8717 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1057 | Classification: 0.0822 | Classification_binary: 0.1363 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0014 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0470 | Regression_eta: 0.0408 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0572 | LR: 2.26e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.6266s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 1.2105 | max_loss: 0.2449 | max_backward: 2.0574 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 5800.5 | uss_MB: 4856.8 | child_rss_MB: 5763.9 | allocated_MB: 498.3 | reserved_MB: 29766.0 | inactive_split_MB: 930.0 +rank2 [2026-10-29 21:24:49,256] INFO : Step 37900/50000 rank2 | Train Loss: 0.9843 | Train Loss Unweighted: 0.4684 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1353 | Classification: 0.0912 | Classification_binary: 0.1705 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0014 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0401 | Regression_eta: 0.0531 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0780 | LR: 2.24e-04 | DataLoad Time: 0.0144s | Model Forward Time: 5.2118s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0297 | max_forward: 0.8081 | max_loss: 0.1547 | max_backward: 2.6708 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5096 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5782.9 | uss_MB: 4859.7 | child_rss_MB: 5732.7 | allocated_MB: 495.6 | reserved_MB: 29766.0 | inactive_split_MB: 912.0 +rank2 [2026-10-29 21:29:44,506] INFO : Step 38000/50000 rank2 | Train Loss: 0.8556 | Train Loss Unweighted: 0.4002 | Train Loss Components: Auxiliary: 0.1833 | Cardinality: 0.1090 | Classification: 0.0730 | Classification_binary: 0.1293 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0972 | Regression_eta: 0.0011 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0877 | Regression_energy: 0.0614 | Regression_eta: 0.0332 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0539 | LR: 2.22e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1664s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7330 | max_loss: 0.2284 | max_backward: 1.8082 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5778.8 | uss_MB: 4857.4 | child_rss_MB: 5727.0 | allocated_MB: 494.5 | reserved_MB: 29766.0 | inactive_split_MB: 847.3 +rank2 [2026-10-29 21:29:45,441] INFO : Running validation on rank2 +rank2 [2026-10-29 21:29:45,441] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 21:29:45,441] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 21:30:41,619] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:30:42,097] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 21:31:09,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 21:31:10,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 21:31:11,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 21:31:12,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 21:31:22,723] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 21:31:23,194] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:31:23,629] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 21:31:39,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 21:31:40,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 21:31:41,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 21:31:42,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 21:31:52,316] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 21:31:52,734] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 21:31:53,235] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 21:32:08,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 21:32:09,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 21:32:10,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 21:32:11,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 21:32:21,313] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 21:37:39,070] INFO : Step 38100/50000 rank2 | Train Loss: 0.9020 | Train Loss Unweighted: 0.4337 | Train Loss Components: Auxiliary: 0.1907 | Cardinality: 0.1231 | Classification: 0.0888 | Classification_binary: 0.1339 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0014 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1538 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0607 | Regression_eta: 0.0444 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0430 | LR: 2.20e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8725s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0963 | max_forward: 0.8445 | max_loss: 0.1406 | max_backward: 1.9197 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.9697 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7028.7 | uss_MB: 6098.3 | child_rss_MB: 5734.5 | allocated_MB: 496.0 | reserved_MB: 27516.0 | inactive_split_MB: 970.5 +rank2 [2026-10-29 21:42:37,687] INFO : Step 38200/50000 rank2 | Train Loss: 0.9174 | Train Loss Unweighted: 0.4351 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.0971 | Classification: 0.0888 | Classification_binary: 0.1526 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0948 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2158 | Regression_cos_phi: 0.1013 | Regression_energy: 0.0672 | Regression_eta: 0.0513 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0714 | LR: 2.18e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.4466s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.8480 | max_loss: 0.2651 | max_backward: 3.1905 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.9151 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5881.2 | uss_MB: 4342.7 | child_rss_MB: 5733.1 | allocated_MB: 499.7 | reserved_MB: 27516.0 | inactive_split_MB: 848.5 +rank2 [2026-10-29 21:47:30,433] INFO : Step 38300/50000 rank2 | Train Loss: 0.9342 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1998 | Cardinality: 0.1099 | Classification: 0.0869 | Classification_binary: 0.1470 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0013 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1551 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0595 | Regression_eta: 0.0317 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0420 | LR: 2.16e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.6855s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0664 | max_forward: 0.8187 | max_loss: 0.1464 | max_backward: 1.8154 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.9517 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5875.2 | uss_MB: 4949.5 | child_rss_MB: 5725.9 | allocated_MB: 495.5 | reserved_MB: 27516.0 | inactive_split_MB: 1085.5 +rank2 [2026-10-29 21:52:14,428] INFO : Step 38400/50000 rank2 | Train Loss: 0.8281 | Train Loss Unweighted: 0.4036 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0915 | Classification: 0.0723 | Classification_binary: 0.1160 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1069 | Regression_eta: 0.0012 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1095 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0506 | Regression_eta: 0.0366 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0286 | LR: 2.14e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.7600s | Phase Times: max_pre_step_barrier: 0.0868 | max_batch_to_device: 0.0062 | max_forward: 0.9937 | max_loss: 0.2500 | max_backward: 1.9690 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5872.2 | uss_MB: 4950.8 | child_rss_MB: 5730.9 | allocated_MB: 496.1 | reserved_MB: 27516.0 | inactive_split_MB: 972.6 +rank2 [2026-10-29 21:57:09,317] INFO : Step 38500/50000 rank2 | Train Loss: 0.9386 | Train Loss Unweighted: 0.4484 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1248 | Classification: 0.1044 | Classification_binary: 0.1389 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1440 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0442 | Regression_eta: 0.0325 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0548 | LR: 2.12e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2029s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.7200 | max_loss: 0.1386 | max_backward: 1.5323 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5868.3 | uss_MB: 4952.0 | child_rss_MB: 5707.9 | allocated_MB: 496.0 | reserved_MB: 27520.0 | inactive_split_MB: 776.0 +rank2 [2026-10-29 22:01:43,604] INFO : Step 38600/50000 rank2 | Train Loss: 0.9227 | Train Loss Unweighted: 0.4436 | Train Loss Components: Auxiliary: 0.1984 | Cardinality: 0.1056 | Classification: 0.0788 | Classification_binary: 0.1660 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0979 | Regression_eta: 0.0013 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0469 | Regression_eta: 0.0443 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0627 | LR: 2.11e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4264s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8785 | max_loss: 0.1473 | max_backward: 2.0276 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5861.9 | uss_MB: 4951.4 | child_rss_MB: 5677.1 | allocated_MB: 499.0 | reserved_MB: 29592.0 | inactive_split_MB: 827.5 +rank2 [2026-10-29 22:06:19,438] INFO : Step 38700/50000 rank2 | Train Loss: 0.8153 | Train Loss Unweighted: 0.3795 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0947 | Classification: 0.0722 | Classification_binary: 0.1256 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0893 | Regression_eta: 0.0013 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1156 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0442 | Regression_eta: 0.0356 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0439 | LR: 2.09e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2103s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7251 | max_loss: 0.2433 | max_backward: 1.8091 | max_optimizer_step: 0.1965 | max_tensorboard_logging: 0.4744 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5858.3 | uss_MB: 4950.6 | child_rss_MB: 5664.7 | allocated_MB: 493.1 | reserved_MB: 29592.0 | inactive_split_MB: 1080.0 +rank2 [2026-10-29 22:11:06,083] INFO : Step 38800/50000 rank2 | Train Loss: 0.8386 | Train Loss Unweighted: 0.4077 | Train Loss Components: Auxiliary: 0.1776 | Cardinality: 0.0905 | Classification: 0.0849 | Classification_binary: 0.1308 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0012 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0522 | Regression_eta: 0.0319 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0603 | LR: 2.07e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1594s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.7346 | max_loss: 0.1426 | max_backward: 1.8320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5857.6 | uss_MB: 4949.0 | child_rss_MB: 5646.6 | allocated_MB: 497.9 | reserved_MB: 29592.0 | inactive_split_MB: 1086.4 +rank2 [2026-10-29 22:15:54,682] INFO : Step 38900/50000 rank2 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1916 | Cardinality: 0.1035 | Classification: 0.0839 | Classification_binary: 0.1479 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0014 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1582 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0691 | Regression_eta: 0.0441 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0693 | LR: 2.05e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.8768s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0144 | max_forward: 2.1610 | max_loss: 0.2966 | max_backward: 2.8203 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4876 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 5858.5 | uss_MB: 4951.7 | child_rss_MB: 5647.2 | allocated_MB: 500.0 | reserved_MB: 29592.0 | inactive_split_MB: 849.6 +rank2 [2026-10-29 22:20:36,723] INFO : Step 39000/50000 rank2 | Train Loss: 0.8505 | Train Loss Unweighted: 0.4196 | Train Loss Components: Auxiliary: 0.1813 | Cardinality: 0.0941 | Classification: 0.0808 | Classification_binary: 0.1368 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0993 | Regression_eta: 0.0015 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0573 | Regression_eta: 0.1127 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0614 | LR: 2.03e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2836s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7392 | max_loss: 0.2445 | max_backward: 1.8584 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5844.6 | uss_MB: 4939.8 | child_rss_MB: 5638.4 | allocated_MB: 496.8 | reserved_MB: 29592.0 | inactive_split_MB: 1086.0 +rank2 [2026-10-29 22:20:37,630] INFO : Running validation on rank2 +rank2 [2026-10-29 22:20:37,630] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 22:20:37,630] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 22:21:31,240] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:21:31,931] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 22:21:58,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 22:21:59,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 22:22:01,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 22:22:02,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 22:22:12,036] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 22:22:12,679] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:22:13,179] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 22:22:29,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 22:22:30,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 22:22:31,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 22:22:32,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 22:22:42,614] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-29 22:22:43,329] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 22:22:43,803] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 22:22:59,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 22:23:00,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 22:23:00,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 22:23:01,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 22:23:11,741] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-29 22:28:30,320] INFO : Step 39100/50000 rank2 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1110 | Classification: 0.0839 | Classification_binary: 0.1198 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1053 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0559 | Regression_eta: 0.0458 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0688 | LR: 2.01e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5489s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7800 | max_loss: 0.2457 | max_backward: 1.9719 | max_optimizer_step: 0.3248 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 7010.9 | uss_MB: 6103.0 | child_rss_MB: 5608.2 | allocated_MB: 497.5 | reserved_MB: 24374.0 | inactive_split_MB: 923.6 +rank2 [2026-10-29 22:33:29,807] INFO : Step 39200/50000 rank2 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4242 | Train Loss Components: Auxiliary: 0.1863 | Cardinality: 0.1210 | Classification: 0.0912 | Classification_binary: 0.1382 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1501 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0439 | Regression_eta: 0.0349 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0420 | LR: 2.00e-04 | DataLoad Time: 0.0142s | Model Forward Time: 4.2605s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0324 | max_forward: 0.7817 | max_loss: 0.1452 | max_backward: 1.9286 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5940.0 | uss_MB: 5036.4 | child_rss_MB: 5609.8 | allocated_MB: 497.9 | reserved_MB: 26338.0 | inactive_split_MB: 912.4 +rank2 [2026-10-29 22:38:28,992] INFO : Step 39300/50000 rank2 | Train Loss: 0.9298 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1080 | Classification: 0.0882 | Classification_binary: 0.1594 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0951 | Regression_eta: 0.0013 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1730 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0513 | Regression_eta: 0.0299 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0600 | LR: 1.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1927s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7922 | max_loss: 0.2572 | max_backward: 1.8558 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 5927.1 | uss_MB: 5036.1 | child_rss_MB: 5613.3 | allocated_MB: 499.4 | reserved_MB: 26338.0 | inactive_split_MB: 1408.0 +rank2 [2026-10-29 22:43:21,461] INFO : Step 39400/50000 rank2 | Train Loss: 0.8421 | Train Loss Unweighted: 0.3964 | Train Loss Components: Auxiliary: 0.1755 | Cardinality: 0.1006 | Classification: 0.0679 | Classification_binary: 0.1346 | HitGrouping: 0.1534 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1202 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0626 | Regression_eta: 0.0323 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0489 | LR: 1.96e-04 | DataLoad Time: 0.0194s | Model Forward Time: 6.2888s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1084 | max_forward: 1.1740 | max_loss: 0.3046 | max_backward: 2.6778 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8613 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5928.1 | uss_MB: 4351.2 | child_rss_MB: 5609.1 | allocated_MB: 495.0 | reserved_MB: 26338.0 | inactive_split_MB: 1095.6 +rank2 [2026-10-29 22:48:17,291] INFO : Step 39500/50000 rank2 | Train Loss: 0.8539 | Train Loss Unweighted: 0.4179 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0879 | Classification: 0.0889 | Classification_binary: 0.1243 | HitGrouping: 0.1565 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0634 | Regression_eta: 0.0294 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0500 | LR: 1.94e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6164s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7373 | max_loss: 0.1406 | max_backward: 1.8719 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.8502 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5922.7 | uss_MB: 5037.5 | child_rss_MB: 5598.5 | allocated_MB: 492.8 | reserved_MB: 28318.0 | inactive_split_MB: 1078.3 +rank2 [2026-10-29 22:53:41,207] INFO : Step 39600/50000 rank2 | Train Loss: 0.9371 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1161 | Classification: 0.0813 | Classification_binary: 0.1596 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1039 | Regression_eta: 0.0013 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1593 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0565 | Regression_eta: 0.0224 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0417 | LR: 1.93e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.3249s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0173 | max_forward: 1.4879 | max_loss: 0.2520 | max_backward: 1.9863 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.8346 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5921.1 | uss_MB: 5038.4 | child_rss_MB: 5602.9 | allocated_MB: 496.0 | reserved_MB: 28318.0 | inactive_split_MB: 858.4 +rank2 [2026-10-29 22:58:48,059] INFO : Step 39700/50000 rank2 | Train Loss: 0.9204 | Train Loss Unweighted: 0.4427 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1101 | Classification: 0.0841 | Classification_binary: 0.1567 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0997 | Regression_eta: 0.0013 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0405 | Regression_eta: 0.0280 | Regression_pt: 0.0424 | Regression_sin_phi: 0.0525 | LR: 1.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2758s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.7571 | max_loss: 0.4946 | max_backward: 1.9147 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 5909.3 | uss_MB: 5037.4 | child_rss_MB: 5602.2 | allocated_MB: 497.0 | reserved_MB: 30324.0 | inactive_split_MB: 1175.5 +rank2 [2026-10-29 23:04:38,178] INFO : Step 39800/50000 rank2 | Train Loss: 0.8155 | Train Loss Unweighted: 0.3780 | Train Loss Components: Auxiliary: 0.1674 | Cardinality: 0.0963 | Classification: 0.0643 | Classification_binary: 0.1275 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1135 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0502 | Regression_eta: 0.0275 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0602 | LR: 1.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0226s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7197 | max_loss: 0.1489 | max_backward: 1.6690 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.4760 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1068 | Memory: rss_MB: 5897.7 | uss_MB: 5034.7 | child_rss_MB: 5594.3 | allocated_MB: 494.4 | reserved_MB: 30324.0 | inactive_split_MB: 897.3 +rank2 [2026-10-29 23:09:43,446] INFO : Step 39900/50000 rank2 | Train Loss: 0.8844 | Train Loss Unweighted: 0.3998 | Train Loss Components: Auxiliary: 0.1871 | Cardinality: 0.1077 | Classification: 0.0758 | Classification_binary: 0.1443 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0883 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1326 | Regression_cos_phi: 0.0332 | Regression_energy: 0.0424 | Regression_eta: 0.0273 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0430 | LR: 1.88e-04 | DataLoad Time: 0.0101s | Model Forward Time: 4.9811s | Phase Times: max_pre_step_barrier: 0.0268 | max_batch_to_device: 0.0727 | max_forward: 1.7016 | max_loss: 0.4968 | max_backward: 2.0829 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.4776 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5898.5 | uss_MB: 5037.5 | child_rss_MB: 5601.6 | allocated_MB: 496.1 | reserved_MB: 30324.0 | inactive_split_MB: 1077.6 +rank2 [2026-10-29 23:14:38,166] INFO : Step 40000/50000 rank2 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1170 | Classification: 0.0871 | Classification_binary: 0.1550 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0516 | Regression_eta: 0.0362 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0607 | LR: 1.86e-04 | DataLoad Time: 0.0114s | Model Forward Time: 5.4246s | Phase Times: max_pre_step_barrier: 0.1084 | max_batch_to_device: 0.0070 | max_forward: 1.1725 | max_loss: 0.2555 | max_backward: 2.4297 | max_optimizer_step: 0.2969 | max_tensorboard_logging: 0.6818 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5889.3 | uss_MB: 5028.2 | child_rss_MB: 5597.1 | allocated_MB: 501.7 | reserved_MB: 30324.0 | inactive_split_MB: 1368.0 +rank2 [2026-10-29 23:14:39,118] INFO : Running validation on rank2 +rank2 [2026-10-29 23:14:39,118] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-29 23:14:39,118] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-29 23:15:31,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:15:32,715] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-29 23:16:03,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-29 23:16:04,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-29 23:16:06,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-29 23:16:07,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-29 23:16:17,960] INFO : Time taken to make predictions on device 2 is: 0.28 min +rank2 [2026-10-29 23:16:28,294] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:16:28,871] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-29 23:16:44,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-29 23:16:45,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-29 23:16:46,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-29 23:16:47,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-29 23:16:57,588] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-29 23:16:58,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-29 23:16:59,026] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-29 23:17:14,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-29 23:17:15,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-29 23:17:16,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-29 23:17:17,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-29 23:17:27,467] INFO : Time taken to make predictions on device 2 is: 0.24 min +rank2 [2026-10-29 23:22:41,683] INFO : Step 40100/50000 rank2 | Train Loss: 0.8115 | Train Loss Unweighted: 0.3794 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.1028 | Classification: 0.0708 | Classification_binary: 0.1239 | HitGrouping: 0.1460 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0909 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0375 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0476 | Regression_eta: 0.0328 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0504 | LR: 1.84e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0766s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.1076 | max_forward: 0.8426 | max_loss: 0.1437 | max_backward: 1.8847 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 7273.7 | uss_MB: 5477.4 | child_rss_MB: 5617.3 | allocated_MB: 493.6 | reserved_MB: 28004.0 | inactive_split_MB: 975.8 +rank2 [2026-10-29 23:27:53,247] INFO : Step 40200/50000 rank2 | Train Loss: 0.8264 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1679 | Cardinality: 0.0984 | Classification: 0.0703 | Classification_binary: 0.1210 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0913 | Regression_eta: 0.0013 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1184 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0508 | Regression_eta: 0.0501 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0615 | LR: 1.83e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.8949s | Phase Times: max_pre_step_barrier: 0.0270 | max_batch_to_device: 0.0062 | max_forward: 0.8324 | max_loss: 0.1802 | max_backward: 2.0569 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0152 | max_post_step_barrier: 0.0867 | Memory: rss_MB: 5955.5 | uss_MB: 5094.1 | child_rss_MB: 5625.1 | allocated_MB: 497.1 | reserved_MB: 28008.0 | inactive_split_MB: 935.5 +rank2 [2026-10-29 23:33:22,807] INFO : Step 40300/50000 rank2 | Train Loss: 0.9351 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1181 | Classification: 0.0959 | Classification_binary: 0.1652 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0014 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0512 | Regression_eta: 0.0313 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0513 | LR: 1.81e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.3104s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8191 | max_loss: 0.1494 | max_backward: 1.9817 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 0.9693 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 5955.1 | uss_MB: 5096.4 | child_rss_MB: 5610.7 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 927.5 +rank2 [2026-10-29 23:38:38,678] INFO : Step 40400/50000 rank2 | Train Loss: 0.9379 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1359 | Classification: 0.0771 | Classification_binary: 0.1554 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0944 | Regression_eta: 0.0014 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0493 | Regression_eta: 0.0349 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0423 | LR: 1.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6736s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0500 | max_forward: 0.8111 | max_loss: 0.5067 | max_backward: 1.8144 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5951.4 | uss_MB: 5092.9 | child_rss_MB: 5616.0 | allocated_MB: 497.0 | reserved_MB: 28008.0 | inactive_split_MB: 931.6 +rank2 [2026-10-29 23:43:43,800] INFO : Step 40500/50000 rank2 | Train Loss: 0.8053 | Train Loss Unweighted: 0.3782 | Train Loss Components: Auxiliary: 0.1649 | Cardinality: 0.0788 | Classification: 0.0749 | Classification_binary: 0.1246 | HitGrouping: 0.1502 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0443 | Regression_eta: 0.0218 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0384 | LR: 1.78e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3984s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.8420 | max_loss: 0.1414 | max_backward: 1.5396 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.9215 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5948.4 | uss_MB: 5092.0 | child_rss_MB: 5597.4 | allocated_MB: 494.5 | reserved_MB: 28012.0 | inactive_split_MB: 979.3 +rank2 [2026-10-29 23:48:43,696] INFO : Step 40600/50000 rank2 | Train Loss: 0.8980 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1108 | Classification: 0.0752 | Classification_binary: 0.1417 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0955 | Regression_eta: 0.0015 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1076 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0387 | Regression_eta: 0.0357 | Regression_pt: 0.0437 | Regression_sin_phi: 0.0686 | LR: 1.76e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3181s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7855 | max_loss: 0.1488 | max_backward: 1.8742 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 5948.2 | uss_MB: 5094.3 | child_rss_MB: 5593.1 | allocated_MB: 499.7 | reserved_MB: 28012.0 | inactive_split_MB: 1102.0 +rank2 [2026-10-29 23:53:39,580] INFO : Step 40700/50000 rank2 | Train Loss: 0.9220 | Train Loss Unweighted: 0.4304 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1148 | Classification: 0.0767 | Classification_binary: 0.1645 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0441 | Regression_eta: 0.0380 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0447 | LR: 1.75e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5452s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.8091 | max_loss: 0.2475 | max_backward: 2.0011 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.5972 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5949.6 | uss_MB: 5096.1 | child_rss_MB: 5593.6 | allocated_MB: 498.9 | reserved_MB: 28012.0 | inactive_split_MB: 776.0 +rank2 [2026-10-30 00:01:00,050] INFO : Step 40800/50000 rank2 | Train Loss: 0.8642 | Train Loss Unweighted: 0.4082 | Train Loss Components: Auxiliary: 0.1815 | Cardinality: 0.1013 | Classification: 0.0834 | Classification_binary: 0.1260 | HitGrouping: 0.1507 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.1363 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0473 | Regression_eta: 0.0671 | Regression_pt: 0.0521 | Regression_sin_phi: 0.1116 | LR: 1.73e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.1141s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7033 | max_loss: 0.1373 | max_backward: 1.5042 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1068 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5935.0 | uss_MB: 4327.0 | child_rss_MB: 5586.6 | allocated_MB: 493.1 | reserved_MB: 28012.0 | inactive_split_MB: 897.3 +rank2 [2026-10-30 00:06:14,592] INFO : Step 40900/50000 rank2 | Train Loss: 0.8749 | Train Loss Unweighted: 0.4038 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1070 | Classification: 0.0854 | Classification_binary: 0.1300 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0923 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1278 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0466 | Regression_eta: 0.0315 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0538 | LR: 1.72e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5716s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7515 | max_loss: 0.2142 | max_backward: 1.7950 | max_optimizer_step: 0.4960 | max_tensorboard_logging: 0.9322 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5926.9 | uss_MB: 5094.2 | child_rss_MB: 5563.3 | allocated_MB: 497.9 | reserved_MB: 30012.0 | inactive_split_MB: 893.0 +rank2 [2026-10-30 00:11:07,452] INFO : Step 41000/50000 rank2 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.0956 | Classification: 0.0893 | Classification_binary: 0.1497 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0014 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0577 | Regression_eta: 0.0326 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0509 | LR: 1.70e-04 | DataLoad Time: 0.0105s | Model Forward Time: 5.0984s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0071 | max_forward: 1.1963 | max_loss: 0.1519 | max_backward: 2.0474 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.0118 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 5918.6 | uss_MB: 5085.3 | child_rss_MB: 5547.4 | allocated_MB: 498.7 | reserved_MB: 30012.0 | inactive_split_MB: 1108.0 +rank2 [2026-10-30 00:11:08,373] INFO : Running validation on rank2 +rank2 [2026-10-30 00:11:08,373] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 00:11:08,373] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 00:11:59,245] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:12:00,022] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 00:12:25,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 00:12:27,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 00:12:28,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 00:12:29,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 00:12:39,110] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 00:12:39,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:12:40,281] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 00:12:56,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 00:12:57,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 00:12:58,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 00:12:59,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 00:13:09,086] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 00:13:09,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 00:13:10,648] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 00:13:26,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 00:13:27,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 00:13:28,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 00:13:29,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 00:13:39,201] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 00:19:15,373] INFO : Step 41100/50000 rank2 | Train Loss: 0.9430 | Train Loss Unweighted: 0.4319 | Train Loss Components: Auxiliary: 0.1995 | Cardinality: 0.1330 | Classification: 0.0721 | Classification_binary: 0.1579 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0013 | Regression_pt: 0.0998 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0317 | Classification_binary: 0.1343 | Regression_cos_phi: 0.0364 | Regression_energy: 0.0424 | Regression_eta: 0.0228 | Regression_pt: 0.0455 | Regression_sin_phi: 0.0488 | LR: 1.69e-04 | DataLoad Time: 0.0149s | Model Forward Time: 6.3662s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8221 | max_loss: 0.4800 | max_backward: 2.2615 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7242.0 | uss_MB: 5759.2 | child_rss_MB: 5549.6 | allocated_MB: 497.6 | reserved_MB: 23400.0 | inactive_split_MB: 991.8 +rank2 [2026-10-30 00:24:27,079] INFO : Step 41200/50000 rank2 | Train Loss: 0.8544 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1097 | Classification: 0.0734 | Classification_binary: 0.1131 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1015 | Regression_eta: 0.0015 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.0917 | Regression_cos_phi: 0.0386 | Regression_energy: 0.0479 | Regression_eta: 0.0316 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0362 | LR: 1.67e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2547s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7030 | max_loss: 0.2408 | max_backward: 1.7750 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5316 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6039.9 | uss_MB: 5208.8 | child_rss_MB: 5554.6 | allocated_MB: 496.1 | reserved_MB: 27308.0 | inactive_split_MB: 984.0 +rank2 [2026-10-30 00:29:20,952] INFO : Step 41300/50000 rank2 | Train Loss: 0.8221 | Train Loss Unweighted: 0.3852 | Train Loss Components: Auxiliary: 0.1744 | Cardinality: 0.0906 | Classification: 0.0754 | Classification_binary: 0.1369 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0849 | Regression_eta: 0.0012 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0348 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0299 | Regression_energy: 0.0431 | Regression_eta: 0.0370 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0468 | LR: 1.66e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4688s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8083 | max_loss: 0.1794 | max_backward: 1.9993 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.5906 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6040.0 | uss_MB: 5213.3 | child_rss_MB: 5551.2 | allocated_MB: 497.0 | reserved_MB: 27308.0 | inactive_split_MB: 1229.0 +rank2 [2026-10-30 00:34:25,619] INFO : Step 41400/50000 rank2 | Train Loss: 0.9216 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1171 | Classification: 0.0901 | Classification_binary: 0.1523 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1010 | Regression_eta: 0.0011 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0518 | Regression_eta: 0.0361 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0506 | LR: 1.64e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6345s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0065 | max_forward: 0.7831 | max_loss: 0.4937 | max_backward: 1.9271 | max_optimizer_step: 0.3071 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6033.3 | uss_MB: 5208.0 | child_rss_MB: 5553.7 | allocated_MB: 498.4 | reserved_MB: 29288.0 | inactive_split_MB: 1180.4 +rank2 [2026-10-30 00:39:29,959] INFO : Step 41500/50000 rank2 | Train Loss: 0.8584 | Train Loss Unweighted: 0.4111 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1030 | Classification: 0.0769 | Classification_binary: 0.1210 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1214 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0410 | Regression_eta: 0.0258 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0451 | LR: 1.63e-04 | DataLoad Time: 0.0169s | Model Forward Time: 6.0408s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.1908 | max_loss: 0.3017 | max_backward: 3.1310 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 6009.7 | uss_MB: 5150.0 | child_rss_MB: 5552.7 | allocated_MB: 494.5 | reserved_MB: 29288.0 | inactive_split_MB: 976.9 +rank2 [2026-10-30 00:44:19,403] INFO : Step 41600/50000 rank2 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4026 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1032 | Classification: 0.0762 | Classification_binary: 0.1246 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0992 | Regression_eta: 0.0013 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0574 | Regression_eta: 0.0314 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0414 | LR: 1.61e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9946s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0065 | max_forward: 0.7279 | max_loss: 0.2339 | max_backward: 2.5338 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.4866 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1404 | Memory: rss_MB: 6001.6 | uss_MB: 5206.5 | child_rss_MB: 5546.1 | allocated_MB: 496.4 | reserved_MB: 29288.0 | inactive_split_MB: 761.6 +rank2 [2026-10-30 00:49:10,876] INFO : Step 41700/50000 rank2 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1941 | Cardinality: 0.1112 | Classification: 0.0767 | Classification_binary: 0.1493 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0988 | Regression_eta: 0.0013 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1652 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0531 | Regression_eta: 0.0231 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0397 | LR: 1.60e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5059s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7998 | max_loss: 0.1827 | max_backward: 1.9774 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6004.5 | uss_MB: 5210.8 | child_rss_MB: 5548.7 | allocated_MB: 497.0 | reserved_MB: 29288.0 | inactive_split_MB: 1220.0 +rank2 [2026-10-30 00:53:58,971] INFO : Step 41800/50000 rank2 | Train Loss: 0.9125 | Train Loss Unweighted: 0.4285 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.1148 | Classification: 0.0784 | Classification_binary: 0.1537 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0519 | Regression_eta: 0.0378 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0589 | LR: 1.58e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8249s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7952 | max_loss: 0.1582 | max_backward: 1.8899 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9392 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1143 | Memory: rss_MB: 5984.7 | uss_MB: 5200.7 | child_rss_MB: 5542.4 | allocated_MB: 499.3 | reserved_MB: 31288.0 | inactive_split_MB: 932.9 +rank2 [2026-10-30 00:59:00,784] INFO : Step 41900/50000 rank2 | Train Loss: 0.8303 | Train Loss Unweighted: 0.3927 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1132 | Classification: 0.0782 | Classification_binary: 0.1167 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0010 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1268 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0617 | Regression_eta: 0.0240 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0357 | LR: 1.57e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1513s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7413 | max_loss: 0.1970 | max_backward: 1.7614 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5208 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5992.5 | uss_MB: 5207.2 | child_rss_MB: 5522.2 | allocated_MB: 497.1 | reserved_MB: 31288.0 | inactive_split_MB: 1188.0 +rank2 [2026-10-30 01:03:51,616] INFO : Step 42000/50000 rank2 | Train Loss: 0.8724 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1847 | Cardinality: 0.0893 | Classification: 0.0874 | Classification_binary: 0.1377 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0958 | Regression_eta: 0.0012 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1452 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0576 | Regression_eta: 0.0628 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0668 | LR: 1.56e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7566s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1147 | max_forward: 0.8308 | max_loss: 0.1392 | max_backward: 1.8336 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5996.6 | uss_MB: 5209.4 | child_rss_MB: 5540.2 | allocated_MB: 496.3 | reserved_MB: 31288.0 | inactive_split_MB: 930.5 +rank2 [2026-10-30 01:03:52,597] INFO : Running validation on rank2 +rank2 [2026-10-30 01:03:52,597] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 01:03:52,597] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 01:04:46,126] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:04:46,670] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 01:05:13,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 01:05:14,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 01:05:15,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 01:05:16,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 01:05:26,881] INFO : Time taken to make predictions on device 2 is: 0.27 min +rank2 [2026-10-30 01:05:27,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:05:27,884] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 01:05:43,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 01:05:44,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 01:05:45,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 01:05:46,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 01:05:56,724] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 01:05:57,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:05:57,664] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 01:06:13,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 01:06:15,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 01:06:16,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 01:06:17,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 01:06:27,083] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 01:11:42,848] INFO : Step 42100/50000 rank2 | Train Loss: 0.9390 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1200 | Classification: 0.0884 | Classification_binary: 0.1619 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0926 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0572 | Regression_eta: 0.0270 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0423 | LR: 1.54e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.0864s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0976 | max_forward: 1.0189 | max_loss: 0.3859 | max_backward: 2.5922 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 1.0368 | max_tensorboard_flush: 0.0149 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7501.8 | uss_MB: 4866.0 | child_rss_MB: 5544.9 | allocated_MB: 497.0 | reserved_MB: 25850.0 | inactive_split_MB: 892.8 +rank2 [2026-10-30 01:16:53,955] INFO : Step 42200/50000 rank2 | Train Loss: 0.9057 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1144 | Classification: 0.0879 | Classification_binary: 0.1381 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0477 | Regression_eta: 0.0223 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0532 | LR: 1.53e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.1525s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7457 | max_loss: 0.1999 | max_backward: 1.6824 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.5916 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6096.0 | uss_MB: 5306.7 | child_rss_MB: 5537.7 | allocated_MB: 497.3 | reserved_MB: 27866.0 | inactive_split_MB: 959.8 +rank2 [2026-10-30 01:18:06,041] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank2 [2026-10-30 01:18:06,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank2 [2026-10-30 01:21:57,449] INFO : Step 42300/50000 rank2 | Train Loss: 0.9052 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1933 | Cardinality: 0.1127 | Classification: 0.0787 | Classification_binary: 0.1569 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0868 | Regression_eta: 0.0013 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1715 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0490 | Regression_eta: 0.0300 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0590 | LR: 1.52e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.1172s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.9764 | max_loss: 0.1904 | max_backward: 2.7671 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.9997 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6118.1 | uss_MB: 5325.9 | child_rss_MB: 5544.6 | allocated_MB: 499.1 | reserved_MB: 27872.0 | inactive_split_MB: 1033.6 +rank2 [2026-10-30 01:26:55,613] INFO : Step 42400/50000 rank2 | Train Loss: 0.9358 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.1976 | Cardinality: 0.1220 | Classification: 0.0815 | Classification_binary: 0.1533 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0695 | Regression_eta: 0.0440 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0484 | LR: 1.50e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8877s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8701 | max_loss: 0.1465 | max_backward: 1.9416 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.8192 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6099.3 | uss_MB: 5308.3 | child_rss_MB: 5546.3 | allocated_MB: 499.8 | reserved_MB: 29908.0 | inactive_split_MB: 939.2 +rank2 [2026-10-30 01:31:49,012] INFO : Step 42500/50000 rank2 | Train Loss: 0.8126 | Train Loss Unweighted: 0.3791 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0975 | Classification: 0.0765 | Classification_binary: 0.1172 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0918 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1307 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0550 | Regression_eta: 0.0356 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0539 | LR: 1.49e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5933s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0858 | max_forward: 0.8156 | max_loss: 0.1407 | max_backward: 1.8570 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7997 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6101.2 | uss_MB: 5308.6 | child_rss_MB: 5538.8 | allocated_MB: 497.5 | reserved_MB: 29908.0 | inactive_split_MB: 943.4 +rank2 [2026-10-30 01:36:44,499] INFO : Step 42600/50000 rank2 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0772 | Classification_binary: 0.1347 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0012 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1167 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0469 | Regression_eta: 0.0358 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0530 | LR: 1.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.0197s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7519 | max_loss: 0.1393 | max_backward: 1.8021 | max_optimizer_step: 0.3023 | max_tensorboard_logging: 0.7973 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6101.5 | uss_MB: 4501.2 | child_rss_MB: 5544.2 | allocated_MB: 497.0 | reserved_MB: 29908.0 | inactive_split_MB: 877.2 +rank2 [2026-10-30 01:41:48,353] INFO : Step 42700/50000 rank2 | Train Loss: 0.9129 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1178 | Classification: 0.0751 | Classification_binary: 0.1632 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0858 | Regression_eta: 0.0014 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0595 | Regression_eta: 0.0424 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0891 | LR: 1.47e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6153s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 1.0869 | max_loss: 0.1547 | max_backward: 1.6854 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7949 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6101.6 | uss_MB: 5308.9 | child_rss_MB: 5545.8 | allocated_MB: 497.3 | reserved_MB: 29908.0 | inactive_split_MB: 953.4 +rank2 [2026-10-30 01:46:39,116] INFO : Step 42800/50000 rank2 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1179 | Classification: 0.0767 | Classification_binary: 0.1362 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0013 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1214 | Regression_cos_phi: 0.0407 | Regression_energy: 0.0467 | Regression_eta: 0.0240 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0518 | LR: 1.45e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3524s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8438 | max_loss: 0.1478 | max_backward: 1.6710 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.6821 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6089.4 | uss_MB: 5295.4 | child_rss_MB: 5546.6 | allocated_MB: 499.0 | reserved_MB: 30126.0 | inactive_split_MB: 875.7 +rank2 [2026-10-30 01:51:40,828] INFO : Step 42900/50000 rank2 | Train Loss: 0.7857 | Train Loss Unweighted: 0.3758 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0951 | Classification: 0.0820 | Classification_binary: 0.1113 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0898 | Regression_eta: 0.0010 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1126 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0401 | Regression_eta: 0.0307 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0678 | LR: 1.44e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.9234s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8792 | max_loss: 0.1461 | max_backward: 1.9928 | max_optimizer_step: 0.1903 | max_tensorboard_logging: 0.9695 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6098.2 | uss_MB: 5308.5 | child_rss_MB: 5544.6 | allocated_MB: 496.5 | reserved_MB: 30126.0 | inactive_split_MB: 951.8 +rank2 [2026-10-30 01:56:37,110] INFO : Step 43000/50000 rank2 | Train Loss: 0.8899 | Train Loss Unweighted: 0.4207 | Train Loss Components: Auxiliary: 0.1911 | Cardinality: 0.1035 | Classification: 0.0847 | Classification_binary: 0.1529 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0405 | Regression_eta: 0.0287 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0456 | LR: 1.43e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7367s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1951 | max_backward: 1.9694 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.8418 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6086.1 | uss_MB: 5311.0 | child_rss_MB: 5508.6 | allocated_MB: 495.5 | reserved_MB: 30126.0 | inactive_split_MB: 1120.4 +rank2 [2026-10-30 01:56:38,137] INFO : Running validation on rank2 +rank2 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 01:57:33,181] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:57:33,675] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 01:57:59,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 01:58:00,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 01:58:01,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 01:58:02,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 01:58:12,629] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 01:58:13,038] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:58:13,512] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 01:58:29,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 01:58:30,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 01:58:31,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 01:58:32,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 01:58:42,623] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 01:58:43,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 01:58:43,759] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 01:58:59,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 01:59:00,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 01:59:01,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 01:59:02,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 01:59:12,090] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 02:04:24,630] INFO : Step 43100/50000 rank2 | Train Loss: 0.8965 | Train Loss Unweighted: 0.4137 | Train Loss Components: Auxiliary: 0.1901 | Cardinality: 0.1074 | Classification: 0.0705 | Classification_binary: 0.1472 | HitGrouping: 0.1435 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1751 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0524 | Regression_eta: 0.0364 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0775 | LR: 1.42e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2199s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.2396 | max_backward: 1.9954 | max_optimizer_step: 0.3209 | max_tensorboard_logging: 1.0803 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7631.1 | uss_MB: 6853.7 | child_rss_MB: 5519.2 | allocated_MB: 496.0 | reserved_MB: 26194.0 | inactive_split_MB: 843.4 +rank2 [2026-10-30 02:09:06,731] INFO : Step 43200/50000 rank2 | Train Loss: 0.8164 | Train Loss Unweighted: 0.3744 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0957 | Classification: 0.0763 | Classification_binary: 0.1197 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1157 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0441 | Regression_eta: 0.0225 | Regression_pt: 0.0478 | Regression_sin_phi: 0.0386 | LR: 1.40e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4252s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7823 | max_loss: 0.4822 | max_backward: 1.6766 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6123.7 | uss_MB: 5349.6 | child_rss_MB: 5511.0 | allocated_MB: 495.8 | reserved_MB: 26686.0 | inactive_split_MB: 848.9 +rank2 [2026-10-30 02:13:48,743] INFO : Step 43300/50000 rank2 | Train Loss: 0.9151 | Train Loss Unweighted: 0.4260 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1202 | Classification: 0.0906 | Classification_binary: 0.1467 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0013 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0411 | Regression_eta: 0.0422 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0824 | LR: 1.39e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.7257s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0155 | max_forward: 0.8938 | max_loss: 0.1464 | max_backward: 2.1010 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.5477 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1243 | Memory: rss_MB: 6097.4 | uss_MB: 5352.9 | child_rss_MB: 5427.7 | allocated_MB: 497.9 | reserved_MB: 28278.0 | inactive_split_MB: 1309.2 +rank2 [2026-10-30 02:18:30,112] INFO : Step 43400/50000 rank2 | Train Loss: 0.9723 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2082 | Cardinality: 0.1222 | Classification: 0.0880 | Classification_binary: 0.1684 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0015 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0620 | Regression_eta: 0.0682 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0620 | LR: 1.38e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.4359s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.5043 | max_backward: 2.0661 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.4824 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6077.9 | uss_MB: 5350.3 | child_rss_MB: 5335.1 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 851.9 +rank2 [2026-10-30 02:23:12,891] INFO : Step 43500/50000 rank2 | Train Loss: 0.8402 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.0947 | Classification: 0.0776 | Classification_binary: 0.1278 | HitGrouping: 0.1501 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0999 | Regression_eta: 0.0012 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1142 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0607 | Regression_eta: 0.0448 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0335 | LR: 1.37e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8612s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8086 | max_loss: 0.1489 | max_backward: 2.0887 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.7978 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1094 | Memory: rss_MB: 6077.6 | uss_MB: 5352.8 | child_rss_MB: 5340.6 | allocated_MB: 498.6 | reserved_MB: 28278.0 | inactive_split_MB: 909.7 +rank2 [2026-10-30 02:28:09,613] INFO : Step 43600/50000 rank2 | Train Loss: 0.7985 | Train Loss Unweighted: 0.3653 | Train Loss Components: Auxiliary: 0.1621 | Cardinality: 0.0887 | Classification: 0.0728 | Classification_binary: 0.1184 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0851 | Regression_eta: 0.0013 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1241 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0465 | Regression_eta: 0.0241 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0369 | LR: 1.36e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.2500s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0170 | max_forward: 1.2777 | max_loss: 0.2064 | max_backward: 1.9835 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 1.1084 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6044.0 | uss_MB: 4957.3 | child_rss_MB: 5260.0 | allocated_MB: 493.5 | reserved_MB: 28278.0 | inactive_split_MB: 1313.2 +rank2 [2026-10-30 02:33:07,295] INFO : Step 43700/50000 rank2 | Train Loss: 0.8599 | Train Loss Unweighted: 0.3884 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1089 | Classification: 0.0797 | Classification_binary: 0.1434 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0812 | Regression_eta: 0.0013 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0435 | Regression_eta: 0.0316 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0521 | LR: 1.35e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6051s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7902 | max_loss: 0.1752 | max_backward: 1.8774 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8044 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6046.0 | uss_MB: 5350.4 | child_rss_MB: 5241.5 | allocated_MB: 498.0 | reserved_MB: 28278.0 | inactive_split_MB: 1046.0 +rank2 [2026-10-30 02:37:45,725] INFO : Step 43800/50000 rank2 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1875 | Cardinality: 0.1049 | Classification: 0.0814 | Classification_binary: 0.1423 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0907 | Regression_eta: 0.0012 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0507 | Regression_eta: 0.0314 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0342 | LR: 1.34e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5747s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7940 | max_loss: 0.4805 | max_backward: 1.8159 | max_optimizer_step: 0.2941 | max_tensorboard_logging: 0.7850 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6040.0 | uss_MB: 5349.3 | child_rss_MB: 5218.8 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 839.4 +rank2 [2026-10-30 02:42:41,735] INFO : Step 43900/50000 rank2 | Train Loss: 0.7987 | Train Loss Unweighted: 0.3885 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.0999 | Classification: 0.0753 | Classification_binary: 0.1166 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0968 | Regression_eta: 0.0010 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.0909 | Regression_cos_phi: 0.0248 | Regression_energy: 0.0563 | Regression_eta: 0.0183 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0429 | LR: 1.33e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5950s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0846 | max_forward: 0.8891 | max_loss: 0.1768 | max_backward: 1.7357 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7516 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6030.6 | uss_MB: 5347.7 | child_rss_MB: 5179.6 | allocated_MB: 494.5 | reserved_MB: 28278.0 | inactive_split_MB: 1129.5 +rank2 [2026-10-30 02:47:33,825] INFO : Step 44000/50000 rank2 | Train Loss: 0.8263 | Train Loss Unweighted: 0.3926 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.1027 | Classification: 0.0840 | Classification_binary: 0.1318 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0868 | Regression_eta: 0.0011 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0457 | Regression_eta: 0.0437 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0661 | LR: 1.32e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4523s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8816 | max_loss: 0.1426 | max_backward: 1.9616 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.4846 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6020.6 | uss_MB: 5352.2 | child_rss_MB: 5134.9 | allocated_MB: 496.3 | reserved_MB: 28762.0 | inactive_split_MB: 1307.2 +rank2 [2026-10-30 02:47:34,860] INFO : Running validation on rank2 +rank2 [2026-10-30 02:47:34,860] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 02:47:34,861] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 02:48:28,154] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 02:48:28,669] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 02:48:55,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 02:48:56,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 02:48:58,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 02:48:59,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 02:49:09,036] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 02:49:09,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 02:49:10,103] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 02:49:25,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 02:49:26,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 02:49:27,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 02:49:28,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 02:49:38,674] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 02:49:39,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 02:49:40,181] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 02:49:55,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 02:49:56,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 02:49:57,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 02:49:58,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 02:50:08,323] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 02:55:20,175] INFO : Step 44100/50000 rank2 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1986 | Cardinality: 0.1334 | Classification: 0.0764 | Classification_binary: 0.1516 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0482 | Regression_eta: 0.0254 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0632 | LR: 1.31e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1234s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0079 | max_forward: 0.8221 | max_loss: 0.1579 | max_backward: 2.0192 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 1.0889 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7539.1 | uss_MB: 6869.4 | child_rss_MB: 5111.3 | allocated_MB: 498.6 | reserved_MB: 25920.0 | inactive_split_MB: 1073.8 +rank2 [2026-10-30 03:00:12,362] INFO : Step 44200/50000 rank2 | Train Loss: 0.9018 | Train Loss Unweighted: 0.4273 | Train Loss Components: Auxiliary: 0.1909 | Cardinality: 0.1109 | Classification: 0.0847 | Classification_binary: 0.1354 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0012 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1216 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0477 | Regression_eta: 0.0286 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0601 | LR: 1.30e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3577s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.1013 | max_forward: 0.8608 | max_loss: 0.1485 | max_backward: 1.6010 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 6119.2 | uss_MB: 5452.1 | child_rss_MB: 5083.8 | allocated_MB: 496.1 | reserved_MB: 27956.0 | inactive_split_MB: 1248.1 +rank2 [2026-10-30 03:04:47,145] INFO : Step 44300/50000 rank2 | Train Loss: 0.8239 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1722 | Cardinality: 0.1007 | Classification: 0.0789 | Classification_binary: 0.1118 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1278 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0535 | Regression_eta: 0.0228 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0492 | LR: 1.29e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3180s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8120 | max_loss: 0.1377 | max_backward: 1.4700 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9055 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6110.2 | uss_MB: 5452.8 | child_rss_MB: 5031.6 | allocated_MB: 492.8 | reserved_MB: 27956.0 | inactive_split_MB: 1035.8 +rank2 [2026-10-30 03:09:18,055] INFO : Step 44400/50000 rank2 | Train Loss: 0.8683 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.0849 | Classification: 0.0836 | Classification_binary: 0.1500 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0473 | Regression_eta: 0.0322 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0793 | LR: 1.28e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9257s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8907 | max_loss: 0.1476 | max_backward: 2.0623 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6106.5 | uss_MB: 5454.6 | child_rss_MB: 5002.7 | allocated_MB: 499.5 | reserved_MB: 27956.0 | inactive_split_MB: 926.0 +rank2 [2026-10-30 03:14:01,913] INFO : Step 44500/50000 rank2 | Train Loss: 0.8672 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.0989 | Classification: 0.0738 | Classification_binary: 0.1455 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1699 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0481 | Regression_eta: 0.0294 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0680 | LR: 1.27e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2942s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0597 | max_forward: 0.8669 | max_loss: 0.1547 | max_backward: 1.7327 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.5628 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6105.7 | uss_MB: 5452.5 | child_rss_MB: 5004.4 | allocated_MB: 498.3 | reserved_MB: 27956.0 | inactive_split_MB: 860.5 +rank2 [2026-10-30 03:18:59,362] INFO : Step 44600/50000 rank2 | Train Loss: 0.7900 | Train Loss Unweighted: 0.3743 | Train Loss Components: Auxiliary: 0.1688 | Cardinality: 0.0956 | Classification: 0.0724 | Classification_binary: 0.1157 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0919 | Regression_eta: 0.0010 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1131 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0459 | Regression_eta: 0.0380 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0357 | LR: 1.26e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5435s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0745 | max_forward: 0.7797 | max_loss: 0.1409 | max_backward: 1.9334 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6022 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6098.6 | uss_MB: 5447.3 | child_rss_MB: 4994.6 | allocated_MB: 494.5 | reserved_MB: 27958.0 | inactive_split_MB: 927.6 +rank2 [2026-10-30 03:23:44,790] INFO : Step 44700/50000 rank2 | Train Loss: 0.8669 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1878 | Cardinality: 0.1160 | Classification: 0.0768 | Classification_binary: 0.1380 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0863 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1222 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0446 | Regression_eta: 0.0268 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0403 | LR: 1.25e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.4951s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8629 | max_loss: 0.1449 | max_backward: 2.0281 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.5242 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6099.2 | uss_MB: 5434.3 | child_rss_MB: 5003.0 | allocated_MB: 498.4 | reserved_MB: 27958.0 | inactive_split_MB: 781.6 +rank2 [2026-10-30 03:28:42,238] INFO : Step 44800/50000 rank2 | Train Loss: 0.8585 | Train Loss Unweighted: 0.4003 | Train Loss Components: Auxiliary: 0.1869 | Cardinality: 0.1024 | Classification: 0.0769 | Classification_binary: 0.1550 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0823 | Regression_eta: 0.0011 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.2055 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0576 | Regression_eta: 0.0345 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0538 | LR: 1.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8035s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0072 | max_forward: 0.7883 | max_loss: 0.5033 | max_backward: 1.9727 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6090.2 | uss_MB: 5455.2 | child_rss_MB: 4956.4 | allocated_MB: 496.5 | reserved_MB: 27958.0 | inactive_split_MB: 778.0 +rank2 [2026-10-30 03:33:12,492] INFO : Step 44900/50000 rank2 | Train Loss: 0.8358 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.1024 | Classification: 0.0797 | Classification_binary: 0.1294 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0859 | Regression_eta: 0.0011 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1130 | Regression_cos_phi: 0.0278 | Regression_energy: 0.0470 | Regression_eta: 0.0299 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0555 | LR: 1.23e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.3697s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7360 | max_loss: 0.1482 | max_backward: 1.6886 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7825 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6084.1 | uss_MB: 5452.9 | child_rss_MB: 4956.9 | allocated_MB: 494.3 | reserved_MB: 27958.0 | inactive_split_MB: 922.0 +rank2 [2026-10-30 03:37:54,705] INFO : Step 45000/50000 rank2 | Train Loss: 0.8678 | Train Loss Unweighted: 0.3957 | Train Loss Components: Auxiliary: 0.1790 | Cardinality: 0.1036 | Classification: 0.0801 | Classification_binary: 0.1237 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0941 | Regression_eta: 0.0013 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.0997 | Regression_cos_phi: 0.1153 | Regression_energy: 0.0518 | Regression_eta: 0.0627 | Regression_pt: 0.0539 | Regression_sin_phi: 0.1038 | LR: 1.22e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5700s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7189 | max_loss: 0.2183 | max_backward: 1.8172 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.8812 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6058.9 | uss_MB: 5454.9 | child_rss_MB: 4844.5 | allocated_MB: 495.0 | reserved_MB: 27960.0 | inactive_split_MB: 978.5 +rank2 [2026-10-30 03:37:55,714] INFO : Running validation on rank2 +rank2 [2026-10-30 03:37:55,714] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 03:37:55,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 03:38:46,291] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 03:38:46,811] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 03:39:14,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 03:39:15,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 03:39:17,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 03:39:18,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 03:39:28,891] INFO : Time taken to make predictions on device 2 is: 0.28 min +rank2 [2026-10-30 03:39:29,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 03:39:30,322] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 03:39:46,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 03:39:47,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 03:39:48,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 03:39:49,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 03:39:59,122] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 03:40:00,144] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 03:40:00,674] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 03:40:16,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 03:40:17,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 03:40:18,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 03:40:18,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 03:40:28,949] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 03:45:39,355] INFO : Step 45100/50000 rank2 | Train Loss: 0.8712 | Train Loss Unweighted: 0.4129 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0997 | Classification: 0.0837 | Classification_binary: 0.1445 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0910 | Regression_eta: 0.0012 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0248 | Classification_binary: 0.1367 | Regression_cos_phi: 0.0424 | Regression_energy: 0.0575 | Regression_eta: 0.0253 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0386 | LR: 1.21e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.1926s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8448 | max_loss: 0.1523 | max_backward: 2.1445 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 1.0003 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7489.1 | uss_MB: 6883.8 | child_rss_MB: 4866.9 | allocated_MB: 499.4 | reserved_MB: 24746.0 | inactive_split_MB: 784.5 +rank2 [2026-10-30 03:50:46,375] INFO : Step 45200/50000 rank2 | Train Loss: 0.9130 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1058 | Classification: 0.0776 | Classification_binary: 0.1464 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0963 | Regression_eta: 0.0015 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1427 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0557 | Regression_eta: 0.0365 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0637 | LR: 1.20e-04 | DataLoad Time: 0.0150s | Model Forward Time: 6.5419s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1574 | max_backward: 1.6440 | max_optimizer_step: 0.2111 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1112 | Memory: rss_MB: 6132.8 | uss_MB: 4597.7 | child_rss_MB: 4866.1 | allocated_MB: 499.7 | reserved_MB: 26730.0 | inactive_split_MB: 1073.5 +rank2 [2026-10-30 03:55:44,003] INFO : Step 45300/50000 rank2 | Train Loss: 0.8299 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.1126 | Classification: 0.0744 | Classification_binary: 0.1207 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0433 | Regression_eta: 0.0323 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0425 | LR: 1.19e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1810s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0827 | max_forward: 0.7966 | max_loss: 0.1432 | max_backward: 1.5141 | max_optimizer_step: 0.2646 | max_tensorboard_logging: 0.6448 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6128.3 | uss_MB: 5538.9 | child_rss_MB: 4864.4 | allocated_MB: 495.4 | reserved_MB: 27170.0 | inactive_split_MB: 881.8 +rank2 [2026-10-30 04:00:47,338] INFO : Step 45400/50000 rank2 | Train Loss: 0.8255 | Train Loss Unweighted: 0.4011 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.0984 | Classification: 0.0907 | Classification_binary: 0.1338 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0870 | Regression_eta: 0.0011 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1033 | Regression_cos_phi: 0.0304 | Regression_energy: 0.0388 | Regression_eta: 0.0308 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0518 | LR: 1.19e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.3356s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7846 | max_loss: 0.1934 | max_backward: 1.9857 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 0.7880 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6124.9 | uss_MB: 4519.9 | child_rss_MB: 4862.5 | allocated_MB: 499.2 | reserved_MB: 27170.0 | inactive_split_MB: 989.3 +rank2 [2026-10-30 04:05:44,853] INFO : Step 45500/50000 rank2 | Train Loss: 0.8945 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1090 | Classification: 0.0825 | Classification_binary: 0.1567 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0012 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1558 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0429 | Regression_eta: 0.0312 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0685 | LR: 1.18e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4593s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8686 | max_loss: 0.1508 | max_backward: 1.9581 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.4909 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6130.5 | uss_MB: 5543.1 | child_rss_MB: 4864.9 | allocated_MB: 498.4 | reserved_MB: 28766.0 | inactive_split_MB: 883.2 +rank2 [2026-10-30 04:10:39,743] INFO : Step 45600/50000 rank2 | Train Loss: 0.8704 | Train Loss Unweighted: 0.4121 | Train Loss Components: Auxiliary: 0.1840 | Cardinality: 0.1126 | Classification: 0.0802 | Classification_binary: 0.1324 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0983 | Regression_eta: 0.0010 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0533 | Regression_eta: 0.0341 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0659 | LR: 1.17e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.8859s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0163 | max_forward: 0.8161 | max_loss: 0.2729 | max_backward: 2.1846 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6122.7 | uss_MB: 5488.3 | child_rss_MB: 4871.5 | allocated_MB: 495.6 | reserved_MB: 28766.0 | inactive_split_MB: 1080.4 +rank2 [2026-10-30 04:15:37,194] INFO : Step 45700/50000 rank2 | Train Loss: 0.8006 | Train Loss Unweighted: 0.3784 | Train Loss Components: Auxiliary: 0.1681 | Cardinality: 0.0873 | Classification: 0.0773 | Classification_binary: 0.1111 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0012 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0338 | Classification_binary: 0.1042 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0564 | Regression_eta: 0.0316 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0591 | LR: 1.16e-04 | DataLoad Time: 0.0097s | Model Forward Time: 4.8297s | Phase Times: max_pre_step_barrier: 0.0327 | max_batch_to_device: 0.0149 | max_forward: 1.1858 | max_loss: 0.1429 | max_backward: 1.6968 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1366 | Memory: rss_MB: 6126.5 | uss_MB: 5540.3 | child_rss_MB: 4873.0 | allocated_MB: 496.6 | reserved_MB: 28766.0 | inactive_split_MB: 903.7 +rank2 [2026-10-30 04:20:28,799] INFO : Step 45800/50000 rank2 | Train Loss: 0.8566 | Train Loss Unweighted: 0.3982 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1097 | Classification: 0.0794 | Classification_binary: 0.1398 | HitGrouping: 0.1406 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.1689 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0673 | Regression_eta: 0.0385 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0546 | LR: 1.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.1206s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.9805 | max_loss: 0.5234 | max_backward: 2.3872 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.8153 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6131.1 | uss_MB: 5547.8 | child_rss_MB: 4874.9 | allocated_MB: 501.7 | reserved_MB: 28766.0 | inactive_split_MB: 1064.3 +rank2 [2026-10-30 04:25:16,810] INFO : Step 45900/50000 rank2 | Train Loss: 0.8754 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1042 | Classification: 0.0696 | Classification_binary: 0.1484 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0620 | Regression_eta: 0.0292 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0501 | LR: 1.15e-04 | DataLoad Time: 0.0148s | Model Forward Time: 5.8455s | Phase Times: max_pre_step_barrier: 0.0308 | max_batch_to_device: 0.0265 | max_forward: 1.7626 | max_loss: 0.1678 | max_backward: 2.6283 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 6124.4 | uss_MB: 5544.6 | child_rss_MB: 4869.8 | allocated_MB: 496.0 | reserved_MB: 29170.0 | inactive_split_MB: 983.4 +rank2 [2026-10-30 04:29:51,933] INFO : Step 46000/50000 rank2 | Train Loss: 0.7531 | Train Loss Unweighted: 0.3556 | Train Loss Components: Auxiliary: 0.1535 | Cardinality: 0.0870 | Classification: 0.0690 | Classification_binary: 0.1026 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0903 | Regression_eta: 0.0011 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1118 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0491 | Regression_eta: 0.0255 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0617 | LR: 1.14e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3114s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0329 | max_forward: 0.7497 | max_loss: 0.1389 | max_backward: 1.9140 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5104 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6122.4 | uss_MB: 5543.2 | child_rss_MB: 4869.8 | allocated_MB: 492.4 | reserved_MB: 29170.0 | inactive_split_MB: 973.8 +rank2 [2026-10-30 04:29:53,062] INFO : Running validation on rank2 +rank2 [2026-10-30 04:29:53,062] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 04:29:53,062] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 04:30:44,348] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 04:30:44,927] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 04:31:11,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 04:31:12,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 04:31:13,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 04:31:14,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 04:31:24,794] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 04:31:25,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 04:31:25,585] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 04:31:41,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 04:31:42,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 04:31:43,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 04:31:44,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 04:31:54,357] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 04:31:54,979] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 04:31:55,463] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 04:32:11,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 04:32:12,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 04:32:13,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 04:32:13,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 04:32:23,838] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 04:37:27,738] INFO : Step 46100/50000 rank2 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3769 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.0999 | Classification: 0.0743 | Classification_binary: 0.1253 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0010 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1471 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0511 | Regression_eta: 0.0390 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0527 | LR: 1.13e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.8723s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.9293 | max_loss: 0.1452 | max_backward: 2.3008 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.4968 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1219 | Memory: rss_MB: 7472.5 | uss_MB: 6889.9 | child_rss_MB: 4882.8 | allocated_MB: 498.3 | reserved_MB: 23298.0 | inactive_split_MB: 776.1 +rank2 [2026-10-30 04:42:19,615] INFO : Step 46200/50000 rank2 | Train Loss: 0.8661 | Train Loss Unweighted: 0.3994 | Train Loss Components: Auxiliary: 0.1947 | Cardinality: 0.1164 | Classification: 0.0793 | Classification_binary: 0.1511 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0833 | Regression_eta: 0.0011 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1658 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0473 | Regression_eta: 0.0243 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0398 | LR: 1.13e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.7269s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8070 | max_loss: 0.2423 | max_backward: 1.9860 | max_optimizer_step: 0.2482 | max_tensorboard_logging: 0.7539 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6199.1 | uss_MB: 5617.8 | child_rss_MB: 4859.8 | allocated_MB: 499.3 | reserved_MB: 25288.0 | inactive_split_MB: 1325.5 +rank2 [2026-10-30 04:46:59,453] INFO : Step 46300/50000 rank2 | Train Loss: 0.8634 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0916 | Classification: 0.0803 | Classification_binary: 0.1320 | HitGrouping: 0.1536 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1031 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1288 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0588 | Regression_eta: 0.0238 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0414 | LR: 1.12e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5158s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7891 | max_loss: 0.1470 | max_backward: 1.7900 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.7709 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6201.3 | uss_MB: 5618.4 | child_rss_MB: 4887.4 | allocated_MB: 498.4 | reserved_MB: 25290.0 | inactive_split_MB: 977.6 +rank2 [2026-10-30 04:51:36,900] INFO : Step 46400/50000 rank2 | Train Loss: 0.8499 | Train Loss Unweighted: 0.4102 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.0956 | Classification: 0.0755 | Classification_binary: 0.1172 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1124 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0518 | Regression_eta: 0.0278 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0501 | LR: 1.11e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9765s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7847 | max_loss: 0.2487 | max_backward: 1.9181 | max_optimizer_step: 0.3148 | max_tensorboard_logging: 0.9408 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6188.6 | uss_MB: 5608.9 | child_rss_MB: 4878.3 | allocated_MB: 501.6 | reserved_MB: 25290.0 | inactive_split_MB: 992.4 +rank2 [2026-10-30 04:56:10,901] INFO : Step 46500/50000 rank2 | Train Loss: 0.8687 | Train Loss Unweighted: 0.4055 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1002 | Classification: 0.0822 | Classification_binary: 0.1448 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0872 | Regression_eta: 0.0013 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0455 | Regression_eta: 0.0567 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0665 | LR: 1.11e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5409s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1839 | max_backward: 1.9629 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6193.9 | uss_MB: 5618.8 | child_rss_MB: 4880.0 | allocated_MB: 498.8 | reserved_MB: 29306.0 | inactive_split_MB: 924.4 +rank2 [2026-10-30 05:00:44,088] INFO : Step 46600/50000 rank2 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4049 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1082 | Classification: 0.0757 | Classification_binary: 0.1508 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1368 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0440 | Regression_eta: 0.0297 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0573 | LR: 1.10e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2103s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7939 | max_loss: 0.2001 | max_backward: 1.6859 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6188.5 | uss_MB: 5615.4 | child_rss_MB: 4846.0 | allocated_MB: 496.0 | reserved_MB: 29306.0 | inactive_split_MB: 901.7 +rank2 [2026-10-30 05:05:18,398] INFO : Step 46700/50000 rank2 | Train Loss: 0.8417 | Train Loss Unweighted: 0.3855 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0982 | Classification: 0.0807 | Classification_binary: 0.1184 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0916 | Regression_eta: 0.0015 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1112 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0515 | Regression_eta: 0.0240 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0450 | LR: 1.10e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7638s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8551 | max_loss: 0.5124 | max_backward: 1.9004 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.7762 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1375 | Memory: rss_MB: 6185.4 | uss_MB: 5616.8 | child_rss_MB: 4868.3 | allocated_MB: 495.0 | reserved_MB: 29308.0 | inactive_split_MB: 1000.4 +rank2 [2026-10-30 05:09:48,389] INFO : Step 46800/50000 rank2 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3911 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0920 | Classification: 0.0920 | Classification_binary: 0.1296 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0828 | Regression_eta: 0.0013 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0486 | Regression_eta: 0.0239 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0417 | LR: 1.09e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4613s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7934 | max_loss: 0.1440 | max_backward: 1.7630 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.7219 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6183.1 | uss_MB: 5617.4 | child_rss_MB: 4871.4 | allocated_MB: 495.6 | reserved_MB: 29308.0 | inactive_split_MB: 1332.0 +rank2 [2026-10-30 05:14:19,738] INFO : Step 46900/50000 rank2 | Train Loss: 0.8496 | Train Loss Unweighted: 0.4142 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0976 | Classification: 0.0776 | Classification_binary: 0.1448 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0946 | Regression_eta: 0.0010 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1524 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0537 | Regression_eta: 0.0265 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0456 | LR: 1.09e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4519s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.9007 | max_loss: 0.1576 | max_backward: 1.8470 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6183.1 | uss_MB: 5616.1 | child_rss_MB: 4872.4 | allocated_MB: 495.4 | reserved_MB: 29310.0 | inactive_split_MB: 850.5 +rank2 [2026-10-30 05:18:49,738] INFO : Step 47000/50000 rank2 | Train Loss: 0.7980 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0982 | Classification: 0.0774 | Classification_binary: 0.1196 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1096 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0484 | Regression_eta: 0.0318 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0354 | LR: 1.08e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8519s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 1.2405 | max_loss: 0.1502 | max_backward: 2.1273 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.7105 | max_tensorboard_flush: 0.0082 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6184.1 | uss_MB: 5617.4 | child_rss_MB: 4877.5 | allocated_MB: 494.9 | reserved_MB: 29310.0 | inactive_split_MB: 852.6 +rank2 [2026-10-30 05:18:50,808] INFO : Running validation on rank2 +rank2 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 05:19:41,787] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 05:19:42,336] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 05:20:07,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 05:20:08,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 05:20:09,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 05:20:10,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 05:20:20,954] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 05:20:21,505] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 05:20:22,014] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 05:20:38,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 05:20:39,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 05:20:40,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 05:20:41,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 05:20:51,048] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 05:20:51,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 05:20:52,198] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 05:21:07,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 05:21:08,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 05:21:09,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 05:21:09,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 05:21:19,843] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 05:26:12,381] INFO : Step 47100/50000 rank2 | Train Loss: 0.8046 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0851 | Classification: 0.0882 | Classification_binary: 0.1121 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0936 | Regression_eta: 0.0010 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0484 | Regression_eta: 0.0370 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0796 | LR: 1.07e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5738s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8250 | max_loss: 0.1453 | max_backward: 1.6267 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8108 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 7449.0 | uss_MB: 6878.6 | child_rss_MB: 4837.4 | allocated_MB: 494.1 | reserved_MB: 25416.0 | inactive_split_MB: 894.5 +rank2 [2026-10-30 05:30:44,527] INFO : Step 47200/50000 rank2 | Train Loss: 0.8884 | Train Loss Unweighted: 0.4191 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1138 | Classification: 0.0793 | Classification_binary: 0.1489 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0010 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1508 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0504 | Regression_eta: 0.0266 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0426 | LR: 1.07e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9171s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7960 | max_loss: 0.1482 | max_backward: 1.9261 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.9435 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0661 | Memory: rss_MB: 6249.9 | uss_MB: 5682.5 | child_rss_MB: 4852.8 | allocated_MB: 496.8 | reserved_MB: 26534.0 | inactive_split_MB: 847.6 +rank2 [2026-10-30 05:35:15,255] INFO : Step 47300/50000 rank2 | Train Loss: 0.8547 | Train Loss Unweighted: 0.3920 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1049 | Classification: 0.0798 | Classification_binary: 0.1368 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0012 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1407 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0535 | Regression_eta: 0.0347 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0475 | LR: 1.06e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5894s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7649 | max_loss: 0.2313 | max_backward: 1.8030 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8438 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6253.9 | uss_MB: 5688.5 | child_rss_MB: 4855.9 | allocated_MB: 496.9 | reserved_MB: 28534.0 | inactive_split_MB: 933.0 +rank2 [2026-10-30 05:39:47,014] INFO : Step 47400/50000 rank2 | Train Loss: 0.7606 | Train Loss Unweighted: 0.3755 | Train Loss Components: Auxiliary: 0.1640 | Cardinality: 0.0704 | Classification: 0.0737 | Classification_binary: 0.1072 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0960 | Regression_eta: 0.0010 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.0884 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0389 | Regression_eta: 0.0224 | Regression_pt: 0.0406 | Regression_sin_phi: 0.0363 | LR: 1.06e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5302s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0275 | max_forward: 0.8465 | max_loss: 0.1482 | max_backward: 1.7103 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.8388 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 6241.7 | uss_MB: 5678.8 | child_rss_MB: 4860.9 | allocated_MB: 496.4 | reserved_MB: 28534.0 | inactive_split_MB: 913.7 +rank2 [2026-10-30 05:44:18,077] INFO : Step 47500/50000 rank2 | Train Loss: 0.8134 | Train Loss Unweighted: 0.3681 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1057 | Classification: 0.0739 | Classification_binary: 0.1303 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0805 | Regression_eta: 0.0011 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1358 | Regression_cos_phi: 0.0367 | Regression_energy: 0.0452 | Regression_eta: 0.0268 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0503 | LR: 1.06e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7788s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1751 | max_backward: 1.9857 | max_optimizer_step: 0.3263 | max_tensorboard_logging: 0.7363 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6249.7 | uss_MB: 5690.9 | child_rss_MB: 4845.9 | allocated_MB: 498.3 | reserved_MB: 28534.0 | inactive_split_MB: 851.5 +rank2 [2026-10-30 05:48:48,523] INFO : Step 47600/50000 rank2 | Train Loss: 0.8663 | Train Loss Unweighted: 0.4076 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1090 | Classification: 0.0820 | Classification_binary: 0.1491 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0865 | Regression_eta: 0.0011 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1422 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0495 | Regression_eta: 0.0247 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0345 | LR: 1.05e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6216s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.9385 | max_loss: 0.1530 | max_backward: 2.1678 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6249.7 | uss_MB: 5691.7 | child_rss_MB: 4853.8 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 843.5 +rank2 [2026-10-30 05:53:19,839] INFO : Step 47700/50000 rank2 | Train Loss: 0.8615 | Train Loss Unweighted: 0.4088 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0954 | Classification: 0.0764 | Classification_binary: 0.1291 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1002 | Regression_eta: 0.0011 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0507 | Regression_eta: 0.0390 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0727 | LR: 1.05e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2184s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7406 | max_loss: 0.2391 | max_backward: 1.7059 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1444 | Memory: rss_MB: 6248.7 | uss_MB: 5687.5 | child_rss_MB: 4851.8 | allocated_MB: 493.9 | reserved_MB: 28534.0 | inactive_split_MB: 895.3 +rank2 [2026-10-30 05:57:52,843] INFO : Step 47800/50000 rank2 | Train Loss: 0.8267 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.0942 | Classification: 0.0882 | Classification_binary: 0.1186 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0870 | Regression_eta: 0.0012 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0471 | Regression_eta: 0.0410 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0506 | LR: 1.04e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.0876s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.7869 | max_loss: 0.5030 | max_backward: 2.0522 | max_optimizer_step: 0.2267 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6249.7 | uss_MB: 5692.3 | child_rss_MB: 4857.3 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 916.9 +rank2 [2026-10-30 06:02:25,106] INFO : Step 47900/50000 rank2 | Train Loss: 0.8423 | Train Loss Unweighted: 0.4000 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.0845 | Classification: 0.0794 | Classification_binary: 0.1416 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1307 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0485 | Regression_eta: 0.0445 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0389 | LR: 1.04e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5073s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7994 | max_loss: 0.1511 | max_backward: 1.9354 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.6685 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6247.4 | uss_MB: 5692.2 | child_rss_MB: 4860.2 | allocated_MB: 496.8 | reserved_MB: 30642.0 | inactive_split_MB: 779.5 +rank2 [2026-10-30 06:06:56,644] INFO : Step 48000/50000 rank2 | Train Loss: 0.8641 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0972 | Classification: 0.0768 | Classification_binary: 0.1396 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0890 | Regression_eta: 0.0015 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1454 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0471 | Regression_eta: 0.0814 | Regression_pt: 0.0476 | Regression_sin_phi: 0.1286 | LR: 1.04e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4068s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0057 | max_forward: 0.8017 | max_loss: 0.1550 | max_backward: 1.6990 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.7498 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6247.1 | uss_MB: 5690.2 | child_rss_MB: 4855.9 | allocated_MB: 495.1 | reserved_MB: 30642.0 | inactive_split_MB: 896.2 +rank2 [2026-10-30 06:06:57,654] INFO : Running validation on rank2 +rank2 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 06:07:49,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:07:49,560] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 06:08:14,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 06:08:15,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 06:08:16,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 06:08:17,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 06:08:27,452] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 06:08:28,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:08:28,620] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 06:08:44,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 06:08:45,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 06:08:46,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 06:08:47,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 06:08:57,570] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 06:08:58,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:08:58,714] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 06:09:13,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 06:09:14,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 06:09:15,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 06:09:16,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 06:09:26,316] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 06:14:16,583] INFO : Step 48100/50000 rank2 | Train Loss: 0.8226 | Train Loss Unweighted: 0.3895 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.1043 | Classification: 0.0717 | Classification_binary: 0.1116 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1023 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0242 | Classification_binary: 0.0757 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0460 | Regression_eta: 0.0307 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0468 | LR: 1.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4971s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7477 | max_loss: 0.1426 | max_backward: 1.7972 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.6718 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1303 | Memory: rss_MB: 7752.6 | uss_MB: 7191.3 | child_rss_MB: 4880.7 | allocated_MB: 494.1 | reserved_MB: 23078.0 | inactive_split_MB: 1096.3 +rank2 [2026-10-30 06:18:47,186] INFO : Step 48200/50000 rank2 | Train Loss: 0.8500 | Train Loss Unweighted: 0.3835 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1027 | Classification: 0.0771 | Classification_binary: 0.1374 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0827 | Regression_eta: 0.0012 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1554 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0457 | Regression_eta: 0.0288 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0628 | LR: 1.03e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2122s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8132 | max_loss: 0.2545 | max_backward: 2.1167 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1241 | Memory: rss_MB: 6310.4 | uss_MB: 5751.6 | child_rss_MB: 4867.3 | allocated_MB: 496.6 | reserved_MB: 25094.0 | inactive_split_MB: 913.4 +rank2 [2026-10-30 06:23:18,411] INFO : Step 48300/50000 rank2 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4303 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1089 | Classification: 0.0832 | Classification_binary: 0.1567 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0013 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0373 | Regression_energy: 0.0473 | Regression_eta: 0.0264 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0416 | LR: 1.03e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4478s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8079 | max_loss: 0.1526 | max_backward: 1.9733 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6317.4 | uss_MB: 5760.9 | child_rss_MB: 4872.1 | allocated_MB: 498.5 | reserved_MB: 25098.0 | inactive_split_MB: 771.0 +rank2 [2026-10-30 06:27:49,027] INFO : Step 48400/50000 rank2 | Train Loss: 0.7961 | Train Loss Unweighted: 0.3733 | Train Loss Components: Auxiliary: 0.1682 | Cardinality: 0.0877 | Classification: 0.0685 | Classification_binary: 0.1191 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0012 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0321 | Classification_binary: 0.1226 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0410 | Regression_eta: 0.0238 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0424 | LR: 1.02e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.2394s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7916 | max_loss: 0.1471 | max_backward: 1.6943 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5940 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6314.4 | uss_MB: 5759.4 | child_rss_MB: 4842.4 | allocated_MB: 496.3 | reserved_MB: 25098.0 | inactive_split_MB: 1087.5 +rank2 [2026-10-30 06:32:21,028] INFO : Step 48500/50000 rank2 | Train Loss: 0.8668 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0816 | Classification: 0.0832 | Classification_binary: 0.1257 | HitGrouping: 0.1542 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0717 | Regression_eta: 0.0323 | Regression_pt: 0.0766 | Regression_sin_phi: 0.0539 | LR: 1.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3834s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7587 | max_loss: 0.1606 | max_backward: 1.7573 | max_optimizer_step: 0.3024 | max_tensorboard_logging: 0.5851 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6309.3 | uss_MB: 5759.8 | child_rss_MB: 4851.4 | allocated_MB: 498.9 | reserved_MB: 25098.0 | inactive_split_MB: 1192.4 +rank2 [2026-10-30 06:36:52,988] INFO : Step 48600/50000 rank2 | Train Loss: 0.8434 | Train Loss Unweighted: 0.4010 | Train Loss Components: Auxiliary: 0.1850 | Cardinality: 0.0964 | Classification: 0.0791 | Classification_binary: 0.1425 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0470 | Regression_eta: 0.0272 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0410 | LR: 1.02e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4859s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7760 | max_loss: 0.1505 | max_backward: 1.8991 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0654 | Memory: rss_MB: 6311.3 | uss_MB: 5762.6 | child_rss_MB: 4856.0 | allocated_MB: 497.0 | reserved_MB: 26848.0 | inactive_split_MB: 777.5 +rank2 [2026-10-30 06:41:29,683] INFO : Step 48700/50000 rank2 | Train Loss: 0.9079 | Train Loss Unweighted: 0.4266 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1195 | Classification: 0.0920 | Classification_binary: 0.1456 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1272 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0590 | Regression_eta: 0.0260 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0666 | LR: 1.02e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5443s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7797 | max_loss: 0.1578 | max_backward: 1.7107 | max_optimizer_step: 0.5280 | max_tensorboard_logging: 0.7650 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6308.9 | uss_MB: 5759.1 | child_rss_MB: 4857.7 | allocated_MB: 496.9 | reserved_MB: 26848.0 | inactive_split_MB: 891.8 +rank2 [2026-10-30 06:46:02,598] INFO : Step 48800/50000 rank2 | Train Loss: 0.8011 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1685 | Cardinality: 0.0863 | Classification: 0.0837 | Classification_binary: 0.1116 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0965 | Regression_eta: 0.0012 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0334 | Classification_binary: 0.0770 | Regression_cos_phi: 0.0359 | Regression_energy: 0.0457 | Regression_eta: 0.0203 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0248 | LR: 1.01e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4997s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7374 | max_loss: 0.2432 | max_backward: 1.8459 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.7427 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6306.9 | uss_MB: 5759.1 | child_rss_MB: 4857.0 | allocated_MB: 493.9 | reserved_MB: 26848.0 | inactive_split_MB: 991.0 +rank2 [2026-10-30 06:50:45,084] INFO : Step 48900/50000 rank2 | Train Loss: 0.8551 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.1109 | Classification: 0.0760 | Classification_binary: 0.1310 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1355 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0425 | Regression_eta: 0.0347 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0726 | LR: 1.01e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5412s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8768 | max_loss: 0.1470 | max_backward: 2.0816 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6306.9 | uss_MB: 5761.7 | child_rss_MB: 4858.9 | allocated_MB: 499.5 | reserved_MB: 26852.0 | inactive_split_MB: 1398.0 +rank2 [2026-10-30 06:55:20,574] INFO : Step 49000/50000 rank2 | Train Loss: 0.9005 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.2010 | Cardinality: 0.1248 | Classification: 0.0800 | Classification_binary: 0.1499 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0939 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0407 | Regression_energy: 0.0444 | Regression_eta: 0.0230 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0435 | LR: 1.01e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9031s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0439 | max_forward: 0.8365 | max_loss: 0.1536 | max_backward: 1.9893 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8890 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6309.3 | uss_MB: 5762.6 | child_rss_MB: 4865.2 | allocated_MB: 497.5 | reserved_MB: 26852.0 | inactive_split_MB: 943.6 +rank2 [2026-10-30 06:55:21,669] INFO : Running validation on rank2 +rank2 [2026-10-30 06:55:21,669] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 06:55:21,669] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 06:56:13,350] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:56:13,862] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 06:56:38,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 06:56:39,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 06:56:41,379] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 06:56:42,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 06:56:52,422] INFO : Time taken to make predictions on device 2 is: 0.26 min +rank2 [2026-10-30 06:56:52,550] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:56:53,029] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 06:57:08,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 06:57:09,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 06:57:10,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 06:57:11,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 06:57:21,471] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 06:57:22,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 06:57:23,149] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 06:57:38,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 06:57:39,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 06:57:40,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 06:57:41,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 06:57:51,240] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 07:02:45,630] INFO : Step 49100/50000 rank2 | Train Loss: 0.8219 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.0880 | Classification: 0.0841 | Classification_binary: 0.1303 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0855 | Regression_eta: 0.0012 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1092 | Regression_cos_phi: 0.0339 | Regression_energy: 0.0454 | Regression_eta: 0.0413 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0524 | LR: 1.01e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3705s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7738 | max_loss: 0.1431 | max_backward: 1.6939 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6276 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 7720.3 | uss_MB: 7170.2 | child_rss_MB: 4829.0 | allocated_MB: 495.9 | reserved_MB: 24730.0 | inactive_split_MB: 936.0 +rank2 [2026-10-30 07:07:18,676] INFO : Step 49200/50000 rank2 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3843 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0856 | Classification: 0.0829 | Classification_binary: 0.1188 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0892 | Regression_eta: 0.0011 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.0893 | Regression_cos_phi: 0.0779 | Regression_energy: 0.0466 | Regression_eta: 0.0316 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0338 | LR: 1.01e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7722s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7590 | max_loss: 0.5032 | max_backward: 1.9826 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.7926 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6365.4 | uss_MB: 5817.0 | child_rss_MB: 4830.4 | allocated_MB: 494.5 | reserved_MB: 26752.0 | inactive_split_MB: 1117.5 +rank2 [2026-10-30 07:11:50,862] INFO : Step 49300/50000 rank2 | Train Loss: 0.9397 | Train Loss Unweighted: 0.4282 | Train Loss Components: Auxiliary: 0.2065 | Cardinality: 0.1351 | Classification: 0.0889 | Classification_binary: 0.1552 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0901 | Regression_eta: 0.0012 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0471 | Regression_eta: 0.0382 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0406 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4922s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0070 | max_forward: 0.7807 | max_loss: 0.1520 | max_backward: 1.8619 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6076 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6376.6 | uss_MB: 5827.0 | child_rss_MB: 4814.1 | allocated_MB: 496.4 | reserved_MB: 26752.0 | inactive_split_MB: 1120.6 +rank2 [2026-10-30 07:16:22,252] INFO : Step 49400/50000 rank2 | Train Loss: 0.8870 | Train Loss Unweighted: 0.4213 | Train Loss Components: Auxiliary: 0.1937 | Cardinality: 0.1005 | Classification: 0.0815 | Classification_binary: 0.1465 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0492 | Regression_eta: 0.0296 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0313 | LR: 1.00e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.2725s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8046 | max_loss: 0.1540 | max_backward: 1.7244 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.6030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6373.7 | uss_MB: 5825.4 | child_rss_MB: 4813.8 | allocated_MB: 498.5 | reserved_MB: 28804.0 | inactive_split_MB: 886.2 +rank2 [2026-10-30 07:20:55,284] INFO : Step 49500/50000 rank2 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3949 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0991 | Classification: 0.0800 | Classification_binary: 0.1101 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0013 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1298 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0626 | Regression_eta: 0.0269 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0727 | LR: 1.00e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.2160s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0074 | max_forward: 0.9924 | max_loss: 0.1865 | max_backward: 2.5853 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.6095 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0429 | Memory: rss_MB: 6377.1 | uss_MB: 5695.1 | child_rss_MB: 4830.9 | allocated_MB: 495.9 | reserved_MB: 28804.0 | inactive_split_MB: 849.6 +rank2 [2026-10-30 07:25:48,132] INFO : Step 49600/50000 rank2 | Train Loss: 0.9075 | Train Loss Unweighted: 0.4421 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1021 | Classification: 0.0939 | Classification_binary: 0.1401 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1483 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0547 | Regression_eta: 0.0422 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0765 | LR: 1.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4389s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7852 | max_loss: 0.1445 | max_backward: 2.0967 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4754 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6378.7 | uss_MB: 5829.2 | child_rss_MB: 4808.2 | allocated_MB: 497.8 | reserved_MB: 28804.0 | inactive_split_MB: 845.2 +rank2 [2026-10-30 07:30:32,223] INFO : Step 49700/50000 rank2 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4307 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1074 | Classification: 0.0931 | Classification_binary: 0.1611 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0865 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0480 | Regression_eta: 0.0290 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0447 | LR: 1.00e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6421s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.8169 | max_loss: 0.1522 | max_backward: 2.0553 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.4937 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1402 | Memory: rss_MB: 6379.2 | uss_MB: 5828.6 | child_rss_MB: 4829.3 | allocated_MB: 499.3 | reserved_MB: 30718.0 | inactive_split_MB: 838.8 +rank2 [2026-10-30 07:35:17,054] INFO : Step 49800/50000 rank2 | Train Loss: 0.8342 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.1036 | Classification: 0.0769 | Classification_binary: 0.1292 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1395 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0668 | Regression_eta: 0.0379 | Regression_pt: 0.0723 | Regression_sin_phi: 0.0454 | LR: 1.00e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.1030s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0329 | max_forward: 0.7490 | max_loss: 0.1889 | max_backward: 2.2249 | max_optimizer_step: 0.2452 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6374.9 | uss_MB: 5812.4 | child_rss_MB: 4828.7 | allocated_MB: 494.4 | reserved_MB: 30718.0 | inactive_split_MB: 1134.4 +rank2 [2026-10-30 07:39:58,302] INFO : Step 49900/50000 rank2 | Train Loss: 0.8436 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.0947 | Classification: 0.0943 | Classification_binary: 0.1179 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0982 | Regression_eta: 0.0012 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1054 | Regression_cos_phi: 0.0360 | Regression_energy: 0.0517 | Regression_eta: 0.0241 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0407 | LR: 1.00e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4173s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7496 | max_loss: 0.5142 | max_backward: 1.9365 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6373.9 | uss_MB: 5825.7 | child_rss_MB: 4831.8 | allocated_MB: 494.8 | reserved_MB: 30718.0 | inactive_split_MB: 1090.4 +rank2 [2026-10-30 07:44:45,888] INFO : Step 50000/50000 rank2 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4040 | Train Loss Components: Auxiliary: 0.1921 | Cardinality: 0.1113 | Classification: 0.0818 | Classification_binary: 0.1436 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0872 | Regression_eta: 0.0014 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1296 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0445 | Regression_eta: 0.0280 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0523 | LR: 1.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.6376s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 1.3997 | max_loss: 0.5678 | max_backward: 3.0655 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6373.9 | uss_MB: 5825.4 | child_rss_MB: 4827.6 | allocated_MB: 497.9 | reserved_MB: 30720.0 | inactive_split_MB: 906.1 +rank2 [2026-10-30 07:44:46,919] INFO : Running validation on rank2 +rank2 [2026-10-30 07:44:46,920] INFO : ResumableSampler physical:valid resetting +rank2 [2026-10-30 07:44:46,920] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank2 [2026-10-30 07:45:38,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 07:45:38,997] INFO : Running predictions on cld_edm_ttbar_hits +rank2 [2026-10-30 07:46:04,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_2_0.parquet +rank2 [2026-10-30 07:46:05,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_2_1.parquet +rank2 [2026-10-30 07:46:06,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_2_2.parquet +rank2 [2026-10-30 07:46:07,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_2_3.parquet +rank2 [2026-10-30 07:46:17,265] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 07:46:17,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 07:46:17,995] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank2 [2026-10-30 07:46:33,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_2_0.parquet +rank2 [2026-10-30 07:46:34,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_2_1.parquet +rank2 [2026-10-30 07:46:35,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_2_2.parquet +rank2 [2026-10-30 07:46:36,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_2_3.parquet +rank2 [2026-10-30 07:46:46,773] INFO : Time taken to make predictions on device 2 is: 0.25 min +rank2 [2026-10-30 07:46:47,616] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank2 [2026-10-30 07:46:48,154] INFO : Running predictions on cld_edm_qq_hits +rank2 [2026-10-30 07:47:03,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_2_0.parquet +rank2 [2026-10-30 07:47:04,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_2_1.parquet +rank2 [2026-10-30 07:47:05,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_2_2.parquet +rank2 [2026-10-30 07:47:06,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_2_3.parquet +rank2 [2026-10-30 07:47:16,113] INFO : Time taken to make predictions on device 2 is: 0.23 min +rank2 [2026-10-30 07:47:17,234] INFO : Training completed. Total time on device 2: 1017.501min diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.3 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.3 new file mode 100644 index 0000000000000000000000000000000000000000..baa668e2e8155009422d674f39d0eec104a5d1cc --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.3 @@ -0,0 +1,2631 @@ +rank3 [2026-10-28 11:12:32,442] INFO : Initializing model with seed=12345; process seed=12348 +rank3 [2026-10-28 11:12:32,442] INFO : configured dtype=torch.bfloat16 for autocast +rank3 [2026-10-28 11:12:54,158] INFO : Instantiating model +rank3 [2026-10-28 11:12:54,158] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank3 [2026-10-28 11:12:54,158] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank3 [2026-10-28 11:12:54,177] INFO : Input encoding initialization took 0.02s +rank3 [2026-10-28 11:12:54,177] INFO : nn0 parameters: 171776 +rank3 [2026-10-28 11:12:54,177] INFO : Initializing shared backbone layers, num_convs=6 +rank3 [2026-10-28 11:12:54,177] INFO : Initializing backbone layer 0 +rank3 [2026-10-28 11:12:54,215] INFO : Initializing backbone layer 1 +rank3 [2026-10-28 11:12:54,230] INFO : Initializing backbone layer 2 +rank3 [2026-10-28 11:12:54,243] INFO : Initializing backbone layer 3 +rank3 [2026-10-28 11:12:54,252] INFO : Initializing backbone layer 4 +rank3 [2026-10-28 11:12:54,260] INFO : Initializing backbone layer 5 +rank3 [2026-10-28 11:12:54,269] INFO : Backbone initialization took 0.09s +rank3 [2026-10-28 11:12:54,269] INFO : backbone parameters: 7901376 +rank3 [2026-10-28 11:12:54,269] INFO : Initializing output DNNs +rank3 [2026-10-28 11:12:54,303] INFO : Output DNNs initialization took 0.03s +rank3 [2026-10-28 11:12:54,303] INFO : backbone_mode=BackboneMode.SHARED +rank3 [2026-10-28 11:12:54,303] INFO : jagged_attention enabled=False +rank3 [2026-10-28 11:12:54,303] INFO : flash_attn_varlen enabled=False +rank3 [2026-10-28 11:12:54,304] INFO : task_queries enabled=False +rank3 [2026-10-28 11:12:54,304] INFO : classification_norm parameters: 0 +rank3 [2026-10-28 11:12:54,304] INFO : regression_norm parameters: 0 +rank3 [2026-10-28 11:12:54,304] INFO : classification_query parameters: 0 +rank3 [2026-10-28 11:12:54,304] INFO : regression_query parameters: 0 +rank3 [2026-10-28 11:12:54,304] INFO : classification_readout parameters: 0 +rank3 [2026-10-28 11:12:54,304] INFO : regression_readout parameters: 0 +rank3 [2026-10-28 11:12:54,304] INFO : output_mode=set +rank3 [2026-10-28 11:12:54,305] INFO : set_decoder parameters: 4713812 +rank3 [2026-10-28 11:12:54,305] INFO : nn_binary_particle parameters: 0 +rank3 [2026-10-28 11:12:54,305] INFO : nn_pid parameters: 0 +rank3 [2026-10-28 11:12:54,305] INFO : nn_pt parameters: 0 +rank3 [2026-10-28 11:12:54,305] INFO : nn_eta parameters: 0 +rank3 [2026-10-28 11:12:54,305] INFO : nn_sin_phi parameters: 0 +rank3 [2026-10-28 11:12:54,305] INFO : nn_cos_phi parameters: 0 +rank3 [2026-10-28 11:12:54,305] INFO : nn_energy parameters: 0 +rank3 [2026-10-28 11:12:54,306] INFO : Total MLPF parameters: 12786964 +rank3 [2026-10-28 11:12:54,307] INFO : MLPF __init__ done +rank3 [2026-10-28 11:12:54,307] INFO : Instantiated model +rank3 [2026-10-28 11:12:54,307] INFO : Moving model to device rank=3 +rank3 [2026-10-28 11:12:54,349] INFO : Moved model to device rank=3 +rank3 [2026-10-28 11:12:54,351] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank3 [2026-10-28 11:12:54,351] INFO : Configured model for SyncBatchNorm rank=3 +rank3 [2026-10-28 11:12:54,648] INFO : Configured model for DistributedDataParallel rank=3 +rank3 [2026-10-28 11:12:54,809] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank3 [2026-10-28 11:12:54,810] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank3 [2026-10-28 11:12:54,810] INFO : Trainable parameters: 12786964 +rank3 [2026-10-28 11:12:54,811] INFO : Non-trainable parameters: 23268 +rank3 [2026-10-28 11:12:54,811] INFO : Total parameters: 12810232 +rank3 [2026-10-28 11:12:54,814] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank3 [2026-10-28 11:12:54,814] INFO : Getting dataloaders +rank3 [2026-10-28 11:12:54,814] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:03,740] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:04,812] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,147] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-28 11:13:06,148] INFO : Creating WeightedMixIterator with 1 data loaders. +rank3 [2026-10-28 11:13:06,148] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,420] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,681] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:13:06,941] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-28 11:13:06,941] INFO : Creating InterleavedIterator with 1 data loaders. +rank3 [2026-10-28 11:13:21,016] INFO : Got dataloaders +rank3 [2026-10-28 11:13:21,016] INFO : loader split=train rank=3 len=21112 +rank3 [2026-10-28 11:13:21,016] INFO : loader split=valid rank=3 len=12 +rank3 [2026-10-28 11:13:21,016] INFO : Starting training from step 1 to 50000 on rank 3 of 8 +rank3 [2026-10-28 11:13:21,016] INFO : Creating train_iterator +rank3 [2026-10-28 11:13:21,017] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-28 11:13:35,797] INFO : Created train_iterator +rank3 [2026-10-28 11:18:11,208] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5408100029081107, 'Classification': 0.8863177470117808, 'Regression_pt': 0.9386234817653895, 'Regression_eta': 0.019238746587652712, 'Regression_sin_phi': 0.006576384851359762, 'Regression_cos_phi': 0.006405904629500583, 'Regression_energy': 1.0145655398070812}, weights={'Classification_binary': 1.0, 'Classification': 0.6101762056350708, 'Regression_pt': 0.5761735439300537, 'Regression_eta': 28.110458374023438, 'Regression_sin_phi': 82.23515319824219, 'Regression_cos_phi': 84.42367553710938, 'Regression_energy': 0.5330458879470825} +rank3 [2026-10-28 11:18:13,718] INFO : Step 100/50000 rank3 | Train Loss: 4.2631 | Train Loss Unweighted: 2.7366 | Train Loss Components: Auxiliary: 0.7941 | Cardinality: 0.3942 | Classification: 0.6312 | Classification_binary: 0.4549 | HitGrouping: 0.3381 | Regression_cos_phi: 0.0047 | Regression_energy: 0.8263 | Regression_eta: 0.0122 | Regression_pt: 0.8026 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.6266 | Classification_binary: 0.4273 | Regression_cos_phi: 0.0038 | Regression_energy: 0.9451 | Regression_eta: 0.0099 | Regression_pt: 0.9091 | Regression_sin_phi: 0.0040 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 3.5379s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6509 | max_loss: 0.1961 | max_backward: 1.4166 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4769.1 | uss_MB: 3475.5 | child_rss_MB: 6372.1 | allocated_MB: 446.7 | reserved_MB: 24804.0 | inactive_split_MB: 1063.6 +rank3 [2026-10-28 11:22:42,538] INFO : Step 200/50000 rank3 | Train Loss: 4.2029 | Train Loss Unweighted: 2.5447 | Train Loss Components: Auxiliary: 0.8074 | Cardinality: 0.4500 | Classification: 0.5373 | Classification_binary: 0.5135 | HitGrouping: 0.3101 | Regression_cos_phi: 0.0042 | Regression_energy: 0.7508 | Regression_eta: 0.0093 | Regression_pt: 0.7252 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.3086 | Classification_binary: 0.5157 | Regression_cos_phi: 0.3517 | Regression_energy: 0.3650 | Regression_eta: 0.2364 | Regression_pt: 0.3770 | Regression_sin_phi: 0.3470 | LR: 1.00e-03 | DataLoad Time: 0.0163s | Model Forward Time: 4.6621s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9214 | max_loss: 0.1502 | max_backward: 2.1220 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 4772.0 | uss_MB: 3480.3 | child_rss_MB: 6374.7 | allocated_MB: 449.0 | reserved_MB: 26252.0 | inactive_split_MB: 1054.4 +rank3 [2026-10-28 11:27:10,390] INFO : Step 300/50000 rank3 | Train Loss: 3.5114 | Train Loss Unweighted: 2.0975 | Train Loss Components: Auxiliary: 0.6791 | Cardinality: 0.3510 | Classification: 0.4201 | Classification_binary: 0.4975 | HitGrouping: 0.2993 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5752 | Regression_eta: 0.0073 | Regression_pt: 0.5905 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2609 | Classification_binary: 0.5018 | Regression_cos_phi: 0.3039 | Regression_energy: 0.3108 | Regression_eta: 0.2276 | Regression_pt: 0.3549 | Regression_sin_phi: 0.3174 | LR: 1.00e-03 | DataLoad Time: 0.0163s | Model Forward Time: 4.2599s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.7369 | max_loss: 0.1526 | max_backward: 1.7330 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.9030 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 4773.0 | uss_MB: 3484.3 | child_rss_MB: 6364.6 | allocated_MB: 446.1 | reserved_MB: 26252.0 | inactive_split_MB: 1068.0 +rank3 [2026-10-28 11:31:37,117] INFO : Step 400/50000 rank3 | Train Loss: 3.3615 | Train Loss Unweighted: 1.9456 | Train Loss Components: Auxiliary: 0.6310 | Cardinality: 0.3555 | Classification: 0.4058 | Classification_binary: 0.4436 | HitGrouping: 0.3006 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5363 | Regression_eta: 0.0080 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2562 | Classification_binary: 0.4648 | Regression_cos_phi: 0.2818 | Regression_energy: 0.2747 | Regression_eta: 0.2611 | Regression_pt: 0.2963 | Regression_sin_phi: 0.3252 | LR: 1.00e-03 | DataLoad Time: 0.0170s | Model Forward Time: 4.5024s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1151 | max_forward: 0.7642 | max_loss: 0.1384 | max_backward: 1.7906 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9359 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0880 | Memory: rss_MB: 4771.8 | uss_MB: 3484.8 | child_rss_MB: 6366.3 | allocated_MB: 446.8 | reserved_MB: 26252.0 | inactive_split_MB: 1064.0 +rank3 [2026-10-28 11:36:08,659] INFO : Step 500/50000 rank3 | Train Loss: 3.8257 | Train Loss Unweighted: 2.0630 | Train Loss Components: Auxiliary: 0.7010 | Cardinality: 0.4740 | Classification: 0.4380 | Classification_binary: 0.4817 | HitGrouping: 0.2961 | Regression_cos_phi: 0.0044 | Regression_energy: 0.5611 | Regression_eta: 0.0087 | Regression_pt: 0.5645 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2420 | Classification_binary: 0.4755 | Regression_cos_phi: 0.3360 | Regression_energy: 0.2793 | Regression_eta: 0.2199 | Regression_pt: 0.3074 | Regression_sin_phi: 0.3826 | LR: 1.00e-03 | DataLoad Time: 0.0155s | Model Forward Time: 5.1259s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0077 | max_forward: 1.2433 | max_loss: 0.1536 | max_backward: 2.4885 | max_optimizer_step: 0.1562 | max_tensorboard_logging: 1.0374 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0034 | Memory: rss_MB: 4768.9 | uss_MB: 3486.1 | child_rss_MB: 6361.5 | allocated_MB: 449.6 | reserved_MB: 26256.0 | inactive_split_MB: 929.2 +rank3 [2026-10-28 11:40:42,093] INFO : Step 600/50000 rank3 | Train Loss: 3.4702 | Train Loss Unweighted: 1.9733 | Train Loss Components: Auxiliary: 0.6509 | Cardinality: 0.3585 | Classification: 0.3825 | Classification_binary: 0.5061 | HitGrouping: 0.2915 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5321 | Regression_eta: 0.0074 | Regression_pt: 0.5377 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2416 | Classification_binary: 0.5321 | Regression_cos_phi: 0.3772 | Regression_energy: 0.3035 | Regression_eta: 0.2151 | Regression_pt: 0.3333 | Regression_sin_phi: 0.3269 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 4.5572s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8098 | max_loss: 0.1544 | max_backward: 1.9541 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 0.9377 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 4772.7 | uss_MB: 3486.1 | child_rss_MB: 6366.7 | allocated_MB: 449.5 | reserved_MB: 27396.0 | inactive_split_MB: 1022.5 +rank3 [2026-10-28 11:45:16,604] INFO : Step 700/50000 rank3 | Train Loss: 2.9693 | Train Loss Unweighted: 1.8072 | Train Loss Components: Auxiliary: 0.5907 | Cardinality: 0.2768 | Classification: 0.3175 | Classification_binary: 0.3969 | HitGrouping: 0.2848 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5321 | Regression_eta: 0.0061 | Regression_pt: 0.5492 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1999 | Classification_binary: 0.3815 | Regression_cos_phi: 0.2129 | Regression_energy: 0.2856 | Regression_eta: 0.1500 | Regression_pt: 0.3241 | Regression_sin_phi: 0.2129 | LR: 1.00e-03 | DataLoad Time: 0.0200s | Model Forward Time: 3.9470s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7910 | max_loss: 0.2436 | max_backward: 1.8672 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.8 | uss_MB: 3484.3 | child_rss_MB: 6369.5 | allocated_MB: 446.1 | reserved_MB: 27396.0 | inactive_split_MB: 1066.2 +rank3 [2026-10-28 11:49:48,175] INFO : Step 800/50000 rank3 | Train Loss: 3.1527 | Train Loss Unweighted: 1.8424 | Train Loss Components: Auxiliary: 0.6214 | Cardinality: 0.3518 | Classification: 0.3758 | Classification_binary: 0.3967 | HitGrouping: 0.2962 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5290 | Regression_eta: 0.0063 | Regression_pt: 0.5287 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2418 | Classification_binary: 0.3943 | Regression_cos_phi: 0.2726 | Regression_energy: 0.2810 | Regression_eta: 0.1763 | Regression_pt: 0.3074 | Regression_sin_phi: 0.2609 | LR: 9.99e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.6449s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0093 | max_forward: 0.8507 | max_loss: 0.1493 | max_backward: 1.9706 | max_optimizer_step: 0.1582 | max_tensorboard_logging: 1.0840 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0072 | Memory: rss_MB: 4754.0 | uss_MB: 3484.3 | child_rss_MB: 6293.3 | allocated_MB: 450.9 | reserved_MB: 27396.0 | inactive_split_MB: 920.9 +rank3 [2026-10-28 11:54:30,570] INFO : Step 900/50000 rank3 | Train Loss: 3.2552 | Train Loss Unweighted: 1.9216 | Train Loss Components: Auxiliary: 0.6208 | Cardinality: 0.2901 | Classification: 0.3818 | Classification_binary: 0.4734 | HitGrouping: 0.2881 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5239 | Regression_eta: 0.0075 | Regression_pt: 0.5284 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2405 | Classification_binary: 0.5001 | Regression_cos_phi: 0.2805 | Regression_energy: 0.2547 | Regression_eta: 0.1948 | Regression_pt: 0.2771 | Regression_sin_phi: 0.2652 | LR: 9.99e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6749s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8050 | max_loss: 0.1486 | max_backward: 1.9075 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.9729 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4751.7 | uss_MB: 3485.4 | child_rss_MB: 6307.4 | allocated_MB: 449.7 | reserved_MB: 27406.0 | inactive_split_MB: 927.3 +rank3 [2026-10-28 11:59:03,649] INFO : Step 1000/50000 rank3 | Train Loss: 2.8964 | Train Loss Unweighted: 1.7179 | Train Loss Components: Auxiliary: 0.5694 | Cardinality: 0.2509 | Classification: 0.3299 | Classification_binary: 0.4129 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4804 | Regression_eta: 0.0063 | Regression_pt: 0.4827 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2045 | Classification_binary: 0.4057 | Regression_cos_phi: 0.1985 | Regression_energy: 0.2720 | Regression_eta: 0.1871 | Regression_pt: 0.2981 | Regression_sin_phi: 0.2084 | LR: 9.99e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4508s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7367 | max_loss: 0.2419 | max_backward: 1.8137 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 1.0408 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 4746.9 | uss_MB: 3485.7 | child_rss_MB: 6281.8 | allocated_MB: 449.1 | reserved_MB: 27406.0 | inactive_split_MB: 959.2 +rank3 [2026-10-28 11:59:04,553] INFO : Running validation on rank3 +rank3 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 11:59:57,026] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 11:59:57,577] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 12:00:26,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 12:00:27,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 12:00:28,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 12:00:29,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 12:00:40,006] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 12:00:40,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:00:41,018] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 12:00:57,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 12:00:58,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 12:00:59,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 12:01:00,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 12:01:10,040] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 12:01:10,610] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:01:11,131] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 12:01:26,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 12:01:27,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 12:01:28,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 12:01:29,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 12:01:39,181] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 12:06:47,467] INFO : Step 1100/50000 rank3 | Train Loss: 2.7351 | Train Loss Unweighted: 1.6531 | Train Loss Components: Auxiliary: 0.5524 | Cardinality: 0.2470 | Classification: 0.3238 | Classification_binary: 0.3516 | HitGrouping: 0.2744 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4831 | Regression_eta: 0.0054 | Regression_pt: 0.4842 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1999 | Classification_binary: 0.3633 | Regression_cos_phi: 0.2522 | Regression_energy: 0.2789 | Regression_eta: 0.1506 | Regression_pt: 0.2981 | Regression_sin_phi: 0.2329 | LR: 9.99e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7323s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7345 | max_loss: 0.2408 | max_backward: 1.8906 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 1.1618 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6205.6 | uss_MB: 4949.9 | child_rss_MB: 6308.4 | allocated_MB: 448.2 | reserved_MB: 24590.0 | inactive_split_MB: 1051.2 +rank3 [2026-10-28 12:11:28,615] INFO : Step 1200/50000 rank3 | Train Loss: 2.9931 | Train Loss Unweighted: 1.7517 | Train Loss Components: Auxiliary: 0.5886 | Cardinality: 0.2847 | Classification: 0.3460 | Classification_binary: 0.4151 | HitGrouping: 0.2795 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4874 | Regression_eta: 0.0065 | Regression_pt: 0.4908 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2119 | Classification_binary: 0.3909 | Regression_cos_phi: 0.2040 | Regression_energy: 0.2437 | Regression_eta: 0.1634 | Regression_pt: 0.2649 | Regression_sin_phi: 0.2271 | LR: 9.99e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9404s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0061 | max_forward: 0.9910 | max_loss: 0.1662 | max_backward: 2.3256 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.8355 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 6205.6 | uss_MB: 4943.2 | child_rss_MB: 6333.1 | allocated_MB: 449.6 | reserved_MB: 24590.0 | inactive_split_MB: 1155.8 +rank3 [2026-10-28 12:16:08,306] INFO : Step 1300/50000 rank3 | Train Loss: 3.0304 | Train Loss Unweighted: 1.7607 | Train Loss Components: Auxiliary: 0.5900 | Cardinality: 0.2909 | Classification: 0.3397 | Classification_binary: 0.4400 | HitGrouping: 0.2683 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4807 | Regression_eta: 0.0062 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1800 | Classification_binary: 0.4559 | Regression_cos_phi: 0.2939 | Regression_energy: 0.2538 | Regression_eta: 0.1687 | Regression_pt: 0.2820 | Regression_sin_phi: 0.2699 | LR: 9.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3294s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8206 | max_loss: 0.1504 | max_backward: 1.9515 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5688 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5595.4 | uss_MB: 4349.7 | child_rss_MB: 6286.2 | allocated_MB: 448.6 | reserved_MB: 25778.0 | inactive_split_MB: 1050.0 +rank3 [2026-10-28 12:20:48,118] INFO : Step 1400/50000 rank3 | Train Loss: 2.5732 | Train Loss Unweighted: 1.5137 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2349 | Classification: 0.2906 | Classification_binary: 0.3573 | HitGrouping: 0.2641 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4252 | Regression_eta: 0.0055 | Regression_pt: 0.4303 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1641 | Classification_binary: 0.3260 | Regression_cos_phi: 0.2311 | Regression_energy: 0.2205 | Regression_eta: 0.1729 | Regression_pt: 0.2451 | Regression_sin_phi: 0.1856 | LR: 9.98e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.1746s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.8166 | max_loss: 0.1962 | max_backward: 1.8388 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4964 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5588.2 | uss_MB: 3765.0 | child_rss_MB: 6275.0 | allocated_MB: 454.5 | reserved_MB: 28272.0 | inactive_split_MB: 1143.7 +rank3 [2026-10-28 12:25:37,857] INFO : Step 1500/50000 rank3 | Train Loss: 2.8564 | Train Loss Unweighted: 1.5902 | Train Loss Components: Auxiliary: 0.5576 | Cardinality: 0.3788 | Classification: 0.3167 | Classification_binary: 0.3534 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4517 | Regression_eta: 0.0055 | Regression_pt: 0.4576 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2420 | Classification_binary: 0.3532 | Regression_cos_phi: 0.3286 | Regression_energy: 0.2531 | Regression_eta: 0.1826 | Regression_pt: 0.2771 | Regression_sin_phi: 0.2584 | LR: 9.98e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6168s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.0272 | max_loss: 0.2459 | max_backward: 1.8682 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.0883 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0105 | Memory: rss_MB: 5566.9 | uss_MB: 4329.3 | child_rss_MB: 6244.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 989.3 +rank3 [2026-10-28 12:30:20,921] INFO : Step 1600/50000 rank3 | Train Loss: 2.8293 | Train Loss Unweighted: 1.6754 | Train Loss Components: Auxiliary: 0.5537 | Cardinality: 0.2456 | Classification: 0.3224 | Classification_binary: 0.4168 | HitGrouping: 0.2726 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4572 | Regression_eta: 0.0058 | Regression_pt: 0.4676 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1913 | Classification_binary: 0.4013 | Regression_cos_phi: 0.2000 | Regression_energy: 0.2524 | Regression_eta: 0.1538 | Regression_pt: 0.2798 | Regression_sin_phi: 0.2141 | LR: 9.98e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5698s | Phase Times: max_pre_step_barrier: 0.1094 | max_batch_to_device: 0.0066 | max_forward: 0.7720 | max_loss: 0.1510 | max_backward: 1.7462 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5578.6 | uss_MB: 4348.9 | child_rss_MB: 6226.8 | allocated_MB: 448.1 | reserved_MB: 28272.0 | inactive_split_MB: 989.9 +rank3 [2026-10-28 12:35:04,109] INFO : Step 1700/50000 rank3 | Train Loss: 2.7295 | Train Loss Unweighted: 1.5885 | Train Loss Components: Auxiliary: 0.5261 | Cardinality: 0.2505 | Classification: 0.2942 | Classification_binary: 0.3852 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4488 | Regression_eta: 0.0059 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1661 | Classification_binary: 0.3995 | Regression_cos_phi: 0.2379 | Regression_energy: 0.2532 | Regression_eta: 0.1504 | Regression_pt: 0.2753 | Regression_sin_phi: 0.2362 | LR: 9.97e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9354s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 1.0304 | max_loss: 0.4846 | max_backward: 1.9957 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0278 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5571.8 | uss_MB: 4349.5 | child_rss_MB: 6226.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 999.4 +rank3 [2026-10-28 12:39:46,514] INFO : Step 1800/50000 rank3 | Train Loss: 2.5541 | Train Loss Unweighted: 1.4989 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2504 | Classification: 0.2799 | Classification_binary: 0.3357 | HitGrouping: 0.2658 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4350 | Regression_eta: 0.0052 | Regression_pt: 0.4384 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1627 | Classification_binary: 0.3347 | Regression_cos_phi: 0.1735 | Regression_energy: 0.2606 | Regression_eta: 0.1657 | Regression_pt: 0.2825 | Regression_sin_phi: 0.1588 | LR: 9.97e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3590s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7316 | max_loss: 0.1436 | max_backward: 1.6570 | max_optimizer_step: 0.2625 | max_tensorboard_logging: 0.9220 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5563.3 | uss_MB: 4348.3 | child_rss_MB: 6170.2 | allocated_MB: 444.7 | reserved_MB: 28272.0 | inactive_split_MB: 991.9 +rank3 [2026-10-28 12:44:29,073] INFO : Step 1900/50000 rank3 | Train Loss: 2.7201 | Train Loss Unweighted: 1.5780 | Train Loss Components: Auxiliary: 0.5257 | Cardinality: 0.2710 | Classification: 0.3152 | Classification_binary: 0.3657 | HitGrouping: 0.2653 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4371 | Regression_eta: 0.0058 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2249 | Classification_binary: 0.3726 | Regression_cos_phi: 0.2377 | Regression_energy: 0.2374 | Regression_eta: 0.1708 | Regression_pt: 0.2602 | Regression_sin_phi: 0.2543 | LR: 9.97e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4047s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.7723 | max_loss: 0.1448 | max_backward: 2.0316 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5561.2 | uss_MB: 4335.4 | child_rss_MB: 6153.7 | allocated_MB: 448.9 | reserved_MB: 28272.0 | inactive_split_MB: 1166.2 +rank3 [2026-10-28 12:49:11,972] INFO : Step 2000/50000 rank3 | Train Loss: 2.7893 | Train Loss Unweighted: 1.5772 | Train Loss Components: Auxiliary: 0.5342 | Cardinality: 0.3149 | Classification: 0.3056 | Classification_binary: 0.4233 | HitGrouping: 0.2623 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4169 | Regression_eta: 0.0058 | Regression_pt: 0.4204 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1666 | Classification_binary: 0.4001 | Regression_cos_phi: 0.2258 | Regression_energy: 0.2264 | Regression_eta: 0.1575 | Regression_pt: 0.2459 | Regression_sin_phi: 0.2047 | LR: 9.96e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.5218s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0070 | max_forward: 1.0424 | max_loss: 0.2160 | max_backward: 2.3184 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.9849 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0132 | Memory: rss_MB: 5555.4 | uss_MB: 4316.1 | child_rss_MB: 6148.5 | allocated_MB: 451.3 | reserved_MB: 28272.0 | inactive_split_MB: 992.4 +rank3 [2026-10-28 12:49:12,926] INFO : Running validation on rank3 +rank3 [2026-10-28 12:49:12,926] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 12:49:12,926] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 12:50:07,790] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:50:08,359] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 12:50:34,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 12:50:35,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 12:50:36,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 12:50:37,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 12:50:47,424] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 12:50:48,335] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:50:48,900] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 12:51:04,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 12:51:05,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 12:51:06,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 12:51:07,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 12:51:17,785] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 12:51:18,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 12:51:18,824] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 12:51:34,361] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 12:51:35,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 12:51:35,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 12:51:36,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 12:51:46,902] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 12:57:01,909] INFO : Step 2100/50000 rank3 | Train Loss: 2.3990 | Train Loss Unweighted: 1.4049 | Train Loss Components: Auxiliary: 0.4671 | Cardinality: 0.2199 | Classification: 0.2491 | Classification_binary: 0.3399 | HitGrouping: 0.2605 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4015 | Regression_eta: 0.0046 | Regression_pt: 0.4053 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1404 | Classification_binary: 0.3146 | Regression_cos_phi: 0.1711 | Regression_energy: 0.2130 | Regression_eta: 0.0921 | Regression_pt: 0.2263 | Regression_sin_phi: 0.1133 | LR: 9.96e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8147s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7238 | max_loss: 0.1490 | max_backward: 1.8098 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.2800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6608.5 | uss_MB: 5401.4 | child_rss_MB: 6187.1 | allocated_MB: 444.8 | reserved_MB: 23654.0 | inactive_split_MB: 964.6 +rank3 [2026-10-28 13:01:41,080] INFO : Step 2200/50000 rank3 | Train Loss: 2.6105 | Train Loss Unweighted: 1.4907 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2489 | Classification: 0.2976 | Classification_binary: 0.3552 | HitGrouping: 0.2519 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4108 | Regression_eta: 0.0054 | Regression_pt: 0.4161 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1904 | Classification_binary: 0.3153 | Regression_cos_phi: 0.1915 | Regression_energy: 0.1885 | Regression_eta: 0.1230 | Regression_pt: 0.2084 | Regression_sin_phi: 0.1979 | LR: 9.96e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9419s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 1.0006 | max_loss: 0.1446 | max_backward: 1.7671 | max_optimizer_step: 0.2923 | max_tensorboard_logging: 1.2733 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6609.6 | uss_MB: 5403.3 | child_rss_MB: 6178.9 | allocated_MB: 445.7 | reserved_MB: 24780.0 | inactive_split_MB: 1132.5 +rank3 [2026-10-28 13:06:12,757] INFO : Step 2300/50000 rank3 | Train Loss: 2.7462 | Train Loss Unweighted: 1.5643 | Train Loss Components: Auxiliary: 0.5175 | Cardinality: 0.2724 | Classification: 0.3064 | Classification_binary: 0.4106 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4145 | Regression_eta: 0.0056 | Regression_pt: 0.4213 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1814 | Classification_binary: 0.4433 | Regression_cos_phi: 0.2095 | Regression_energy: 0.2077 | Regression_eta: 0.1630 | Regression_pt: 0.2300 | Regression_sin_phi: 0.2293 | LR: 9.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7162s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0353 | max_forward: 0.7930 | max_loss: 0.1572 | max_backward: 1.9535 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0140 | Memory: rss_MB: 5702.4 | uss_MB: 4498.0 | child_rss_MB: 6178.9 | allocated_MB: 451.2 | reserved_MB: 26626.0 | inactive_split_MB: 1036.8 +rank3 [2026-10-28 13:10:49,216] INFO : Step 2400/50000 rank3 | Train Loss: 2.5721 | Train Loss Unweighted: 1.4496 | Train Loss Components: Auxiliary: 0.4832 | Cardinality: 0.2667 | Classification: 0.2677 | Classification_binary: 0.3686 | HitGrouping: 0.2521 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3975 | Regression_eta: 0.0053 | Regression_pt: 0.4051 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1212 | Classification_binary: 0.3405 | Regression_cos_phi: 0.2360 | Regression_energy: 0.1993 | Regression_eta: 0.1349 | Regression_pt: 0.2191 | Regression_sin_phi: 0.1738 | LR: 9.95e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.0725s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7367 | max_loss: 0.1564 | max_backward: 1.6902 | max_optimizer_step: 0.2413 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5703.1 | uss_MB: 4496.2 | child_rss_MB: 6167.5 | allocated_MB: 448.4 | reserved_MB: 26626.0 | inactive_split_MB: 1138.1 +rank3 [2026-10-28 13:15:33,529] INFO : Step 2500/50000 rank3 | Train Loss: 2.3709 | Train Loss Unweighted: 1.3478 | Train Loss Components: Auxiliary: 0.4634 | Cardinality: 0.2742 | Classification: 0.2528 | Classification_binary: 0.3100 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3861 | Regression_eta: 0.0042 | Regression_pt: 0.3903 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1672 | Classification_binary: 0.3209 | Regression_cos_phi: 0.1586 | Regression_energy: 0.1863 | Regression_eta: 0.1194 | Regression_pt: 0.2019 | Regression_sin_phi: 0.2246 | LR: 9.94e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3396s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7598 | max_loss: 0.2442 | max_backward: 1.9617 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5783 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5689.7 | uss_MB: 4483.6 | child_rss_MB: 6173.2 | allocated_MB: 447.2 | reserved_MB: 27052.0 | inactive_split_MB: 1262.0 +rank3 [2026-10-28 13:20:26,044] INFO : Step 2600/50000 rank3 | Train Loss: 2.6216 | Train Loss Unweighted: 1.5111 | Train Loss Components: Auxiliary: 0.4988 | Cardinality: 0.2823 | Classification: 0.2863 | Classification_binary: 0.3723 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4174 | Regression_eta: 0.0052 | Regression_pt: 0.4247 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1560 | Classification_binary: 0.3615 | Regression_cos_phi: 0.2396 | Regression_energy: 0.2161 | Regression_eta: 0.1139 | Regression_pt: 0.2381 | Regression_sin_phi: 0.1959 | LR: 9.94e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2876s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8125 | max_loss: 0.1494 | max_backward: 1.9790 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5698.8 | uss_MB: 4500.2 | child_rss_MB: 6171.1 | allocated_MB: 448.2 | reserved_MB: 28734.0 | inactive_split_MB: 1300.2 +rank3 [2026-10-28 13:25:15,884] INFO : Step 2700/50000 rank3 | Train Loss: 2.5735 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4801 | Cardinality: 0.2457 | Classification: 0.2692 | Classification_binary: 0.4021 | HitGrouping: 0.2514 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3948 | Regression_eta: 0.0060 | Regression_pt: 0.3979 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1421 | Classification_binary: 0.4039 | Regression_cos_phi: 0.1963 | Regression_energy: 0.1967 | Regression_eta: 0.1508 | Regression_pt: 0.2143 | Regression_sin_phi: 0.1710 | LR: 9.94e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3217s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0070 | max_forward: 0.8435 | max_loss: 0.1556 | max_backward: 1.9409 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5695.7 | uss_MB: 4500.5 | child_rss_MB: 6182.0 | allocated_MB: 447.4 | reserved_MB: 28734.0 | inactive_split_MB: 980.7 +rank3 [2026-10-28 13:29:49,852] INFO : Step 2800/50000 rank3 | Train Loss: 2.3517 | Train Loss Unweighted: 1.3315 | Train Loss Components: Auxiliary: 0.4396 | Cardinality: 0.2218 | Classification: 0.2269 | Classification_binary: 0.3206 | HitGrouping: 0.2464 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3829 | Regression_eta: 0.0052 | Regression_pt: 0.3909 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1318 | Classification_binary: 0.3076 | Regression_cos_phi: 0.1664 | Regression_energy: 0.2034 | Regression_eta: 0.0928 | Regression_pt: 0.2261 | Regression_sin_phi: 0.1571 | LR: 9.93e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3283s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0062 | max_forward: 0.8898 | max_loss: 0.1952 | max_backward: 1.8882 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0163 | Memory: rss_MB: 5687.7 | uss_MB: 4493.8 | child_rss_MB: 6179.6 | allocated_MB: 445.9 | reserved_MB: 28734.0 | inactive_split_MB: 1208.5 +rank3 [2026-10-28 13:34:34,297] INFO : Step 2900/50000 rank3 | Train Loss: 2.4505 | Train Loss Unweighted: 1.3674 | Train Loss Components: Auxiliary: 0.4603 | Cardinality: 0.2746 | Classification: 0.2611 | Classification_binary: 0.3388 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3750 | Regression_eta: 0.0049 | Regression_pt: 0.3827 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1462 | Classification_binary: 0.3290 | Regression_cos_phi: 0.1840 | Regression_energy: 0.2039 | Regression_eta: 0.1381 | Regression_pt: 0.2254 | Regression_sin_phi: 0.2099 | LR: 9.93e-04 | DataLoad Time: 0.0178s | Model Forward Time: 6.3925s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7484 | max_loss: 0.6089 | max_backward: 2.2353 | max_optimizer_step: 0.1775 | max_tensorboard_logging: 2.4313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0155 | Memory: rss_MB: 5691.8 | uss_MB: 4497.3 | child_rss_MB: 6175.9 | allocated_MB: 446.7 | reserved_MB: 28734.0 | inactive_split_MB: 1133.2 +rank3 [2026-10-28 13:39:22,258] INFO : Step 3000/50000 rank3 | Train Loss: 2.6179 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4760 | Cardinality: 0.2600 | Classification: 0.3170 | Classification_binary: 0.3907 | HitGrouping: 0.2509 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3743 | Regression_eta: 0.0054 | Regression_pt: 0.3819 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1857 | Classification_binary: 0.3675 | Regression_cos_phi: 0.2423 | Regression_energy: 0.2075 | Regression_eta: 0.1347 | Regression_pt: 0.2318 | Regression_sin_phi: 0.2328 | LR: 9.92e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8191s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0842 | max_forward: 0.8203 | max_loss: 0.1511 | max_backward: 1.9714 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.0307 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5691.4 | uss_MB: 4499.3 | child_rss_MB: 6154.8 | allocated_MB: 451.4 | reserved_MB: 28740.0 | inactive_split_MB: 1256.0 +rank3 [2026-10-28 13:39:23,128] INFO : Running validation on rank3 +rank3 [2026-10-28 13:39:23,128] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 13:39:23,129] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 13:40:15,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:40:16,490] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 13:40:43,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 13:40:44,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 13:40:45,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 13:40:46,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 13:40:56,940] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 13:40:57,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:40:58,512] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 13:41:14,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 13:41:15,720] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 13:41:16,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 13:41:17,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 13:41:27,685] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 13:41:28,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 13:41:28,598] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 13:41:44,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 13:41:45,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 13:41:46,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 13:41:46,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 13:41:56,916] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 13:46:51,409] INFO : Step 3100/50000 rank3 | Train Loss: 2.3456 | Train Loss Unweighted: 1.3820 | Train Loss Components: Auxiliary: 0.4397 | Cardinality: 0.2017 | Classification: 0.2549 | Classification_binary: 0.3432 | HitGrouping: 0.2399 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3852 | Regression_eta: 0.0046 | Regression_pt: 0.3892 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1712 | Classification_binary: 0.3714 | Regression_cos_phi: 0.2391 | Regression_energy: 0.2112 | Regression_eta: 0.1695 | Regression_pt: 0.2309 | Regression_sin_phi: 0.2612 | LR: 9.91e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4917s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0425 | max_forward: 0.7824 | max_loss: 0.1439 | max_backward: 1.7183 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.8666 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6905.9 | uss_MB: 5708.9 | child_rss_MB: 6176.0 | allocated_MB: 447.6 | reserved_MB: 24468.0 | inactive_split_MB: 1269.5 +rank3 [2026-10-28 13:51:36,981] INFO : Step 3200/50000 rank3 | Train Loss: 2.2453 | Train Loss Unweighted: 1.2729 | Train Loss Components: Auxiliary: 0.4171 | Cardinality: 0.2143 | Classification: 0.2366 | Classification_binary: 0.3116 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3569 | Regression_eta: 0.0047 | Regression_pt: 0.3584 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1559 | Classification_binary: 0.3109 | Regression_cos_phi: 0.2039 | Regression_energy: 0.2134 | Regression_eta: 0.1246 | Regression_pt: 0.2316 | Regression_sin_phi: 0.1913 | LR: 9.91e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6649s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7985 | max_loss: 0.1379 | max_backward: 1.7895 | max_optimizer_step: 0.2027 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6908.5 | uss_MB: 5710.9 | child_rss_MB: 6167.1 | allocated_MB: 448.6 | reserved_MB: 26448.0 | inactive_split_MB: 1391.4 +rank3 [2026-10-28 13:56:16,618] INFO : Step 3300/50000 rank3 | Train Loss: 2.5474 | Train Loss Unweighted: 1.4472 | Train Loss Components: Auxiliary: 0.4779 | Cardinality: 0.2732 | Classification: 0.2824 | Classification_binary: 0.3640 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3911 | Regression_eta: 0.0054 | Regression_pt: 0.3992 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1900 | Classification_binary: 0.3916 | Regression_cos_phi: 0.2563 | Regression_energy: 0.2028 | Regression_eta: 0.2058 | Regression_pt: 0.2266 | Regression_sin_phi: 0.2040 | LR: 9.90e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6653s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8323 | max_loss: 0.1479 | max_backward: 2.0543 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.7834 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5785.1 | uss_MB: 4587.8 | child_rss_MB: 6143.0 | allocated_MB: 448.7 | reserved_MB: 26460.0 | inactive_split_MB: 1087.4 +rank3 [2026-10-28 14:00:49,247] INFO : Step 3400/50000 rank3 | Train Loss: 2.5743 | Train Loss Unweighted: 1.4358 | Train Loss Components: Auxiliary: 0.4720 | Cardinality: 0.2664 | Classification: 0.2699 | Classification_binary: 0.3915 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3784 | Regression_eta: 0.0053 | Regression_pt: 0.3851 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1539 | Classification_binary: 0.3968 | Regression_cos_phi: 0.1889 | Regression_energy: 0.2071 | Regression_eta: 0.1190 | Regression_pt: 0.2283 | Regression_sin_phi: 0.1926 | LR: 9.90e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2590s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7827 | max_loss: 0.1630 | max_backward: 1.9191 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5606 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5794.1 | uss_MB: 4597.7 | child_rss_MB: 6161.5 | allocated_MB: 450.4 | reserved_MB: 28138.0 | inactive_split_MB: 1261.0 +rank3 [2026-10-28 14:05:23,603] INFO : Step 3500/50000 rank3 | Train Loss: 2.1916 | Train Loss Unweighted: 1.2418 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2204 | Classification: 0.2212 | Classification_binary: 0.3041 | HitGrouping: 0.2426 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3517 | Regression_eta: 0.0047 | Regression_pt: 0.3558 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1505 | Classification_binary: 0.2787 | Regression_cos_phi: 0.1967 | Regression_energy: 0.1713 | Regression_eta: 0.1515 | Regression_pt: 0.1906 | Regression_sin_phi: 0.1800 | LR: 9.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2264s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7972 | max_loss: 0.1441 | max_backward: 1.7398 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6845 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5780.0 | uss_MB: 4584.6 | child_rss_MB: 6167.3 | allocated_MB: 443.7 | reserved_MB: 28532.0 | inactive_split_MB: 1176.3 +rank3 [2026-10-28 14:10:06,692] INFO : Step 3600/50000 rank3 | Train Loss: 2.3195 | Train Loss Unweighted: 1.3024 | Train Loss Components: Auxiliary: 0.4248 | Cardinality: 0.2221 | Classification: 0.2518 | Classification_binary: 0.3172 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3610 | Regression_eta: 0.0052 | Regression_pt: 0.3623 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1533 | Classification_binary: 0.3255 | Regression_cos_phi: 0.1553 | Regression_energy: 0.1895 | Regression_eta: 0.1132 | Regression_pt: 0.2062 | Regression_sin_phi: 0.1834 | LR: 9.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3082s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8456 | max_loss: 0.1388 | max_backward: 1.9466 | max_optimizer_step: 0.1811 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5787.8 | uss_MB: 4598.8 | child_rss_MB: 6163.2 | allocated_MB: 449.1 | reserved_MB: 28532.0 | inactive_split_MB: 1277.8 +rank3 [2026-10-28 14:14:47,797] INFO : Step 3700/50000 rank3 | Train Loss: 2.5861 | Train Loss Unweighted: 1.4427 | Train Loss Components: Auxiliary: 0.4700 | Cardinality: 0.2678 | Classification: 0.2978 | Classification_binary: 0.3844 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3719 | Regression_eta: 0.0058 | Regression_pt: 0.3773 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2033 | Classification_binary: 0.3866 | Regression_cos_phi: 0.2196 | Regression_energy: 0.1992 | Regression_eta: 0.1629 | Regression_pt: 0.2173 | Regression_sin_phi: 0.2374 | LR: 9.88e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5013s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.8001 | max_loss: 0.1519 | max_backward: 1.8239 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.7916 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0181 | Memory: rss_MB: 5775.1 | uss_MB: 4600.5 | child_rss_MB: 6165.4 | allocated_MB: 450.6 | reserved_MB: 30118.0 | inactive_split_MB: 1097.6 +rank3 [2026-10-28 14:19:24,598] INFO : Step 3800/50000 rank3 | Train Loss: 2.3269 | Train Loss Unweighted: 1.3016 | Train Loss Components: Auxiliary: 0.4225 | Cardinality: 0.2364 | Classification: 0.2400 | Classification_binary: 0.3389 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3534 | Regression_eta: 0.0054 | Regression_pt: 0.3591 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1488 | Classification_binary: 0.3477 | Regression_cos_phi: 0.2066 | Regression_energy: 0.1822 | Regression_eta: 0.1004 | Regression_pt: 0.2022 | Regression_sin_phi: 0.1780 | LR: 9.87e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.4706s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7555 | max_loss: 0.1601 | max_backward: 1.7612 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.9394 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5772.0 | uss_MB: 4595.7 | child_rss_MB: 6158.5 | allocated_MB: 447.4 | reserved_MB: 30118.0 | inactive_split_MB: 1355.2 +rank3 [2026-10-28 14:24:05,269] INFO : Step 3900/50000 rank3 | Train Loss: 2.0276 | Train Loss Unweighted: 1.1874 | Train Loss Components: Auxiliary: 0.3861 | Cardinality: 0.1825 | Classification: 0.2116 | Classification_binary: 0.2707 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3485 | Regression_eta: 0.0038 | Regression_pt: 0.3489 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0930 | Classification_binary: 0.2260 | Regression_cos_phi: 0.1556 | Regression_energy: 0.1723 | Regression_eta: 0.0913 | Regression_pt: 0.1842 | Regression_sin_phi: 0.1058 | LR: 9.87e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4541s | Phase Times: max_pre_step_barrier: 0.0095 | max_batch_to_device: 0.0140 | max_forward: 1.1007 | max_loss: 0.1867 | max_backward: 2.0952 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 0.5091 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5772.2 | uss_MB: 4598.5 | child_rss_MB: 6165.7 | allocated_MB: 448.6 | reserved_MB: 30118.0 | inactive_split_MB: 1119.4 +rank3 [2026-10-28 14:28:52,324] INFO : Step 4000/50000 rank3 | Train Loss: 2.3921 | Train Loss Unweighted: 1.3683 | Train Loss Components: Auxiliary: 0.4366 | Cardinality: 0.2254 | Classification: 0.2720 | Classification_binary: 0.3524 | HitGrouping: 0.2405 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3647 | Regression_eta: 0.0048 | Regression_pt: 0.3692 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1634 | Classification_binary: 0.3325 | Regression_cos_phi: 0.1882 | Regression_energy: 0.1987 | Regression_eta: 0.1313 | Regression_pt: 0.2232 | Regression_sin_phi: 0.1916 | LR: 9.86e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8908s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.8532 | max_loss: 0.1476 | max_backward: 2.4362 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.6138 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5773.5 | uss_MB: 4601.5 | child_rss_MB: 6164.9 | allocated_MB: 449.7 | reserved_MB: 30118.0 | inactive_split_MB: 899.8 +rank3 [2026-10-28 14:28:53,307] INFO : Running validation on rank3 +rank3 [2026-10-28 14:28:53,307] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 14:28:53,307] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 14:29:46,118] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:29:46,699] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 14:30:12,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 14:30:13,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 14:30:15,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 14:30:16,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 14:30:26,115] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 14:30:26,980] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:30:27,511] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 14:30:43,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 14:30:44,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 14:30:45,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 14:30:46,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 14:30:56,492] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 14:30:57,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 14:30:57,799] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 14:31:13,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 14:31:14,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 14:31:15,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 14:31:16,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 14:31:26,102] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 14:36:30,160] INFO : Step 4100/50000 rank3 | Train Loss: 2.5062 | Train Loss Unweighted: 1.3582 | Train Loss Components: Auxiliary: 0.4559 | Cardinality: 0.3208 | Classification: 0.2565 | Classification_binary: 0.3823 | HitGrouping: 0.2386 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3543 | Regression_eta: 0.0046 | Regression_pt: 0.3553 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1560 | Classification_binary: 0.4031 | Regression_cos_phi: 0.2476 | Regression_energy: 0.2021 | Regression_eta: 0.1417 | Regression_pt: 0.2172 | Regression_sin_phi: 0.2004 | LR: 9.85e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.2036s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0069 | max_forward: 0.8943 | max_loss: 0.2518 | max_backward: 1.9865 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 1.1537 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6976.0 | uss_MB: 5798.7 | child_rss_MB: 6190.7 | allocated_MB: 449.4 | reserved_MB: 25148.0 | inactive_split_MB: 1028.6 +rank3 [2026-10-28 14:41:18,155] INFO : Step 4200/50000 rank3 | Train Loss: 2.0961 | Train Loss Unweighted: 1.2232 | Train Loss Components: Auxiliary: 0.3927 | Cardinality: 0.2101 | Classification: 0.2242 | Classification_binary: 0.2902 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3497 | Regression_eta: 0.0042 | Regression_pt: 0.3510 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1065 | Classification_binary: 0.2627 | Regression_cos_phi: 0.1495 | Regression_energy: 0.1850 | Regression_eta: 0.1485 | Regression_pt: 0.1985 | Regression_sin_phi: 0.1117 | LR: 9.84e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.6561s | Phase Times: max_pre_step_barrier: 0.1131 | max_batch_to_device: 0.0199 | max_forward: 1.5911 | max_loss: 0.3393 | max_backward: 3.0707 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5197 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6973.9 | uss_MB: 5797.7 | child_rss_MB: 6186.7 | allocated_MB: 446.0 | reserved_MB: 25296.0 | inactive_split_MB: 922.1 +rank3 [2026-10-28 14:46:18,702] INFO : Step 4300/50000 rank3 | Train Loss: 2.2792 | Train Loss Unweighted: 1.3065 | Train Loss Components: Auxiliary: 0.4224 | Cardinality: 0.2158 | Classification: 0.2339 | Classification_binary: 0.3250 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3669 | Regression_eta: 0.0044 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1292 | Classification_binary: 0.3109 | Regression_cos_phi: 0.2031 | Regression_energy: 0.1959 | Regression_eta: 0.1159 | Regression_pt: 0.2144 | Regression_sin_phi: 0.2072 | LR: 9.84e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5465s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0938 | max_forward: 0.8451 | max_loss: 0.1396 | max_backward: 1.9669 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.6639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5862.7 | uss_MB: 4685.4 | child_rss_MB: 6195.4 | allocated_MB: 448.4 | reserved_MB: 28580.0 | inactive_split_MB: 1346.0 +rank3 [2026-10-28 14:50:58,886] INFO : Step 4400/50000 rank3 | Train Loss: 2.4955 | Train Loss Unweighted: 1.3385 | Train Loss Components: Auxiliary: 0.4510 | Cardinality: 0.3020 | Classification: 0.2394 | Classification_binary: 0.3842 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3482 | Regression_eta: 0.0051 | Regression_pt: 0.3563 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1501 | Classification_binary: 0.3705 | Regression_cos_phi: 0.1809 | Regression_energy: 0.1787 | Regression_eta: 0.1092 | Regression_pt: 0.1972 | Regression_sin_phi: 0.1501 | LR: 9.83e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2053s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0083 | max_forward: 0.7889 | max_loss: 0.1505 | max_backward: 1.8572 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4868 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5870.1 | uss_MB: 4697.1 | child_rss_MB: 6175.1 | allocated_MB: 449.1 | reserved_MB: 28582.0 | inactive_split_MB: 1105.9 +rank3 [2026-10-28 14:55:40,224] INFO : Step 4500/50000 rank3 | Train Loss: 2.3283 | Train Loss Unweighted: 1.2750 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.3015 | Classification: 0.2318 | Classification_binary: 0.3495 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3409 | Regression_eta: 0.0044 | Regression_pt: 0.3441 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1411 | Classification_binary: 0.3251 | Regression_cos_phi: 0.1437 | Regression_energy: 0.1773 | Regression_eta: 0.1290 | Regression_pt: 0.1867 | Regression_sin_phi: 0.2092 | LR: 9.82e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0172s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7326 | max_loss: 0.2428 | max_backward: 1.7841 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5868.2 | uss_MB: 4696.0 | child_rss_MB: 6178.6 | allocated_MB: 447.8 | reserved_MB: 28582.0 | inactive_split_MB: 1258.0 +rank3 [2026-10-28 15:00:16,448] INFO : Step 4600/50000 rank3 | Train Loss: 2.0353 | Train Loss Unweighted: 1.1777 | Train Loss Components: Auxiliary: 0.3804 | Cardinality: 0.1920 | Classification: 0.2053 | Classification_binary: 0.2642 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3490 | Regression_eta: 0.0039 | Regression_pt: 0.3512 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1085 | Classification_binary: 0.2840 | Regression_cos_phi: 0.1964 | Regression_energy: 0.1739 | Regression_eta: 0.1100 | Regression_pt: 0.1920 | Regression_sin_phi: 0.1811 | LR: 9.81e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0847s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0402 | max_forward: 0.7143 | max_loss: 0.1412 | max_backward: 1.6726 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 5861.4 | uss_MB: 4694.6 | child_rss_MB: 6186.8 | allocated_MB: 446.4 | reserved_MB: 28582.0 | inactive_split_MB: 981.4 +rank3 [2026-10-28 15:05:12,090] INFO : Step 4700/50000 rank3 | Train Loss: 2.2426 | Train Loss Unweighted: 1.2576 | Train Loss Components: Auxiliary: 0.4073 | Cardinality: 0.2222 | Classification: 0.2485 | Classification_binary: 0.3333 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3307 | Regression_eta: 0.0049 | Regression_pt: 0.3356 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1510 | Classification_binary: 0.2948 | Regression_cos_phi: 0.1970 | Regression_energy: 0.1882 | Regression_eta: 0.1330 | Regression_pt: 0.2078 | Regression_sin_phi: 0.1701 | LR: 9.81e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6201s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.1471 | max_backward: 2.0627 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.7062 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0769 | Memory: rss_MB: 5860.5 | uss_MB: 4699.2 | child_rss_MB: 6169.2 | allocated_MB: 447.7 | reserved_MB: 28652.0 | inactive_split_MB: 1164.9 +rank3 [2026-10-28 15:09:59,907] INFO : Step 4800/50000 rank3 | Train Loss: 2.2697 | Train Loss Unweighted: 1.2637 | Train Loss Components: Auxiliary: 0.4132 | Cardinality: 0.2493 | Classification: 0.2262 | Classification_binary: 0.3667 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3287 | Regression_eta: 0.0044 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1368 | Classification_binary: 0.3780 | Regression_cos_phi: 0.1944 | Regression_energy: 0.1942 | Regression_eta: 0.1219 | Regression_pt: 0.2104 | Regression_sin_phi: 0.1977 | LR: 9.80e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.0700s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0065 | max_forward: 0.7626 | max_loss: 0.1571 | max_backward: 1.6465 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5855.3 | uss_MB: 4693.4 | child_rss_MB: 6188.9 | allocated_MB: 450.7 | reserved_MB: 28652.0 | inactive_split_MB: 1088.6 +rank3 [2026-10-28 15:14:51,789] INFO : Step 4900/50000 rank3 | Train Loss: 2.0529 | Train Loss Unweighted: 1.1598 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.1942 | Classification: 0.2269 | Classification_binary: 0.2874 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3172 | Regression_eta: 0.0041 | Regression_pt: 0.3199 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1821 | Classification_binary: 0.3160 | Regression_cos_phi: 0.2663 | Regression_energy: 0.1508 | Regression_eta: 0.1409 | Regression_pt: 0.1641 | Regression_sin_phi: 0.2409 | LR: 9.79e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3243s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7293 | max_loss: 0.1974 | max_backward: 1.8633 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5858.6 | uss_MB: 4697.0 | child_rss_MB: 6189.0 | allocated_MB: 447.4 | reserved_MB: 28652.0 | inactive_split_MB: 1266.1 +rank3 [2026-10-28 15:19:59,464] INFO : Step 5000/50000 rank3 | Train Loss: 2.0827 | Train Loss Unweighted: 1.1872 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2154 | Classification: 0.2462 | Classification_binary: 0.2941 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3169 | Regression_eta: 0.0042 | Regression_pt: 0.3219 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1471 | Classification_binary: 0.2748 | Regression_cos_phi: 0.1288 | Regression_energy: 0.1774 | Regression_eta: 0.1083 | Regression_pt: 0.1956 | Regression_sin_phi: 0.1626 | LR: 9.78e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3257s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.8382 | max_loss: 0.1450 | max_backward: 1.9472 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0203 | Memory: rss_MB: 5858.9 | uss_MB: 4697.5 | child_rss_MB: 6177.0 | allocated_MB: 446.9 | reserved_MB: 29316.0 | inactive_split_MB: 980.8 +rank3 [2026-10-28 15:20:00,328] INFO : Running validation on rank3 +rank3 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 15:20:55,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:20:56,481] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 15:21:22,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 15:21:23,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 15:21:24,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 15:21:25,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 15:21:35,881] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 15:21:36,445] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:21:36,966] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 15:21:53,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 15:21:54,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 15:21:55,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 15:21:56,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 15:22:06,449] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 15:22:06,552] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 15:22:07,049] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 15:22:23,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 15:22:24,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 15:22:25,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 15:22:26,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 15:22:36,583] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 15:28:09,086] INFO : Step 5100/50000 rank3 | Train Loss: 2.4098 | Train Loss Unweighted: 1.3457 | Train Loss Components: Auxiliary: 0.4318 | Cardinality: 0.2397 | Classification: 0.2711 | Classification_binary: 0.3798 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3391 | Regression_eta: 0.0051 | Regression_pt: 0.3453 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1885 | Classification_binary: 0.3933 | Regression_cos_phi: 0.2402 | Regression_energy: 0.1795 | Regression_eta: 0.1665 | Regression_pt: 0.1949 | Regression_sin_phi: 0.2097 | LR: 9.77e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.8388s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1511 | max_backward: 1.7914 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 1.0544 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5949.7 | uss_MB: 4786.0 | child_rss_MB: 6159.3 | allocated_MB: 447.6 | reserved_MB: 25650.0 | inactive_split_MB: 1122.4 +rank3 [2026-10-28 15:32:58,661] INFO : Step 5200/50000 rank3 | Train Loss: 2.0929 | Train Loss Unweighted: 1.1974 | Train Loss Components: Auxiliary: 0.3875 | Cardinality: 0.2007 | Classification: 0.2193 | Classification_binary: 0.3278 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3194 | Regression_eta: 0.0041 | Regression_pt: 0.3227 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1514 | Classification_binary: 0.3468 | Regression_cos_phi: 0.1939 | Regression_energy: 0.1686 | Regression_eta: 0.1373 | Regression_pt: 0.1856 | Regression_sin_phi: 0.1969 | LR: 9.76e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4919s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7458 | max_loss: 0.2423 | max_backward: 1.7123 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9648 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5957.0 | uss_MB: 4796.7 | child_rss_MB: 6179.0 | allocated_MB: 450.7 | reserved_MB: 28708.0 | inactive_split_MB: 1139.3 +rank3 [2026-10-28 15:37:52,596] INFO : Step 5300/50000 rank3 | Train Loss: 1.9905 | Train Loss Unweighted: 1.1352 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.2083 | Classification: 0.2239 | Classification_binary: 0.2818 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0037 | Regression_pt: 0.3135 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1204 | Classification_binary: 0.2759 | Regression_cos_phi: 0.1376 | Regression_energy: 0.1626 | Regression_eta: 0.0766 | Regression_pt: 0.1818 | Regression_sin_phi: 0.1146 | LR: 9.75e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1039s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 1.2453 | max_loss: 0.2326 | max_backward: 2.1392 | max_optimizer_step: 0.1883 | max_tensorboard_logging: 1.0947 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5945.0 | uss_MB: 4784.6 | child_rss_MB: 6156.3 | allocated_MB: 445.9 | reserved_MB: 30688.0 | inactive_split_MB: 925.3 +rank3 [2026-10-28 15:42:39,663] INFO : Step 5400/50000 rank3 | Train Loss: 2.2932 | Train Loss Unweighted: 1.2957 | Train Loss Components: Auxiliary: 0.4189 | Cardinality: 0.2411 | Classification: 0.2689 | Classification_binary: 0.3420 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3351 | Regression_eta: 0.0045 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1736 | Classification_binary: 0.3539 | Regression_cos_phi: 0.2240 | Regression_energy: 0.1731 | Regression_eta: 0.1136 | Regression_pt: 0.1890 | Regression_sin_phi: 0.2172 | LR: 9.74e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5707s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7688 | max_loss: 0.1697 | max_backward: 1.8189 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8641 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5955.2 | uss_MB: 4796.8 | child_rss_MB: 6163.2 | allocated_MB: 450.8 | reserved_MB: 30688.0 | inactive_split_MB: 915.1 +rank3 [2026-10-28 15:47:42,229] INFO : Step 5500/50000 rank3 | Train Loss: 2.1734 | Train Loss Unweighted: 1.2295 | Train Loss Components: Auxiliary: 0.3970 | Cardinality: 0.2269 | Classification: 0.2218 | Classification_binary: 0.3571 | HitGrouping: 0.2200 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3188 | Regression_eta: 0.0041 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1316 | Classification_binary: 0.3403 | Regression_cos_phi: 0.1891 | Regression_energy: 0.1753 | Regression_eta: 0.0935 | Regression_pt: 0.1910 | Regression_sin_phi: 0.1584 | LR: 9.73e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.9285s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0153 | max_forward: 1.1913 | max_loss: 0.6135 | max_backward: 3.0599 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.6982 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5952.5 | uss_MB: 4797.4 | child_rss_MB: 6170.1 | allocated_MB: 446.9 | reserved_MB: 32804.0 | inactive_split_MB: 913.2 +rank3 [2026-10-28 15:52:30,288] INFO : Step 5600/50000 rank3 | Train Loss: 2.0307 | Train Loss Unweighted: 1.1169 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.1966 | Classification: 0.2085 | Classification_binary: 0.3031 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2975 | Regression_eta: 0.0047 | Regression_pt: 0.2988 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1577 | Classification_binary: 0.3045 | Regression_cos_phi: 0.2276 | Regression_energy: 0.1657 | Regression_eta: 0.1566 | Regression_pt: 0.1779 | Regression_sin_phi: 0.2313 | LR: 9.72e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.7957s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7794 | max_loss: 0.1497 | max_backward: 1.9410 | max_optimizer_step: 0.5154 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0061 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5920.9 | uss_MB: 4797.0 | child_rss_MB: 6174.4 | allocated_MB: 445.4 | reserved_MB: 32804.0 | inactive_split_MB: 1164.2 +rank3 [2026-10-28 15:57:16,862] INFO : Step 5700/50000 rank3 | Train Loss: 2.2064 | Train Loss Unweighted: 1.2224 | Train Loss Components: Auxiliary: 0.4023 | Cardinality: 0.2320 | Classification: 0.2526 | Classification_binary: 0.3174 | HitGrouping: 0.2343 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3173 | Regression_eta: 0.0045 | Regression_pt: 0.3259 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1597 | Classification_binary: 0.3056 | Regression_cos_phi: 0.1902 | Regression_energy: 0.1761 | Regression_eta: 0.1100 | Regression_pt: 0.1946 | Regression_sin_phi: 0.2243 | LR: 9.71e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.6652s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7746 | max_loss: 0.4928 | max_backward: 1.9590 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8719 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 5893.1 | uss_MB: 3755.9 | child_rss_MB: 6168.2 | allocated_MB: 448.4 | reserved_MB: 32804.0 | inactive_split_MB: 913.0 +rank3 [2026-10-28 16:02:01,174] INFO : Step 5800/50000 rank3 | Train Loss: 2.1438 | Train Loss Unweighted: 1.2123 | Train Loss Components: Auxiliary: 0.3898 | Cardinality: 0.2335 | Classification: 0.2404 | Classification_binary: 0.3575 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3006 | Regression_eta: 0.0042 | Regression_pt: 0.3054 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1479 | Classification_binary: 0.3527 | Regression_cos_phi: 0.1529 | Regression_energy: 0.1577 | Regression_eta: 0.1322 | Regression_pt: 0.1691 | Regression_sin_phi: 0.1844 | LR: 9.70e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.5885s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8028 | max_loss: 0.1540 | max_backward: 1.8645 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.8379 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5883.5 | uss_MB: 4797.1 | child_rss_MB: 6169.4 | allocated_MB: 448.3 | reserved_MB: 32804.0 | inactive_split_MB: 916.5 +rank3 [2026-10-28 16:06:46,107] INFO : Step 5900/50000 rank3 | Train Loss: 2.2094 | Train Loss Unweighted: 1.1453 | Train Loss Components: Auxiliary: 0.3936 | Cardinality: 0.2891 | Classification: 0.2198 | Classification_binary: 0.3337 | HitGrouping: 0.2238 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2890 | Regression_eta: 0.0046 | Regression_pt: 0.2935 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1264 | Classification_binary: 0.3433 | Regression_cos_phi: 0.1645 | Regression_energy: 0.1635 | Regression_eta: 0.1265 | Regression_pt: 0.1823 | Regression_sin_phi: 0.2107 | LR: 9.69e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5920s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0065 | max_forward: 1.0922 | max_loss: 0.1477 | max_backward: 1.8468 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8251 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0991 | Memory: rss_MB: 5881.4 | uss_MB: 4798.1 | child_rss_MB: 6148.2 | allocated_MB: 446.5 | reserved_MB: 32804.0 | inactive_split_MB: 927.1 +rank3 [2026-10-28 16:11:31,999] INFO : Step 6000/50000 rank3 | Train Loss: 1.8458 | Train Loss Unweighted: 1.0363 | Train Loss Components: Auxiliary: 0.3398 | Cardinality: 0.1796 | Classification: 0.1819 | Classification_binary: 0.2710 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2865 | Regression_eta: 0.0032 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.2568 | Regression_cos_phi: 0.1217 | Regression_energy: 0.1562 | Regression_eta: 0.0782 | Regression_pt: 0.1719 | Regression_sin_phi: 0.1027 | LR: 9.68e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5090s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7380 | max_loss: 0.2483 | max_backward: 1.8094 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.9243 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5850.9 | uss_MB: 4795.2 | child_rss_MB: 6173.4 | allocated_MB: 448.1 | reserved_MB: 32804.0 | inactive_split_MB: 1232.3 +rank3 [2026-10-28 16:11:32,848] INFO : Running validation on rank3 +rank3 [2026-10-28 16:11:32,848] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 16:12:26,409] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:12:26,927] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 16:12:53,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 16:12:55,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 16:12:56,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 16:12:57,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 16:13:07,414] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 16:13:07,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:13:08,327] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 16:13:23,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 16:13:24,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 16:13:25,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 16:13:26,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 16:13:36,927] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 16:13:38,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 16:13:38,687] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 16:13:54,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 16:13:55,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 16:13:56,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 16:13:57,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 16:14:07,122] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 16:19:13,933] INFO : Step 6100/50000 rank3 | Train Loss: 2.0818 | Train Loss Unweighted: 1.1297 | Train Loss Components: Auxiliary: 0.3734 | Cardinality: 0.2287 | Classification: 0.2079 | Classification_binary: 0.3249 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2929 | Regression_eta: 0.0044 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1391 | Classification_binary: 0.3582 | Regression_cos_phi: 0.1642 | Regression_energy: 0.1611 | Regression_eta: 0.1148 | Regression_pt: 0.1765 | Regression_sin_phi: 0.2014 | LR: 9.67e-04 | DataLoad Time: 0.0157s | Model Forward Time: 6.5157s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8130 | max_loss: 0.1524 | max_backward: 3.1921 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 1.4719 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5941.1 | uss_MB: 4883.2 | child_rss_MB: 6189.9 | allocated_MB: 448.4 | reserved_MB: 24434.0 | inactive_split_MB: 1287.8 +rank3 [2026-10-28 16:23:58,090] INFO : Step 6200/50000 rank3 | Train Loss: 2.3716 | Train Loss Unweighted: 1.2065 | Train Loss Components: Auxiliary: 0.4163 | Cardinality: 0.3927 | Classification: 0.2343 | Classification_binary: 0.3679 | HitGrouping: 0.2165 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2973 | Regression_eta: 0.0043 | Regression_pt: 0.2981 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1532 | Classification_binary: 0.4004 | Regression_cos_phi: 0.2045 | Regression_energy: 0.1599 | Regression_eta: 0.1452 | Regression_pt: 0.1742 | Regression_sin_phi: 0.2379 | LR: 9.66e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5585s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8556 | max_loss: 0.1996 | max_backward: 1.9593 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.5823 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5946.6 | uss_MB: 3767.3 | child_rss_MB: 6181.8 | allocated_MB: 447.9 | reserved_MB: 27020.0 | inactive_split_MB: 1185.4 +rank3 [2026-10-28 16:28:38,484] INFO : Step 6300/50000 rank3 | Train Loss: 1.9346 | Train Loss Unweighted: 1.0816 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.2071 | Classification: 0.2053 | Classification_binary: 0.2781 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2941 | Regression_eta: 0.0037 | Regression_pt: 0.2966 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1239 | Classification_binary: 0.2829 | Regression_cos_phi: 0.1598 | Regression_energy: 0.1530 | Regression_eta: 0.1070 | Regression_pt: 0.1632 | Regression_sin_phi: 0.1670 | LR: 9.65e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5962s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0068 | max_forward: 0.7412 | max_loss: 0.1444 | max_backward: 1.7431 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.9045 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5941.6 | uss_MB: 4896.5 | child_rss_MB: 6164.9 | allocated_MB: 444.6 | reserved_MB: 27022.0 | inactive_split_MB: 1185.4 +rank3 [2026-10-28 16:33:23,144] INFO : Step 6400/50000 rank3 | Train Loss: 2.0123 | Train Loss Unweighted: 1.0992 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2082 | Classification: 0.2291 | Classification_binary: 0.2885 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2842 | Regression_eta: 0.0040 | Regression_pt: 0.2891 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1173 | Classification_binary: 0.2553 | Regression_cos_phi: 0.1872 | Regression_energy: 0.1560 | Regression_eta: 0.1023 | Regression_pt: 0.1706 | Regression_sin_phi: 0.1404 | LR: 9.64e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.7588s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0069 | max_forward: 1.1063 | max_loss: 0.2471 | max_backward: 1.9949 | max_optimizer_step: 0.2726 | max_tensorboard_logging: 0.8233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5936.1 | uss_MB: 4896.3 | child_rss_MB: 6176.5 | allocated_MB: 451.6 | reserved_MB: 27022.0 | inactive_split_MB: 1113.0 +rank3 [2026-10-28 16:37:57,182] INFO : Step 6500/50000 rank3 | Train Loss: 2.1791 | Train Loss Unweighted: 1.2220 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2499 | Classification: 0.2271 | Classification_binary: 0.3508 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3177 | Regression_eta: 0.0042 | Regression_pt: 0.3181 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1525 | Classification_binary: 0.3678 | Regression_cos_phi: 0.1619 | Regression_energy: 0.1761 | Regression_eta: 0.1275 | Regression_pt: 0.1905 | Regression_sin_phi: 0.1734 | LR: 9.63e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.9968s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0548 | max_forward: 0.8069 | max_loss: 0.1518 | max_backward: 2.0692 | max_optimizer_step: 0.1860 | max_tensorboard_logging: 0.9414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5926.6 | uss_MB: 4895.5 | child_rss_MB: 6160.0 | allocated_MB: 449.6 | reserved_MB: 28336.0 | inactive_split_MB: 1199.2 +rank3 [2026-10-28 16:42:33,043] INFO : Step 6600/50000 rank3 | Train Loss: 1.9690 | Train Loss Unweighted: 1.0745 | Train Loss Components: Auxiliary: 0.3537 | Cardinality: 0.2132 | Classification: 0.1982 | Classification_binary: 0.3095 | HitGrouping: 0.2136 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2771 | Regression_eta: 0.0040 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1334 | Classification_binary: 0.3090 | Regression_cos_phi: 0.1760 | Regression_energy: 0.1449 | Regression_eta: 0.1268 | Regression_pt: 0.1626 | Regression_sin_phi: 0.2242 | LR: 9.62e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8258s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7382 | max_loss: 0.1467 | max_backward: 1.8335 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 1.1664 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5923.1 | uss_MB: 4893.3 | child_rss_MB: 6174.8 | allocated_MB: 447.4 | reserved_MB: 28340.0 | inactive_split_MB: 1012.8 +rank3 [2026-10-28 16:47:06,903] INFO : Step 6700/50000 rank3 | Train Loss: 1.9470 | Train Loss Unweighted: 1.0781 | Train Loss Components: Auxiliary: 0.3616 | Cardinality: 0.2363 | Classification: 0.1923 | Classification_binary: 0.2661 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3028 | Regression_eta: 0.0035 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0997 | Classification_binary: 0.2119 | Regression_cos_phi: 0.1112 | Regression_energy: 0.1781 | Regression_eta: 0.0973 | Regression_pt: 0.1995 | Regression_sin_phi: 0.1278 | LR: 9.61e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6592s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7926 | max_loss: 0.2421 | max_backward: 2.0577 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.7621 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5925.5 | uss_MB: 4898.4 | child_rss_MB: 6176.2 | allocated_MB: 450.1 | reserved_MB: 29806.0 | inactive_split_MB: 1002.0 +rank3 [2026-10-28 16:51:50,914] INFO : Step 6800/50000 rank3 | Train Loss: 2.1609 | Train Loss Unweighted: 1.1507 | Train Loss Components: Auxiliary: 0.3771 | Cardinality: 0.2180 | Classification: 0.2315 | Classification_binary: 0.3225 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0025 | Regression_energy: 0.2916 | Regression_eta: 0.0048 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1251 | Classification_binary: 0.3026 | Regression_cos_phi: 0.2396 | Regression_energy: 0.1658 | Regression_eta: 0.1453 | Regression_pt: 0.1788 | Regression_sin_phi: 0.2093 | LR: 9.60e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.1905s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0070 | max_forward: 0.9144 | max_loss: 0.4866 | max_backward: 2.1484 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 1.2438 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5920.8 | uss_MB: 4896.0 | child_rss_MB: 6174.1 | allocated_MB: 451.3 | reserved_MB: 29806.0 | inactive_split_MB: 1098.9 +rank3 [2026-10-28 16:56:32,287] INFO : Step 6900/50000 rank3 | Train Loss: 1.9936 | Train Loss Unweighted: 1.1069 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.1991 | Classification: 0.2085 | Classification_binary: 0.3385 | HitGrouping: 0.2125 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2742 | Regression_eta: 0.0041 | Regression_pt: 0.2777 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1378 | Classification_binary: 0.3736 | Regression_cos_phi: 0.2138 | Regression_energy: 0.1745 | Regression_eta: 0.1590 | Regression_pt: 0.1891 | Regression_sin_phi: 0.1645 | LR: 9.58e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6486s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7764 | max_loss: 0.2225 | max_backward: 1.6637 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1184 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5918.2 | uss_MB: 4895.4 | child_rss_MB: 6170.7 | allocated_MB: 450.8 | reserved_MB: 29806.0 | inactive_split_MB: 1193.4 +rank3 [2026-10-28 17:01:16,519] INFO : Step 7000/50000 rank3 | Train Loss: 1.9256 | Train Loss Unweighted: 1.0803 | Train Loss Components: Auxiliary: 0.3608 | Cardinality: 0.2036 | Classification: 0.1946 | Classification_binary: 0.2738 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2998 | Regression_eta: 0.0039 | Regression_pt: 0.3044 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0943 | Classification_binary: 0.2566 | Regression_cos_phi: 0.1265 | Regression_energy: 0.1541 | Regression_eta: 0.0960 | Regression_pt: 0.1698 | Regression_sin_phi: 0.1231 | LR: 9.57e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.7627s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7896 | max_loss: 0.1557 | max_backward: 1.7415 | max_optimizer_step: 0.2946 | max_tensorboard_logging: 1.0959 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 5913.7 | uss_MB: 4893.6 | child_rss_MB: 6162.0 | allocated_MB: 450.3 | reserved_MB: 29806.0 | inactive_split_MB: 1283.4 +rank3 [2026-10-28 17:01:17,464] INFO : Running validation on rank3 +rank3 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 17:02:11,958] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:02:12,537] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 17:02:38,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 17:02:39,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 17:02:40,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 17:02:41,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 17:02:51,987] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 17:02:52,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:02:53,233] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 17:03:09,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 17:03:10,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 17:03:11,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 17:03:12,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 17:03:22,418] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 17:03:22,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:03:23,343] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 17:03:38,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 17:03:39,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 17:03:40,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 17:03:40,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 17:03:50,926] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 17:08:58,934] INFO : Step 7100/50000 rank3 | Train Loss: 1.8251 | Train Loss Unweighted: 1.0448 | Train Loss Components: Auxiliary: 0.3403 | Cardinality: 0.1731 | Classification: 0.2040 | Classification_binary: 0.2675 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2809 | Regression_eta: 0.0035 | Regression_pt: 0.2853 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0962 | Classification_binary: 0.2339 | Regression_cos_phi: 0.1472 | Regression_energy: 0.1342 | Regression_eta: 0.0855 | Regression_pt: 0.1469 | Regression_sin_phi: 0.1228 | LR: 9.56e-04 | DataLoad Time: 0.0214s | Model Forward Time: 5.2535s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0622 | max_forward: 0.8282 | max_loss: 0.1540 | max_backward: 1.9352 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.3584 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6015.2 | uss_MB: 4988.2 | child_rss_MB: 6155.9 | allocated_MB: 449.9 | reserved_MB: 22498.0 | inactive_split_MB: 1193.4 +rank3 [2026-10-28 17:13:31,724] INFO : Step 7200/50000 rank3 | Train Loss: 2.1146 | Train Loss Unweighted: 1.1506 | Train Loss Components: Auxiliary: 0.3887 | Cardinality: 0.2411 | Classification: 0.2243 | Classification_binary: 0.3469 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2844 | Regression_eta: 0.0044 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1420 | Classification_binary: 0.3415 | Regression_cos_phi: 0.1814 | Regression_energy: 0.1343 | Regression_eta: 0.1209 | Regression_pt: 0.1457 | Regression_sin_phi: 0.1540 | LR: 9.55e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.5732s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0369 | max_forward: 0.7969 | max_loss: 0.1517 | max_backward: 1.9380 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6030.2 | uss_MB: 5003.5 | child_rss_MB: 6152.6 | allocated_MB: 448.6 | reserved_MB: 22906.0 | inactive_split_MB: 1087.6 +rank3 [2026-10-28 17:18:11,974] INFO : Step 7300/50000 rank3 | Train Loss: 1.8657 | Train Loss Unweighted: 1.0129 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2046 | Classification: 0.1759 | Classification_binary: 0.2802 | HitGrouping: 0.2112 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2746 | Regression_eta: 0.0037 | Regression_pt: 0.2747 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1311 | Classification_binary: 0.2943 | Regression_cos_phi: 0.1671 | Regression_energy: 0.1414 | Regression_eta: 0.1087 | Regression_pt: 0.1523 | Regression_sin_phi: 0.1780 | LR: 9.53e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4773s | Phase Times: max_pre_step_barrier: 0.0082 | max_batch_to_device: 0.0150 | max_forward: 0.7713 | max_loss: 0.1507 | max_backward: 1.8303 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.6113 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6029.9 | uss_MB: 5003.9 | child_rss_MB: 6151.8 | allocated_MB: 449.9 | reserved_MB: 25044.0 | inactive_split_MB: 918.5 +rank3 [2026-10-28 17:22:47,474] INFO : Step 7400/50000 rank3 | Train Loss: 1.8326 | Train Loss Unweighted: 1.0006 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2227 | Classification: 0.1795 | Classification_binary: 0.2563 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2767 | Regression_eta: 0.0034 | Regression_pt: 0.2813 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1152 | Classification_binary: 0.2749 | Regression_cos_phi: 0.1124 | Regression_energy: 0.1479 | Regression_eta: 0.0812 | Regression_pt: 0.1614 | Regression_sin_phi: 0.1594 | LR: 9.52e-04 | DataLoad Time: 0.0224s | Model Forward Time: 4.2032s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0684 | max_forward: 0.7762 | max_loss: 0.1384 | max_backward: 1.7881 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6026.9 | uss_MB: 5000.5 | child_rss_MB: 6151.8 | allocated_MB: 447.4 | reserved_MB: 25044.0 | inactive_split_MB: 1152.4 +rank3 [2026-10-28 17:27:28,847] INFO : Step 7500/50000 rank3 | Train Loss: 2.1479 | Train Loss Unweighted: 1.1510 | Train Loss Components: Auxiliary: 0.3847 | Cardinality: 0.2329 | Classification: 0.2181 | Classification_binary: 0.3323 | HitGrouping: 0.2143 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2947 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1375 | Classification_binary: 0.3283 | Regression_cos_phi: 0.2088 | Regression_energy: 0.1427 | Regression_eta: 0.1289 | Regression_pt: 0.1539 | Regression_sin_phi: 0.1627 | LR: 9.51e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2515s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7726 | max_loss: 0.2518 | max_backward: 1.8458 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1040 | Memory: rss_MB: 6026.8 | uss_MB: 5002.5 | child_rss_MB: 6153.2 | allocated_MB: 452.3 | reserved_MB: 26846.0 | inactive_split_MB: 1117.1 +rank3 [2026-10-28 17:32:29,655] INFO : Step 7600/50000 rank3 | Train Loss: 1.9984 | Train Loss Unweighted: 1.0873 | Train Loss Components: Auxiliary: 0.3667 | Cardinality: 0.2088 | Classification: 0.2053 | Classification_binary: 0.3405 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2647 | Regression_eta: 0.0038 | Regression_pt: 0.2689 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1300 | Classification_binary: 0.3389 | Regression_cos_phi: 0.2250 | Regression_energy: 0.1246 | Regression_eta: 0.1349 | Regression_pt: 0.1357 | Regression_sin_phi: 0.2141 | LR: 9.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5775s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7834 | max_loss: 0.1641 | max_backward: 1.8407 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.8255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6029.1 | uss_MB: 5003.6 | child_rss_MB: 6150.1 | allocated_MB: 448.9 | reserved_MB: 26846.0 | inactive_split_MB: 1232.3 +rank3 [2026-10-28 17:37:19,852] INFO : Step 7700/50000 rank3 | Train Loss: 1.7524 | Train Loss Unweighted: 0.9761 | Train Loss Components: Auxiliary: 0.3200 | Cardinality: 0.1783 | Classification: 0.1756 | Classification_binary: 0.2521 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2705 | Regression_eta: 0.0035 | Regression_pt: 0.2709 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1078 | Classification_binary: 0.2177 | Regression_cos_phi: 0.1194 | Regression_energy: 0.1381 | Regression_eta: 0.0723 | Regression_pt: 0.1480 | Regression_sin_phi: 0.1226 | LR: 9.48e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4848s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7659 | max_loss: 0.1551 | max_backward: 1.8103 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6027.2 | uss_MB: 5004.4 | child_rss_MB: 6154.6 | allocated_MB: 448.8 | reserved_MB: 26846.0 | inactive_split_MB: 1042.2 +rank3 [2026-10-28 17:42:10,706] INFO : Step 7800/50000 rank3 | Train Loss: 1.8039 | Train Loss Unweighted: 1.0226 | Train Loss Components: Auxiliary: 0.3378 | Cardinality: 0.1686 | Classification: 0.1822 | Classification_binary: 0.2556 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2863 | Regression_eta: 0.0034 | Regression_pt: 0.2914 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1425 | Classification_binary: 0.2431 | Regression_cos_phi: 0.1471 | Regression_energy: 0.1522 | Regression_eta: 0.0802 | Regression_pt: 0.1656 | Regression_sin_phi: 0.1225 | LR: 9.47e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6884s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.8680 | max_loss: 0.1424 | max_backward: 1.9949 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.8163 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 6024.0 | uss_MB: 5003.5 | child_rss_MB: 6156.5 | allocated_MB: 448.6 | reserved_MB: 26846.0 | inactive_split_MB: 1396.9 +rank3 [2026-10-28 17:47:05,102] INFO : Step 7900/50000 rank3 | Train Loss: 2.0641 | Train Loss Unweighted: 1.0960 | Train Loss Components: Auxiliary: 0.3855 | Cardinality: 0.2401 | Classification: 0.2031 | Classification_binary: 0.3367 | HitGrouping: 0.2085 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2721 | Regression_eta: 0.0041 | Regression_pt: 0.2759 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1093 | Classification_binary: 0.3409 | Regression_cos_phi: 0.1556 | Regression_energy: 0.1528 | Regression_eta: 0.1147 | Regression_pt: 0.1673 | Regression_sin_phi: 0.1658 | LR: 9.46e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6373s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7971 | max_loss: 0.2430 | max_backward: 1.9131 | max_optimizer_step: 0.2681 | max_tensorboard_logging: 0.7971 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6025.3 | uss_MB: 5004.3 | child_rss_MB: 6161.5 | allocated_MB: 451.8 | reserved_MB: 27044.0 | inactive_split_MB: 827.4 +rank3 [2026-10-28 17:51:56,121] INFO : Step 8000/50000 rank3 | Train Loss: 1.8391 | Train Loss Unweighted: 0.9813 | Train Loss Components: Auxiliary: 0.3355 | Cardinality: 0.2153 | Classification: 0.1625 | Classification_binary: 0.2896 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2605 | Regression_eta: 0.0035 | Regression_pt: 0.2616 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2890 | Regression_cos_phi: 0.1354 | Regression_energy: 0.1393 | Regression_eta: 0.0978 | Regression_pt: 0.1539 | Regression_sin_phi: 0.1366 | LR: 9.44e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.5541s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 1.2188 | max_loss: 0.4570 | max_backward: 3.9408 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.5566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6013.9 | uss_MB: 4999.4 | child_rss_MB: 6155.0 | allocated_MB: 448.4 | reserved_MB: 27044.0 | inactive_split_MB: 1156.9 +rank3 [2026-10-28 17:51:56,979] INFO : Running validation on rank3 +rank3 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 17:52:50,098] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:52:50,644] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 17:53:16,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 17:53:17,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 17:53:18,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 17:53:19,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 17:53:29,897] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 17:53:30,250] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:53:30,810] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 17:53:47,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 17:53:48,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 17:53:49,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 17:53:50,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 17:54:00,398] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-28 17:54:01,630] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 17:54:02,175] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 17:54:17,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 17:54:18,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 17:54:19,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 17:54:20,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 17:54:30,315] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 17:59:43,744] INFO : Step 8100/50000 rank3 | Train Loss: 1.7101 | Train Loss Unweighted: 0.9276 | Train Loss Components: Auxiliary: 0.3166 | Cardinality: 0.1787 | Classification: 0.1664 | Classification_binary: 0.2381 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2564 | Regression_eta: 0.0032 | Regression_pt: 0.2599 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0962 | Classification_binary: 0.2261 | Regression_cos_phi: 0.1431 | Regression_energy: 0.1380 | Regression_eta: 0.0862 | Regression_pt: 0.1535 | Regression_sin_phi: 0.1873 | LR: 9.43e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6982s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7628 | max_loss: 0.1353 | max_backward: 2.0123 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.8300 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6970.2 | uss_MB: 5951.6 | child_rss_MB: 6190.8 | allocated_MB: 447.2 | reserved_MB: 25306.0 | inactive_split_MB: 1261.3 +rank3 [2026-10-28 18:04:56,111] INFO : Step 8200/50000 rank3 | Train Loss: 1.8745 | Train Loss Unweighted: 1.0293 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.1946 | Classification: 0.1986 | Classification_binary: 0.2966 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2609 | Regression_eta: 0.0035 | Regression_pt: 0.2657 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1141 | Classification_binary: 0.3067 | Regression_cos_phi: 0.1419 | Regression_energy: 0.1202 | Regression_eta: 0.0839 | Regression_pt: 0.1319 | Regression_sin_phi: 0.1129 | LR: 9.42e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.3083s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7790 | max_loss: 0.1535 | max_backward: 1.9181 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.5352 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6080.6 | uss_MB: 5064.4 | child_rss_MB: 6184.5 | allocated_MB: 448.2 | reserved_MB: 25360.0 | inactive_split_MB: 1159.7 +rank3 [2026-10-28 18:09:37,297] INFO : Step 8300/50000 rank3 | Train Loss: 1.8727 | Train Loss Unweighted: 1.0119 | Train Loss Components: Auxiliary: 0.3401 | Cardinality: 0.1867 | Classification: 0.1728 | Classification_binary: 0.3272 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2513 | Regression_eta: 0.0033 | Regression_pt: 0.2533 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0912 | Classification_binary: 0.3246 | Regression_cos_phi: 0.1708 | Regression_energy: 0.1341 | Regression_eta: 0.0769 | Regression_pt: 0.1446 | Regression_sin_phi: 0.1657 | LR: 9.40e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5436s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0436 | max_forward: 1.1303 | max_loss: 0.1503 | max_backward: 2.0213 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6082.4 | uss_MB: 5067.6 | child_rss_MB: 6182.5 | allocated_MB: 449.6 | reserved_MB: 27362.0 | inactive_split_MB: 1355.2 +rank3 [2026-10-28 18:14:10,520] INFO : Step 8400/50000 rank3 | Train Loss: 1.6705 | Train Loss Unweighted: 0.9205 | Train Loss Components: Auxiliary: 0.3113 | Cardinality: 0.1938 | Classification: 0.1656 | Classification_binary: 0.2512 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2485 | Regression_eta: 0.0028 | Regression_pt: 0.2494 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0846 | Classification_binary: 0.2457 | Regression_cos_phi: 0.1388 | Regression_energy: 0.1216 | Regression_eta: 0.0831 | Regression_pt: 0.1344 | Regression_sin_phi: 0.1415 | LR: 9.39e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4478s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8079 | max_loss: 0.1514 | max_backward: 1.6942 | max_optimizer_step: 0.2670 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6074.8 | uss_MB: 5060.4 | child_rss_MB: 6188.9 | allocated_MB: 446.4 | reserved_MB: 27366.0 | inactive_split_MB: 1386.9 +rank3 [2026-10-28 18:18:45,092] INFO : Step 8500/50000 rank3 | Train Loss: 1.8447 | Train Loss Unweighted: 0.9684 | Train Loss Components: Auxiliary: 0.3299 | Cardinality: 0.2197 | Classification: 0.1758 | Classification_binary: 0.2823 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2478 | Regression_eta: 0.0040 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1084 | Classification_binary: 0.2527 | Regression_cos_phi: 0.2055 | Regression_energy: 0.1319 | Regression_eta: 0.1114 | Regression_pt: 0.1459 | Regression_sin_phi: 0.1760 | LR: 9.37e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.1675s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0066 | max_forward: 0.7185 | max_loss: 0.1465 | max_backward: 1.6783 | max_optimizer_step: 0.2239 | max_tensorboard_logging: 0.6269 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 6077.8 | uss_MB: 5064.0 | child_rss_MB: 6185.3 | allocated_MB: 448.4 | reserved_MB: 27368.0 | inactive_split_MB: 1257.8 +rank3 [2026-10-28 18:23:30,152] INFO : Step 8600/50000 rank3 | Train Loss: 1.9431 | Train Loss Unweighted: 1.0237 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.2483 | Classification: 0.1814 | Classification_binary: 0.3328 | HitGrouping: 0.2024 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2495 | Regression_eta: 0.0034 | Regression_pt: 0.2527 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1069 | Classification_binary: 0.3699 | Regression_cos_phi: 0.1420 | Regression_energy: 0.1395 | Regression_eta: 0.1159 | Regression_pt: 0.1537 | Regression_sin_phi: 0.1669 | LR: 9.36e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5563s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1011 | max_forward: 0.9340 | max_loss: 0.1508 | max_backward: 1.7704 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.7686 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 6082.0 | uss_MB: 5062.1 | child_rss_MB: 6191.5 | allocated_MB: 448.9 | reserved_MB: 27368.0 | inactive_split_MB: 1268.3 +rank3 [2026-10-28 18:28:08,445] INFO : Step 8700/50000 rank3 | Train Loss: 1.9739 | Train Loss Unweighted: 1.0707 | Train Loss Components: Auxiliary: 0.3725 | Cardinality: 0.2322 | Classification: 0.1854 | Classification_binary: 0.2934 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2900 | Regression_eta: 0.0037 | Regression_pt: 0.2944 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1246 | Classification_binary: 0.2901 | Regression_cos_phi: 0.1400 | Regression_energy: 0.1523 | Regression_eta: 0.1048 | Regression_pt: 0.1688 | Regression_sin_phi: 0.1570 | LR: 9.34e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.1627s | Phase Times: max_pre_step_barrier: 0.0890 | max_batch_to_device: 0.0067 | max_forward: 0.7726 | max_loss: 0.1574 | max_backward: 1.6332 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 2.5948 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 6082.7 | uss_MB: 5062.6 | child_rss_MB: 6185.4 | allocated_MB: 448.0 | reserved_MB: 29420.0 | inactive_split_MB: 876.0 +rank3 [2026-10-28 18:32:55,587] INFO : Step 8800/50000 rank3 | Train Loss: 1.5247 | Train Loss Unweighted: 0.8421 | Train Loss Components: Auxiliary: 0.2808 | Cardinality: 0.1524 | Classification: 0.1581 | Classification_binary: 0.2323 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2217 | Regression_eta: 0.0029 | Regression_pt: 0.2242 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1171 | Classification_binary: 0.2430 | Regression_cos_phi: 0.1326 | Regression_energy: 0.1245 | Regression_eta: 0.0989 | Regression_pt: 0.1347 | Regression_sin_phi: 0.1201 | LR: 9.33e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.9908s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7382 | max_loss: 0.2414 | max_backward: 1.7741 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 6078.3 | uss_MB: 5061.0 | child_rss_MB: 6166.0 | allocated_MB: 447.9 | reserved_MB: 29420.0 | inactive_split_MB: 1260.3 +rank3 [2026-10-28 18:37:36,895] INFO : Step 8900/50000 rank3 | Train Loss: 1.7540 | Train Loss Unweighted: 0.9436 | Train Loss Components: Auxiliary: 0.3227 | Cardinality: 0.2072 | Classification: 0.1816 | Classification_binary: 0.2735 | HitGrouping: 0.1904 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2393 | Regression_eta: 0.0031 | Regression_pt: 0.2426 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1098 | Classification_binary: 0.2778 | Regression_cos_phi: 0.1094 | Regression_energy: 0.1286 | Regression_eta: 0.0904 | Regression_pt: 0.1401 | Regression_sin_phi: 0.1214 | LR: 9.31e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4484s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8077 | max_loss: 0.2089 | max_backward: 1.9976 | max_optimizer_step: 0.2211 | max_tensorboard_logging: 0.5747 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 6080.9 | uss_MB: 5000.4 | child_rss_MB: 6188.7 | allocated_MB: 451.8 | reserved_MB: 29420.0 | inactive_split_MB: 1133.8 +rank3 [2026-10-28 18:42:20,745] INFO : Step 9000/50000 rank3 | Train Loss: 1.7807 | Train Loss Unweighted: 0.9795 | Train Loss Components: Auxiliary: 0.3326 | Cardinality: 0.1942 | Classification: 0.1600 | Classification_binary: 0.3157 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2461 | Regression_eta: 0.0033 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1196 | Classification_binary: 0.3219 | Regression_cos_phi: 0.1247 | Regression_energy: 0.1465 | Regression_eta: 0.0834 | Regression_pt: 0.1636 | Regression_sin_phi: 0.1242 | LR: 9.30e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8569s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.8696 | max_loss: 0.1559 | max_backward: 1.8245 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.9520 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 6064.6 | uss_MB: 5051.1 | child_rss_MB: 6187.5 | allocated_MB: 448.9 | reserved_MB: 29420.0 | inactive_split_MB: 1051.6 +rank3 [2026-10-28 18:42:21,625] INFO : Running validation on rank3 +rank3 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 18:43:19,661] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:43:20,246] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 18:43:46,998] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 18:43:48,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 18:43:49,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 18:43:50,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 18:44:00,134] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 18:44:00,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:44:01,306] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 18:44:17,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 18:44:18,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 18:44:19,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 18:44:20,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 18:44:30,362] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 18:44:31,056] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 18:44:31,605] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 18:44:47,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 18:44:48,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 18:44:49,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 18:44:49,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 18:45:00,006] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 18:50:14,979] INFO : Step 9100/50000 rank3 | Train Loss: 1.6002 | Train Loss Unweighted: 0.8685 | Train Loss Components: Auxiliary: 0.3013 | Cardinality: 0.1975 | Classification: 0.1658 | Classification_binary: 0.2349 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2304 | Regression_eta: 0.0027 | Regression_pt: 0.2318 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1067 | Classification_binary: 0.2467 | Regression_cos_phi: 0.0935 | Regression_energy: 0.1292 | Regression_eta: 0.0701 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0944 | LR: 9.28e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5183s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7981 | max_loss: 0.1460 | max_backward: 1.6475 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.9178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7065.6 | uss_MB: 6052.1 | child_rss_MB: 6167.4 | allocated_MB: 449.7 | reserved_MB: 25788.0 | inactive_split_MB: 957.3 +rank3 [2026-10-28 18:55:02,312] INFO : Step 9200/50000 rank3 | Train Loss: 1.6859 | Train Loss Unweighted: 0.8867 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.1906 | Classification: 0.1447 | Classification_binary: 0.2542 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2389 | Regression_eta: 0.0033 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0998 | Classification_binary: 0.2602 | Regression_cos_phi: 0.1152 | Regression_energy: 0.1250 | Regression_eta: 0.0808 | Regression_pt: 0.1361 | Regression_sin_phi: 0.1524 | LR: 9.27e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3390s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7865 | max_loss: 0.1441 | max_backward: 1.8922 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6128.5 | uss_MB: 5123.6 | child_rss_MB: 6161.7 | allocated_MB: 450.8 | reserved_MB: 25788.0 | inactive_split_MB: 1040.1 +rank3 [2026-10-28 18:59:49,908] INFO : Step 9300/50000 rank3 | Train Loss: 1.8980 | Train Loss Unweighted: 0.9669 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.3104 | Classification: 0.1729 | Classification_binary: 0.3073 | HitGrouping: 0.1936 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2397 | Regression_eta: 0.0030 | Regression_pt: 0.2407 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0987 | Classification_binary: 0.3530 | Regression_cos_phi: 0.1911 | Regression_energy: 0.1228 | Regression_eta: 0.1043 | Regression_pt: 0.1351 | Regression_sin_phi: 0.1643 | LR: 9.25e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.1852s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0143 | max_forward: 0.7819 | max_loss: 0.1515 | max_backward: 1.8209 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6071.1 | uss_MB: 5121.9 | child_rss_MB: 5951.4 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1031.1 +rank3 [2026-10-28 19:04:46,838] INFO : Step 9400/50000 rank3 | Train Loss: 1.5878 | Train Loss Unweighted: 0.8563 | Train Loss Components: Auxiliary: 0.2880 | Cardinality: 0.1552 | Classification: 0.1580 | Classification_binary: 0.2625 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2141 | Regression_eta: 0.0030 | Regression_pt: 0.2153 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1125 | Classification_binary: 0.2609 | Regression_cos_phi: 0.1535 | Regression_energy: 0.1131 | Regression_eta: 0.0809 | Regression_pt: 0.1214 | Regression_sin_phi: 0.1238 | LR: 9.24e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0182s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7342 | max_loss: 0.2471 | max_backward: 1.7320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.4671 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6067.3 | uss_MB: 5118.7 | child_rss_MB: 5945.7 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1121.1 +rank3 [2026-10-28 19:09:47,343] INFO : Step 9500/50000 rank3 | Train Loss: 1.5519 | Train Loss Unweighted: 0.8141 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1719 | Classification: 0.1393 | Classification_binary: 0.2430 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2129 | Regression_eta: 0.0030 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.2383 | Regression_cos_phi: 0.1178 | Regression_energy: 0.1227 | Regression_eta: 0.0835 | Regression_pt: 0.1309 | Regression_sin_phi: 0.1088 | LR: 9.22e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1298s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0076 | max_forward: 0.7261 | max_loss: 0.1376 | max_backward: 1.8267 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6069.1 | uss_MB: 5122.8 | child_rss_MB: 5952.7 | allocated_MB: 446.7 | reserved_MB: 27752.0 | inactive_split_MB: 1128.4 +rank3 [2026-10-28 19:14:55,455] INFO : Step 9600/50000 rank3 | Train Loss: 1.7379 | Train Loss Unweighted: 0.9171 | Train Loss Components: Auxiliary: 0.3221 | Cardinality: 0.2199 | Classification: 0.1679 | Classification_binary: 0.2886 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2245 | Regression_eta: 0.0032 | Regression_pt: 0.2297 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.2655 | Regression_cos_phi: 0.1215 | Regression_energy: 0.1217 | Regression_eta: 0.0868 | Regression_pt: 0.1339 | Regression_sin_phi: 0.1212 | LR: 9.21e-04 | DataLoad Time: 0.0187s | Model Forward Time: 7.7255s | Phase Times: max_pre_step_barrier: 0.0081 | max_batch_to_device: 0.0183 | max_forward: 3.1116 | max_loss: 0.7194 | max_backward: 4.4829 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6068.7 | uss_MB: 5120.4 | child_rss_MB: 5924.4 | allocated_MB: 449.4 | reserved_MB: 29732.0 | inactive_split_MB: 1322.6 +rank3 [2026-10-28 19:20:02,940] INFO : Step 9700/50000 rank3 | Train Loss: 1.8683 | Train Loss Unweighted: 0.9602 | Train Loss Components: Auxiliary: 0.3449 | Cardinality: 0.2461 | Classification: 0.1512 | Classification_binary: 0.3271 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2364 | Regression_eta: 0.0037 | Regression_pt: 0.2383 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.3195 | Regression_cos_phi: 0.1532 | Regression_energy: 0.1276 | Regression_eta: 0.1093 | Regression_pt: 0.1369 | Regression_sin_phi: 0.1346 | LR: 9.19e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.0320s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0069 | max_forward: 0.8299 | max_loss: 0.7776 | max_backward: 2.5571 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5726 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6067.5 | uss_MB: 5123.7 | child_rss_MB: 5943.1 | allocated_MB: 450.3 | reserved_MB: 29734.0 | inactive_split_MB: 1128.1 +rank3 [2026-10-28 19:25:00,573] INFO : Step 9800/50000 rank3 | Train Loss: 1.6119 | Train Loss Unweighted: 0.8200 | Train Loss Components: Auxiliary: 0.3036 | Cardinality: 0.2266 | Classification: 0.1371 | Classification_binary: 0.2410 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2176 | Regression_eta: 0.0027 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2315 | Regression_cos_phi: 0.1117 | Regression_energy: 0.1332 | Regression_eta: 0.0787 | Regression_pt: 0.1452 | Regression_sin_phi: 0.1075 | LR: 9.17e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.5935s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7430 | max_loss: 0.2467 | max_backward: 1.8961 | max_optimizer_step: 0.2008 | max_tensorboard_logging: 0.9056 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6064.8 | uss_MB: 5122.0 | child_rss_MB: 5925.9 | allocated_MB: 446.9 | reserved_MB: 29734.0 | inactive_split_MB: 905.1 +rank3 [2026-10-28 19:29:40,210] INFO : Step 9900/50000 rank3 | Train Loss: 1.5408 | Train Loss Unweighted: 0.8391 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1631 | Classification: 0.1532 | Classification_binary: 0.2416 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2178 | Regression_eta: 0.0029 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0949 | Classification_binary: 0.2511 | Regression_cos_phi: 0.1420 | Regression_energy: 0.1189 | Regression_eta: 0.0768 | Regression_pt: 0.1283 | Regression_sin_phi: 0.1169 | LR: 9.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6048s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7611 | max_loss: 0.1425 | max_backward: 1.8994 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6066.1 | uss_MB: 5122.8 | child_rss_MB: 5945.5 | allocated_MB: 448.3 | reserved_MB: 29734.0 | inactive_split_MB: 1119.7 +rank3 [2026-10-28 19:34:23,377] INFO : Step 10000/50000 rank3 | Train Loss: 1.7147 | Train Loss Unweighted: 0.9225 | Train Loss Components: Auxiliary: 0.3261 | Cardinality: 0.1969 | Classification: 0.1590 | Classification_binary: 0.2912 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2305 | Regression_eta: 0.0029 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1114 | Classification_binary: 0.2971 | Regression_cos_phi: 0.1392 | Regression_energy: 0.1187 | Regression_eta: 0.0794 | Regression_pt: 0.1288 | Regression_sin_phi: 0.1159 | LR: 9.14e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8789s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1140 | max_forward: 0.8951 | max_loss: 0.1695 | max_backward: 1.8837 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 1.0339 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 6055.8 | uss_MB: 5112.2 | child_rss_MB: 5922.2 | allocated_MB: 449.4 | reserved_MB: 29734.0 | inactive_split_MB: 1320.6 +rank3 [2026-10-28 19:34:24,225] INFO : Running validation on rank3 +rank3 [2026-10-28 19:34:24,225] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 19:34:24,225] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 19:35:21,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:35:22,180] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 19:35:48,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 19:35:49,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 19:35:50,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 19:35:51,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 19:36:01,790] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 19:36:02,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:36:02,954] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 19:36:21,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 19:36:24,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 19:36:25,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 19:36:26,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 19:36:36,459] INFO : Time taken to make predictions on device 3 is: 0.32 min +rank3 [2026-10-28 19:36:36,471] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 19:36:36,973] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 19:36:52,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 19:36:53,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 19:36:54,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 19:36:54,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 19:37:04,949] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 19:42:23,380] INFO : Step 10100/50000 rank3 | Train Loss: 1.6697 | Train Loss Unweighted: 0.8627 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.2287 | Classification: 0.1283 | Classification_binary: 0.2543 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2359 | Regression_eta: 0.0032 | Regression_pt: 0.2378 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0976 | Classification_binary: 0.2993 | Regression_cos_phi: 0.1294 | Regression_energy: 0.1325 | Regression_eta: 0.0918 | Regression_pt: 0.1454 | Regression_sin_phi: 0.1324 | LR: 9.12e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7402s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7652 | max_loss: 0.1496 | max_backward: 1.8791 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 0.9326 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 7065.4 | uss_MB: 6119.6 | child_rss_MB: 5927.2 | allocated_MB: 448.1 | reserved_MB: 26360.0 | inactive_split_MB: 1080.0 +rank3 [2026-10-28 19:47:10,586] INFO : Step 10200/50000 rank3 | Train Loss: 1.3968 | Train Loss Unweighted: 0.7619 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1483 | Classification: 0.1293 | Classification_binary: 0.2070 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2090 | Regression_eta: 0.0026 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0810 | Classification_binary: 0.1834 | Regression_cos_phi: 0.1071 | Regression_energy: 0.1170 | Regression_eta: 0.0660 | Regression_pt: 0.1262 | Regression_sin_phi: 0.1341 | LR: 9.11e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.4823s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0330 | max_forward: 0.8960 | max_loss: 0.5080 | max_backward: 1.9040 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.6816 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 6137.8 | uss_MB: 5194.1 | child_rss_MB: 5912.5 | allocated_MB: 450.3 | reserved_MB: 27130.0 | inactive_split_MB: 1088.5 +rank3 [2026-10-28 19:51:56,122] INFO : Step 10300/50000 rank3 | Train Loss: 1.6039 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.2943 | Cardinality: 0.1656 | Classification: 0.1487 | Classification_binary: 0.2679 | HitGrouping: 0.1881 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2132 | Regression_eta: 0.0030 | Regression_pt: 0.2158 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0891 | Classification_binary: 0.2622 | Regression_cos_phi: 0.0936 | Regression_energy: 0.1209 | Regression_eta: 0.0750 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0912 | LR: 9.09e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7926s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8152 | max_loss: 0.2478 | max_backward: 2.0483 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6143.7 | uss_MB: 5198.5 | child_rss_MB: 5930.4 | allocated_MB: 448.7 | reserved_MB: 28404.0 | inactive_split_MB: 991.9 +rank3 [2026-10-28 19:56:39,452] INFO : Step 10400/50000 rank3 | Train Loss: 1.7188 | Train Loss Unweighted: 0.8798 | Train Loss Components: Auxiliary: 0.3209 | Cardinality: 0.2578 | Classification: 0.1544 | Classification_binary: 0.2910 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2139 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0899 | Classification_binary: 0.3280 | Regression_cos_phi: 0.1593 | Regression_energy: 0.1210 | Regression_eta: 0.1050 | Regression_pt: 0.1322 | Regression_sin_phi: 0.1506 | LR: 9.07e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6757s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7821 | max_loss: 0.2448 | max_backward: 1.8782 | max_optimizer_step: 0.2932 | max_tensorboard_logging: 0.8061 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 6134.3 | uss_MB: 5193.2 | child_rss_MB: 5938.4 | allocated_MB: 448.9 | reserved_MB: 28404.0 | inactive_split_MB: 983.3 +rank3 [2026-10-28 20:01:29,563] INFO : Step 10500/50000 rank3 | Train Loss: 1.4652 | Train Loss Unweighted: 0.7986 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1697 | Classification: 0.1280 | Classification_binary: 0.2159 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2232 | Regression_eta: 0.0026 | Regression_pt: 0.2263 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0825 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0563 | Regression_energy: 0.1374 | Regression_eta: 0.0605 | Regression_pt: 0.1511 | Regression_sin_phi: 0.1004 | LR: 9.06e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.2156s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.7113 | max_loss: 0.1455 | max_backward: 1.6308 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.7760 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6134.1 | uss_MB: 5192.0 | child_rss_MB: 5916.0 | allocated_MB: 444.7 | reserved_MB: 28404.0 | inactive_split_MB: 1307.3 +rank3 [2026-10-28 20:06:14,520] INFO : Step 10600/50000 rank3 | Train Loss: 1.5657 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2802 | Cardinality: 0.1297 | Classification: 0.1617 | Classification_binary: 0.2471 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2084 | Regression_eta: 0.0037 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1119 | Classification_binary: 0.2692 | Regression_cos_phi: 0.1649 | Regression_energy: 0.1115 | Regression_eta: 0.0820 | Regression_pt: 0.1240 | Regression_sin_phi: 0.1579 | LR: 9.04e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.9399s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0173 | max_forward: 0.7875 | max_loss: 0.2211 | max_backward: 1.9019 | max_optimizer_step: 0.2814 | max_tensorboard_logging: 0.8518 | max_tensorboard_flush: 0.0072 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6133.2 | uss_MB: 5193.0 | child_rss_MB: 5934.7 | allocated_MB: 449.8 | reserved_MB: 28404.0 | inactive_split_MB: 1130.2 +rank3 [2026-10-28 20:10:59,505] INFO : Step 10700/50000 rank3 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8676 | Train Loss Components: Auxiliary: 0.3003 | Cardinality: 0.1749 | Classification: 0.1453 | Classification_binary: 0.2956 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2085 | Regression_eta: 0.0033 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0823 | Classification_binary: 0.3072 | Regression_cos_phi: 0.1456 | Regression_energy: 0.1078 | Regression_eta: 0.0925 | Regression_pt: 0.1206 | Regression_sin_phi: 0.1763 | LR: 9.02e-04 | DataLoad Time: 0.0118s | Model Forward Time: 5.1705s | Phase Times: max_pre_step_barrier: 0.0305 | max_batch_to_device: 0.0154 | max_forward: 2.0246 | max_loss: 0.2864 | max_backward: 2.7424 | max_optimizer_step: 0.1980 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6136.9 | uss_MB: 5195.8 | child_rss_MB: 5906.4 | allocated_MB: 446.2 | reserved_MB: 29148.0 | inactive_split_MB: 1134.5 +rank3 [2026-10-28 20:15:41,623] INFO : Step 10800/50000 rank3 | Train Loss: 1.5849 | Train Loss Unweighted: 0.7936 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1942 | Classification: 0.1383 | Classification_binary: 0.2480 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2007 | Regression_eta: 0.0032 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0869 | Classification_binary: 0.2748 | Regression_cos_phi: 0.1136 | Regression_energy: 0.1110 | Regression_eta: 0.0999 | Regression_pt: 0.1172 | Regression_sin_phi: 0.2261 | LR: 9.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 8.1982s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2408 | max_backward: 1.7977 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 1.7470 | max_tensorboard_flush: 0.0028 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 6126.4 | uss_MB: 4363.9 | child_rss_MB: 5910.3 | allocated_MB: 448.6 | reserved_MB: 30628.0 | inactive_split_MB: 1292.6 +rank3 [2026-10-28 20:20:32,833] INFO : Step 10900/50000 rank3 | Train Loss: 1.3552 | Train Loss Unweighted: 0.7247 | Train Loss Components: Auxiliary: 0.2531 | Cardinality: 0.1449 | Classification: 0.1275 | Classification_binary: 0.2074 | HitGrouping: 0.1874 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1917 | Regression_eta: 0.0024 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1802 | Regression_cos_phi: 0.0908 | Regression_energy: 0.1137 | Regression_eta: 0.0603 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0960 | LR: 8.99e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4556s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7076 | max_loss: 0.2425 | max_backward: 1.6536 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6131.8 | uss_MB: 5191.3 | child_rss_MB: 5932.5 | allocated_MB: 446.4 | reserved_MB: 30628.0 | inactive_split_MB: 984.9 +rank3 [2026-10-28 20:25:36,608] INFO : Step 11000/50000 rank3 | Train Loss: 1.5999 | Train Loss Unweighted: 0.8322 | Train Loss Components: Auxiliary: 0.2959 | Cardinality: 0.1737 | Classification: 0.1477 | Classification_binary: 0.2877 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0017 | Regression_energy: 0.1943 | Regression_eta: 0.0030 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1055 | Classification_binary: 0.3070 | Regression_cos_phi: 0.1678 | Regression_energy: 0.1196 | Regression_eta: 0.1072 | Regression_pt: 0.1293 | Regression_sin_phi: 0.1639 | LR: 8.97e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9318s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8260 | max_loss: 0.1503 | max_backward: 1.9603 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6124.9 | uss_MB: 5185.7 | child_rss_MB: 5918.3 | allocated_MB: 451.8 | reserved_MB: 30628.0 | inactive_split_MB: 996.6 +rank3 [2026-10-28 20:25:37,479] INFO : Running validation on rank3 +rank3 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 20:26:32,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:26:33,362] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 20:27:00,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 20:27:01,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 20:27:02,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 20:27:03,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 20:27:13,390] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 20:27:14,319] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:27:14,834] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 20:27:31,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 20:27:32,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 20:27:34,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 20:27:35,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 20:27:45,714] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-28 20:27:46,110] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 20:27:46,649] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 20:28:02,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 20:28:03,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 20:28:03,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 20:28:04,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 20:28:14,804] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 20:33:29,189] INFO : Step 11100/50000 rank3 | Train Loss: 1.6214 | Train Loss Unweighted: 0.8328 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.2186 | Classification: 0.1350 | Classification_binary: 0.2802 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2042 | Regression_eta: 0.0030 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0847 | Classification_binary: 0.3046 | Regression_cos_phi: 0.0980 | Regression_energy: 0.1217 | Regression_eta: 0.0838 | Regression_pt: 0.1336 | Regression_sin_phi: 0.1402 | LR: 8.95e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8818s | Phase Times: max_pre_step_barrier: 0.0088 | max_batch_to_device: 0.0068 | max_forward: 0.8488 | max_loss: 0.1548 | max_backward: 1.7645 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 1.1046 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7069.9 | uss_MB: 6125.7 | child_rss_MB: 5940.1 | allocated_MB: 448.1 | reserved_MB: 24340.0 | inactive_split_MB: 1081.9 +rank3 [2026-10-28 20:38:18,705] INFO : Step 11200/50000 rank3 | Train Loss: 1.7050 | Train Loss Unweighted: 0.8927 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2640 | Classification: 0.1534 | Classification_binary: 0.2523 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2404 | Regression_eta: 0.0030 | Regression_pt: 0.2410 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0827 | Classification_binary: 0.2069 | Regression_cos_phi: 0.0832 | Regression_energy: 0.1165 | Regression_eta: 0.0652 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0647 | LR: 8.93e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.8959s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8707 | max_loss: 0.2001 | max_backward: 2.2037 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.7912 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 6163.4 | uss_MB: 5218.5 | child_rss_MB: 5949.2 | allocated_MB: 447.8 | reserved_MB: 26392.0 | inactive_split_MB: 938.7 +rank3 [2026-10-28 20:43:01,723] INFO : Step 11300/50000 rank3 | Train Loss: 1.3885 | Train Loss Unweighted: 0.7511 | Train Loss Components: Auxiliary: 0.2594 | Cardinality: 0.1357 | Classification: 0.1316 | Classification_binary: 0.2357 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1874 | Regression_eta: 0.0024 | Regression_pt: 0.1915 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0846 | Classification_binary: 0.2577 | Regression_cos_phi: 0.1113 | Regression_energy: 0.0876 | Regression_eta: 0.0665 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0909 | LR: 8.91e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4289s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8053 | max_loss: 0.1469 | max_backward: 2.0062 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.4618 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1345 | Memory: rss_MB: 6165.0 | uss_MB: 5222.0 | child_rss_MB: 5948.8 | allocated_MB: 450.6 | reserved_MB: 26400.0 | inactive_split_MB: 1092.9 +rank3 [2026-10-28 20:48:05,522] INFO : Step 11400/50000 rank3 | Train Loss: 1.5865 | Train Loss Unweighted: 0.8323 | Train Loss Components: Auxiliary: 0.2981 | Cardinality: 0.1866 | Classification: 0.1338 | Classification_binary: 0.2899 | HitGrouping: 0.1840 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1999 | Regression_eta: 0.0028 | Regression_pt: 0.2030 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0989 | Classification_binary: 0.3262 | Regression_cos_phi: 0.1527 | Regression_energy: 0.1149 | Regression_eta: 0.1006 | Regression_pt: 0.1273 | Regression_sin_phi: 0.1519 | LR: 8.89e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.7658s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0769 | max_forward: 1.3800 | max_loss: 0.6715 | max_backward: 3.0339 | max_optimizer_step: 0.2660 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6163.4 | uss_MB: 5219.6 | child_rss_MB: 5946.4 | allocated_MB: 448.1 | reserved_MB: 27144.0 | inactive_split_MB: 1081.9 +rank3 [2026-10-28 20:52:57,975] INFO : Step 11500/50000 rank3 | Train Loss: 1.4825 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2779 | Cardinality: 0.1999 | Classification: 0.1273 | Classification_binary: 0.2463 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1956 | Regression_eta: 0.0026 | Regression_pt: 0.1972 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2435 | Regression_cos_phi: 0.1343 | Regression_energy: 0.1074 | Regression_eta: 0.0680 | Regression_pt: 0.1163 | Regression_sin_phi: 0.1003 | LR: 8.88e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.0553s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.7822 | max_loss: 0.1471 | max_backward: 1.7308 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 1.4583 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6164.2 | uss_MB: 5219.4 | child_rss_MB: 5944.4 | allocated_MB: 450.8 | reserved_MB: 27144.0 | inactive_split_MB: 1342.3 +rank3 [2026-10-28 20:57:41,224] INFO : Step 11600/50000 rank3 | Train Loss: 1.4219 | Train Loss Unweighted: 0.7444 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1679 | Classification: 0.1415 | Classification_binary: 0.2106 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1945 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2146 | Regression_cos_phi: 0.1456 | Regression_energy: 0.1219 | Regression_eta: 0.0654 | Regression_pt: 0.1334 | Regression_sin_phi: 0.1319 | LR: 8.86e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7003s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8362 | max_loss: 0.1428 | max_backward: 2.0109 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6156.9 | uss_MB: 5220.2 | child_rss_MB: 5947.3 | allocated_MB: 448.6 | reserved_MB: 27146.0 | inactive_split_MB: 1315.4 +rank3 [2026-10-28 21:02:35,116] INFO : Step 11700/50000 rank3 | Train Loss: 1.5203 | Train Loss Unweighted: 0.8017 | Train Loss Components: Auxiliary: 0.2866 | Cardinality: 0.1889 | Classification: 0.1302 | Classification_binary: 0.2577 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2024 | Regression_eta: 0.0026 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.2460 | Regression_cos_phi: 0.0892 | Regression_energy: 0.1092 | Regression_eta: 0.0579 | Regression_pt: 0.1201 | Regression_sin_phi: 0.1341 | LR: 8.84e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.3160s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 1.0014 | max_loss: 0.1975 | max_backward: 2.5347 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0688 | Memory: rss_MB: 6157.0 | uss_MB: 5144.6 | child_rss_MB: 5922.6 | allocated_MB: 448.7 | reserved_MB: 27266.0 | inactive_split_MB: 1090.9 +rank3 [2026-10-28 21:07:42,591] INFO : Step 11800/50000 rank3 | Train Loss: 1.5591 | Train Loss Unweighted: 0.7907 | Train Loss Components: Auxiliary: 0.2873 | Cardinality: 0.2051 | Classification: 0.1346 | Classification_binary: 0.2655 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1918 | Regression_eta: 0.0030 | Regression_pt: 0.1929 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0892 | Classification_binary: 0.2586 | Regression_cos_phi: 0.1333 | Regression_energy: 0.1091 | Regression_eta: 0.1070 | Regression_pt: 0.1184 | Regression_sin_phi: 0.1480 | LR: 8.82e-04 | DataLoad Time: 0.0215s | Model Forward Time: 7.4825s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.0966 | max_forward: 1.7907 | max_loss: 0.6496 | max_backward: 2.7103 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 2.3587 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6136.9 | uss_MB: 5209.7 | child_rss_MB: 5946.4 | allocated_MB: 450.4 | reserved_MB: 27266.0 | inactive_split_MB: 1187.2 +rank3 [2026-10-28 21:12:51,257] INFO : Step 11900/50000 rank3 | Train Loss: 1.3283 | Train Loss Unweighted: 0.7113 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1476 | Classification: 0.1193 | Classification_binary: 0.2094 | HitGrouping: 0.1824 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0021 | Regression_pt: 0.1901 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0872 | Classification_binary: 0.2127 | Regression_cos_phi: 0.0918 | Regression_energy: 0.1011 | Regression_eta: 0.0529 | Regression_pt: 0.1148 | Regression_sin_phi: 0.1092 | LR: 8.80e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.1682s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0785 | max_forward: 0.7917 | max_loss: 0.1465 | max_backward: 1.5595 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6137.9 | uss_MB: 5216.4 | child_rss_MB: 5947.1 | allocated_MB: 443.9 | reserved_MB: 27266.0 | inactive_split_MB: 955.3 +rank3 [2026-10-28 21:17:48,734] INFO : Step 12000/50000 rank3 | Train Loss: 1.4074 | Train Loss Unweighted: 0.7415 | Train Loss Components: Auxiliary: 0.2683 | Cardinality: 0.1719 | Classification: 0.1235 | Classification_binary: 0.2283 | HitGrouping: 0.1812 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1909 | Regression_eta: 0.0022 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0834 | Classification_binary: 0.1862 | Regression_cos_phi: 0.1026 | Regression_energy: 0.0985 | Regression_eta: 0.0593 | Regression_pt: 0.1081 | Regression_sin_phi: 0.0871 | LR: 8.78e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2509s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0303 | max_forward: 0.7589 | max_loss: 0.1459 | max_backward: 1.9073 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6136.6 | uss_MB: 5219.4 | child_rss_MB: 5952.0 | allocated_MB: 447.6 | reserved_MB: 27266.0 | inactive_split_MB: 1310.4 +rank3 [2026-10-28 21:17:49,718] INFO : Running validation on rank3 +rank3 [2026-10-28 21:17:49,718] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 21:17:49,718] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 21:18:44,915] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:18:45,602] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 21:19:11,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 21:19:12,613] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 21:19:13,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 21:19:14,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 21:19:24,708] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 21:19:25,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:19:25,829] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 21:19:42,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 21:19:43,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 21:19:44,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 21:19:45,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 21:19:55,036] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 21:19:55,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 21:19:56,383] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 21:20:12,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 21:20:13,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 21:20:13,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 21:20:14,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 21:20:24,858] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 21:25:33,195] INFO : Step 12100/50000 rank3 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8521 | Train Loss Components: Auxiliary: 0.3093 | Cardinality: 0.2088 | Classification: 0.1337 | Classification_binary: 0.2971 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0029 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0713 | Classification_binary: 0.3015 | Regression_cos_phi: 0.1074 | Regression_energy: 0.1076 | Regression_eta: 0.0595 | Regression_pt: 0.1184 | Regression_sin_phi: 0.1155 | LR: 8.76e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9398s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8139 | max_loss: 0.1586 | max_backward: 1.9999 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9439 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7027.3 | uss_MB: 6104.0 | child_rss_MB: 5925.8 | allocated_MB: 450.8 | reserved_MB: 28008.0 | inactive_split_MB: 1125.6 +rank3 [2026-10-28 21:30:25,391] INFO : Step 12200/50000 rank3 | Train Loss: 1.4655 | Train Loss Unweighted: 0.7431 | Train Loss Components: Auxiliary: 0.2757 | Cardinality: 0.1845 | Classification: 0.1184 | Classification_binary: 0.2450 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1862 | Regression_eta: 0.0026 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.2138 | Regression_cos_phi: 0.1132 | Regression_energy: 0.1074 | Regression_eta: 0.0612 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0686 | LR: 8.74e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4990s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0364 | max_forward: 0.7684 | max_loss: 0.1438 | max_backward: 1.8240 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.7243 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0697 | Memory: rss_MB: 6148.5 | uss_MB: 5233.4 | child_rss_MB: 5949.1 | allocated_MB: 445.9 | reserved_MB: 28008.0 | inactive_split_MB: 995.3 +rank3 [2026-10-28 21:35:16,328] INFO : Step 12300/50000 rank3 | Train Loss: 1.3845 | Train Loss Unweighted: 0.7270 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1523 | Classification: 0.1341 | Classification_binary: 0.2109 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1869 | Regression_eta: 0.0026 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.2295 | Regression_cos_phi: 0.1029 | Regression_energy: 0.1110 | Regression_eta: 0.0892 | Regression_pt: 0.1197 | Regression_sin_phi: 0.1765 | LR: 8.72e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5876s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7264 | max_loss: 0.1976 | max_backward: 1.8534 | max_optimizer_step: 0.2993 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6147.3 | uss_MB: 5233.4 | child_rss_MB: 5944.1 | allocated_MB: 448.1 | reserved_MB: 29988.0 | inactive_split_MB: 1298.2 +rank3 [2026-10-28 21:40:15,077] INFO : Step 12400/50000 rank3 | Train Loss: 1.4006 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.2654 | Cardinality: 0.1602 | Classification: 0.1310 | Classification_binary: 0.2363 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0773 | Classification_binary: 0.2316 | Regression_cos_phi: 0.1233 | Regression_energy: 0.0923 | Regression_eta: 0.0792 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0805 | LR: 8.70e-04 | DataLoad Time: 0.0149s | Model Forward Time: 7.1297s | Phase Times: max_pre_step_barrier: 0.0107 | max_batch_to_device: 0.0207 | max_forward: 1.2284 | max_loss: 0.2740 | max_backward: 2.6428 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8714 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 6147.7 | uss_MB: 5022.1 | child_rss_MB: 5936.9 | allocated_MB: 448.8 | reserved_MB: 29988.0 | inactive_split_MB: 1332.2 +rank3 [2026-10-28 21:45:07,596] INFO : Step 12500/50000 rank3 | Train Loss: 1.4931 | Train Loss Unweighted: 0.7691 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1768 | Classification: 0.1197 | Classification_binary: 0.2587 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1910 | Regression_eta: 0.0024 | Regression_pt: 0.1944 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.2708 | Regression_cos_phi: 0.1414 | Regression_energy: 0.0917 | Regression_eta: 0.0750 | Regression_pt: 0.0995 | Regression_sin_phi: 0.1229 | LR: 8.68e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7451s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0333 | max_forward: 0.8806 | max_loss: 0.1518 | max_backward: 2.1554 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.5983 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6144.5 | uss_MB: 5232.3 | child_rss_MB: 5946.8 | allocated_MB: 452.8 | reserved_MB: 29990.0 | inactive_split_MB: 1046.7 +rank3 [2026-10-28 21:50:03,650] INFO : Step 12600/50000 rank3 | Train Loss: 1.3547 | Train Loss Unweighted: 0.7028 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1629 | Classification: 0.1033 | Classification_binary: 0.2112 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1912 | Regression_eta: 0.0024 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.2210 | Regression_cos_phi: 0.1104 | Regression_energy: 0.1183 | Regression_eta: 0.0732 | Regression_pt: 0.1332 | Regression_sin_phi: 0.1146 | LR: 8.66e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5167s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.7639 | max_loss: 0.2516 | max_backward: 1.8947 | max_optimizer_step: 0.2049 | max_tensorboard_logging: 0.7350 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6141.1 | uss_MB: 5233.0 | child_rss_MB: 5920.5 | allocated_MB: 446.9 | reserved_MB: 29990.0 | inactive_split_MB: 1034.9 +rank3 [2026-10-28 21:54:54,644] INFO : Step 12700/50000 rank3 | Train Loss: 1.3557 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1573 | Classification: 0.1262 | Classification_binary: 0.2132 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1863 | Regression_eta: 0.0022 | Regression_pt: 0.1887 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0911 | Classification_binary: 0.1922 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0840 | Regression_eta: 0.0566 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0783 | LR: 8.64e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.1256s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7499 | max_loss: 0.2453 | max_backward: 3.4334 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6139.6 | uss_MB: 5232.1 | child_rss_MB: 5948.4 | allocated_MB: 450.2 | reserved_MB: 29990.0 | inactive_split_MB: 1245.6 +rank3 [2026-10-28 21:59:39,371] INFO : Step 12800/50000 rank3 | Train Loss: 1.5298 | Train Loss Unweighted: 0.7754 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1939 | Classification: 0.1237 | Classification_binary: 0.2821 | HitGrouping: 0.1822 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1810 | Regression_eta: 0.0027 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0921 | Classification_binary: 0.2866 | Regression_cos_phi: 0.1200 | Regression_energy: 0.0896 | Regression_eta: 0.0918 | Regression_pt: 0.0965 | Regression_sin_phi: 0.1256 | LR: 8.62e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.2917s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.5299 | max_backward: 2.3185 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6133.4 | uss_MB: 4813.1 | child_rss_MB: 5927.2 | allocated_MB: 449.9 | reserved_MB: 32006.0 | inactive_split_MB: 889.3 +rank3 [2026-10-28 22:04:39,764] INFO : Step 12900/50000 rank3 | Train Loss: 1.3302 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1576 | Classification: 0.1054 | Classification_binary: 0.2343 | HitGrouping: 0.1811 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1758 | Regression_eta: 0.0020 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2212 | Regression_cos_phi: 0.0857 | Regression_energy: 0.0885 | Regression_eta: 0.0465 | Regression_pt: 0.0992 | Regression_sin_phi: 0.1181 | LR: 8.60e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.6340s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0137 | max_forward: 0.9836 | max_loss: 0.2293 | max_backward: 2.2888 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.9302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6139.6 | uss_MB: 5231.4 | child_rss_MB: 5950.5 | allocated_MB: 445.9 | reserved_MB: 32006.0 | inactive_split_MB: 1247.6 +rank3 [2026-10-28 22:09:25,338] INFO : Step 13000/50000 rank3 | Train Loss: 1.3124 | Train Loss Unweighted: 0.7073 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1406 | Classification: 0.1147 | Classification_binary: 0.1973 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1944 | Regression_eta: 0.0023 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0851 | Regression_energy: 0.1147 | Regression_eta: 0.0610 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0608 | LR: 8.58e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2796s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0065 | max_forward: 0.7321 | max_loss: 0.1616 | max_backward: 1.7374 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6140.2 | uss_MB: 5231.6 | child_rss_MB: 5949.6 | allocated_MB: 444.7 | reserved_MB: 32006.0 | inactive_split_MB: 1300.2 +rank3 [2026-10-28 22:09:26,329] INFO : Running validation on rank3 +rank3 [2026-10-28 22:09:26,329] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 22:09:26,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 22:10:21,902] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:10:22,475] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 22:10:48,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 22:10:49,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 22:10:50,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 22:10:51,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 22:11:01,678] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 22:11:02,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:11:02,874] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 22:11:19,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 22:11:20,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 22:11:20,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 22:11:21,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 22:11:31,957] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 22:11:32,690] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 22:11:33,217] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 22:11:48,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 22:11:49,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 22:11:50,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 22:11:51,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 22:12:01,618] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-28 22:17:08,485] INFO : Step 13100/50000 rank3 | Train Loss: 1.6165 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3082 | Cardinality: 0.2635 | Classification: 0.1364 | Classification_binary: 0.2628 | HitGrouping: 0.1835 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0026 | Regression_pt: 0.1957 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0788 | Classification_binary: 0.2536 | Regression_cos_phi: 0.0841 | Regression_energy: 0.1167 | Regression_eta: 0.0772 | Regression_pt: 0.1284 | Regression_sin_phi: 0.1068 | LR: 8.56e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2713s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0479 | max_forward: 0.8994 | max_loss: 0.2807 | max_backward: 2.1108 | max_optimizer_step: 0.2760 | max_tensorboard_logging: 0.9690 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 7057.8 | uss_MB: 6143.5 | child_rss_MB: 5948.1 | allocated_MB: 446.5 | reserved_MB: 23120.0 | inactive_split_MB: 920.0 +rank3 [2026-10-28 22:22:01,650] INFO : Step 13200/50000 rank3 | Train Loss: 1.4928 | Train Loss Unweighted: 0.7469 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.2211 | Classification: 0.1278 | Classification_binary: 0.2505 | HitGrouping: 0.1746 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1808 | Regression_eta: 0.0025 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0756 | Classification_binary: 0.2458 | Regression_cos_phi: 0.1496 | Regression_energy: 0.1015 | Regression_eta: 0.0853 | Regression_pt: 0.1108 | Regression_sin_phi: 0.1181 | LR: 8.54e-04 | DataLoad Time: 0.0190s | Model Forward Time: 6.5992s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.7755 | max_loss: 0.2864 | max_backward: 3.4589 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6222.1 | uss_MB: 5074.0 | child_rss_MB: 5943.3 | allocated_MB: 448.3 | reserved_MB: 25156.0 | inactive_split_MB: 978.0 +rank3 [2026-10-28 22:26:52,973] INFO : Step 13300/50000 rank3 | Train Loss: 1.3251 | Train Loss Unweighted: 0.6741 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1607 | Classification: 0.1132 | Classification_binary: 0.1927 | HitGrouping: 0.1773 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1805 | Regression_eta: 0.0024 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2176 | Regression_cos_phi: 0.0881 | Regression_energy: 0.1103 | Regression_eta: 0.0636 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0837 | LR: 8.52e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3784s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8118 | max_loss: 0.1467 | max_backward: 1.5503 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.9039 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6221.5 | uss_MB: 5307.8 | child_rss_MB: 5952.8 | allocated_MB: 446.0 | reserved_MB: 26736.0 | inactive_split_MB: 965.8 +rank3 [2026-10-28 22:31:44,118] INFO : Step 13400/50000 rank3 | Train Loss: 1.2635 | Train Loss Unweighted: 0.6770 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1436 | Classification: 0.1317 | Classification_binary: 0.2016 | HitGrouping: 0.1790 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1692 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2032 | Regression_cos_phi: 0.0846 | Regression_energy: 0.0861 | Regression_eta: 0.0676 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0669 | LR: 8.50e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7370s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8396 | max_loss: 0.1412 | max_backward: 2.0130 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.8047 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6225.0 | uss_MB: 5310.9 | child_rss_MB: 5925.4 | allocated_MB: 445.8 | reserved_MB: 26736.0 | inactive_split_MB: 1014.4 +rank3 [2026-10-28 22:36:27,627] INFO : Step 13500/50000 rank3 | Train Loss: 1.3935 | Train Loss Unweighted: 0.7401 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1713 | Classification: 0.1245 | Classification_binary: 0.2541 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0021 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2420 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0903 | Regression_eta: 0.0484 | Regression_pt: 0.0983 | Regression_sin_phi: 0.0860 | LR: 8.48e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5759s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0859 | max_forward: 0.8721 | max_loss: 0.1498 | max_backward: 1.7581 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.7855 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6221.6 | uss_MB: 5308.9 | child_rss_MB: 5951.2 | allocated_MB: 449.1 | reserved_MB: 26736.0 | inactive_split_MB: 922.5 +rank3 [2026-10-28 22:41:19,303] INFO : Step 13600/50000 rank3 | Train Loss: 1.3997 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1870 | Classification: 0.1206 | Classification_binary: 0.2286 | HitGrouping: 0.1765 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1702 | Regression_eta: 0.0026 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0749 | Classification_binary: 0.2297 | Regression_cos_phi: 0.1211 | Regression_energy: 0.0972 | Regression_eta: 0.0809 | Regression_pt: 0.1050 | Regression_sin_phi: 0.0959 | LR: 8.45e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.2263s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7703 | max_loss: 0.2484 | max_backward: 1.6943 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 6210.7 | uss_MB: 5299.9 | child_rss_MB: 5953.4 | allocated_MB: 450.3 | reserved_MB: 26736.0 | inactive_split_MB: 922.0 +rank3 [2026-10-28 22:46:12,326] INFO : Step 13700/50000 rank3 | Train Loss: 1.2648 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2420 | Cardinality: 0.1315 | Classification: 0.1136 | Classification_binary: 0.1964 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1793 | Regression_eta: 0.0020 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.1796 | Regression_cos_phi: 0.1062 | Regression_energy: 0.0784 | Regression_eta: 0.0574 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0967 | LR: 8.43e-04 | DataLoad Time: 0.0123s | Model Forward Time: 6.2912s | Phase Times: max_pre_step_barrier: 0.0406 | max_batch_to_device: 0.0119 | max_forward: 1.2281 | max_loss: 0.2480 | max_backward: 3.6472 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.6460 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0767 | Memory: rss_MB: 6216.0 | uss_MB: 5307.4 | child_rss_MB: 5948.4 | allocated_MB: 446.9 | reserved_MB: 26742.0 | inactive_split_MB: 1058.7 +rank3 [2026-10-28 22:51:09,233] INFO : Step 13800/50000 rank3 | Train Loss: 1.3469 | Train Loss Unweighted: 0.7266 | Train Loss Components: Auxiliary: 0.2604 | Cardinality: 0.1436 | Classification: 0.1184 | Classification_binary: 0.2382 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1812 | Regression_eta: 0.0021 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.2641 | Regression_cos_phi: 0.1010 | Regression_energy: 0.0992 | Regression_eta: 0.0661 | Regression_pt: 0.1070 | Regression_sin_phi: 0.1139 | LR: 8.41e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.8522s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7995 | max_loss: 0.1458 | max_backward: 3.4735 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6221.0 | uss_MB: 5310.1 | child_rss_MB: 5919.7 | allocated_MB: 446.4 | reserved_MB: 27082.0 | inactive_split_MB: 1370.3 +rank3 [2026-10-28 22:56:04,349] INFO : Step 13900/50000 rank3 | Train Loss: 1.4882 | Train Loss Unweighted: 0.7682 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1631 | Classification: 0.1199 | Classification_binary: 0.2624 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1900 | Regression_eta: 0.0029 | Regression_pt: 0.1902 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0824 | Classification_binary: 0.3013 | Regression_cos_phi: 0.1481 | Regression_energy: 0.1110 | Regression_eta: 0.0943 | Regression_pt: 0.1199 | Regression_sin_phi: 0.1413 | LR: 8.39e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4523s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.7750 | max_loss: 0.1563 | max_backward: 1.5716 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.8868 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6217.9 | uss_MB: 5307.2 | child_rss_MB: 5946.5 | allocated_MB: 449.9 | reserved_MB: 27316.0 | inactive_split_MB: 918.0 +rank3 [2026-10-28 23:00:54,751] INFO : Step 14000/50000 rank3 | Train Loss: 1.2239 | Train Loss Unweighted: 0.6384 | Train Loss Components: Auxiliary: 0.2362 | Cardinality: 0.1573 | Classification: 0.0898 | Classification_binary: 0.1853 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0021 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1825 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0835 | Regression_eta: 0.0518 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0586 | LR: 8.37e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5037s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7611 | max_loss: 0.1442 | max_backward: 1.7887 | max_optimizer_step: 0.2976 | max_tensorboard_logging: 0.7418 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6219.2 | uss_MB: 5309.9 | child_rss_MB: 5948.9 | allocated_MB: 446.8 | reserved_MB: 29264.0 | inactive_split_MB: 1150.9 +rank3 [2026-10-28 23:00:55,745] INFO : Running validation on rank3 +rank3 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 23:01:50,597] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:01:51,151] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 23:02:17,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 23:02:18,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 23:02:20,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 23:02:21,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 23:02:31,854] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-28 23:02:33,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:02:33,998] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 23:02:50,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 23:02:51,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 23:02:52,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 23:02:52,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 23:03:03,000] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 23:03:03,406] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:03:03,862] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 23:03:19,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 23:03:20,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 23:03:21,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 23:03:22,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 23:03:32,450] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 23:08:22,316] INFO : Step 14100/50000 rank3 | Train Loss: 1.3472 | Train Loss Unweighted: 0.6964 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1511 | Classification: 0.1164 | Classification_binary: 0.2090 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1819 | Regression_eta: 0.0023 | Regression_pt: 0.1843 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.2482 | Regression_cos_phi: 0.1446 | Regression_energy: 0.1036 | Regression_eta: 0.1217 | Regression_pt: 0.1111 | Regression_sin_phi: 0.1629 | LR: 8.35e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2199s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8885 | max_loss: 0.1430 | max_backward: 2.1784 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 1.1214 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7149.4 | uss_MB: 6238.2 | child_rss_MB: 5960.3 | allocated_MB: 451.8 | reserved_MB: 25710.0 | inactive_split_MB: 1103.7 +rank3 [2026-10-28 23:12:52,861] INFO : Step 14200/50000 rank3 | Train Loss: 1.4358 | Train Loss Unweighted: 0.7354 | Train Loss Components: Auxiliary: 0.2744 | Cardinality: 0.1685 | Classification: 0.1248 | Classification_binary: 0.2623 | HitGrouping: 0.1814 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1709 | Regression_eta: 0.0023 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 0.2752 | Regression_cos_phi: 0.0730 | Regression_energy: 0.0824 | Regression_eta: 0.0602 | Regression_pt: 0.0884 | Regression_sin_phi: 0.1105 | LR: 8.32e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3433s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0875 | max_forward: 0.8579 | max_loss: 0.1577 | max_backward: 1.8303 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6333.7 | uss_MB: 5422.0 | child_rss_MB: 5960.5 | allocated_MB: 449.2 | reserved_MB: 27638.0 | inactive_split_MB: 1319.6 +rank3 [2026-10-28 23:17:26,477] INFO : Step 14300/50000 rank3 | Train Loss: 1.2642 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2452 | Cardinality: 0.1652 | Classification: 0.1069 | Classification_binary: 0.2108 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0018 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.2072 | Regression_cos_phi: 0.0994 | Regression_energy: 0.0749 | Regression_eta: 0.0609 | Regression_pt: 0.0826 | Regression_sin_phi: 0.1166 | LR: 8.30e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3450s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7599 | max_loss: 0.1477 | max_backward: 1.7369 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6331.9 | uss_MB: 5419.4 | child_rss_MB: 5959.7 | allocated_MB: 446.3 | reserved_MB: 29582.0 | inactive_split_MB: 949.7 +rank3 [2026-10-28 23:22:05,869] INFO : Step 14400/50000 rank3 | Train Loss: 1.2457 | Train Loss Unweighted: 0.6253 | Train Loss Components: Auxiliary: 0.2300 | Cardinality: 0.1408 | Classification: 0.1072 | Classification_binary: 0.1847 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1629 | Regression_eta: 0.0022 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1756 | Regression_cos_phi: 0.1670 | Regression_energy: 0.0919 | Regression_eta: 0.0933 | Regression_pt: 0.1003 | Regression_sin_phi: 0.1337 | LR: 8.28e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.9716s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.9667 | max_loss: 0.2306 | max_backward: 2.7434 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6331.8 | uss_MB: 4969.8 | child_rss_MB: 5956.2 | allocated_MB: 450.7 | reserved_MB: 29584.0 | inactive_split_MB: 1252.8 +rank3 [2026-10-28 23:26:50,539] INFO : Step 14500/50000 rank3 | Train Loss: 1.4093 | Train Loss Unweighted: 0.7194 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1803 | Classification: 0.1211 | Classification_binary: 0.2406 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.2524 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0878 | Regression_eta: 0.0535 | Regression_pt: 0.0963 | Regression_sin_phi: 0.0925 | LR: 8.26e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8484s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8204 | max_loss: 0.1455 | max_backward: 2.0838 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7592 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1066 | Memory: rss_MB: 6329.0 | uss_MB: 5423.5 | child_rss_MB: 5932.6 | allocated_MB: 449.1 | reserved_MB: 29584.0 | inactive_split_MB: 1312.1 +rank3 [2026-10-28 23:31:26,129] INFO : Step 14600/50000 rank3 | Train Loss: 1.4466 | Train Loss Unweighted: 0.7628 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1607 | Classification: 0.1257 | Classification_binary: 0.2761 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1767 | Regression_eta: 0.0023 | Regression_pt: 0.1796 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0837 | Classification_binary: 0.3095 | Regression_cos_phi: 0.1081 | Regression_energy: 0.0846 | Regression_eta: 0.0780 | Regression_pt: 0.0941 | Regression_sin_phi: 0.1306 | LR: 8.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8556s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0622 | max_forward: 0.8658 | max_loss: 0.1496 | max_backward: 1.9094 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.9894 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6320.2 | uss_MB: 5414.6 | child_rss_MB: 5962.0 | allocated_MB: 450.2 | reserved_MB: 29584.0 | inactive_split_MB: 939.8 +rank3 [2026-10-28 23:35:59,706] INFO : Step 14700/50000 rank3 | Train Loss: 1.2298 | Train Loss Unweighted: 0.6544 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1321 | Classification: 0.0995 | Classification_binary: 0.1882 | HitGrouping: 0.1788 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1813 | Regression_eta: 0.0021 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1682 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0943 | Regression_eta: 0.0497 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0850 | LR: 8.21e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9552s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8361 | max_loss: 0.2462 | max_backward: 1.9700 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.8314 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6323.9 | uss_MB: 5421.4 | child_rss_MB: 5964.3 | allocated_MB: 445.8 | reserved_MB: 31548.0 | inactive_split_MB: 1127.1 +rank3 [2026-10-28 23:40:34,506] INFO : Step 14800/50000 rank3 | Train Loss: 1.2682 | Train Loss Unweighted: 0.6574 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1516 | Classification: 0.1063 | Classification_binary: 0.2120 | HitGrouping: 0.1767 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1664 | Regression_eta: 0.0020 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.2424 | Regression_cos_phi: 0.0940 | Regression_energy: 0.0907 | Regression_eta: 0.0715 | Regression_pt: 0.0981 | Regression_sin_phi: 0.1440 | LR: 8.19e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6971s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7530 | max_loss: 0.1476 | max_backward: 1.8229 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6323.4 | uss_MB: 5420.5 | child_rss_MB: 5957.9 | allocated_MB: 449.4 | reserved_MB: 33554.0 | inactive_split_MB: 1332.6 +rank3 [2026-10-28 23:45:11,571] INFO : Step 14900/50000 rank3 | Train Loss: 1.2753 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2409 | Cardinality: 0.1305 | Classification: 0.1126 | Classification_binary: 0.2387 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1497 | Regression_eta: 0.0022 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.2295 | Regression_cos_phi: 0.1091 | Regression_energy: 0.0860 | Regression_eta: 0.0813 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0937 | LR: 8.17e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8750s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.7955 | max_loss: 0.1633 | max_backward: 1.8376 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.9623 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1206 | Memory: rss_MB: 6322.3 | uss_MB: 5420.8 | child_rss_MB: 5961.8 | allocated_MB: 449.7 | reserved_MB: 33554.0 | inactive_split_MB: 1310.1 +rank3 [2026-10-28 23:49:45,306] INFO : Step 15000/50000 rank3 | Train Loss: 1.2701 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1516 | Classification: 0.1012 | Classification_binary: 0.2121 | HitGrouping: 0.1755 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2029 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0921 | Regression_eta: 0.0448 | Regression_pt: 0.0983 | Regression_sin_phi: 0.0845 | LR: 8.15e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1017s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7505 | max_loss: 0.2449 | max_backward: 1.7335 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.4870 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6318.2 | uss_MB: 5419.0 | child_rss_MB: 5953.6 | allocated_MB: 447.3 | reserved_MB: 33554.0 | inactive_split_MB: 1038.0 +rank3 [2026-10-28 23:49:46,285] INFO : Running validation on rank3 +rank3 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-28 23:50:37,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:50:38,006] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-28 23:51:04,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-28 23:51:05,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-28 23:51:06,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-28 23:51:07,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-28 23:51:17,667] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 23:51:18,491] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:51:18,969] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-28 23:51:35,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-28 23:51:36,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-28 23:51:37,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-28 23:51:38,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-28 23:51:48,274] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-28 23:51:48,549] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-28 23:51:49,026] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-28 23:52:04,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-28 23:52:05,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-28 23:52:06,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-28 23:52:07,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-28 23:52:17,430] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-28 23:57:18,880] INFO : Step 15100/50000 rank3 | Train Loss: 1.2388 | Train Loss Unweighted: 0.6590 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1366 | Classification: 0.1178 | Classification_binary: 0.1841 | HitGrouping: 0.1810 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1743 | Regression_eta: 0.0018 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0656 | Regression_energy: 0.1156 | Regression_eta: 0.0378 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0627 | LR: 8.12e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8273s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.8010 | max_loss: 0.1660 | max_backward: 1.9832 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.8565 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7422.7 | uss_MB: 6518.8 | child_rss_MB: 5938.1 | allocated_MB: 446.2 | reserved_MB: 25230.0 | inactive_split_MB: 1087.3 +rank3 [2026-10-29 00:02:05,559] INFO : Step 15200/50000 rank3 | Train Loss: 1.3362 | Train Loss Unweighted: 0.6611 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1596 | Classification: 0.1086 | Classification_binary: 0.2256 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1600 | Regression_eta: 0.0022 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.2151 | Regression_cos_phi: 0.1259 | Regression_energy: 0.0751 | Regression_eta: 0.0653 | Regression_pt: 0.0848 | Regression_sin_phi: 0.1074 | LR: 8.10e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6171s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1077 | max_forward: 0.8572 | max_loss: 0.1505 | max_backward: 1.8951 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7089 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6402.3 | uss_MB: 5500.4 | child_rss_MB: 5939.7 | allocated_MB: 449.9 | reserved_MB: 25448.0 | inactive_split_MB: 1183.4 +rank3 [2026-10-29 00:06:54,749] INFO : Step 15300/50000 rank3 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6783 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1400 | Classification: 0.1062 | Classification_binary: 0.2357 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1646 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.2379 | Regression_cos_phi: 0.1064 | Regression_energy: 0.0775 | Regression_eta: 0.0678 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0727 | LR: 8.08e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7476s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7962 | max_loss: 0.2606 | max_backward: 1.9831 | max_optimizer_step: 0.2491 | max_tensorboard_logging: 0.7840 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6402.1 | uss_MB: 5500.3 | child_rss_MB: 5940.3 | allocated_MB: 446.7 | reserved_MB: 27282.0 | inactive_split_MB: 1189.4 +rank3 [2026-10-29 00:11:34,440] INFO : Step 15400/50000 rank3 | Train Loss: 1.2329 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1583 | Classification: 0.1127 | Classification_binary: 0.1891 | HitGrouping: 0.1750 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1648 | Regression_eta: 0.0018 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0777 | Regression_eta: 0.0354 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0541 | LR: 8.05e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2507s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.7153 | max_loss: 0.1508 | max_backward: 1.6501 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.5965 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6395.9 | uss_MB: 5491.2 | child_rss_MB: 5941.0 | allocated_MB: 444.7 | reserved_MB: 27520.0 | inactive_split_MB: 1102.9 +rank3 [2026-10-29 00:16:23,332] INFO : Step 15500/50000 rank3 | Train Loss: 1.2944 | Train Loss Unweighted: 0.6657 | Train Loss Components: Auxiliary: 0.2480 | Cardinality: 0.1645 | Classification: 0.1174 | Classification_binary: 0.2036 | HitGrouping: 0.1739 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1691 | Regression_eta: 0.0021 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.2115 | Regression_cos_phi: 0.1003 | Regression_energy: 0.1025 | Regression_eta: 0.0567 | Regression_pt: 0.1116 | Regression_sin_phi: 0.1078 | LR: 8.03e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.2292s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.9970 | max_loss: 0.1740 | max_backward: 2.3916 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.8759 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 6406.1 | uss_MB: 5472.3 | child_rss_MB: 5938.5 | allocated_MB: 449.9 | reserved_MB: 27520.0 | inactive_split_MB: 1174.5 +rank3 [2026-10-29 00:21:07,249] INFO : Step 15600/50000 rank3 | Train Loss: 1.4613 | Train Loss Unweighted: 0.7617 | Train Loss Components: Auxiliary: 0.2872 | Cardinality: 0.1801 | Classification: 0.1276 | Classification_binary: 0.2641 | HitGrouping: 0.1785 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1795 | Regression_eta: 0.0025 | Regression_pt: 0.1856 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0814 | Classification_binary: 0.2572 | Regression_cos_phi: 0.1133 | Regression_energy: 0.1087 | Regression_eta: 0.0947 | Regression_pt: 0.1201 | Regression_sin_phi: 0.1050 | LR: 8.01e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.7188s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.9893 | max_loss: 0.1680 | max_backward: 2.1731 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.0419 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6396.7 | uss_MB: 5034.1 | child_rss_MB: 5938.2 | allocated_MB: 448.1 | reserved_MB: 27520.0 | inactive_split_MB: 1080.9 +rank3 [2026-10-29 00:25:54,170] INFO : Step 15700/50000 rank3 | Train Loss: 1.3299 | Train Loss Unweighted: 0.6547 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1880 | Classification: 0.1192 | Classification_binary: 0.2203 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1547 | Regression_eta: 0.0020 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.2237 | Regression_cos_phi: 0.1043 | Regression_energy: 0.0855 | Regression_eta: 0.0591 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0705 | LR: 7.98e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.0315s | Phase Times: max_pre_step_barrier: 0.0061 | max_batch_to_device: 0.0068 | max_forward: 0.9761 | max_loss: 0.2543 | max_backward: 2.0225 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8016 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.1020 | Memory: rss_MB: 6402.6 | uss_MB: 5496.7 | child_rss_MB: 5941.3 | allocated_MB: 448.3 | reserved_MB: 27520.0 | inactive_split_MB: 1390.5 +rank3 [2026-10-29 00:30:33,525] INFO : Step 15800/50000 rank3 | Train Loss: 1.2079 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1572 | Classification: 0.0909 | Classification_binary: 0.1814 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1704 | Regression_eta: 0.0019 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1497 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0868 | Regression_eta: 0.0433 | Regression_pt: 0.0924 | Regression_sin_phi: 0.0513 | LR: 7.96e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3441s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1066 | max_forward: 0.8154 | max_loss: 0.1440 | max_backward: 1.6020 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.7740 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6405.3 | uss_MB: 5497.2 | child_rss_MB: 5941.3 | allocated_MB: 447.1 | reserved_MB: 27520.0 | inactive_split_MB: 1038.3 +rank3 [2026-10-29 00:35:19,264] INFO : Step 15900/50000 rank3 | Train Loss: 1.4055 | Train Loss Unweighted: 0.7313 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1808 | Classification: 0.1304 | Classification_binary: 0.2426 | HitGrouping: 0.1763 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1752 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.2545 | Regression_cos_phi: 0.0882 | Regression_energy: 0.1122 | Regression_eta: 0.0599 | Regression_pt: 0.1253 | Regression_sin_phi: 0.1237 | LR: 7.94e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.7741s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 1.0653 | max_loss: 0.1515 | max_backward: 2.0024 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.6957 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6402.5 | uss_MB: 5500.0 | child_rss_MB: 5940.1 | allocated_MB: 447.4 | reserved_MB: 27520.0 | inactive_split_MB: 1172.5 +rank3 [2026-10-29 00:40:00,684] INFO : Step 16000/50000 rank3 | Train Loss: 1.3107 | Train Loss Unweighted: 0.6580 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1760 | Classification: 0.1019 | Classification_binary: 0.2391 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1548 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2270 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0759 | Regression_eta: 0.0622 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0811 | LR: 7.91e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7642s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.1053 | max_forward: 1.1703 | max_loss: 0.1516 | max_backward: 1.8974 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6403.0 | uss_MB: 5502.7 | child_rss_MB: 5941.0 | allocated_MB: 447.2 | reserved_MB: 27520.0 | inactive_split_MB: 1195.6 +rank3 [2026-10-29 00:40:01,695] INFO : Running validation on rank3 +rank3 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 00:40:58,074] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:40:58,625] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 00:41:23,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 00:41:24,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 00:41:25,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 00:41:26,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 00:41:36,473] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 00:41:36,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:41:37,530] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 00:41:53,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 00:41:54,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 00:41:55,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 00:41:56,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 00:42:06,834] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 00:42:07,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 00:42:07,840] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 00:42:23,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 00:42:23,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 00:42:24,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 00:42:25,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 00:42:35,578] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 00:47:34,083] INFO : Step 16100/50000 rank3 | Train Loss: 1.2037 | Train Loss Unweighted: 0.6025 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1467 | Classification: 0.1080 | Classification_binary: 0.1878 | HitGrouping: 0.1747 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1512 | Regression_eta: 0.0018 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0777 | Classification_binary: 0.1772 | Regression_cos_phi: 0.1007 | Regression_energy: 0.0832 | Regression_eta: 0.0463 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0872 | LR: 7.89e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.0451s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7665 | max_loss: 0.1405 | max_backward: 1.4602 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7547.3 | uss_MB: 6643.5 | child_rss_MB: 5941.5 | allocated_MB: 446.1 | reserved_MB: 27680.0 | inactive_split_MB: 1038.2 +rank3 [2026-10-29 00:52:15,156] INFO : Step 16200/50000 rank3 | Train Loss: 1.2083 | Train Loss Unweighted: 0.6192 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1499 | Classification: 0.1107 | Classification_binary: 0.2017 | HitGrouping: 0.1711 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1506 | Regression_eta: 0.0018 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0939 | Regression_energy: 0.0974 | Regression_eta: 0.0471 | Regression_pt: 0.1050 | Regression_sin_phi: 0.1100 | LR: 7.86e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9528s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0473 | max_forward: 0.8343 | max_loss: 0.2530 | max_backward: 2.1136 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6458.0 | uss_MB: 5555.6 | child_rss_MB: 5961.3 | allocated_MB: 446.7 | reserved_MB: 27682.0 | inactive_split_MB: 1157.2 +rank3 [2026-10-29 00:56:51,718] INFO : Step 16300/50000 rank3 | Train Loss: 1.2745 | Train Loss Unweighted: 0.6291 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1604 | Classification: 0.1062 | Classification_binary: 0.2417 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1373 | Regression_eta: 0.0022 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.2436 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0632 | Regression_eta: 0.0455 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0752 | LR: 7.84e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8124s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.8059 | max_loss: 0.2506 | max_backward: 2.1763 | max_optimizer_step: 0.5213 | max_tensorboard_logging: 0.5923 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6459.9 | uss_MB: 5555.3 | child_rss_MB: 5936.4 | allocated_MB: 450.9 | reserved_MB: 27682.0 | inactive_split_MB: 950.8 +rank3 [2026-10-29 01:01:26,580] INFO : Step 16400/50000 rank3 | Train Loss: 1.2654 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2380 | Cardinality: 0.1554 | Classification: 0.1095 | Classification_binary: 0.2018 | HitGrouping: 0.1699 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1561 | Regression_eta: 0.0023 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1848 | Regression_cos_phi: 0.0862 | Regression_energy: 0.0804 | Regression_eta: 0.0617 | Regression_pt: 0.0873 | Regression_sin_phi: 0.1173 | LR: 7.81e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.2776s | Phase Times: max_pre_step_barrier: 0.0533 | max_batch_to_device: 0.0064 | max_forward: 0.7459 | max_loss: 0.1486 | max_backward: 1.5352 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.6677 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6452.2 | uss_MB: 5545.1 | child_rss_MB: 5962.3 | allocated_MB: 446.4 | reserved_MB: 27682.0 | inactive_split_MB: 949.1 +rank3 [2026-10-29 01:06:05,048] INFO : Step 16500/50000 rank3 | Train Loss: 1.1728 | Train Loss Unweighted: 0.5918 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1444 | Classification: 0.0927 | Classification_binary: 0.1686 | HitGrouping: 0.1723 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0021 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0816 | Regression_energy: 0.0890 | Regression_eta: 0.0793 | Regression_pt: 0.0981 | Regression_sin_phi: 0.1086 | LR: 7.79e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4903s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8043 | max_loss: 0.1772 | max_backward: 2.0250 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6458.1 | uss_MB: 5554.8 | child_rss_MB: 5927.7 | allocated_MB: 448.6 | reserved_MB: 29698.0 | inactive_split_MB: 1167.6 +rank3 [2026-10-29 01:10:53,703] INFO : Step 16600/50000 rank3 | Train Loss: 1.2920 | Train Loss Unweighted: 0.6427 | Train Loss Components: Auxiliary: 0.2455 | Cardinality: 0.1794 | Classification: 0.1148 | Classification_binary: 0.2098 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1561 | Regression_eta: 0.0021 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.2335 | Regression_cos_phi: 0.1545 | Regression_energy: 0.0825 | Regression_eta: 0.0989 | Regression_pt: 0.0893 | Regression_sin_phi: 0.1094 | LR: 7.77e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2547s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0377 | max_forward: 1.0830 | max_loss: 0.1480 | max_backward: 1.7919 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6457.2 | uss_MB: 5552.9 | child_rss_MB: 5951.6 | allocated_MB: 449.3 | reserved_MB: 29698.0 | inactive_split_MB: 1014.7 +rank3 [2026-10-29 01:15:45,847] INFO : Step 16700/50000 rank3 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6325 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1731 | Classification: 0.1072 | Classification_binary: 0.2252 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1469 | Regression_eta: 0.0019 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.2298 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0720 | Regression_eta: 0.0512 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0609 | LR: 7.74e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4559s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0722 | max_loss: 0.1506 | max_backward: 2.0365 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5067 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6456.8 | uss_MB: 5555.5 | child_rss_MB: 5918.8 | allocated_MB: 448.3 | reserved_MB: 29698.0 | inactive_split_MB: 893.3 +rank3 [2026-10-29 01:20:20,822] INFO : Step 16800/50000 rank3 | Train Loss: 1.1947 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1542 | Classification: 0.1036 | Classification_binary: 0.1800 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0023 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.1833 | Regression_cos_phi: 0.1238 | Regression_energy: 0.0967 | Regression_eta: 0.0846 | Regression_pt: 0.1033 | Regression_sin_phi: 0.0909 | LR: 7.72e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.2731s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0166 | max_forward: 0.8224 | max_loss: 0.1466 | max_backward: 1.6608 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6428.0 | uss_MB: 5552.9 | child_rss_MB: 5812.3 | allocated_MB: 446.4 | reserved_MB: 29698.0 | inactive_split_MB: 1161.6 +rank3 [2026-10-29 01:25:06,863] INFO : Step 16900/50000 rank3 | Train Loss: 1.2025 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1406 | Classification: 0.1120 | Classification_binary: 0.1884 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1446 | Regression_eta: 0.0023 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1868 | Regression_cos_phi: 0.1125 | Regression_energy: 0.0717 | Regression_eta: 0.0621 | Regression_pt: 0.0791 | Regression_sin_phi: 0.1010 | LR: 7.69e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3232s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7445 | max_loss: 0.1453 | max_backward: 1.8211 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6418.8 | uss_MB: 5553.4 | child_rss_MB: 5783.4 | allocated_MB: 448.9 | reserved_MB: 29698.0 | inactive_split_MB: 901.3 +rank3 [2026-10-29 01:29:52,082] INFO : Step 17000/50000 rank3 | Train Loss: 1.2049 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2349 | Cardinality: 0.1467 | Classification: 0.1018 | Classification_binary: 0.2228 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2369 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0843 | Regression_eta: 0.0510 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0768 | LR: 7.67e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.1424s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0141 | max_forward: 1.3980 | max_loss: 0.5129 | max_backward: 2.5664 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.5255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1218 | Memory: rss_MB: 6382.7 | uss_MB: 5556.2 | child_rss_MB: 5576.1 | allocated_MB: 448.2 | reserved_MB: 29698.0 | inactive_split_MB: 955.8 +rank3 [2026-10-29 01:29:53,074] INFO : Running validation on rank3 +rank3 [2026-10-29 01:29:53,074] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 01:29:53,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 01:30:46,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:30:46,580] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 01:31:13,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 01:31:14,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 01:31:16,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 01:31:17,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 01:31:27,243] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 01:31:28,036] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:31:28,587] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 01:31:44,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 01:31:45,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 01:31:46,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 01:31:47,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 01:31:57,619] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 01:31:58,252] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 01:31:58,744] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 01:32:14,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 01:32:15,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 01:32:16,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 01:32:16,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 01:32:26,998] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 01:37:36,199] INFO : Step 17100/50000 rank3 | Train Loss: 1.2386 | Train Loss Unweighted: 0.5919 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1931 | Classification: 0.0995 | Classification_binary: 0.1969 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0911 | Regression_energy: 0.0719 | Regression_eta: 0.0495 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0503 | LR: 7.64e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.3603s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0067 | max_forward: 1.0304 | max_loss: 0.1869 | max_backward: 2.3467 | max_optimizer_step: 0.3175 | max_tensorboard_logging: 0.7405 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 7631.0 | uss_MB: 6798.9 | child_rss_MB: 5579.4 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 1373.9 +rank3 [2026-10-29 01:42:27,134] INFO : Step 17200/50000 rank3 | Train Loss: 1.1810 | Train Loss Unweighted: 0.5844 | Train Loss Components: Auxiliary: 0.2270 | Cardinality: 0.1495 | Classification: 0.1085 | Classification_binary: 0.1683 | HitGrouping: 0.1731 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0022 | Regression_pt: 0.1515 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0782 | Classification_binary: 0.1938 | Regression_cos_phi: 0.0935 | Regression_energy: 0.0776 | Regression_eta: 0.0973 | Regression_pt: 0.0840 | Regression_sin_phi: 0.1308 | LR: 7.62e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.1051s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7638 | max_loss: 0.2424 | max_backward: 1.9393 | max_optimizer_step: 0.2921 | max_tensorboard_logging: 2.1267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6444.8 | uss_MB: 5633.5 | child_rss_MB: 5577.8 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 988.1 +rank3 [2026-10-29 01:47:11,311] INFO : Step 17300/50000 rank3 | Train Loss: 1.2271 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2397 | Cardinality: 0.1372 | Classification: 0.1102 | Classification_binary: 0.2127 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1512 | Regression_eta: 0.0022 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2131 | Regression_cos_phi: 0.1025 | Regression_energy: 0.0647 | Regression_eta: 0.0510 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0718 | LR: 7.59e-04 | DataLoad Time: 0.0204s | Model Forward Time: 6.1231s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0243 | max_forward: 1.4273 | max_loss: 0.6053 | max_backward: 3.4810 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6426.5 | uss_MB: 5175.0 | child_rss_MB: 5586.3 | allocated_MB: 449.6 | reserved_MB: 26752.0 | inactive_split_MB: 1120.1 +rank3 [2026-10-29 01:52:04,203] INFO : Step 17400/50000 rank3 | Train Loss: 1.2297 | Train Loss Unweighted: 0.6013 | Train Loss Components: Auxiliary: 0.2404 | Cardinality: 0.1709 | Classification: 0.1017 | Classification_binary: 0.2232 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1354 | Regression_eta: 0.0019 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.2309 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0782 | Regression_eta: 0.0502 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0809 | LR: 7.57e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.6396s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0185 | max_forward: 1.0000 | max_loss: 0.1504 | max_backward: 1.9673 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6331 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6409.7 | uss_MB: 5631.7 | child_rss_MB: 5583.3 | allocated_MB: 447.1 | reserved_MB: 28732.0 | inactive_split_MB: 1125.6 +rank3 [2026-10-29 01:56:48,791] INFO : Step 17500/50000 rank3 | Train Loss: 1.2152 | Train Loss Unweighted: 0.6044 | Train Loss Components: Auxiliary: 0.2290 | Cardinality: 0.1485 | Classification: 0.1023 | Classification_binary: 0.1789 | HitGrouping: 0.1757 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1590 | Regression_eta: 0.0026 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.1784 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0881 | Regression_eta: 0.0548 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0824 | LR: 7.54e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.2356s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0167 | max_forward: 0.7711 | max_loss: 0.1449 | max_backward: 1.8153 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.4947 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6388.7 | uss_MB: 5638.8 | child_rss_MB: 5515.3 | allocated_MB: 450.1 | reserved_MB: 28732.0 | inactive_split_MB: 1375.8 +rank3 [2026-10-29 02:01:39,791] INFO : Step 17600/50000 rank3 | Train Loss: 1.1560 | Train Loss Unweighted: 0.6042 | Train Loss Components: Auxiliary: 0.2286 | Cardinality: 0.1294 | Classification: 0.1057 | Classification_binary: 0.1993 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1471 | Regression_eta: 0.0018 | Regression_pt: 0.1484 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1991 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0868 | Regression_eta: 0.0606 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0866 | LR: 7.52e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.8153s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.8868 | max_loss: 0.1705 | max_backward: 2.0183 | max_optimizer_step: 0.2066 | max_tensorboard_logging: 0.7222 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1349 | Memory: rss_MB: 6384.6 | uss_MB: 5639.3 | child_rss_MB: 5506.2 | allocated_MB: 448.3 | reserved_MB: 28732.0 | inactive_split_MB: 1319.7 +rank3 [2026-10-29 02:06:28,891] INFO : Step 17700/50000 rank3 | Train Loss: 1.7848 | Train Loss Unweighted: 0.7433 | Train Loss Components: Auxiliary: 0.3415 | Cardinality: 0.4306 | Classification: 0.1200 | Classification_binary: 0.2341 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1920 | Regression_eta: 0.0037 | Regression_pt: 0.1910 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.2537 | Regression_cos_phi: 0.0996 | Regression_energy: 0.1011 | Regression_eta: 0.1079 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0896 | LR: 7.49e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.5793s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8788 | max_loss: 0.4738 | max_backward: 1.9859 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.5305 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6359.7 | uss_MB: 5640.4 | child_rss_MB: 5400.4 | allocated_MB: 450.1 | reserved_MB: 28738.0 | inactive_split_MB: 1125.6 +rank3 [2026-10-29 02:11:14,773] INFO : Step 17800/50000 rank3 | Train Loss: 1.1828 | Train Loss Unweighted: 0.6014 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1456 | Classification: 0.0982 | Classification_binary: 0.1996 | HitGrouping: 0.1749 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1503 | Regression_eta: 0.0019 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0809 | Classification_binary: 0.2320 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0798 | Regression_eta: 0.0806 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0746 | LR: 7.47e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1940s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.3167 | max_backward: 2.2751 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 6343.6 | uss_MB: 5261.8 | child_rss_MB: 5331.5 | allocated_MB: 444.9 | reserved_MB: 28738.0 | inactive_split_MB: 1155.9 +rank3 [2026-10-29 02:16:10,311] INFO : Step 17900/50000 rank3 | Train Loss: 1.0765 | Train Loss Unweighted: 0.5646 | Train Loss Components: Auxiliary: 0.2108 | Cardinality: 0.1097 | Classification: 0.0932 | Classification_binary: 0.1585 | HitGrouping: 0.1686 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1536 | Regression_eta: 0.0016 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.1744 | Regression_cos_phi: 0.1274 | Regression_energy: 0.0684 | Regression_eta: 0.0604 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0928 | LR: 7.44e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2960s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7593 | max_loss: 0.4918 | max_backward: 1.7987 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.6052 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6290.9 | uss_MB: 5640.2 | child_rss_MB: 5109.0 | allocated_MB: 447.1 | reserved_MB: 28738.0 | inactive_split_MB: 1154.4 +rank3 [2026-10-29 02:20:55,875] INFO : Step 18000/50000 rank3 | Train Loss: 1.2227 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2303 | Cardinality: 0.1450 | Classification: 0.1088 | Classification_binary: 0.2099 | HitGrouping: 0.1650 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1447 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.1802 | Regression_cos_phi: 0.1046 | Regression_energy: 0.0769 | Regression_eta: 0.0558 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0811 | LR: 7.42e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.7017s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9293 | max_loss: 0.1567 | max_backward: 2.2476 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6283.0 | uss_MB: 5642.9 | child_rss_MB: 5063.5 | allocated_MB: 449.4 | reserved_MB: 28738.0 | inactive_split_MB: 1324.2 +rank3 [2026-10-29 02:20:56,966] INFO : Running validation on rank3 +rank3 [2026-10-29 02:20:56,966] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 02:20:56,967] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 02:21:57,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:21:58,045] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 02:22:25,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 02:22:26,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 02:22:27,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 02:22:28,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 02:22:38,270] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 02:22:38,913] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:22:39,418] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 02:22:55,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 02:22:56,508] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 02:22:57,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 02:22:58,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 02:23:08,468] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 02:23:08,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 02:23:09,543] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 02:23:25,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 02:23:26,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 02:23:26,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 02:23:27,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 02:23:37,811] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 02:28:43,002] INFO : Step 18100/50000 rank3 | Train Loss: 1.2922 | Train Loss Unweighted: 0.6330 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1763 | Classification: 0.1061 | Classification_binary: 0.2257 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1467 | Regression_eta: 0.0022 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.2468 | Regression_cos_phi: 0.0972 | Regression_energy: 0.0795 | Regression_eta: 0.0610 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0959 | LR: 7.39e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.0668s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.9150 | max_loss: 0.1573 | max_backward: 2.1915 | max_optimizer_step: 0.2256 | max_tensorboard_logging: 0.8426 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7551.6 | uss_MB: 6907.4 | child_rss_MB: 5063.5 | allocated_MB: 448.8 | reserved_MB: 25396.0 | inactive_split_MB: 1192.4 +rank3 [2026-10-29 02:33:21,588] INFO : Step 18200/50000 rank3 | Train Loss: 1.1403 | Train Loss Unweighted: 0.5693 | Train Loss Components: Auxiliary: 0.2140 | Cardinality: 0.1362 | Classification: 0.1099 | Classification_binary: 0.1658 | HitGrouping: 0.1698 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1444 | Regression_eta: 0.0021 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0849 | Regression_eta: 0.0442 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0664 | LR: 7.36e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.5290s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0059 | max_forward: 0.7843 | max_loss: 0.1427 | max_backward: 1.4948 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 1.0509 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6418.0 | uss_MB: 5773.9 | child_rss_MB: 5038.4 | allocated_MB: 446.0 | reserved_MB: 27340.0 | inactive_split_MB: 922.1 +rank3 [2026-10-29 02:37:53,621] INFO : Step 18300/50000 rank3 | Train Loss: 1.1938 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2339 | Cardinality: 0.1526 | Classification: 0.1133 | Classification_binary: 0.1878 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1493 | Regression_eta: 0.0019 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.1584 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0862 | Regression_eta: 0.0573 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0977 | LR: 7.34e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9182s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.9058 | max_loss: 0.1527 | max_backward: 2.1689 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6422.1 | uss_MB: 5784.1 | child_rss_MB: 5032.3 | allocated_MB: 448.4 | reserved_MB: 27340.0 | inactive_split_MB: 1185.8 +rank3 [2026-10-29 02:42:24,375] INFO : Step 18400/50000 rank3 | Train Loss: 1.1959 | Train Loss Unweighted: 0.6109 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1130 | Classification: 0.1042 | Classification_binary: 0.2213 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1400 | Regression_eta: 0.0022 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2097 | Regression_cos_phi: 0.0911 | Regression_energy: 0.0728 | Regression_eta: 0.0720 | Regression_pt: 0.0787 | Regression_sin_phi: 0.1250 | LR: 7.31e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7064s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8434 | max_loss: 0.1474 | max_backward: 1.9441 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6423.0 | uss_MB: 5783.9 | child_rss_MB: 5008.1 | allocated_MB: 448.8 | reserved_MB: 29306.0 | inactive_split_MB: 1091.4 +rank3 [2026-10-29 02:47:17,191] INFO : Step 18500/50000 rank3 | Train Loss: 1.2719 | Train Loss Unweighted: 0.6036 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1990 | Classification: 0.1087 | Classification_binary: 0.1793 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1538 | Regression_eta: 0.0024 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0860 | Classification_binary: 0.2031 | Regression_cos_phi: 0.1171 | Regression_energy: 0.0973 | Regression_eta: 0.0857 | Regression_pt: 0.1058 | Regression_sin_phi: 0.1073 | LR: 7.29e-04 | DataLoad Time: 0.0166s | Model Forward Time: 6.4221s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 1.6245 | max_loss: 0.3527 | max_backward: 3.4112 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.7932 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 6423.2 | uss_MB: 5781.7 | child_rss_MB: 5035.9 | allocated_MB: 447.9 | reserved_MB: 29306.0 | inactive_split_MB: 1193.4 +rank3 [2026-10-29 02:52:01,281] INFO : Step 18600/50000 rank3 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5476 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1351 | Classification: 0.0870 | Classification_binary: 0.1697 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1433 | Regression_eta: 0.0017 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1628 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0688 | Regression_eta: 0.0477 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0696 | LR: 7.26e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2418s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0527 | max_forward: 0.7639 | max_loss: 0.1373 | max_backward: 1.5273 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.7424 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6420.8 | uss_MB: 5779.1 | child_rss_MB: 5038.0 | allocated_MB: 445.6 | reserved_MB: 29306.0 | inactive_split_MB: 1194.4 +rank3 [2026-10-29 02:56:35,338] INFO : Step 18700/50000 rank3 | Train Loss: 1.2376 | Train Loss Unweighted: 0.5906 | Train Loss Components: Auxiliary: 0.2405 | Cardinality: 0.1841 | Classification: 0.1039 | Classification_binary: 0.2012 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1400 | Regression_eta: 0.0020 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1816 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0650 | Regression_eta: 0.0473 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0736 | LR: 7.24e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7555s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8903 | max_loss: 0.1469 | max_backward: 2.1433 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4822 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1128 | Memory: rss_MB: 6419.7 | uss_MB: 5784.3 | child_rss_MB: 5013.1 | allocated_MB: 449.2 | reserved_MB: 31324.0 | inactive_split_MB: 1093.4 +rank3 [2026-10-29 03:01:09,007] INFO : Step 18800/50000 rank3 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1585 | Classification: 0.0943 | Classification_binary: 0.2038 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1342 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.2003 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0859 | Regression_eta: 0.0545 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0981 | LR: 7.21e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4216s | Phase Times: max_pre_step_barrier: 0.1142 | max_batch_to_device: 0.0068 | max_forward: 0.7808 | max_loss: 0.2502 | max_backward: 1.8229 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0996 | Memory: rss_MB: 6421.2 | uss_MB: 5781.7 | child_rss_MB: 5036.2 | allocated_MB: 450.4 | reserved_MB: 31336.0 | inactive_split_MB: 1098.4 +rank3 [2026-10-29 03:05:45,769] INFO : Step 18900/50000 rank3 | Train Loss: 1.0747 | Train Loss Unweighted: 0.5385 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.1306 | Classification: 0.0887 | Classification_binary: 0.1613 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1414 | Regression_eta: 0.0017 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0711 | Regression_eta: 0.0401 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0762 | LR: 7.18e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3938s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7400 | max_loss: 0.1656 | max_backward: 1.8003 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.7596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6421.5 | uss_MB: 5781.2 | child_rss_MB: 5039.7 | allocated_MB: 443.5 | reserved_MB: 31336.0 | inactive_split_MB: 928.5 +rank3 [2026-10-29 03:10:20,956] INFO : Step 19000/50000 rank3 | Train Loss: 1.1726 | Train Loss Unweighted: 0.5966 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1418 | Classification: 0.1102 | Classification_binary: 0.1870 | HitGrouping: 0.1705 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1463 | Regression_eta: 0.0019 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0748 | Regression_eta: 0.0567 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0578 | LR: 7.16e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8903s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7637 | max_loss: 0.5203 | max_backward: 1.8720 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 1.1055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6422.5 | uss_MB: 5783.8 | child_rss_MB: 5041.8 | allocated_MB: 446.6 | reserved_MB: 31336.0 | inactive_split_MB: 1097.4 +rank3 [2026-10-29 03:10:21,982] INFO : Running validation on rank3 +rank3 [2026-10-29 03:10:21,982] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 03:11:18,711] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:11:19,247] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 03:11:45,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 03:11:46,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 03:11:48,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 03:11:49,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 03:11:59,774] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 03:12:01,384] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:12:01,928] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 03:12:17,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 03:12:18,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 03:12:19,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 03:12:20,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 03:12:30,599] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 03:12:31,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 03:12:31,972] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 03:12:47,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 03:12:48,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 03:12:49,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 03:12:50,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 03:13:00,017] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 03:17:58,240] INFO : Step 19100/50000 rank3 | Train Loss: 1.2150 | Train Loss Unweighted: 0.6107 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1439 | Classification: 0.1037 | Classification_binary: 0.2230 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1395 | Regression_eta: 0.0020 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2073 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0894 | Regression_eta: 0.0391 | Regression_pt: 0.0989 | Regression_sin_phi: 0.0867 | LR: 7.13e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7831s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8630 | max_loss: 0.1532 | max_backward: 1.9622 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.7474 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7697.6 | uss_MB: 7053.9 | child_rss_MB: 5049.6 | allocated_MB: 449.4 | reserved_MB: 23972.0 | inactive_split_MB: 1371.8 +rank3 [2026-10-29 03:22:35,412] INFO : Step 19200/50000 rank3 | Train Loss: 1.1075 | Train Loss Unweighted: 0.5303 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1382 | Classification: 0.0958 | Classification_binary: 0.1758 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1270 | Regression_eta: 0.0019 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1981 | Regression_cos_phi: 0.1224 | Regression_energy: 0.0780 | Regression_eta: 0.0700 | Regression_pt: 0.0853 | Regression_sin_phi: 0.1077 | LR: 7.10e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.0051s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8287 | max_loss: 0.1556 | max_backward: 1.8127 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6504.0 | uss_MB: 5863.3 | child_rss_MB: 5048.7 | allocated_MB: 445.3 | reserved_MB: 27640.0 | inactive_split_MB: 940.7 +rank3 [2026-10-29 03:27:09,998] INFO : Step 19300/50000 rank3 | Train Loss: 1.0792 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2111 | Cardinality: 0.1254 | Classification: 0.0911 | Classification_binary: 0.1561 | HitGrouping: 0.1662 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0879 | Regression_eta: 0.0461 | Regression_pt: 0.0952 | Regression_sin_phi: 0.0538 | LR: 7.08e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5149s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8891 | max_loss: 0.1513 | max_backward: 1.7637 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6508.5 | uss_MB: 5869.2 | child_rss_MB: 5044.9 | allocated_MB: 445.1 | reserved_MB: 27640.0 | inactive_split_MB: 1366.9 +rank3 [2026-10-29 03:31:44,570] INFO : Step 19400/50000 rank3 | Train Loss: 1.2411 | Train Loss Unweighted: 0.6183 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1555 | Classification: 0.1055 | Classification_binary: 0.2145 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1463 | Regression_eta: 0.0020 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.2179 | Regression_cos_phi: 0.0799 | Regression_energy: 0.0646 | Regression_eta: 0.0556 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0798 | LR: 7.05e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.6742s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0092 | max_forward: 0.8079 | max_loss: 0.1550 | max_backward: 1.7710 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 2.8282 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6511.0 | uss_MB: 5872.2 | child_rss_MB: 5045.1 | allocated_MB: 452.5 | reserved_MB: 28010.0 | inactive_split_MB: 1195.9 +rank3 [2026-10-29 03:36:21,472] INFO : Step 19500/50000 rank3 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6008 | Train Loss Components: Auxiliary: 0.2448 | Cardinality: 0.1836 | Classification: 0.1087 | Classification_binary: 0.2123 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1374 | Regression_eta: 0.0023 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.2429 | Regression_cos_phi: 0.0997 | Regression_energy: 0.0873 | Regression_eta: 0.0610 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0794 | LR: 7.02e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6194s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7880 | max_loss: 0.1506 | max_backward: 1.7760 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.7620 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6510.7 | uss_MB: 5870.2 | child_rss_MB: 5041.2 | allocated_MB: 449.4 | reserved_MB: 29680.0 | inactive_split_MB: 1368.8 +rank3 [2026-10-29 03:40:54,624] INFO : Step 19600/50000 rank3 | Train Loss: 1.1451 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1464 | Classification: 0.0933 | Classification_binary: 0.1736 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1533 | Regression_eta: 0.0018 | Regression_pt: 0.1557 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0812 | Regression_eta: 0.0448 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0656 | LR: 7.00e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.3539s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8082 | max_loss: 0.1437 | max_backward: 1.6649 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.6567 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6510.9 | uss_MB: 5869.4 | child_rss_MB: 5045.0 | allocated_MB: 444.5 | reserved_MB: 29680.0 | inactive_split_MB: 1016.8 +rank3 [2026-10-29 03:45:30,311] INFO : Step 19700/50000 rank3 | Train Loss: 1.1424 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2236 | Cardinality: 0.1385 | Classification: 0.1093 | Classification_binary: 0.1797 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1379 | Regression_eta: 0.0020 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1862 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0722 | Regression_eta: 0.0519 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0642 | LR: 6.97e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4882s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7612 | max_loss: 0.2297 | max_backward: 1.9668 | max_optimizer_step: 0.5229 | max_tensorboard_logging: 0.5756 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6508.5 | uss_MB: 5869.4 | child_rss_MB: 5047.4 | allocated_MB: 449.1 | reserved_MB: 29680.0 | inactive_split_MB: 1100.9 +rank3 [2026-10-29 03:50:08,780] INFO : Step 19800/50000 rank3 | Train Loss: 1.2806 | Train Loss Unweighted: 0.6390 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1432 | Classification: 0.1110 | Classification_binary: 0.2282 | HitGrouping: 0.1668 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0030 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.2614 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0679 | Regression_eta: 0.0933 | Regression_pt: 0.0742 | Regression_sin_phi: 0.1105 | LR: 6.94e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5077s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8034 | max_loss: 0.1812 | max_backward: 1.9502 | max_optimizer_step: 0.2119 | max_tensorboard_logging: 0.5857 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6510.7 | uss_MB: 5873.6 | child_rss_MB: 5045.7 | allocated_MB: 452.3 | reserved_MB: 29680.0 | inactive_split_MB: 1110.9 +rank3 [2026-10-29 03:54:46,663] INFO : Step 19900/50000 rank3 | Train Loss: 1.0692 | Train Loss Unweighted: 0.5407 | Train Loss Components: Auxiliary: 0.2077 | Cardinality: 0.1171 | Classification: 0.0922 | Classification_binary: 0.1812 | HitGrouping: 0.1610 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1793 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0522 | Regression_eta: 0.0361 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0498 | LR: 6.92e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6125s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7533 | max_loss: 0.2412 | max_backward: 2.0720 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5848 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6505.8 | uss_MB: 5870.0 | child_rss_MB: 5057.5 | allocated_MB: 444.7 | reserved_MB: 29682.0 | inactive_split_MB: 1000.0 +rank3 [2026-10-29 03:59:25,168] INFO : Step 20000/50000 rank3 | Train Loss: 1.0304 | Train Loss Unweighted: 0.5275 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1060 | Classification: 0.0940 | Classification_binary: 0.1604 | HitGrouping: 0.1656 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1344 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1763 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0641 | Regression_eta: 0.0493 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0967 | LR: 6.89e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.8366s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.8721 | max_loss: 0.1507 | max_backward: 2.1861 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6505.7 | uss_MB: 5870.7 | child_rss_MB: 5049.5 | allocated_MB: 445.9 | reserved_MB: 30966.0 | inactive_split_MB: 1179.2 +rank3 [2026-10-29 03:59:26,239] INFO : Running validation on rank3 +rank3 [2026-10-29 03:59:26,239] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 03:59:26,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 04:00:19,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:00:19,531] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 04:00:44,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 04:00:45,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 04:00:46,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 04:00:47,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 04:00:57,963] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 04:00:59,044] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:00:59,500] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 04:01:15,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 04:01:16,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 04:01:17,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 04:01:18,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 04:01:28,478] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 04:01:28,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:01:29,373] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 04:01:44,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 04:01:45,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 04:01:46,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 04:01:47,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 04:01:57,211] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 04:06:53,362] INFO : Step 20100/50000 rank3 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2437 | Cardinality: 0.1586 | Classification: 0.1055 | Classification_binary: 0.2238 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1360 | Regression_eta: 0.0021 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.2352 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0747 | Regression_eta: 0.0766 | Regression_pt: 0.0809 | Regression_sin_phi: 0.1033 | LR: 6.86e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6017s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0372 | max_forward: 0.8293 | max_loss: 0.1584 | max_backward: 1.7738 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.7139 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7674.6 | uss_MB: 7034.7 | child_rss_MB: 5034.6 | allocated_MB: 448.4 | reserved_MB: 29328.0 | inactive_split_MB: 969.2 +rank3 [2026-10-29 04:11:25,836] INFO : Step 20200/50000 rank3 | Train Loss: 1.3163 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.1695 | Classification: 0.1000 | Classification_binary: 0.2250 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1433 | Regression_eta: 0.0029 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.2227 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0802 | Regression_eta: 0.0745 | Regression_pt: 0.0856 | Regression_sin_phi: 0.1050 | LR: 6.84e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6420s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8823 | max_loss: 0.1595 | max_backward: 2.1030 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5136 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7674.6 | uss_MB: 7036.8 | child_rss_MB: 5027.8 | allocated_MB: 449.6 | reserved_MB: 29340.0 | inactive_split_MB: 938.7 +rank3 [2026-10-29 04:16:03,708] INFO : Step 20300/50000 rank3 | Train Loss: 1.1420 | Train Loss Unweighted: 0.5734 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1487 | Classification: 0.1087 | Classification_binary: 0.1713 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1438 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0750 | Regression_eta: 0.0385 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0545 | LR: 6.81e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.9956s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.9650 | max_loss: 0.1904 | max_backward: 2.0801 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.8679 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6555.4 | uss_MB: 5913.2 | child_rss_MB: 5021.5 | allocated_MB: 444.8 | reserved_MB: 29344.0 | inactive_split_MB: 986.2 +rank3 [2026-10-29 04:20:44,976] INFO : Step 20400/50000 rank3 | Train Loss: 1.1926 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1642 | Classification: 0.1091 | Classification_binary: 0.1922 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1329 | Regression_eta: 0.0018 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.1953 | Regression_cos_phi: 0.0981 | Regression_energy: 0.0711 | Regression_eta: 0.0509 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0818 | LR: 6.78e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.2521s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0167 | max_forward: 1.2639 | max_loss: 0.1873 | max_backward: 2.4261 | max_optimizer_step: 0.3105 | max_tensorboard_logging: 0.7562 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6556.4 | uss_MB: 5916.7 | child_rss_MB: 5031.4 | allocated_MB: 447.7 | reserved_MB: 29344.0 | inactive_split_MB: 985.2 +rank3 [2026-10-29 04:25:27,116] INFO : Step 20500/50000 rank3 | Train Loss: 1.2110 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1591 | Classification: 0.1022 | Classification_binary: 0.2117 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1333 | Regression_eta: 0.0017 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.2034 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0618 | Regression_eta: 0.0347 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0751 | LR: 6.76e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5483s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7705 | max_loss: 0.1525 | max_backward: 1.8060 | max_optimizer_step: 0.2622 | max_tensorboard_logging: 0.7595 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6553.9 | uss_MB: 5915.2 | child_rss_MB: 5013.6 | allocated_MB: 450.6 | reserved_MB: 29344.0 | inactive_split_MB: 1123.4 +rank3 [2026-10-29 04:30:07,866] INFO : Step 20600/50000 rank3 | Train Loss: 1.1161 | Train Loss Unweighted: 0.5564 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1436 | Classification: 0.0978 | Classification_binary: 0.1795 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0804 | Regression_eta: 0.0477 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0756 | LR: 6.73e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4375s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8358 | max_loss: 0.5148 | max_backward: 1.8393 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6556.9 | uss_MB: 5916.8 | child_rss_MB: 5024.7 | allocated_MB: 445.4 | reserved_MB: 31380.0 | inactive_split_MB: 917.0 +rank3 [2026-10-29 04:34:51,728] INFO : Step 20700/50000 rank3 | Train Loss: 1.0740 | Train Loss Unweighted: 0.5459 | Train Loss Components: Auxiliary: 0.2086 | Cardinality: 0.1252 | Classification: 0.0962 | Classification_binary: 0.1560 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1712 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0714 | Regression_eta: 0.0443 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0690 | LR: 6.70e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0702s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7111 | max_loss: 0.2396 | max_backward: 1.6829 | max_optimizer_step: 0.2121 | max_tensorboard_logging: 0.4996 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6553.0 | uss_MB: 5912.9 | child_rss_MB: 5017.8 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1185.7 +rank3 [2026-10-29 04:39:29,327] INFO : Step 20800/50000 rank3 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5426 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1263 | Classification: 0.0965 | Classification_binary: 0.1999 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1207 | Regression_eta: 0.0017 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1713 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0639 | Regression_eta: 0.0319 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0611 | LR: 6.67e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7618s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.9079 | max_loss: 0.1550 | max_backward: 2.1083 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.5897 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6553.9 | uss_MB: 5918.9 | child_rss_MB: 5012.5 | allocated_MB: 448.2 | reserved_MB: 31380.0 | inactive_split_MB: 1157.8 +rank3 [2026-10-29 04:44:09,835] INFO : Step 20900/50000 rank3 | Train Loss: 1.2188 | Train Loss Unweighted: 0.5970 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1525 | Classification: 0.1062 | Classification_binary: 0.2022 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0022 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2003 | Regression_cos_phi: 0.1215 | Regression_energy: 0.0803 | Regression_eta: 0.0582 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0808 | LR: 6.65e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.7887s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0138 | max_forward: 0.8922 | max_loss: 0.1535 | max_backward: 2.0483 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6550.7 | uss_MB: 5916.4 | child_rss_MB: 5027.7 | allocated_MB: 447.8 | reserved_MB: 31380.0 | inactive_split_MB: 1126.3 +rank3 [2026-10-29 04:48:48,418] INFO : Step 21000/50000 rank3 | Train Loss: 1.0503 | Train Loss Unweighted: 0.5338 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1267 | Classification: 0.0900 | Classification_binary: 0.1573 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1405 | Regression_eta: 0.0016 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1685 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0787 | Regression_eta: 0.0338 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0756 | LR: 6.62e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8549s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7596 | max_loss: 0.2536 | max_backward: 1.8713 | max_optimizer_step: 0.3177 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6551.7 | uss_MB: 5915.9 | child_rss_MB: 5032.5 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1148.0 +rank3 [2026-10-29 04:48:49,285] INFO : Running validation on rank3 +rank3 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 04:49:44,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:49:44,825] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 04:50:11,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 04:50:12,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 04:50:13,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 04:50:14,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 04:50:24,466] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 04:50:25,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:50:25,768] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 04:50:42,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 04:50:43,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 04:50:44,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 04:50:45,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 04:50:55,247] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 04:50:55,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 04:50:55,853] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 04:51:11,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 04:51:12,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 04:51:13,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 04:51:13,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 04:51:23,911] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 04:56:25,805] INFO : Step 21100/50000 rank3 | Train Loss: 1.1646 | Train Loss Unweighted: 0.5785 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1401 | Classification: 0.1095 | Classification_binary: 0.1927 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1344 | Regression_eta: 0.0022 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.2063 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0610 | Regression_eta: 0.0599 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0946 | LR: 6.59e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.4052s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.9258 | max_loss: 0.1506 | max_backward: 2.4347 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.8145 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7847.0 | uss_MB: 5890.3 | child_rss_MB: 5018.3 | allocated_MB: 448.7 | reserved_MB: 23438.0 | inactive_split_MB: 911.7 +rank3 [2026-10-29 04:56:59,983] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank3 [2026-10-29 04:56:59,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-29 05:01:22,612] INFO : Step 21200/50000 rank3 | Train Loss: 1.1437 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1413 | Classification: 0.0985 | Classification_binary: 0.1768 | HitGrouping: 0.1669 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0020 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1663 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0707 | Regression_eta: 0.0537 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0706 | LR: 6.56e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5647s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7993 | max_loss: 0.1450 | max_backward: 1.7427 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7842.9 | uss_MB: 7208.8 | child_rss_MB: 5012.9 | allocated_MB: 446.9 | reserved_MB: 23590.0 | inactive_split_MB: 1086.9 +rank3 [2026-10-29 05:06:02,215] INFO : Step 21300/50000 rank3 | Train Loss: 1.1586 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1648 | Classification: 0.1118 | Classification_binary: 0.1760 | HitGrouping: 0.1681 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1392 | Regression_eta: 0.0021 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.1793 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0719 | Regression_eta: 0.0598 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0813 | LR: 6.54e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.2686s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0558 | max_forward: 0.9498 | max_loss: 0.1699 | max_backward: 2.4288 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.7788 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1270 | Memory: rss_MB: 6605.4 | uss_MB: 5978.0 | child_rss_MB: 5012.8 | allocated_MB: 447.4 | reserved_MB: 25440.0 | inactive_split_MB: 1003.9 +rank3 [2026-10-29 05:10:41,727] INFO : Step 21400/50000 rank3 | Train Loss: 1.1843 | Train Loss Unweighted: 0.5823 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1516 | Classification: 0.1038 | Classification_binary: 0.2065 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1335 | Regression_eta: 0.0020 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2095 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0815 | Regression_eta: 0.0502 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0838 | LR: 6.51e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.0967s | Phase Times: max_pre_step_barrier: 0.0085 | max_batch_to_device: 0.0640 | max_forward: 0.9920 | max_loss: 0.5036 | max_backward: 2.0711 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.8200 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6606.3 | uss_MB: 5978.8 | child_rss_MB: 5018.5 | allocated_MB: 448.9 | reserved_MB: 25440.0 | inactive_split_MB: 994.4 +rank3 [2026-10-29 05:15:15,087] INFO : Step 21500/50000 rank3 | Train Loss: 1.1484 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1478 | Classification: 0.0953 | Classification_binary: 0.2062 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1299 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.2213 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0687 | Regression_eta: 0.0439 | Regression_pt: 0.0756 | Regression_sin_phi: 0.0743 | LR: 6.48e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.9372s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8371 | max_loss: 0.1537 | max_backward: 2.1150 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.8633 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6603.8 | uss_MB: 5979.6 | child_rss_MB: 5017.0 | allocated_MB: 447.2 | reserved_MB: 27056.0 | inactive_split_MB: 1468.9 +rank3 [2026-10-29 05:19:51,738] INFO : Step 21600/50000 rank3 | Train Loss: 1.0171 | Train Loss Unweighted: 0.5098 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1167 | Classification: 0.0973 | Classification_binary: 0.1559 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1265 | Regression_eta: 0.0014 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0614 | Regression_eta: 0.0409 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0669 | LR: 6.45e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.5927s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0062 | max_forward: 0.7285 | max_loss: 0.1940 | max_backward: 1.8148 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.8819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6602.2 | uss_MB: 5978.2 | child_rss_MB: 5014.0 | allocated_MB: 445.7 | reserved_MB: 27056.0 | inactive_split_MB: 1196.5 +rank3 [2026-10-29 05:24:30,476] INFO : Step 21700/50000 rank3 | Train Loss: 1.1623 | Train Loss Unweighted: 0.5535 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1498 | Classification: 0.1025 | Classification_binary: 0.1819 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0903 | Regression_energy: 0.0627 | Regression_eta: 0.0566 | Regression_pt: 0.0698 | Regression_sin_phi: 0.1135 | LR: 6.43e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4944s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7947 | max_loss: 0.1379 | max_backward: 1.9808 | max_optimizer_step: 0.2157 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6600.2 | uss_MB: 5978.2 | child_rss_MB: 5010.8 | allocated_MB: 448.2 | reserved_MB: 27056.0 | inactive_split_MB: 1045.2 +rank3 [2026-10-29 05:29:08,471] INFO : Step 21800/50000 rank3 | Train Loss: 1.0898 | Train Loss Unweighted: 0.5518 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1233 | Classification: 0.0977 | Classification_binary: 0.1967 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1269 | Regression_eta: 0.0015 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.2103 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0787 | Regression_eta: 0.0452 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0658 | LR: 6.40e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.7239s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8952 | max_loss: 0.1765 | max_backward: 2.1628 | max_optimizer_step: 0.2138 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6602.0 | uss_MB: 5872.4 | child_rss_MB: 5012.0 | allocated_MB: 448.4 | reserved_MB: 27056.0 | inactive_split_MB: 1001.4 +rank3 [2026-10-29 05:33:46,455] INFO : Step 21900/50000 rank3 | Train Loss: 1.0699 | Train Loss Unweighted: 0.5306 | Train Loss Components: Auxiliary: 0.2121 | Cardinality: 0.1436 | Classification: 0.0991 | Classification_binary: 0.1704 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1276 | Regression_eta: 0.0015 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0614 | Regression_eta: 0.0329 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0560 | LR: 6.37e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4508s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7813 | max_loss: 0.1495 | max_backward: 1.8703 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 6600.0 | uss_MB: 5976.9 | child_rss_MB: 5013.0 | allocated_MB: 447.4 | reserved_MB: 27056.0 | inactive_split_MB: 1002.4 +rank3 [2026-10-29 05:38:23,934] INFO : Step 22000/50000 rank3 | Train Loss: 1.0659 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2100 | Cardinality: 0.1294 | Classification: 0.0955 | Classification_binary: 0.1521 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1360 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1510 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0675 | Regression_eta: 0.0420 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0772 | LR: 6.34e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4516s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7855 | max_loss: 0.1383 | max_backward: 1.9648 | max_optimizer_step: 0.2132 | max_tensorboard_logging: 0.5884 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6605.4 | uss_MB: 5979.9 | child_rss_MB: 5015.2 | allocated_MB: 449.1 | reserved_MB: 27056.0 | inactive_split_MB: 1180.9 +rank3 [2026-10-29 05:38:24,834] INFO : Running validation on rank3 +rank3 [2026-10-29 05:38:24,834] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 05:39:16,674] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:39:17,216] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 05:39:42,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 05:39:43,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 05:39:44,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 05:39:45,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 05:39:55,946] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 05:39:56,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:39:57,074] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 05:40:13,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 05:40:14,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 05:40:15,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 05:40:16,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 05:40:26,492] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 05:40:27,478] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 05:40:27,967] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 05:40:43,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 05:40:44,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 05:40:45,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 05:40:45,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 05:40:56,003] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 05:45:56,688] INFO : Step 22100/50000 rank3 | Train Loss: 1.1235 | Train Loss Unweighted: 0.5530 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1528 | Classification: 0.0997 | Classification_binary: 0.1905 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1304 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0846 | Classification_binary: 0.2130 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0728 | Regression_eta: 0.0574 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0801 | LR: 6.32e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5233s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8135 | max_loss: 0.1488 | max_backward: 1.7877 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5979 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7900.1 | uss_MB: 7271.0 | child_rss_MB: 5019.5 | allocated_MB: 451.8 | reserved_MB: 29136.0 | inactive_split_MB: 1136.2 +rank3 [2026-10-29 05:50:45,342] INFO : Step 22200/50000 rank3 | Train Loss: 1.1922 | Train Loss Unweighted: 0.5684 | Train Loss Components: Auxiliary: 0.2361 | Cardinality: 0.1699 | Classification: 0.0906 | Classification_binary: 0.2085 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.2148 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0645 | Regression_eta: 0.0534 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0736 | LR: 6.29e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5693s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8616 | max_loss: 0.1545 | max_backward: 1.6969 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 0.7120 | max_tensorboard_flush: 0.0150 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7899.1 | uss_MB: 7269.8 | child_rss_MB: 5025.9 | allocated_MB: 448.1 | reserved_MB: 29136.0 | inactive_split_MB: 1371.0 +rank3 [2026-10-29 05:55:21,940] INFO : Step 22300/50000 rank3 | Train Loss: 1.0221 | Train Loss Unweighted: 0.5006 | Train Loss Components: Auxiliary: 0.1973 | Cardinality: 0.1169 | Classification: 0.0876 | Classification_binary: 0.1576 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1252 | Regression_eta: 0.0021 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0609 | Regression_eta: 0.0529 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0479 | LR: 6.26e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5039s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7903 | max_loss: 0.1446 | max_backward: 1.7078 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 0.8083 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6690.1 | uss_MB: 6062.0 | child_rss_MB: 5023.7 | allocated_MB: 446.4 | reserved_MB: 29136.0 | inactive_split_MB: 1159.3 +rank3 [2026-10-29 06:00:08,368] INFO : Step 22400/50000 rank3 | Train Loss: 1.1170 | Train Loss Unweighted: 0.5609 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1391 | Classification: 0.1065 | Classification_binary: 0.1807 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1348 | Regression_eta: 0.0019 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1513 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0662 | Regression_eta: 0.0339 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0664 | LR: 6.23e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9542s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7658 | max_loss: 0.2479 | max_backward: 1.9808 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9928 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6691.0 | uss_MB: 6065.0 | child_rss_MB: 5027.0 | allocated_MB: 447.1 | reserved_MB: 31152.0 | inactive_split_MB: 925.7 +rank3 [2026-10-29 06:04:41,560] INFO : Step 22500/50000 rank3 | Train Loss: 1.1245 | Train Loss Unweighted: 0.5482 | Train Loss Components: Auxiliary: 0.2231 | Cardinality: 0.1463 | Classification: 0.0856 | Classification_binary: 0.2083 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1244 | Regression_eta: 0.0016 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.2377 | Regression_cos_phi: 0.0918 | Regression_energy: 0.0638 | Regression_eta: 0.0425 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0820 | LR: 6.20e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.0103s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 1.1916 | max_loss: 0.1613 | max_backward: 2.1225 | max_optimizer_step: 0.2176 | max_tensorboard_logging: 0.7664 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6685.2 | uss_MB: 6063.6 | child_rss_MB: 5003.5 | allocated_MB: 447.4 | reserved_MB: 31152.0 | inactive_split_MB: 959.7 +rank3 [2026-10-29 06:09:13,551] INFO : Step 22600/50000 rank3 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5177 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1260 | Classification: 0.0928 | Classification_binary: 0.1789 | HitGrouping: 0.1619 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1208 | Regression_eta: 0.0019 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0580 | Regression_eta: 0.0373 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0665 | LR: 6.18e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.8025s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0651 | max_forward: 1.1064 | max_loss: 0.1531 | max_backward: 1.9236 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.8515 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6676.3 | uss_MB: 6063.0 | child_rss_MB: 5024.2 | allocated_MB: 448.3 | reserved_MB: 31154.0 | inactive_split_MB: 961.5 +rank3 [2026-10-29 06:13:51,718] INFO : Step 22700/50000 rank3 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5090 | Train Loss Components: Auxiliary: 0.1959 | Cardinality: 0.1101 | Classification: 0.0877 | Classification_binary: 0.1479 | HitGrouping: 0.1634 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1339 | Regression_eta: 0.0015 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1471 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0572 | Regression_eta: 0.0322 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0487 | LR: 6.15e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.4578s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 1.0743 | max_loss: 0.1383 | max_backward: 1.7235 | max_optimizer_step: 0.2204 | max_tensorboard_logging: 0.5170 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 6672.9 | uss_MB: 6061.8 | child_rss_MB: 5030.8 | allocated_MB: 446.5 | reserved_MB: 31154.0 | inactive_split_MB: 972.7 +rank3 [2026-10-29 06:18:32,627] INFO : Step 22800/50000 rank3 | Train Loss: 1.0730 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2215 | Cardinality: 0.1332 | Classification: 0.1095 | Classification_binary: 0.1782 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1289 | Regression_eta: 0.0014 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1936 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0664 | Regression_eta: 0.0387 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0592 | LR: 6.12e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.7946s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.2500 | max_backward: 2.0107 | max_optimizer_step: 0.2761 | max_tensorboard_logging: 0.6168 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 6673.9 | uss_MB: 6062.6 | child_rss_MB: 5029.6 | allocated_MB: 447.2 | reserved_MB: 31154.0 | inactive_split_MB: 922.1 +rank3 [2026-10-29 06:23:13,203] INFO : Step 22900/50000 rank3 | Train Loss: 1.1830 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2389 | Cardinality: 0.1788 | Classification: 0.0948 | Classification_binary: 0.2023 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1393 | Regression_eta: 0.0018 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2176 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0703 | Regression_eta: 0.0503 | Regression_pt: 0.0767 | Regression_sin_phi: 0.0770 | LR: 6.09e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.9942s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0073 | max_forward: 0.7992 | max_loss: 0.1478 | max_backward: 1.9051 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.9858 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 6664.2 | uss_MB: 6055.9 | child_rss_MB: 5028.8 | allocated_MB: 448.8 | reserved_MB: 31154.0 | inactive_split_MB: 1109.2 +rank3 [2026-10-29 06:27:51,864] INFO : Step 23000/50000 rank3 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5014 | Train Loss Components: Auxiliary: 0.1951 | Cardinality: 0.1184 | Classification: 0.0781 | Classification_binary: 0.1455 | HitGrouping: 0.1653 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1597 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0772 | Regression_eta: 0.0538 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0736 | LR: 6.06e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.6097s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7528 | max_loss: 0.2292 | max_backward: 1.7818 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.8884 | max_tensorboard_flush: 0.0060 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6672.0 | uss_MB: 6063.2 | child_rss_MB: 5028.5 | allocated_MB: 443.9 | reserved_MB: 31154.0 | inactive_split_MB: 985.1 +rank3 [2026-10-29 06:27:52,750] INFO : Running validation on rank3 +rank3 [2026-10-29 06:27:52,750] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 06:27:52,750] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 06:28:45,240] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:28:45,726] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 06:29:12,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 06:29:13,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 06:29:14,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 06:29:15,234] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 06:29:25,244] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 06:29:26,162] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:29:26,632] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 06:29:42,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 06:29:43,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 06:29:44,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 06:29:45,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 06:29:55,212] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 06:29:56,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 06:29:56,643] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 06:30:12,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 06:30:13,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 06:30:13,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 06:30:14,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 06:30:24,865] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 06:35:29,455] INFO : Step 23100/50000 rank3 | Train Loss: 1.1240 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1320 | Classification: 0.1122 | Classification_binary: 0.1745 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1274 | Regression_eta: 0.0021 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0887 | Regression_energy: 0.0876 | Regression_eta: 0.0437 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0669 | LR: 6.04e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8780s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7914 | max_loss: 0.1430 | max_backward: 1.7255 | max_optimizer_step: 0.3295 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0695 | Memory: rss_MB: 7843.1 | uss_MB: 7233.5 | child_rss_MB: 5030.9 | allocated_MB: 449.4 | reserved_MB: 25680.0 | inactive_split_MB: 1046.0 +rank3 [2026-10-29 06:40:11,956] INFO : Step 23200/50000 rank3 | Train Loss: 1.1736 | Train Loss Unweighted: 0.5745 | Train Loss Components: Auxiliary: 0.2341 | Cardinality: 0.1586 | Classification: 0.1057 | Classification_binary: 0.2033 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1304 | Regression_eta: 0.0018 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0738 | Classification_binary: 0.1887 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0626 | Regression_eta: 0.0375 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0607 | LR: 6.01e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.1132s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9381 | max_loss: 0.1692 | max_backward: 2.2545 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6161 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 7846.1 | uss_MB: 5861.1 | child_rss_MB: 5025.2 | allocated_MB: 447.8 | reserved_MB: 25680.0 | inactive_split_MB: 1169.7 +rank3 [2026-10-29 06:44:52,827] INFO : Step 23300/50000 rank3 | Train Loss: 1.0647 | Train Loss Unweighted: 0.5262 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1268 | Classification: 0.0958 | Classification_binary: 0.1665 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1301 | Regression_eta: 0.0017 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0675 | Regression_eta: 0.0393 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0770 | LR: 5.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6089s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8613 | max_loss: 0.1501 | max_backward: 2.0035 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 0.6274 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6729.0 | uss_MB: 6118.5 | child_rss_MB: 5031.7 | allocated_MB: 447.3 | reserved_MB: 29702.0 | inactive_split_MB: 1252.4 +rank3 [2026-10-29 06:49:32,125] INFO : Step 23400/50000 rank3 | Train Loss: 0.9916 | Train Loss Unweighted: 0.5023 | Train Loss Components: Auxiliary: 0.1957 | Cardinality: 0.1164 | Classification: 0.1006 | Classification_binary: 0.1374 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1345 | Regression_cos_phi: 0.1169 | Regression_energy: 0.0753 | Regression_eta: 0.0523 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0596 | LR: 5.95e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3951s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7229 | max_loss: 0.2412 | max_backward: 1.6798 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.4992 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1172 | Memory: rss_MB: 6727.8 | uss_MB: 6117.0 | child_rss_MB: 5031.9 | allocated_MB: 445.7 | reserved_MB: 29702.0 | inactive_split_MB: 1158.3 +rank3 [2026-10-29 06:54:42,355] INFO : Step 23500/50000 rank3 | Train Loss: 1.0349 | Train Loss Unweighted: 0.5234 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1083 | Classification: 0.0851 | Classification_binary: 0.1870 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1235 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1712 | Regression_cos_phi: 0.1136 | Regression_energy: 0.0569 | Regression_eta: 0.0462 | Regression_pt: 0.0624 | Regression_sin_phi: 0.1156 | LR: 5.92e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7754s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7924 | max_loss: 0.2020 | max_backward: 1.9787 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.7987 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6728.4 | uss_MB: 6111.2 | child_rss_MB: 5026.6 | allocated_MB: 446.9 | reserved_MB: 31774.0 | inactive_split_MB: 1042.0 +rank3 [2026-10-29 06:59:38,279] INFO : Step 23600/50000 rank3 | Train Loss: 1.1326 | Train Loss Unweighted: 0.5418 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1458 | Classification: 0.0933 | Classification_binary: 0.1900 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1267 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0898 | Regression_energy: 0.0707 | Regression_eta: 0.0518 | Regression_pt: 0.0768 | Regression_sin_phi: 0.1022 | LR: 5.90e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.9840s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0063 | max_forward: 0.7946 | max_loss: 0.1855 | max_backward: 1.9114 | max_optimizer_step: 0.2664 | max_tensorboard_logging: 0.9547 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6730.1 | uss_MB: 6120.3 | child_rss_MB: 5028.3 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1333.8 +rank3 [2026-10-29 07:04:20,062] INFO : Step 23700/50000 rank3 | Train Loss: 1.0167 | Train Loss Unweighted: 0.4968 | Train Loss Components: Auxiliary: 0.1932 | Cardinality: 0.1158 | Classification: 0.0853 | Classification_binary: 0.1475 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1285 | Regression_eta: 0.0017 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1323 | Regression_cos_phi: 0.0972 | Regression_energy: 0.0650 | Regression_eta: 0.0348 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0470 | LR: 5.87e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.7173s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0221 | max_forward: 0.7952 | max_loss: 0.1461 | max_backward: 1.8308 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7718 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6725.6 | uss_MB: 6118.0 | child_rss_MB: 5028.2 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1160.4 +rank3 [2026-10-29 07:08:57,915] INFO : Step 23800/50000 rank3 | Train Loss: 1.0651 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1325 | Classification: 0.0886 | Classification_binary: 0.1616 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1318 | Regression_eta: 0.0015 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1576 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0556 | Regression_eta: 0.0387 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0916 | LR: 5.84e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.6719s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8562 | max_loss: 0.1426 | max_backward: 1.8700 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6730.2 | uss_MB: 6120.2 | child_rss_MB: 5026.7 | allocated_MB: 449.4 | reserved_MB: 31776.0 | inactive_split_MB: 1046.7 +rank3 [2026-10-29 07:13:41,573] INFO : Step 23900/50000 rank3 | Train Loss: 1.1515 | Train Loss Unweighted: 0.5718 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1646 | Classification: 0.0962 | Classification_binary: 0.2086 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1938 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0611 | Regression_eta: 0.0437 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0684 | LR: 5.81e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8374s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.8887 | max_loss: 0.2019 | max_backward: 2.1185 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6224 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6714.4 | uss_MB: 6108.3 | child_rss_MB: 5032.9 | allocated_MB: 447.1 | reserved_MB: 31776.0 | inactive_split_MB: 1168.8 +rank3 [2026-10-29 07:18:20,710] INFO : Step 24000/50000 rank3 | Train Loss: 1.0174 | Train Loss Unweighted: 0.5187 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1133 | Classification: 0.0873 | Classification_binary: 0.1674 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0014 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1841 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0722 | Regression_eta: 0.0385 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0705 | LR: 5.78e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.6304s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 1.3237 | max_loss: 0.2403 | max_backward: 1.8319 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5068 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6721.2 | uss_MB: 6114.9 | child_rss_MB: 5029.4 | allocated_MB: 446.4 | reserved_MB: 31776.0 | inactive_split_MB: 957.1 +rank3 [2026-10-29 07:18:21,624] INFO : Running validation on rank3 +rank3 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 07:19:13,461] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:19:13,969] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 07:19:41,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 07:19:42,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 07:19:43,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 07:19:44,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 07:19:54,479] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 07:19:55,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:19:55,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 07:20:12,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 07:20:13,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 07:20:13,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 07:20:14,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 07:20:24,957] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 07:20:25,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 07:20:25,604] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 07:20:41,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 07:20:42,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 07:20:43,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 07:20:43,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 07:20:53,930] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 07:26:06,615] INFO : Step 24100/50000 rank3 | Train Loss: 1.0669 | Train Loss Unweighted: 0.5331 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1324 | Classification: 0.0928 | Classification_binary: 0.1578 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1390 | Regression_eta: 0.0017 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0834 | Regression_eta: 0.0807 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0610 | LR: 5.75e-04 | DataLoad Time: 0.0182s | Model Forward Time: 7.5185s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.7836 | max_loss: 0.1442 | max_backward: 1.8747 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 3.6030 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0626 | Memory: rss_MB: 7857.5 | uss_MB: 7243.1 | child_rss_MB: 5051.0 | allocated_MB: 448.1 | reserved_MB: 24490.0 | inactive_split_MB: 1198.5 +rank3 [2026-10-29 07:30:59,926] INFO : Step 24200/50000 rank3 | Train Loss: 1.1691 | Train Loss Unweighted: 0.5645 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1546 | Classification: 0.1031 | Classification_binary: 0.1915 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1325 | Regression_eta: 0.0019 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2117 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0687 | Regression_eta: 0.0499 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0736 | LR: 5.73e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2336s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.9207 | max_loss: 0.1533 | max_backward: 2.2288 | max_optimizer_step: 0.2436 | max_tensorboard_logging: 0.8522 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7858.2 | uss_MB: 7247.2 | child_rss_MB: 5039.0 | allocated_MB: 450.2 | reserved_MB: 26490.0 | inactive_split_MB: 976.9 +rank3 [2026-10-29 07:35:42,203] INFO : Step 24300/50000 rank3 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5336 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1380 | Classification: 0.0927 | Classification_binary: 0.1863 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1247 | Regression_eta: 0.0020 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1822 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0825 | Regression_eta: 0.0645 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0837 | LR: 5.70e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.6716s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8648 | max_loss: 0.1481 | max_backward: 1.7152 | max_optimizer_step: 0.2781 | max_tensorboard_logging: 0.7831 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6810.0 | uss_MB: 6199.1 | child_rss_MB: 5043.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 983.3 +rank3 [2026-10-29 07:40:19,144] INFO : Step 24400/50000 rank3 | Train Loss: 0.9707 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0879 | Classification_binary: 0.1454 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0014 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0811 | Regression_eta: 0.0339 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0666 | LR: 5.67e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.8494s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7628 | max_loss: 0.1940 | max_backward: 2.4253 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.9481 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6811.6 | uss_MB: 5803.5 | child_rss_MB: 5042.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 1129.3 +rank3 [2026-10-29 07:45:00,050] INFO : Step 24500/50000 rank3 | Train Loss: 1.0937 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1319 | Classification: 0.0941 | Classification_binary: 0.1676 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1334 | Regression_eta: 0.0016 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.1825 | Regression_cos_phi: 0.0873 | Regression_energy: 0.0607 | Regression_eta: 0.0583 | Regression_pt: 0.0655 | Regression_sin_phi: 0.1184 | LR: 5.64e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4284s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7565 | max_loss: 0.5105 | max_backward: 1.8057 | max_optimizer_step: 0.2534 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6807.0 | uss_MB: 6198.6 | child_rss_MB: 5045.1 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 945.2 +rank3 [2026-10-29 07:49:44,236] INFO : Step 24600/50000 rank3 | Train Loss: 1.0924 | Train Loss Unweighted: 0.5511 | Train Loss Components: Auxiliary: 0.2237 | Cardinality: 0.1233 | Classification: 0.0963 | Classification_binary: 0.1991 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1255 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0607 | Regression_eta: 0.0318 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0441 | LR: 5.61e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6477s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.9060 | max_loss: 0.1523 | max_backward: 2.0894 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.5102 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6811.1 | uss_MB: 6200.6 | child_rss_MB: 5046.3 | allocated_MB: 449.5 | reserved_MB: 26492.0 | inactive_split_MB: 943.2 +rank3 [2026-10-29 07:54:29,279] INFO : Step 24700/50000 rank3 | Train Loss: 0.9972 | Train Loss Unweighted: 0.4984 | Train Loss Components: Auxiliary: 0.1950 | Cardinality: 0.1160 | Classification: 0.0782 | Classification_binary: 0.1692 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0708 | Regression_eta: 0.0342 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0551 | LR: 5.58e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.5151s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0060 | max_forward: 0.7847 | max_loss: 0.1519 | max_backward: 1.7159 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.8325 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6813.0 | uss_MB: 6199.7 | child_rss_MB: 5050.6 | allocated_MB: 447.9 | reserved_MB: 27708.0 | inactive_split_MB: 1169.6 +rank3 [2026-10-29 07:59:09,068] INFO : Step 24800/50000 rank3 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4722 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1202 | Classification: 0.0808 | Classification_binary: 0.1484 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0016 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0597 | Regression_eta: 0.0408 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0790 | LR: 5.56e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7183s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0127 | max_forward: 0.7903 | max_loss: 0.1427 | max_backward: 1.7798 | max_optimizer_step: 0.2260 | max_tensorboard_logging: 0.8943 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6813.0 | uss_MB: 6199.9 | child_rss_MB: 5051.5 | allocated_MB: 447.6 | reserved_MB: 28508.0 | inactive_split_MB: 989.1 +rank3 [2026-10-29 08:03:50,794] INFO : Step 24900/50000 rank3 | Train Loss: 1.0773 | Train Loss Unweighted: 0.5300 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1506 | Classification: 0.0964 | Classification_binary: 0.1776 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1257 | Regression_eta: 0.0016 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2187 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0857 | Regression_eta: 0.0477 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0788 | LR: 5.53e-04 | DataLoad Time: 0.0144s | Model Forward Time: 5.3181s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8100 | max_loss: 0.1483 | max_backward: 1.9423 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 1.0834 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6802.2 | uss_MB: 5593.7 | child_rss_MB: 5050.0 | allocated_MB: 449.1 | reserved_MB: 28508.0 | inactive_split_MB: 993.6 +rank3 [2026-10-29 08:08:37,678] INFO : Step 25000/50000 rank3 | Train Loss: 1.1315 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1671 | Classification: 0.0915 | Classification_binary: 0.1899 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1201 | Regression_eta: 0.0020 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1890 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0678 | Regression_eta: 0.0387 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0633 | LR: 5.50e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.7060s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8152 | max_loss: 0.2476 | max_backward: 1.7290 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.8750 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6807.0 | uss_MB: 6195.6 | child_rss_MB: 5052.2 | allocated_MB: 448.5 | reserved_MB: 28508.0 | inactive_split_MB: 953.9 +rank3 [2026-10-29 08:08:38,676] INFO : Running validation on rank3 +rank3 [2026-10-29 08:08:38,676] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 08:08:38,676] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 08:09:30,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:09:31,356] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 08:09:58,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 08:09:59,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 08:10:00,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 08:10:01,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 08:10:11,137] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 08:10:11,661] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:10:12,184] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 08:10:27,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 08:10:28,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 08:10:29,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 08:10:30,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 08:10:40,890] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 08:10:42,218] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 08:10:42,693] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 08:10:58,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 08:10:59,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 08:11:00,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 08:11:01,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 08:11:11,070] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 08:16:19,774] INFO : Step 25100/50000 rank3 | Train Loss: 1.0264 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1288 | Classification: 0.0850 | Classification_binary: 0.1497 | HitGrouping: 0.1664 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1378 | Regression_eta: 0.0016 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.1747 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0589 | Regression_eta: 0.0458 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0695 | LR: 5.47e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.5426s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0059 | max_forward: 0.7878 | max_loss: 0.1437 | max_backward: 2.2383 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9692 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7903.8 | uss_MB: 7286.7 | child_rss_MB: 5042.9 | allocated_MB: 446.8 | reserved_MB: 24062.0 | inactive_split_MB: 1159.3 +rank3 [2026-10-29 08:21:20,698] INFO : Step 25200/50000 rank3 | Train Loss: 1.0775 | Train Loss Unweighted: 0.5434 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1397 | Classification: 0.0996 | Classification_binary: 0.1706 | HitGrouping: 0.1584 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1340 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.1383 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0654 | Regression_eta: 0.0339 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0465 | LR: 5.44e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2306s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8831 | max_loss: 0.1456 | max_backward: 2.1320 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7903.3 | uss_MB: 7287.2 | child_rss_MB: 5042.2 | allocated_MB: 448.0 | reserved_MB: 26042.0 | inactive_split_MB: 1093.3 +rank3 [2026-10-29 08:26:17,211] INFO : Step 25300/50000 rank3 | Train Loss: 1.0761 | Train Loss Unweighted: 0.5109 | Train Loss Components: Auxiliary: 0.2132 | Cardinality: 0.1361 | Classification: 0.0961 | Classification_binary: 0.1821 | HitGrouping: 0.1562 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1138 | Regression_eta: 0.0018 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1692 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0495 | Regression_eta: 0.0542 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0788 | LR: 5.42e-04 | DataLoad Time: 0.0189s | Model Forward Time: 6.1313s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.9122 | max_loss: 0.1821 | max_backward: 2.1499 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.9506 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6897.2 | uss_MB: 5222.8 | child_rss_MB: 5036.8 | allocated_MB: 446.7 | reserved_MB: 28062.0 | inactive_split_MB: 1098.3 +rank3 [2026-10-29 08:31:10,694] INFO : Step 25400/50000 rank3 | Train Loss: 1.0222 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1238 | Classification: 0.0928 | Classification_binary: 0.1653 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1221 | Regression_eta: 0.0015 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0645 | Regression_eta: 0.0493 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0751 | LR: 5.39e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.7080s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1954 | max_backward: 1.8415 | max_optimizer_step: 0.3277 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6896.2 | uss_MB: 6278.8 | child_rss_MB: 5052.2 | allocated_MB: 449.3 | reserved_MB: 28062.0 | inactive_split_MB: 945.6 +rank3 [2026-10-29 08:36:13,579] INFO : Step 25500/50000 rank3 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4596 | Train Loss Components: Auxiliary: 0.1817 | Cardinality: 0.1001 | Classification: 0.0953 | Classification_binary: 0.1334 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0016 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1255 | Regression_cos_phi: 0.0352 | Regression_energy: 0.0742 | Regression_eta: 0.0391 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0598 | LR: 5.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7855s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7884 | max_loss: 0.1475 | max_backward: 2.0272 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.7862 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0483 | Memory: rss_MB: 6898.2 | uss_MB: 6283.2 | child_rss_MB: 5050.7 | allocated_MB: 450.7 | reserved_MB: 28062.0 | inactive_split_MB: 1166.9 +rank3 [2026-10-29 08:41:08,958] INFO : Step 25600/50000 rank3 | Train Loss: 1.0724 | Train Loss Unweighted: 0.5330 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1190 | Classification: 0.0985 | Classification_binary: 0.1843 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1231 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0760 | Regression_eta: 0.0438 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0766 | LR: 5.33e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.1604s | Phase Times: max_pre_step_barrier: 0.0194 | max_batch_to_device: 0.0083 | max_forward: 1.0622 | max_loss: 0.3311 | max_backward: 2.5999 | max_optimizer_step: 0.3019 | max_tensorboard_logging: 0.8123 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6895.2 | uss_MB: 5585.2 | child_rss_MB: 5023.7 | allocated_MB: 450.4 | reserved_MB: 28064.0 | inactive_split_MB: 1163.5 +rank3 [2026-10-29 08:46:09,678] INFO : Step 25700/50000 rank3 | Train Loss: 1.0595 | Train Loss Unweighted: 0.5120 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.1394 | Classification: 0.0920 | Classification_binary: 0.1853 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0553 | Regression_eta: 0.0487 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0691 | LR: 5.30e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.8085s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.8320 | max_loss: 0.2520 | max_backward: 2.6962 | max_optimizer_step: 0.2699 | max_tensorboard_logging: 0.7057 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6885.0 | uss_MB: 6272.6 | child_rss_MB: 5051.6 | allocated_MB: 448.6 | reserved_MB: 29942.0 | inactive_split_MB: 1165.2 +rank3 [2026-10-29 08:51:40,159] INFO : Step 25800/50000 rank3 | Train Loss: 0.9592 | Train Loss Unweighted: 0.4733 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1038 | Classification: 0.0787 | Classification_binary: 0.1531 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1182 | Regression_eta: 0.0016 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0387 | Regression_energy: 0.0520 | Regression_eta: 0.0469 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0653 | LR: 5.27e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.4843s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7549 | max_loss: 0.1480 | max_backward: 1.5296 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 0.8796 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1361 | Memory: rss_MB: 6889.1 | uss_MB: 6277.5 | child_rss_MB: 5023.6 | allocated_MB: 445.9 | reserved_MB: 30100.0 | inactive_split_MB: 961.8 +rank3 [2026-10-29 08:56:50,110] INFO : Step 25900/50000 rank3 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4613 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1142 | Classification: 0.0768 | Classification_binary: 0.1501 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1145 | Regression_eta: 0.0015 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0847 | Regression_energy: 0.0491 | Regression_eta: 0.0555 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0811 | LR: 5.25e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8310s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0124 | max_forward: 0.7901 | max_loss: 0.1430 | max_backward: 1.9362 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8933 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 6880.5 | uss_MB: 6281.6 | child_rss_MB: 5008.5 | allocated_MB: 447.7 | reserved_MB: 30100.0 | inactive_split_MB: 1167.6 +rank3 [2026-10-29 09:01:47,508] INFO : Step 26000/50000 rank3 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5169 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1347 | Classification: 0.0896 | Classification_binary: 0.1938 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0015 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2389 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0716 | Regression_eta: 0.0581 | Regression_pt: 0.0768 | Regression_sin_phi: 0.1021 | LR: 5.22e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.4836s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8125 | max_loss: 0.2612 | max_backward: 1.8771 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.8972 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6845.2 | uss_MB: 5700.0 | child_rss_MB: 4882.0 | allocated_MB: 450.3 | reserved_MB: 30108.0 | inactive_split_MB: 1161.3 +rank3 [2026-10-29 09:01:48,414] INFO : Running validation on rank3 +rank3 [2026-10-29 09:01:48,414] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 09:01:48,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 09:02:42,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:02:42,921] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 09:03:09,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 09:03:10,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 09:03:11,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 09:03:12,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 09:03:22,141] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 09:03:22,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:03:23,247] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 09:03:39,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 09:03:40,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 09:03:41,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 09:03:42,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 09:03:52,559] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 09:03:52,917] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:03:53,429] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 09:04:08,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 09:04:09,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 09:04:10,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 09:04:11,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 09:04:21,182] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 09:09:37,593] INFO : Step 26100/50000 rank3 | Train Loss: 0.9519 | Train Loss Unweighted: 0.4738 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1091 | Classification: 0.0871 | Classification_binary: 0.1547 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1147 | Regression_eta: 0.0013 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0667 | Regression_eta: 0.0329 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0616 | LR: 5.19e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2091s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0321 | max_forward: 0.8143 | max_loss: 0.1540 | max_backward: 1.7669 | max_optimizer_step: 0.2382 | max_tensorboard_logging: 1.1185 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 8060.0 | uss_MB: 7433.5 | child_rss_MB: 4850.5 | allocated_MB: 445.1 | reserved_MB: 25850.0 | inactive_split_MB: 1376.9 +rank3 [2026-10-29 09:14:36,073] INFO : Step 26200/50000 rank3 | Train Loss: 0.9758 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.0970 | Classification: 0.0858 | Classification_binary: 0.1501 | HitGrouping: 0.1665 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0735 | Regression_eta: 0.0457 | Regression_pt: 0.0824 | Regression_sin_phi: 0.0961 | LR: 5.16e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6744s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7728 | max_loss: 0.1384 | max_backward: 1.8195 | max_optimizer_step: 0.3036 | max_tensorboard_logging: 0.7010 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0602 | Memory: rss_MB: 8060.1 | uss_MB: 7488.7 | child_rss_MB: 4848.5 | allocated_MB: 447.4 | reserved_MB: 27830.0 | inactive_split_MB: 1117.7 +rank3 [2026-10-29 09:19:51,626] INFO : Step 26300/50000 rank3 | Train Loss: 1.0568 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2146 | Cardinality: 0.1416 | Classification: 0.0935 | Classification_binary: 0.1772 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0501 | Regression_eta: 0.0336 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0506 | LR: 5.13e-04 | DataLoad Time: 0.0177s | Model Forward Time: 6.1530s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.8205 | max_loss: 0.1808 | max_backward: 2.8872 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 1.1024 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6927.6 | uss_MB: 6354.4 | child_rss_MB: 4842.0 | allocated_MB: 450.8 | reserved_MB: 29830.0 | inactive_split_MB: 1114.5 +rank3 [2026-10-29 09:24:55,631] INFO : Step 26400/50000 rank3 | Train Loss: 1.0536 | Train Loss Unweighted: 0.5239 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1323 | Classification: 0.0937 | Classification_binary: 0.1845 | HitGrouping: 0.1559 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1209 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0856 | Classification_binary: 0.1986 | Regression_cos_phi: 0.0939 | Regression_energy: 0.0663 | Regression_eta: 0.0424 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0858 | LR: 5.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.5430s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8639 | max_loss: 0.1569 | max_backward: 2.4710 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 1.0049 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6924.7 | uss_MB: 6353.2 | child_rss_MB: 4853.6 | allocated_MB: 447.6 | reserved_MB: 29832.0 | inactive_split_MB: 1186.4 +rank3 [2026-10-29 09:29:59,514] INFO : Step 26500/50000 rank3 | Train Loss: 0.9812 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1299 | Classification: 0.0807 | Classification_binary: 0.1400 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1217 | Regression_eta: 0.0017 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0526 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0643 | Regression_eta: 0.0423 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0576 | LR: 5.08e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8056s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7436 | max_loss: 0.1579 | max_backward: 1.8417 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.9265 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6921.4 | uss_MB: 6354.1 | child_rss_MB: 4851.7 | allocated_MB: 444.4 | reserved_MB: 29832.0 | inactive_split_MB: 1229.5 +rank3 [2026-10-29 09:34:58,506] INFO : Step 26600/50000 rank3 | Train Loss: 1.0108 | Train Loss Unweighted: 0.5066 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.1182 | Classification: 0.0957 | Classification_binary: 0.1588 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1242 | Regression_eta: 0.0014 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0644 | Regression_eta: 0.0446 | Regression_pt: 0.0716 | Regression_sin_phi: 0.1049 | LR: 5.05e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6969s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7561 | max_loss: 0.1430 | max_backward: 1.8427 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6921.4 | uss_MB: 6356.0 | child_rss_MB: 4827.0 | allocated_MB: 447.9 | reserved_MB: 29832.0 | inactive_split_MB: 1037.2 +rank3 [2026-10-29 09:40:07,140] INFO : Step 26700/50000 rank3 | Train Loss: 1.0878 | Train Loss Unweighted: 0.5266 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1269 | Classification: 0.0956 | Classification_binary: 0.1961 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1144 | Regression_eta: 0.0018 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1781 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0633 | Regression_eta: 0.0312 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0603 | LR: 5.02e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1889s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.9788 | max_loss: 0.1540 | max_backward: 1.9407 | max_optimizer_step: 0.2295 | max_tensorboard_logging: 1.0076 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6911.8 | uss_MB: 6344.9 | child_rss_MB: 4859.9 | allocated_MB: 450.1 | reserved_MB: 29832.0 | inactive_split_MB: 1190.4 +rank3 [2026-10-29 09:48:41,511] INFO : Step 26800/50000 rank3 | Train Loss: 1.0077 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1102 | Classification: 0.0787 | Classification_binary: 0.1553 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1359 | Regression_eta: 0.0015 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0632 | Regression_eta: 0.0383 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0795 | LR: 4.99e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.8138s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.7774 | max_loss: 0.2523 | max_backward: 1.8741 | max_optimizer_step: 0.2873 | max_tensorboard_logging: 0.8206 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6909.2 | uss_MB: 6351.0 | child_rss_MB: 4816.3 | allocated_MB: 444.7 | reserved_MB: 29832.0 | inactive_split_MB: 1121.3 +rank3 [2026-10-29 09:53:40,042] INFO : Step 26900/50000 rank3 | Train Loss: 0.9524 | Train Loss Unweighted: 0.4793 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1099 | Classification: 0.0833 | Classification_binary: 0.1305 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1313 | Regression_eta: 0.0016 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0628 | Regression_eta: 0.0682 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0664 | LR: 4.96e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.4194s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7292 | max_loss: 0.1373 | max_backward: 1.7442 | max_optimizer_step: 0.3152 | max_tensorboard_logging: 0.6764 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 6905.2 | uss_MB: 5520.6 | child_rss_MB: 4788.1 | allocated_MB: 446.9 | reserved_MB: 29832.0 | inactive_split_MB: 1016.7 +rank3 [2026-10-29 09:58:37,162] INFO : Step 27000/50000 rank3 | Train Loss: 1.0285 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1243 | Classification: 0.0862 | Classification_binary: 0.1888 | HitGrouping: 0.1539 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0016 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0487 | Regression_energy: 0.0508 | Regression_eta: 0.0264 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0457 | LR: 4.94e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.1618s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.9134 | max_loss: 0.1497 | max_backward: 2.1803 | max_optimizer_step: 0.2254 | max_tensorboard_logging: 0.9490 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6904.1 | uss_MB: 6354.5 | child_rss_MB: 4772.3 | allocated_MB: 449.7 | reserved_MB: 29832.0 | inactive_split_MB: 1024.3 +rank3 [2026-10-29 09:58:38,108] INFO : Running validation on rank3 +rank3 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 09:59:32,785] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 09:59:33,284] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 10:00:00,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 10:00:01,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 10:00:02,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 10:00:03,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 10:00:13,264] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 10:00:14,009] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:00:14,539] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 10:00:31,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 10:00:33,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 10:00:34,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 10:00:35,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 10:00:45,798] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 10:00:46,267] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:00:46,863] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 10:01:02,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 10:01:03,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 10:01:04,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 10:01:05,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 10:01:15,299] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 10:06:37,478] INFO : Step 27100/50000 rank3 | Train Loss: 1.0614 | Train Loss Unweighted: 0.4989 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1449 | Classification: 0.0752 | Classification_binary: 0.1877 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.2056 | Regression_cos_phi: 0.0780 | Regression_energy: 0.0648 | Regression_eta: 0.0605 | Regression_pt: 0.0711 | Regression_sin_phi: 0.0961 | LR: 4.91e-04 | DataLoad Time: 0.0209s | Model Forward Time: 7.1178s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0966 | max_forward: 2.1724 | max_loss: 0.6705 | max_backward: 3.1723 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 1.5888 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 8047.0 | uss_MB: 7496.7 | child_rss_MB: 4778.7 | allocated_MB: 447.0 | reserved_MB: 27292.0 | inactive_split_MB: 983.4 +rank3 [2026-10-29 10:11:19,567] INFO : Step 27200/50000 rank3 | Train Loss: 0.9045 | Train Loss Unweighted: 0.4389 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1028 | Classification: 0.0770 | Classification_binary: 0.1287 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1150 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0594 | Regression_eta: 0.0370 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0720 | LR: 4.88e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7910s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7725 | max_loss: 0.5032 | max_backward: 1.6788 | max_optimizer_step: 0.2815 | max_tensorboard_logging: 0.9686 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8047.0 | uss_MB: 7493.7 | child_rss_MB: 4786.7 | allocated_MB: 445.7 | reserved_MB: 29236.0 | inactive_split_MB: 1232.8 +rank3 [2026-10-29 10:15:58,638] INFO : Step 27300/50000 rank3 | Train Loss: 1.0726 | Train Loss Unweighted: 0.5095 | Train Loss Components: Auxiliary: 0.2138 | Cardinality: 0.1337 | Classification: 0.1081 | Classification_binary: 0.1640 | HitGrouping: 0.1596 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1159 | Regression_eta: 0.0019 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0704 | Classification_binary: 0.1617 | Regression_cos_phi: 0.0827 | Regression_energy: 0.0783 | Regression_eta: 0.0538 | Regression_pt: 0.0853 | Regression_sin_phi: 0.1015 | LR: 4.85e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1435s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8896 | max_loss: 0.1460 | max_backward: 2.0153 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6974.0 | uss_MB: 6424.6 | child_rss_MB: 4784.6 | allocated_MB: 450.9 | reserved_MB: 29330.0 | inactive_split_MB: 1103.1 +rank3 [2026-10-29 10:20:31,815] INFO : Step 27400/50000 rank3 | Train Loss: 1.0758 | Train Loss Unweighted: 0.5282 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1344 | Classification: 0.0930 | Classification_binary: 0.1899 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1207 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1972 | Regression_cos_phi: 0.0928 | Regression_energy: 0.0540 | Regression_eta: 0.0418 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0637 | LR: 4.82e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.0219s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8446 | max_loss: 0.1488 | max_backward: 2.0519 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.8725 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6977.0 | uss_MB: 5627.2 | child_rss_MB: 4777.8 | allocated_MB: 448.4 | reserved_MB: 31310.0 | inactive_split_MB: 895.7 +rank3 [2026-10-29 10:25:07,332] INFO : Step 27500/50000 rank3 | Train Loss: 0.9697 | Train Loss Unweighted: 0.4574 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1111 | Classification: 0.0831 | Classification_binary: 0.1473 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1114 | Regression_eta: 0.0018 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1707 | Regression_cos_phi: 0.0985 | Regression_energy: 0.0577 | Regression_eta: 0.0600 | Regression_pt: 0.0626 | Regression_sin_phi: 0.1415 | LR: 4.80e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4620s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.7904 | max_loss: 0.1563 | max_backward: 1.6291 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.7457 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6966.1 | uss_MB: 4710.9 | child_rss_MB: 4769.8 | allocated_MB: 447.3 | reserved_MB: 31310.0 | inactive_split_MB: 986.8 +rank3 [2026-10-29 10:30:03,960] INFO : Step 27600/50000 rank3 | Train Loss: 0.9616 | Train Loss Unweighted: 0.4618 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1092 | Classification: 0.0865 | Classification_binary: 0.1284 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0019 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1429 | Regression_cos_phi: 0.1022 | Regression_energy: 0.0628 | Regression_eta: 0.0572 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0635 | LR: 4.77e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5703s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0380 | max_forward: 0.7674 | max_loss: 0.1463 | max_backward: 1.9164 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.5145 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6974.1 | uss_MB: 6422.9 | child_rss_MB: 4778.4 | allocated_MB: 446.7 | reserved_MB: 31310.0 | inactive_split_MB: 1182.0 +rank3 [2026-10-29 10:34:46,993] INFO : Step 27700/50000 rank3 | Train Loss: 1.0693 | Train Loss Unweighted: 0.5190 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1414 | Classification: 0.0939 | Classification_binary: 0.1783 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0015 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1519 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0625 | Regression_eta: 0.0345 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0628 | LR: 4.74e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.6751s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.8151 | max_loss: 0.1750 | max_backward: 2.0398 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.5213 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0659 | Memory: rss_MB: 6974.1 | uss_MB: 6424.8 | child_rss_MB: 4756.0 | allocated_MB: 448.4 | reserved_MB: 33312.0 | inactive_split_MB: 986.3 +rank3 [2026-10-29 10:39:29,303] INFO : Step 27800/50000 rank3 | Train Loss: 1.0875 | Train Loss Unweighted: 0.5294 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1226 | Classification: 0.0924 | Classification_binary: 0.1842 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1237 | Regression_eta: 0.0019 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1854 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0572 | Regression_eta: 0.0372 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0632 | LR: 4.71e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.8085s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0152 | max_forward: 0.8463 | max_loss: 0.1744 | max_backward: 1.7379 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.9397 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 6971.2 | uss_MB: 6421.2 | child_rss_MB: 4773.7 | allocated_MB: 452.2 | reserved_MB: 33312.0 | inactive_split_MB: 1320.1 +rank3 [2026-10-29 10:44:11,938] INFO : Step 27900/50000 rank3 | Train Loss: 0.9483 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0988 | Classification: 0.0859 | Classification_binary: 0.1439 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1244 | Regression_eta: 0.0014 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0649 | Regression_eta: 0.0549 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0860 | LR: 4.68e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.4742s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.7626 | max_loss: 0.1370 | max_backward: 1.8424 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6972.2 | uss_MB: 6423.3 | child_rss_MB: 4773.1 | allocated_MB: 447.1 | reserved_MB: 33312.0 | inactive_split_MB: 1112.6 +rank3 [2026-10-29 10:48:55,971] INFO : Step 28000/50000 rank3 | Train Loss: 0.9840 | Train Loss Unweighted: 0.4736 | Train Loss Components: Auxiliary: 0.1949 | Cardinality: 0.1114 | Classification: 0.0927 | Classification_binary: 0.1564 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1099 | Regression_eta: 0.0015 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1365 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0490 | Regression_eta: 0.0313 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0580 | LR: 4.66e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9426s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8192 | max_loss: 0.2552 | max_backward: 2.1299 | max_optimizer_step: 0.2920 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 6969.4 | uss_MB: 6424.2 | child_rss_MB: 4776.8 | allocated_MB: 448.2 | reserved_MB: 33312.0 | inactive_split_MB: 1178.0 +rank3 [2026-10-29 10:48:56,835] INFO : Running validation on rank3 +rank3 [2026-10-29 10:48:56,835] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 10:49:51,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:49:52,004] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 10:50:18,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 10:50:19,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 10:50:20,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 10:50:21,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 10:50:31,604] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 10:50:32,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:50:32,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 10:50:49,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 10:50:50,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 10:50:50,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 10:50:51,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 10:51:01,955] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 10:51:02,905] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 10:51:03,444] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 10:51:19,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 10:51:19,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 10:51:20,702] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 10:51:21,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 10:51:31,606] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 10:56:32,745] INFO : Step 28100/50000 rank3 | Train Loss: 1.0915 | Train Loss Unweighted: 0.5085 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1649 | Classification: 0.0833 | Classification_binary: 0.1834 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1187 | Regression_eta: 0.0017 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1973 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0601 | Regression_eta: 0.0398 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0731 | LR: 4.63e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8675s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8571 | max_loss: 0.1603 | max_backward: 1.9112 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 0.7670 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0838 | Memory: rss_MB: 8288.3 | uss_MB: 7733.7 | child_rss_MB: 4773.7 | allocated_MB: 448.9 | reserved_MB: 25946.0 | inactive_split_MB: 1313.1 +rank3 [2026-10-29 11:01:18,173] INFO : Step 28200/50000 rank3 | Train Loss: 0.9241 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1144 | Classification: 0.0770 | Classification_binary: 0.1441 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1163 | Regression_eta: 0.0014 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1564 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0740 | Regression_eta: 0.0366 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0382 | LR: 4.60e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8981s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0751 | max_forward: 0.8691 | max_loss: 0.5236 | max_backward: 1.8483 | max_optimizer_step: 0.2480 | max_tensorboard_logging: 0.8546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 8289.0 | uss_MB: 7733.3 | child_rss_MB: 4770.2 | allocated_MB: 445.0 | reserved_MB: 25948.0 | inactive_split_MB: 976.4 +rank3 [2026-10-29 11:07:40,703] INFO : Step 28300/50000 rank3 | Train Loss: 0.9513 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1028 | Classification: 0.0882 | Classification_binary: 0.1368 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1147 | Regression_eta: 0.0016 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0702 | Regression_energy: 0.0851 | Regression_eta: 0.0505 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0889 | LR: 4.57e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9473s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.7916 | max_loss: 0.1449 | max_backward: 2.0850 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.8583 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7055.4 | uss_MB: 6496.0 | child_rss_MB: 4774.9 | allocated_MB: 447.2 | reserved_MB: 25952.0 | inactive_split_MB: 1165.3 +rank3 [2026-10-29 11:12:27,615] INFO : Step 28400/50000 rank3 | Train Loss: 1.0604 | Train Loss Unweighted: 0.4982 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1382 | Classification: 0.0847 | Classification_binary: 0.1862 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1111 | Regression_eta: 0.0018 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.2117 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0620 | Regression_eta: 0.0455 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0807 | LR: 4.55e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.0316s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9139 | max_loss: 0.1474 | max_backward: 2.1323 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.6846 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0517 | Memory: rss_MB: 7055.5 | uss_MB: 6496.2 | child_rss_MB: 4770.0 | allocated_MB: 448.2 | reserved_MB: 26968.0 | inactive_split_MB: 835.9 +rank3 [2026-10-29 11:17:11,503] INFO : Step 28500/50000 rank3 | Train Loss: 1.0282 | Train Loss Unweighted: 0.5005 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1237 | Classification: 0.0911 | Classification_binary: 0.1756 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0013 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1731 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0553 | Regression_eta: 0.0363 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0725 | LR: 4.52e-04 | DataLoad Time: 0.0208s | Model Forward Time: 9.4591s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0886 | max_forward: 0.9758 | max_loss: 0.5123 | max_backward: 2.0811 | max_optimizer_step: 0.2797 | max_tensorboard_logging: 1.5644 | max_tensorboard_flush: 0.0047 | max_post_step_barrier: 0.0889 | Memory: rss_MB: 7042.2 | uss_MB: 4349.9 | child_rss_MB: 4774.7 | allocated_MB: 452.0 | reserved_MB: 27968.0 | inactive_split_MB: 923.2 +rank3 [2026-10-29 11:21:56,254] INFO : Step 28600/50000 rank3 | Train Loss: 0.9753 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1202 | Classification: 0.0804 | Classification_binary: 0.1446 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1132 | Regression_eta: 0.0017 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1159 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0576 | Regression_eta: 0.0325 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0348 | LR: 4.49e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.9451s | Phase Times: max_pre_step_barrier: 0.1127 | max_batch_to_device: 0.0146 | max_forward: 0.7731 | max_loss: 0.2404 | max_backward: 1.6352 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 1.0006 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7048.8 | uss_MB: 5469.7 | child_rss_MB: 4776.7 | allocated_MB: 446.8 | reserved_MB: 30056.0 | inactive_split_MB: 896.4 +rank3 [2026-10-29 11:26:34,427] INFO : Step 28700/50000 rank3 | Train Loss: 1.0558 | Train Loss Unweighted: 0.5173 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1082 | Classification: 0.1065 | Classification_binary: 0.1612 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1217 | Regression_eta: 0.0024 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1558 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0622 | Regression_eta: 0.1062 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0852 | LR: 4.46e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.9558s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0686 | max_forward: 0.8423 | max_loss: 0.1475 | max_backward: 2.0170 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.8830 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 7053.8 | uss_MB: 6496.4 | child_rss_MB: 4774.2 | allocated_MB: 447.9 | reserved_MB: 30056.0 | inactive_split_MB: 925.2 +rank3 [2026-10-29 11:31:18,243] INFO : Step 28800/50000 rank3 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5167 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1474 | Classification: 0.0969 | Classification_binary: 0.1939 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2148 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0584 | Regression_eta: 0.0347 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0517 | LR: 4.44e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.8775s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0075 | max_forward: 0.8427 | max_loss: 0.2661 | max_backward: 2.8464 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7050.1 | uss_MB: 4315.7 | child_rss_MB: 4755.7 | allocated_MB: 448.2 | reserved_MB: 30058.0 | inactive_split_MB: 976.8 +rank3 [2026-10-29 11:35:53,801] INFO : Step 28900/50000 rank3 | Train Loss: 0.9389 | Train Loss Unweighted: 0.4547 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.1078 | Classification: 0.0733 | Classification_binary: 0.1521 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0580 | Regression_eta: 0.0445 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0789 | LR: 4.41e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.5679s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0062 | max_forward: 0.7499 | max_loss: 0.2426 | max_backward: 1.8556 | max_optimizer_step: 0.3135 | max_tensorboard_logging: 0.5831 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7048.4 | uss_MB: 6494.0 | child_rss_MB: 4774.7 | allocated_MB: 444.3 | reserved_MB: 30058.0 | inactive_split_MB: 1133.2 +rank3 [2026-10-29 11:40:53,319] INFO : Step 29000/50000 rank3 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4535 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0939 | Classification: 0.0855 | Classification_binary: 0.1308 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0015 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1213 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0594 | Regression_eta: 0.0289 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0486 | LR: 4.38e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.8488s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0062 | max_forward: 0.9270 | max_loss: 0.2073 | max_backward: 2.1896 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6044 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7047.7 | uss_MB: 6496.6 | child_rss_MB: 4759.3 | allocated_MB: 450.2 | reserved_MB: 30060.0 | inactive_split_MB: 1402.9 +rank3 [2026-10-29 11:40:54,336] INFO : Running validation on rank3 +rank3 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 11:41:46,667] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:41:47,174] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 11:42:15,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 11:42:17,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 11:42:18,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 11:42:19,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 11:42:29,746] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 11:42:31,032] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:42:31,704] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 11:42:47,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 11:42:48,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 11:42:49,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 11:42:50,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 11:43:00,262] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 11:43:00,864] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 11:43:01,398] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 11:43:16,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 11:43:17,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 11:43:18,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 11:43:19,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 11:43:29,111] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 11:48:35,108] INFO : Step 29100/50000 rank3 | Train Loss: 1.0559 | Train Loss Unweighted: 0.5040 | Train Loss Components: Auxiliary: 0.2142 | Cardinality: 0.1307 | Classification: 0.0939 | Classification_binary: 0.1851 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1098 | Regression_eta: 0.0015 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0773 | Regression_energy: 0.0557 | Regression_eta: 0.0469 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0714 | LR: 4.35e-04 | DataLoad Time: 0.0158s | Model Forward Time: 7.6739s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.6779 | max_loss: 0.3815 | max_backward: 4.2661 | max_optimizer_step: 0.3470 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8253.0 | uss_MB: 6395.3 | child_rss_MB: 4770.6 | allocated_MB: 448.8 | reserved_MB: 24188.0 | inactive_split_MB: 1162.1 +rank3 [2026-10-29 11:53:34,049] INFO : Step 29200/50000 rank3 | Train Loss: 0.9909 | Train Loss Unweighted: 0.4801 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1204 | Classification: 0.0923 | Classification_binary: 0.1746 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1054 | Regression_eta: 0.0014 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1555 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0453 | Regression_eta: 0.0260 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0496 | LR: 4.33e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7766s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8354 | max_loss: 0.1558 | max_backward: 2.1201 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0677 | Memory: rss_MB: 8251.2 | uss_MB: 7692.4 | child_rss_MB: 4763.1 | allocated_MB: 447.9 | reserved_MB: 24190.0 | inactive_split_MB: 1166.5 +rank3 [2026-10-29 11:58:27,982] INFO : Step 29300/50000 rank3 | Train Loss: 0.9337 | Train Loss Unweighted: 0.4462 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1121 | Classification: 0.0773 | Classification_binary: 0.1359 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1146 | Regression_eta: 0.0015 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0574 | Regression_eta: 0.0346 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0575 | LR: 4.30e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.9133s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0277 | max_forward: 0.8212 | max_loss: 0.1457 | max_backward: 1.8414 | max_optimizer_step: 0.2550 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7135.1 | uss_MB: 5997.0 | child_rss_MB: 4761.5 | allocated_MB: 444.9 | reserved_MB: 24190.0 | inactive_split_MB: 1038.6 +rank3 [2026-10-29 12:03:07,471] INFO : Step 29400/50000 rank3 | Train Loss: 1.0186 | Train Loss Unweighted: 0.4675 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1372 | Classification: 0.0948 | Classification_binary: 0.1561 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0020 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0460 | Regression_eta: 0.0385 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0496 | LR: 4.27e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.8023s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8495 | max_loss: 0.1434 | max_backward: 1.9359 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7145.4 | uss_MB: 6588.5 | child_rss_MB: 4762.3 | allocated_MB: 447.9 | reserved_MB: 26208.0 | inactive_split_MB: 957.3 +rank3 [2026-10-29 12:08:16,204] INFO : Step 29500/50000 rank3 | Train Loss: 1.0909 | Train Loss Unweighted: 0.5362 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.1071 | Classification_binary: 0.1896 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1176 | Regression_eta: 0.0019 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1531 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0563 | Regression_eta: 0.0445 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0486 | LR: 4.24e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9670s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0876 | max_forward: 0.8925 | max_loss: 0.1527 | max_backward: 1.9621 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.8162 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7137.1 | uss_MB: 6580.0 | child_rss_MB: 4760.9 | allocated_MB: 448.9 | reserved_MB: 26214.0 | inactive_split_MB: 1166.5 +rank3 [2026-10-29 12:13:12,652] INFO : Step 29600/50000 rank3 | Train Loss: 0.9608 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.1935 | Cardinality: 0.1093 | Classification: 0.0850 | Classification_binary: 0.1542 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1355 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0725 | Regression_eta: 0.0574 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0416 | LR: 4.22e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.8675s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7848 | max_loss: 0.1495 | max_backward: 1.8042 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.0178 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7115.2 | uss_MB: 6585.9 | child_rss_MB: 4743.4 | allocated_MB: 445.5 | reserved_MB: 26214.0 | inactive_split_MB: 1171.9 +rank3 [2026-10-29 12:18:07,441] INFO : Step 29700/50000 rank3 | Train Loss: 0.9511 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1260 | Classification: 0.0829 | Classification_binary: 0.1357 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1183 | Regression_eta: 0.0014 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1391 | Regression_cos_phi: 0.1008 | Regression_energy: 0.0590 | Regression_eta: 0.0474 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0945 | LR: 4.19e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.2346s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7531 | max_loss: 0.2308 | max_backward: 2.2313 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.9708 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7117.3 | uss_MB: 6587.7 | child_rss_MB: 4741.9 | allocated_MB: 446.6 | reserved_MB: 26214.0 | inactive_split_MB: 1160.6 +rank3 [2026-10-29 12:23:08,943] INFO : Step 29800/50000 rank3 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4915 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1104 | Classification: 0.0864 | Classification_binary: 0.1759 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0592 | Regression_eta: 0.0381 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0723 | LR: 4.16e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.4232s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8210 | max_loss: 0.1528 | max_backward: 2.4753 | max_optimizer_step: 0.3576 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0038 | max_post_step_barrier: 0.1798 | Memory: rss_MB: 7115.5 | uss_MB: 5536.1 | child_rss_MB: 4741.9 | allocated_MB: 450.3 | reserved_MB: 26214.0 | inactive_split_MB: 938.8 +rank3 [2026-10-29 12:28:10,142] INFO : Step 29900/50000 rank3 | Train Loss: 0.9604 | Train Loss Unweighted: 0.4559 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1243 | Classification: 0.0825 | Classification_binary: 0.1616 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1040 | Regression_eta: 0.0015 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0533 | Regression_eta: 0.0420 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0661 | LR: 4.14e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9479s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1142 | max_forward: 0.9271 | max_loss: 0.1531 | max_backward: 1.8698 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.8620 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7091.0 | uss_MB: 6437.5 | child_rss_MB: 4745.0 | allocated_MB: 450.0 | reserved_MB: 26214.0 | inactive_split_MB: 1166.0 +rank3 [2026-10-29 12:32:58,888] INFO : Step 30000/50000 rank3 | Train Loss: 0.9450 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.1132 | Classification: 0.0847 | Classification_binary: 0.1394 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0877 | Regression_energy: 0.0568 | Regression_eta: 0.0389 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0570 | LR: 4.11e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1877s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 1.0327 | max_loss: 0.1419 | max_backward: 1.6503 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.4878 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7039.0 | uss_MB: 6586.5 | child_rss_MB: 4745.4 | allocated_MB: 443.8 | reserved_MB: 26214.0 | inactive_split_MB: 1152.2 +rank3 [2026-10-29 12:47:12,790] INFO : Running validation on rank3 +rank3 [2026-10-29 12:47:12,790] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 12:47:12,790] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 14:49:00,140] INFO : Initializing model with seed=12345; process seed=12348 +rank3 [2026-10-29 14:49:00,158] INFO : configured dtype=torch.bfloat16 for autocast +rank3 [2026-10-29 14:49:20,929] INFO : Instantiating model +rank3 [2026-10-29 14:49:20,930] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank3 [2026-10-29 14:49:20,930] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank3 [2026-10-29 14:49:20,948] INFO : Input encoding initialization took 0.02s +rank3 [2026-10-29 14:49:20,949] INFO : nn0 parameters: 171776 +rank3 [2026-10-29 14:49:20,949] INFO : Initializing shared backbone layers, num_convs=6 +rank3 [2026-10-29 14:49:20,949] INFO : Initializing backbone layer 0 +rank3 [2026-10-29 14:49:20,986] INFO : Initializing backbone layer 1 +rank3 [2026-10-29 14:49:21,001] INFO : Initializing backbone layer 2 +rank3 [2026-10-29 14:49:21,015] INFO : Initializing backbone layer 3 +rank3 [2026-10-29 14:49:21,029] INFO : Initializing backbone layer 4 +rank3 [2026-10-29 14:49:21,044] INFO : Initializing backbone layer 5 +rank3 [2026-10-29 14:49:21,058] INFO : Backbone initialization took 0.11s +rank3 [2026-10-29 14:49:21,058] INFO : backbone parameters: 7901376 +rank3 [2026-10-29 14:49:21,059] INFO : Initializing output DNNs +rank3 [2026-10-29 14:49:21,114] INFO : Output DNNs initialization took 0.06s +rank3 [2026-10-29 14:49:21,114] INFO : backbone_mode=BackboneMode.SHARED +rank3 [2026-10-29 14:49:21,114] INFO : jagged_attention enabled=False +rank3 [2026-10-29 14:49:21,114] INFO : flash_attn_varlen enabled=False +rank3 [2026-10-29 14:49:21,114] INFO : task_queries enabled=False +rank3 [2026-10-29 14:49:21,114] INFO : classification_norm parameters: 0 +rank3 [2026-10-29 14:49:21,114] INFO : regression_norm parameters: 0 +rank3 [2026-10-29 14:49:21,114] INFO : classification_query parameters: 0 +rank3 [2026-10-29 14:49:21,115] INFO : regression_query parameters: 0 +rank3 [2026-10-29 14:49:21,116] INFO : classification_readout parameters: 0 +rank3 [2026-10-29 14:49:21,116] INFO : regression_readout parameters: 0 +rank3 [2026-10-29 14:49:21,116] INFO : output_mode=set +rank3 [2026-10-29 14:49:21,116] INFO : set_decoder parameters: 4713812 +rank3 [2026-10-29 14:49:21,116] INFO : nn_binary_particle parameters: 0 +rank3 [2026-10-29 14:49:21,117] INFO : nn_pid parameters: 0 +rank3 [2026-10-29 14:49:21,117] INFO : nn_pt parameters: 0 +rank3 [2026-10-29 14:49:21,117] INFO : nn_eta parameters: 0 +rank3 [2026-10-29 14:49:21,117] INFO : nn_sin_phi parameters: 0 +rank3 [2026-10-29 14:49:21,117] INFO : nn_cos_phi parameters: 0 +rank3 [2026-10-29 14:49:21,117] INFO : nn_energy parameters: 0 +rank3 [2026-10-29 14:49:21,118] INFO : Total MLPF parameters: 12786964 +rank3 [2026-10-29 14:49:21,118] INFO : MLPF __init__ done +rank3 [2026-10-29 14:49:21,118] INFO : Instantiated model +rank3 [2026-10-29 14:49:21,118] INFO : Moving model to device rank=3 +rank3 [2026-10-29 14:49:21,146] INFO : Moved model to device rank=3 +rank3 [2026-10-29 14:49:21,149] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank3 [2026-10-29 14:49:21,631] INFO : Restoring training from step 30001 +rank3 [2026-10-29 14:49:21,644] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth +rank3 [2026-10-29 14:49:21,645] INFO : Configured model for SyncBatchNorm rank=3 +rank3 [2026-10-29 14:49:22,135] INFO : Configured model for DistributedDataParallel rank=3 +rank3 [2026-10-29 14:49:22,287] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank3 [2026-10-29 14:49:22,287] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank3 [2026-10-29 14:49:22,288] INFO : Trainable parameters: 12786964 +rank3 [2026-10-29 14:49:22,288] INFO : Non-trainable parameters: 23268 +rank3 [2026-10-29 14:49:22,288] INFO : Total parameters: 12810232 +rank3 [2026-10-29 14:49:22,292] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank3 [2026-10-29 14:49:22,292] INFO : Getting dataloaders +rank3 [2026-10-29 14:49:22,292] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:49:28,733] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:49:29,765] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:49:30,957] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank3 [2026-10-29 14:49:30,957] INFO : Creating WeightedMixIterator with 1 data loaders. +rank3 [2026-10-29 14:49:30,957] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:49:31,225] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:49:31,477] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 14:49:31,733] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank3 [2026-10-29 14:49:31,734] INFO : Creating InterleavedIterator with 1 data loaders. +rank3 [2026-10-29 14:49:45,494] INFO : Got dataloaders +rank3 [2026-10-29 14:49:45,494] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [8888] of epochs [1] +rank3 [2026-10-29 14:49:45,494] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank3 [2026-10-29 14:49:45,494] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank3 [2026-10-29 14:49:45,495] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank3 [2026-10-29 14:49:47,207] INFO : loader split=train rank=3 len=21112 +rank3 [2026-10-29 14:49:47,207] INFO : loader split=valid rank=3 len=12 +rank3 [2026-10-29 14:49:47,207] INFO : Starting training from step 30001 to 50000 on rank 3 of 8 +rank3 [2026-10-29 14:49:47,208] INFO : Creating train_iterator +rank3 [2026-10-29 14:49:47,208] INFO : ResumableSampler physical:train loading state: start_index=142208 +rank3 [2026-10-29 14:50:01,299] INFO : Created train_iterator +rank3 [2026-10-29 14:54:42,004] INFO : Step 30100/50000 rank3 | Train Loss: 0.9854 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.2024 | Cardinality: 0.1295 | Classification: 0.0965 | Classification_binary: 0.1561 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1723 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0543 | Regression_eta: 0.0370 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0572 | LR: 4.08e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.6365s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7011 | max_loss: 0.1473 | max_backward: 1.5273 | max_optimizer_step: 0.1574 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0061 | Memory: rss_MB: 4775.9 | uss_MB: 3472.5 | child_rss_MB: 6376.2 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 850.1 +rank3 [2026-10-29 14:59:12,793] INFO : Step 30200/50000 rank3 | Train Loss: 1.0165 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2044 | Cardinality: 0.1520 | Classification: 0.0856 | Classification_binary: 0.1718 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1600 | Regression_cos_phi: 0.0929 | Regression_energy: 0.0562 | Regression_eta: 0.0377 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0592 | LR: 4.06e-04 | DataLoad Time: 0.0166s | Model Forward Time: 3.9879s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.7915 | max_loss: 0.2568 | max_backward: 1.8926 | max_optimizer_step: 0.1551 | max_tensorboard_logging: 0.4604 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4776.3 | uss_MB: 3476.7 | child_rss_MB: 6367.6 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 842.0 +rank3 [2026-10-29 15:03:42,965] INFO : Step 30300/50000 rank3 | Train Loss: 0.9183 | Train Loss Unweighted: 0.4449 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1232 | Classification: 0.0820 | Classification_binary: 0.1459 | HitGrouping: 0.1516 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0012 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1366 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0600 | Regression_eta: 0.0263 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0546 | LR: 4.03e-04 | DataLoad Time: 0.0169s | Model Forward Time: 3.8572s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7525 | max_loss: 0.2476 | max_backward: 1.8262 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.4574 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0039 | Memory: rss_MB: 4773.5 | uss_MB: 3475.1 | child_rss_MB: 6385.6 | allocated_MB: 495.8 | reserved_MB: 28760.0 | inactive_split_MB: 848.2 +rank3 [2026-10-29 15:08:13,476] INFO : Step 30400/50000 rank3 | Train Loss: 0.9035 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1137 | Classification: 0.0848 | Classification_binary: 0.1358 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1287 | Regression_cos_phi: 0.0769 | Regression_energy: 0.0505 | Regression_eta: 0.0337 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0791 | LR: 4.00e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9729s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7769 | max_loss: 0.1398 | max_backward: 1.9002 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.4925 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4770.8 | uss_MB: 3475.5 | child_rss_MB: 6366.2 | allocated_MB: 498.8 | reserved_MB: 28760.0 | inactive_split_MB: 878.9 +rank3 [2026-10-29 15:12:45,608] INFO : Step 30500/50000 rank3 | Train Loss: 1.0399 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1239 | Classification: 0.0901 | Classification_binary: 0.1841 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0018 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1795 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0580 | Regression_eta: 0.0656 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0869 | LR: 3.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3867s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9688 | max_loss: 0.1627 | max_backward: 2.3054 | max_optimizer_step: 0.1555 | max_tensorboard_logging: 0.4908 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4768.4 | uss_MB: 3475.6 | child_rss_MB: 6405.5 | allocated_MB: 501.7 | reserved_MB: 28760.0 | inactive_split_MB: 944.3 +rank3 [2026-10-29 15:17:13,866] INFO : Step 30600/50000 rank3 | Train Loss: 0.9987 | Train Loss Unweighted: 0.4959 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1112 | Classification: 0.0932 | Classification_binary: 0.1720 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1137 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2013 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0656 | Regression_eta: 0.0358 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0780 | LR: 3.95e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2239s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7736 | max_loss: 0.1566 | max_backward: 1.7922 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.6994 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4770.8 | uss_MB: 3476.1 | child_rss_MB: 6399.5 | allocated_MB: 500.1 | reserved_MB: 28760.0 | inactive_split_MB: 845.9 +rank3 [2026-10-29 15:21:45,752] INFO : Step 30700/50000 rank3 | Train Loss: 0.9490 | Train Loss Unweighted: 0.4575 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1237 | Classification: 0.0844 | Classification_binary: 0.1399 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0014 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0788 | Classification_binary: 0.1778 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0544 | Regression_eta: 0.0434 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0609 | LR: 3.92e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1855s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7215 | max_loss: 0.5560 | max_backward: 2.0906 | max_optimizer_step: 0.1553 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4768.7 | uss_MB: 3474.9 | child_rss_MB: 6379.2 | allocated_MB: 493.8 | reserved_MB: 28760.0 | inactive_split_MB: 827.5 +rank3 [2026-10-29 15:26:16,185] INFO : Step 30800/50000 rank3 | Train Loss: 1.0320 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1331 | Classification: 0.0916 | Classification_binary: 0.1632 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1173 | Regression_eta: 0.0015 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0327 | Classification_binary: 0.1298 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0656 | Regression_eta: 0.0274 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0450 | LR: 3.90e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6902s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1016 | max_forward: 0.8041 | max_loss: 0.1462 | max_backward: 1.9351 | max_optimizer_step: 0.1552 | max_tensorboard_logging: 1.1007 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4769.5 | uss_MB: 3475.6 | child_rss_MB: 6407.8 | allocated_MB: 497.9 | reserved_MB: 28760.0 | inactive_split_MB: 842.0 +rank3 [2026-10-29 15:30:49,188] INFO : Step 30900/50000 rank3 | Train Loss: 1.0226 | Train Loss Unweighted: 0.4952 | Train Loss Components: Auxiliary: 0.2139 | Cardinality: 0.1339 | Classification: 0.0875 | Classification_binary: 0.1859 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1092 | Regression_eta: 0.0013 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1693 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0516 | Regression_eta: 0.0278 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0564 | LR: 3.87e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4011s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7610 | max_loss: 0.2067 | max_backward: 1.7028 | max_optimizer_step: 0.1599 | max_tensorboard_logging: 1.0327 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4771.1 | uss_MB: 3476.3 | child_rss_MB: 6385.8 | allocated_MB: 497.2 | reserved_MB: 28760.0 | inactive_split_MB: 838.0 +rank3 [2026-10-29 15:35:26,545] INFO : Step 31000/50000 rank3 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4630 | Train Loss Components: Auxiliary: 0.2006 | Cardinality: 0.1217 | Classification: 0.0821 | Classification_binary: 0.1592 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1085 | Regression_eta: 0.0018 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0557 | Regression_eta: 0.0360 | Regression_pt: 0.0614 | Regression_sin_phi: 0.1101 | LR: 3.84e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4968s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7780 | max_loss: 0.2475 | max_backward: 1.8803 | max_optimizer_step: 0.1588 | max_tensorboard_logging: 0.9947 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4772.1 | uss_MB: 3477.0 | child_rss_MB: 6402.3 | allocated_MB: 498.2 | reserved_MB: 28760.0 | inactive_split_MB: 820.4 +rank3 [2026-10-29 15:35:27,414] INFO : Running validation on rank3 +rank3 [2026-10-29 15:35:27,414] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 15:35:27,414] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 15:36:19,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:36:20,267] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 15:36:45,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 15:36:46,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 15:36:48,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 15:36:49,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 15:36:59,074] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-29 15:36:59,566] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:37:00,107] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 15:37:15,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 15:37:16,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 15:37:17,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 15:37:18,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 15:37:28,429] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 15:37:29,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 15:37:30,051] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 15:37:45,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 15:37:46,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 15:37:47,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 15:37:48,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 15:37:58,047] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 15:42:50,284] INFO : Step 31100/50000 rank3 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4264 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1054 | Classification: 0.0853 | Classification_binary: 0.1288 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.0941 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0502 | Regression_eta: 0.0226 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0333 | LR: 3.82e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5235s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7353 | max_loss: 0.1410 | max_backward: 1.7959 | max_optimizer_step: 0.2754 | max_tensorboard_logging: 0.9599 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 6463.6 | uss_MB: 5188.4 | child_rss_MB: 6394.5 | allocated_MB: 495.8 | reserved_MB: 27508.0 | inactive_split_MB: 851.2 +rank3 [2026-10-29 15:47:34,510] INFO : Step 31200/50000 rank3 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4709 | Train Loss Components: Auxiliary: 0.2045 | Cardinality: 0.1286 | Classification: 0.0862 | Classification_binary: 0.1646 | HitGrouping: 0.1524 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1077 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0540 | Regression_eta: 0.0408 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0744 | LR: 3.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3216s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8441 | max_loss: 0.1469 | max_backward: 2.0813 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 6462.4 | uss_MB: 5189.4 | child_rss_MB: 6422.3 | allocated_MB: 496.9 | reserved_MB: 27508.0 | inactive_split_MB: 935.6 +rank3 [2026-10-29 15:52:19,964] INFO : Step 31300/50000 rank3 | Train Loss: 1.0631 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1394 | Classification: 0.0917 | Classification_binary: 0.1731 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1638 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0634 | Regression_eta: 0.0414 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0602 | LR: 3.76e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2495s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7950 | max_loss: 0.1940 | max_backward: 1.8810 | max_optimizer_step: 0.2519 | max_tensorboard_logging: 0.5719 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5602.7 | uss_MB: 4330.1 | child_rss_MB: 6421.1 | allocated_MB: 498.6 | reserved_MB: 29580.0 | inactive_split_MB: 773.5 +rank3 [2026-10-29 15:56:52,522] INFO : Step 31400/50000 rank3 | Train Loss: 0.9019 | Train Loss Unweighted: 0.4258 | Train Loss Components: Auxiliary: 0.1823 | Cardinality: 0.1101 | Classification: 0.0746 | Classification_binary: 0.1337 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1070 | Regression_eta: 0.0014 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0550 | Regression_eta: 0.0315 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0687 | LR: 3.74e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2763s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8478 | max_loss: 0.2460 | max_backward: 2.0403 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.4770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5595.4 | uss_MB: 4331.0 | child_rss_MB: 6422.8 | allocated_MB: 495.2 | reserved_MB: 29580.0 | inactive_split_MB: 986.6 +rank3 [2026-10-29 16:01:34,248] INFO : Step 31500/50000 rank3 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4434 | Train Loss Components: Auxiliary: 0.1867 | Cardinality: 0.1036 | Classification: 0.0834 | Classification_binary: 0.1437 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1064 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0497 | Regression_eta: 0.0268 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0399 | LR: 3.71e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.7503s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7930 | max_loss: 0.1458 | max_backward: 1.7855 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 2.2800 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5572.8 | uss_MB: 4310.0 | child_rss_MB: 6426.4 | allocated_MB: 496.8 | reserved_MB: 29580.0 | inactive_split_MB: 850.8 +rank3 [2026-10-29 16:06:05,475] INFO : Step 31600/50000 rank3 | Train Loss: 1.0288 | Train Loss Unweighted: 0.4730 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1317 | Classification: 0.0906 | Classification_binary: 0.1743 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1012 | Regression_eta: 0.0017 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0980 | Regression_energy: 0.0537 | Regression_eta: 0.0729 | Regression_pt: 0.0608 | Regression_sin_phi: 0.1032 | LR: 3.69e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1939s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1021 | max_forward: 0.8041 | max_loss: 0.5000 | max_backward: 1.8881 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0133 | Memory: rss_MB: 5567.6 | uss_MB: 4330.2 | child_rss_MB: 6421.3 | allocated_MB: 494.5 | reserved_MB: 29580.0 | inactive_split_MB: 850.5 +rank3 [2026-10-29 16:10:48,123] INFO : Step 31700/50000 rank3 | Train Loss: 0.9206 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1106 | Classification: 0.0729 | Classification_binary: 0.1465 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0331 | Regression_energy: 0.0560 | Regression_eta: 0.0257 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0476 | LR: 3.66e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.7510s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0062 | max_forward: 0.7097 | max_loss: 0.1806 | max_backward: 1.6117 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0100 | Memory: rss_MB: 5567.0 | uss_MB: 4328.6 | child_rss_MB: 6424.1 | allocated_MB: 495.3 | reserved_MB: 29580.0 | inactive_split_MB: 969.6 +rank3 [2026-10-29 16:15:56,265] INFO : Step 31800/50000 rank3 | Train Loss: 0.9090 | Train Loss Unweighted: 0.4310 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1140 | Classification: 0.0748 | Classification_binary: 0.1332 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1094 | Regression_eta: 0.0013 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0611 | Regression_eta: 0.0435 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0454 | LR: 3.64e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8745s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7102 | max_loss: 0.1436 | max_backward: 1.6093 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5562.0 | uss_MB: 4326.9 | child_rss_MB: 6424.4 | allocated_MB: 499.4 | reserved_MB: 29584.0 | inactive_split_MB: 1068.5 +rank3 [2026-10-29 16:20:39,712] INFO : Step 31900/50000 rank3 | Train Loss: 0.9919 | Train Loss Unweighted: 0.4645 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1269 | Classification: 0.0911 | Classification_binary: 0.1745 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0975 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1992 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0502 | Regression_eta: 0.0398 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0682 | LR: 3.61e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3412s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.2538 | max_backward: 1.9768 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.5597 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5568.8 | uss_MB: 4333.1 | child_rss_MB: 6433.0 | allocated_MB: 496.2 | reserved_MB: 29584.0 | inactive_split_MB: 829.8 +rank3 [2026-10-29 16:25:21,506] INFO : Step 32000/50000 rank3 | Train Loss: 0.9662 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1172 | Classification: 0.0861 | Classification_binary: 0.1650 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1010 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1552 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0517 | Regression_eta: 0.0333 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0435 | LR: 3.58e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2283s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7801 | max_loss: 0.1534 | max_backward: 1.8397 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5561.3 | uss_MB: 4329.3 | child_rss_MB: 6422.4 | allocated_MB: 497.7 | reserved_MB: 29584.0 | inactive_split_MB: 927.6 +rank3 [2026-10-29 16:25:22,456] INFO : Running validation on rank3 +rank3 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 16:26:16,179] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:26:16,711] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 16:26:42,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 16:26:43,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 16:26:44,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 16:26:45,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 16:26:55,919] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 16:26:56,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:26:57,013] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 16:27:13,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 16:27:14,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 16:27:15,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 16:27:16,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 16:27:26,119] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 16:27:26,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 16:27:27,382] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 16:27:42,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 16:27:43,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 16:27:44,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 16:27:45,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 16:27:55,126] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 16:32:59,848] INFO : Step 32100/50000 rank3 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4572 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1125 | Classification: 0.0829 | Classification_binary: 0.1389 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1159 | Regression_eta: 0.0016 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1337 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0578 | Regression_eta: 0.0344 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0514 | LR: 3.56e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8029s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0431 | max_forward: 0.7694 | max_loss: 0.1379 | max_backward: 1.5662 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 1.3675 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6985.8 | uss_MB: 5757.8 | child_rss_MB: 6447.8 | allocated_MB: 499.0 | reserved_MB: 25862.0 | inactive_split_MB: 892.0 +rank3 [2026-10-29 16:37:33,176] INFO : Step 32200/50000 rank3 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1193 | Classification: 0.0958 | Classification_binary: 0.1508 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1092 | Regression_eta: 0.0016 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0676 | Regression_eta: 0.0521 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0667 | LR: 3.53e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1393s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8207 | max_loss: 0.1504 | max_backward: 2.0613 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.9148 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6989.2 | uss_MB: 5763.2 | child_rss_MB: 6457.2 | allocated_MB: 499.4 | reserved_MB: 25864.0 | inactive_split_MB: 778.6 +rank3 [2026-10-29 16:42:04,507] INFO : Step 32300/50000 rank3 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4909 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1295 | Classification: 0.0875 | Classification_binary: 0.1834 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0017 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0392 | Regression_energy: 0.0583 | Regression_eta: 0.0335 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0493 | LR: 3.51e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7855s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8382 | max_loss: 0.2412 | max_backward: 2.0637 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.9540 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5664.5 | uss_MB: 4450.0 | child_rss_MB: 6462.2 | allocated_MB: 499.0 | reserved_MB: 27716.0 | inactive_split_MB: 775.0 +rank3 [2026-10-29 16:46:37,460] INFO : Step 32400/50000 rank3 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1089 | Classification: 0.0860 | Classification_binary: 0.1494 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1059 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1625 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0484 | Regression_eta: 0.0318 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0592 | LR: 3.48e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5410s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1137 | max_forward: 0.8338 | max_loss: 0.1463 | max_backward: 1.8285 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 0.8970 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5664.1 | uss_MB: 4449.3 | child_rss_MB: 6459.6 | allocated_MB: 497.5 | reserved_MB: 27716.0 | inactive_split_MB: 964.6 +rank3 [2026-10-29 16:51:11,722] INFO : Step 32500/50000 rank3 | Train Loss: 0.9126 | Train Loss Unweighted: 0.4568 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.0989 | Classification: 0.0980 | Classification_binary: 0.1302 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1128 | Regression_eta: 0.0017 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0565 | Regression_eta: 0.0473 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0434 | LR: 3.46e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0954s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7619 | max_loss: 0.1497 | max_backward: 1.5269 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.7665 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5643.1 | uss_MB: 4429.2 | child_rss_MB: 6454.9 | allocated_MB: 495.6 | reserved_MB: 27718.0 | inactive_split_MB: 1084.2 +rank3 [2026-10-29 16:55:48,603] INFO : Step 32600/50000 rank3 | Train Loss: 0.9821 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2025 | Cardinality: 0.1270 | Classification: 0.0917 | Classification_binary: 0.1687 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1000 | Regression_eta: 0.0014 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0514 | Regression_eta: 0.0451 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0613 | LR: 3.43e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2674s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1505 | max_backward: 1.8823 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 5658.5 | uss_MB: 4448.5 | child_rss_MB: 6453.0 | allocated_MB: 496.5 | reserved_MB: 27844.0 | inactive_split_MB: 835.5 +rank3 [2026-10-29 17:00:16,962] INFO : Step 32700/50000 rank3 | Train Loss: 1.0218 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2103 | Cardinality: 0.1308 | Classification: 0.0908 | Classification_binary: 0.1761 | HitGrouping: 0.1579 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0560 | Regression_eta: 0.0313 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0497 | LR: 3.41e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2016s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7654 | max_loss: 0.2523 | max_backward: 1.8639 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4904 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.1101 | Memory: rss_MB: 5657.8 | uss_MB: 4448.3 | child_rss_MB: 6459.1 | allocated_MB: 495.8 | reserved_MB: 27844.0 | inactive_split_MB: 1177.8 +rank3 [2026-10-29 17:04:50,646] INFO : Step 32800/50000 rank3 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4798 | Train Loss Components: Auxiliary: 0.1977 | Cardinality: 0.1139 | Classification: 0.0957 | Classification_binary: 0.1480 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1163 | Regression_eta: 0.0019 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0613 | Regression_eta: 0.0548 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0505 | LR: 3.38e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0475s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7437 | max_loss: 0.1443 | max_backward: 1.7603 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.5989 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5618.6 | uss_MB: 4447.1 | child_rss_MB: 6328.3 | allocated_MB: 497.6 | reserved_MB: 27846.0 | inactive_split_MB: 1089.8 +rank3 [2026-10-29 17:09:25,012] INFO : Step 32900/50000 rank3 | Train Loss: 1.0585 | Train Loss Unweighted: 0.4917 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1472 | Classification: 0.0991 | Classification_binary: 0.1566 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0020 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1588 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0674 | Regression_eta: 0.0500 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0684 | LR: 3.36e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.2350s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0141 | max_forward: 0.7853 | max_loss: 0.1442 | max_backward: 1.9716 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.5450 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5622.7 | uss_MB: 4451.0 | child_rss_MB: 6346.8 | allocated_MB: 498.5 | reserved_MB: 27846.0 | inactive_split_MB: 844.8 +rank3 [2026-10-29 17:13:57,544] INFO : Step 33000/50000 rank3 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4896 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1109 | Classification: 0.0902 | Classification_binary: 0.1757 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1096 | Regression_eta: 0.0015 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0723 | Regression_eta: 0.0312 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0522 | LR: 3.33e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1386s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0372 | max_forward: 0.7798 | max_loss: 0.1504 | max_backward: 1.8342 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5614.6 | uss_MB: 4448.7 | child_rss_MB: 6348.5 | allocated_MB: 497.7 | reserved_MB: 27846.0 | inactive_split_MB: 857.6 +rank3 [2026-10-29 17:13:58,424] INFO : Running validation on rank3 +rank3 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 17:14:49,526] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:14:50,069] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 17:15:17,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 17:15:18,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 17:15:19,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 17:15:20,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 17:15:30,161] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 17:15:31,022] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:15:31,507] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 17:15:47,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 17:15:48,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 17:15:49,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 17:15:50,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 17:16:00,310] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 17:16:01,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 17:16:01,831] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 17:16:16,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 17:16:17,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 17:16:18,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 17:16:19,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 17:16:29,401] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 17:21:40,219] INFO : Step 33100/50000 rank3 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4666 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1297 | Classification: 0.0842 | Classification_binary: 0.1537 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1125 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1312 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0516 | Regression_eta: 0.0290 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0354 | LR: 3.31e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1417s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7607 | max_loss: 0.1464 | max_backward: 1.8352 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6888.0 | uss_MB: 5749.0 | child_rss_MB: 6329.7 | allocated_MB: 496.1 | reserved_MB: 24572.0 | inactive_split_MB: 941.9 +rank3 [2026-10-29 17:26:33,026] INFO : Step 33200/50000 rank3 | Train Loss: 0.9345 | Train Loss Unweighted: 0.4417 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.1121 | Classification: 0.0825 | Classification_binary: 0.1333 | HitGrouping: 0.1549 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0017 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1275 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0599 | Regression_eta: 0.0339 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0695 | LR: 3.28e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2177s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0447 | max_forward: 0.7619 | max_loss: 0.4589 | max_backward: 1.9181 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.4922 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6887.2 | uss_MB: 5750.0 | child_rss_MB: 6329.3 | allocated_MB: 497.6 | reserved_MB: 24578.0 | inactive_split_MB: 835.2 +rank3 [2026-10-29 17:31:31,555] INFO : Step 33300/50000 rank3 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1270 | Classification: 0.0780 | Classification_binary: 0.1518 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0985 | Regression_eta: 0.0012 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1717 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0602 | Regression_eta: 0.0422 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0547 | LR: 3.26e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.6840s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0334 | max_forward: 0.7851 | max_loss: 0.1434 | max_backward: 2.0284 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.7975 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5654.9 | uss_MB: 4170.1 | child_rss_MB: 6324.3 | allocated_MB: 496.9 | reserved_MB: 24578.0 | inactive_split_MB: 903.1 +rank3 [2026-10-29 17:36:22,673] INFO : Step 33400/50000 rank3 | Train Loss: 0.9537 | Train Loss Unweighted: 0.4634 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1239 | Classification: 0.0872 | Classification_binary: 0.1563 | HitGrouping: 0.1500 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0013 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0571 | Regression_eta: 0.0389 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0531 | LR: 3.23e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2707s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7951 | max_loss: 0.2522 | max_backward: 1.9349 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0134 | Memory: rss_MB: 5664.7 | uss_MB: 4533.0 | child_rss_MB: 6325.6 | allocated_MB: 500.7 | reserved_MB: 26578.0 | inactive_split_MB: 843.2 +rank3 [2026-10-29 17:41:05,304] INFO : Step 33500/50000 rank3 | Train Loss: 0.9039 | Train Loss Unweighted: 0.4494 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.1057 | Classification: 0.0795 | Classification_binary: 0.1278 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0011 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0592 | Regression_eta: 0.0336 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0603 | LR: 3.21e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1835s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0177 | max_forward: 0.7696 | max_loss: 0.1421 | max_backward: 1.9122 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 0.5562 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5654.7 | uss_MB: 4519.8 | child_rss_MB: 6324.9 | allocated_MB: 494.9 | reserved_MB: 26578.0 | inactive_split_MB: 839.3 +rank3 [2026-10-29 17:45:56,711] INFO : Step 33600/50000 rank3 | Train Loss: 0.9179 | Train Loss Unweighted: 0.4415 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.0997 | Classification: 0.0812 | Classification_binary: 0.1440 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1298 | Regression_cos_phi: 0.0447 | Regression_energy: 0.0467 | Regression_eta: 0.0303 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0476 | LR: 3.19e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4034s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7783 | max_loss: 0.2418 | max_backward: 2.0392 | max_optimizer_step: 0.2787 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5667.7 | uss_MB: 4535.9 | child_rss_MB: 6324.1 | allocated_MB: 497.9 | reserved_MB: 28594.0 | inactive_split_MB: 1127.0 +rank3 [2026-10-29 17:50:43,454] INFO : Step 33700/50000 rank3 | Train Loss: 1.0564 | Train Loss Unweighted: 0.4962 | Train Loss Components: Auxiliary: 0.2235 | Cardinality: 0.1517 | Classification: 0.0917 | Classification_binary: 0.1786 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1761 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0493 | Regression_eta: 0.0378 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0520 | LR: 3.16e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1466s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0121 | max_forward: 0.7981 | max_loss: 0.1554 | max_backward: 1.9054 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5667.6 | uss_MB: 4535.1 | child_rss_MB: 6300.3 | allocated_MB: 501.8 | reserved_MB: 28594.0 | inactive_split_MB: 1103.2 +rank3 [2026-10-29 17:55:24,637] INFO : Step 33800/50000 rank3 | Train Loss: 0.9305 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1894 | Cardinality: 0.1080 | Classification: 0.0751 | Classification_binary: 0.1369 | HitGrouping: 0.1527 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1201 | Regression_eta: 0.0014 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1199 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0622 | Regression_eta: 0.0360 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0404 | LR: 3.14e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1055s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8033 | max_loss: 0.1441 | max_backward: 1.7197 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0176 | Memory: rss_MB: 5662.9 | uss_MB: 4531.8 | child_rss_MB: 6320.7 | allocated_MB: 496.9 | reserved_MB: 28594.0 | inactive_split_MB: 909.5 +rank3 [2026-10-29 18:00:01,586] INFO : Step 33900/50000 rank3 | Train Loss: 0.9360 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.1942 | Cardinality: 0.1035 | Classification: 0.0805 | Classification_binary: 0.1361 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1124 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1177 | Regression_cos_phi: 0.0634 | Regression_energy: 0.0485 | Regression_eta: 0.0564 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0674 | LR: 3.11e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2971s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7557 | max_loss: 0.2439 | max_backward: 1.8923 | max_optimizer_step: 0.2623 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5665.6 | uss_MB: 4533.9 | child_rss_MB: 6306.4 | allocated_MB: 496.5 | reserved_MB: 28598.0 | inactive_split_MB: 941.5 +rank3 [2026-10-29 18:04:39,053] INFO : Step 34000/50000 rank3 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0930 | Classification: 0.0875 | Classification_binary: 0.1606 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.1836 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0590 | Regression_eta: 0.0325 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0528 | LR: 3.09e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3484s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0943 | max_forward: 0.8289 | max_loss: 0.4616 | max_backward: 2.0219 | max_optimizer_step: 0.1767 | max_tensorboard_logging: 0.4773 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5666.0 | uss_MB: 4534.2 | child_rss_MB: 6326.4 | allocated_MB: 496.0 | reserved_MB: 28598.0 | inactive_split_MB: 1123.0 +rank3 [2026-10-29 18:04:39,968] INFO : Running validation on rank3 +rank3 [2026-10-29 18:04:39,968] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 18:04:39,968] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 18:05:30,627] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:05:31,101] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 18:05:54,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 18:05:55,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 18:05:56,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 18:05:57,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 18:06:07,898] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 18:06:08,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:06:09,213] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 18:06:25,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 18:06:26,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 18:06:27,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 18:06:27,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 18:06:38,007] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 18:06:39,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:06:39,483] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 18:06:54,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 18:06:55,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 18:06:56,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 18:06:57,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 18:07:07,070] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 18:12:12,107] INFO : Step 34100/50000 rank3 | Train Loss: 1.0240 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2153 | Cardinality: 0.1202 | Classification: 0.0979 | Classification_binary: 0.1668 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1210 | Regression_eta: 0.0013 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1735 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0777 | Regression_eta: 0.0314 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0470 | LR: 3.06e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5014s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0078 | max_forward: 0.9085 | max_loss: 0.1531 | max_backward: 2.1727 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.4680 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6900.4 | uss_MB: 5761.6 | child_rss_MB: 6326.4 | allocated_MB: 496.4 | reserved_MB: 24560.0 | inactive_split_MB: 1172.2 +rank3 [2026-10-29 18:17:11,666] INFO : Step 34200/50000 rank3 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4225 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.1030 | Classification: 0.0694 | Classification_binary: 0.1279 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0016 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0632 | Regression_eta: 0.0340 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0456 | LR: 3.04e-04 | DataLoad Time: 0.0178s | Model Forward Time: 5.8725s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 1.1401 | max_loss: 1.0887 | max_backward: 3.3565 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5402 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6897.3 | uss_MB: 5759.3 | child_rss_MB: 6328.4 | allocated_MB: 493.1 | reserved_MB: 27668.0 | inactive_split_MB: 842.0 +rank3 [2026-10-29 18:21:53,343] INFO : Step 34300/50000 rank3 | Train Loss: 0.8999 | Train Loss Unweighted: 0.4291 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.1046 | Classification: 0.0782 | Classification_binary: 0.1421 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0015 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0424 | Regression_eta: 0.0332 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0622 | LR: 3.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5616s | Phase Times: max_pre_step_barrier: 0.1089 | max_batch_to_device: 0.0070 | max_forward: 0.7738 | max_loss: 0.4754 | max_backward: 1.8888 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5721.4 | uss_MB: 4584.0 | child_rss_MB: 6315.8 | allocated_MB: 498.9 | reserved_MB: 29668.0 | inactive_split_MB: 777.1 +rank3 [2026-10-29 18:26:42,330] INFO : Step 34400/50000 rank3 | Train Loss: 0.9382 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2026 | Cardinality: 0.1181 | Classification: 0.0846 | Classification_binary: 0.1681 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0565 | Regression_eta: 0.0322 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0631 | LR: 2.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6571s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.8411 | max_loss: 0.2739 | max_backward: 2.1385 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5734.9 | uss_MB: 4598.3 | child_rss_MB: 6315.2 | allocated_MB: 496.8 | reserved_MB: 29668.0 | inactive_split_MB: 858.8 +rank3 [2026-10-29 18:31:40,822] INFO : Step 34500/50000 rank3 | Train Loss: 0.9189 | Train Loss Unweighted: 0.4456 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1063 | Classification: 0.0767 | Classification_binary: 0.1421 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1523 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0725 | Regression_eta: 0.0502 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0451 | LR: 2.97e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.2126s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.7430 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5727.7 | uss_MB: 4595.8 | child_rss_MB: 6313.8 | allocated_MB: 494.3 | reserved_MB: 29668.0 | inactive_split_MB: 926.0 +rank3 [2026-10-29 18:36:27,182] INFO : Step 34600/50000 rank3 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4340 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1046 | Classification: 0.0794 | Classification_binary: 0.1340 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0016 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0502 | Regression_eta: 0.0507 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0415 | LR: 2.95e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.6945s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0170 | max_forward: 0.9895 | max_loss: 0.4622 | max_backward: 2.2541 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5731.7 | uss_MB: 4597.0 | child_rss_MB: 6311.8 | allocated_MB: 498.6 | reserved_MB: 29672.0 | inactive_split_MB: 765.1 +rank3 [2026-10-29 18:40:58,825] INFO : Step 34700/50000 rank3 | Train Loss: 0.9586 | Train Loss Unweighted: 0.4539 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1185 | Classification: 0.0862 | Classification_binary: 0.1624 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0446 | Regression_eta: 0.0407 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0497 | LR: 2.92e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.3671s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7862 | max_loss: 0.2628 | max_backward: 1.9231 | max_optimizer_step: 0.2860 | max_tensorboard_logging: 0.4599 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5730.7 | uss_MB: 4596.4 | child_rss_MB: 6303.0 | allocated_MB: 499.0 | reserved_MB: 29672.0 | inactive_split_MB: 1074.3 +rank3 [2026-10-29 18:45:38,651] INFO : Step 34800/50000 rank3 | Train Loss: 0.9346 | Train Loss Unweighted: 0.4373 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1276 | Classification: 0.0867 | Classification_binary: 0.1551 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0959 | Regression_eta: 0.0013 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0511 | Regression_eta: 0.0257 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0510 | LR: 2.90e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2169s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0770 | max_forward: 0.8404 | max_loss: 0.1556 | max_backward: 1.8035 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.5351 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5727.0 | uss_MB: 4594.2 | child_rss_MB: 6312.4 | allocated_MB: 497.4 | reserved_MB: 29672.0 | inactive_split_MB: 1113.6 +rank3 [2026-10-29 18:50:17,935] INFO : Step 34900/50000 rank3 | Train Loss: 0.8719 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1154 | Classification: 0.0681 | Classification_binary: 0.1220 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1093 | Regression_eta: 0.0013 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1137 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0604 | Regression_eta: 0.0231 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0421 | LR: 2.88e-04 | DataLoad Time: 0.0189s | Model Forward Time: 6.1481s | Phase Times: max_pre_step_barrier: 0.0080 | max_batch_to_device: 0.0693 | max_forward: 1.2127 | max_loss: 0.2834 | max_backward: 3.0996 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.6125 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 5724.6 | uss_MB: 4576.6 | child_rss_MB: 6292.2 | allocated_MB: 495.3 | reserved_MB: 29672.0 | inactive_split_MB: 864.7 +rank3 [2026-10-29 18:55:19,194] INFO : Step 35000/50000 rank3 | Train Loss: 0.9157 | Train Loss Unweighted: 0.4220 | Train Loss Components: Auxiliary: 0.1897 | Cardinality: 0.1230 | Classification: 0.0771 | Classification_binary: 0.1455 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0980 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1414 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0571 | Regression_eta: 0.0500 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0486 | LR: 2.85e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5128s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.8243 | max_loss: 0.4759 | max_backward: 2.1181 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5728.2 | uss_MB: 4598.3 | child_rss_MB: 6307.6 | allocated_MB: 498.3 | reserved_MB: 29672.0 | inactive_split_MB: 766.0 +rank3 [2026-10-29 18:55:20,127] INFO : Running validation on rank3 +rank3 [2026-10-29 18:55:20,127] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 18:55:20,127] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 18:56:11,605] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:56:12,118] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 18:56:36,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 18:56:37,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 18:56:38,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 18:56:39,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 18:56:49,683] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 18:56:49,876] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:56:50,324] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 18:57:05,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 18:57:07,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 18:57:08,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 18:57:09,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 18:57:19,881] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 18:57:20,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 18:57:20,904] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 18:57:36,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 18:57:36,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 18:57:37,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 18:57:38,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 18:57:48,729] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 19:02:56,669] INFO : Step 35100/50000 rank3 | Train Loss: 0.9622 | Train Loss Unweighted: 0.4512 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1150 | Classification: 0.0857 | Classification_binary: 0.1724 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0948 | Regression_eta: 0.0014 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0355 | Regression_energy: 0.0513 | Regression_eta: 0.0242 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0523 | LR: 2.83e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.8767s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1018 | max_forward: 0.8921 | max_loss: 0.1518 | max_backward: 2.0328 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8633 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5815.2 | uss_MB: 4686.8 | child_rss_MB: 6305.0 | allocated_MB: 499.4 | reserved_MB: 23366.0 | inactive_split_MB: 1243.8 +rank3 [2026-10-29 19:07:43,278] INFO : Step 35200/50000 rank3 | Train Loss: 0.9331 | Train Loss Unweighted: 0.4385 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.1234 | Classification: 0.0814 | Classification_binary: 0.1441 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0689 | Regression_eta: 0.0306 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0726 | LR: 2.81e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.1468s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0577 | max_forward: 0.7693 | max_loss: 0.1423 | max_backward: 1.7209 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5822.3 | uss_MB: 4693.9 | child_rss_MB: 6304.7 | allocated_MB: 494.4 | reserved_MB: 25346.0 | inactive_split_MB: 1054.2 +rank3 [2026-10-29 19:12:30,668] INFO : Step 35300/50000 rank3 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4075 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1105 | Classification: 0.0819 | Classification_binary: 0.1245 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0989 | Regression_eta: 0.0014 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1305 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0567 | Regression_eta: 0.0502 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0350 | LR: 2.79e-04 | DataLoad Time: 0.0181s | Model Forward Time: 6.8652s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 1.2856 | max_loss: 0.4079 | max_backward: 4.4969 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5749 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5814.5 | uss_MB: 4685.2 | child_rss_MB: 6307.8 | allocated_MB: 498.3 | reserved_MB: 25346.0 | inactive_split_MB: 1140.5 +rank3 [2026-10-29 19:17:14,808] INFO : Step 35400/50000 rank3 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4888 | Train Loss Components: Auxiliary: 0.2130 | Cardinality: 0.1296 | Classification: 0.0908 | Classification_binary: 0.1718 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0014 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1578 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0497 | Regression_eta: 0.0459 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0800 | LR: 2.76e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3434s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8312 | max_loss: 0.1490 | max_backward: 2.0287 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.4708 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5820.8 | uss_MB: 4696.7 | child_rss_MB: 6308.5 | allocated_MB: 497.8 | reserved_MB: 25346.0 | inactive_split_MB: 1132.0 +rank3 [2026-10-29 19:21:58,249] INFO : Step 35500/50000 rank3 | Train Loss: 0.9444 | Train Loss Unweighted: 0.4557 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1201 | Classification: 0.0924 | Classification_binary: 0.1605 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0368 | Regression_energy: 0.0471 | Regression_eta: 0.0265 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0414 | LR: 2.74e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3616s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7670 | max_loss: 0.2318 | max_backward: 1.8405 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5820.9 | uss_MB: 4694.9 | child_rss_MB: 6312.3 | allocated_MB: 497.5 | reserved_MB: 26942.0 | inactive_split_MB: 1130.5 +rank3 [2026-10-29 19:26:41,939] INFO : Step 35600/50000 rank3 | Train Loss: 0.9280 | Train Loss Unweighted: 0.4451 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1095 | Classification: 0.0861 | Classification_binary: 0.1334 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0274 | Classification_binary: 0.1117 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0494 | Regression_eta: 0.0269 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0468 | LR: 2.72e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7394s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7460 | max_loss: 0.1440 | max_backward: 1.8206 | max_optimizer_step: 0.2911 | max_tensorboard_logging: 0.6036 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 5820.5 | uss_MB: 3919.7 | child_rss_MB: 6316.0 | allocated_MB: 496.1 | reserved_MB: 26942.0 | inactive_split_MB: 907.6 +rank3 [2026-10-29 19:31:29,665] INFO : Step 35700/50000 rank3 | Train Loss: 0.9116 | Train Loss Unweighted: 0.4301 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1148 | Classification: 0.0829 | Classification_binary: 0.1432 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0998 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1529 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0561 | Regression_eta: 0.0361 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0742 | LR: 2.70e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6309s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8312 | max_loss: 0.2568 | max_backward: 2.2622 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5823.2 | uss_MB: 4699.5 | child_rss_MB: 6323.6 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1241.0 +rank3 [2026-10-29 19:36:11,896] INFO : Step 35800/50000 rank3 | Train Loss: 1.0095 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1252 | Classification: 0.0857 | Classification_binary: 0.1786 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1066 | Regression_eta: 0.0014 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1826 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0670 | Regression_eta: 0.0417 | Regression_pt: 0.0722 | Regression_sin_phi: 0.0755 | LR: 2.68e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1759s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7644 | max_loss: 0.5090 | max_backward: 1.8855 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4861 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5816.8 | uss_MB: 4694.8 | child_rss_MB: 6319.7 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1129.0 +rank3 [2026-10-29 19:40:48,761] INFO : Step 35900/50000 rank3 | Train Loss: 0.9596 | Train Loss Unweighted: 0.4571 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1173 | Classification: 0.0913 | Classification_binary: 0.1468 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0014 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0446 | Regression_eta: 0.0345 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0577 | LR: 2.65e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.0315s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7217 | max_loss: 0.1443 | max_backward: 1.6894 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.4586 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5816.9 | uss_MB: 4695.3 | child_rss_MB: 6319.9 | allocated_MB: 495.4 | reserved_MB: 29024.0 | inactive_split_MB: 934.6 +rank3 [2026-10-29 19:45:19,092] INFO : Step 36000/50000 rank3 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0912 | Classification: 0.0938 | Classification_binary: 0.1314 | HitGrouping: 0.1587 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1135 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.1529 | Regression_cos_phi: 0.1353 | Regression_energy: 0.0659 | Regression_eta: 0.0603 | Regression_pt: 0.0717 | Regression_sin_phi: 0.1005 | LR: 2.63e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3007s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0073 | max_forward: 0.7630 | max_loss: 0.4769 | max_backward: 1.8869 | max_optimizer_step: 0.2587 | max_tensorboard_logging: 0.5842 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5815.2 | uss_MB: 4695.9 | child_rss_MB: 6324.3 | allocated_MB: 495.0 | reserved_MB: 29024.0 | inactive_split_MB: 937.5 +rank3 [2026-10-29 19:45:19,981] INFO : Running validation on rank3 +rank3 [2026-10-29 19:45:19,981] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 19:45:19,981] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 19:46:10,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:46:11,092] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 19:46:35,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 19:46:36,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 19:46:37,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 19:46:38,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 19:46:48,915] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 19:46:49,453] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:46:49,932] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 19:47:05,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 19:47:06,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 19:47:07,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 19:47:08,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 19:47:18,785] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 19:47:19,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 19:47:19,885] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 19:47:35,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 19:47:36,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 19:47:36,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 19:47:37,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 19:47:47,782] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 19:53:12,007] INFO : Step 36100/50000 rank3 | Train Loss: 0.9292 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1100 | Classification: 0.0906 | Classification_binary: 0.1490 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0540 | Regression_eta: 0.0348 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0668 | LR: 2.61e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.7847s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7768 | max_loss: 0.2455 | max_backward: 1.8077 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 1.0653 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5880.7 | uss_MB: 4756.1 | child_rss_MB: 6293.3 | allocated_MB: 499.2 | reserved_MB: 26012.0 | inactive_split_MB: 1080.2 +rank3 [2026-10-29 19:58:21,470] INFO : Step 36200/50000 rank3 | Train Loss: 0.9030 | Train Loss Unweighted: 0.4308 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1163 | Classification: 0.0762 | Classification_binary: 0.1596 | HitGrouping: 0.1506 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0548 | Regression_eta: 0.0382 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0518 | LR: 2.59e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5365s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0777 | max_forward: 0.8125 | max_loss: 0.1541 | max_backward: 1.8492 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 5884.8 | uss_MB: 4767.2 | child_rss_MB: 6290.6 | allocated_MB: 494.4 | reserved_MB: 26016.0 | inactive_split_MB: 1377.4 +rank3 [2026-10-29 20:03:20,658] INFO : Step 36300/50000 rank3 | Train Loss: 0.9576 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1236 | Classification: 0.0876 | Classification_binary: 0.1347 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1100 | Regression_eta: 0.0017 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0556 | Regression_eta: 0.0624 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0583 | LR: 2.57e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.4521s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.8065 | max_loss: 0.1484 | max_backward: 1.9315 | max_optimizer_step: 0.1930 | max_tensorboard_logging: 0.6801 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5878.7 | uss_MB: 4768.0 | child_rss_MB: 6273.7 | allocated_MB: 496.0 | reserved_MB: 28008.0 | inactive_split_MB: 1156.5 +rank3 [2026-10-29 20:08:10,865] INFO : Step 36400/50000 rank3 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4147 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.0999 | Classification: 0.0796 | Classification_binary: 0.1372 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0478 | Regression_eta: 0.0350 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0559 | LR: 2.55e-04 | DataLoad Time: 0.0222s | Model Forward Time: 5.8716s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0192 | max_forward: 1.1829 | max_loss: 0.3790 | max_backward: 3.0917 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 5880.4 | uss_MB: 4768.9 | child_rss_MB: 6270.8 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 845.2 +rank3 [2026-10-29 20:12:41,930] INFO : Step 36500/50000 rank3 | Train Loss: 0.9762 | Train Loss Unweighted: 0.4543 | Train Loss Components: Auxiliary: 0.2088 | Cardinality: 0.1239 | Classification: 0.0805 | Classification_binary: 0.1682 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1393 | Regression_cos_phi: 0.0390 | Regression_energy: 0.0453 | Regression_eta: 0.0286 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0496 | LR: 2.52e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2754s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7782 | max_loss: 0.1554 | max_backward: 1.9378 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.5103 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5877.6 | uss_MB: 4769.9 | child_rss_MB: 6273.0 | allocated_MB: 499.0 | reserved_MB: 30008.0 | inactive_split_MB: 913.8 +rank3 [2026-10-29 20:17:47,229] INFO : Step 36600/50000 rank3 | Train Loss: 0.8699 | Train Loss Unweighted: 0.4190 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.1138 | Classification: 0.0734 | Classification_binary: 0.1317 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0010 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1140 | Regression_cos_phi: 0.0325 | Regression_energy: 0.0647 | Regression_eta: 0.0245 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0615 | LR: 2.50e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.2963s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7561 | max_loss: 0.1454 | max_backward: 1.9017 | max_optimizer_step: 0.5115 | max_tensorboard_logging: 0.5672 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5872.6 | uss_MB: 4766.8 | child_rss_MB: 6277.6 | allocated_MB: 494.4 | reserved_MB: 30008.0 | inactive_split_MB: 856.0 +rank3 [2026-10-29 20:22:56,452] INFO : Step 36700/50000 rank3 | Train Loss: 0.9065 | Train Loss Unweighted: 0.4327 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.1102 | Classification: 0.0849 | Classification_binary: 0.1262 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1087 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1332 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0523 | Regression_eta: 0.0425 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0659 | LR: 2.48e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.0803s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.7169 | max_loss: 0.1507 | max_backward: 1.7104 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5871.4 | uss_MB: 4766.4 | child_rss_MB: 6270.7 | allocated_MB: 496.5 | reserved_MB: 30008.0 | inactive_split_MB: 1154.6 +rank3 [2026-10-29 20:28:04,050] INFO : Step 36800/50000 rank3 | Train Loss: 0.9068 | Train Loss Unweighted: 0.4227 | Train Loss Components: Auxiliary: 0.1925 | Cardinality: 0.1100 | Classification: 0.0739 | Classification_binary: 0.1571 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0945 | Regression_eta: 0.0013 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1264 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0633 | Regression_eta: 0.0223 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0311 | LR: 2.46e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4674s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8487 | max_loss: 0.1525 | max_backward: 2.1121 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5877.1 | uss_MB: 4770.1 | child_rss_MB: 6274.6 | allocated_MB: 499.3 | reserved_MB: 30008.0 | inactive_split_MB: 939.6 +rank3 [2026-10-29 20:33:09,546] INFO : Step 36900/50000 rank3 | Train Loss: 0.8934 | Train Loss Unweighted: 0.4126 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1194 | Classification: 0.0721 | Classification_binary: 0.1507 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0013 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0471 | Regression_eta: 0.0529 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0496 | LR: 2.44e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.5698s | Phase Times: max_pre_step_barrier: 0.0139 | max_batch_to_device: 0.0068 | max_forward: 1.0975 | max_loss: 0.2241 | max_backward: 2.7278 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.6216 | max_tensorboard_flush: 0.0085 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5871.7 | uss_MB: 4750.7 | child_rss_MB: 6260.8 | allocated_MB: 498.9 | reserved_MB: 30008.0 | inactive_split_MB: 771.1 +rank3 [2026-10-29 20:38:05,959] INFO : Step 37000/50000 rank3 | Train Loss: 0.9040 | Train Loss Unweighted: 0.4182 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1192 | Classification: 0.0807 | Classification_binary: 0.1318 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1328 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0532 | Regression_eta: 0.0366 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0398 | LR: 2.42e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.8501s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7055 | max_loss: 0.1505 | max_backward: 1.5664 | max_optimizer_step: 0.1900 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5866.9 | uss_MB: 4763.2 | child_rss_MB: 6273.0 | allocated_MB: 493.9 | reserved_MB: 30008.0 | inactive_split_MB: 1169.8 +rank3 [2026-10-29 20:38:06,931] INFO : Running validation on rank3 +rank3 [2026-10-29 20:38:06,931] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 20:38:06,931] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 20:39:05,962] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:39:06,530] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 20:39:31,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 20:39:32,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 20:39:33,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 20:39:34,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 20:39:44,595] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 20:39:45,042] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:39:45,534] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 20:40:02,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 20:40:03,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 20:40:04,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 20:40:05,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 20:40:15,401] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 20:40:16,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 20:40:16,608] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 20:40:32,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 20:40:32,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 20:40:33,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 20:40:35,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 20:40:45,053] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-29 20:45:59,178] INFO : Step 37100/50000 rank3 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4614 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1255 | Classification: 0.0999 | Classification_binary: 0.1492 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1041 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0610 | Regression_eta: 0.0313 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0757 | LR: 2.40e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.8454s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0061 | max_forward: 0.8553 | max_loss: 0.1461 | max_backward: 2.1723 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5959.4 | uss_MB: 4852.0 | child_rss_MB: 6311.9 | allocated_MB: 499.3 | reserved_MB: 25668.0 | inactive_split_MB: 852.4 +rank3 [2026-10-29 20:50:39,579] INFO : Step 37200/50000 rank3 | Train Loss: 0.9288 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.1928 | Cardinality: 0.1108 | Classification: 0.0836 | Classification_binary: 0.1643 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0015 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1851 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0496 | Regression_eta: 0.0468 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0624 | LR: 2.38e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6394s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8460 | max_loss: 0.2603 | max_backward: 2.0959 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5966.0 | uss_MB: 4861.5 | child_rss_MB: 6312.1 | allocated_MB: 497.9 | reserved_MB: 25674.0 | inactive_split_MB: 846.0 +rank3 [2026-10-29 20:55:11,587] INFO : Step 37300/50000 rank3 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1275 | Classification: 0.0781 | Classification_binary: 0.1424 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0982 | Regression_eta: 0.0013 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0342 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0420 | Regression_eta: 0.0277 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0454 | LR: 2.36e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0206s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7653 | max_loss: 0.1472 | max_backward: 1.7105 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.4778 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0692 | Memory: rss_MB: 5962.2 | uss_MB: 4859.4 | child_rss_MB: 6302.2 | allocated_MB: 497.3 | reserved_MB: 27616.0 | inactive_split_MB: 974.4 +rank3 [2026-10-29 21:00:09,948] INFO : Step 37400/50000 rank3 | Train Loss: 0.8836 | Train Loss Unweighted: 0.4228 | Train Loss Components: Auxiliary: 0.1825 | Cardinality: 0.0983 | Classification: 0.0851 | Classification_binary: 0.1289 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1025 | Regression_eta: 0.0015 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1147 | Regression_cos_phi: 0.0266 | Regression_energy: 0.0469 | Regression_eta: 0.0639 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0591 | LR: 2.34e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5393s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7611 | max_loss: 0.1956 | max_backward: 1.9276 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7505 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5960.0 | uss_MB: 4860.1 | child_rss_MB: 6301.8 | allocated_MB: 495.0 | reserved_MB: 27690.0 | inactive_split_MB: 915.5 +rank3 [2026-10-29 21:05:01,764] INFO : Step 37500/50000 rank3 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1260 | Classification: 0.0804 | Classification_binary: 0.1577 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1005 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1377 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0615 | Regression_eta: 0.0276 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0537 | LR: 2.32e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4456s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.9030 | max_loss: 0.1494 | max_backward: 1.9871 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5938.4 | uss_MB: 4859.4 | child_rss_MB: 6224.4 | allocated_MB: 496.1 | reserved_MB: 27690.0 | inactive_split_MB: 923.5 +rank3 [2026-10-29 21:09:54,197] INFO : Step 37600/50000 rank3 | Train Loss: 0.9091 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.0995 | Classification: 0.0683 | Classification_binary: 0.1534 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0957 | Regression_eta: 0.0015 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1535 | Regression_cos_phi: 0.1074 | Regression_energy: 0.0419 | Regression_eta: 0.0430 | Regression_pt: 0.0459 | Regression_sin_phi: 0.0648 | LR: 2.30e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3494s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0070 | max_forward: 0.8280 | max_loss: 0.1484 | max_backward: 1.8328 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5933.4 | uss_MB: 4856.7 | child_rss_MB: 6230.2 | allocated_MB: 497.1 | reserved_MB: 27690.0 | inactive_split_MB: 927.5 +rank3 [2026-10-29 21:14:45,920] INFO : Step 37700/50000 rank3 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3995 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.0864 | Classification: 0.0813 | Classification_binary: 0.1231 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0012 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1242 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0534 | Regression_eta: 0.0447 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0681 | LR: 2.28e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4208s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7582 | max_loss: 0.2486 | max_backward: 1.9197 | max_optimizer_step: 0.3005 | max_tensorboard_logging: 0.5767 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5802.8 | uss_MB: 4857.6 | child_rss_MB: 5767.0 | allocated_MB: 495.2 | reserved_MB: 29766.0 | inactive_split_MB: 925.5 +rank3 [2026-10-29 21:19:50,095] INFO : Step 37800/50000 rank3 | Train Loss: 0.8717 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1057 | Classification: 0.0822 | Classification_binary: 0.1363 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0014 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1222 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0528 | Regression_eta: 0.0406 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0635 | LR: 2.26e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.6246s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 1.2105 | max_loss: 0.2449 | max_backward: 2.0574 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 5800.5 | uss_MB: 4856.8 | child_rss_MB: 5763.9 | allocated_MB: 498.3 | reserved_MB: 29766.0 | inactive_split_MB: 930.0 +rank3 [2026-10-29 21:24:49,255] INFO : Step 37900/50000 rank3 | Train Loss: 0.9843 | Train Loss Unweighted: 0.4684 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1353 | Classification: 0.0912 | Classification_binary: 0.1705 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0014 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0549 | Regression_eta: 0.0322 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0834 | LR: 2.24e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2095s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0297 | max_forward: 0.8081 | max_loss: 0.1547 | max_backward: 2.6708 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5096 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5782.9 | uss_MB: 4859.7 | child_rss_MB: 5732.7 | allocated_MB: 495.6 | reserved_MB: 29766.0 | inactive_split_MB: 912.0 +rank3 [2026-10-29 21:29:44,506] INFO : Step 38000/50000 rank3 | Train Loss: 0.8556 | Train Loss Unweighted: 0.4002 | Train Loss Components: Auxiliary: 0.1833 | Cardinality: 0.1090 | Classification: 0.0730 | Classification_binary: 0.1293 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0972 | Regression_eta: 0.0011 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0436 | Regression_eta: 0.0274 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0248 | LR: 2.22e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1651s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7330 | max_loss: 0.2284 | max_backward: 1.8082 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5778.8 | uss_MB: 4857.4 | child_rss_MB: 5727.0 | allocated_MB: 494.5 | reserved_MB: 29766.0 | inactive_split_MB: 847.3 +rank3 [2026-10-29 21:29:45,441] INFO : Running validation on rank3 +rank3 [2026-10-29 21:29:45,441] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 21:29:45,441] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 21:30:41,620] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:30:42,097] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 21:31:09,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 21:31:10,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 21:31:11,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 21:31:12,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 21:31:22,594] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 21:31:23,194] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:31:23,629] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 21:31:39,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 21:31:40,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 21:31:41,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 21:31:42,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 21:31:52,659] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 21:31:52,734] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 21:31:53,235] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 21:32:08,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 21:32:09,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 21:32:09,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 21:32:10,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 21:32:20,896] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 21:37:39,070] INFO : Step 38100/50000 rank3 | Train Loss: 0.9020 | Train Loss Unweighted: 0.4337 | Train Loss Components: Auxiliary: 0.1907 | Cardinality: 0.1231 | Classification: 0.0888 | Classification_binary: 0.1339 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0014 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0419 | Classification_binary: 0.1340 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0584 | Regression_eta: 0.0312 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0281 | LR: 2.20e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8726s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0963 | max_forward: 0.8445 | max_loss: 0.1406 | max_backward: 1.9197 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.9697 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7028.7 | uss_MB: 6098.3 | child_rss_MB: 5734.5 | allocated_MB: 496.0 | reserved_MB: 27516.0 | inactive_split_MB: 970.5 +rank3 [2026-10-29 21:42:37,686] INFO : Step 38200/50000 rank3 | Train Loss: 0.9174 | Train Loss Unweighted: 0.4351 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.0971 | Classification: 0.0888 | Classification_binary: 0.1526 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0948 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0723 | Regression_energy: 0.0511 | Regression_eta: 0.0372 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0997 | LR: 2.18e-04 | DataLoad Time: 0.0174s | Model Forward Time: 6.4455s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.8480 | max_loss: 0.2651 | max_backward: 3.1905 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.9151 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5881.2 | uss_MB: 4342.7 | child_rss_MB: 5733.1 | allocated_MB: 499.7 | reserved_MB: 27516.0 | inactive_split_MB: 848.5 +rank3 [2026-10-29 21:47:30,433] INFO : Step 38300/50000 rank3 | Train Loss: 0.9342 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1998 | Cardinality: 0.1099 | Classification: 0.0869 | Classification_binary: 0.1470 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0013 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0716 | Regression_eta: 0.0263 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0484 | LR: 2.16e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6858s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0664 | max_forward: 0.8187 | max_loss: 0.1464 | max_backward: 1.8154 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.9517 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5875.2 | uss_MB: 4949.5 | child_rss_MB: 5725.9 | allocated_MB: 495.5 | reserved_MB: 27516.0 | inactive_split_MB: 1085.5 +rank3 [2026-10-29 21:52:14,429] INFO : Step 38400/50000 rank3 | Train Loss: 0.8281 | Train Loss Unweighted: 0.4036 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0915 | Classification: 0.0723 | Classification_binary: 0.1160 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1069 | Regression_eta: 0.0012 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1012 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0629 | Regression_eta: 0.0389 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0432 | LR: 2.14e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.7613s | Phase Times: max_pre_step_barrier: 0.0868 | max_batch_to_device: 0.0062 | max_forward: 0.9937 | max_loss: 0.2500 | max_backward: 1.9690 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5872.2 | uss_MB: 4950.8 | child_rss_MB: 5730.9 | allocated_MB: 496.1 | reserved_MB: 27516.0 | inactive_split_MB: 972.6 +rank3 [2026-10-29 21:57:09,318] INFO : Step 38500/50000 rank3 | Train Loss: 0.9386 | Train Loss Unweighted: 0.4484 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1248 | Classification: 0.1044 | Classification_binary: 0.1389 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0801 | Regression_eta: 0.0617 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0724 | LR: 2.12e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2057s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.7200 | max_loss: 0.1386 | max_backward: 1.5323 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5868.3 | uss_MB: 4952.0 | child_rss_MB: 5707.9 | allocated_MB: 496.0 | reserved_MB: 27520.0 | inactive_split_MB: 776.0 +rank3 [2026-10-29 22:01:43,605] INFO : Step 38600/50000 rank3 | Train Loss: 0.9227 | Train Loss Unweighted: 0.4436 | Train Loss Components: Auxiliary: 0.1984 | Cardinality: 0.1056 | Classification: 0.0788 | Classification_binary: 0.1660 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0979 | Regression_eta: 0.0013 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0555 | Regression_eta: 0.0374 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0738 | LR: 2.11e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4277s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8785 | max_loss: 0.1473 | max_backward: 2.0276 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5861.9 | uss_MB: 4951.4 | child_rss_MB: 5677.1 | allocated_MB: 499.0 | reserved_MB: 29592.0 | inactive_split_MB: 827.5 +rank3 [2026-10-29 22:06:19,438] INFO : Step 38700/50000 rank3 | Train Loss: 0.8153 | Train Loss Unweighted: 0.3795 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0947 | Classification: 0.0722 | Classification_binary: 0.1256 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0893 | Regression_eta: 0.0013 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0304 | Classification_binary: 0.1134 | Regression_cos_phi: 0.0337 | Regression_energy: 0.0385 | Regression_eta: 0.0351 | Regression_pt: 0.0420 | Regression_sin_phi: 0.0380 | LR: 2.09e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2105s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7251 | max_loss: 0.2433 | max_backward: 1.8091 | max_optimizer_step: 0.1965 | max_tensorboard_logging: 0.4744 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5858.3 | uss_MB: 4950.6 | child_rss_MB: 5664.7 | allocated_MB: 493.1 | reserved_MB: 29592.0 | inactive_split_MB: 1080.0 +rank3 [2026-10-29 22:11:06,083] INFO : Step 38800/50000 rank3 | Train Loss: 0.8386 | Train Loss Unweighted: 0.4077 | Train Loss Components: Auxiliary: 0.1776 | Cardinality: 0.0905 | Classification: 0.0849 | Classification_binary: 0.1308 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0012 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0447 | Regression_eta: 0.0294 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0554 | LR: 2.07e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1585s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.7346 | max_loss: 0.1426 | max_backward: 1.8320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5857.6 | uss_MB: 4949.0 | child_rss_MB: 5646.6 | allocated_MB: 497.9 | reserved_MB: 29592.0 | inactive_split_MB: 1086.4 +rank3 [2026-10-29 22:15:54,682] INFO : Step 38900/50000 rank3 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1916 | Cardinality: 0.1035 | Classification: 0.0839 | Classification_binary: 0.1479 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0014 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1482 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0422 | Regression_eta: 0.0341 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0721 | LR: 2.05e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.8779s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0144 | max_forward: 2.1610 | max_loss: 0.2966 | max_backward: 2.8203 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4876 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 5858.5 | uss_MB: 4951.7 | child_rss_MB: 5647.2 | allocated_MB: 500.0 | reserved_MB: 29592.0 | inactive_split_MB: 849.6 +rank3 [2026-10-29 22:20:36,723] INFO : Step 39000/50000 rank3 | Train Loss: 0.8505 | Train Loss Unweighted: 0.4196 | Train Loss Components: Auxiliary: 0.1813 | Cardinality: 0.0941 | Classification: 0.0808 | Classification_binary: 0.1368 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0993 | Regression_eta: 0.0015 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0555 | Regression_eta: 0.0370 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0527 | LR: 2.03e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2843s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7392 | max_loss: 0.2445 | max_backward: 1.8584 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5844.6 | uss_MB: 4939.8 | child_rss_MB: 5638.4 | allocated_MB: 496.8 | reserved_MB: 29592.0 | inactive_split_MB: 1086.0 +rank3 [2026-10-29 22:20:37,630] INFO : Running validation on rank3 +rank3 [2026-10-29 22:20:37,630] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 22:20:37,630] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 22:21:31,242] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:21:31,759] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 22:21:58,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 22:21:59,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 22:22:01,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 22:22:02,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 22:22:12,056] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 22:22:12,679] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:22:13,174] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 22:22:29,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 22:22:30,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 22:22:31,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 22:22:32,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 22:22:42,751] INFO : Time taken to make predictions on device 3 is: 0.26 min +rank3 [2026-10-29 22:22:43,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 22:22:43,803] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 22:22:59,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 22:23:00,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 22:23:01,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 22:23:02,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 22:23:12,037] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 22:28:30,320] INFO : Step 39100/50000 rank3 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1110 | Classification: 0.0839 | Classification_binary: 0.1198 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.0972 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0441 | Regression_eta: 0.0210 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0345 | LR: 2.01e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5505s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7800 | max_loss: 0.2457 | max_backward: 1.9719 | max_optimizer_step: 0.3248 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 7010.9 | uss_MB: 6103.0 | child_rss_MB: 5608.2 | allocated_MB: 497.5 | reserved_MB: 24374.0 | inactive_split_MB: 923.6 +rank3 [2026-10-29 22:33:29,807] INFO : Step 39200/50000 rank3 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4242 | Train Loss Components: Auxiliary: 0.1863 | Cardinality: 0.1210 | Classification: 0.0912 | Classification_binary: 0.1382 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0418 | Regression_eta: 0.0376 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0593 | LR: 2.00e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2588s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0324 | max_forward: 0.7817 | max_loss: 0.1452 | max_backward: 1.9286 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5940.0 | uss_MB: 5036.4 | child_rss_MB: 5609.8 | allocated_MB: 497.9 | reserved_MB: 26338.0 | inactive_split_MB: 912.4 +rank3 [2026-10-29 22:38:28,991] INFO : Step 39300/50000 rank3 | Train Loss: 0.9298 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1080 | Classification: 0.0882 | Classification_binary: 0.1594 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0951 | Regression_eta: 0.0013 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1591 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0513 | Regression_eta: 0.0327 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0539 | LR: 1.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.1934s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7922 | max_loss: 0.2572 | max_backward: 1.8558 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 5927.1 | uss_MB: 5036.1 | child_rss_MB: 5613.3 | allocated_MB: 499.4 | reserved_MB: 26338.0 | inactive_split_MB: 1408.0 +rank3 [2026-10-29 22:43:21,466] INFO : Step 39400/50000 rank3 | Train Loss: 0.8421 | Train Loss Unweighted: 0.3964 | Train Loss Components: Auxiliary: 0.1755 | Cardinality: 0.1006 | Classification: 0.0679 | Classification_binary: 0.1346 | HitGrouping: 0.1534 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1144 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0516 | Regression_eta: 0.0306 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0296 | LR: 1.96e-04 | DataLoad Time: 0.0188s | Model Forward Time: 6.2906s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1084 | max_forward: 1.1740 | max_loss: 0.3046 | max_backward: 2.6778 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8613 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5928.1 | uss_MB: 4351.2 | child_rss_MB: 5609.1 | allocated_MB: 495.0 | reserved_MB: 26338.0 | inactive_split_MB: 1095.6 +rank3 [2026-10-29 22:48:17,291] INFO : Step 39500/50000 rank3 | Train Loss: 0.8539 | Train Loss Unweighted: 0.4179 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0879 | Classification: 0.0889 | Classification_binary: 0.1243 | HitGrouping: 0.1565 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1168 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0515 | Regression_eta: 0.0646 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0372 | LR: 1.94e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6175s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7373 | max_loss: 0.1406 | max_backward: 1.8719 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.8502 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5922.7 | uss_MB: 5037.5 | child_rss_MB: 5598.5 | allocated_MB: 492.8 | reserved_MB: 28318.0 | inactive_split_MB: 1078.3 +rank3 [2026-10-29 22:53:41,207] INFO : Step 39600/50000 rank3 | Train Loss: 0.9371 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1161 | Classification: 0.0813 | Classification_binary: 0.1596 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1039 | Regression_eta: 0.0013 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0407 | Regression_eta: 0.0372 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0491 | LR: 1.93e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.3251s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0173 | max_forward: 1.4879 | max_loss: 0.2520 | max_backward: 1.9863 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.8346 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5921.1 | uss_MB: 5038.4 | child_rss_MB: 5602.9 | allocated_MB: 496.0 | reserved_MB: 28318.0 | inactive_split_MB: 858.4 +rank3 [2026-10-29 22:58:48,059] INFO : Step 39700/50000 rank3 | Train Loss: 0.9204 | Train Loss Unweighted: 0.4427 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1101 | Classification: 0.0841 | Classification_binary: 0.1567 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0997 | Regression_eta: 0.0013 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0594 | Regression_eta: 0.0331 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0587 | LR: 1.91e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.2777s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.7571 | max_loss: 0.4946 | max_backward: 1.9147 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 5909.3 | uss_MB: 5037.4 | child_rss_MB: 5602.2 | allocated_MB: 497.0 | reserved_MB: 30324.0 | inactive_split_MB: 1175.5 +rank3 [2026-10-29 23:04:38,179] INFO : Step 39800/50000 rank3 | Train Loss: 0.8155 | Train Loss Unweighted: 0.3780 | Train Loss Components: Auxiliary: 0.1674 | Cardinality: 0.0963 | Classification: 0.0643 | Classification_binary: 0.1275 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0308 | Classification_binary: 0.1106 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0460 | Regression_eta: 0.0243 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0427 | LR: 1.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0234s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7197 | max_loss: 0.1489 | max_backward: 1.6690 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.4760 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1068 | Memory: rss_MB: 5897.7 | uss_MB: 5034.7 | child_rss_MB: 5594.3 | allocated_MB: 494.4 | reserved_MB: 30324.0 | inactive_split_MB: 897.3 +rank3 [2026-10-29 23:09:43,446] INFO : Step 39900/50000 rank3 | Train Loss: 0.8844 | Train Loss Unweighted: 0.3998 | Train Loss Components: Auxiliary: 0.1871 | Cardinality: 0.1077 | Classification: 0.0758 | Classification_binary: 0.1443 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0883 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1478 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0513 | Regression_eta: 0.0370 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0565 | LR: 1.88e-04 | DataLoad Time: 0.0120s | Model Forward Time: 4.9911s | Phase Times: max_pre_step_barrier: 0.0268 | max_batch_to_device: 0.0727 | max_forward: 1.7016 | max_loss: 0.4968 | max_backward: 2.0829 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.4776 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5898.5 | uss_MB: 5037.5 | child_rss_MB: 5601.6 | allocated_MB: 496.1 | reserved_MB: 30324.0 | inactive_split_MB: 1077.6 +rank3 [2026-10-29 23:14:38,166] INFO : Step 40000/50000 rank3 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1170 | Classification: 0.0871 | Classification_binary: 0.1550 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0551 | Regression_eta: 0.0250 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0400 | LR: 1.86e-04 | DataLoad Time: 0.0134s | Model Forward Time: 5.4268s | Phase Times: max_pre_step_barrier: 0.1084 | max_batch_to_device: 0.0070 | max_forward: 1.1725 | max_loss: 0.2555 | max_backward: 2.4297 | max_optimizer_step: 0.2969 | max_tensorboard_logging: 0.6818 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5889.3 | uss_MB: 5028.2 | child_rss_MB: 5597.1 | allocated_MB: 501.7 | reserved_MB: 30324.0 | inactive_split_MB: 1368.0 +rank3 [2026-10-29 23:14:39,118] INFO : Running validation on rank3 +rank3 [2026-10-29 23:14:39,118] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-29 23:14:39,118] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-29 23:15:31,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:15:32,369] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-29 23:16:03,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-29 23:16:04,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-29 23:16:06,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-29 23:16:08,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-29 23:16:18,035] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-29 23:16:28,294] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:16:28,835] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-29 23:16:45,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-29 23:16:46,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-29 23:16:47,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-29 23:16:48,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-29 23:16:58,189] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-29 23:16:58,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-29 23:16:59,026] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-29 23:17:14,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-29 23:17:14,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-29 23:17:15,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-29 23:17:16,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-29 23:17:26,724] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-29 23:22:41,683] INFO : Step 40100/50000 rank3 | Train Loss: 0.8115 | Train Loss Unweighted: 0.3794 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.1028 | Classification: 0.0708 | Classification_binary: 0.1239 | HitGrouping: 0.1460 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0909 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1131 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0418 | Regression_eta: 0.0306 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0437 | LR: 1.84e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0768s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.1076 | max_forward: 0.8426 | max_loss: 0.1437 | max_backward: 1.8847 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 7273.7 | uss_MB: 5477.4 | child_rss_MB: 5617.3 | allocated_MB: 493.6 | reserved_MB: 28004.0 | inactive_split_MB: 975.8 +rank3 [2026-10-29 23:27:53,247] INFO : Step 40200/50000 rank3 | Train Loss: 0.8264 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1679 | Cardinality: 0.0984 | Classification: 0.0703 | Classification_binary: 0.1210 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0913 | Regression_eta: 0.0013 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1272 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0523 | Regression_eta: 0.0326 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0671 | LR: 1.83e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8933s | Phase Times: max_pre_step_barrier: 0.0270 | max_batch_to_device: 0.0062 | max_forward: 0.8324 | max_loss: 0.1802 | max_backward: 2.0569 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0152 | max_post_step_barrier: 0.0867 | Memory: rss_MB: 5955.5 | uss_MB: 5094.1 | child_rss_MB: 5625.1 | allocated_MB: 497.1 | reserved_MB: 28008.0 | inactive_split_MB: 935.5 +rank3 [2026-10-29 23:33:22,807] INFO : Step 40300/50000 rank3 | Train Loss: 0.9351 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1181 | Classification: 0.0959 | Classification_binary: 0.1652 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0014 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0562 | Regression_eta: 0.0400 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0732 | LR: 1.81e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.3105s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8191 | max_loss: 0.1494 | max_backward: 1.9817 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 0.9693 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 5955.1 | uss_MB: 5096.4 | child_rss_MB: 5610.7 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 927.5 +rank3 [2026-10-29 23:38:38,678] INFO : Step 40400/50000 rank3 | Train Loss: 0.9379 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1359 | Classification: 0.0771 | Classification_binary: 0.1554 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0944 | Regression_eta: 0.0014 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0351 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0596 | Regression_eta: 0.0398 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0579 | LR: 1.79e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6741s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0500 | max_forward: 0.8111 | max_loss: 0.5067 | max_backward: 1.8144 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5951.4 | uss_MB: 5092.9 | child_rss_MB: 5616.0 | allocated_MB: 497.0 | reserved_MB: 28008.0 | inactive_split_MB: 931.6 +rank3 [2026-10-29 23:43:43,799] INFO : Step 40500/50000 rank3 | Train Loss: 0.8053 | Train Loss Unweighted: 0.3782 | Train Loss Components: Auxiliary: 0.1649 | Cardinality: 0.0788 | Classification: 0.0749 | Classification_binary: 0.1246 | HitGrouping: 0.1502 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1048 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0539 | Regression_eta: 0.0291 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0526 | LR: 1.78e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3971s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.8420 | max_loss: 0.1414 | max_backward: 1.5396 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.9215 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5948.4 | uss_MB: 5092.0 | child_rss_MB: 5597.4 | allocated_MB: 494.5 | reserved_MB: 28012.0 | inactive_split_MB: 979.3 +rank3 [2026-10-29 23:48:43,695] INFO : Step 40600/50000 rank3 | Train Loss: 0.8980 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1108 | Classification: 0.0752 | Classification_binary: 0.1417 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0955 | Regression_eta: 0.0015 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1584 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0647 | Regression_eta: 0.0528 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0438 | LR: 1.76e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3177s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7855 | max_loss: 0.1488 | max_backward: 1.8742 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 5948.2 | uss_MB: 5094.3 | child_rss_MB: 5593.1 | allocated_MB: 499.7 | reserved_MB: 28012.0 | inactive_split_MB: 1102.0 +rank3 [2026-10-29 23:53:39,580] INFO : Step 40700/50000 rank3 | Train Loss: 0.9220 | Train Loss Unweighted: 0.4304 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1148 | Classification: 0.0767 | Classification_binary: 0.1645 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1764 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0492 | Regression_eta: 0.0274 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0634 | LR: 1.75e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5451s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.8091 | max_loss: 0.2475 | max_backward: 2.0011 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.5972 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5949.6 | uss_MB: 5096.1 | child_rss_MB: 5593.6 | allocated_MB: 498.9 | reserved_MB: 28012.0 | inactive_split_MB: 776.0 +rank3 [2026-10-30 00:01:00,041] INFO : Step 40800/50000 rank3 | Train Loss: 0.8642 | Train Loss Unweighted: 0.4082 | Train Loss Components: Auxiliary: 0.1815 | Cardinality: 0.1013 | Classification: 0.0834 | Classification_binary: 0.1260 | HitGrouping: 0.1507 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0536 | Regression_eta: 0.0407 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0451 | LR: 1.73e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.1137s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7033 | max_loss: 0.1373 | max_backward: 1.5042 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1068 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5935.0 | uss_MB: 4327.0 | child_rss_MB: 5586.6 | allocated_MB: 493.1 | reserved_MB: 28012.0 | inactive_split_MB: 897.3 +rank3 [2026-10-30 00:06:14,592] INFO : Step 40900/50000 rank3 | Train Loss: 0.8749 | Train Loss Unweighted: 0.4038 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1070 | Classification: 0.0854 | Classification_binary: 0.1300 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0923 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0508 | Regression_eta: 0.0445 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0648 | LR: 1.72e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5723s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7515 | max_loss: 0.2142 | max_backward: 1.7950 | max_optimizer_step: 0.4960 | max_tensorboard_logging: 0.9322 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5926.9 | uss_MB: 5094.2 | child_rss_MB: 5563.3 | allocated_MB: 497.9 | reserved_MB: 30012.0 | inactive_split_MB: 893.0 +rank3 [2026-10-30 00:11:07,451] INFO : Step 41000/50000 rank3 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.0956 | Classification: 0.0893 | Classification_binary: 0.1497 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0014 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0490 | Regression_eta: 0.0373 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0617 | LR: 1.70e-04 | DataLoad Time: 0.0120s | Model Forward Time: 5.0994s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0071 | max_forward: 1.1963 | max_loss: 0.1519 | max_backward: 2.0474 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.0118 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 5918.6 | uss_MB: 5085.3 | child_rss_MB: 5547.4 | allocated_MB: 498.7 | reserved_MB: 30012.0 | inactive_split_MB: 1108.0 +rank3 [2026-10-30 00:11:08,373] INFO : Running validation on rank3 +rank3 [2026-10-30 00:11:08,373] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 00:11:08,373] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 00:11:59,247] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:12:00,024] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 00:12:25,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 00:12:26,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 00:12:27,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 00:12:28,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 00:12:38,945] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 00:12:39,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:12:40,281] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 00:12:56,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 00:12:57,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 00:12:58,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 00:12:59,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 00:13:09,117] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 00:13:09,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 00:13:10,647] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 00:13:25,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 00:13:27,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 00:13:27,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 00:13:28,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 00:13:38,780] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 00:19:15,373] INFO : Step 41100/50000 rank3 | Train Loss: 0.9430 | Train Loss Unweighted: 0.4319 | Train Loss Components: Auxiliary: 0.1995 | Cardinality: 0.1330 | Classification: 0.0721 | Classification_binary: 0.1579 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0013 | Regression_pt: 0.0998 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1678 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0517 | Regression_eta: 0.0423 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0670 | LR: 1.69e-04 | DataLoad Time: 0.0157s | Model Forward Time: 6.3662s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8221 | max_loss: 0.4800 | max_backward: 2.2615 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7242.0 | uss_MB: 5759.2 | child_rss_MB: 5549.6 | allocated_MB: 497.6 | reserved_MB: 23400.0 | inactive_split_MB: 991.8 +rank3 [2026-10-30 00:24:27,079] INFO : Step 41200/50000 rank3 | Train Loss: 0.8544 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1097 | Classification: 0.0734 | Classification_binary: 0.1131 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1015 | Regression_eta: 0.0015 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1229 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0477 | Regression_eta: 0.0658 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0557 | LR: 1.67e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2545s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7030 | max_loss: 0.2408 | max_backward: 1.7750 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5316 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6039.9 | uss_MB: 5208.8 | child_rss_MB: 5554.6 | allocated_MB: 496.1 | reserved_MB: 27308.0 | inactive_split_MB: 984.0 +rank3 [2026-10-30 00:29:20,952] INFO : Step 41300/50000 rank3 | Train Loss: 0.8221 | Train Loss Unweighted: 0.3852 | Train Loss Components: Auxiliary: 0.1744 | Cardinality: 0.0906 | Classification: 0.0754 | Classification_binary: 0.1369 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0849 | Regression_eta: 0.0012 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0586 | Regression_energy: 0.0581 | Regression_eta: 0.0258 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0342 | LR: 1.66e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4683s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8083 | max_loss: 0.1794 | max_backward: 1.9993 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.5906 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6040.0 | uss_MB: 5213.3 | child_rss_MB: 5551.2 | allocated_MB: 497.0 | reserved_MB: 27308.0 | inactive_split_MB: 1229.0 +rank3 [2026-10-30 00:34:25,620] INFO : Step 41400/50000 rank3 | Train Loss: 0.9216 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1171 | Classification: 0.0901 | Classification_binary: 0.1523 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1010 | Regression_eta: 0.0011 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1738 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0480 | Regression_eta: 0.0393 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0609 | LR: 1.64e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6355s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0065 | max_forward: 0.7831 | max_loss: 0.4937 | max_backward: 1.9271 | max_optimizer_step: 0.3071 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6033.3 | uss_MB: 5208.0 | child_rss_MB: 5553.7 | allocated_MB: 498.4 | reserved_MB: 29288.0 | inactive_split_MB: 1180.4 +rank3 [2026-10-30 00:39:29,959] INFO : Step 41500/50000 rank3 | Train Loss: 0.8584 | Train Loss Unweighted: 0.4111 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1030 | Classification: 0.0769 | Classification_binary: 0.1210 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0294 | Classification_binary: 0.1107 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0557 | Regression_eta: 0.0308 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0392 | LR: 1.63e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.0407s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.1908 | max_loss: 0.3017 | max_backward: 3.1310 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 6009.7 | uss_MB: 5150.0 | child_rss_MB: 5552.7 | allocated_MB: 494.5 | reserved_MB: 29288.0 | inactive_split_MB: 976.9 +rank3 [2026-10-30 00:44:19,403] INFO : Step 41600/50000 rank3 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4026 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1032 | Classification: 0.0762 | Classification_binary: 0.1246 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0992 | Regression_eta: 0.0013 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1021 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0487 | Regression_eta: 0.0404 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0602 | LR: 1.61e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.9946s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0065 | max_forward: 0.7279 | max_loss: 0.2339 | max_backward: 2.5338 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.4866 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1404 | Memory: rss_MB: 6001.6 | uss_MB: 5206.5 | child_rss_MB: 5546.1 | allocated_MB: 496.4 | reserved_MB: 29288.0 | inactive_split_MB: 761.6 +rank3 [2026-10-30 00:49:10,876] INFO : Step 41700/50000 rank3 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1941 | Cardinality: 0.1112 | Classification: 0.0767 | Classification_binary: 0.1493 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0988 | Regression_eta: 0.0013 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0394 | Regression_eta: 0.0275 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0789 | LR: 1.60e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5062s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7998 | max_loss: 0.1827 | max_backward: 1.9774 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6004.5 | uss_MB: 5210.8 | child_rss_MB: 5548.7 | allocated_MB: 497.0 | reserved_MB: 29288.0 | inactive_split_MB: 1220.0 +rank3 [2026-10-30 00:53:58,971] INFO : Step 41800/50000 rank3 | Train Loss: 0.9125 | Train Loss Unweighted: 0.4285 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.1148 | Classification: 0.0784 | Classification_binary: 0.1537 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1302 | Regression_cos_phi: 0.0432 | Regression_energy: 0.0509 | Regression_eta: 0.0310 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0620 | LR: 1.58e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8246s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7952 | max_loss: 0.1582 | max_backward: 1.8899 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9392 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1143 | Memory: rss_MB: 5984.7 | uss_MB: 5200.7 | child_rss_MB: 5542.4 | allocated_MB: 499.3 | reserved_MB: 31288.0 | inactive_split_MB: 932.9 +rank3 [2026-10-30 00:59:00,784] INFO : Step 41900/50000 rank3 | Train Loss: 0.8303 | Train Loss Unweighted: 0.3927 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1132 | Classification: 0.0782 | Classification_binary: 0.1167 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0010 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1092 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0583 | Regression_eta: 0.0252 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0380 | LR: 1.57e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1523s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7413 | max_loss: 0.1970 | max_backward: 1.7614 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5208 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5992.5 | uss_MB: 5207.2 | child_rss_MB: 5522.2 | allocated_MB: 497.1 | reserved_MB: 31288.0 | inactive_split_MB: 1188.0 +rank3 [2026-10-30 01:03:51,616] INFO : Step 42000/50000 rank3 | Train Loss: 0.8724 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1847 | Cardinality: 0.0893 | Classification: 0.0874 | Classification_binary: 0.1377 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0958 | Regression_eta: 0.0012 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1350 | Regression_cos_phi: 0.0672 | Regression_energy: 0.0532 | Regression_eta: 0.0344 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0480 | LR: 1.56e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7567s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1147 | max_forward: 0.8308 | max_loss: 0.1392 | max_backward: 1.8336 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5996.6 | uss_MB: 5209.4 | child_rss_MB: 5540.2 | allocated_MB: 496.3 | reserved_MB: 31288.0 | inactive_split_MB: 930.5 +rank3 [2026-10-30 01:03:52,597] INFO : Running validation on rank3 +rank3 [2026-10-30 01:03:52,597] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 01:03:52,597] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 01:04:46,128] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:04:46,673] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 01:05:13,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 01:05:14,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 01:05:15,826] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 01:05:16,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 01:05:26,861] INFO : Time taken to make predictions on device 3 is: 0.27 min +rank3 [2026-10-30 01:05:27,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:05:27,884] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 01:05:44,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 01:05:45,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 01:05:46,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 01:05:47,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 01:05:57,139] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 01:05:57,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:05:57,664] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 01:06:13,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 01:06:14,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 01:06:15,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 01:06:16,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 01:06:26,292] INFO : Time taken to make predictions on device 3 is: 0.24 min +rank3 [2026-10-30 01:11:42,851] INFO : Step 42100/50000 rank3 | Train Loss: 0.9390 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1200 | Classification: 0.0884 | Classification_binary: 0.1619 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0926 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1715 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0487 | Regression_eta: 0.0430 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0576 | LR: 1.54e-04 | DataLoad Time: 0.0173s | Model Forward Time: 6.0859s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0976 | max_forward: 1.0189 | max_loss: 0.3859 | max_backward: 2.5922 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 1.0368 | max_tensorboard_flush: 0.0149 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7501.8 | uss_MB: 4866.0 | child_rss_MB: 5544.9 | allocated_MB: 497.0 | reserved_MB: 25850.0 | inactive_split_MB: 892.8 +rank3 [2026-10-30 01:16:53,956] INFO : Step 42200/50000 rank3 | Train Loss: 0.9057 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1144 | Classification: 0.0879 | Classification_binary: 0.1381 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0572 | Regression_eta: 0.0381 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0739 | LR: 1.53e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.1527s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7457 | max_loss: 0.1999 | max_backward: 1.6824 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.5916 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6096.0 | uss_MB: 5306.7 | child_rss_MB: 5537.7 | allocated_MB: 497.3 | reserved_MB: 27866.0 | inactive_split_MB: 959.8 +rank3 [2026-10-30 01:18:06,041] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank3 [2026-10-30 01:18:06,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank3 [2026-10-30 01:21:57,454] INFO : Step 42300/50000 rank3 | Train Loss: 0.9052 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1933 | Cardinality: 0.1127 | Classification: 0.0787 | Classification_binary: 0.1569 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0868 | Regression_eta: 0.0013 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0389 | Regression_eta: 0.0613 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0550 | LR: 1.52e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.1177s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.9764 | max_loss: 0.1904 | max_backward: 2.7671 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.9997 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6118.1 | uss_MB: 5325.9 | child_rss_MB: 5544.6 | allocated_MB: 499.1 | reserved_MB: 27872.0 | inactive_split_MB: 1033.6 +rank3 [2026-10-30 01:26:55,612] INFO : Step 42400/50000 rank3 | Train Loss: 0.9358 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.1976 | Cardinality: 0.1220 | Classification: 0.0815 | Classification_binary: 0.1533 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0514 | Regression_eta: 0.0282 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0477 | LR: 1.50e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.8879s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8701 | max_loss: 0.1465 | max_backward: 1.9416 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.8192 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6099.3 | uss_MB: 5308.3 | child_rss_MB: 5546.3 | allocated_MB: 499.8 | reserved_MB: 29908.0 | inactive_split_MB: 939.2 +rank3 [2026-10-30 01:31:49,012] INFO : Step 42500/50000 rank3 | Train Loss: 0.8126 | Train Loss Unweighted: 0.3791 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0975 | Classification: 0.0765 | Classification_binary: 0.1172 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0918 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0590 | Regression_eta: 0.0444 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0412 | LR: 1.49e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.5922s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0858 | max_forward: 0.8156 | max_loss: 0.1407 | max_backward: 1.8570 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7997 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6101.2 | uss_MB: 5308.6 | child_rss_MB: 5538.8 | allocated_MB: 497.5 | reserved_MB: 29908.0 | inactive_split_MB: 943.4 +rank3 [2026-10-30 01:36:44,505] INFO : Step 42600/50000 rank3 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0772 | Classification_binary: 0.1347 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0012 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1169 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0379 | Regression_eta: 0.0292 | Regression_pt: 0.0403 | Regression_sin_phi: 0.0505 | LR: 1.48e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.0219s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7519 | max_loss: 0.1393 | max_backward: 1.8021 | max_optimizer_step: 0.3023 | max_tensorboard_logging: 0.7973 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6101.5 | uss_MB: 4501.2 | child_rss_MB: 5544.2 | allocated_MB: 497.0 | reserved_MB: 29908.0 | inactive_split_MB: 877.2 +rank3 [2026-10-30 01:41:48,354] INFO : Step 42700/50000 rank3 | Train Loss: 0.9129 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1178 | Classification: 0.0751 | Classification_binary: 0.1632 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0858 | Regression_eta: 0.0014 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1545 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0467 | Regression_eta: 0.0380 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0516 | LR: 1.47e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6157s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 1.0869 | max_loss: 0.1547 | max_backward: 1.6854 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7949 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6101.6 | uss_MB: 5308.9 | child_rss_MB: 5545.8 | allocated_MB: 497.3 | reserved_MB: 29908.0 | inactive_split_MB: 953.4 +rank3 [2026-10-30 01:46:39,116] INFO : Step 42800/50000 rank3 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1179 | Classification: 0.0767 | Classification_binary: 0.1362 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0013 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1296 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0494 | Regression_eta: 0.0358 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0706 | LR: 1.45e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3529s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8438 | max_loss: 0.1478 | max_backward: 1.6710 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.6821 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6089.4 | uss_MB: 5295.4 | child_rss_MB: 5546.6 | allocated_MB: 499.0 | reserved_MB: 30126.0 | inactive_split_MB: 875.7 +rank3 [2026-10-30 01:51:40,828] INFO : Step 42900/50000 rank3 | Train Loss: 0.7857 | Train Loss Unweighted: 0.3758 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0951 | Classification: 0.0820 | Classification_binary: 0.1113 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0898 | Regression_eta: 0.0010 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1074 | Regression_cos_phi: 0.0312 | Regression_energy: 0.0478 | Regression_eta: 0.0309 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0433 | LR: 1.44e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.9230s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8792 | max_loss: 0.1461 | max_backward: 1.9928 | max_optimizer_step: 0.1903 | max_tensorboard_logging: 0.9695 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6098.2 | uss_MB: 5308.5 | child_rss_MB: 5544.6 | allocated_MB: 496.5 | reserved_MB: 30126.0 | inactive_split_MB: 951.8 +rank3 [2026-10-30 01:56:37,111] INFO : Step 43000/50000 rank3 | Train Loss: 0.8899 | Train Loss Unweighted: 0.4207 | Train Loss Components: Auxiliary: 0.1911 | Cardinality: 0.1035 | Classification: 0.0847 | Classification_binary: 0.1529 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0447 | Regression_eta: 0.0425 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0533 | LR: 1.43e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7367s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1951 | max_backward: 1.9694 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.8418 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6086.1 | uss_MB: 5311.0 | child_rss_MB: 5508.6 | allocated_MB: 495.5 | reserved_MB: 30126.0 | inactive_split_MB: 1120.4 +rank3 [2026-10-30 01:56:38,136] INFO : Running validation on rank3 +rank3 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 01:57:33,184] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:57:33,675] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 01:57:59,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 01:58:00,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 01:58:01,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 01:58:02,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 01:58:12,594] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 01:58:13,038] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:58:13,512] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 01:58:29,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 01:58:30,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 01:58:31,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 01:58:32,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 01:58:42,096] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 01:58:43,304] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 01:58:43,756] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 01:58:59,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 01:58:59,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 01:59:00,768] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 01:59:01,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 01:59:11,677] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 02:04:24,630] INFO : Step 43100/50000 rank3 | Train Loss: 0.8965 | Train Loss Unweighted: 0.4137 | Train Loss Components: Auxiliary: 0.1901 | Cardinality: 0.1074 | Classification: 0.0705 | Classification_binary: 0.1472 | HitGrouping: 0.1435 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0357 | Regression_energy: 0.0644 | Regression_eta: 0.0470 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0567 | LR: 1.42e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.2202s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.2396 | max_backward: 1.9954 | max_optimizer_step: 0.3209 | max_tensorboard_logging: 1.0803 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7631.1 | uss_MB: 6853.7 | child_rss_MB: 5519.2 | allocated_MB: 496.0 | reserved_MB: 26194.0 | inactive_split_MB: 843.4 +rank3 [2026-10-30 02:09:06,731] INFO : Step 43200/50000 rank3 | Train Loss: 0.8164 | Train Loss Unweighted: 0.3744 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0957 | Classification: 0.0763 | Classification_binary: 0.1197 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1073 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0568 | Regression_eta: 0.0254 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0450 | LR: 1.40e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4253s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7823 | max_loss: 0.4822 | max_backward: 1.6766 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6123.7 | uss_MB: 5349.6 | child_rss_MB: 5511.0 | allocated_MB: 495.8 | reserved_MB: 26686.0 | inactive_split_MB: 848.9 +rank3 [2026-10-30 02:13:48,743] INFO : Step 43300/50000 rank3 | Train Loss: 0.9151 | Train Loss Unweighted: 0.4260 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1202 | Classification: 0.0906 | Classification_binary: 0.1467 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0013 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0447 | Regression_eta: 0.0535 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0800 | LR: 1.39e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.7253s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0155 | max_forward: 0.8938 | max_loss: 0.1464 | max_backward: 2.1010 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.5477 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1243 | Memory: rss_MB: 6097.4 | uss_MB: 5352.9 | child_rss_MB: 5427.7 | allocated_MB: 497.9 | reserved_MB: 28278.0 | inactive_split_MB: 1309.2 +rank3 [2026-10-30 02:18:30,112] INFO : Step 43400/50000 rank3 | Train Loss: 0.9723 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2082 | Cardinality: 0.1222 | Classification: 0.0880 | Classification_binary: 0.1684 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0015 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1966 | Regression_cos_phi: 0.0833 | Regression_energy: 0.0530 | Regression_eta: 0.0438 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0958 | LR: 1.38e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4354s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.5043 | max_backward: 2.0661 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.4824 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6077.9 | uss_MB: 5350.3 | child_rss_MB: 5335.1 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 851.9 +rank3 [2026-10-30 02:23:12,892] INFO : Step 43500/50000 rank3 | Train Loss: 0.8402 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.0947 | Classification: 0.0776 | Classification_binary: 0.1278 | HitGrouping: 0.1501 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0999 | Regression_eta: 0.0012 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1130 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0695 | Regression_eta: 0.0256 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0493 | LR: 1.37e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8632s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8086 | max_loss: 0.1489 | max_backward: 2.0887 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.7978 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1094 | Memory: rss_MB: 6077.6 | uss_MB: 5352.8 | child_rss_MB: 5340.6 | allocated_MB: 498.6 | reserved_MB: 28278.0 | inactive_split_MB: 909.7 +rank3 [2026-10-30 02:28:09,613] INFO : Step 43600/50000 rank3 | Train Loss: 0.7985 | Train Loss Unweighted: 0.3653 | Train Loss Components: Auxiliary: 0.1621 | Cardinality: 0.0887 | Classification: 0.0728 | Classification_binary: 0.1184 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0851 | Regression_eta: 0.0013 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0411 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0482 | Regression_eta: 0.0391 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0386 | LR: 1.36e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.2494s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0170 | max_forward: 1.2777 | max_loss: 0.2064 | max_backward: 1.9835 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 1.1084 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6044.0 | uss_MB: 4957.3 | child_rss_MB: 5260.0 | allocated_MB: 493.5 | reserved_MB: 28278.0 | inactive_split_MB: 1313.2 +rank3 [2026-10-30 02:33:07,295] INFO : Step 43700/50000 rank3 | Train Loss: 0.8599 | Train Loss Unweighted: 0.3884 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1089 | Classification: 0.0797 | Classification_binary: 0.1434 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0812 | Regression_eta: 0.0013 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1315 | Regression_cos_phi: 0.0405 | Regression_energy: 0.0362 | Regression_eta: 0.0270 | Regression_pt: 0.0391 | Regression_sin_phi: 0.0467 | LR: 1.35e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6055s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7902 | max_loss: 0.1752 | max_backward: 1.8774 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8044 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6046.0 | uss_MB: 5350.4 | child_rss_MB: 5241.5 | allocated_MB: 498.0 | reserved_MB: 28278.0 | inactive_split_MB: 1046.0 +rank3 [2026-10-30 02:37:45,726] INFO : Step 43800/50000 rank3 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1875 | Cardinality: 0.1049 | Classification: 0.0814 | Classification_binary: 0.1423 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0907 | Regression_eta: 0.0012 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1366 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0387 | Regression_eta: 0.0348 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0380 | LR: 1.34e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5743s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7940 | max_loss: 0.4805 | max_backward: 1.8159 | max_optimizer_step: 0.2941 | max_tensorboard_logging: 0.7850 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6040.0 | uss_MB: 5349.3 | child_rss_MB: 5218.8 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 839.4 +rank3 [2026-10-30 02:42:41,735] INFO : Step 43900/50000 rank3 | Train Loss: 0.7987 | Train Loss Unweighted: 0.3885 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.0999 | Classification: 0.0753 | Classification_binary: 0.1166 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0968 | Regression_eta: 0.0010 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1368 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0438 | Regression_eta: 0.0252 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0477 | LR: 1.33e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.5932s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0846 | max_forward: 0.8891 | max_loss: 0.1768 | max_backward: 1.7357 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7516 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6030.6 | uss_MB: 5347.7 | child_rss_MB: 5179.6 | allocated_MB: 494.5 | reserved_MB: 28278.0 | inactive_split_MB: 1129.5 +rank3 [2026-10-30 02:47:33,827] INFO : Step 44000/50000 rank3 | Train Loss: 0.8263 | Train Loss Unweighted: 0.3926 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.1027 | Classification: 0.0840 | Classification_binary: 0.1318 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0868 | Regression_eta: 0.0011 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1591 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0504 | Regression_eta: 0.0367 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0562 | LR: 1.32e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4526s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8816 | max_loss: 0.1426 | max_backward: 1.9616 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.4846 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6020.6 | uss_MB: 5352.2 | child_rss_MB: 5134.9 | allocated_MB: 496.3 | reserved_MB: 28762.0 | inactive_split_MB: 1307.2 +rank3 [2026-10-30 02:47:34,860] INFO : Running validation on rank3 +rank3 [2026-10-30 02:47:34,860] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 02:47:34,860] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 02:48:28,156] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 02:48:28,669] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 02:48:55,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 02:48:56,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 02:48:57,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 02:48:59,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 02:49:09,044] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 02:49:09,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 02:49:10,103] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 02:49:25,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 02:49:26,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 02:49:27,668] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 02:49:28,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 02:49:38,652] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 02:49:39,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 02:49:40,181] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 02:49:55,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 02:49:56,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 02:49:57,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 02:49:58,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 02:50:08,216] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 02:55:20,175] INFO : Step 44100/50000 rank3 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1986 | Cardinality: 0.1334 | Classification: 0.0764 | Classification_binary: 0.1516 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0412 | Regression_energy: 0.0456 | Regression_eta: 0.0377 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0628 | LR: 1.31e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.1235s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0079 | max_forward: 0.8221 | max_loss: 0.1579 | max_backward: 2.0192 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 1.0889 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7539.1 | uss_MB: 6869.4 | child_rss_MB: 5111.3 | allocated_MB: 498.6 | reserved_MB: 25920.0 | inactive_split_MB: 1073.8 +rank3 [2026-10-30 03:00:12,362] INFO : Step 44200/50000 rank3 | Train Loss: 0.9018 | Train Loss Unweighted: 0.4273 | Train Loss Components: Auxiliary: 0.1909 | Cardinality: 0.1109 | Classification: 0.0847 | Classification_binary: 0.1354 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0012 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1463 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0601 | Regression_eta: 0.0287 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0491 | LR: 1.30e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3583s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.1013 | max_forward: 0.8608 | max_loss: 0.1485 | max_backward: 1.6010 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 6119.2 | uss_MB: 5452.1 | child_rss_MB: 5083.8 | allocated_MB: 496.1 | reserved_MB: 27956.0 | inactive_split_MB: 1248.1 +rank3 [2026-10-30 03:04:47,144] INFO : Step 44300/50000 rank3 | Train Loss: 0.8239 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1722 | Cardinality: 0.1007 | Classification: 0.0789 | Classification_binary: 0.1118 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1244 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0472 | Regression_eta: 0.0555 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0957 | LR: 1.29e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.3171s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8120 | max_loss: 0.1377 | max_backward: 1.4700 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9055 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6110.2 | uss_MB: 5452.8 | child_rss_MB: 5031.6 | allocated_MB: 492.8 | reserved_MB: 27956.0 | inactive_split_MB: 1035.8 +rank3 [2026-10-30 03:09:18,055] INFO : Step 44400/50000 rank3 | Train Loss: 0.8683 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.0849 | Classification: 0.0836 | Classification_binary: 0.1500 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1347 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0496 | Regression_eta: 0.0289 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0735 | LR: 1.28e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9254s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8907 | max_loss: 0.1476 | max_backward: 2.0623 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6106.5 | uss_MB: 5454.6 | child_rss_MB: 5002.7 | allocated_MB: 499.5 | reserved_MB: 27956.0 | inactive_split_MB: 926.0 +rank3 [2026-10-30 03:14:01,913] INFO : Step 44500/50000 rank3 | Train Loss: 0.8672 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.0989 | Classification: 0.0738 | Classification_binary: 0.1455 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1237 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0473 | Regression_eta: 0.0270 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0417 | LR: 1.27e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2934s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0597 | max_forward: 0.8669 | max_loss: 0.1547 | max_backward: 1.7327 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.5628 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6105.7 | uss_MB: 5452.5 | child_rss_MB: 5004.4 | allocated_MB: 498.3 | reserved_MB: 27956.0 | inactive_split_MB: 860.5 +rank3 [2026-10-30 03:18:59,362] INFO : Step 44600/50000 rank3 | Train Loss: 0.7900 | Train Loss Unweighted: 0.3743 | Train Loss Components: Auxiliary: 0.1688 | Cardinality: 0.0956 | Classification: 0.0724 | Classification_binary: 0.1157 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0919 | Regression_eta: 0.0010 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1102 | Regression_cos_phi: 0.0384 | Regression_energy: 0.0429 | Regression_eta: 0.0243 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0569 | LR: 1.26e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5435s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0745 | max_forward: 0.7797 | max_loss: 0.1409 | max_backward: 1.9334 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6022 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6098.6 | uss_MB: 5447.3 | child_rss_MB: 4994.6 | allocated_MB: 494.5 | reserved_MB: 27958.0 | inactive_split_MB: 927.6 +rank3 [2026-10-30 03:23:44,789] INFO : Step 44700/50000 rank3 | Train Loss: 0.8669 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1878 | Cardinality: 0.1160 | Classification: 0.0768 | Classification_binary: 0.1380 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0863 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0577 | Regression_eta: 0.0341 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0655 | LR: 1.25e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.4946s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8629 | max_loss: 0.1449 | max_backward: 2.0281 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.5242 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6099.2 | uss_MB: 5434.3 | child_rss_MB: 5003.0 | allocated_MB: 498.4 | reserved_MB: 27958.0 | inactive_split_MB: 781.6 +rank3 [2026-10-30 03:28:42,238] INFO : Step 44800/50000 rank3 | Train Loss: 0.8585 | Train Loss Unweighted: 0.4003 | Train Loss Components: Auxiliary: 0.1869 | Cardinality: 0.1024 | Classification: 0.0769 | Classification_binary: 0.1550 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0823 | Regression_eta: 0.0011 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0466 | Regression_eta: 0.0362 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0708 | LR: 1.24e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8034s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0072 | max_forward: 0.7883 | max_loss: 0.5033 | max_backward: 1.9727 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6090.2 | uss_MB: 5455.2 | child_rss_MB: 4956.4 | allocated_MB: 496.5 | reserved_MB: 27958.0 | inactive_split_MB: 778.0 +rank3 [2026-10-30 03:33:12,492] INFO : Step 44900/50000 rank3 | Train Loss: 0.8358 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.1024 | Classification: 0.0797 | Classification_binary: 0.1294 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0859 | Regression_eta: 0.0011 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0468 | Regression_eta: 0.0373 | Regression_pt: 0.0515 | Regression_sin_phi: 0.0459 | LR: 1.23e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3689s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7360 | max_loss: 0.1482 | max_backward: 1.6886 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7825 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6084.1 | uss_MB: 5452.9 | child_rss_MB: 4956.9 | allocated_MB: 494.3 | reserved_MB: 27958.0 | inactive_split_MB: 922.0 +rank3 [2026-10-30 03:37:54,705] INFO : Step 45000/50000 rank3 | Train Loss: 0.8678 | Train Loss Unweighted: 0.3957 | Train Loss Components: Auxiliary: 0.1790 | Cardinality: 0.1036 | Classification: 0.0801 | Classification_binary: 0.1237 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0941 | Regression_eta: 0.0013 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1129 | Regression_cos_phi: 0.0299 | Regression_energy: 0.0514 | Regression_eta: 0.0191 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0422 | LR: 1.22e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5700s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7189 | max_loss: 0.2183 | max_backward: 1.8172 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.8812 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6058.9 | uss_MB: 5454.9 | child_rss_MB: 4844.5 | allocated_MB: 495.0 | reserved_MB: 27960.0 | inactive_split_MB: 978.5 +rank3 [2026-10-30 03:37:55,714] INFO : Running validation on rank3 +rank3 [2026-10-30 03:37:55,714] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 03:37:55,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 03:38:46,293] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 03:38:46,811] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 03:39:14,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 03:39:15,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 03:39:16,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 03:39:19,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 03:39:29,097] INFO : Time taken to make predictions on device 3 is: 0.28 min +rank3 [2026-10-30 03:39:29,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 03:39:30,322] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 03:39:46,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 03:39:47,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 03:39:48,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 03:39:49,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 03:39:59,532] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 03:40:00,144] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 03:40:00,673] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 03:40:16,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 03:40:17,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 03:40:18,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 03:40:18,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 03:40:29,002] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 03:45:39,355] INFO : Step 45100/50000 rank3 | Train Loss: 0.8712 | Train Loss Unweighted: 0.4129 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0997 | Classification: 0.0837 | Classification_binary: 0.1445 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0910 | Regression_eta: 0.0012 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1577 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0548 | Regression_eta: 0.0372 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0477 | LR: 1.21e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1923s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8448 | max_loss: 0.1523 | max_backward: 2.1445 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 1.0003 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7489.1 | uss_MB: 6883.8 | child_rss_MB: 4866.9 | allocated_MB: 499.4 | reserved_MB: 24746.0 | inactive_split_MB: 784.5 +rank3 [2026-10-30 03:50:46,377] INFO : Step 45200/50000 rank3 | Train Loss: 0.9130 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1058 | Classification: 0.0776 | Classification_binary: 0.1464 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0963 | Regression_eta: 0.0015 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.1477 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0503 | Regression_eta: 0.0242 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0540 | LR: 1.20e-04 | DataLoad Time: 0.0154s | Model Forward Time: 6.5432s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1574 | max_backward: 1.6440 | max_optimizer_step: 0.2111 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1112 | Memory: rss_MB: 6132.8 | uss_MB: 4597.7 | child_rss_MB: 4866.1 | allocated_MB: 499.7 | reserved_MB: 26730.0 | inactive_split_MB: 1073.5 +rank3 [2026-10-30 03:55:44,003] INFO : Step 45300/50000 rank3 | Train Loss: 0.8299 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.1126 | Classification: 0.0744 | Classification_binary: 0.1207 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.1052 | Regression_cos_phi: 0.0419 | Regression_energy: 0.0410 | Regression_eta: 0.0258 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0632 | LR: 1.19e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1815s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0827 | max_forward: 0.7966 | max_loss: 0.1432 | max_backward: 1.5141 | max_optimizer_step: 0.2646 | max_tensorboard_logging: 0.6448 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6128.3 | uss_MB: 5538.9 | child_rss_MB: 4864.4 | allocated_MB: 495.4 | reserved_MB: 27170.0 | inactive_split_MB: 881.8 +rank3 [2026-10-30 04:00:47,331] INFO : Step 45400/50000 rank3 | Train Loss: 0.8255 | Train Loss Unweighted: 0.4011 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.0984 | Classification: 0.0907 | Classification_binary: 0.1338 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0870 | Regression_eta: 0.0011 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0534 | Regression_eta: 0.0412 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0600 | LR: 1.19e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.3357s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7846 | max_loss: 0.1934 | max_backward: 1.9857 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 0.7880 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6124.9 | uss_MB: 4519.9 | child_rss_MB: 4862.5 | allocated_MB: 499.2 | reserved_MB: 27170.0 | inactive_split_MB: 989.3 +rank3 [2026-10-30 04:05:44,853] INFO : Step 45500/50000 rank3 | Train Loss: 0.8945 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1090 | Classification: 0.0825 | Classification_binary: 0.1567 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0012 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0565 | Regression_eta: 0.0320 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0671 | LR: 1.18e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4574s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8686 | max_loss: 0.1508 | max_backward: 1.9581 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.4909 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6130.5 | uss_MB: 5543.1 | child_rss_MB: 4864.9 | allocated_MB: 498.4 | reserved_MB: 28766.0 | inactive_split_MB: 883.2 +rank3 [2026-10-30 04:10:39,744] INFO : Step 45600/50000 rank3 | Train Loss: 0.8704 | Train Loss Unweighted: 0.4121 | Train Loss Components: Auxiliary: 0.1840 | Cardinality: 0.1126 | Classification: 0.0802 | Classification_binary: 0.1324 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0983 | Regression_eta: 0.0010 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0358 | Classification_binary: 0.1251 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0600 | Regression_eta: 0.0192 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0586 | LR: 1.17e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.8863s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0163 | max_forward: 0.8161 | max_loss: 0.2729 | max_backward: 2.1846 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6122.7 | uss_MB: 5488.3 | child_rss_MB: 4871.5 | allocated_MB: 495.6 | reserved_MB: 28766.0 | inactive_split_MB: 1080.4 +rank3 [2026-10-30 04:15:37,194] INFO : Step 45700/50000 rank3 | Train Loss: 0.8006 | Train Loss Unweighted: 0.3784 | Train Loss Components: Auxiliary: 0.1681 | Cardinality: 0.0873 | Classification: 0.0773 | Classification_binary: 0.1111 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0012 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1273 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0548 | Regression_eta: 0.0334 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0374 | LR: 1.16e-04 | DataLoad Time: 0.0109s | Model Forward Time: 4.8414s | Phase Times: max_pre_step_barrier: 0.0327 | max_batch_to_device: 0.0149 | max_forward: 1.1858 | max_loss: 0.1429 | max_backward: 1.6968 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1366 | Memory: rss_MB: 6126.5 | uss_MB: 5540.3 | child_rss_MB: 4873.0 | allocated_MB: 496.6 | reserved_MB: 28766.0 | inactive_split_MB: 903.7 +rank3 [2026-10-30 04:20:28,800] INFO : Step 45800/50000 rank3 | Train Loss: 0.8566 | Train Loss Unweighted: 0.3982 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1097 | Classification: 0.0794 | Classification_binary: 0.1398 | HitGrouping: 0.1406 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0428 | Regression_eta: 0.0281 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0466 | LR: 1.16e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1215s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.9805 | max_loss: 0.5234 | max_backward: 2.3872 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.8153 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6131.1 | uss_MB: 5547.8 | child_rss_MB: 4874.9 | allocated_MB: 501.7 | reserved_MB: 28766.0 | inactive_split_MB: 1064.3 +rank3 [2026-10-30 04:25:16,811] INFO : Step 45900/50000 rank3 | Train Loss: 0.8754 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1042 | Classification: 0.0696 | Classification_binary: 0.1484 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1393 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0513 | Regression_eta: 0.0233 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0455 | LR: 1.15e-04 | DataLoad Time: 0.0144s | Model Forward Time: 5.8493s | Phase Times: max_pre_step_barrier: 0.0308 | max_batch_to_device: 0.0265 | max_forward: 1.7626 | max_loss: 0.1678 | max_backward: 2.6283 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 6124.4 | uss_MB: 5544.6 | child_rss_MB: 4869.8 | allocated_MB: 496.0 | reserved_MB: 29170.0 | inactive_split_MB: 983.4 +rank3 [2026-10-30 04:29:51,933] INFO : Step 46000/50000 rank3 | Train Loss: 0.7531 | Train Loss Unweighted: 0.3556 | Train Loss Components: Auxiliary: 0.1535 | Cardinality: 0.0870 | Classification: 0.0690 | Classification_binary: 0.1026 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0903 | Regression_eta: 0.0011 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.0952 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0410 | Regression_eta: 0.0335 | Regression_pt: 0.0433 | Regression_sin_phi: 0.0450 | LR: 1.14e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3119s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0329 | max_forward: 0.7497 | max_loss: 0.1389 | max_backward: 1.9140 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5104 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6122.4 | uss_MB: 5543.2 | child_rss_MB: 4869.8 | allocated_MB: 492.4 | reserved_MB: 29170.0 | inactive_split_MB: 973.8 +rank3 [2026-10-30 04:29:53,062] INFO : Running validation on rank3 +rank3 [2026-10-30 04:29:53,062] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 04:29:53,062] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 04:30:44,349] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 04:30:44,926] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 04:31:11,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 04:31:12,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 04:31:13,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 04:31:14,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 04:31:24,575] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 04:31:25,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 04:31:25,579] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 04:31:41,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 04:31:42,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 04:31:43,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 04:31:44,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 04:31:54,421] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 04:31:54,978] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 04:31:55,463] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 04:32:10,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 04:32:11,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 04:32:12,458] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 04:32:13,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 04:32:23,371] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 04:37:27,738] INFO : Step 46100/50000 rank3 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3769 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.0999 | Classification: 0.0743 | Classification_binary: 0.1253 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0010 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1203 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0457 | Regression_eta: 0.0269 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0494 | LR: 1.13e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8713s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.9293 | max_loss: 0.1452 | max_backward: 2.3008 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.4968 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1219 | Memory: rss_MB: 7472.5 | uss_MB: 6889.9 | child_rss_MB: 4882.8 | allocated_MB: 498.3 | reserved_MB: 23298.0 | inactive_split_MB: 776.1 +rank3 [2026-10-30 04:42:19,615] INFO : Step 46200/50000 rank3 | Train Loss: 0.8661 | Train Loss Unweighted: 0.3994 | Train Loss Components: Auxiliary: 0.1947 | Cardinality: 0.1164 | Classification: 0.0793 | Classification_binary: 0.1511 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0833 | Regression_eta: 0.0011 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1503 | Regression_cos_phi: 0.0331 | Regression_energy: 0.0365 | Regression_eta: 0.0280 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0385 | LR: 1.13e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.7274s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8070 | max_loss: 0.2423 | max_backward: 1.9860 | max_optimizer_step: 0.2482 | max_tensorboard_logging: 0.7539 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6199.1 | uss_MB: 5617.8 | child_rss_MB: 4859.8 | allocated_MB: 499.3 | reserved_MB: 25288.0 | inactive_split_MB: 1325.5 +rank3 [2026-10-30 04:46:59,453] INFO : Step 46300/50000 rank3 | Train Loss: 0.8634 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0916 | Classification: 0.0803 | Classification_binary: 0.1320 | HitGrouping: 0.1536 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1031 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1236 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0535 | Regression_eta: 0.0336 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0538 | LR: 1.12e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5159s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7891 | max_loss: 0.1470 | max_backward: 1.7900 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.7709 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6201.3 | uss_MB: 5618.4 | child_rss_MB: 4887.4 | allocated_MB: 498.4 | reserved_MB: 25290.0 | inactive_split_MB: 977.6 +rank3 [2026-10-30 04:51:36,901] INFO : Step 46400/50000 rank3 | Train Loss: 0.8499 | Train Loss Unweighted: 0.4102 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.0956 | Classification: 0.0755 | Classification_binary: 0.1172 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1024 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0712 | Regression_eta: 0.0292 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0661 | LR: 1.11e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9768s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7847 | max_loss: 0.2487 | max_backward: 1.9181 | max_optimizer_step: 0.3148 | max_tensorboard_logging: 0.9408 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6188.6 | uss_MB: 5608.9 | child_rss_MB: 4878.3 | allocated_MB: 501.6 | reserved_MB: 25290.0 | inactive_split_MB: 992.4 +rank3 [2026-10-30 04:56:10,903] INFO : Step 46500/50000 rank3 | Train Loss: 0.8687 | Train Loss Unweighted: 0.4055 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1002 | Classification: 0.0822 | Classification_binary: 0.1448 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0872 | Regression_eta: 0.0013 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1609 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0366 | Regression_eta: 0.0274 | Regression_pt: 0.0401 | Regression_sin_phi: 0.0503 | LR: 1.11e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5426s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1839 | max_backward: 1.9629 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6193.9 | uss_MB: 5618.8 | child_rss_MB: 4880.0 | allocated_MB: 498.8 | reserved_MB: 29306.0 | inactive_split_MB: 924.4 +rank3 [2026-10-30 05:00:44,087] INFO : Step 46600/50000 rank3 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4049 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1082 | Classification: 0.0757 | Classification_binary: 0.1508 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0409 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0390 | Regression_energy: 0.0426 | Regression_eta: 0.0218 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0458 | LR: 1.10e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2095s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7939 | max_loss: 0.2001 | max_backward: 1.6859 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6188.5 | uss_MB: 5615.4 | child_rss_MB: 4846.0 | allocated_MB: 496.0 | reserved_MB: 29306.0 | inactive_split_MB: 901.7 +rank3 [2026-10-30 05:05:18,398] INFO : Step 46700/50000 rank3 | Train Loss: 0.8417 | Train Loss Unweighted: 0.3855 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0982 | Classification: 0.0807 | Classification_binary: 0.1184 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0916 | Regression_eta: 0.0015 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1061 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0446 | Regression_eta: 0.0330 | Regression_pt: 0.0478 | Regression_sin_phi: 0.0478 | LR: 1.10e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7628s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8551 | max_loss: 0.5124 | max_backward: 1.9004 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.7762 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1375 | Memory: rss_MB: 6185.4 | uss_MB: 5616.8 | child_rss_MB: 4868.3 | allocated_MB: 495.0 | reserved_MB: 29308.0 | inactive_split_MB: 1000.4 +rank3 [2026-10-30 05:09:48,388] INFO : Step 46800/50000 rank3 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3911 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0920 | Classification: 0.0920 | Classification_binary: 0.1296 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0828 | Regression_eta: 0.0013 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0396 | Regression_eta: 0.0401 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0496 | LR: 1.09e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4617s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7934 | max_loss: 0.1440 | max_backward: 1.7630 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.7219 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6183.1 | uss_MB: 5617.4 | child_rss_MB: 4871.4 | allocated_MB: 495.6 | reserved_MB: 29308.0 | inactive_split_MB: 1332.0 +rank3 [2026-10-30 05:14:19,737] INFO : Step 46900/50000 rank3 | Train Loss: 0.8496 | Train Loss Unweighted: 0.4142 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0976 | Classification: 0.0776 | Classification_binary: 0.1448 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0946 | Regression_eta: 0.0010 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1639 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0561 | Regression_eta: 0.0297 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0444 | LR: 1.09e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4518s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.9007 | max_loss: 0.1576 | max_backward: 1.8470 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6183.1 | uss_MB: 5616.1 | child_rss_MB: 4872.4 | allocated_MB: 495.4 | reserved_MB: 29310.0 | inactive_split_MB: 850.5 +rank3 [2026-10-30 05:18:49,738] INFO : Step 47000/50000 rank3 | Train Loss: 0.7980 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0982 | Classification: 0.0774 | Classification_binary: 0.1196 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1167 | Regression_cos_phi: 0.0340 | Regression_energy: 0.0432 | Regression_eta: 0.0272 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0304 | LR: 1.08e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8522s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 1.2405 | max_loss: 0.1502 | max_backward: 2.1273 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.7105 | max_tensorboard_flush: 0.0082 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6184.1 | uss_MB: 5617.4 | child_rss_MB: 4877.5 | allocated_MB: 494.9 | reserved_MB: 29310.0 | inactive_split_MB: 852.6 +rank3 [2026-10-30 05:18:50,807] INFO : Running validation on rank3 +rank3 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 05:19:41,789] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 05:19:42,337] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 05:20:07,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 05:20:08,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 05:20:09,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 05:20:11,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 05:20:21,015] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 05:20:21,504] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 05:20:22,014] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 05:20:37,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 05:20:38,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 05:20:39,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 05:20:40,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 05:20:50,844] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 05:20:51,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 05:20:52,199] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 05:21:07,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 05:21:08,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 05:21:09,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 05:21:10,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 05:21:20,049] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 05:26:12,382] INFO : Step 47100/50000 rank3 | Train Loss: 0.8046 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0851 | Classification: 0.0882 | Classification_binary: 0.1121 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0936 | Regression_eta: 0.0010 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1154 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0583 | Regression_eta: 0.0228 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0395 | LR: 1.07e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5739s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8250 | max_loss: 0.1453 | max_backward: 1.6267 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8108 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 7449.0 | uss_MB: 6878.6 | child_rss_MB: 4837.4 | allocated_MB: 494.1 | reserved_MB: 25416.0 | inactive_split_MB: 894.5 +rank3 [2026-10-30 05:30:44,527] INFO : Step 47200/50000 rank3 | Train Loss: 0.8884 | Train Loss Unweighted: 0.4191 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1138 | Classification: 0.0793 | Classification_binary: 0.1489 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0010 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0427 | Regression_eta: 0.0341 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0435 | LR: 1.07e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9171s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7960 | max_loss: 0.1482 | max_backward: 1.9261 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.9435 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0661 | Memory: rss_MB: 6249.9 | uss_MB: 5682.5 | child_rss_MB: 4852.8 | allocated_MB: 496.8 | reserved_MB: 26534.0 | inactive_split_MB: 847.6 +rank3 [2026-10-30 05:35:15,256] INFO : Step 47300/50000 rank3 | Train Loss: 0.8547 | Train Loss Unweighted: 0.3920 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1049 | Classification: 0.0798 | Classification_binary: 0.1368 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0012 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1347 | Regression_cos_phi: 0.0441 | Regression_energy: 0.0390 | Regression_eta: 0.0413 | Regression_pt: 0.0416 | Regression_sin_phi: 0.0689 | LR: 1.06e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5902s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7649 | max_loss: 0.2313 | max_backward: 1.8030 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8438 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6253.9 | uss_MB: 5688.5 | child_rss_MB: 4855.9 | allocated_MB: 496.9 | reserved_MB: 28534.0 | inactive_split_MB: 933.0 +rank3 [2026-10-30 05:39:47,014] INFO : Step 47400/50000 rank3 | Train Loss: 0.7606 | Train Loss Unweighted: 0.3755 | Train Loss Components: Auxiliary: 0.1640 | Cardinality: 0.0704 | Classification: 0.0737 | Classification_binary: 0.1072 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0960 | Regression_eta: 0.0010 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1068 | Regression_cos_phi: 0.0344 | Regression_energy: 0.0570 | Regression_eta: 0.0222 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0334 | LR: 1.06e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5313s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0275 | max_forward: 0.8465 | max_loss: 0.1482 | max_backward: 1.7103 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.8388 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 6241.7 | uss_MB: 5678.8 | child_rss_MB: 4860.9 | allocated_MB: 496.4 | reserved_MB: 28534.0 | inactive_split_MB: 913.7 +rank3 [2026-10-30 05:44:18,077] INFO : Step 47500/50000 rank3 | Train Loss: 0.8134 | Train Loss Unweighted: 0.3681 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1057 | Classification: 0.0739 | Classification_binary: 0.1303 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0805 | Regression_eta: 0.0011 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1460 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0449 | Regression_eta: 0.0241 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0554 | LR: 1.06e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.7795s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1751 | max_backward: 1.9857 | max_optimizer_step: 0.3263 | max_tensorboard_logging: 0.7363 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6249.7 | uss_MB: 5690.9 | child_rss_MB: 4845.9 | allocated_MB: 498.3 | reserved_MB: 28534.0 | inactive_split_MB: 851.5 +rank3 [2026-10-30 05:48:48,524] INFO : Step 47600/50000 rank3 | Train Loss: 0.8663 | Train Loss Unweighted: 0.4076 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1090 | Classification: 0.0820 | Classification_binary: 0.1491 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0865 | Regression_eta: 0.0011 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0529 | Regression_eta: 0.0266 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0319 | LR: 1.05e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.6235s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.9385 | max_loss: 0.1530 | max_backward: 2.1678 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6249.7 | uss_MB: 5691.7 | child_rss_MB: 4853.8 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 843.5 +rank3 [2026-10-30 05:53:19,839] INFO : Step 47700/50000 rank3 | Train Loss: 0.8615 | Train Loss Unweighted: 0.4088 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0954 | Classification: 0.0764 | Classification_binary: 0.1291 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1002 | Regression_eta: 0.0011 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.0909 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0492 | Regression_eta: 0.0179 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0490 | LR: 1.05e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2191s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7406 | max_loss: 0.2391 | max_backward: 1.7059 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1444 | Memory: rss_MB: 6248.7 | uss_MB: 5687.5 | child_rss_MB: 4851.8 | allocated_MB: 493.9 | reserved_MB: 28534.0 | inactive_split_MB: 895.3 +rank3 [2026-10-30 05:57:52,842] INFO : Step 47800/50000 rank3 | Train Loss: 0.8267 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.0942 | Classification: 0.0882 | Classification_binary: 0.1186 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0870 | Regression_eta: 0.0012 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.1277 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0415 | Regression_eta: 0.0355 | Regression_pt: 0.0439 | Regression_sin_phi: 0.0653 | LR: 1.04e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0865s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.7869 | max_loss: 0.5030 | max_backward: 2.0522 | max_optimizer_step: 0.2267 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6249.7 | uss_MB: 5692.3 | child_rss_MB: 4857.3 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 916.9 +rank3 [2026-10-30 06:02:25,107] INFO : Step 47900/50000 rank3 | Train Loss: 0.8423 | Train Loss Unweighted: 0.4000 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.0845 | Classification: 0.0794 | Classification_binary: 0.1416 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1233 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0467 | Regression_eta: 0.0466 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0534 | LR: 1.04e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5084s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7994 | max_loss: 0.1511 | max_backward: 1.9354 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.6685 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6247.4 | uss_MB: 5692.2 | child_rss_MB: 4860.2 | allocated_MB: 496.8 | reserved_MB: 30642.0 | inactive_split_MB: 779.5 +rank3 [2026-10-30 06:06:56,644] INFO : Step 48000/50000 rank3 | Train Loss: 0.8641 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0972 | Classification: 0.0768 | Classification_binary: 0.1396 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0890 | Regression_eta: 0.0015 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0488 | Regression_eta: 0.0557 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0417 | LR: 1.04e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4077s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0057 | max_forward: 0.8017 | max_loss: 0.1550 | max_backward: 1.6990 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.7498 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6247.1 | uss_MB: 5690.2 | child_rss_MB: 4855.9 | allocated_MB: 495.1 | reserved_MB: 30642.0 | inactive_split_MB: 896.2 +rank3 [2026-10-30 06:06:57,653] INFO : Running validation on rank3 +rank3 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 06:07:49,054] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:07:49,560] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 06:08:14,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 06:08:15,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 06:08:16,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 06:08:17,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 06:08:27,193] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 06:08:28,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:08:28,619] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 06:08:44,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 06:08:45,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 06:08:46,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 06:08:47,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 06:08:57,362] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 06:08:58,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:08:58,714] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 06:09:14,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 06:09:15,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 06:09:15,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 06:09:16,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 06:09:26,818] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 06:14:16,583] INFO : Step 48100/50000 rank3 | Train Loss: 0.8226 | Train Loss Unweighted: 0.3895 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.1043 | Classification: 0.0717 | Classification_binary: 0.1116 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1023 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1139 | Regression_cos_phi: 0.0314 | Regression_energy: 0.0628 | Regression_eta: 0.0271 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0375 | LR: 1.03e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4966s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7477 | max_loss: 0.1426 | max_backward: 1.7972 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.6718 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1303 | Memory: rss_MB: 7752.6 | uss_MB: 7191.3 | child_rss_MB: 4880.7 | allocated_MB: 494.1 | reserved_MB: 23078.0 | inactive_split_MB: 1096.3 +rank3 [2026-10-30 06:18:47,187] INFO : Step 48200/50000 rank3 | Train Loss: 0.8500 | Train Loss Unweighted: 0.3835 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1027 | Classification: 0.0771 | Classification_binary: 0.1374 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0827 | Regression_eta: 0.0012 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0434 | Regression_eta: 0.0315 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0558 | LR: 1.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2131s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8132 | max_loss: 0.2545 | max_backward: 2.1167 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1241 | Memory: rss_MB: 6310.4 | uss_MB: 5751.6 | child_rss_MB: 4867.3 | allocated_MB: 496.6 | reserved_MB: 25094.0 | inactive_split_MB: 913.4 +rank3 [2026-10-30 06:23:18,411] INFO : Step 48300/50000 rank3 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4303 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1089 | Classification: 0.0832 | Classification_binary: 0.1567 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0013 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1873 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0568 | Regression_eta: 0.0344 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0636 | LR: 1.03e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.4481s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8079 | max_loss: 0.1526 | max_backward: 1.9733 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6317.4 | uss_MB: 5760.9 | child_rss_MB: 4872.1 | allocated_MB: 498.5 | reserved_MB: 25098.0 | inactive_split_MB: 771.0 +rank3 [2026-10-30 06:27:49,028] INFO : Step 48400/50000 rank3 | Train Loss: 0.7961 | Train Loss Unweighted: 0.3733 | Train Loss Components: Auxiliary: 0.1682 | Cardinality: 0.0877 | Classification: 0.0685 | Classification_binary: 0.1191 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0012 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1169 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0449 | Regression_eta: 0.0314 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0462 | LR: 1.02e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2393s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7916 | max_loss: 0.1471 | max_backward: 1.6943 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5940 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6314.4 | uss_MB: 5759.4 | child_rss_MB: 4842.4 | allocated_MB: 496.3 | reserved_MB: 25098.0 | inactive_split_MB: 1087.5 +rank3 [2026-10-30 06:32:21,027] INFO : Step 48500/50000 rank3 | Train Loss: 0.8668 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0816 | Classification: 0.0832 | Classification_binary: 0.1257 | HitGrouping: 0.1542 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1320 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0587 | Regression_eta: 0.0473 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0715 | LR: 1.02e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3823s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7587 | max_loss: 0.1606 | max_backward: 1.7573 | max_optimizer_step: 0.3024 | max_tensorboard_logging: 0.5851 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6309.3 | uss_MB: 5759.8 | child_rss_MB: 4851.4 | allocated_MB: 498.9 | reserved_MB: 25098.0 | inactive_split_MB: 1192.4 +rank3 [2026-10-30 06:36:52,986] INFO : Step 48600/50000 rank3 | Train Loss: 0.8434 | Train Loss Unweighted: 0.4010 | Train Loss Components: Auxiliary: 0.1850 | Cardinality: 0.0964 | Classification: 0.0791 | Classification_binary: 0.1425 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1326 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0517 | Regression_eta: 0.0286 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0582 | LR: 1.02e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4850s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7760 | max_loss: 0.1505 | max_backward: 1.8991 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0654 | Memory: rss_MB: 6311.3 | uss_MB: 5762.6 | child_rss_MB: 4856.0 | allocated_MB: 497.0 | reserved_MB: 26848.0 | inactive_split_MB: 777.5 +rank3 [2026-10-30 06:41:29,683] INFO : Step 48700/50000 rank3 | Train Loss: 0.9079 | Train Loss Unweighted: 0.4266 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1195 | Classification: 0.0920 | Classification_binary: 0.1456 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1481 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0537 | Regression_eta: 0.0312 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0623 | LR: 1.02e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5445s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7797 | max_loss: 0.1578 | max_backward: 1.7107 | max_optimizer_step: 0.5280 | max_tensorboard_logging: 0.7650 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6308.9 | uss_MB: 5759.1 | child_rss_MB: 4857.7 | allocated_MB: 496.9 | reserved_MB: 26848.0 | inactive_split_MB: 891.8 +rank3 [2026-10-30 06:46:02,597] INFO : Step 48800/50000 rank3 | Train Loss: 0.8011 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1685 | Cardinality: 0.0863 | Classification: 0.0837 | Classification_binary: 0.1116 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0965 | Regression_eta: 0.0012 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1098 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0587 | Regression_eta: 0.0535 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0380 | LR: 1.01e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4993s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7374 | max_loss: 0.2432 | max_backward: 1.8459 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.7427 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6306.9 | uss_MB: 5759.1 | child_rss_MB: 4857.0 | allocated_MB: 493.9 | reserved_MB: 26848.0 | inactive_split_MB: 991.0 +rank3 [2026-10-30 06:50:45,084] INFO : Step 48900/50000 rank3 | Train Loss: 0.8551 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.1109 | Classification: 0.0760 | Classification_binary: 0.1310 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1161 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0427 | Regression_eta: 0.0301 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0643 | LR: 1.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5415s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8768 | max_loss: 0.1470 | max_backward: 2.0816 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6306.9 | uss_MB: 5761.7 | child_rss_MB: 4858.9 | allocated_MB: 499.5 | reserved_MB: 26852.0 | inactive_split_MB: 1398.0 +rank3 [2026-10-30 06:55:20,574] INFO : Step 49000/50000 rank3 | Train Loss: 0.9005 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.2010 | Cardinality: 0.1248 | Classification: 0.0800 | Classification_binary: 0.1499 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0939 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1795 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0530 | Regression_eta: 0.0316 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0549 | LR: 1.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9025s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0439 | max_forward: 0.8365 | max_loss: 0.1536 | max_backward: 1.9893 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8890 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6309.3 | uss_MB: 5762.6 | child_rss_MB: 4865.2 | allocated_MB: 497.5 | reserved_MB: 26852.0 | inactive_split_MB: 943.6 +rank3 [2026-10-30 06:55:21,669] INFO : Running validation on rank3 +rank3 [2026-10-30 06:55:21,669] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 06:55:21,669] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 06:56:13,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:56:13,862] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 06:56:38,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 06:56:39,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 06:56:41,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 06:56:42,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 06:56:52,039] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 06:56:52,550] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:56:53,030] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 06:57:08,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 06:57:09,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 06:57:10,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 06:57:11,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 06:57:21,852] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 06:57:22,632] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 06:57:23,150] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 06:57:38,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 06:57:39,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 06:57:40,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 06:57:41,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 06:57:51,199] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 07:02:45,629] INFO : Step 49100/50000 rank3 | Train Loss: 0.8219 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.0880 | Classification: 0.0841 | Classification_binary: 0.1303 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0855 | Regression_eta: 0.0012 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0389 | Regression_eta: 0.0459 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0579 | LR: 1.01e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3699s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7738 | max_loss: 0.1431 | max_backward: 1.6939 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6276 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 7720.3 | uss_MB: 7170.2 | child_rss_MB: 4829.0 | allocated_MB: 495.9 | reserved_MB: 24730.0 | inactive_split_MB: 936.0 +rank3 [2026-10-30 07:07:18,676] INFO : Step 49200/50000 rank3 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3843 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0856 | Classification: 0.0829 | Classification_binary: 0.1188 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0892 | Regression_eta: 0.0011 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0420 | Regression_eta: 0.0302 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0638 | LR: 1.01e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.7720s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7590 | max_loss: 0.5032 | max_backward: 1.9826 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.7926 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6365.4 | uss_MB: 5817.0 | child_rss_MB: 4830.4 | allocated_MB: 494.5 | reserved_MB: 26752.0 | inactive_split_MB: 1117.5 +rank3 [2026-10-30 07:11:50,862] INFO : Step 49300/50000 rank3 | Train Loss: 0.9397 | Train Loss Unweighted: 0.4282 | Train Loss Components: Auxiliary: 0.2065 | Cardinality: 0.1351 | Classification: 0.0889 | Classification_binary: 0.1552 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0901 | Regression_eta: 0.0012 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1715 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0445 | Regression_eta: 0.0231 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0620 | LR: 1.00e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4919s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0070 | max_forward: 0.7807 | max_loss: 0.1520 | max_backward: 1.8619 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6076 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6376.6 | uss_MB: 5827.0 | child_rss_MB: 4814.1 | allocated_MB: 496.4 | reserved_MB: 26752.0 | inactive_split_MB: 1120.6 +rank3 [2026-10-30 07:16:22,252] INFO : Step 49400/50000 rank3 | Train Loss: 0.8870 | Train Loss Unweighted: 0.4213 | Train Loss Components: Auxiliary: 0.1937 | Cardinality: 0.1005 | Classification: 0.0815 | Classification_binary: 0.1465 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0529 | Regression_eta: 0.0363 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0649 | LR: 1.00e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2712s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8046 | max_loss: 0.1540 | max_backward: 1.7244 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.6030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6373.7 | uss_MB: 5825.4 | child_rss_MB: 4813.8 | allocated_MB: 498.5 | reserved_MB: 28804.0 | inactive_split_MB: 886.2 +rank3 [2026-10-30 07:20:55,284] INFO : Step 49500/50000 rank3 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3949 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0991 | Classification: 0.0800 | Classification_binary: 0.1101 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0013 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0327 | Classification_binary: 0.0759 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0503 | Regression_eta: 0.0242 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0270 | LR: 1.00e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.2143s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0074 | max_forward: 0.9924 | max_loss: 0.1865 | max_backward: 2.5853 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.6095 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0429 | Memory: rss_MB: 6377.1 | uss_MB: 5695.1 | child_rss_MB: 4830.9 | allocated_MB: 495.9 | reserved_MB: 28804.0 | inactive_split_MB: 849.6 +rank3 [2026-10-30 07:25:48,132] INFO : Step 49600/50000 rank3 | Train Loss: 0.9075 | Train Loss Unweighted: 0.4421 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1021 | Classification: 0.0939 | Classification_binary: 0.1401 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1131 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0455 | Regression_eta: 0.0283 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0527 | LR: 1.00e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4384s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7852 | max_loss: 0.1445 | max_backward: 2.0967 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4754 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6378.7 | uss_MB: 5829.2 | child_rss_MB: 4808.2 | allocated_MB: 497.8 | reserved_MB: 28804.0 | inactive_split_MB: 845.2 +rank3 [2026-10-30 07:30:32,223] INFO : Step 49700/50000 rank3 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4307 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1074 | Classification: 0.0931 | Classification_binary: 0.1611 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0865 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0499 | Regression_eta: 0.0318 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0532 | LR: 1.00e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6428s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.8169 | max_loss: 0.1522 | max_backward: 2.0553 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.4937 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1402 | Memory: rss_MB: 6379.2 | uss_MB: 5828.6 | child_rss_MB: 4829.3 | allocated_MB: 499.3 | reserved_MB: 30718.0 | inactive_split_MB: 838.8 +rank3 [2026-10-30 07:35:17,063] INFO : Step 49800/50000 rank3 | Train Loss: 0.8342 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.1036 | Classification: 0.0769 | Classification_binary: 0.1292 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.0981 | Regression_cos_phi: 0.0406 | Regression_energy: 0.0403 | Regression_eta: 0.0310 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0503 | LR: 1.00e-04 | DataLoad Time: 0.0181s | Model Forward Time: 5.1019s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0329 | max_forward: 0.7490 | max_loss: 0.1889 | max_backward: 2.2249 | max_optimizer_step: 0.2452 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6374.9 | uss_MB: 5812.4 | child_rss_MB: 4828.7 | allocated_MB: 494.4 | reserved_MB: 30718.0 | inactive_split_MB: 1134.4 +rank3 [2026-10-30 07:39:58,302] INFO : Step 49900/50000 rank3 | Train Loss: 0.8436 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.0947 | Classification: 0.0943 | Classification_binary: 0.1179 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0982 | Regression_eta: 0.0012 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.0939 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0396 | Regression_eta: 0.0261 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0573 | LR: 1.00e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.4162s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7496 | max_loss: 0.5142 | max_backward: 1.9365 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6373.9 | uss_MB: 5825.7 | child_rss_MB: 4831.8 | allocated_MB: 494.8 | reserved_MB: 30718.0 | inactive_split_MB: 1090.4 +rank3 [2026-10-30 07:44:45,887] INFO : Step 50000/50000 rank3 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4040 | Train Loss Components: Auxiliary: 0.1921 | Cardinality: 0.1113 | Classification: 0.0818 | Classification_binary: 0.1436 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0872 | Regression_eta: 0.0014 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0614 | Regression_energy: 0.0352 | Regression_eta: 0.0539 | Regression_pt: 0.0402 | Regression_sin_phi: 0.0513 | LR: 1.00e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.6368s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 1.3997 | max_loss: 0.5678 | max_backward: 3.0655 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6373.9 | uss_MB: 5825.4 | child_rss_MB: 4827.6 | allocated_MB: 497.9 | reserved_MB: 30720.0 | inactive_split_MB: 906.1 +rank3 [2026-10-30 07:44:46,919] INFO : Running validation on rank3 +rank3 [2026-10-30 07:44:46,919] INFO : ResumableSampler physical:valid resetting +rank3 [2026-10-30 07:44:46,919] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank3 [2026-10-30 07:45:38,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 07:45:38,998] INFO : Running predictions on cld_edm_ttbar_hits +rank3 [2026-10-30 07:46:04,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_3_0.parquet +rank3 [2026-10-30 07:46:05,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_3_1.parquet +rank3 [2026-10-30 07:46:06,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_3_2.parquet +rank3 [2026-10-30 07:46:07,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_3_3.parquet +rank3 [2026-10-30 07:46:17,182] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 07:46:17,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 07:46:17,996] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank3 [2026-10-30 07:46:34,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_3_0.parquet +rank3 [2026-10-30 07:46:35,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_3_1.parquet +rank3 [2026-10-30 07:46:36,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_3_2.parquet +rank3 [2026-10-30 07:46:37,303] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_3_3.parquet +rank3 [2026-10-30 07:46:47,314] INFO : Time taken to make predictions on device 3 is: 0.25 min +rank3 [2026-10-30 07:46:47,615] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank3 [2026-10-30 07:46:48,154] INFO : Running predictions on cld_edm_qq_hits +rank3 [2026-10-30 07:47:03,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_3_0.parquet +rank3 [2026-10-30 07:47:04,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_3_1.parquet +rank3 [2026-10-30 07:47:04,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_3_2.parquet +rank3 [2026-10-30 07:47:05,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_3_3.parquet +rank3 [2026-10-30 07:47:15,861] INFO : Time taken to make predictions on device 3 is: 0.23 min +rank3 [2026-10-30 07:47:17,219] INFO : Training completed. Total time on device 3: 1017.500min diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.4 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.4 new file mode 100644 index 0000000000000000000000000000000000000000..b99ea06e277886ae74c9979490c4140ce07b3439 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.4 @@ -0,0 +1,2631 @@ +rank4 [2026-10-28 11:12:32,389] INFO : Initializing model with seed=12345; process seed=12349 +rank4 [2026-10-28 11:12:32,389] INFO : configured dtype=torch.bfloat16 for autocast +rank4 [2026-10-28 11:12:54,158] INFO : Instantiating model +rank4 [2026-10-28 11:12:54,158] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank4 [2026-10-28 11:12:54,158] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank4 [2026-10-28 11:12:54,176] INFO : Input encoding initialization took 0.02s +rank4 [2026-10-28 11:12:54,176] INFO : nn0 parameters: 171776 +rank4 [2026-10-28 11:12:54,176] INFO : Initializing shared backbone layers, num_convs=6 +rank4 [2026-10-28 11:12:54,176] INFO : Initializing backbone layer 0 +rank4 [2026-10-28 11:12:54,212] INFO : Initializing backbone layer 1 +rank4 [2026-10-28 11:12:54,221] INFO : Initializing backbone layer 2 +rank4 [2026-10-28 11:12:54,230] INFO : Initializing backbone layer 3 +rank4 [2026-10-28 11:12:54,238] INFO : Initializing backbone layer 4 +rank4 [2026-10-28 11:12:54,247] INFO : Initializing backbone layer 5 +rank4 [2026-10-28 11:12:54,255] INFO : Backbone initialization took 0.08s +rank4 [2026-10-28 11:12:54,256] INFO : backbone parameters: 7901376 +rank4 [2026-10-28 11:12:54,256] INFO : Initializing output DNNs +rank4 [2026-10-28 11:12:54,292] INFO : Output DNNs initialization took 0.04s +rank4 [2026-10-28 11:12:54,292] INFO : backbone_mode=BackboneMode.SHARED +rank4 [2026-10-28 11:12:54,292] INFO : jagged_attention enabled=False +rank4 [2026-10-28 11:12:54,292] INFO : flash_attn_varlen enabled=False +rank4 [2026-10-28 11:12:54,292] INFO : task_queries enabled=False +rank4 [2026-10-28 11:12:54,292] INFO : classification_norm parameters: 0 +rank4 [2026-10-28 11:12:54,292] INFO : regression_norm parameters: 0 +rank4 [2026-10-28 11:12:54,292] INFO : classification_query parameters: 0 +rank4 [2026-10-28 11:12:54,292] INFO : regression_query parameters: 0 +rank4 [2026-10-28 11:12:54,293] INFO : classification_readout parameters: 0 +rank4 [2026-10-28 11:12:54,293] INFO : regression_readout parameters: 0 +rank4 [2026-10-28 11:12:54,293] INFO : output_mode=set +rank4 [2026-10-28 11:12:54,293] INFO : set_decoder parameters: 4713812 +rank4 [2026-10-28 11:12:54,293] INFO : nn_binary_particle parameters: 0 +rank4 [2026-10-28 11:12:54,293] INFO : nn_pid parameters: 0 +rank4 [2026-10-28 11:12:54,293] INFO : nn_pt parameters: 0 +rank4 [2026-10-28 11:12:54,293] INFO : nn_eta parameters: 0 +rank4 [2026-10-28 11:12:54,293] INFO : nn_sin_phi parameters: 0 +rank4 [2026-10-28 11:12:54,293] INFO : nn_cos_phi parameters: 0 +rank4 [2026-10-28 11:12:54,293] INFO : nn_energy parameters: 0 +rank4 [2026-10-28 11:12:54,294] INFO : Total MLPF parameters: 12786964 +rank4 [2026-10-28 11:12:54,294] INFO : MLPF __init__ done +rank4 [2026-10-28 11:12:54,294] INFO : Instantiated model +rank4 [2026-10-28 11:12:54,294] INFO : Moving model to device rank=4 +rank4 [2026-10-28 11:12:54,330] INFO : Moved model to device rank=4 +rank4 [2026-10-28 11:12:54,334] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank4 [2026-10-28 11:12:54,335] INFO : Configured model for SyncBatchNorm rank=4 +rank4 [2026-10-28 11:12:54,648] INFO : Configured model for DistributedDataParallel rank=4 +rank4 [2026-10-28 11:12:54,808] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank4 [2026-10-28 11:12:54,809] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank4 [2026-10-28 11:12:54,810] INFO : Trainable parameters: 12786964 +rank4 [2026-10-28 11:12:54,810] INFO : Non-trainable parameters: 23268 +rank4 [2026-10-28 11:12:54,810] INFO : Total parameters: 12810232 +rank4 [2026-10-28 11:12:54,813] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank4 [2026-10-28 11:12:54,813] INFO : Getting dataloaders +rank4 [2026-10-28 11:12:54,813] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:03,738] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:04,812] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,146] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-28 11:13:06,146] INFO : Creating WeightedMixIterator with 1 data loaders. +rank4 [2026-10-28 11:13:06,146] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,417] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,680] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:13:06,932] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-28 11:13:06,932] INFO : Creating InterleavedIterator with 1 data loaders. +rank4 [2026-10-28 11:13:20,752] INFO : Got dataloaders +rank4 [2026-10-28 11:13:20,752] INFO : loader split=train rank=4 len=21112 +rank4 [2026-10-28 11:13:20,752] INFO : loader split=valid rank=4 len=12 +rank4 [2026-10-28 11:13:20,753] INFO : Starting training from step 1 to 50000 on rank 4 of 8 +rank4 [2026-10-28 11:13:20,753] INFO : Creating train_iterator +rank4 [2026-10-28 11:13:20,753] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-28 11:13:35,248] INFO : Created train_iterator +rank4 [2026-10-28 11:18:11,207] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5408100029081107, 'Classification': 0.8863177470117808, 'Regression_pt': 0.9386234817653895, 'Regression_eta': 0.019238746587652712, 'Regression_sin_phi': 0.006576384851359762, 'Regression_cos_phi': 0.006405904629500583, 'Regression_energy': 1.0145655398070812}, weights={'Classification_binary': 1.0, 'Classification': 0.6101762056350708, 'Regression_pt': 0.5761735439300537, 'Regression_eta': 28.110458374023438, 'Regression_sin_phi': 82.23515319824219, 'Regression_cos_phi': 84.42367553710938, 'Regression_energy': 0.5330458879470825} +rank4 [2026-10-28 11:18:13,718] INFO : Step 100/50000 rank4 | Train Loss: 4.2631 | Train Loss Unweighted: 2.7366 | Train Loss Components: Auxiliary: 0.7941 | Cardinality: 0.3942 | Classification: 0.6312 | Classification_binary: 0.4549 | HitGrouping: 0.3381 | Regression_cos_phi: 0.0047 | Regression_energy: 0.8263 | Regression_eta: 0.0122 | Regression_pt: 0.8026 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.5936 | Classification_binary: 0.4406 | Regression_cos_phi: 0.0051 | Regression_energy: 0.9207 | Regression_eta: 0.0132 | Regression_pt: 0.8850 | Regression_sin_phi: 0.0052 | LR: 1.00e-03 | DataLoad Time: 0.0170s | Model Forward Time: 3.5362s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6509 | max_loss: 0.1961 | max_backward: 1.4166 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4769.1 | uss_MB: 3475.5 | child_rss_MB: 6372.1 | allocated_MB: 446.7 | reserved_MB: 24804.0 | inactive_split_MB: 1063.6 +rank4 [2026-10-28 11:22:42,538] INFO : Step 200/50000 rank4 | Train Loss: 4.2029 | Train Loss Unweighted: 2.5447 | Train Loss Components: Auxiliary: 0.8074 | Cardinality: 0.4500 | Classification: 0.5373 | Classification_binary: 0.5135 | HitGrouping: 0.3101 | Regression_cos_phi: 0.0042 | Regression_energy: 0.7508 | Regression_eta: 0.0093 | Regression_pt: 0.7252 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.3200 | Classification_binary: 0.5403 | Regression_cos_phi: 0.3819 | Regression_energy: 0.3678 | Regression_eta: 0.3056 | Regression_pt: 0.3844 | Regression_sin_phi: 0.3818 | LR: 1.00e-03 | DataLoad Time: 0.0158s | Model Forward Time: 4.6629s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9214 | max_loss: 0.1502 | max_backward: 2.1220 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 4772.0 | uss_MB: 3480.3 | child_rss_MB: 6374.7 | allocated_MB: 449.0 | reserved_MB: 26252.0 | inactive_split_MB: 1054.4 +rank4 [2026-10-28 11:27:10,390] INFO : Step 300/50000 rank4 | Train Loss: 3.5114 | Train Loss Unweighted: 2.0975 | Train Loss Components: Auxiliary: 0.6791 | Cardinality: 0.3510 | Classification: 0.4201 | Classification_binary: 0.4975 | HitGrouping: 0.2993 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5752 | Regression_eta: 0.0073 | Regression_pt: 0.5905 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2297 | Classification_binary: 0.5047 | Regression_cos_phi: 0.2695 | Regression_energy: 0.2677 | Regression_eta: 0.1922 | Regression_pt: 0.2975 | Regression_sin_phi: 0.3107 | LR: 1.00e-03 | DataLoad Time: 0.0159s | Model Forward Time: 4.2605s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.7369 | max_loss: 0.1526 | max_backward: 1.7330 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.9030 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 4773.0 | uss_MB: 3484.3 | child_rss_MB: 6364.6 | allocated_MB: 446.1 | reserved_MB: 26252.0 | inactive_split_MB: 1068.0 +rank4 [2026-10-28 11:31:37,117] INFO : Step 400/50000 rank4 | Train Loss: 3.3615 | Train Loss Unweighted: 1.9456 | Train Loss Components: Auxiliary: 0.6310 | Cardinality: 0.3555 | Classification: 0.4058 | Classification_binary: 0.4436 | HitGrouping: 0.3006 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5363 | Regression_eta: 0.0080 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2261 | Classification_binary: 0.4498 | Regression_cos_phi: 0.3697 | Regression_energy: 0.3548 | Regression_eta: 0.3168 | Regression_pt: 0.3796 | Regression_sin_phi: 0.4140 | LR: 1.00e-03 | DataLoad Time: 0.0161s | Model Forward Time: 4.5036s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1151 | max_forward: 0.7642 | max_loss: 0.1384 | max_backward: 1.7906 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9359 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0880 | Memory: rss_MB: 4771.8 | uss_MB: 3484.8 | child_rss_MB: 6366.3 | allocated_MB: 446.8 | reserved_MB: 26252.0 | inactive_split_MB: 1064.0 +rank4 [2026-10-28 11:36:08,659] INFO : Step 500/50000 rank4 | Train Loss: 3.8257 | Train Loss Unweighted: 2.0630 | Train Loss Components: Auxiliary: 0.7010 | Cardinality: 0.4740 | Classification: 0.4380 | Classification_binary: 0.4817 | HitGrouping: 0.2961 | Regression_cos_phi: 0.0044 | Regression_energy: 0.5611 | Regression_eta: 0.0087 | Regression_pt: 0.5645 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2493 | Classification_binary: 0.4665 | Regression_cos_phi: 0.3532 | Regression_energy: 0.3308 | Regression_eta: 0.2297 | Regression_pt: 0.3565 | Regression_sin_phi: 0.2838 | LR: 1.00e-03 | DataLoad Time: 0.0163s | Model Forward Time: 5.1254s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0077 | max_forward: 1.2433 | max_loss: 0.1536 | max_backward: 2.4885 | max_optimizer_step: 0.1562 | max_tensorboard_logging: 1.0374 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0034 | Memory: rss_MB: 4768.9 | uss_MB: 3486.1 | child_rss_MB: 6361.5 | allocated_MB: 449.6 | reserved_MB: 26256.0 | inactive_split_MB: 929.2 +rank4 [2026-10-28 11:40:42,094] INFO : Step 600/50000 rank4 | Train Loss: 3.4702 | Train Loss Unweighted: 1.9733 | Train Loss Components: Auxiliary: 0.6509 | Cardinality: 0.3585 | Classification: 0.3825 | Classification_binary: 0.5061 | HitGrouping: 0.2915 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5321 | Regression_eta: 0.0074 | Regression_pt: 0.5377 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2160 | Classification_binary: 0.4956 | Regression_cos_phi: 0.2905 | Regression_energy: 0.2661 | Regression_eta: 0.1739 | Regression_pt: 0.2912 | Regression_sin_phi: 0.2683 | LR: 1.00e-03 | DataLoad Time: 0.0159s | Model Forward Time: 4.5579s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8098 | max_loss: 0.1544 | max_backward: 1.9541 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 0.9377 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 4772.7 | uss_MB: 3486.1 | child_rss_MB: 6366.7 | allocated_MB: 449.5 | reserved_MB: 27396.0 | inactive_split_MB: 1022.5 +rank4 [2026-10-28 11:45:16,605] INFO : Step 700/50000 rank4 | Train Loss: 2.9693 | Train Loss Unweighted: 1.8072 | Train Loss Components: Auxiliary: 0.5907 | Cardinality: 0.2768 | Classification: 0.3175 | Classification_binary: 0.3969 | HitGrouping: 0.2848 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5321 | Regression_eta: 0.0061 | Regression_pt: 0.5492 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1726 | Classification_binary: 0.3856 | Regression_cos_phi: 0.2246 | Regression_energy: 0.2905 | Regression_eta: 0.1404 | Regression_pt: 0.3281 | Regression_sin_phi: 0.1951 | LR: 1.00e-03 | DataLoad Time: 0.0216s | Model Forward Time: 3.9464s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7910 | max_loss: 0.2436 | max_backward: 1.8672 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.8 | uss_MB: 3484.3 | child_rss_MB: 6369.5 | allocated_MB: 446.1 | reserved_MB: 27396.0 | inactive_split_MB: 1066.2 +rank4 [2026-10-28 11:49:48,175] INFO : Step 800/50000 rank4 | Train Loss: 3.1527 | Train Loss Unweighted: 1.8424 | Train Loss Components: Auxiliary: 0.6214 | Cardinality: 0.3518 | Classification: 0.3758 | Classification_binary: 0.3967 | HitGrouping: 0.2962 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5290 | Regression_eta: 0.0063 | Regression_pt: 0.5287 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2101 | Classification_binary: 0.3443 | Regression_cos_phi: 0.2127 | Regression_energy: 0.3117 | Regression_eta: 0.2044 | Regression_pt: 0.3400 | Regression_sin_phi: 0.3045 | LR: 9.99e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.6442s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0093 | max_forward: 0.8507 | max_loss: 0.1493 | max_backward: 1.9706 | max_optimizer_step: 0.1582 | max_tensorboard_logging: 1.0840 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0072 | Memory: rss_MB: 4754.0 | uss_MB: 3484.3 | child_rss_MB: 6293.3 | allocated_MB: 450.9 | reserved_MB: 27396.0 | inactive_split_MB: 920.9 +rank4 [2026-10-28 11:54:30,570] INFO : Step 900/50000 rank4 | Train Loss: 3.2552 | Train Loss Unweighted: 1.9216 | Train Loss Components: Auxiliary: 0.6208 | Cardinality: 0.2901 | Classification: 0.3818 | Classification_binary: 0.4734 | HitGrouping: 0.2881 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5239 | Regression_eta: 0.0075 | Regression_pt: 0.5284 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2544 | Classification_binary: 0.4821 | Regression_cos_phi: 0.2371 | Regression_energy: 0.2853 | Regression_eta: 0.1792 | Regression_pt: 0.3082 | Regression_sin_phi: 0.2468 | LR: 9.99e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6744s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8050 | max_loss: 0.1486 | max_backward: 1.9075 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.9729 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4751.7 | uss_MB: 3485.4 | child_rss_MB: 6307.4 | allocated_MB: 449.7 | reserved_MB: 27406.0 | inactive_split_MB: 927.3 +rank4 [2026-10-28 11:59:03,650] INFO : Step 1000/50000 rank4 | Train Loss: 2.8964 | Train Loss Unweighted: 1.7179 | Train Loss Components: Auxiliary: 0.5694 | Cardinality: 0.2509 | Classification: 0.3299 | Classification_binary: 0.4129 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4804 | Regression_eta: 0.0063 | Regression_pt: 0.4827 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2148 | Classification_binary: 0.4777 | Regression_cos_phi: 0.3164 | Regression_energy: 0.2389 | Regression_eta: 0.2012 | Regression_pt: 0.2624 | Regression_sin_phi: 0.3219 | LR: 9.99e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4527s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7367 | max_loss: 0.2419 | max_backward: 1.8137 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 1.0408 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 4746.9 | uss_MB: 3485.7 | child_rss_MB: 6281.8 | allocated_MB: 449.1 | reserved_MB: 27406.0 | inactive_split_MB: 959.2 +rank4 [2026-10-28 11:59:04,553] INFO : Running validation on rank4 +rank4 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 11:59:57,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 11:59:57,577] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 12:00:26,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 12:00:27,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 12:00:28,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 12:00:29,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 12:00:39,920] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-28 12:00:40,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:00:41,218] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 12:00:57,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 12:00:57,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 12:00:58,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 12:00:59,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 12:01:09,879] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 12:01:10,610] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:01:11,131] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 12:01:26,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 12:01:27,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 12:01:28,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 12:01:29,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 12:01:39,473] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 12:06:47,466] INFO : Step 1100/50000 rank4 | Train Loss: 2.7351 | Train Loss Unweighted: 1.6531 | Train Loss Components: Auxiliary: 0.5524 | Cardinality: 0.2470 | Classification: 0.3238 | Classification_binary: 0.3516 | HitGrouping: 0.2744 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4831 | Regression_eta: 0.0054 | Regression_pt: 0.4842 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1968 | Classification_binary: 0.3410 | Regression_cos_phi: 0.2240 | Regression_energy: 0.2421 | Regression_eta: 0.1691 | Regression_pt: 0.2617 | Regression_sin_phi: 0.2217 | LR: 9.99e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7316s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7345 | max_loss: 0.2408 | max_backward: 1.8906 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 1.1618 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6205.6 | uss_MB: 4949.9 | child_rss_MB: 6308.4 | allocated_MB: 448.2 | reserved_MB: 24590.0 | inactive_split_MB: 1051.2 +rank4 [2026-10-28 12:11:28,615] INFO : Step 1200/50000 rank4 | Train Loss: 2.9931 | Train Loss Unweighted: 1.7517 | Train Loss Components: Auxiliary: 0.5886 | Cardinality: 0.2847 | Classification: 0.3460 | Classification_binary: 0.4151 | HitGrouping: 0.2795 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4874 | Regression_eta: 0.0065 | Regression_pt: 0.4908 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2103 | Classification_binary: 0.4392 | Regression_cos_phi: 0.2673 | Regression_energy: 0.2573 | Regression_eta: 0.2015 | Regression_pt: 0.2804 | Regression_sin_phi: 0.2735 | LR: 9.99e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.9389s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0061 | max_forward: 0.9910 | max_loss: 0.1662 | max_backward: 2.3256 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.8355 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 6205.6 | uss_MB: 4943.2 | child_rss_MB: 6333.1 | allocated_MB: 449.6 | reserved_MB: 24590.0 | inactive_split_MB: 1155.8 +rank4 [2026-10-28 12:16:08,306] INFO : Step 1300/50000 rank4 | Train Loss: 3.0304 | Train Loss Unweighted: 1.7607 | Train Loss Components: Auxiliary: 0.5900 | Cardinality: 0.2909 | Classification: 0.3397 | Classification_binary: 0.4400 | HitGrouping: 0.2683 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4807 | Regression_eta: 0.0062 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2032 | Classification_binary: 0.4347 | Regression_cos_phi: 0.2516 | Regression_energy: 0.2501 | Regression_eta: 0.1458 | Regression_pt: 0.2744 | Regression_sin_phi: 0.2025 | LR: 9.98e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.3316s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8206 | max_loss: 0.1504 | max_backward: 1.9515 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5688 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5595.4 | uss_MB: 4349.7 | child_rss_MB: 6286.2 | allocated_MB: 448.6 | reserved_MB: 25778.0 | inactive_split_MB: 1050.0 +rank4 [2026-10-28 12:20:48,119] INFO : Step 1400/50000 rank4 | Train Loss: 2.5732 | Train Loss Unweighted: 1.5137 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2349 | Classification: 0.2906 | Classification_binary: 0.3573 | HitGrouping: 0.2641 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4252 | Regression_eta: 0.0055 | Regression_pt: 0.4303 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1837 | Classification_binary: 0.3591 | Regression_cos_phi: 0.2360 | Regression_energy: 0.2152 | Regression_eta: 0.2460 | Regression_pt: 0.2337 | Regression_sin_phi: 0.2388 | LR: 9.98e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.1759s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.8166 | max_loss: 0.1962 | max_backward: 1.8388 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4964 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5588.2 | uss_MB: 3765.0 | child_rss_MB: 6275.0 | allocated_MB: 454.5 | reserved_MB: 28272.0 | inactive_split_MB: 1143.7 +rank4 [2026-10-28 12:25:37,857] INFO : Step 1500/50000 rank4 | Train Loss: 2.8564 | Train Loss Unweighted: 1.5902 | Train Loss Components: Auxiliary: 0.5576 | Cardinality: 0.3788 | Classification: 0.3167 | Classification_binary: 0.3534 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4517 | Regression_eta: 0.0055 | Regression_pt: 0.4576 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1685 | Classification_binary: 0.3456 | Regression_cos_phi: 0.3053 | Regression_energy: 0.2198 | Regression_eta: 0.1568 | Regression_pt: 0.2395 | Regression_sin_phi: 0.2695 | LR: 9.98e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6161s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.0272 | max_loss: 0.2459 | max_backward: 1.8682 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.0883 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0105 | Memory: rss_MB: 5566.9 | uss_MB: 4329.3 | child_rss_MB: 6244.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 989.3 +rank4 [2026-10-28 12:30:20,922] INFO : Step 1600/50000 rank4 | Train Loss: 2.8293 | Train Loss Unweighted: 1.6754 | Train Loss Components: Auxiliary: 0.5537 | Cardinality: 0.2456 | Classification: 0.3224 | Classification_binary: 0.4168 | HitGrouping: 0.2726 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4572 | Regression_eta: 0.0058 | Regression_pt: 0.4676 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1794 | Classification_binary: 0.4011 | Regression_cos_phi: 0.2051 | Regression_energy: 0.2537 | Regression_eta: 0.1320 | Regression_pt: 0.2787 | Regression_sin_phi: 0.1842 | LR: 9.98e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5690s | Phase Times: max_pre_step_barrier: 0.1094 | max_batch_to_device: 0.0066 | max_forward: 0.7720 | max_loss: 0.1510 | max_backward: 1.7462 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5578.6 | uss_MB: 4348.9 | child_rss_MB: 6226.8 | allocated_MB: 448.1 | reserved_MB: 28272.0 | inactive_split_MB: 989.9 +rank4 [2026-10-28 12:35:04,111] INFO : Step 1700/50000 rank4 | Train Loss: 2.7295 | Train Loss Unweighted: 1.5885 | Train Loss Components: Auxiliary: 0.5261 | Cardinality: 0.2505 | Classification: 0.2942 | Classification_binary: 0.3852 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4488 | Regression_eta: 0.0059 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2002 | Classification_binary: 0.3827 | Regression_cos_phi: 0.2265 | Regression_energy: 0.2166 | Regression_eta: 0.2347 | Regression_pt: 0.2328 | Regression_sin_phi: 0.2318 | LR: 9.97e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9357s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 1.0304 | max_loss: 0.4846 | max_backward: 1.9957 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0278 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5571.8 | uss_MB: 4349.5 | child_rss_MB: 6226.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 999.4 +rank4 [2026-10-28 12:39:46,515] INFO : Step 1800/50000 rank4 | Train Loss: 2.5541 | Train Loss Unweighted: 1.4989 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2504 | Classification: 0.2799 | Classification_binary: 0.3357 | HitGrouping: 0.2658 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4350 | Regression_eta: 0.0052 | Regression_pt: 0.4384 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1778 | Classification_binary: 0.3436 | Regression_cos_phi: 0.1566 | Regression_energy: 0.2072 | Regression_eta: 0.1309 | Regression_pt: 0.2237 | Regression_sin_phi: 0.1869 | LR: 9.97e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3590s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7316 | max_loss: 0.1436 | max_backward: 1.6570 | max_optimizer_step: 0.2625 | max_tensorboard_logging: 0.9220 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5563.3 | uss_MB: 4348.3 | child_rss_MB: 6170.2 | allocated_MB: 444.7 | reserved_MB: 28272.0 | inactive_split_MB: 991.9 +rank4 [2026-10-28 12:44:29,073] INFO : Step 1900/50000 rank4 | Train Loss: 2.7201 | Train Loss Unweighted: 1.5780 | Train Loss Components: Auxiliary: 0.5257 | Cardinality: 0.2710 | Classification: 0.3152 | Classification_binary: 0.3657 | HitGrouping: 0.2653 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4371 | Regression_eta: 0.0058 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1593 | Classification_binary: 0.3250 | Regression_cos_phi: 0.2534 | Regression_energy: 0.2216 | Regression_eta: 0.1812 | Regression_pt: 0.2449 | Regression_sin_phi: 0.1794 | LR: 9.97e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4045s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.7723 | max_loss: 0.1448 | max_backward: 2.0316 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5561.2 | uss_MB: 4335.4 | child_rss_MB: 6153.7 | allocated_MB: 448.9 | reserved_MB: 28272.0 | inactive_split_MB: 1166.2 +rank4 [2026-10-28 12:49:11,978] INFO : Step 2000/50000 rank4 | Train Loss: 2.7893 | Train Loss Unweighted: 1.5772 | Train Loss Components: Auxiliary: 0.5342 | Cardinality: 0.3149 | Classification: 0.3056 | Classification_binary: 0.4233 | HitGrouping: 0.2623 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4169 | Regression_eta: 0.0058 | Regression_pt: 0.4204 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1927 | Classification_binary: 0.4517 | Regression_cos_phi: 0.2561 | Regression_energy: 0.2354 | Regression_eta: 0.1844 | Regression_pt: 0.2579 | Regression_sin_phi: 0.2542 | LR: 9.96e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.5206s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0070 | max_forward: 1.0424 | max_loss: 0.2160 | max_backward: 2.3184 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.9849 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0132 | Memory: rss_MB: 5555.4 | uss_MB: 4316.1 | child_rss_MB: 6148.5 | allocated_MB: 451.3 | reserved_MB: 28272.0 | inactive_split_MB: 992.4 +rank4 [2026-10-28 12:49:12,938] INFO : Running validation on rank4 +rank4 [2026-10-28 12:49:12,938] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 12:49:12,938] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 12:50:07,789] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:50:08,359] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 12:50:34,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 12:50:35,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 12:50:36,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 12:50:37,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 12:50:47,320] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 12:50:48,335] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:50:48,900] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 12:51:04,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 12:51:05,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 12:51:06,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 12:51:07,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 12:51:17,688] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 12:51:18,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 12:51:18,824] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 12:51:34,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 12:51:35,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 12:51:36,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 12:51:37,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 12:51:47,448] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 12:57:01,909] INFO : Step 2100/50000 rank4 | Train Loss: 2.3990 | Train Loss Unweighted: 1.4049 | Train Loss Components: Auxiliary: 0.4671 | Cardinality: 0.2199 | Classification: 0.2491 | Classification_binary: 0.3399 | HitGrouping: 0.2605 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4015 | Regression_eta: 0.0046 | Regression_pt: 0.4053 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1635 | Classification_binary: 0.3204 | Regression_cos_phi: 0.1933 | Regression_energy: 0.2073 | Regression_eta: 0.1562 | Regression_pt: 0.2270 | Regression_sin_phi: 0.2109 | LR: 9.96e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8144s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7238 | max_loss: 0.1490 | max_backward: 1.8098 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.2800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6608.5 | uss_MB: 5401.4 | child_rss_MB: 6187.1 | allocated_MB: 444.8 | reserved_MB: 23654.0 | inactive_split_MB: 964.6 +rank4 [2026-10-28 13:01:41,080] INFO : Step 2200/50000 rank4 | Train Loss: 2.6105 | Train Loss Unweighted: 1.4907 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2489 | Classification: 0.2976 | Classification_binary: 0.3552 | HitGrouping: 0.2519 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4108 | Regression_eta: 0.0054 | Regression_pt: 0.4161 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1872 | Classification_binary: 0.3836 | Regression_cos_phi: 0.2598 | Regression_energy: 0.1979 | Regression_eta: 0.1403 | Regression_pt: 0.2170 | Regression_sin_phi: 0.2104 | LR: 9.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9416s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 1.0006 | max_loss: 0.1446 | max_backward: 1.7671 | max_optimizer_step: 0.2923 | max_tensorboard_logging: 1.2733 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6609.6 | uss_MB: 5403.3 | child_rss_MB: 6178.9 | allocated_MB: 445.7 | reserved_MB: 24780.0 | inactive_split_MB: 1132.5 +rank4 [2026-10-28 13:06:12,757] INFO : Step 2300/50000 rank4 | Train Loss: 2.7462 | Train Loss Unweighted: 1.5643 | Train Loss Components: Auxiliary: 0.5175 | Cardinality: 0.2724 | Classification: 0.3064 | Classification_binary: 0.4106 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4145 | Regression_eta: 0.0056 | Regression_pt: 0.4213 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1810 | Classification_binary: 0.4342 | Regression_cos_phi: 0.2859 | Regression_energy: 0.2077 | Regression_eta: 0.1552 | Regression_pt: 0.2289 | Regression_sin_phi: 0.2024 | LR: 9.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7160s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0353 | max_forward: 0.7930 | max_loss: 0.1572 | max_backward: 1.9535 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0140 | Memory: rss_MB: 5702.4 | uss_MB: 4498.0 | child_rss_MB: 6178.9 | allocated_MB: 451.2 | reserved_MB: 26626.0 | inactive_split_MB: 1036.8 +rank4 [2026-10-28 13:10:49,216] INFO : Step 2400/50000 rank4 | Train Loss: 2.5721 | Train Loss Unweighted: 1.4496 | Train Loss Components: Auxiliary: 0.4832 | Cardinality: 0.2667 | Classification: 0.2677 | Classification_binary: 0.3686 | HitGrouping: 0.2521 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3975 | Regression_eta: 0.0053 | Regression_pt: 0.4051 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1708 | Classification_binary: 0.3742 | Regression_cos_phi: 0.2010 | Regression_energy: 0.2042 | Regression_eta: 0.1401 | Regression_pt: 0.2241 | Regression_sin_phi: 0.2618 | LR: 9.95e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.0736s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7367 | max_loss: 0.1564 | max_backward: 1.6902 | max_optimizer_step: 0.2413 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5703.1 | uss_MB: 4496.2 | child_rss_MB: 6167.5 | allocated_MB: 448.4 | reserved_MB: 26626.0 | inactive_split_MB: 1138.1 +rank4 [2026-10-28 13:15:33,528] INFO : Step 2500/50000 rank4 | Train Loss: 2.3709 | Train Loss Unweighted: 1.3478 | Train Loss Components: Auxiliary: 0.4634 | Cardinality: 0.2742 | Classification: 0.2528 | Classification_binary: 0.3100 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3861 | Regression_eta: 0.0042 | Regression_pt: 0.3903 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1105 | Classification_binary: 0.2742 | Regression_cos_phi: 0.1322 | Regression_energy: 0.2083 | Regression_eta: 0.0768 | Regression_pt: 0.2306 | Regression_sin_phi: 0.1110 | LR: 9.94e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3399s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7598 | max_loss: 0.2442 | max_backward: 1.9617 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5783 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5689.7 | uss_MB: 4483.6 | child_rss_MB: 6173.2 | allocated_MB: 447.2 | reserved_MB: 27052.0 | inactive_split_MB: 1262.0 +rank4 [2026-10-28 13:20:26,044] INFO : Step 2600/50000 rank4 | Train Loss: 2.6216 | Train Loss Unweighted: 1.5111 | Train Loss Components: Auxiliary: 0.4988 | Cardinality: 0.2823 | Classification: 0.2863 | Classification_binary: 0.3723 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4174 | Regression_eta: 0.0052 | Regression_pt: 0.4247 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1493 | Classification_binary: 0.3340 | Regression_cos_phi: 0.2327 | Regression_energy: 0.2330 | Regression_eta: 0.1247 | Regression_pt: 0.2561 | Regression_sin_phi: 0.1659 | LR: 9.94e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2875s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8125 | max_loss: 0.1494 | max_backward: 1.9790 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5698.8 | uss_MB: 4500.2 | child_rss_MB: 6171.1 | allocated_MB: 448.2 | reserved_MB: 28734.0 | inactive_split_MB: 1300.2 +rank4 [2026-10-28 13:25:15,883] INFO : Step 2700/50000 rank4 | Train Loss: 2.5735 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4801 | Cardinality: 0.2457 | Classification: 0.2692 | Classification_binary: 0.4021 | HitGrouping: 0.2514 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3948 | Regression_eta: 0.0060 | Regression_pt: 0.3979 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1954 | Classification_binary: 0.3978 | Regression_cos_phi: 0.2828 | Regression_energy: 0.2350 | Regression_eta: 0.1972 | Regression_pt: 0.2546 | Regression_sin_phi: 0.2073 | LR: 9.94e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3195s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0070 | max_forward: 0.8435 | max_loss: 0.1556 | max_backward: 1.9409 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5695.7 | uss_MB: 4500.5 | child_rss_MB: 6182.0 | allocated_MB: 447.4 | reserved_MB: 28734.0 | inactive_split_MB: 980.7 +rank4 [2026-10-28 13:29:49,852] INFO : Step 2800/50000 rank4 | Train Loss: 2.3517 | Train Loss Unweighted: 1.3315 | Train Loss Components: Auxiliary: 0.4396 | Cardinality: 0.2218 | Classification: 0.2269 | Classification_binary: 0.3206 | HitGrouping: 0.2464 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3829 | Regression_eta: 0.0052 | Regression_pt: 0.3909 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1476 | Classification_binary: 0.3060 | Regression_cos_phi: 0.2507 | Regression_energy: 0.2124 | Regression_eta: 0.1425 | Regression_pt: 0.2296 | Regression_sin_phi: 0.1821 | LR: 9.93e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3211s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0062 | max_forward: 0.8898 | max_loss: 0.1952 | max_backward: 1.8882 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0163 | Memory: rss_MB: 5687.7 | uss_MB: 4493.8 | child_rss_MB: 6179.6 | allocated_MB: 445.9 | reserved_MB: 28734.0 | inactive_split_MB: 1208.5 +rank4 [2026-10-28 13:34:34,296] INFO : Step 2900/50000 rank4 | Train Loss: 2.4505 | Train Loss Unweighted: 1.3674 | Train Loss Components: Auxiliary: 0.4603 | Cardinality: 0.2746 | Classification: 0.2611 | Classification_binary: 0.3388 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3750 | Regression_eta: 0.0049 | Regression_pt: 0.3827 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1698 | Classification_binary: 0.3548 | Regression_cos_phi: 0.2009 | Regression_energy: 0.1964 | Regression_eta: 0.1249 | Regression_pt: 0.2146 | Regression_sin_phi: 0.1760 | LR: 9.93e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.3942s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7484 | max_loss: 0.6089 | max_backward: 2.2353 | max_optimizer_step: 0.1775 | max_tensorboard_logging: 2.4313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0155 | Memory: rss_MB: 5691.8 | uss_MB: 4497.3 | child_rss_MB: 6175.9 | allocated_MB: 446.7 | reserved_MB: 28734.0 | inactive_split_MB: 1133.2 +rank4 [2026-10-28 13:39:22,258] INFO : Step 3000/50000 rank4 | Train Loss: 2.6179 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4760 | Cardinality: 0.2600 | Classification: 0.3170 | Classification_binary: 0.3907 | HitGrouping: 0.2509 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3743 | Regression_eta: 0.0054 | Regression_pt: 0.3819 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1795 | Classification_binary: 0.3719 | Regression_cos_phi: 0.2041 | Regression_energy: 0.1899 | Regression_eta: 0.1597 | Regression_pt: 0.2110 | Regression_sin_phi: 0.2306 | LR: 9.92e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8199s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0842 | max_forward: 0.8203 | max_loss: 0.1511 | max_backward: 1.9714 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.0307 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5691.4 | uss_MB: 4499.3 | child_rss_MB: 6154.8 | allocated_MB: 451.4 | reserved_MB: 28740.0 | inactive_split_MB: 1256.0 +rank4 [2026-10-28 13:39:23,129] INFO : Running validation on rank4 +rank4 [2026-10-28 13:39:23,129] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 13:39:23,129] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 13:40:15,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:40:16,490] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 13:40:43,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 13:40:44,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 13:40:45,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 13:40:46,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 13:40:56,914] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 13:40:57,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:40:58,635] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 13:41:14,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 13:41:15,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 13:41:16,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 13:41:17,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 13:41:27,638] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 13:41:28,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 13:41:28,598] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 13:41:44,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 13:41:45,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 13:41:45,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 13:41:46,690] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 13:41:56,700] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 13:46:51,409] INFO : Step 3100/50000 rank4 | Train Loss: 2.3456 | Train Loss Unweighted: 1.3820 | Train Loss Components: Auxiliary: 0.4397 | Cardinality: 0.2017 | Classification: 0.2549 | Classification_binary: 0.3432 | HitGrouping: 0.2399 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3852 | Regression_eta: 0.0046 | Regression_pt: 0.3892 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1538 | Classification_binary: 0.3524 | Regression_cos_phi: 0.1976 | Regression_energy: 0.2098 | Regression_eta: 0.1377 | Regression_pt: 0.2290 | Regression_sin_phi: 0.2534 | LR: 9.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4922s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0425 | max_forward: 0.7824 | max_loss: 0.1439 | max_backward: 1.7183 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.8666 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6905.9 | uss_MB: 5708.9 | child_rss_MB: 6176.0 | allocated_MB: 447.6 | reserved_MB: 24468.0 | inactive_split_MB: 1269.5 +rank4 [2026-10-28 13:51:36,981] INFO : Step 3200/50000 rank4 | Train Loss: 2.2453 | Train Loss Unweighted: 1.2729 | Train Loss Components: Auxiliary: 0.4171 | Cardinality: 0.2143 | Classification: 0.2366 | Classification_binary: 0.3116 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3569 | Regression_eta: 0.0047 | Regression_pt: 0.3584 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1618 | Classification_binary: 0.3398 | Regression_cos_phi: 0.2377 | Regression_energy: 0.1847 | Regression_eta: 0.1510 | Regression_pt: 0.1984 | Regression_sin_phi: 0.1952 | LR: 9.91e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6646s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7985 | max_loss: 0.1379 | max_backward: 1.7895 | max_optimizer_step: 0.2027 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6908.5 | uss_MB: 5710.9 | child_rss_MB: 6167.1 | allocated_MB: 448.6 | reserved_MB: 26448.0 | inactive_split_MB: 1391.4 +rank4 [2026-10-28 13:56:16,618] INFO : Step 3300/50000 rank4 | Train Loss: 2.5474 | Train Loss Unweighted: 1.4472 | Train Loss Components: Auxiliary: 0.4779 | Cardinality: 0.2732 | Classification: 0.2824 | Classification_binary: 0.3640 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3911 | Regression_eta: 0.0054 | Regression_pt: 0.3992 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1527 | Classification_binary: 0.3341 | Regression_cos_phi: 0.1861 | Regression_energy: 0.2181 | Regression_eta: 0.1183 | Regression_pt: 0.2414 | Regression_sin_phi: 0.1542 | LR: 9.90e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6662s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8323 | max_loss: 0.1479 | max_backward: 2.0543 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.7834 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5785.1 | uss_MB: 4587.8 | child_rss_MB: 6143.0 | allocated_MB: 448.7 | reserved_MB: 26460.0 | inactive_split_MB: 1087.4 +rank4 [2026-10-28 14:00:49,247] INFO : Step 3400/50000 rank4 | Train Loss: 2.5743 | Train Loss Unweighted: 1.4358 | Train Loss Components: Auxiliary: 0.4720 | Cardinality: 0.2664 | Classification: 0.2699 | Classification_binary: 0.3915 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3784 | Regression_eta: 0.0053 | Regression_pt: 0.3851 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1869 | Classification_binary: 0.3895 | Regression_cos_phi: 0.2924 | Regression_energy: 0.2089 | Regression_eta: 0.1782 | Regression_pt: 0.2298 | Regression_sin_phi: 0.2212 | LR: 9.90e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2594s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7827 | max_loss: 0.1630 | max_backward: 1.9191 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5606 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5794.1 | uss_MB: 4597.7 | child_rss_MB: 6161.5 | allocated_MB: 450.4 | reserved_MB: 28138.0 | inactive_split_MB: 1261.0 +rank4 [2026-10-28 14:05:23,603] INFO : Step 3500/50000 rank4 | Train Loss: 2.1916 | Train Loss Unweighted: 1.2418 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2204 | Classification: 0.2212 | Classification_binary: 0.3041 | HitGrouping: 0.2426 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3517 | Regression_eta: 0.0047 | Regression_pt: 0.3558 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1289 | Classification_binary: 0.3337 | Regression_cos_phi: 0.1742 | Regression_energy: 0.1755 | Regression_eta: 0.1287 | Regression_pt: 0.1898 | Regression_sin_phi: 0.1854 | LR: 9.89e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2270s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7972 | max_loss: 0.1441 | max_backward: 1.7398 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6845 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5780.0 | uss_MB: 4584.6 | child_rss_MB: 6167.3 | allocated_MB: 443.7 | reserved_MB: 28532.0 | inactive_split_MB: 1176.3 +rank4 [2026-10-28 14:10:06,691] INFO : Step 3600/50000 rank4 | Train Loss: 2.3195 | Train Loss Unweighted: 1.3024 | Train Loss Components: Auxiliary: 0.4248 | Cardinality: 0.2221 | Classification: 0.2518 | Classification_binary: 0.3172 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3610 | Regression_eta: 0.0052 | Regression_pt: 0.3623 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1105 | Classification_binary: 0.2640 | Regression_cos_phi: 0.1661 | Regression_energy: 0.2039 | Regression_eta: 0.1321 | Regression_pt: 0.2201 | Regression_sin_phi: 0.1440 | LR: 9.89e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3064s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8456 | max_loss: 0.1388 | max_backward: 1.9466 | max_optimizer_step: 0.1811 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5787.8 | uss_MB: 4598.8 | child_rss_MB: 6163.2 | allocated_MB: 449.1 | reserved_MB: 28532.0 | inactive_split_MB: 1277.8 +rank4 [2026-10-28 14:14:47,797] INFO : Step 3700/50000 rank4 | Train Loss: 2.5861 | Train Loss Unweighted: 1.4427 | Train Loss Components: Auxiliary: 0.4700 | Cardinality: 0.2678 | Classification: 0.2978 | Classification_binary: 0.3844 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3719 | Regression_eta: 0.0058 | Regression_pt: 0.3773 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1873 | Classification_binary: 0.3585 | Regression_cos_phi: 0.2073 | Regression_energy: 0.2076 | Regression_eta: 0.1799 | Regression_pt: 0.2300 | Regression_sin_phi: 0.2168 | LR: 9.88e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5011s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.8001 | max_loss: 0.1519 | max_backward: 1.8239 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.7916 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0181 | Memory: rss_MB: 5775.1 | uss_MB: 4600.5 | child_rss_MB: 6165.4 | allocated_MB: 450.6 | reserved_MB: 30118.0 | inactive_split_MB: 1097.6 +rank4 [2026-10-28 14:19:24,599] INFO : Step 3800/50000 rank4 | Train Loss: 2.3269 | Train Loss Unweighted: 1.3016 | Train Loss Components: Auxiliary: 0.4225 | Cardinality: 0.2364 | Classification: 0.2400 | Classification_binary: 0.3389 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3534 | Regression_eta: 0.0054 | Regression_pt: 0.3591 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1373 | Classification_binary: 0.3488 | Regression_cos_phi: 0.1513 | Regression_energy: 0.1716 | Regression_eta: 0.1266 | Regression_pt: 0.1884 | Regression_sin_phi: 0.1901 | LR: 9.87e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.4711s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7555 | max_loss: 0.1601 | max_backward: 1.7612 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.9394 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5772.0 | uss_MB: 4595.7 | child_rss_MB: 6158.5 | allocated_MB: 447.4 | reserved_MB: 30118.0 | inactive_split_MB: 1355.2 +rank4 [2026-10-28 14:24:05,269] INFO : Step 3900/50000 rank4 | Train Loss: 2.0276 | Train Loss Unweighted: 1.1874 | Train Loss Components: Auxiliary: 0.3861 | Cardinality: 0.1825 | Classification: 0.2116 | Classification_binary: 0.2707 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3485 | Regression_eta: 0.0038 | Regression_pt: 0.3489 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1140 | Classification_binary: 0.2213 | Regression_cos_phi: 0.1260 | Regression_energy: 0.1902 | Regression_eta: 0.0697 | Regression_pt: 0.2061 | Regression_sin_phi: 0.1014 | LR: 9.87e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.4478s | Phase Times: max_pre_step_barrier: 0.0095 | max_batch_to_device: 0.0140 | max_forward: 1.1007 | max_loss: 0.1867 | max_backward: 2.0952 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 0.5091 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5772.2 | uss_MB: 4598.5 | child_rss_MB: 6165.7 | allocated_MB: 448.6 | reserved_MB: 30118.0 | inactive_split_MB: 1119.4 +rank4 [2026-10-28 14:28:52,325] INFO : Step 4000/50000 rank4 | Train Loss: 2.3921 | Train Loss Unweighted: 1.3683 | Train Loss Components: Auxiliary: 0.4366 | Cardinality: 0.2254 | Classification: 0.2720 | Classification_binary: 0.3524 | HitGrouping: 0.2405 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3647 | Regression_eta: 0.0048 | Regression_pt: 0.3692 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1666 | Classification_binary: 0.3604 | Regression_cos_phi: 0.2271 | Regression_energy: 0.2059 | Regression_eta: 0.1246 | Regression_pt: 0.2256 | Regression_sin_phi: 0.2056 | LR: 9.86e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8905s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.8532 | max_loss: 0.1476 | max_backward: 2.4362 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.6138 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5773.5 | uss_MB: 4601.5 | child_rss_MB: 6164.9 | allocated_MB: 449.7 | reserved_MB: 30118.0 | inactive_split_MB: 899.8 +rank4 [2026-10-28 14:28:53,308] INFO : Running validation on rank4 +rank4 [2026-10-28 14:28:53,308] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 14:28:53,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 14:29:46,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:29:46,700] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 14:30:12,961] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 14:30:13,950] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 14:30:14,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 14:30:16,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 14:30:26,032] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 14:30:26,980] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:30:27,512] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 14:30:43,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 14:30:44,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 14:30:45,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 14:30:46,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 14:30:56,218] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 14:30:57,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 14:30:58,054] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 14:31:13,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 14:31:14,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 14:31:15,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 14:31:16,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 14:31:26,295] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 14:36:30,159] INFO : Step 4100/50000 rank4 | Train Loss: 2.5062 | Train Loss Unweighted: 1.3582 | Train Loss Components: Auxiliary: 0.4559 | Cardinality: 0.3208 | Classification: 0.2565 | Classification_binary: 0.3823 | HitGrouping: 0.2386 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3543 | Regression_eta: 0.0046 | Regression_pt: 0.3553 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1604 | Classification_binary: 0.4276 | Regression_cos_phi: 0.2255 | Regression_energy: 0.1702 | Regression_eta: 0.1266 | Regression_pt: 0.1860 | Regression_sin_phi: 0.2092 | LR: 9.85e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.2036s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0069 | max_forward: 0.8943 | max_loss: 0.2518 | max_backward: 1.9865 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 1.1537 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6976.0 | uss_MB: 5798.7 | child_rss_MB: 6190.7 | allocated_MB: 449.4 | reserved_MB: 25148.0 | inactive_split_MB: 1028.6 +rank4 [2026-10-28 14:41:18,155] INFO : Step 4200/50000 rank4 | Train Loss: 2.0961 | Train Loss Unweighted: 1.2232 | Train Loss Components: Auxiliary: 0.3927 | Cardinality: 0.2101 | Classification: 0.2242 | Classification_binary: 0.2902 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3497 | Regression_eta: 0.0042 | Regression_pt: 0.3510 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1245 | Classification_binary: 0.2494 | Regression_cos_phi: 0.1445 | Regression_energy: 0.1778 | Regression_eta: 0.0994 | Regression_pt: 0.1975 | Regression_sin_phi: 0.1846 | LR: 9.84e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.6547s | Phase Times: max_pre_step_barrier: 0.1131 | max_batch_to_device: 0.0199 | max_forward: 1.5911 | max_loss: 0.3393 | max_backward: 3.0707 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5197 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6973.9 | uss_MB: 5797.7 | child_rss_MB: 6186.7 | allocated_MB: 446.0 | reserved_MB: 25296.0 | inactive_split_MB: 922.1 +rank4 [2026-10-28 14:46:18,703] INFO : Step 4300/50000 rank4 | Train Loss: 2.2792 | Train Loss Unweighted: 1.3065 | Train Loss Components: Auxiliary: 0.4224 | Cardinality: 0.2158 | Classification: 0.2339 | Classification_binary: 0.3250 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3669 | Regression_eta: 0.0044 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1389 | Classification_binary: 0.3235 | Regression_cos_phi: 0.1521 | Regression_energy: 0.2098 | Regression_eta: 0.1153 | Regression_pt: 0.2296 | Regression_sin_phi: 0.2058 | LR: 9.84e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5472s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0938 | max_forward: 0.8451 | max_loss: 0.1396 | max_backward: 1.9669 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.6639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5862.7 | uss_MB: 4685.4 | child_rss_MB: 6195.4 | allocated_MB: 448.4 | reserved_MB: 28580.0 | inactive_split_MB: 1346.0 +rank4 [2026-10-28 14:50:58,886] INFO : Step 4400/50000 rank4 | Train Loss: 2.4955 | Train Loss Unweighted: 1.3385 | Train Loss Components: Auxiliary: 0.4510 | Cardinality: 0.3020 | Classification: 0.2394 | Classification_binary: 0.3842 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3482 | Regression_eta: 0.0051 | Regression_pt: 0.3563 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1665 | Classification_binary: 0.4264 | Regression_cos_phi: 0.2974 | Regression_energy: 0.1941 | Regression_eta: 0.1721 | Regression_pt: 0.2143 | Regression_sin_phi: 0.1981 | LR: 9.83e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2054s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0083 | max_forward: 0.7889 | max_loss: 0.1505 | max_backward: 1.8572 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4868 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5870.1 | uss_MB: 4697.1 | child_rss_MB: 6175.1 | allocated_MB: 449.1 | reserved_MB: 28582.0 | inactive_split_MB: 1105.9 +rank4 [2026-10-28 14:55:40,224] INFO : Step 4500/50000 rank4 | Train Loss: 2.3283 | Train Loss Unweighted: 1.2750 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.3015 | Classification: 0.2318 | Classification_binary: 0.3495 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3409 | Regression_eta: 0.0044 | Regression_pt: 0.3441 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1432 | Classification_binary: 0.3686 | Regression_cos_phi: 0.2017 | Regression_energy: 0.1832 | Regression_eta: 0.1330 | Regression_pt: 0.1986 | Regression_sin_phi: 0.2335 | LR: 9.82e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0171s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7326 | max_loss: 0.2428 | max_backward: 1.7841 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5868.2 | uss_MB: 4696.0 | child_rss_MB: 6178.6 | allocated_MB: 447.8 | reserved_MB: 28582.0 | inactive_split_MB: 1258.0 +rank4 [2026-10-28 15:00:16,448] INFO : Step 4600/50000 rank4 | Train Loss: 2.0353 | Train Loss Unweighted: 1.1777 | Train Loss Components: Auxiliary: 0.3804 | Cardinality: 0.1920 | Classification: 0.2053 | Classification_binary: 0.2642 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3490 | Regression_eta: 0.0039 | Regression_pt: 0.3512 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1554 | Classification_binary: 0.2700 | Regression_cos_phi: 0.1518 | Regression_energy: 0.1841 | Regression_eta: 0.1081 | Regression_pt: 0.1980 | Regression_sin_phi: 0.1779 | LR: 9.81e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0847s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0402 | max_forward: 0.7143 | max_loss: 0.1412 | max_backward: 1.6726 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 5861.4 | uss_MB: 4694.6 | child_rss_MB: 6186.8 | allocated_MB: 446.4 | reserved_MB: 28582.0 | inactive_split_MB: 981.4 +rank4 [2026-10-28 15:05:12,090] INFO : Step 4700/50000 rank4 | Train Loss: 2.2426 | Train Loss Unweighted: 1.2576 | Train Loss Components: Auxiliary: 0.4073 | Cardinality: 0.2222 | Classification: 0.2485 | Classification_binary: 0.3333 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3307 | Regression_eta: 0.0049 | Regression_pt: 0.3356 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1465 | Classification_binary: 0.3081 | Regression_cos_phi: 0.1941 | Regression_energy: 0.1680 | Regression_eta: 0.1578 | Regression_pt: 0.1821 | Regression_sin_phi: 0.2362 | LR: 9.81e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6201s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.1471 | max_backward: 2.0627 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.7062 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0769 | Memory: rss_MB: 5860.5 | uss_MB: 4699.2 | child_rss_MB: 6169.2 | allocated_MB: 447.7 | reserved_MB: 28652.0 | inactive_split_MB: 1164.9 +rank4 [2026-10-28 15:09:59,906] INFO : Step 4800/50000 rank4 | Train Loss: 2.2697 | Train Loss Unweighted: 1.2637 | Train Loss Components: Auxiliary: 0.4132 | Cardinality: 0.2493 | Classification: 0.2262 | Classification_binary: 0.3667 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3287 | Regression_eta: 0.0044 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1334 | Classification_binary: 0.3745 | Regression_cos_phi: 0.2169 | Regression_energy: 0.1673 | Regression_eta: 0.1493 | Regression_pt: 0.1807 | Regression_sin_phi: 0.1955 | LR: 9.80e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0687s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0065 | max_forward: 0.7626 | max_loss: 0.1571 | max_backward: 1.6465 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5855.3 | uss_MB: 4693.4 | child_rss_MB: 6188.9 | allocated_MB: 450.7 | reserved_MB: 28652.0 | inactive_split_MB: 1088.6 +rank4 [2026-10-28 15:14:51,789] INFO : Step 4900/50000 rank4 | Train Loss: 2.0529 | Train Loss Unweighted: 1.1598 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.1942 | Classification: 0.2269 | Classification_binary: 0.2874 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3172 | Regression_eta: 0.0041 | Regression_pt: 0.3199 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1227 | Classification_binary: 0.3045 | Regression_cos_phi: 0.1891 | Regression_energy: 0.1795 | Regression_eta: 0.0905 | Regression_pt: 0.1969 | Regression_sin_phi: 0.1417 | LR: 9.79e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3248s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7293 | max_loss: 0.1974 | max_backward: 1.8633 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5858.6 | uss_MB: 4697.0 | child_rss_MB: 6189.0 | allocated_MB: 447.4 | reserved_MB: 28652.0 | inactive_split_MB: 1266.1 +rank4 [2026-10-28 15:19:59,464] INFO : Step 5000/50000 rank4 | Train Loss: 2.0827 | Train Loss Unweighted: 1.1872 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2154 | Classification: 0.2462 | Classification_binary: 0.2941 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3169 | Regression_eta: 0.0042 | Regression_pt: 0.3219 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1538 | Classification_binary: 0.3001 | Regression_cos_phi: 0.1466 | Regression_energy: 0.1453 | Regression_eta: 0.0888 | Regression_pt: 0.1604 | Regression_sin_phi: 0.1709 | LR: 9.78e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3253s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.8382 | max_loss: 0.1450 | max_backward: 1.9472 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0203 | Memory: rss_MB: 5858.9 | uss_MB: 4697.5 | child_rss_MB: 6177.0 | allocated_MB: 446.9 | reserved_MB: 29316.0 | inactive_split_MB: 980.8 +rank4 [2026-10-28 15:20:00,329] INFO : Running validation on rank4 +rank4 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 15:20:55,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:20:56,481] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 15:21:22,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 15:21:23,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 15:21:24,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 15:21:25,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 15:21:35,861] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 15:21:36,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:21:36,965] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 15:21:53,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 15:21:54,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 15:21:55,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 15:21:56,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 15:22:06,020] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 15:22:06,553] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 15:22:07,053] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 15:22:22,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 15:22:24,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 15:22:25,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 15:22:26,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 15:22:36,104] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 15:28:09,086] INFO : Step 5100/50000 rank4 | Train Loss: 2.4098 | Train Loss Unweighted: 1.3457 | Train Loss Components: Auxiliary: 0.4318 | Cardinality: 0.2397 | Classification: 0.2711 | Classification_binary: 0.3798 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3391 | Regression_eta: 0.0051 | Regression_pt: 0.3453 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1456 | Classification_binary: 0.3687 | Regression_cos_phi: 0.1649 | Regression_energy: 0.1847 | Regression_eta: 0.1169 | Regression_pt: 0.2035 | Regression_sin_phi: 0.2035 | LR: 9.77e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.8407s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1511 | max_backward: 1.7914 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 1.0544 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5949.7 | uss_MB: 4786.0 | child_rss_MB: 6159.3 | allocated_MB: 447.6 | reserved_MB: 25650.0 | inactive_split_MB: 1122.4 +rank4 [2026-10-28 15:32:58,661] INFO : Step 5200/50000 rank4 | Train Loss: 2.0929 | Train Loss Unweighted: 1.1974 | Train Loss Components: Auxiliary: 0.3875 | Cardinality: 0.2007 | Classification: 0.2193 | Classification_binary: 0.3278 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3194 | Regression_eta: 0.0041 | Regression_pt: 0.3227 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1145 | Classification_binary: 0.3020 | Regression_cos_phi: 0.1408 | Regression_energy: 0.1761 | Regression_eta: 0.1196 | Regression_pt: 0.1881 | Regression_sin_phi: 0.1674 | LR: 9.76e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4915s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7458 | max_loss: 0.2423 | max_backward: 1.7123 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9648 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5957.0 | uss_MB: 4796.7 | child_rss_MB: 6179.0 | allocated_MB: 450.7 | reserved_MB: 28708.0 | inactive_split_MB: 1139.3 +rank4 [2026-10-28 15:37:52,596] INFO : Step 5300/50000 rank4 | Train Loss: 1.9905 | Train Loss Unweighted: 1.1352 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.2083 | Classification: 0.2239 | Classification_binary: 0.2818 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0037 | Regression_pt: 0.3135 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1528 | Classification_binary: 0.2928 | Regression_cos_phi: 0.1362 | Regression_energy: 0.1637 | Regression_eta: 0.1114 | Regression_pt: 0.1770 | Regression_sin_phi: 0.1649 | LR: 9.75e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1040s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 1.2453 | max_loss: 0.2326 | max_backward: 2.1392 | max_optimizer_step: 0.1883 | max_tensorboard_logging: 1.0947 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5945.0 | uss_MB: 4784.6 | child_rss_MB: 6156.3 | allocated_MB: 445.9 | reserved_MB: 30688.0 | inactive_split_MB: 925.3 +rank4 [2026-10-28 15:42:39,663] INFO : Step 5400/50000 rank4 | Train Loss: 2.2932 | Train Loss Unweighted: 1.2957 | Train Loss Components: Auxiliary: 0.4189 | Cardinality: 0.2411 | Classification: 0.2689 | Classification_binary: 0.3420 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3351 | Regression_eta: 0.0045 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1245 | Classification_binary: 0.3288 | Regression_cos_phi: 0.1257 | Regression_energy: 0.1849 | Regression_eta: 0.1220 | Regression_pt: 0.2006 | Regression_sin_phi: 0.2199 | LR: 9.74e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5700s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7688 | max_loss: 0.1697 | max_backward: 1.8189 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8641 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5955.2 | uss_MB: 4796.8 | child_rss_MB: 6163.2 | allocated_MB: 450.8 | reserved_MB: 30688.0 | inactive_split_MB: 915.1 +rank4 [2026-10-28 15:47:42,229] INFO : Step 5500/50000 rank4 | Train Loss: 2.1734 | Train Loss Unweighted: 1.2295 | Train Loss Components: Auxiliary: 0.3970 | Cardinality: 0.2269 | Classification: 0.2218 | Classification_binary: 0.3571 | HitGrouping: 0.2200 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3188 | Regression_eta: 0.0041 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1711 | Classification_binary: 0.4193 | Regression_cos_phi: 0.2208 | Regression_energy: 0.1750 | Regression_eta: 0.1470 | Regression_pt: 0.1917 | Regression_sin_phi: 0.2207 | LR: 9.73e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.9284s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0153 | max_forward: 1.1913 | max_loss: 0.6135 | max_backward: 3.0599 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.6982 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5952.5 | uss_MB: 4797.4 | child_rss_MB: 6170.1 | allocated_MB: 446.9 | reserved_MB: 32804.0 | inactive_split_MB: 913.2 +rank4 [2026-10-28 15:52:30,289] INFO : Step 5600/50000 rank4 | Train Loss: 2.0307 | Train Loss Unweighted: 1.1169 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.1966 | Classification: 0.2085 | Classification_binary: 0.3031 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2975 | Regression_eta: 0.0047 | Regression_pt: 0.2988 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1008 | Classification_binary: 0.3250 | Regression_cos_phi: 0.1696 | Regression_energy: 0.1490 | Regression_eta: 0.0993 | Regression_pt: 0.1618 | Regression_sin_phi: 0.2085 | LR: 9.72e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7958s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7794 | max_loss: 0.1497 | max_backward: 1.9410 | max_optimizer_step: 0.5154 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0061 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5920.9 | uss_MB: 4797.0 | child_rss_MB: 6174.4 | allocated_MB: 445.4 | reserved_MB: 32804.0 | inactive_split_MB: 1164.2 +rank4 [2026-10-28 15:57:16,862] INFO : Step 5700/50000 rank4 | Train Loss: 2.2064 | Train Loss Unweighted: 1.2224 | Train Loss Components: Auxiliary: 0.4023 | Cardinality: 0.2320 | Classification: 0.2526 | Classification_binary: 0.3174 | HitGrouping: 0.2343 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3173 | Regression_eta: 0.0045 | Regression_pt: 0.3259 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1630 | Classification_binary: 0.3311 | Regression_cos_phi: 0.1833 | Regression_energy: 0.1883 | Regression_eta: 0.1353 | Regression_pt: 0.2083 | Regression_sin_phi: 0.2062 | LR: 9.71e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6667s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7746 | max_loss: 0.4928 | max_backward: 1.9590 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8719 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 5893.1 | uss_MB: 3755.9 | child_rss_MB: 6168.2 | allocated_MB: 448.4 | reserved_MB: 32804.0 | inactive_split_MB: 913.0 +rank4 [2026-10-28 16:02:01,175] INFO : Step 5800/50000 rank4 | Train Loss: 2.1438 | Train Loss Unweighted: 1.2123 | Train Loss Components: Auxiliary: 0.3898 | Cardinality: 0.2335 | Classification: 0.2404 | Classification_binary: 0.3575 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3006 | Regression_eta: 0.0042 | Regression_pt: 0.3054 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1315 | Classification_binary: 0.3773 | Regression_cos_phi: 0.1780 | Regression_energy: 0.1564 | Regression_eta: 0.1039 | Regression_pt: 0.1733 | Regression_sin_phi: 0.1731 | LR: 9.70e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.5894s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8028 | max_loss: 0.1540 | max_backward: 1.8645 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.8379 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5883.5 | uss_MB: 4797.1 | child_rss_MB: 6169.4 | allocated_MB: 448.3 | reserved_MB: 32804.0 | inactive_split_MB: 916.5 +rank4 [2026-10-28 16:06:46,106] INFO : Step 5900/50000 rank4 | Train Loss: 2.2094 | Train Loss Unweighted: 1.1453 | Train Loss Components: Auxiliary: 0.3936 | Cardinality: 0.2891 | Classification: 0.2198 | Classification_binary: 0.3337 | HitGrouping: 0.2238 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2890 | Regression_eta: 0.0046 | Regression_pt: 0.2935 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1354 | Classification_binary: 0.3176 | Regression_cos_phi: 0.1635 | Regression_energy: 0.1276 | Regression_eta: 0.1139 | Regression_pt: 0.1403 | Regression_sin_phi: 0.1639 | LR: 9.69e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.5878s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0065 | max_forward: 1.0922 | max_loss: 0.1477 | max_backward: 1.8468 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8251 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0991 | Memory: rss_MB: 5881.4 | uss_MB: 4798.1 | child_rss_MB: 6148.2 | allocated_MB: 446.5 | reserved_MB: 32804.0 | inactive_split_MB: 927.1 +rank4 [2026-10-28 16:11:31,999] INFO : Step 6000/50000 rank4 | Train Loss: 1.8458 | Train Loss Unweighted: 1.0363 | Train Loss Components: Auxiliary: 0.3398 | Cardinality: 0.1796 | Classification: 0.1819 | Classification_binary: 0.2710 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2865 | Regression_eta: 0.0032 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1168 | Classification_binary: 0.2717 | Regression_cos_phi: 0.1903 | Regression_energy: 0.1673 | Regression_eta: 0.0954 | Regression_pt: 0.1805 | Regression_sin_phi: 0.1804 | LR: 9.68e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5099s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7380 | max_loss: 0.2483 | max_backward: 1.8094 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.9243 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5850.9 | uss_MB: 4795.2 | child_rss_MB: 6173.4 | allocated_MB: 448.1 | reserved_MB: 32804.0 | inactive_split_MB: 1232.3 +rank4 [2026-10-28 16:11:32,849] INFO : Running validation on rank4 +rank4 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 16:12:26,407] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:12:26,924] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 16:12:53,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 16:12:54,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 16:12:55,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 16:12:57,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 16:13:07,085] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 16:13:07,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:13:08,323] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 16:13:24,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 16:13:25,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 16:13:26,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 16:13:27,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 16:13:37,217] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 16:13:38,169] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 16:13:38,690] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 16:13:54,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 16:13:55,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 16:13:55,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 16:13:56,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 16:14:06,734] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 16:19:13,933] INFO : Step 6100/50000 rank4 | Train Loss: 2.0818 | Train Loss Unweighted: 1.1297 | Train Loss Components: Auxiliary: 0.3734 | Cardinality: 0.2287 | Classification: 0.2079 | Classification_binary: 0.3249 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2929 | Regression_eta: 0.0044 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1267 | Classification_binary: 0.3103 | Regression_cos_phi: 0.1812 | Regression_energy: 0.1403 | Regression_eta: 0.1383 | Regression_pt: 0.1524 | Regression_sin_phi: 0.1745 | LR: 9.67e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.5156s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8130 | max_loss: 0.1524 | max_backward: 3.1921 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 1.4719 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5941.1 | uss_MB: 4883.2 | child_rss_MB: 6189.9 | allocated_MB: 448.4 | reserved_MB: 24434.0 | inactive_split_MB: 1287.8 +rank4 [2026-10-28 16:23:58,090] INFO : Step 6200/50000 rank4 | Train Loss: 2.3716 | Train Loss Unweighted: 1.2065 | Train Loss Components: Auxiliary: 0.4163 | Cardinality: 0.3927 | Classification: 0.2343 | Classification_binary: 0.3679 | HitGrouping: 0.2165 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2973 | Regression_eta: 0.0043 | Regression_pt: 0.2981 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1457 | Classification_binary: 0.3714 | Regression_cos_phi: 0.2742 | Regression_energy: 0.1662 | Regression_eta: 0.1298 | Regression_pt: 0.1790 | Regression_sin_phi: 0.2443 | LR: 9.66e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5606s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8556 | max_loss: 0.1996 | max_backward: 1.9593 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.5823 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5946.6 | uss_MB: 3767.3 | child_rss_MB: 6181.8 | allocated_MB: 447.9 | reserved_MB: 27020.0 | inactive_split_MB: 1185.4 +rank4 [2026-10-28 16:28:38,485] INFO : Step 6300/50000 rank4 | Train Loss: 1.9346 | Train Loss Unweighted: 1.0816 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.2071 | Classification: 0.2053 | Classification_binary: 0.2781 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2941 | Regression_eta: 0.0037 | Regression_pt: 0.2966 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1363 | Classification_binary: 0.2693 | Regression_cos_phi: 0.1356 | Regression_energy: 0.1683 | Regression_eta: 0.0979 | Regression_pt: 0.1750 | Regression_sin_phi: 0.1624 | LR: 9.65e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5897s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0068 | max_forward: 0.7412 | max_loss: 0.1444 | max_backward: 1.7431 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.9045 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5941.6 | uss_MB: 4896.5 | child_rss_MB: 6164.9 | allocated_MB: 444.6 | reserved_MB: 27022.0 | inactive_split_MB: 1185.4 +rank4 [2026-10-28 16:33:23,144] INFO : Step 6400/50000 rank4 | Train Loss: 2.0123 | Train Loss Unweighted: 1.0992 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2082 | Classification: 0.2291 | Classification_binary: 0.2885 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2842 | Regression_eta: 0.0040 | Regression_pt: 0.2891 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1021 | Classification_binary: 0.2359 | Regression_cos_phi: 0.0881 | Regression_energy: 0.1385 | Regression_eta: 0.0669 | Regression_pt: 0.1520 | Regression_sin_phi: 0.0821 | LR: 9.64e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.7580s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0069 | max_forward: 1.1063 | max_loss: 0.2471 | max_backward: 1.9949 | max_optimizer_step: 0.2726 | max_tensorboard_logging: 0.8233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5936.1 | uss_MB: 4896.3 | child_rss_MB: 6176.5 | allocated_MB: 451.6 | reserved_MB: 27022.0 | inactive_split_MB: 1113.0 +rank4 [2026-10-28 16:37:57,182] INFO : Step 6500/50000 rank4 | Train Loss: 2.1791 | Train Loss Unweighted: 1.2220 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2499 | Classification: 0.2271 | Classification_binary: 0.3508 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3177 | Regression_eta: 0.0042 | Regression_pt: 0.3181 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1194 | Classification_binary: 0.3334 | Regression_cos_phi: 0.1510 | Regression_energy: 0.1550 | Regression_eta: 0.1097 | Regression_pt: 0.1671 | Regression_sin_phi: 0.1387 | LR: 9.63e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.9960s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0548 | max_forward: 0.8069 | max_loss: 0.1518 | max_backward: 2.0692 | max_optimizer_step: 0.1860 | max_tensorboard_logging: 0.9414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5926.6 | uss_MB: 4895.5 | child_rss_MB: 6160.0 | allocated_MB: 449.6 | reserved_MB: 28336.0 | inactive_split_MB: 1199.2 +rank4 [2026-10-28 16:42:33,043] INFO : Step 6600/50000 rank4 | Train Loss: 1.9690 | Train Loss Unweighted: 1.0745 | Train Loss Components: Auxiliary: 0.3537 | Cardinality: 0.2132 | Classification: 0.1982 | Classification_binary: 0.3095 | HitGrouping: 0.2136 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2771 | Regression_eta: 0.0040 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1357 | Classification_binary: 0.3496 | Regression_cos_phi: 0.1821 | Regression_energy: 0.1450 | Regression_eta: 0.1158 | Regression_pt: 0.1591 | Regression_sin_phi: 0.1678 | LR: 9.62e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8257s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7382 | max_loss: 0.1467 | max_backward: 1.8335 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 1.1664 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5923.1 | uss_MB: 4893.3 | child_rss_MB: 6174.8 | allocated_MB: 447.4 | reserved_MB: 28340.0 | inactive_split_MB: 1012.8 +rank4 [2026-10-28 16:47:06,903] INFO : Step 6700/50000 rank4 | Train Loss: 1.9470 | Train Loss Unweighted: 1.0781 | Train Loss Components: Auxiliary: 0.3616 | Cardinality: 0.2363 | Classification: 0.1923 | Classification_binary: 0.2661 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3028 | Regression_eta: 0.0035 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1211 | Classification_binary: 0.2843 | Regression_cos_phi: 0.1431 | Regression_energy: 0.1586 | Regression_eta: 0.1043 | Regression_pt: 0.1759 | Regression_sin_phi: 0.1640 | LR: 9.61e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6598s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7926 | max_loss: 0.2421 | max_backward: 2.0577 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.7621 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5925.5 | uss_MB: 4898.4 | child_rss_MB: 6176.2 | allocated_MB: 450.1 | reserved_MB: 29806.0 | inactive_split_MB: 1002.0 +rank4 [2026-10-28 16:51:50,914] INFO : Step 6800/50000 rank4 | Train Loss: 2.1609 | Train Loss Unweighted: 1.1507 | Train Loss Components: Auxiliary: 0.3771 | Cardinality: 0.2180 | Classification: 0.2315 | Classification_binary: 0.3225 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0025 | Regression_energy: 0.2916 | Regression_eta: 0.0048 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1422 | Classification_binary: 0.3382 | Regression_cos_phi: 0.1978 | Regression_energy: 0.1569 | Regression_eta: 0.1214 | Regression_pt: 0.1731 | Regression_sin_phi: 0.1977 | LR: 9.60e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.1894s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0070 | max_forward: 0.9144 | max_loss: 0.4866 | max_backward: 2.1484 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 1.2438 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5920.8 | uss_MB: 4896.0 | child_rss_MB: 6174.1 | allocated_MB: 451.3 | reserved_MB: 29806.0 | inactive_split_MB: 1098.9 +rank4 [2026-10-28 16:56:32,288] INFO : Step 6900/50000 rank4 | Train Loss: 1.9936 | Train Loss Unweighted: 1.1069 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.1991 | Classification: 0.2085 | Classification_binary: 0.3385 | HitGrouping: 0.2125 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2742 | Regression_eta: 0.0041 | Regression_pt: 0.2777 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1229 | Classification_binary: 0.3313 | Regression_cos_phi: 0.1756 | Regression_energy: 0.1592 | Regression_eta: 0.1057 | Regression_pt: 0.1750 | Regression_sin_phi: 0.1523 | LR: 9.58e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.6499s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7764 | max_loss: 0.2225 | max_backward: 1.6637 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1184 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5918.2 | uss_MB: 4895.4 | child_rss_MB: 6170.7 | allocated_MB: 450.8 | reserved_MB: 29806.0 | inactive_split_MB: 1193.4 +rank4 [2026-10-28 17:01:16,519] INFO : Step 7000/50000 rank4 | Train Loss: 1.9256 | Train Loss Unweighted: 1.0803 | Train Loss Components: Auxiliary: 0.3608 | Cardinality: 0.2036 | Classification: 0.1946 | Classification_binary: 0.2738 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2998 | Regression_eta: 0.0039 | Regression_pt: 0.3044 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1173 | Classification_binary: 0.2674 | Regression_cos_phi: 0.1821 | Regression_energy: 0.1738 | Regression_eta: 0.1377 | Regression_pt: 0.1926 | Regression_sin_phi: 0.1425 | LR: 9.57e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7630s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7896 | max_loss: 0.1557 | max_backward: 1.7415 | max_optimizer_step: 0.2946 | max_tensorboard_logging: 1.0959 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 5913.7 | uss_MB: 4893.6 | child_rss_MB: 6162.0 | allocated_MB: 450.3 | reserved_MB: 29806.0 | inactive_split_MB: 1283.4 +rank4 [2026-10-28 17:01:17,465] INFO : Running validation on rank4 +rank4 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 17:02:11,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:02:12,536] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 17:02:38,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 17:02:39,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 17:02:40,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 17:02:41,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 17:02:51,944] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 17:02:52,592] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:02:53,110] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 17:03:09,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 17:03:10,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 17:03:11,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 17:03:12,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 17:03:22,537] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 17:03:22,840] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:03:23,346] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 17:03:38,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 17:03:39,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 17:03:40,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 17:03:41,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 17:03:51,355] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 17:08:58,935] INFO : Step 7100/50000 rank4 | Train Loss: 1.8251 | Train Loss Unweighted: 1.0448 | Train Loss Components: Auxiliary: 0.3403 | Cardinality: 0.1731 | Classification: 0.2040 | Classification_binary: 0.2675 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2809 | Regression_eta: 0.0035 | Regression_pt: 0.2853 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1553 | Classification_binary: 0.3167 | Regression_cos_phi: 0.1676 | Regression_energy: 0.1420 | Regression_eta: 0.1134 | Regression_pt: 0.1565 | Regression_sin_phi: 0.1855 | LR: 9.56e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.2554s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0622 | max_forward: 0.8282 | max_loss: 0.1540 | max_backward: 1.9352 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.3584 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6015.2 | uss_MB: 4988.2 | child_rss_MB: 6155.9 | allocated_MB: 449.9 | reserved_MB: 22498.0 | inactive_split_MB: 1193.4 +rank4 [2026-10-28 17:13:31,725] INFO : Step 7200/50000 rank4 | Train Loss: 2.1146 | Train Loss Unweighted: 1.1506 | Train Loss Components: Auxiliary: 0.3887 | Cardinality: 0.2411 | Classification: 0.2243 | Classification_binary: 0.3469 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2844 | Regression_eta: 0.0044 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1341 | Classification_binary: 0.3284 | Regression_cos_phi: 0.1756 | Regression_energy: 0.1519 | Regression_eta: 0.1165 | Regression_pt: 0.1672 | Regression_sin_phi: 0.1948 | LR: 9.55e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.5721s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0369 | max_forward: 0.7969 | max_loss: 0.1517 | max_backward: 1.9380 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6030.2 | uss_MB: 5003.5 | child_rss_MB: 6152.6 | allocated_MB: 448.6 | reserved_MB: 22906.0 | inactive_split_MB: 1087.6 +rank4 [2026-10-28 17:18:11,974] INFO : Step 7300/50000 rank4 | Train Loss: 1.8657 | Train Loss Unweighted: 1.0129 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2046 | Classification: 0.1759 | Classification_binary: 0.2802 | HitGrouping: 0.2112 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2746 | Regression_eta: 0.0037 | Regression_pt: 0.2747 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0849 | Classification_binary: 0.2344 | Regression_cos_phi: 0.1297 | Regression_energy: 0.1567 | Regression_eta: 0.0892 | Regression_pt: 0.1695 | Regression_sin_phi: 0.1119 | LR: 9.53e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4764s | Phase Times: max_pre_step_barrier: 0.0082 | max_batch_to_device: 0.0150 | max_forward: 0.7713 | max_loss: 0.1507 | max_backward: 1.8303 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.6113 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6029.9 | uss_MB: 5003.9 | child_rss_MB: 6151.8 | allocated_MB: 449.9 | reserved_MB: 25044.0 | inactive_split_MB: 918.5 +rank4 [2026-10-28 17:22:47,474] INFO : Step 7400/50000 rank4 | Train Loss: 1.8326 | Train Loss Unweighted: 1.0006 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2227 | Classification: 0.1795 | Classification_binary: 0.2563 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2767 | Regression_eta: 0.0034 | Regression_pt: 0.2813 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1093 | Classification_binary: 0.2586 | Regression_cos_phi: 0.1362 | Regression_energy: 0.1567 | Regression_eta: 0.0984 | Regression_pt: 0.1730 | Regression_sin_phi: 0.1431 | LR: 9.52e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.2062s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0684 | max_forward: 0.7762 | max_loss: 0.1384 | max_backward: 1.7881 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6026.9 | uss_MB: 5000.5 | child_rss_MB: 6151.8 | allocated_MB: 447.4 | reserved_MB: 25044.0 | inactive_split_MB: 1152.4 +rank4 [2026-10-28 17:27:28,847] INFO : Step 7500/50000 rank4 | Train Loss: 2.1479 | Train Loss Unweighted: 1.1510 | Train Loss Components: Auxiliary: 0.3847 | Cardinality: 0.2329 | Classification: 0.2181 | Classification_binary: 0.3323 | HitGrouping: 0.2143 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2947 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1058 | Classification_binary: 0.3030 | Regression_cos_phi: 0.1797 | Regression_energy: 0.1532 | Regression_eta: 0.0977 | Regression_pt: 0.1663 | Regression_sin_phi: 0.1406 | LR: 9.51e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2524s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7726 | max_loss: 0.2518 | max_backward: 1.8458 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1040 | Memory: rss_MB: 6026.8 | uss_MB: 5002.5 | child_rss_MB: 6153.2 | allocated_MB: 452.3 | reserved_MB: 26846.0 | inactive_split_MB: 1117.1 +rank4 [2026-10-28 17:32:29,655] INFO : Step 7600/50000 rank4 | Train Loss: 1.9984 | Train Loss Unweighted: 1.0873 | Train Loss Components: Auxiliary: 0.3667 | Cardinality: 0.2088 | Classification: 0.2053 | Classification_binary: 0.3405 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2647 | Regression_eta: 0.0038 | Regression_pt: 0.2689 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1174 | Classification_binary: 0.3868 | Regression_cos_phi: 0.2062 | Regression_energy: 0.1411 | Regression_eta: 0.1048 | Regression_pt: 0.1563 | Regression_sin_phi: 0.1881 | LR: 9.50e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5789s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7834 | max_loss: 0.1641 | max_backward: 1.8407 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.8255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6029.1 | uss_MB: 5003.6 | child_rss_MB: 6150.1 | allocated_MB: 448.9 | reserved_MB: 26846.0 | inactive_split_MB: 1232.3 +rank4 [2026-10-28 17:37:19,852] INFO : Step 7700/50000 rank4 | Train Loss: 1.7524 | Train Loss Unweighted: 0.9761 | Train Loss Components: Auxiliary: 0.3200 | Cardinality: 0.1783 | Classification: 0.1756 | Classification_binary: 0.2521 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2705 | Regression_eta: 0.0035 | Regression_pt: 0.2709 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0821 | Classification_binary: 0.2296 | Regression_cos_phi: 0.0988 | Regression_energy: 0.1444 | Regression_eta: 0.0980 | Regression_pt: 0.1553 | Regression_sin_phi: 0.1490 | LR: 9.48e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.4844s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7659 | max_loss: 0.1551 | max_backward: 1.8103 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6027.2 | uss_MB: 5004.4 | child_rss_MB: 6154.6 | allocated_MB: 448.8 | reserved_MB: 26846.0 | inactive_split_MB: 1042.2 +rank4 [2026-10-28 17:42:10,706] INFO : Step 7800/50000 rank4 | Train Loss: 1.8039 | Train Loss Unweighted: 1.0226 | Train Loss Components: Auxiliary: 0.3378 | Cardinality: 0.1686 | Classification: 0.1822 | Classification_binary: 0.2556 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2863 | Regression_eta: 0.0034 | Regression_pt: 0.2914 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0881 | Classification_binary: 0.2229 | Regression_cos_phi: 0.1124 | Regression_energy: 0.1436 | Regression_eta: 0.0734 | Regression_pt: 0.1576 | Regression_sin_phi: 0.1370 | LR: 9.47e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6876s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.8680 | max_loss: 0.1424 | max_backward: 1.9949 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.8163 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 6024.0 | uss_MB: 5003.5 | child_rss_MB: 6156.5 | allocated_MB: 448.6 | reserved_MB: 26846.0 | inactive_split_MB: 1396.9 +rank4 [2026-10-28 17:47:05,102] INFO : Step 7900/50000 rank4 | Train Loss: 2.0641 | Train Loss Unweighted: 1.0960 | Train Loss Components: Auxiliary: 0.3855 | Cardinality: 0.2401 | Classification: 0.2031 | Classification_binary: 0.3367 | HitGrouping: 0.2085 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2721 | Regression_eta: 0.0041 | Regression_pt: 0.2759 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0863 | Classification_binary: 0.2943 | Regression_cos_phi: 0.1406 | Regression_energy: 0.1526 | Regression_eta: 0.0860 | Regression_pt: 0.1677 | Regression_sin_phi: 0.1376 | LR: 9.46e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6366s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7971 | max_loss: 0.2430 | max_backward: 1.9131 | max_optimizer_step: 0.2681 | max_tensorboard_logging: 0.7971 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6025.3 | uss_MB: 5004.3 | child_rss_MB: 6161.5 | allocated_MB: 451.8 | reserved_MB: 27044.0 | inactive_split_MB: 827.4 +rank4 [2026-10-28 17:51:56,121] INFO : Step 8000/50000 rank4 | Train Loss: 1.8391 | Train Loss Unweighted: 0.9813 | Train Loss Components: Auxiliary: 0.3355 | Cardinality: 0.2153 | Classification: 0.1625 | Classification_binary: 0.2896 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2605 | Regression_eta: 0.0035 | Regression_pt: 0.2616 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0944 | Classification_binary: 0.3002 | Regression_cos_phi: 0.1547 | Regression_energy: 0.1358 | Regression_eta: 0.1004 | Regression_pt: 0.1487 | Regression_sin_phi: 0.1482 | LR: 9.44e-04 | DataLoad Time: 0.0174s | Model Forward Time: 6.5537s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 1.2188 | max_loss: 0.4570 | max_backward: 3.9408 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.5566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6013.9 | uss_MB: 4999.4 | child_rss_MB: 6155.0 | allocated_MB: 448.4 | reserved_MB: 27044.0 | inactive_split_MB: 1156.9 +rank4 [2026-10-28 17:51:56,979] INFO : Running validation on rank4 +rank4 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 17:52:50,096] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:52:50,645] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 17:53:16,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 17:53:17,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 17:53:18,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 17:53:19,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 17:53:29,838] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 17:53:30,251] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:53:30,810] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 17:53:47,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 17:53:48,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 17:53:49,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 17:53:50,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 17:54:00,444] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 17:54:01,631] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 17:54:02,178] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 17:54:17,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 17:54:18,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 17:54:19,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 17:54:20,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 17:54:30,045] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 17:59:43,744] INFO : Step 8100/50000 rank4 | Train Loss: 1.7101 | Train Loss Unweighted: 0.9276 | Train Loss Components: Auxiliary: 0.3166 | Cardinality: 0.1787 | Classification: 0.1664 | Classification_binary: 0.2381 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2564 | Regression_eta: 0.0032 | Regression_pt: 0.2599 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0958 | Classification_binary: 0.2438 | Regression_cos_phi: 0.0927 | Regression_energy: 0.1301 | Regression_eta: 0.0765 | Regression_pt: 0.1368 | Regression_sin_phi: 0.1101 | LR: 9.43e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6998s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7628 | max_loss: 0.1353 | max_backward: 2.0123 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.8300 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6970.2 | uss_MB: 5951.6 | child_rss_MB: 6190.8 | allocated_MB: 447.2 | reserved_MB: 25306.0 | inactive_split_MB: 1261.3 +rank4 [2026-10-28 18:04:56,111] INFO : Step 8200/50000 rank4 | Train Loss: 1.8745 | Train Loss Unweighted: 1.0293 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.1946 | Classification: 0.1986 | Classification_binary: 0.2966 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2609 | Regression_eta: 0.0035 | Regression_pt: 0.2657 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1172 | Classification_binary: 0.2523 | Regression_cos_phi: 0.1460 | Regression_energy: 0.1399 | Regression_eta: 0.0905 | Regression_pt: 0.1531 | Regression_sin_phi: 0.1765 | LR: 9.42e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3072s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7790 | max_loss: 0.1535 | max_backward: 1.9181 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.5352 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6080.6 | uss_MB: 5064.4 | child_rss_MB: 6184.5 | allocated_MB: 448.2 | reserved_MB: 25360.0 | inactive_split_MB: 1159.7 +rank4 [2026-10-28 18:09:37,298] INFO : Step 8300/50000 rank4 | Train Loss: 1.8727 | Train Loss Unweighted: 1.0119 | Train Loss Components: Auxiliary: 0.3401 | Cardinality: 0.1867 | Classification: 0.1728 | Classification_binary: 0.3272 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2513 | Regression_eta: 0.0033 | Regression_pt: 0.2533 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1055 | Classification_binary: 0.3484 | Regression_cos_phi: 0.1500 | Regression_energy: 0.1270 | Regression_eta: 0.0985 | Regression_pt: 0.1411 | Regression_sin_phi: 0.1723 | LR: 9.40e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5443s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0436 | max_forward: 1.1303 | max_loss: 0.1503 | max_backward: 2.0213 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6082.4 | uss_MB: 5067.6 | child_rss_MB: 6182.5 | allocated_MB: 449.6 | reserved_MB: 27362.0 | inactive_split_MB: 1355.2 +rank4 [2026-10-28 18:14:10,520] INFO : Step 8400/50000 rank4 | Train Loss: 1.6705 | Train Loss Unweighted: 0.9205 | Train Loss Components: Auxiliary: 0.3113 | Cardinality: 0.1938 | Classification: 0.1656 | Classification_binary: 0.2512 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2485 | Regression_eta: 0.0028 | Regression_pt: 0.2494 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1072 | Classification_binary: 0.2543 | Regression_cos_phi: 0.1333 | Regression_energy: 0.1252 | Regression_eta: 0.0751 | Regression_pt: 0.1362 | Regression_sin_phi: 0.1037 | LR: 9.39e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4492s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8079 | max_loss: 0.1514 | max_backward: 1.6942 | max_optimizer_step: 0.2670 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6074.8 | uss_MB: 5060.4 | child_rss_MB: 6188.9 | allocated_MB: 446.4 | reserved_MB: 27366.0 | inactive_split_MB: 1386.9 +rank4 [2026-10-28 18:18:45,092] INFO : Step 8500/50000 rank4 | Train Loss: 1.8447 | Train Loss Unweighted: 0.9684 | Train Loss Components: Auxiliary: 0.3299 | Cardinality: 0.2197 | Classification: 0.1758 | Classification_binary: 0.2823 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2478 | Regression_eta: 0.0040 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0985 | Classification_binary: 0.2911 | Regression_cos_phi: 0.1321 | Regression_energy: 0.1483 | Regression_eta: 0.1183 | Regression_pt: 0.1650 | Regression_sin_phi: 0.1318 | LR: 9.37e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.1680s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0066 | max_forward: 0.7185 | max_loss: 0.1465 | max_backward: 1.6783 | max_optimizer_step: 0.2239 | max_tensorboard_logging: 0.6269 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 6077.8 | uss_MB: 5064.0 | child_rss_MB: 6185.3 | allocated_MB: 448.4 | reserved_MB: 27368.0 | inactive_split_MB: 1257.8 +rank4 [2026-10-28 18:23:30,153] INFO : Step 8600/50000 rank4 | Train Loss: 1.9431 | Train Loss Unweighted: 1.0237 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.2483 | Classification: 0.1814 | Classification_binary: 0.3328 | HitGrouping: 0.2024 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2495 | Regression_eta: 0.0034 | Regression_pt: 0.2527 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1056 | Classification_binary: 0.3156 | Regression_cos_phi: 0.1532 | Regression_energy: 0.1160 | Regression_eta: 0.0810 | Regression_pt: 0.1275 | Regression_sin_phi: 0.1048 | LR: 9.36e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.5569s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1011 | max_forward: 0.9340 | max_loss: 0.1508 | max_backward: 1.7704 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.7686 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 6082.0 | uss_MB: 5062.1 | child_rss_MB: 6191.5 | allocated_MB: 448.9 | reserved_MB: 27368.0 | inactive_split_MB: 1268.3 +rank4 [2026-10-28 18:28:08,445] INFO : Step 8700/50000 rank4 | Train Loss: 1.9739 | Train Loss Unweighted: 1.0707 | Train Loss Components: Auxiliary: 0.3725 | Cardinality: 0.2322 | Classification: 0.1854 | Classification_binary: 0.2934 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2900 | Regression_eta: 0.0037 | Regression_pt: 0.2944 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1047 | Classification_binary: 0.2887 | Regression_cos_phi: 0.1369 | Regression_energy: 0.1734 | Regression_eta: 0.1089 | Regression_pt: 0.1888 | Regression_sin_phi: 0.1740 | LR: 9.34e-04 | DataLoad Time: 0.0202s | Model Forward Time: 6.1628s | Phase Times: max_pre_step_barrier: 0.0890 | max_batch_to_device: 0.0067 | max_forward: 0.7726 | max_loss: 0.1574 | max_backward: 1.6332 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 2.5948 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 6082.7 | uss_MB: 5062.6 | child_rss_MB: 6185.4 | allocated_MB: 448.0 | reserved_MB: 29420.0 | inactive_split_MB: 876.0 +rank4 [2026-10-28 18:32:55,587] INFO : Step 8800/50000 rank4 | Train Loss: 1.5247 | Train Loss Unweighted: 0.8421 | Train Loss Components: Auxiliary: 0.2808 | Cardinality: 0.1524 | Classification: 0.1581 | Classification_binary: 0.2323 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2217 | Regression_eta: 0.0029 | Regression_pt: 0.2242 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1169 | Classification_binary: 0.2668 | Regression_cos_phi: 0.1438 | Regression_energy: 0.1453 | Regression_eta: 0.0855 | Regression_pt: 0.1600 | Regression_sin_phi: 0.1382 | LR: 9.33e-04 | DataLoad Time: 0.0164s | Model Forward Time: 3.9920s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7382 | max_loss: 0.2414 | max_backward: 1.7741 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 6078.3 | uss_MB: 5061.0 | child_rss_MB: 6166.0 | allocated_MB: 447.9 | reserved_MB: 29420.0 | inactive_split_MB: 1260.3 +rank4 [2026-10-28 18:37:36,891] INFO : Step 8900/50000 rank4 | Train Loss: 1.7540 | Train Loss Unweighted: 0.9436 | Train Loss Components: Auxiliary: 0.3227 | Cardinality: 0.2072 | Classification: 0.1816 | Classification_binary: 0.2735 | HitGrouping: 0.1904 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2393 | Regression_eta: 0.0031 | Regression_pt: 0.2426 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1501 | Classification_binary: 0.2743 | Regression_cos_phi: 0.1501 | Regression_energy: 0.1133 | Regression_eta: 0.0929 | Regression_pt: 0.1257 | Regression_sin_phi: 0.1390 | LR: 9.31e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4484s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8077 | max_loss: 0.2089 | max_backward: 1.9976 | max_optimizer_step: 0.2211 | max_tensorboard_logging: 0.5747 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 6080.9 | uss_MB: 5000.4 | child_rss_MB: 6188.7 | allocated_MB: 451.8 | reserved_MB: 29420.0 | inactive_split_MB: 1133.8 +rank4 [2026-10-28 18:42:20,745] INFO : Step 9000/50000 rank4 | Train Loss: 1.7807 | Train Loss Unweighted: 0.9795 | Train Loss Components: Auxiliary: 0.3326 | Cardinality: 0.1942 | Classification: 0.1600 | Classification_binary: 0.3157 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2461 | Regression_eta: 0.0033 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.3038 | Regression_cos_phi: 0.1098 | Regression_energy: 0.1411 | Regression_eta: 0.1130 | Regression_pt: 0.1539 | Regression_sin_phi: 0.1109 | LR: 9.30e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8573s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.8696 | max_loss: 0.1559 | max_backward: 1.8245 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.9520 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 6064.6 | uss_MB: 5051.1 | child_rss_MB: 6187.5 | allocated_MB: 448.9 | reserved_MB: 29420.0 | inactive_split_MB: 1051.6 +rank4 [2026-10-28 18:42:21,625] INFO : Running validation on rank4 +rank4 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 18:43:19,669] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:43:20,247] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 18:43:47,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 18:43:48,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 18:43:49,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 18:43:50,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 18:44:00,124] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 18:44:00,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:44:01,639] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 18:44:17,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 18:44:18,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 18:44:19,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 18:44:20,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 18:44:30,720] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 18:44:31,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 18:44:31,608] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 18:44:47,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 18:44:48,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 18:44:49,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 18:44:50,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 18:45:00,354] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 18:50:14,979] INFO : Step 9100/50000 rank4 | Train Loss: 1.6002 | Train Loss Unweighted: 0.8685 | Train Loss Components: Auxiliary: 0.3013 | Cardinality: 0.1975 | Classification: 0.1658 | Classification_binary: 0.2349 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2304 | Regression_eta: 0.0027 | Regression_pt: 0.2318 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0880 | Classification_binary: 0.2396 | Regression_cos_phi: 0.1069 | Regression_energy: 0.1270 | Regression_eta: 0.0622 | Regression_pt: 0.1380 | Regression_sin_phi: 0.1059 | LR: 9.28e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5183s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7981 | max_loss: 0.1460 | max_backward: 1.6475 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.9178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7065.6 | uss_MB: 6052.1 | child_rss_MB: 6167.4 | allocated_MB: 449.7 | reserved_MB: 25788.0 | inactive_split_MB: 957.3 +rank4 [2026-10-28 18:55:02,312] INFO : Step 9200/50000 rank4 | Train Loss: 1.6859 | Train Loss Unweighted: 0.8867 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.1906 | Classification: 0.1447 | Classification_binary: 0.2542 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2389 | Regression_eta: 0.0033 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0872 | Classification_binary: 0.2617 | Regression_cos_phi: 0.1381 | Regression_energy: 0.1210 | Regression_eta: 0.0966 | Regression_pt: 0.1359 | Regression_sin_phi: 0.1204 | LR: 9.27e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3390s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7865 | max_loss: 0.1441 | max_backward: 1.8922 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6128.5 | uss_MB: 5123.6 | child_rss_MB: 6161.7 | allocated_MB: 450.8 | reserved_MB: 25788.0 | inactive_split_MB: 1040.1 +rank4 [2026-10-28 18:59:49,908] INFO : Step 9300/50000 rank4 | Train Loss: 1.8980 | Train Loss Unweighted: 0.9669 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.3104 | Classification: 0.1729 | Classification_binary: 0.3073 | HitGrouping: 0.1936 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2397 | Regression_eta: 0.0030 | Regression_pt: 0.2407 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0982 | Classification_binary: 0.3024 | Regression_cos_phi: 0.1177 | Regression_energy: 0.1302 | Regression_eta: 0.0680 | Regression_pt: 0.1427 | Regression_sin_phi: 0.1342 | LR: 9.25e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.1841s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0143 | max_forward: 0.7819 | max_loss: 0.1515 | max_backward: 1.8209 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6071.1 | uss_MB: 5121.9 | child_rss_MB: 5951.4 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1031.1 +rank4 [2026-10-28 19:04:46,838] INFO : Step 9400/50000 rank4 | Train Loss: 1.5878 | Train Loss Unweighted: 0.8563 | Train Loss Components: Auxiliary: 0.2880 | Cardinality: 0.1552 | Classification: 0.1580 | Classification_binary: 0.2625 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2141 | Regression_eta: 0.0030 | Regression_pt: 0.2153 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0940 | Classification_binary: 0.2520 | Regression_cos_phi: 0.2416 | Regression_energy: 0.1183 | Regression_eta: 0.1392 | Regression_pt: 0.1252 | Regression_sin_phi: 0.1654 | LR: 9.24e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0185s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7342 | max_loss: 0.2471 | max_backward: 1.7320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.4671 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6067.3 | uss_MB: 5118.7 | child_rss_MB: 5945.7 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1121.1 +rank4 [2026-10-28 19:09:47,342] INFO : Step 9500/50000 rank4 | Train Loss: 1.5519 | Train Loss Unweighted: 0.8141 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1719 | Classification: 0.1393 | Classification_binary: 0.2430 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2129 | Regression_eta: 0.0030 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.2153 | Regression_cos_phi: 0.1340 | Regression_energy: 0.1049 | Regression_eta: 0.0885 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0976 | LR: 9.22e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1289s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0076 | max_forward: 0.7261 | max_loss: 0.1376 | max_backward: 1.8267 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6069.1 | uss_MB: 5122.8 | child_rss_MB: 5952.7 | allocated_MB: 446.7 | reserved_MB: 27752.0 | inactive_split_MB: 1128.4 +rank4 [2026-10-28 19:14:55,454] INFO : Step 9600/50000 rank4 | Train Loss: 1.7379 | Train Loss Unweighted: 0.9171 | Train Loss Components: Auxiliary: 0.3221 | Cardinality: 0.2199 | Classification: 0.1679 | Classification_binary: 0.2886 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2245 | Regression_eta: 0.0032 | Regression_pt: 0.2297 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1223 | Classification_binary: 0.3043 | Regression_cos_phi: 0.1421 | Regression_energy: 0.1176 | Regression_eta: 0.0764 | Regression_pt: 0.1314 | Regression_sin_phi: 0.1222 | LR: 9.21e-04 | DataLoad Time: 0.0178s | Model Forward Time: 7.7258s | Phase Times: max_pre_step_barrier: 0.0081 | max_batch_to_device: 0.0183 | max_forward: 3.1116 | max_loss: 0.7194 | max_backward: 4.4829 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6068.7 | uss_MB: 5120.4 | child_rss_MB: 5924.4 | allocated_MB: 449.4 | reserved_MB: 29732.0 | inactive_split_MB: 1322.6 +rank4 [2026-10-28 19:20:02,940] INFO : Step 9700/50000 rank4 | Train Loss: 1.8683 | Train Loss Unweighted: 0.9602 | Train Loss Components: Auxiliary: 0.3449 | Cardinality: 0.2461 | Classification: 0.1512 | Classification_binary: 0.3271 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2364 | Regression_eta: 0.0037 | Regression_pt: 0.2383 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1095 | Classification_binary: 0.3166 | Regression_cos_phi: 0.1461 | Regression_energy: 0.1356 | Regression_eta: 0.1006 | Regression_pt: 0.1487 | Regression_sin_phi: 0.1611 | LR: 9.19e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.0329s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0069 | max_forward: 0.8299 | max_loss: 0.7776 | max_backward: 2.5571 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5726 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6067.5 | uss_MB: 5123.7 | child_rss_MB: 5943.1 | allocated_MB: 450.3 | reserved_MB: 29734.0 | inactive_split_MB: 1128.1 +rank4 [2026-10-28 19:25:00,573] INFO : Step 9800/50000 rank4 | Train Loss: 1.6119 | Train Loss Unweighted: 0.8200 | Train Loss Components: Auxiliary: 0.3036 | Cardinality: 0.2266 | Classification: 0.1371 | Classification_binary: 0.2410 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2176 | Regression_eta: 0.0027 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0861 | Classification_binary: 0.2363 | Regression_cos_phi: 0.1329 | Regression_energy: 0.1172 | Regression_eta: 0.0695 | Regression_pt: 0.1268 | Regression_sin_phi: 0.1412 | LR: 9.17e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5945s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7430 | max_loss: 0.2467 | max_backward: 1.8961 | max_optimizer_step: 0.2008 | max_tensorboard_logging: 0.9056 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6064.8 | uss_MB: 5122.0 | child_rss_MB: 5925.9 | allocated_MB: 446.9 | reserved_MB: 29734.0 | inactive_split_MB: 905.1 +rank4 [2026-10-28 19:29:40,210] INFO : Step 9900/50000 rank4 | Train Loss: 1.5408 | Train Loss Unweighted: 0.8391 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1631 | Classification: 0.1532 | Classification_binary: 0.2416 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2178 | Regression_eta: 0.0029 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0832 | Regression_energy: 0.1195 | Regression_eta: 0.0509 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0766 | LR: 9.16e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6038s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7611 | max_loss: 0.1425 | max_backward: 1.8994 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6066.1 | uss_MB: 5122.8 | child_rss_MB: 5945.5 | allocated_MB: 448.3 | reserved_MB: 29734.0 | inactive_split_MB: 1119.7 +rank4 [2026-10-28 19:34:23,378] INFO : Step 10000/50000 rank4 | Train Loss: 1.7147 | Train Loss Unweighted: 0.9225 | Train Loss Components: Auxiliary: 0.3261 | Cardinality: 0.1969 | Classification: 0.1590 | Classification_binary: 0.2912 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2305 | Regression_eta: 0.0029 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1047 | Classification_binary: 0.3360 | Regression_cos_phi: 0.1635 | Regression_energy: 0.1285 | Regression_eta: 0.1185 | Regression_pt: 0.1414 | Regression_sin_phi: 0.1638 | LR: 9.14e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8791s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1140 | max_forward: 0.8951 | max_loss: 0.1695 | max_backward: 1.8837 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 1.0339 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 6055.8 | uss_MB: 5112.2 | child_rss_MB: 5922.2 | allocated_MB: 449.4 | reserved_MB: 29734.0 | inactive_split_MB: 1320.6 +rank4 [2026-10-28 19:34:24,225] INFO : Running validation on rank4 +rank4 [2026-10-28 19:34:24,225] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 19:34:24,226] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 19:35:21,693] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:35:22,179] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 19:35:48,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 19:35:49,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 19:35:50,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 19:35:51,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 19:36:01,814] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 19:36:02,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:36:02,979] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 19:36:18,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 19:36:19,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 19:36:20,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 19:36:21,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 19:36:31,774] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 19:36:36,471] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 19:36:37,122] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 19:36:52,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 19:36:53,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 19:36:54,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 19:36:55,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 19:37:05,322] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 19:42:23,380] INFO : Step 10100/50000 rank4 | Train Loss: 1.6697 | Train Loss Unweighted: 0.8627 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.2287 | Classification: 0.1283 | Classification_binary: 0.2543 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2359 | Regression_eta: 0.0032 | Regression_pt: 0.2378 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.2403 | Regression_cos_phi: 0.1021 | Regression_energy: 0.1162 | Regression_eta: 0.0901 | Regression_pt: 0.1260 | Regression_sin_phi: 0.1000 | LR: 9.12e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.7385s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7652 | max_loss: 0.1496 | max_backward: 1.8791 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 0.9326 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 7065.4 | uss_MB: 6119.6 | child_rss_MB: 5927.2 | allocated_MB: 448.1 | reserved_MB: 26360.0 | inactive_split_MB: 1080.0 +rank4 [2026-10-28 19:47:10,586] INFO : Step 10200/50000 rank4 | Train Loss: 1.3968 | Train Loss Unweighted: 0.7619 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1483 | Classification: 0.1293 | Classification_binary: 0.2070 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2090 | Regression_eta: 0.0026 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.1979 | Regression_cos_phi: 0.0855 | Regression_energy: 0.1357 | Regression_eta: 0.0560 | Regression_pt: 0.1502 | Regression_sin_phi: 0.1044 | LR: 9.11e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.4826s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0330 | max_forward: 0.8960 | max_loss: 0.5080 | max_backward: 1.9040 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.6816 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 6137.8 | uss_MB: 5194.1 | child_rss_MB: 5912.5 | allocated_MB: 450.3 | reserved_MB: 27130.0 | inactive_split_MB: 1088.5 +rank4 [2026-10-28 19:51:56,122] INFO : Step 10300/50000 rank4 | Train Loss: 1.6039 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.2943 | Cardinality: 0.1656 | Classification: 0.1487 | Classification_binary: 0.2679 | HitGrouping: 0.1881 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2132 | Regression_eta: 0.0030 | Regression_pt: 0.2158 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0922 | Classification_binary: 0.2550 | Regression_cos_phi: 0.1410 | Regression_energy: 0.1243 | Regression_eta: 0.0728 | Regression_pt: 0.1366 | Regression_sin_phi: 0.1060 | LR: 9.09e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7931s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8152 | max_loss: 0.2478 | max_backward: 2.0483 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6143.7 | uss_MB: 5198.5 | child_rss_MB: 5930.4 | allocated_MB: 448.7 | reserved_MB: 28404.0 | inactive_split_MB: 991.9 +rank4 [2026-10-28 19:56:39,452] INFO : Step 10400/50000 rank4 | Train Loss: 1.7188 | Train Loss Unweighted: 0.8798 | Train Loss Components: Auxiliary: 0.3209 | Cardinality: 0.2578 | Classification: 0.1544 | Classification_binary: 0.2910 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2139 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0747 | Classification_binary: 0.2711 | Regression_cos_phi: 0.1194 | Regression_energy: 0.0977 | Regression_eta: 0.0682 | Regression_pt: 0.1071 | Regression_sin_phi: 0.1035 | LR: 9.07e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6758s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7821 | max_loss: 0.2448 | max_backward: 1.8782 | max_optimizer_step: 0.2932 | max_tensorboard_logging: 0.8061 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 6134.3 | uss_MB: 5193.2 | child_rss_MB: 5938.4 | allocated_MB: 448.9 | reserved_MB: 28404.0 | inactive_split_MB: 983.3 +rank4 [2026-10-28 20:01:29,564] INFO : Step 10500/50000 rank4 | Train Loss: 1.4652 | Train Loss Unweighted: 0.7986 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1697 | Classification: 0.1280 | Classification_binary: 0.2159 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2232 | Regression_eta: 0.0026 | Regression_pt: 0.2263 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0884 | Classification_binary: 0.2152 | Regression_cos_phi: 0.1420 | Regression_energy: 0.1069 | Regression_eta: 0.0856 | Regression_pt: 0.1196 | Regression_sin_phi: 0.1221 | LR: 9.06e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.2162s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.7113 | max_loss: 0.1455 | max_backward: 1.6308 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.7760 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6134.1 | uss_MB: 5192.0 | child_rss_MB: 5916.0 | allocated_MB: 444.7 | reserved_MB: 28404.0 | inactive_split_MB: 1307.3 +rank4 [2026-10-28 20:06:14,521] INFO : Step 10600/50000 rank4 | Train Loss: 1.5657 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2802 | Cardinality: 0.1297 | Classification: 0.1617 | Classification_binary: 0.2471 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2084 | Regression_eta: 0.0037 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0996 | Classification_binary: 0.2246 | Regression_cos_phi: 0.1715 | Regression_energy: 0.1026 | Regression_eta: 0.0700 | Regression_pt: 0.1118 | Regression_sin_phi: 0.1191 | LR: 9.04e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.9403s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0173 | max_forward: 0.7875 | max_loss: 0.2211 | max_backward: 1.9019 | max_optimizer_step: 0.2814 | max_tensorboard_logging: 0.8518 | max_tensorboard_flush: 0.0072 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6133.2 | uss_MB: 5193.0 | child_rss_MB: 5934.7 | allocated_MB: 449.8 | reserved_MB: 28404.0 | inactive_split_MB: 1130.2 +rank4 [2026-10-28 20:10:59,505] INFO : Step 10700/50000 rank4 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8676 | Train Loss Components: Auxiliary: 0.3003 | Cardinality: 0.1749 | Classification: 0.1453 | Classification_binary: 0.2956 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2085 | Regression_eta: 0.0033 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0849 | Classification_binary: 0.2703 | Regression_cos_phi: 0.1227 | Regression_energy: 0.0963 | Regression_eta: 0.0850 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0895 | LR: 9.02e-04 | DataLoad Time: 0.0127s | Model Forward Time: 5.1531s | Phase Times: max_pre_step_barrier: 0.0305 | max_batch_to_device: 0.0154 | max_forward: 2.0246 | max_loss: 0.2864 | max_backward: 2.7424 | max_optimizer_step: 0.1980 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6136.9 | uss_MB: 5195.8 | child_rss_MB: 5906.4 | allocated_MB: 446.2 | reserved_MB: 29148.0 | inactive_split_MB: 1134.5 +rank4 [2026-10-28 20:15:41,623] INFO : Step 10800/50000 rank4 | Train Loss: 1.5849 | Train Loss Unweighted: 0.7936 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1942 | Classification: 0.1383 | Classification_binary: 0.2480 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2007 | Regression_eta: 0.0032 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0866 | Classification_binary: 0.2064 | Regression_cos_phi: 0.1586 | Regression_energy: 0.1081 | Regression_eta: 0.0837 | Regression_pt: 0.1161 | Regression_sin_phi: 0.1310 | LR: 9.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 8.1989s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2408 | max_backward: 1.7977 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 1.7470 | max_tensorboard_flush: 0.0028 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 6126.4 | uss_MB: 4363.9 | child_rss_MB: 5910.3 | allocated_MB: 448.6 | reserved_MB: 30628.0 | inactive_split_MB: 1292.6 +rank4 [2026-10-28 20:20:32,833] INFO : Step 10900/50000 rank4 | Train Loss: 1.3552 | Train Loss Unweighted: 0.7247 | Train Loss Components: Auxiliary: 0.2531 | Cardinality: 0.1449 | Classification: 0.1275 | Classification_binary: 0.2074 | HitGrouping: 0.1874 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1917 | Regression_eta: 0.0024 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0942 | Classification_binary: 0.2164 | Regression_cos_phi: 0.1465 | Regression_energy: 0.0868 | Regression_eta: 0.0829 | Regression_pt: 0.0950 | Regression_sin_phi: 0.1191 | LR: 8.99e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4552s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7076 | max_loss: 0.2425 | max_backward: 1.6536 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6131.8 | uss_MB: 5191.3 | child_rss_MB: 5932.5 | allocated_MB: 446.4 | reserved_MB: 30628.0 | inactive_split_MB: 984.9 +rank4 [2026-10-28 20:25:36,608] INFO : Step 11000/50000 rank4 | Train Loss: 1.5999 | Train Loss Unweighted: 0.8322 | Train Loss Components: Auxiliary: 0.2959 | Cardinality: 0.1737 | Classification: 0.1477 | Classification_binary: 0.2877 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0017 | Regression_energy: 0.1943 | Regression_eta: 0.0030 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.2634 | Regression_cos_phi: 0.1538 | Regression_energy: 0.0989 | Regression_eta: 0.0703 | Regression_pt: 0.1089 | Regression_sin_phi: 0.1188 | LR: 8.97e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9319s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8260 | max_loss: 0.1503 | max_backward: 1.9603 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6124.9 | uss_MB: 5185.7 | child_rss_MB: 5918.3 | allocated_MB: 451.8 | reserved_MB: 30628.0 | inactive_split_MB: 996.6 +rank4 [2026-10-28 20:25:37,479] INFO : Running validation on rank4 +rank4 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 20:26:32,846] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:26:33,361] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 20:27:00,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 20:27:01,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 20:27:02,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 20:27:03,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 20:27:13,275] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 20:27:14,319] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:27:14,833] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 20:27:31,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 20:27:32,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 20:27:33,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 20:27:34,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 20:27:44,927] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-28 20:27:46,111] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 20:27:46,648] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 20:28:01,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 20:28:02,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 20:28:03,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 20:28:04,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 20:28:14,470] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 20:33:29,191] INFO : Step 11100/50000 rank4 | Train Loss: 1.6214 | Train Loss Unweighted: 0.8328 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.2186 | Classification: 0.1350 | Classification_binary: 0.2802 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2042 | Regression_eta: 0.0030 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1003 | Classification_binary: 0.3089 | Regression_cos_phi: 0.1215 | Regression_energy: 0.1174 | Regression_eta: 0.0884 | Regression_pt: 0.1289 | Regression_sin_phi: 0.1438 | LR: 8.95e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8819s | Phase Times: max_pre_step_barrier: 0.0088 | max_batch_to_device: 0.0068 | max_forward: 0.8488 | max_loss: 0.1548 | max_backward: 1.7645 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 1.1046 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7069.9 | uss_MB: 6125.7 | child_rss_MB: 5940.1 | allocated_MB: 448.1 | reserved_MB: 24340.0 | inactive_split_MB: 1081.9 +rank4 [2026-10-28 20:38:18,706] INFO : Step 11200/50000 rank4 | Train Loss: 1.7050 | Train Loss Unweighted: 0.8927 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2640 | Classification: 0.1534 | Classification_binary: 0.2523 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2404 | Regression_eta: 0.0030 | Regression_pt: 0.2410 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.2851 | Regression_cos_phi: 0.1304 | Regression_energy: 0.1363 | Regression_eta: 0.0937 | Regression_pt: 0.1487 | Regression_sin_phi: 0.1168 | LR: 8.93e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.8973s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8707 | max_loss: 0.2001 | max_backward: 2.2037 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.7912 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 6163.4 | uss_MB: 5218.5 | child_rss_MB: 5949.2 | allocated_MB: 447.8 | reserved_MB: 26392.0 | inactive_split_MB: 938.7 +rank4 [2026-10-28 20:43:01,723] INFO : Step 11300/50000 rank4 | Train Loss: 1.3885 | Train Loss Unweighted: 0.7511 | Train Loss Components: Auxiliary: 0.2594 | Cardinality: 0.1357 | Classification: 0.1316 | Classification_binary: 0.2357 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1874 | Regression_eta: 0.0024 | Regression_pt: 0.1915 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0751 | Classification_binary: 0.1869 | Regression_cos_phi: 0.1131 | Regression_energy: 0.0956 | Regression_eta: 0.0540 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0847 | LR: 8.91e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4294s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8053 | max_loss: 0.1469 | max_backward: 2.0062 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.4618 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1345 | Memory: rss_MB: 6165.0 | uss_MB: 5222.0 | child_rss_MB: 5948.8 | allocated_MB: 450.6 | reserved_MB: 26400.0 | inactive_split_MB: 1092.9 +rank4 [2026-10-28 20:48:05,522] INFO : Step 11400/50000 rank4 | Train Loss: 1.5865 | Train Loss Unweighted: 0.8323 | Train Loss Components: Auxiliary: 0.2981 | Cardinality: 0.1866 | Classification: 0.1338 | Classification_binary: 0.2899 | HitGrouping: 0.1840 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1999 | Regression_eta: 0.0028 | Regression_pt: 0.2030 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.2660 | Regression_cos_phi: 0.1275 | Regression_energy: 0.0850 | Regression_eta: 0.0771 | Regression_pt: 0.0927 | Regression_sin_phi: 0.1005 | LR: 8.89e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.7655s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0769 | max_forward: 1.3800 | max_loss: 0.6715 | max_backward: 3.0339 | max_optimizer_step: 0.2660 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6163.4 | uss_MB: 5219.6 | child_rss_MB: 5946.4 | allocated_MB: 448.1 | reserved_MB: 27144.0 | inactive_split_MB: 1081.9 +rank4 [2026-10-28 20:52:57,975] INFO : Step 11500/50000 rank4 | Train Loss: 1.4825 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2779 | Cardinality: 0.1999 | Classification: 0.1273 | Classification_binary: 0.2463 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1956 | Regression_eta: 0.0026 | Regression_pt: 0.1972 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.2740 | Regression_cos_phi: 0.1156 | Regression_energy: 0.0988 | Regression_eta: 0.0764 | Regression_pt: 0.1080 | Regression_sin_phi: 0.1007 | LR: 8.88e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0561s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.7822 | max_loss: 0.1471 | max_backward: 1.7308 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 1.4583 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6164.2 | uss_MB: 5219.4 | child_rss_MB: 5944.4 | allocated_MB: 450.8 | reserved_MB: 27144.0 | inactive_split_MB: 1342.3 +rank4 [2026-10-28 20:57:41,224] INFO : Step 11600/50000 rank4 | Train Loss: 1.4219 | Train Loss Unweighted: 0.7444 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1679 | Classification: 0.1415 | Classification_binary: 0.2106 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1945 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.2199 | Regression_cos_phi: 0.1134 | Regression_energy: 0.0971 | Regression_eta: 0.0725 | Regression_pt: 0.1065 | Regression_sin_phi: 0.1329 | LR: 8.86e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6998s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8362 | max_loss: 0.1428 | max_backward: 2.0109 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6156.9 | uss_MB: 5220.2 | child_rss_MB: 5947.3 | allocated_MB: 448.6 | reserved_MB: 27146.0 | inactive_split_MB: 1315.4 +rank4 [2026-10-28 21:02:35,116] INFO : Step 11700/50000 rank4 | Train Loss: 1.5203 | Train Loss Unweighted: 0.8017 | Train Loss Components: Auxiliary: 0.2866 | Cardinality: 0.1889 | Classification: 0.1302 | Classification_binary: 0.2577 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2024 | Regression_eta: 0.0026 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0859 | Classification_binary: 0.2408 | Regression_cos_phi: 0.1199 | Regression_energy: 0.0981 | Regression_eta: 0.0845 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0832 | LR: 8.84e-04 | DataLoad Time: 0.0180s | Model Forward Time: 5.3171s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 1.0014 | max_loss: 0.1975 | max_backward: 2.5347 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0688 | Memory: rss_MB: 6157.0 | uss_MB: 5144.6 | child_rss_MB: 5922.6 | allocated_MB: 448.7 | reserved_MB: 27266.0 | inactive_split_MB: 1090.9 +rank4 [2026-10-28 21:07:42,591] INFO : Step 11800/50000 rank4 | Train Loss: 1.5591 | Train Loss Unweighted: 0.7907 | Train Loss Components: Auxiliary: 0.2873 | Cardinality: 0.2051 | Classification: 0.1346 | Classification_binary: 0.2655 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1918 | Regression_eta: 0.0030 | Regression_pt: 0.1929 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1056 | Classification_binary: 0.2881 | Regression_cos_phi: 0.1576 | Regression_energy: 0.1026 | Regression_eta: 0.0951 | Regression_pt: 0.1126 | Regression_sin_phi: 0.1308 | LR: 8.82e-04 | DataLoad Time: 0.0200s | Model Forward Time: 7.4840s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.0966 | max_forward: 1.7907 | max_loss: 0.6496 | max_backward: 2.7103 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 2.3587 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6136.9 | uss_MB: 5209.7 | child_rss_MB: 5946.4 | allocated_MB: 450.4 | reserved_MB: 27266.0 | inactive_split_MB: 1187.2 +rank4 [2026-10-28 21:12:51,258] INFO : Step 11900/50000 rank4 | Train Loss: 1.3283 | Train Loss Unweighted: 0.7113 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1476 | Classification: 0.1193 | Classification_binary: 0.2094 | HitGrouping: 0.1824 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0021 | Regression_pt: 0.1901 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.1875 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0779 | Regression_eta: 0.0707 | Regression_pt: 0.0849 | Regression_sin_phi: 0.0556 | LR: 8.80e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.1701s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0785 | max_forward: 0.7917 | max_loss: 0.1465 | max_backward: 1.5595 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6137.9 | uss_MB: 5216.4 | child_rss_MB: 5947.1 | allocated_MB: 443.9 | reserved_MB: 27266.0 | inactive_split_MB: 955.3 +rank4 [2026-10-28 21:17:48,734] INFO : Step 12000/50000 rank4 | Train Loss: 1.4074 | Train Loss Unweighted: 0.7415 | Train Loss Components: Auxiliary: 0.2683 | Cardinality: 0.1719 | Classification: 0.1235 | Classification_binary: 0.2283 | HitGrouping: 0.1812 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1909 | Regression_eta: 0.0022 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.2125 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0950 | Regression_eta: 0.0361 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0782 | LR: 8.78e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2510s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0303 | max_forward: 0.7589 | max_loss: 0.1459 | max_backward: 1.9073 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6136.6 | uss_MB: 5219.4 | child_rss_MB: 5952.0 | allocated_MB: 447.6 | reserved_MB: 27266.0 | inactive_split_MB: 1310.4 +rank4 [2026-10-28 21:17:49,718] INFO : Running validation on rank4 +rank4 [2026-10-28 21:17:49,719] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 21:17:49,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 21:18:44,923] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:18:45,602] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 21:19:11,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 21:19:12,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 21:19:13,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 21:19:14,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 21:19:24,518] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 21:19:25,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:19:25,829] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 21:19:42,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 21:19:42,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 21:19:43,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 21:19:44,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 21:19:54,862] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 21:19:55,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 21:19:56,383] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 21:20:12,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 21:20:12,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 21:20:13,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 21:20:14,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 21:20:24,640] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 21:25:33,196] INFO : Step 12100/50000 rank4 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8521 | Train Loss Components: Auxiliary: 0.3093 | Cardinality: 0.2088 | Classification: 0.1337 | Classification_binary: 0.2971 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0029 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0946 | Classification_binary: 0.3447 | Regression_cos_phi: 0.1528 | Regression_energy: 0.1491 | Regression_eta: 0.0933 | Regression_pt: 0.1637 | Regression_sin_phi: 0.1505 | LR: 8.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9405s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8139 | max_loss: 0.1586 | max_backward: 1.9999 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9439 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7027.3 | uss_MB: 6104.0 | child_rss_MB: 5925.8 | allocated_MB: 450.8 | reserved_MB: 28008.0 | inactive_split_MB: 1125.6 +rank4 [2026-10-28 21:30:25,392] INFO : Step 12200/50000 rank4 | Train Loss: 1.4655 | Train Loss Unweighted: 0.7431 | Train Loss Components: Auxiliary: 0.2757 | Cardinality: 0.1845 | Classification: 0.1184 | Classification_binary: 0.2450 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1862 | Regression_eta: 0.0026 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0798 | Classification_binary: 0.2668 | Regression_cos_phi: 0.1166 | Regression_energy: 0.0928 | Regression_eta: 0.0676 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0943 | LR: 8.74e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4996s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0364 | max_forward: 0.7684 | max_loss: 0.1438 | max_backward: 1.8240 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.7243 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0697 | Memory: rss_MB: 6148.5 | uss_MB: 5233.4 | child_rss_MB: 5949.1 | allocated_MB: 445.9 | reserved_MB: 28008.0 | inactive_split_MB: 995.3 +rank4 [2026-10-28 21:35:16,328] INFO : Step 12300/50000 rank4 | Train Loss: 1.3845 | Train Loss Unweighted: 0.7270 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1523 | Classification: 0.1341 | Classification_binary: 0.2109 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1869 | Regression_eta: 0.0026 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0896 | Regression_eta: 0.0623 | Regression_pt: 0.0983 | Regression_sin_phi: 0.0882 | LR: 8.72e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5875s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7264 | max_loss: 0.1976 | max_backward: 1.8534 | max_optimizer_step: 0.2993 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6147.3 | uss_MB: 5233.4 | child_rss_MB: 5944.1 | allocated_MB: 448.1 | reserved_MB: 29988.0 | inactive_split_MB: 1298.2 +rank4 [2026-10-28 21:40:15,076] INFO : Step 12400/50000 rank4 | Train Loss: 1.4006 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.2654 | Cardinality: 0.1602 | Classification: 0.1310 | Classification_binary: 0.2363 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0710 | Classification_binary: 0.2235 | Regression_cos_phi: 0.1013 | Regression_energy: 0.1157 | Regression_eta: 0.0629 | Regression_pt: 0.1252 | Regression_sin_phi: 0.0916 | LR: 8.70e-04 | DataLoad Time: 0.0169s | Model Forward Time: 7.1207s | Phase Times: max_pre_step_barrier: 0.0107 | max_batch_to_device: 0.0207 | max_forward: 1.2284 | max_loss: 0.2740 | max_backward: 2.6428 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8714 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 6147.7 | uss_MB: 5022.1 | child_rss_MB: 5936.9 | allocated_MB: 448.8 | reserved_MB: 29988.0 | inactive_split_MB: 1332.2 +rank4 [2026-10-28 21:45:07,596] INFO : Step 12500/50000 rank4 | Train Loss: 1.4931 | Train Loss Unweighted: 0.7691 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1768 | Classification: 0.1197 | Classification_binary: 0.2587 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1910 | Regression_eta: 0.0024 | Regression_pt: 0.1944 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.2687 | Regression_cos_phi: 0.1256 | Regression_energy: 0.0889 | Regression_eta: 0.0867 | Regression_pt: 0.0995 | Regression_sin_phi: 0.1594 | LR: 8.68e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.7418s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0333 | max_forward: 0.8806 | max_loss: 0.1518 | max_backward: 2.1554 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.5983 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6144.5 | uss_MB: 5232.3 | child_rss_MB: 5946.8 | allocated_MB: 452.8 | reserved_MB: 29990.0 | inactive_split_MB: 1046.7 +rank4 [2026-10-28 21:50:03,651] INFO : Step 12600/50000 rank4 | Train Loss: 1.3547 | Train Loss Unweighted: 0.7028 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1629 | Classification: 0.1033 | Classification_binary: 0.2112 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1912 | Regression_eta: 0.0024 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.1977 | Regression_cos_phi: 0.1461 | Regression_energy: 0.1231 | Regression_eta: 0.0534 | Regression_pt: 0.1297 | Regression_sin_phi: 0.1077 | LR: 8.66e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5168s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.7639 | max_loss: 0.2516 | max_backward: 1.8947 | max_optimizer_step: 0.2049 | max_tensorboard_logging: 0.7350 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6141.1 | uss_MB: 5233.0 | child_rss_MB: 5920.5 | allocated_MB: 446.9 | reserved_MB: 29990.0 | inactive_split_MB: 1034.9 +rank4 [2026-10-28 21:54:54,644] INFO : Step 12700/50000 rank4 | Train Loss: 1.3557 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1573 | Classification: 0.1262 | Classification_binary: 0.2132 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1863 | Regression_eta: 0.0022 | Regression_pt: 0.1887 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0863 | Classification_binary: 0.2359 | Regression_cos_phi: 0.0939 | Regression_energy: 0.0969 | Regression_eta: 0.0690 | Regression_pt: 0.1063 | Regression_sin_phi: 0.1216 | LR: 8.64e-04 | DataLoad Time: 0.0147s | Model Forward Time: 6.1276s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7499 | max_loss: 0.2453 | max_backward: 3.4334 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6139.6 | uss_MB: 5232.1 | child_rss_MB: 5948.4 | allocated_MB: 450.2 | reserved_MB: 29990.0 | inactive_split_MB: 1245.6 +rank4 [2026-10-28 21:59:39,371] INFO : Step 12800/50000 rank4 | Train Loss: 1.5298 | Train Loss Unweighted: 0.7754 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1939 | Classification: 0.1237 | Classification_binary: 0.2821 | HitGrouping: 0.1822 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1810 | Regression_eta: 0.0027 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0884 | Classification_binary: 0.2984 | Regression_cos_phi: 0.1544 | Regression_energy: 0.1018 | Regression_eta: 0.0867 | Regression_pt: 0.1105 | Regression_sin_phi: 0.1459 | LR: 8.62e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.2924s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.5299 | max_backward: 2.3185 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6133.4 | uss_MB: 4813.1 | child_rss_MB: 5927.2 | allocated_MB: 449.9 | reserved_MB: 32006.0 | inactive_split_MB: 889.3 +rank4 [2026-10-28 22:04:39,764] INFO : Step 12900/50000 rank4 | Train Loss: 1.3302 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1576 | Classification: 0.1054 | Classification_binary: 0.2343 | HitGrouping: 0.1811 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1758 | Regression_eta: 0.0020 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2546 | Regression_cos_phi: 0.1173 | Regression_energy: 0.0850 | Regression_eta: 0.0752 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0867 | LR: 8.60e-04 | DataLoad Time: 0.0198s | Model Forward Time: 5.6347s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0137 | max_forward: 0.9836 | max_loss: 0.2293 | max_backward: 2.2888 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.9302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6139.6 | uss_MB: 5231.4 | child_rss_MB: 5950.5 | allocated_MB: 445.9 | reserved_MB: 32006.0 | inactive_split_MB: 1247.6 +rank4 [2026-10-28 22:09:25,338] INFO : Step 13000/50000 rank4 | Train Loss: 1.3124 | Train Loss Unweighted: 0.7073 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1406 | Classification: 0.1147 | Classification_binary: 0.1973 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1944 | Regression_eta: 0.0023 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.1843 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0893 | Regression_eta: 0.0444 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0721 | LR: 8.58e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2805s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0065 | max_forward: 0.7321 | max_loss: 0.1616 | max_backward: 1.7374 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6140.2 | uss_MB: 5231.6 | child_rss_MB: 5949.6 | allocated_MB: 444.7 | reserved_MB: 32006.0 | inactive_split_MB: 1300.2 +rank4 [2026-10-28 22:09:26,329] INFO : Running validation on rank4 +rank4 [2026-10-28 22:09:26,330] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 22:09:26,330] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 22:10:21,901] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:10:22,475] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 22:10:48,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 22:10:49,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 22:10:50,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 22:10:51,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 22:11:01,691] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 22:11:02,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:11:02,874] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 22:11:19,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 22:11:20,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 22:11:21,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 22:11:22,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 22:11:32,278] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 22:11:32,690] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 22:11:33,217] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 22:11:49,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 22:11:50,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 22:11:50,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 22:11:51,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 22:12:01,727] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-28 22:17:08,486] INFO : Step 13100/50000 rank4 | Train Loss: 1.6165 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3082 | Cardinality: 0.2635 | Classification: 0.1364 | Classification_binary: 0.2628 | HitGrouping: 0.1835 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0026 | Regression_pt: 0.1957 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0953 | Classification_binary: 0.2852 | Regression_cos_phi: 0.1077 | Regression_energy: 0.1048 | Regression_eta: 0.0812 | Regression_pt: 0.1118 | Regression_sin_phi: 0.1231 | LR: 8.56e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2712s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0479 | max_forward: 0.8994 | max_loss: 0.2807 | max_backward: 2.1108 | max_optimizer_step: 0.2760 | max_tensorboard_logging: 0.9690 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 7057.8 | uss_MB: 6143.5 | child_rss_MB: 5948.1 | allocated_MB: 446.5 | reserved_MB: 23120.0 | inactive_split_MB: 920.0 +rank4 [2026-10-28 22:22:01,651] INFO : Step 13200/50000 rank4 | Train Loss: 1.4928 | Train Loss Unweighted: 0.7469 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.2211 | Classification: 0.1278 | Classification_binary: 0.2505 | HitGrouping: 0.1746 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1808 | Regression_eta: 0.0025 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0734 | Classification_binary: 0.2540 | Regression_cos_phi: 0.0939 | Regression_energy: 0.0935 | Regression_eta: 0.0741 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0885 | LR: 8.54e-04 | DataLoad Time: 0.0191s | Model Forward Time: 6.5995s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.7755 | max_loss: 0.2864 | max_backward: 3.4589 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6222.1 | uss_MB: 5074.0 | child_rss_MB: 5943.3 | allocated_MB: 448.3 | reserved_MB: 25156.0 | inactive_split_MB: 978.0 +rank4 [2026-10-28 22:26:52,973] INFO : Step 13300/50000 rank4 | Train Loss: 1.3251 | Train Loss Unweighted: 0.6741 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1607 | Classification: 0.1132 | Classification_binary: 0.1927 | HitGrouping: 0.1773 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1805 | Regression_eta: 0.0024 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1782 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0875 | Regression_eta: 0.0776 | Regression_pt: 0.0938 | Regression_sin_phi: 0.1310 | LR: 8.52e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3786s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8118 | max_loss: 0.1467 | max_backward: 1.5503 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.9039 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6221.5 | uss_MB: 5307.8 | child_rss_MB: 5952.8 | allocated_MB: 446.0 | reserved_MB: 26736.0 | inactive_split_MB: 965.8 +rank4 [2026-10-28 22:31:44,118] INFO : Step 13400/50000 rank4 | Train Loss: 1.2635 | Train Loss Unweighted: 0.6770 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1436 | Classification: 0.1317 | Classification_binary: 0.2016 | HitGrouping: 0.1790 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1692 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0898 | Classification_binary: 0.2159 | Regression_cos_phi: 0.0673 | Regression_energy: 0.1013 | Regression_eta: 0.0453 | Regression_pt: 0.1122 | Regression_sin_phi: 0.0967 | LR: 8.50e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7363s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8396 | max_loss: 0.1412 | max_backward: 2.0130 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.8047 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6225.0 | uss_MB: 5310.9 | child_rss_MB: 5925.4 | allocated_MB: 445.8 | reserved_MB: 26736.0 | inactive_split_MB: 1014.4 +rank4 [2026-10-28 22:36:27,626] INFO : Step 13500/50000 rank4 | Train Loss: 1.3935 | Train Loss Unweighted: 0.7401 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1713 | Classification: 0.1245 | Classification_binary: 0.2541 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0021 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0759 | Classification_binary: 0.2440 | Regression_cos_phi: 0.1104 | Regression_energy: 0.1022 | Regression_eta: 0.0594 | Regression_pt: 0.1100 | Regression_sin_phi: 0.0770 | LR: 8.48e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5761s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0859 | max_forward: 0.8721 | max_loss: 0.1498 | max_backward: 1.7581 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.7855 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6221.6 | uss_MB: 5308.9 | child_rss_MB: 5951.2 | allocated_MB: 449.1 | reserved_MB: 26736.0 | inactive_split_MB: 922.5 +rank4 [2026-10-28 22:41:19,303] INFO : Step 13600/50000 rank4 | Train Loss: 1.3997 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1870 | Classification: 0.1206 | Classification_binary: 0.2286 | HitGrouping: 0.1765 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1702 | Regression_eta: 0.0026 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0897 | Classification_binary: 0.2291 | Regression_cos_phi: 0.1074 | Regression_energy: 0.0978 | Regression_eta: 0.0809 | Regression_pt: 0.1080 | Regression_sin_phi: 0.1241 | LR: 8.45e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.2254s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7703 | max_loss: 0.2484 | max_backward: 1.6943 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 6210.7 | uss_MB: 5299.9 | child_rss_MB: 5953.4 | allocated_MB: 450.3 | reserved_MB: 26736.0 | inactive_split_MB: 922.0 +rank4 [2026-10-28 22:46:12,326] INFO : Step 13700/50000 rank4 | Train Loss: 1.2648 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2420 | Cardinality: 0.1315 | Classification: 0.1136 | Classification_binary: 0.1964 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1793 | Regression_eta: 0.0020 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.1881 | Regression_cos_phi: 0.0782 | Regression_energy: 0.1037 | Regression_eta: 0.0759 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0889 | LR: 8.43e-04 | DataLoad Time: 0.0174s | Model Forward Time: 6.2729s | Phase Times: max_pre_step_barrier: 0.0406 | max_batch_to_device: 0.0119 | max_forward: 1.2281 | max_loss: 0.2480 | max_backward: 3.6472 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.6460 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0767 | Memory: rss_MB: 6216.0 | uss_MB: 5307.4 | child_rss_MB: 5948.4 | allocated_MB: 446.9 | reserved_MB: 26742.0 | inactive_split_MB: 1058.7 +rank4 [2026-10-28 22:51:09,233] INFO : Step 13800/50000 rank4 | Train Loss: 1.3469 | Train Loss Unweighted: 0.7266 | Train Loss Components: Auxiliary: 0.2604 | Cardinality: 0.1436 | Classification: 0.1184 | Classification_binary: 0.2382 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1812 | Regression_eta: 0.0021 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.2224 | Regression_cos_phi: 0.0801 | Regression_energy: 0.0889 | Regression_eta: 0.0530 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0958 | LR: 8.41e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.8519s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7995 | max_loss: 0.1458 | max_backward: 3.4735 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6221.0 | uss_MB: 5310.1 | child_rss_MB: 5919.7 | allocated_MB: 446.4 | reserved_MB: 27082.0 | inactive_split_MB: 1370.3 +rank4 [2026-10-28 22:56:04,349] INFO : Step 13900/50000 rank4 | Train Loss: 1.4882 | Train Loss Unweighted: 0.7682 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1631 | Classification: 0.1199 | Classification_binary: 0.2624 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1900 | Regression_eta: 0.0029 | Regression_pt: 0.1902 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0823 | Classification_binary: 0.2873 | Regression_cos_phi: 0.1328 | Regression_energy: 0.0975 | Regression_eta: 0.0758 | Regression_pt: 0.1041 | Regression_sin_phi: 0.1266 | LR: 8.39e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4535s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.7750 | max_loss: 0.1563 | max_backward: 1.5716 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.8868 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6217.9 | uss_MB: 5307.2 | child_rss_MB: 5946.5 | allocated_MB: 449.9 | reserved_MB: 27316.0 | inactive_split_MB: 918.0 +rank4 [2026-10-28 23:00:54,751] INFO : Step 14000/50000 rank4 | Train Loss: 1.2239 | Train Loss Unweighted: 0.6384 | Train Loss Components: Auxiliary: 0.2362 | Cardinality: 0.1573 | Classification: 0.0898 | Classification_binary: 0.1853 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0021 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2069 | Regression_cos_phi: 0.1044 | Regression_energy: 0.1098 | Regression_eta: 0.1079 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0720 | LR: 8.37e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5027s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7611 | max_loss: 0.1442 | max_backward: 1.7887 | max_optimizer_step: 0.2976 | max_tensorboard_logging: 0.7418 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6219.2 | uss_MB: 5309.9 | child_rss_MB: 5948.9 | allocated_MB: 446.8 | reserved_MB: 29264.0 | inactive_split_MB: 1150.9 +rank4 [2026-10-28 23:00:55,745] INFO : Running validation on rank4 +rank4 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 23:01:50,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:01:51,151] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 23:02:17,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 23:02:18,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 23:02:20,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 23:02:21,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 23:02:31,998] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-28 23:02:33,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:02:33,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 23:02:50,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 23:02:51,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 23:02:51,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 23:02:52,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 23:03:02,970] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 23:03:03,407] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:03:03,862] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 23:03:19,310] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 23:03:20,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 23:03:20,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 23:03:21,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 23:03:31,805] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 23:08:22,314] INFO : Step 14100/50000 rank4 | Train Loss: 1.3472 | Train Loss Unweighted: 0.6964 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1511 | Classification: 0.1164 | Classification_binary: 0.2090 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1819 | Regression_eta: 0.0023 | Regression_pt: 0.1843 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.2370 | Regression_cos_phi: 0.1290 | Regression_energy: 0.0857 | Regression_eta: 0.0707 | Regression_pt: 0.0961 | Regression_sin_phi: 0.1184 | LR: 8.35e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2184s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8885 | max_loss: 0.1430 | max_backward: 2.1784 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 1.1214 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7149.4 | uss_MB: 6238.2 | child_rss_MB: 5960.3 | allocated_MB: 451.8 | reserved_MB: 25710.0 | inactive_split_MB: 1103.7 +rank4 [2026-10-28 23:12:52,861] INFO : Step 14200/50000 rank4 | Train Loss: 1.4358 | Train Loss Unweighted: 0.7354 | Train Loss Components: Auxiliary: 0.2744 | Cardinality: 0.1685 | Classification: 0.1248 | Classification_binary: 0.2623 | HitGrouping: 0.1814 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1709 | Regression_eta: 0.0023 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0834 | Classification_binary: 0.3017 | Regression_cos_phi: 0.1248 | Regression_energy: 0.0945 | Regression_eta: 0.0842 | Regression_pt: 0.1042 | Regression_sin_phi: 0.1613 | LR: 8.32e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3436s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0875 | max_forward: 0.8579 | max_loss: 0.1577 | max_backward: 1.8303 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6333.7 | uss_MB: 5422.0 | child_rss_MB: 5960.5 | allocated_MB: 449.2 | reserved_MB: 27638.0 | inactive_split_MB: 1319.6 +rank4 [2026-10-28 23:17:26,477] INFO : Step 14300/50000 rank4 | Train Loss: 1.2642 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2452 | Cardinality: 0.1652 | Classification: 0.1069 | Classification_binary: 0.2108 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0018 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0658 | Classification_binary: 0.2170 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0823 | Regression_eta: 0.0511 | Regression_pt: 0.0889 | Regression_sin_phi: 0.1092 | LR: 8.30e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3455s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7599 | max_loss: 0.1477 | max_backward: 1.7369 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6331.9 | uss_MB: 5419.4 | child_rss_MB: 5959.7 | allocated_MB: 446.3 | reserved_MB: 29582.0 | inactive_split_MB: 949.7 +rank4 [2026-10-28 23:22:05,870] INFO : Step 14400/50000 rank4 | Train Loss: 1.2457 | Train Loss Unweighted: 0.6253 | Train Loss Components: Auxiliary: 0.2300 | Cardinality: 0.1408 | Classification: 0.1072 | Classification_binary: 0.1847 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1629 | Regression_eta: 0.0022 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0868 | Regression_energy: 0.0797 | Regression_eta: 0.0780 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0947 | LR: 8.28e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.9700s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.9667 | max_loss: 0.2306 | max_backward: 2.7434 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6331.8 | uss_MB: 4969.8 | child_rss_MB: 5956.2 | allocated_MB: 450.7 | reserved_MB: 29584.0 | inactive_split_MB: 1252.8 +rank4 [2026-10-28 23:26:50,539] INFO : Step 14500/50000 rank4 | Train Loss: 1.4093 | Train Loss Unweighted: 0.7194 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1803 | Classification: 0.1211 | Classification_binary: 0.2406 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.2009 | Regression_cos_phi: 0.0892 | Regression_energy: 0.1024 | Regression_eta: 0.0531 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0594 | LR: 8.26e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8479s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8204 | max_loss: 0.1455 | max_backward: 2.0838 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7592 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1066 | Memory: rss_MB: 6329.0 | uss_MB: 5423.5 | child_rss_MB: 5932.6 | allocated_MB: 449.1 | reserved_MB: 29584.0 | inactive_split_MB: 1312.1 +rank4 [2026-10-28 23:31:26,129] INFO : Step 14600/50000 rank4 | Train Loss: 1.4466 | Train Loss Unweighted: 0.7628 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1607 | Classification: 0.1257 | Classification_binary: 0.2761 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1767 | Regression_eta: 0.0023 | Regression_pt: 0.1796 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2525 | Regression_cos_phi: 0.0990 | Regression_energy: 0.0917 | Regression_eta: 0.0497 | Regression_pt: 0.1006 | Regression_sin_phi: 0.1144 | LR: 8.24e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8560s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0622 | max_forward: 0.8658 | max_loss: 0.1496 | max_backward: 1.9094 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.9894 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6320.2 | uss_MB: 5414.6 | child_rss_MB: 5962.0 | allocated_MB: 450.2 | reserved_MB: 29584.0 | inactive_split_MB: 939.8 +rank4 [2026-10-28 23:35:59,706] INFO : Step 14700/50000 rank4 | Train Loss: 1.2298 | Train Loss Unweighted: 0.6544 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1321 | Classification: 0.0995 | Classification_binary: 0.1882 | HitGrouping: 0.1788 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1813 | Regression_eta: 0.0021 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0703 | Classification_binary: 0.1777 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0880 | Regression_eta: 0.0610 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0740 | LR: 8.21e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.9550s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8361 | max_loss: 0.2462 | max_backward: 1.9700 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.8314 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6323.9 | uss_MB: 5421.4 | child_rss_MB: 5964.3 | allocated_MB: 445.8 | reserved_MB: 31548.0 | inactive_split_MB: 1127.1 +rank4 [2026-10-28 23:40:34,505] INFO : Step 14800/50000 rank4 | Train Loss: 1.2682 | Train Loss Unweighted: 0.6574 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1516 | Classification: 0.1063 | Classification_binary: 0.2120 | HitGrouping: 0.1767 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1664 | Regression_eta: 0.0020 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.2165 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0958 | Regression_eta: 0.0484 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0728 | LR: 8.19e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.6958s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7530 | max_loss: 0.1476 | max_backward: 1.8229 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6323.4 | uss_MB: 5420.5 | child_rss_MB: 5957.9 | allocated_MB: 449.4 | reserved_MB: 33554.0 | inactive_split_MB: 1332.6 +rank4 [2026-10-28 23:45:11,571] INFO : Step 14900/50000 rank4 | Train Loss: 1.2753 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2409 | Cardinality: 0.1305 | Classification: 0.1126 | Classification_binary: 0.2387 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1497 | Regression_eta: 0.0022 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.2358 | Regression_cos_phi: 0.1037 | Regression_energy: 0.0777 | Regression_eta: 0.0626 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0971 | LR: 8.17e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.8754s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.7955 | max_loss: 0.1633 | max_backward: 1.8376 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.9623 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1206 | Memory: rss_MB: 6322.3 | uss_MB: 5420.8 | child_rss_MB: 5961.8 | allocated_MB: 449.7 | reserved_MB: 33554.0 | inactive_split_MB: 1310.1 +rank4 [2026-10-28 23:49:45,306] INFO : Step 15000/50000 rank4 | Train Loss: 1.2701 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1516 | Classification: 0.1012 | Classification_binary: 0.2121 | HitGrouping: 0.1755 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2268 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0845 | Regression_eta: 0.0870 | Regression_pt: 0.0921 | Regression_sin_phi: 0.1321 | LR: 8.15e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1019s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7505 | max_loss: 0.2449 | max_backward: 1.7335 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.4870 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6318.2 | uss_MB: 5419.0 | child_rss_MB: 5953.6 | allocated_MB: 447.3 | reserved_MB: 33554.0 | inactive_split_MB: 1038.0 +rank4 [2026-10-28 23:49:46,286] INFO : Running validation on rank4 +rank4 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-28 23:50:37,476] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:50:38,005] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-28 23:51:04,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-28 23:51:05,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-28 23:51:06,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-28 23:51:07,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-28 23:51:17,463] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 23:51:18,491] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:51:18,969] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-28 23:51:34,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-28 23:51:35,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-28 23:51:36,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-28 23:51:37,674] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-28 23:51:47,684] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-28 23:51:48,549] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-28 23:51:49,026] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-28 23:52:04,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-28 23:52:05,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-28 23:52:06,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-28 23:52:07,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-28 23:52:17,430] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-28 23:57:18,881] INFO : Step 15100/50000 rank4 | Train Loss: 1.2388 | Train Loss Unweighted: 0.6590 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1366 | Classification: 0.1178 | Classification_binary: 0.1841 | HitGrouping: 0.1810 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1743 | Regression_eta: 0.0018 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.1750 | Regression_cos_phi: 0.0843 | Regression_energy: 0.0932 | Regression_eta: 0.0573 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0934 | LR: 8.12e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8271s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.8010 | max_loss: 0.1660 | max_backward: 1.9832 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.8565 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7422.7 | uss_MB: 6518.8 | child_rss_MB: 5938.1 | allocated_MB: 446.2 | reserved_MB: 25230.0 | inactive_split_MB: 1087.3 +rank4 [2026-10-29 00:02:05,559] INFO : Step 15200/50000 rank4 | Train Loss: 1.3362 | Train Loss Unweighted: 0.6611 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1596 | Classification: 0.1086 | Classification_binary: 0.2256 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1600 | Regression_eta: 0.0022 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.2048 | Regression_cos_phi: 0.1062 | Regression_energy: 0.0742 | Regression_eta: 0.0547 | Regression_pt: 0.0812 | Regression_sin_phi: 0.1175 | LR: 8.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6162s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1077 | max_forward: 0.8572 | max_loss: 0.1505 | max_backward: 1.8951 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7089 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6402.3 | uss_MB: 5500.4 | child_rss_MB: 5939.7 | allocated_MB: 449.9 | reserved_MB: 25448.0 | inactive_split_MB: 1183.4 +rank4 [2026-10-29 00:06:54,749] INFO : Step 15300/50000 rank4 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6783 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1400 | Classification: 0.1062 | Classification_binary: 0.2357 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1646 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.2348 | Regression_cos_phi: 0.1041 | Regression_energy: 0.0918 | Regression_eta: 0.0567 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0874 | LR: 8.08e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7476s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7962 | max_loss: 0.2606 | max_backward: 1.9831 | max_optimizer_step: 0.2491 | max_tensorboard_logging: 0.7840 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6402.1 | uss_MB: 5500.3 | child_rss_MB: 5940.3 | allocated_MB: 446.7 | reserved_MB: 27282.0 | inactive_split_MB: 1189.4 +rank4 [2026-10-29 00:11:34,440] INFO : Step 15400/50000 rank4 | Train Loss: 1.2329 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1583 | Classification: 0.1127 | Classification_binary: 0.1891 | HitGrouping: 0.1750 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1648 | Regression_eta: 0.0018 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2149 | Regression_cos_phi: 0.1194 | Regression_energy: 0.0864 | Regression_eta: 0.0503 | Regression_pt: 0.0919 | Regression_sin_phi: 0.1078 | LR: 8.05e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2514s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.7153 | max_loss: 0.1508 | max_backward: 1.6501 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.5965 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6395.9 | uss_MB: 5491.2 | child_rss_MB: 5941.0 | allocated_MB: 444.7 | reserved_MB: 27520.0 | inactive_split_MB: 1102.9 +rank4 [2026-10-29 00:16:23,332] INFO : Step 15500/50000 rank4 | Train Loss: 1.2944 | Train Loss Unweighted: 0.6657 | Train Loss Components: Auxiliary: 0.2480 | Cardinality: 0.1645 | Classification: 0.1174 | Classification_binary: 0.2036 | HitGrouping: 0.1739 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1691 | Regression_eta: 0.0021 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.2234 | Regression_cos_phi: 0.1072 | Regression_energy: 0.0808 | Regression_eta: 0.0720 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0800 | LR: 8.03e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.2295s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.9970 | max_loss: 0.1740 | max_backward: 2.3916 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.8759 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 6406.1 | uss_MB: 5472.3 | child_rss_MB: 5938.5 | allocated_MB: 449.9 | reserved_MB: 27520.0 | inactive_split_MB: 1174.5 +rank4 [2026-10-29 00:21:07,252] INFO : Step 15600/50000 rank4 | Train Loss: 1.4613 | Train Loss Unweighted: 0.7617 | Train Loss Components: Auxiliary: 0.2872 | Cardinality: 0.1801 | Classification: 0.1276 | Classification_binary: 0.2641 | HitGrouping: 0.1785 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1795 | Regression_eta: 0.0025 | Regression_pt: 0.1856 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.2918 | Regression_cos_phi: 0.1019 | Regression_energy: 0.0977 | Regression_eta: 0.0616 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0837 | LR: 8.01e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.7232s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.9893 | max_loss: 0.1680 | max_backward: 2.1731 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.0419 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6396.7 | uss_MB: 5034.1 | child_rss_MB: 5938.2 | allocated_MB: 448.1 | reserved_MB: 27520.0 | inactive_split_MB: 1080.9 +rank4 [2026-10-29 00:25:54,170] INFO : Step 15700/50000 rank4 | Train Loss: 1.3299 | Train Loss Unweighted: 0.6547 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1880 | Classification: 0.1192 | Classification_binary: 0.2203 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1547 | Regression_eta: 0.0020 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.2340 | Regression_cos_phi: 0.1093 | Regression_energy: 0.0984 | Regression_eta: 0.0752 | Regression_pt: 0.1072 | Regression_sin_phi: 0.1097 | LR: 7.98e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.0309s | Phase Times: max_pre_step_barrier: 0.0061 | max_batch_to_device: 0.0068 | max_forward: 0.9761 | max_loss: 0.2543 | max_backward: 2.0225 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8016 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.1020 | Memory: rss_MB: 6402.6 | uss_MB: 5496.7 | child_rss_MB: 5941.3 | allocated_MB: 448.3 | reserved_MB: 27520.0 | inactive_split_MB: 1390.5 +rank4 [2026-10-29 00:30:33,525] INFO : Step 15800/50000 rank4 | Train Loss: 1.2079 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1572 | Classification: 0.0909 | Classification_binary: 0.1814 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1704 | Regression_eta: 0.0019 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0646 | Classification_binary: 0.1776 | Regression_cos_phi: 0.1066 | Regression_energy: 0.0890 | Regression_eta: 0.0794 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0808 | LR: 7.96e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3427s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1066 | max_forward: 0.8154 | max_loss: 0.1440 | max_backward: 1.6020 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.7740 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6405.3 | uss_MB: 5497.2 | child_rss_MB: 5941.3 | allocated_MB: 447.1 | reserved_MB: 27520.0 | inactive_split_MB: 1038.3 +rank4 [2026-10-29 00:35:19,264] INFO : Step 15900/50000 rank4 | Train Loss: 1.4055 | Train Loss Unweighted: 0.7313 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1808 | Classification: 0.1304 | Classification_binary: 0.2426 | HitGrouping: 0.1763 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1752 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0909 | Classification_binary: 0.2550 | Regression_cos_phi: 0.1110 | Regression_energy: 0.0921 | Regression_eta: 0.0680 | Regression_pt: 0.1017 | Regression_sin_phi: 0.1077 | LR: 7.94e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7725s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 1.0653 | max_loss: 0.1515 | max_backward: 2.0024 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.6957 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6402.5 | uss_MB: 5500.0 | child_rss_MB: 5940.1 | allocated_MB: 447.4 | reserved_MB: 27520.0 | inactive_split_MB: 1172.5 +rank4 [2026-10-29 00:40:00,683] INFO : Step 16000/50000 rank4 | Train Loss: 1.3107 | Train Loss Unweighted: 0.6580 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1760 | Classification: 0.1019 | Classification_binary: 0.2391 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1548 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.2351 | Regression_cos_phi: 0.0860 | Regression_energy: 0.0981 | Regression_eta: 0.0492 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0812 | LR: 7.91e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7630s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.1053 | max_forward: 1.1703 | max_loss: 0.1516 | max_backward: 1.8974 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6403.0 | uss_MB: 5502.7 | child_rss_MB: 5941.0 | allocated_MB: 447.2 | reserved_MB: 27520.0 | inactive_split_MB: 1195.6 +rank4 [2026-10-29 00:40:01,696] INFO : Running validation on rank4 +rank4 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 00:40:58,076] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:40:58,630] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 00:41:23,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 00:41:24,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 00:41:25,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 00:41:26,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 00:41:36,361] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 00:41:36,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:41:37,530] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 00:41:53,277] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 00:41:54,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 00:41:55,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 00:41:56,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 00:42:06,198] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 00:42:07,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 00:42:07,840] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 00:42:23,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 00:42:24,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 00:42:25,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 00:42:26,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 00:42:36,110] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 00:47:34,083] INFO : Step 16100/50000 rank4 | Train Loss: 1.2037 | Train Loss Unweighted: 0.6025 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1467 | Classification: 0.1080 | Classification_binary: 0.1878 | HitGrouping: 0.1747 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1512 | Regression_eta: 0.0018 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0892 | Regression_energy: 0.0684 | Regression_eta: 0.0552 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0927 | LR: 7.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0447s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7665 | max_loss: 0.1405 | max_backward: 1.4602 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7547.3 | uss_MB: 6643.5 | child_rss_MB: 5941.5 | allocated_MB: 446.1 | reserved_MB: 27680.0 | inactive_split_MB: 1038.2 +rank4 [2026-10-29 00:52:15,156] INFO : Step 16200/50000 rank4 | Train Loss: 1.2083 | Train Loss Unweighted: 0.6192 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1499 | Classification: 0.1107 | Classification_binary: 0.2017 | HitGrouping: 0.1711 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1506 | Regression_eta: 0.0018 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1911 | Regression_cos_phi: 0.1044 | Regression_energy: 0.0841 | Regression_eta: 0.0398 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0801 | LR: 7.86e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9524s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0473 | max_forward: 0.8343 | max_loss: 0.2530 | max_backward: 2.1136 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6458.0 | uss_MB: 5555.6 | child_rss_MB: 5961.3 | allocated_MB: 446.7 | reserved_MB: 27682.0 | inactive_split_MB: 1157.2 +rank4 [2026-10-29 00:56:51,718] INFO : Step 16300/50000 rank4 | Train Loss: 1.2745 | Train Loss Unweighted: 0.6291 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1604 | Classification: 0.1062 | Classification_binary: 0.2417 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1373 | Regression_eta: 0.0022 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2300 | Regression_cos_phi: 0.0932 | Regression_energy: 0.0778 | Regression_eta: 0.0522 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0818 | LR: 7.84e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.8118s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.8059 | max_loss: 0.2506 | max_backward: 2.1763 | max_optimizer_step: 0.5213 | max_tensorboard_logging: 0.5923 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6459.9 | uss_MB: 5555.3 | child_rss_MB: 5936.4 | allocated_MB: 450.9 | reserved_MB: 27682.0 | inactive_split_MB: 950.8 +rank4 [2026-10-29 01:01:26,581] INFO : Step 16400/50000 rank4 | Train Loss: 1.2654 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2380 | Cardinality: 0.1554 | Classification: 0.1095 | Classification_binary: 0.2018 | HitGrouping: 0.1699 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1561 | Regression_eta: 0.0023 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1925 | Regression_cos_phi: 0.1131 | Regression_energy: 0.0906 | Regression_eta: 0.0725 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0952 | LR: 7.81e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2795s | Phase Times: max_pre_step_barrier: 0.0533 | max_batch_to_device: 0.0064 | max_forward: 0.7459 | max_loss: 0.1486 | max_backward: 1.5352 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.6677 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6452.2 | uss_MB: 5545.1 | child_rss_MB: 5962.3 | allocated_MB: 446.4 | reserved_MB: 27682.0 | inactive_split_MB: 949.1 +rank4 [2026-10-29 01:06:05,049] INFO : Step 16500/50000 rank4 | Train Loss: 1.1728 | Train Loss Unweighted: 0.5918 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1444 | Classification: 0.0927 | Classification_binary: 0.1686 | HitGrouping: 0.1723 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0021 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1203 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0847 | Regression_eta: 0.0454 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0621 | LR: 7.79e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.4920s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8043 | max_loss: 0.1772 | max_backward: 2.0250 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6458.1 | uss_MB: 5554.8 | child_rss_MB: 5927.7 | allocated_MB: 448.6 | reserved_MB: 29698.0 | inactive_split_MB: 1167.6 +rank4 [2026-10-29 01:10:53,703] INFO : Step 16600/50000 rank4 | Train Loss: 1.2920 | Train Loss Unweighted: 0.6427 | Train Loss Components: Auxiliary: 0.2455 | Cardinality: 0.1794 | Classification: 0.1148 | Classification_binary: 0.2098 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1561 | Regression_eta: 0.0021 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.1791 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0736 | Regression_eta: 0.0470 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0731 | LR: 7.77e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2551s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0377 | max_forward: 1.0830 | max_loss: 0.1480 | max_backward: 1.7919 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6457.2 | uss_MB: 5552.9 | child_rss_MB: 5951.6 | allocated_MB: 449.3 | reserved_MB: 29698.0 | inactive_split_MB: 1014.7 +rank4 [2026-10-29 01:15:45,847] INFO : Step 16700/50000 rank4 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6325 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1731 | Classification: 0.1072 | Classification_binary: 0.2252 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1469 | Regression_eta: 0.0019 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.2499 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0801 | Regression_eta: 0.0507 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0981 | LR: 7.74e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4552s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0722 | max_loss: 0.1506 | max_backward: 2.0365 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5067 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6456.8 | uss_MB: 5555.5 | child_rss_MB: 5918.8 | allocated_MB: 448.3 | reserved_MB: 29698.0 | inactive_split_MB: 893.3 +rank4 [2026-10-29 01:20:20,822] INFO : Step 16800/50000 rank4 | Train Loss: 1.1947 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1542 | Classification: 0.1036 | Classification_binary: 0.1800 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0023 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1820 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0786 | Regression_eta: 0.0476 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0675 | LR: 7.72e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.2730s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0166 | max_forward: 0.8224 | max_loss: 0.1466 | max_backward: 1.6608 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6428.0 | uss_MB: 5552.9 | child_rss_MB: 5812.3 | allocated_MB: 446.4 | reserved_MB: 29698.0 | inactive_split_MB: 1161.6 +rank4 [2026-10-29 01:25:06,862] INFO : Step 16900/50000 rank4 | Train Loss: 1.2025 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1406 | Classification: 0.1120 | Classification_binary: 0.1884 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1446 | Regression_eta: 0.0023 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1844 | Regression_cos_phi: 0.0856 | Regression_energy: 0.0776 | Regression_eta: 0.0606 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0690 | LR: 7.69e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.3245s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7445 | max_loss: 0.1453 | max_backward: 1.8211 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6418.8 | uss_MB: 5553.4 | child_rss_MB: 5783.4 | allocated_MB: 448.9 | reserved_MB: 29698.0 | inactive_split_MB: 901.3 +rank4 [2026-10-29 01:29:52,082] INFO : Step 17000/50000 rank4 | Train Loss: 1.2049 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2349 | Cardinality: 0.1467 | Classification: 0.1018 | Classification_binary: 0.2228 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0728 | Classification_binary: 0.2210 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0741 | Regression_eta: 0.0363 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0739 | LR: 7.67e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.1416s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0141 | max_forward: 1.3980 | max_loss: 0.5129 | max_backward: 2.5664 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.5255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1218 | Memory: rss_MB: 6382.7 | uss_MB: 5556.2 | child_rss_MB: 5576.1 | allocated_MB: 448.2 | reserved_MB: 29698.0 | inactive_split_MB: 955.8 +rank4 [2026-10-29 01:29:53,074] INFO : Running validation on rank4 +rank4 [2026-10-29 01:29:53,074] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 01:29:53,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 01:30:46,066] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:30:46,579] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 01:31:13,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 01:31:14,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 01:31:15,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 01:31:17,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 01:31:27,188] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 01:31:28,037] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:31:28,588] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 01:31:44,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 01:31:45,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 01:31:46,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 01:31:47,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 01:31:57,675] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 01:31:58,252] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 01:31:58,743] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 01:32:14,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 01:32:15,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 01:32:15,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 01:32:16,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 01:32:26,720] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 01:37:36,198] INFO : Step 17100/50000 rank4 | Train Loss: 1.2386 | Train Loss Unweighted: 0.5919 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1931 | Classification: 0.0995 | Classification_binary: 0.1969 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1920 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0730 | Regression_eta: 0.0393 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0704 | LR: 7.64e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.3600s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0067 | max_forward: 1.0304 | max_loss: 0.1869 | max_backward: 2.3467 | max_optimizer_step: 0.3175 | max_tensorboard_logging: 0.7405 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 7631.0 | uss_MB: 6798.9 | child_rss_MB: 5579.4 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 1373.9 +rank4 [2026-10-29 01:42:27,134] INFO : Step 17200/50000 rank4 | Train Loss: 1.1810 | Train Loss Unweighted: 0.5844 | Train Loss Components: Auxiliary: 0.2270 | Cardinality: 0.1495 | Classification: 0.1085 | Classification_binary: 0.1683 | HitGrouping: 0.1731 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0022 | Regression_pt: 0.1515 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0876 | Regression_eta: 0.0381 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0631 | LR: 7.62e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.1050s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7638 | max_loss: 0.2424 | max_backward: 1.9393 | max_optimizer_step: 0.2921 | max_tensorboard_logging: 2.1267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6444.8 | uss_MB: 5633.5 | child_rss_MB: 5577.8 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 988.1 +rank4 [2026-10-29 01:47:11,312] INFO : Step 17300/50000 rank4 | Train Loss: 1.2271 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2397 | Cardinality: 0.1372 | Classification: 0.1102 | Classification_binary: 0.2127 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1512 | Regression_eta: 0.0022 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.2169 | Regression_cos_phi: 0.1282 | Regression_energy: 0.0785 | Regression_eta: 0.0785 | Regression_pt: 0.0857 | Regression_sin_phi: 0.1050 | LR: 7.59e-04 | DataLoad Time: 0.0128s | Model Forward Time: 6.1169s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0243 | max_forward: 1.4273 | max_loss: 0.6053 | max_backward: 3.4810 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6426.5 | uss_MB: 5175.0 | child_rss_MB: 5586.3 | allocated_MB: 449.6 | reserved_MB: 26752.0 | inactive_split_MB: 1120.1 +rank4 [2026-10-29 01:52:04,203] INFO : Step 17400/50000 rank4 | Train Loss: 1.2297 | Train Loss Unweighted: 0.6013 | Train Loss Components: Auxiliary: 0.2404 | Cardinality: 0.1709 | Classification: 0.1017 | Classification_binary: 0.2232 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1354 | Regression_eta: 0.0019 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.2119 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0758 | Regression_eta: 0.0525 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0761 | LR: 7.57e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.6382s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0185 | max_forward: 1.0000 | max_loss: 0.1504 | max_backward: 1.9673 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6331 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6409.7 | uss_MB: 5631.7 | child_rss_MB: 5583.3 | allocated_MB: 447.1 | reserved_MB: 28732.0 | inactive_split_MB: 1125.6 +rank4 [2026-10-29 01:56:48,791] INFO : Step 17500/50000 rank4 | Train Loss: 1.2152 | Train Loss Unweighted: 0.6044 | Train Loss Components: Auxiliary: 0.2290 | Cardinality: 0.1485 | Classification: 0.1023 | Classification_binary: 0.1789 | HitGrouping: 0.1757 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1590 | Regression_eta: 0.0026 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0732 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0730 | Regression_eta: 0.0376 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0704 | LR: 7.54e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.2346s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0167 | max_forward: 0.7711 | max_loss: 0.1449 | max_backward: 1.8153 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.4947 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6388.7 | uss_MB: 5638.8 | child_rss_MB: 5515.3 | allocated_MB: 450.1 | reserved_MB: 28732.0 | inactive_split_MB: 1375.8 +rank4 [2026-10-29 02:01:39,791] INFO : Step 17600/50000 rank4 | Train Loss: 1.1560 | Train Loss Unweighted: 0.6042 | Train Loss Components: Auxiliary: 0.2286 | Cardinality: 0.1294 | Classification: 0.1057 | Classification_binary: 0.1993 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1471 | Regression_eta: 0.0018 | Regression_pt: 0.1484 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0577 | Regression_eta: 0.0510 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0677 | LR: 7.52e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.8141s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.8868 | max_loss: 0.1705 | max_backward: 2.0183 | max_optimizer_step: 0.2066 | max_tensorboard_logging: 0.7222 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1349 | Memory: rss_MB: 6384.6 | uss_MB: 5639.3 | child_rss_MB: 5506.2 | allocated_MB: 448.3 | reserved_MB: 28732.0 | inactive_split_MB: 1319.7 +rank4 [2026-10-29 02:06:28,891] INFO : Step 17700/50000 rank4 | Train Loss: 1.7848 | Train Loss Unweighted: 0.7433 | Train Loss Components: Auxiliary: 0.3415 | Cardinality: 0.4306 | Classification: 0.1200 | Classification_binary: 0.2341 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1920 | Regression_eta: 0.0037 | Regression_pt: 0.1910 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2334 | Regression_cos_phi: 0.0982 | Regression_energy: 0.1150 | Regression_eta: 0.1176 | Regression_pt: 0.1219 | Regression_sin_phi: 0.1038 | LR: 7.49e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.5785s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8788 | max_loss: 0.4738 | max_backward: 1.9859 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.5305 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6359.7 | uss_MB: 5640.4 | child_rss_MB: 5400.4 | allocated_MB: 450.1 | reserved_MB: 28738.0 | inactive_split_MB: 1125.6 +rank4 [2026-10-29 02:11:14,774] INFO : Step 17800/50000 rank4 | Train Loss: 1.1828 | Train Loss Unweighted: 0.6014 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1456 | Classification: 0.0982 | Classification_binary: 0.1996 | HitGrouping: 0.1749 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1503 | Regression_eta: 0.0019 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0403 | Classification_binary: 0.2037 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0846 | Regression_eta: 0.0383 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0751 | LR: 7.47e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.1942s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.3167 | max_backward: 2.2751 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 6343.6 | uss_MB: 5261.8 | child_rss_MB: 5331.5 | allocated_MB: 444.9 | reserved_MB: 28738.0 | inactive_split_MB: 1155.9 +rank4 [2026-10-29 02:16:10,312] INFO : Step 17900/50000 rank4 | Train Loss: 1.0765 | Train Loss Unweighted: 0.5646 | Train Loss Components: Auxiliary: 0.2108 | Cardinality: 0.1097 | Classification: 0.0932 | Classification_binary: 0.1585 | HitGrouping: 0.1686 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1536 | Regression_eta: 0.0016 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1459 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0730 | Regression_eta: 0.0360 | Regression_pt: 0.0795 | Regression_sin_phi: 0.0908 | LR: 7.44e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2962s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7593 | max_loss: 0.4918 | max_backward: 1.7987 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.6052 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6290.9 | uss_MB: 5640.2 | child_rss_MB: 5109.0 | allocated_MB: 447.1 | reserved_MB: 28738.0 | inactive_split_MB: 1154.4 +rank4 [2026-10-29 02:20:55,876] INFO : Step 18000/50000 rank4 | Train Loss: 1.2227 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2303 | Cardinality: 0.1450 | Classification: 0.1088 | Classification_binary: 0.2099 | HitGrouping: 0.1650 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1447 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2389 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0797 | Regression_eta: 0.0700 | Regression_pt: 0.0854 | Regression_sin_phi: 0.1044 | LR: 7.42e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7011s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9293 | max_loss: 0.1567 | max_backward: 2.2476 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6283.0 | uss_MB: 5642.9 | child_rss_MB: 5063.5 | allocated_MB: 449.4 | reserved_MB: 28738.0 | inactive_split_MB: 1324.2 +rank4 [2026-10-29 02:20:56,966] INFO : Running validation on rank4 +rank4 [2026-10-29 02:20:56,967] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 02:20:56,967] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 02:21:57,514] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:21:58,045] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 02:22:25,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 02:22:25,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 02:22:27,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 02:22:28,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 02:22:38,114] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 02:22:38,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:22:39,418] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 02:22:55,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 02:22:56,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 02:22:57,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 02:22:58,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 02:23:08,468] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 02:23:08,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 02:23:09,547] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 02:23:25,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 02:23:26,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 02:23:26,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 02:23:27,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 02:23:37,748] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 02:28:43,003] INFO : Step 18100/50000 rank4 | Train Loss: 1.2922 | Train Loss Unweighted: 0.6330 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1763 | Classification: 0.1061 | Classification_binary: 0.2257 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1467 | Regression_eta: 0.0022 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.2313 | Regression_cos_phi: 0.0892 | Regression_energy: 0.0667 | Regression_eta: 0.0445 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0687 | LR: 7.39e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.0681s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.9150 | max_loss: 0.1573 | max_backward: 2.1915 | max_optimizer_step: 0.2256 | max_tensorboard_logging: 0.8426 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7551.6 | uss_MB: 6907.4 | child_rss_MB: 5063.5 | allocated_MB: 448.8 | reserved_MB: 25396.0 | inactive_split_MB: 1192.4 +rank4 [2026-10-29 02:33:21,588] INFO : Step 18200/50000 rank4 | Train Loss: 1.1403 | Train Loss Unweighted: 0.5693 | Train Loss Components: Auxiliary: 0.2140 | Cardinality: 0.1362 | Classification: 0.1099 | Classification_binary: 0.1658 | HitGrouping: 0.1698 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1444 | Regression_eta: 0.0021 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.1556 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0839 | Regression_eta: 0.0438 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0789 | LR: 7.36e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.5263s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0059 | max_forward: 0.7843 | max_loss: 0.1427 | max_backward: 1.4948 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 1.0509 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6418.0 | uss_MB: 5773.9 | child_rss_MB: 5038.4 | allocated_MB: 446.0 | reserved_MB: 27340.0 | inactive_split_MB: 922.1 +rank4 [2026-10-29 02:37:53,622] INFO : Step 18300/50000 rank4 | Train Loss: 1.1938 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2339 | Cardinality: 0.1526 | Classification: 0.1133 | Classification_binary: 0.1878 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1493 | Regression_eta: 0.0019 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0661 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0813 | Regression_eta: 0.0430 | Regression_pt: 0.0895 | Regression_sin_phi: 0.0813 | LR: 7.34e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9190s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.9058 | max_loss: 0.1527 | max_backward: 2.1689 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6422.1 | uss_MB: 5784.1 | child_rss_MB: 5032.3 | allocated_MB: 448.4 | reserved_MB: 27340.0 | inactive_split_MB: 1185.8 +rank4 [2026-10-29 02:42:24,376] INFO : Step 18400/50000 rank4 | Train Loss: 1.1959 | Train Loss Unweighted: 0.6109 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1130 | Classification: 0.1042 | Classification_binary: 0.2213 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1400 | Regression_eta: 0.0022 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.2288 | Regression_cos_phi: 0.0997 | Regression_energy: 0.0759 | Regression_eta: 0.0508 | Regression_pt: 0.0820 | Regression_sin_phi: 0.1128 | LR: 7.31e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.7085s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8434 | max_loss: 0.1474 | max_backward: 1.9441 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6423.0 | uss_MB: 5783.9 | child_rss_MB: 5008.1 | allocated_MB: 448.8 | reserved_MB: 29306.0 | inactive_split_MB: 1091.4 +rank4 [2026-10-29 02:47:17,191] INFO : Step 18500/50000 rank4 | Train Loss: 1.2719 | Train Loss Unweighted: 0.6036 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1990 | Classification: 0.1087 | Classification_binary: 0.1793 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1538 | Regression_eta: 0.0024 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0779 | Regression_eta: 0.0555 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0620 | LR: 7.29e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.4229s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 1.6245 | max_loss: 0.3527 | max_backward: 3.4112 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.7932 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 6423.2 | uss_MB: 5781.7 | child_rss_MB: 5035.9 | allocated_MB: 447.9 | reserved_MB: 29306.0 | inactive_split_MB: 1193.4 +rank4 [2026-10-29 02:52:01,280] INFO : Step 18600/50000 rank4 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5476 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1351 | Classification: 0.0870 | Classification_binary: 0.1697 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1433 | Regression_eta: 0.0017 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0771 | Regression_eta: 0.0498 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0757 | LR: 7.26e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2409s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0527 | max_forward: 0.7639 | max_loss: 0.1373 | max_backward: 1.5273 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.7424 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6420.8 | uss_MB: 5779.1 | child_rss_MB: 5038.0 | allocated_MB: 445.6 | reserved_MB: 29306.0 | inactive_split_MB: 1194.4 +rank4 [2026-10-29 02:56:35,338] INFO : Step 18700/50000 rank4 | Train Loss: 1.2376 | Train Loss Unweighted: 0.5906 | Train Loss Components: Auxiliary: 0.2405 | Cardinality: 0.1841 | Classification: 0.1039 | Classification_binary: 0.2012 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1400 | Regression_eta: 0.0020 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0716 | Regression_eta: 0.0554 | Regression_pt: 0.0781 | Regression_sin_phi: 0.1045 | LR: 7.24e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7558s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8903 | max_loss: 0.1469 | max_backward: 2.1433 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4822 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1128 | Memory: rss_MB: 6419.7 | uss_MB: 5784.3 | child_rss_MB: 5013.1 | allocated_MB: 449.2 | reserved_MB: 31324.0 | inactive_split_MB: 1093.4 +rank4 [2026-10-29 03:01:09,007] INFO : Step 18800/50000 rank4 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1585 | Classification: 0.0943 | Classification_binary: 0.2038 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1342 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1983 | Regression_cos_phi: 0.0748 | Regression_energy: 0.0749 | Regression_eta: 0.0376 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0550 | LR: 7.21e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4227s | Phase Times: max_pre_step_barrier: 0.1142 | max_batch_to_device: 0.0068 | max_forward: 0.7808 | max_loss: 0.2502 | max_backward: 1.8229 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0996 | Memory: rss_MB: 6421.2 | uss_MB: 5781.7 | child_rss_MB: 5036.2 | allocated_MB: 450.4 | reserved_MB: 31336.0 | inactive_split_MB: 1098.4 +rank4 [2026-10-29 03:05:45,769] INFO : Step 18900/50000 rank4 | Train Loss: 1.0747 | Train Loss Unweighted: 0.5385 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.1306 | Classification: 0.0887 | Classification_binary: 0.1613 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1414 | Regression_eta: 0.0017 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0698 | Regression_energy: 0.1040 | Regression_eta: 0.0337 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0782 | LR: 7.18e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3933s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7400 | max_loss: 0.1656 | max_backward: 1.8003 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.7596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6421.5 | uss_MB: 5781.2 | child_rss_MB: 5039.7 | allocated_MB: 443.5 | reserved_MB: 31336.0 | inactive_split_MB: 928.5 +rank4 [2026-10-29 03:10:20,956] INFO : Step 19000/50000 rank4 | Train Loss: 1.1726 | Train Loss Unweighted: 0.5966 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1418 | Classification: 0.1102 | Classification_binary: 0.1870 | HitGrouping: 0.1705 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1463 | Regression_eta: 0.0019 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.1871 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0713 | Regression_eta: 0.0635 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0938 | LR: 7.16e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8905s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7637 | max_loss: 0.5203 | max_backward: 1.8720 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 1.1055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6422.5 | uss_MB: 5783.8 | child_rss_MB: 5041.8 | allocated_MB: 446.6 | reserved_MB: 31336.0 | inactive_split_MB: 1097.4 +rank4 [2026-10-29 03:10:21,983] INFO : Running validation on rank4 +rank4 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 03:11:18,714] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:11:19,247] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 03:11:45,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 03:11:46,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 03:11:48,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 03:11:49,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 03:11:59,563] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 03:12:01,384] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:12:01,928] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 03:12:17,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 03:12:18,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 03:12:19,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 03:12:20,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 03:12:30,819] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 03:12:31,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 03:12:31,971] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 03:12:47,618] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 03:12:48,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 03:12:49,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 03:12:50,119] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 03:13:00,130] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 03:17:58,240] INFO : Step 19100/50000 rank4 | Train Loss: 1.2150 | Train Loss Unweighted: 0.6107 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1439 | Classification: 0.1037 | Classification_binary: 0.2230 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1395 | Regression_eta: 0.0020 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2369 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0827 | Regression_eta: 0.0505 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0764 | LR: 7.13e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7841s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8630 | max_loss: 0.1532 | max_backward: 1.9622 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.7474 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7697.6 | uss_MB: 7053.9 | child_rss_MB: 5049.6 | allocated_MB: 449.4 | reserved_MB: 23972.0 | inactive_split_MB: 1371.8 +rank4 [2026-10-29 03:22:35,411] INFO : Step 19200/50000 rank4 | Train Loss: 1.1075 | Train Loss Unweighted: 0.5303 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1382 | Classification: 0.0958 | Classification_binary: 0.1758 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1270 | Regression_eta: 0.0019 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0630 | Regression_eta: 0.0579 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0852 | LR: 7.10e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0052s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8287 | max_loss: 0.1556 | max_backward: 1.8127 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6504.0 | uss_MB: 5863.3 | child_rss_MB: 5048.7 | allocated_MB: 445.3 | reserved_MB: 27640.0 | inactive_split_MB: 940.7 +rank4 [2026-10-29 03:27:09,997] INFO : Step 19300/50000 rank4 | Train Loss: 1.0792 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2111 | Cardinality: 0.1254 | Classification: 0.0911 | Classification_binary: 0.1561 | HitGrouping: 0.1662 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0685 | Regression_eta: 0.0532 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0727 | LR: 7.08e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.5134s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8891 | max_loss: 0.1513 | max_backward: 1.7637 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6508.5 | uss_MB: 5869.2 | child_rss_MB: 5044.9 | allocated_MB: 445.1 | reserved_MB: 27640.0 | inactive_split_MB: 1366.9 +rank4 [2026-10-29 03:31:44,571] INFO : Step 19400/50000 rank4 | Train Loss: 1.2411 | Train Loss Unweighted: 0.6183 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1555 | Classification: 0.1055 | Classification_binary: 0.2145 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1463 | Regression_eta: 0.0020 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.2165 | Regression_cos_phi: 0.0968 | Regression_energy: 0.0791 | Regression_eta: 0.0462 | Regression_pt: 0.0865 | Regression_sin_phi: 0.0648 | LR: 7.05e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.6748s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0092 | max_forward: 0.8079 | max_loss: 0.1550 | max_backward: 1.7710 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 2.8282 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6511.0 | uss_MB: 5872.2 | child_rss_MB: 5045.1 | allocated_MB: 452.5 | reserved_MB: 28010.0 | inactive_split_MB: 1195.9 +rank4 [2026-10-29 03:36:21,473] INFO : Step 19500/50000 rank4 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6008 | Train Loss Components: Auxiliary: 0.2448 | Cardinality: 0.1836 | Classification: 0.1087 | Classification_binary: 0.2123 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1374 | Regression_eta: 0.0023 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.2256 | Regression_cos_phi: 0.1072 | Regression_energy: 0.0821 | Regression_eta: 0.0863 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0872 | LR: 7.02e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6203s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7880 | max_loss: 0.1506 | max_backward: 1.7760 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.7620 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6510.7 | uss_MB: 5870.2 | child_rss_MB: 5041.2 | allocated_MB: 449.4 | reserved_MB: 29680.0 | inactive_split_MB: 1368.8 +rank4 [2026-10-29 03:40:54,623] INFO : Step 19600/50000 rank4 | Train Loss: 1.1451 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1464 | Classification: 0.0933 | Classification_binary: 0.1736 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1533 | Regression_eta: 0.0018 | Regression_pt: 0.1557 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0799 | Regression_eta: 0.0621 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0882 | LR: 7.00e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.3543s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8082 | max_loss: 0.1437 | max_backward: 1.6649 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.6567 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6510.9 | uss_MB: 5869.4 | child_rss_MB: 5045.0 | allocated_MB: 444.5 | reserved_MB: 29680.0 | inactive_split_MB: 1016.8 +rank4 [2026-10-29 03:45:30,311] INFO : Step 19700/50000 rank4 | Train Loss: 1.1424 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2236 | Cardinality: 0.1385 | Classification: 0.1093 | Classification_binary: 0.1797 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1379 | Regression_eta: 0.0020 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0830 | Regression_energy: 0.0764 | Regression_eta: 0.0712 | Regression_pt: 0.0824 | Regression_sin_phi: 0.0976 | LR: 6.97e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4879s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7612 | max_loss: 0.2297 | max_backward: 1.9668 | max_optimizer_step: 0.5229 | max_tensorboard_logging: 0.5756 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6508.5 | uss_MB: 5869.4 | child_rss_MB: 5047.4 | allocated_MB: 449.1 | reserved_MB: 29680.0 | inactive_split_MB: 1100.9 +rank4 [2026-10-29 03:50:08,779] INFO : Step 19800/50000 rank4 | Train Loss: 1.2806 | Train Loss Unweighted: 0.6390 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1432 | Classification: 0.1110 | Classification_binary: 0.2282 | HitGrouping: 0.1668 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0030 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.2048 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0787 | Regression_eta: 0.0804 | Regression_pt: 0.0877 | Regression_sin_phi: 0.1073 | LR: 6.94e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5053s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8034 | max_loss: 0.1812 | max_backward: 1.9502 | max_optimizer_step: 0.2119 | max_tensorboard_logging: 0.5857 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6510.7 | uss_MB: 5873.6 | child_rss_MB: 5045.7 | allocated_MB: 452.3 | reserved_MB: 29680.0 | inactive_split_MB: 1110.9 +rank4 [2026-10-29 03:54:46,663] INFO : Step 19900/50000 rank4 | Train Loss: 1.0692 | Train Loss Unweighted: 0.5407 | Train Loss Components: Auxiliary: 0.2077 | Cardinality: 0.1171 | Classification: 0.0922 | Classification_binary: 0.1812 | HitGrouping: 0.1610 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1862 | Regression_cos_phi: 0.0929 | Regression_energy: 0.0785 | Regression_eta: 0.0455 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0858 | LR: 6.92e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6132s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7533 | max_loss: 0.2412 | max_backward: 2.0720 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5848 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6505.8 | uss_MB: 5870.0 | child_rss_MB: 5057.5 | allocated_MB: 444.7 | reserved_MB: 29682.0 | inactive_split_MB: 1000.0 +rank4 [2026-10-29 03:59:25,168] INFO : Step 20000/50000 rank4 | Train Loss: 1.0304 | Train Loss Unweighted: 0.5275 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1060 | Classification: 0.0940 | Classification_binary: 0.1604 | HitGrouping: 0.1656 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1344 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0854 | Regression_energy: 0.0822 | Regression_eta: 0.0404 | Regression_pt: 0.0872 | Regression_sin_phi: 0.1108 | LR: 6.89e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8379s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.8721 | max_loss: 0.1507 | max_backward: 2.1861 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6505.7 | uss_MB: 5870.7 | child_rss_MB: 5049.5 | allocated_MB: 445.9 | reserved_MB: 30966.0 | inactive_split_MB: 1179.2 +rank4 [2026-10-29 03:59:26,239] INFO : Running validation on rank4 +rank4 [2026-10-29 03:59:26,239] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 03:59:26,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 04:00:19,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:00:19,530] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 04:00:44,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 04:00:45,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 04:00:46,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 04:00:47,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 04:00:58,006] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 04:00:59,045] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:00:59,502] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 04:01:15,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 04:01:16,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 04:01:17,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 04:01:18,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 04:01:28,461] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 04:01:28,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:01:29,373] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 04:01:45,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 04:01:46,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 04:01:46,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 04:01:47,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 04:01:57,642] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 04:06:53,361] INFO : Step 20100/50000 rank4 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2437 | Cardinality: 0.1586 | Classification: 0.1055 | Classification_binary: 0.2238 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1360 | Regression_eta: 0.0021 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0766 | Classification_binary: 0.2672 | Regression_cos_phi: 0.0895 | Regression_energy: 0.0753 | Regression_eta: 0.0855 | Regression_pt: 0.0828 | Regression_sin_phi: 0.1051 | LR: 6.86e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6020s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0372 | max_forward: 0.8293 | max_loss: 0.1584 | max_backward: 1.7738 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.7139 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7674.6 | uss_MB: 7034.7 | child_rss_MB: 5034.6 | allocated_MB: 448.4 | reserved_MB: 29328.0 | inactive_split_MB: 969.2 +rank4 [2026-10-29 04:11:25,836] INFO : Step 20200/50000 rank4 | Train Loss: 1.3163 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.1695 | Classification: 0.1000 | Classification_binary: 0.2250 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1433 | Regression_eta: 0.0029 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.2137 | Regression_cos_phi: 0.1121 | Regression_energy: 0.0779 | Regression_eta: 0.0827 | Regression_pt: 0.0850 | Regression_sin_phi: 0.1001 | LR: 6.84e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6422s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8823 | max_loss: 0.1595 | max_backward: 2.1030 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5136 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7674.6 | uss_MB: 7036.8 | child_rss_MB: 5027.8 | allocated_MB: 449.6 | reserved_MB: 29340.0 | inactive_split_MB: 938.7 +rank4 [2026-10-29 04:16:03,709] INFO : Step 20300/50000 rank4 | Train Loss: 1.1420 | Train Loss Unweighted: 0.5734 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1487 | Classification: 0.1087 | Classification_binary: 0.1713 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1438 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0694 | Classification_binary: 0.1780 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0748 | Regression_eta: 0.0486 | Regression_pt: 0.0832 | Regression_sin_phi: 0.0733 | LR: 6.81e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.9977s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.9650 | max_loss: 0.1904 | max_backward: 2.0801 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.8679 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6555.4 | uss_MB: 5913.2 | child_rss_MB: 5021.5 | allocated_MB: 444.8 | reserved_MB: 29344.0 | inactive_split_MB: 986.2 +rank4 [2026-10-29 04:20:44,976] INFO : Step 20400/50000 rank4 | Train Loss: 1.1926 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1642 | Classification: 0.1091 | Classification_binary: 0.1922 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1329 | Regression_eta: 0.0018 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1836 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0677 | Regression_eta: 0.0381 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0631 | LR: 6.78e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.2542s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0167 | max_forward: 1.2639 | max_loss: 0.1873 | max_backward: 2.4261 | max_optimizer_step: 0.3105 | max_tensorboard_logging: 0.7562 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6556.4 | uss_MB: 5916.7 | child_rss_MB: 5031.4 | allocated_MB: 447.7 | reserved_MB: 29344.0 | inactive_split_MB: 985.2 +rank4 [2026-10-29 04:25:27,116] INFO : Step 20500/50000 rank4 | Train Loss: 1.2110 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1591 | Classification: 0.1022 | Classification_binary: 0.2117 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1333 | Regression_eta: 0.0017 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2398 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0704 | Regression_eta: 0.0491 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0873 | LR: 6.76e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5487s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7705 | max_loss: 0.1525 | max_backward: 1.8060 | max_optimizer_step: 0.2622 | max_tensorboard_logging: 0.7595 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6553.9 | uss_MB: 5915.2 | child_rss_MB: 5013.6 | allocated_MB: 450.6 | reserved_MB: 29344.0 | inactive_split_MB: 1123.4 +rank4 [2026-10-29 04:30:07,866] INFO : Step 20600/50000 rank4 | Train Loss: 1.1161 | Train Loss Unweighted: 0.5564 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1436 | Classification: 0.0978 | Classification_binary: 0.1795 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1919 | Regression_cos_phi: 0.0949 | Regression_energy: 0.0798 | Regression_eta: 0.0485 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0767 | LR: 6.73e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4384s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8358 | max_loss: 0.5148 | max_backward: 1.8393 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6556.9 | uss_MB: 5916.8 | child_rss_MB: 5024.7 | allocated_MB: 445.4 | reserved_MB: 31380.0 | inactive_split_MB: 917.0 +rank4 [2026-10-29 04:34:51,728] INFO : Step 20700/50000 rank4 | Train Loss: 1.0740 | Train Loss Unweighted: 0.5459 | Train Loss Components: Auxiliary: 0.2086 | Cardinality: 0.1252 | Classification: 0.0962 | Classification_binary: 0.1560 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0938 | Regression_energy: 0.0652 | Regression_eta: 0.0391 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0536 | LR: 6.70e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0704s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7111 | max_loss: 0.2396 | max_backward: 1.6829 | max_optimizer_step: 0.2121 | max_tensorboard_logging: 0.4996 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6553.0 | uss_MB: 5912.9 | child_rss_MB: 5017.8 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1185.7 +rank4 [2026-10-29 04:39:29,327] INFO : Step 20800/50000 rank4 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5426 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1263 | Classification: 0.0965 | Classification_binary: 0.1999 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1207 | Regression_eta: 0.0017 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0752 | Classification_binary: 0.2050 | Regression_cos_phi: 0.0857 | Regression_energy: 0.0593 | Regression_eta: 0.0483 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0877 | LR: 6.67e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7629s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.9079 | max_loss: 0.1550 | max_backward: 2.1083 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.5897 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6553.9 | uss_MB: 5918.9 | child_rss_MB: 5012.5 | allocated_MB: 448.2 | reserved_MB: 31380.0 | inactive_split_MB: 1157.8 +rank4 [2026-10-29 04:44:09,835] INFO : Step 20900/50000 rank4 | Train Loss: 1.2188 | Train Loss Unweighted: 0.5970 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1525 | Classification: 0.1062 | Classification_binary: 0.2022 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0022 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.2046 | Regression_cos_phi: 0.0954 | Regression_energy: 0.0630 | Regression_eta: 0.0583 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0697 | LR: 6.65e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.7898s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0138 | max_forward: 0.8922 | max_loss: 0.1535 | max_backward: 2.0483 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6550.7 | uss_MB: 5916.4 | child_rss_MB: 5027.7 | allocated_MB: 447.8 | reserved_MB: 31380.0 | inactive_split_MB: 1126.3 +rank4 [2026-10-29 04:48:48,418] INFO : Step 21000/50000 rank4 | Train Loss: 1.0503 | Train Loss Unweighted: 0.5338 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1267 | Classification: 0.0900 | Classification_binary: 0.1573 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1405 | Regression_eta: 0.0016 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1267 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0794 | Regression_eta: 0.0331 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0474 | LR: 6.62e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8539s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7596 | max_loss: 0.2536 | max_backward: 1.8713 | max_optimizer_step: 0.3177 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6551.7 | uss_MB: 5915.9 | child_rss_MB: 5032.5 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1148.0 +rank4 [2026-10-29 04:48:49,285] INFO : Running validation on rank4 +rank4 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 04:49:44,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:49:44,825] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 04:50:11,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 04:50:12,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 04:50:13,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 04:50:14,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 04:50:24,430] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 04:50:25,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:50:25,768] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 04:50:41,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 04:50:42,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 04:50:43,695] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 04:50:44,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 04:50:54,685] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 04:50:55,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 04:50:55,843] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 04:51:11,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 04:51:11,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 04:51:12,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 04:51:13,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 04:51:23,562] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 04:56:25,805] INFO : Step 21100/50000 rank4 | Train Loss: 1.1646 | Train Loss Unweighted: 0.5785 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1401 | Classification: 0.1095 | Classification_binary: 0.1927 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1344 | Regression_eta: 0.0022 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2017 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0592 | Regression_eta: 0.0616 | Regression_pt: 0.0686 | Regression_sin_phi: 0.1055 | LR: 6.59e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.4043s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.9258 | max_loss: 0.1506 | max_backward: 2.4347 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.8145 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7847.0 | uss_MB: 5890.3 | child_rss_MB: 5018.3 | allocated_MB: 448.7 | reserved_MB: 23438.0 | inactive_split_MB: 911.7 +rank4 [2026-10-29 04:56:59,983] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank4 [2026-10-29 04:56:59,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-29 05:01:22,612] INFO : Step 21200/50000 rank4 | Train Loss: 1.1437 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1413 | Classification: 0.0985 | Classification_binary: 0.1768 | HitGrouping: 0.1669 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0020 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1748 | Regression_cos_phi: 0.1104 | Regression_energy: 0.0739 | Regression_eta: 0.0674 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0739 | LR: 6.56e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5647s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7993 | max_loss: 0.1450 | max_backward: 1.7427 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7842.9 | uss_MB: 7208.8 | child_rss_MB: 5012.9 | allocated_MB: 446.9 | reserved_MB: 23590.0 | inactive_split_MB: 1086.9 +rank4 [2026-10-29 05:06:02,215] INFO : Step 21300/50000 rank4 | Train Loss: 1.1586 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1648 | Classification: 0.1118 | Classification_binary: 0.1760 | HitGrouping: 0.1681 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1392 | Regression_eta: 0.0021 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0919 | Regression_energy: 0.0692 | Regression_eta: 0.0524 | Regression_pt: 0.0755 | Regression_sin_phi: 0.1179 | LR: 6.54e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.2684s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0558 | max_forward: 0.9498 | max_loss: 0.1699 | max_backward: 2.4288 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.7788 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1270 | Memory: rss_MB: 6605.4 | uss_MB: 5978.0 | child_rss_MB: 5012.8 | allocated_MB: 447.4 | reserved_MB: 25440.0 | inactive_split_MB: 1003.9 +rank4 [2026-10-29 05:10:41,726] INFO : Step 21400/50000 rank4 | Train Loss: 1.1843 | Train Loss Unweighted: 0.5823 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1516 | Classification: 0.1038 | Classification_binary: 0.2065 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1335 | Regression_eta: 0.0020 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0661 | Regression_eta: 0.0582 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0721 | LR: 6.51e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.0965s | Phase Times: max_pre_step_barrier: 0.0085 | max_batch_to_device: 0.0640 | max_forward: 0.9920 | max_loss: 0.5036 | max_backward: 2.0711 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.8200 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6606.3 | uss_MB: 5978.8 | child_rss_MB: 5018.5 | allocated_MB: 448.9 | reserved_MB: 25440.0 | inactive_split_MB: 994.4 +rank4 [2026-10-29 05:15:15,087] INFO : Step 21500/50000 rank4 | Train Loss: 1.1484 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1478 | Classification: 0.0953 | Classification_binary: 0.2062 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1299 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0630 | Regression_eta: 0.0563 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0881 | LR: 6.48e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.9368s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8371 | max_loss: 0.1537 | max_backward: 2.1150 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.8633 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6603.8 | uss_MB: 5979.6 | child_rss_MB: 5017.0 | allocated_MB: 447.2 | reserved_MB: 27056.0 | inactive_split_MB: 1468.9 +rank4 [2026-10-29 05:19:51,737] INFO : Step 21600/50000 rank4 | Train Loss: 1.0171 | Train Loss Unweighted: 0.5098 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1167 | Classification: 0.0973 | Classification_binary: 0.1559 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1265 | Regression_eta: 0.0014 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0494 | Regression_eta: 0.0371 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0648 | LR: 6.45e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5900s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0062 | max_forward: 0.7285 | max_loss: 0.1940 | max_backward: 1.8148 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.8819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6602.2 | uss_MB: 5978.2 | child_rss_MB: 5014.0 | allocated_MB: 445.7 | reserved_MB: 27056.0 | inactive_split_MB: 1196.5 +rank4 [2026-10-29 05:24:30,476] INFO : Step 21700/50000 rank4 | Train Loss: 1.1623 | Train Loss Unweighted: 0.5535 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1498 | Classification: 0.1025 | Classification_binary: 0.1819 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1815 | Regression_cos_phi: 0.0976 | Regression_energy: 0.0719 | Regression_eta: 0.0583 | Regression_pt: 0.0790 | Regression_sin_phi: 0.1083 | LR: 6.43e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4944s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7947 | max_loss: 0.1379 | max_backward: 1.9808 | max_optimizer_step: 0.2157 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6600.2 | uss_MB: 5978.2 | child_rss_MB: 5010.8 | allocated_MB: 448.2 | reserved_MB: 27056.0 | inactive_split_MB: 1045.2 +rank4 [2026-10-29 05:29:08,484] INFO : Step 21800/50000 rank4 | Train Loss: 1.0898 | Train Loss Unweighted: 0.5518 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1233 | Classification: 0.0977 | Classification_binary: 0.1967 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1269 | Regression_eta: 0.0015 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2031 | Regression_cos_phi: 0.0866 | Regression_energy: 0.0667 | Regression_eta: 0.0488 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0815 | LR: 6.40e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.7218s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8952 | max_loss: 0.1765 | max_backward: 2.1628 | max_optimizer_step: 0.2138 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6602.0 | uss_MB: 5872.4 | child_rss_MB: 5012.0 | allocated_MB: 448.4 | reserved_MB: 27056.0 | inactive_split_MB: 1001.4 +rank4 [2026-10-29 05:33:46,455] INFO : Step 21900/50000 rank4 | Train Loss: 1.0699 | Train Loss Unweighted: 0.5306 | Train Loss Components: Auxiliary: 0.2121 | Cardinality: 0.1436 | Classification: 0.0991 | Classification_binary: 0.1704 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1276 | Regression_eta: 0.0015 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1358 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0657 | Regression_eta: 0.0340 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0576 | LR: 6.37e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4509s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7813 | max_loss: 0.1495 | max_backward: 1.8703 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 6600.0 | uss_MB: 5976.9 | child_rss_MB: 5013.0 | allocated_MB: 447.4 | reserved_MB: 27056.0 | inactive_split_MB: 1002.4 +rank4 [2026-10-29 05:38:23,934] INFO : Step 22000/50000 rank4 | Train Loss: 1.0659 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2100 | Cardinality: 0.1294 | Classification: 0.0955 | Classification_binary: 0.1521 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1360 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0751 | Regression_eta: 0.0435 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0611 | LR: 6.34e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4503s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7855 | max_loss: 0.1383 | max_backward: 1.9648 | max_optimizer_step: 0.2132 | max_tensorboard_logging: 0.5884 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6605.4 | uss_MB: 5979.9 | child_rss_MB: 5015.2 | allocated_MB: 449.1 | reserved_MB: 27056.0 | inactive_split_MB: 1180.9 +rank4 [2026-10-29 05:38:24,835] INFO : Running validation on rank4 +rank4 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 05:39:16,672] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:39:17,215] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 05:39:42,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 05:39:43,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 05:39:44,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 05:39:45,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 05:39:55,898] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 05:39:56,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:39:57,074] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 05:40:13,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 05:40:14,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 05:40:15,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 05:40:16,660] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 05:40:26,670] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 05:40:27,479] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 05:40:27,972] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 05:40:43,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 05:40:44,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 05:40:45,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 05:40:45,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 05:40:56,010] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 05:45:56,687] INFO : Step 22100/50000 rank4 | Train Loss: 1.1235 | Train Loss Unweighted: 0.5530 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1528 | Classification: 0.0997 | Classification_binary: 0.1905 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1304 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.2000 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0677 | Regression_eta: 0.0447 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0778 | LR: 6.32e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5226s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8135 | max_loss: 0.1488 | max_backward: 1.7877 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5979 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7900.1 | uss_MB: 7271.0 | child_rss_MB: 5019.5 | allocated_MB: 451.8 | reserved_MB: 29136.0 | inactive_split_MB: 1136.2 +rank4 [2026-10-29 05:50:45,341] INFO : Step 22200/50000 rank4 | Train Loss: 1.1922 | Train Loss Unweighted: 0.5684 | Train Loss Components: Auxiliary: 0.2361 | Cardinality: 0.1699 | Classification: 0.0906 | Classification_binary: 0.2085 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.2167 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0722 | Regression_eta: 0.0528 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0780 | LR: 6.29e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5685s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8616 | max_loss: 0.1545 | max_backward: 1.6969 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 0.7120 | max_tensorboard_flush: 0.0150 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7899.1 | uss_MB: 7269.8 | child_rss_MB: 5025.9 | allocated_MB: 448.1 | reserved_MB: 29136.0 | inactive_split_MB: 1371.0 +rank4 [2026-10-29 05:55:21,939] INFO : Step 22300/50000 rank4 | Train Loss: 1.0221 | Train Loss Unweighted: 0.5006 | Train Loss Components: Auxiliary: 0.1973 | Cardinality: 0.1169 | Classification: 0.0876 | Classification_binary: 0.1576 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1252 | Regression_eta: 0.0021 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0666 | Regression_eta: 0.0671 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0824 | LR: 6.26e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5021s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7903 | max_loss: 0.1446 | max_backward: 1.7078 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 0.8083 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6690.1 | uss_MB: 6062.0 | child_rss_MB: 5023.7 | allocated_MB: 446.4 | reserved_MB: 29136.0 | inactive_split_MB: 1159.3 +rank4 [2026-10-29 06:00:08,367] INFO : Step 22400/50000 rank4 | Train Loss: 1.1170 | Train Loss Unweighted: 0.5609 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1391 | Classification: 0.1065 | Classification_binary: 0.1807 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1348 | Regression_eta: 0.0019 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0776 | Classification_binary: 0.1733 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0863 | Regression_eta: 0.0440 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0667 | LR: 6.23e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9532s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7658 | max_loss: 0.2479 | max_backward: 1.9808 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9928 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6691.0 | uss_MB: 6065.0 | child_rss_MB: 5027.0 | allocated_MB: 447.1 | reserved_MB: 31152.0 | inactive_split_MB: 925.7 +rank4 [2026-10-29 06:04:41,559] INFO : Step 22500/50000 rank4 | Train Loss: 1.1245 | Train Loss Unweighted: 0.5482 | Train Loss Components: Auxiliary: 0.2231 | Cardinality: 0.1463 | Classification: 0.0856 | Classification_binary: 0.2083 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1244 | Regression_eta: 0.0016 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.2232 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0739 | Regression_eta: 0.0468 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0809 | LR: 6.20e-04 | DataLoad Time: 0.0146s | Model Forward Time: 5.0102s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 1.1916 | max_loss: 0.1613 | max_backward: 2.1225 | max_optimizer_step: 0.2176 | max_tensorboard_logging: 0.7664 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6685.2 | uss_MB: 6063.6 | child_rss_MB: 5003.5 | allocated_MB: 447.4 | reserved_MB: 31152.0 | inactive_split_MB: 959.7 +rank4 [2026-10-29 06:09:13,551] INFO : Step 22600/50000 rank4 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5177 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1260 | Classification: 0.0928 | Classification_binary: 0.1789 | HitGrouping: 0.1619 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1208 | Regression_eta: 0.0019 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1740 | Regression_cos_phi: 0.1047 | Regression_energy: 0.0750 | Regression_eta: 0.0651 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0909 | LR: 6.18e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.8018s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0651 | max_forward: 1.1064 | max_loss: 0.1531 | max_backward: 1.9236 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.8515 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6676.3 | uss_MB: 6063.0 | child_rss_MB: 5024.2 | allocated_MB: 448.3 | reserved_MB: 31154.0 | inactive_split_MB: 961.5 +rank4 [2026-10-29 06:13:51,719] INFO : Step 22700/50000 rank4 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5090 | Train Loss Components: Auxiliary: 0.1959 | Cardinality: 0.1101 | Classification: 0.0877 | Classification_binary: 0.1479 | HitGrouping: 0.1634 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1339 | Regression_eta: 0.0015 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1234 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0697 | Regression_eta: 0.0332 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0519 | LR: 6.15e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.4583s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 1.0743 | max_loss: 0.1383 | max_backward: 1.7235 | max_optimizer_step: 0.2204 | max_tensorboard_logging: 0.5170 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 6672.9 | uss_MB: 6061.8 | child_rss_MB: 5030.8 | allocated_MB: 446.5 | reserved_MB: 31154.0 | inactive_split_MB: 972.7 +rank4 [2026-10-29 06:18:32,627] INFO : Step 22800/50000 rank4 | Train Loss: 1.0730 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2215 | Cardinality: 0.1332 | Classification: 0.1095 | Classification_binary: 0.1782 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1289 | Regression_eta: 0.0014 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1526 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0798 | Regression_eta: 0.0436 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0652 | LR: 6.12e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.7944s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.2500 | max_backward: 2.0107 | max_optimizer_step: 0.2761 | max_tensorboard_logging: 0.6168 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 6673.9 | uss_MB: 6062.6 | child_rss_MB: 5029.6 | allocated_MB: 447.2 | reserved_MB: 31154.0 | inactive_split_MB: 922.1 +rank4 [2026-10-29 06:23:13,203] INFO : Step 22900/50000 rank4 | Train Loss: 1.1830 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2389 | Cardinality: 0.1788 | Classification: 0.0948 | Classification_binary: 0.2023 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1393 | Regression_eta: 0.0018 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1751 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0701 | Regression_eta: 0.0536 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0694 | LR: 6.09e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.9953s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0073 | max_forward: 0.7992 | max_loss: 0.1478 | max_backward: 1.9051 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.9858 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 6664.2 | uss_MB: 6055.9 | child_rss_MB: 5028.8 | allocated_MB: 448.8 | reserved_MB: 31154.0 | inactive_split_MB: 1109.2 +rank4 [2026-10-29 06:27:51,863] INFO : Step 23000/50000 rank4 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5014 | Train Loss Components: Auxiliary: 0.1951 | Cardinality: 0.1184 | Classification: 0.0781 | Classification_binary: 0.1455 | HitGrouping: 0.1653 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0629 | Regression_eta: 0.0477 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0562 | LR: 6.06e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6089s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7528 | max_loss: 0.2292 | max_backward: 1.7818 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.8884 | max_tensorboard_flush: 0.0060 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6672.0 | uss_MB: 6063.2 | child_rss_MB: 5028.5 | allocated_MB: 443.9 | reserved_MB: 31154.0 | inactive_split_MB: 985.1 +rank4 [2026-10-29 06:27:52,750] INFO : Running validation on rank4 +rank4 [2026-10-29 06:27:52,750] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 06:27:52,751] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 06:28:45,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:28:45,725] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 06:29:12,096] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 06:29:13,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 06:29:14,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 06:29:15,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 06:29:25,142] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 06:29:26,163] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:29:26,632] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 06:29:42,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 06:29:43,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 06:29:44,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 06:29:45,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 06:29:55,426] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 06:29:56,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 06:29:56,643] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 06:30:12,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 06:30:13,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 06:30:13,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 06:30:14,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 06:30:24,765] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 06:35:29,455] INFO : Step 23100/50000 rank4 | Train Loss: 1.1240 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1320 | Classification: 0.1122 | Classification_binary: 0.1745 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1274 | Regression_eta: 0.0021 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.1729 | Regression_cos_phi: 0.1488 | Regression_energy: 0.0711 | Regression_eta: 0.0706 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0886 | LR: 6.04e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8781s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7914 | max_loss: 0.1430 | max_backward: 1.7255 | max_optimizer_step: 0.3295 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0695 | Memory: rss_MB: 7843.1 | uss_MB: 7233.5 | child_rss_MB: 5030.9 | allocated_MB: 449.4 | reserved_MB: 25680.0 | inactive_split_MB: 1046.0 +rank4 [2026-10-29 06:40:11,955] INFO : Step 23200/50000 rank4 | Train Loss: 1.1736 | Train Loss Unweighted: 0.5745 | Train Loss Components: Auxiliary: 0.2341 | Cardinality: 0.1586 | Classification: 0.1057 | Classification_binary: 0.2033 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1304 | Regression_eta: 0.0018 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.2101 | Regression_cos_phi: 0.0802 | Regression_energy: 0.0772 | Regression_eta: 0.0427 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0651 | LR: 6.01e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.1118s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9381 | max_loss: 0.1692 | max_backward: 2.2545 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6161 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 7846.1 | uss_MB: 5861.1 | child_rss_MB: 5025.2 | allocated_MB: 447.8 | reserved_MB: 25680.0 | inactive_split_MB: 1169.7 +rank4 [2026-10-29 06:44:52,827] INFO : Step 23300/50000 rank4 | Train Loss: 1.0647 | Train Loss Unweighted: 0.5262 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1268 | Classification: 0.0958 | Classification_binary: 0.1665 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1301 | Regression_eta: 0.0017 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1358 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0643 | Regression_eta: 0.0383 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0430 | LR: 5.98e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6087s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8613 | max_loss: 0.1501 | max_backward: 2.0035 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 0.6274 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6729.0 | uss_MB: 6118.5 | child_rss_MB: 5031.7 | allocated_MB: 447.3 | reserved_MB: 29702.0 | inactive_split_MB: 1252.4 +rank4 [2026-10-29 06:49:32,126] INFO : Step 23400/50000 rank4 | Train Loss: 0.9916 | Train Loss Unweighted: 0.5023 | Train Loss Components: Auxiliary: 0.1957 | Cardinality: 0.1164 | Classification: 0.1006 | Classification_binary: 0.1374 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1304 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0731 | Regression_eta: 0.0337 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0538 | LR: 5.95e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3953s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7229 | max_loss: 0.2412 | max_backward: 1.6798 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.4992 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1172 | Memory: rss_MB: 6727.8 | uss_MB: 6117.0 | child_rss_MB: 5031.9 | allocated_MB: 445.7 | reserved_MB: 29702.0 | inactive_split_MB: 1158.3 +rank4 [2026-10-29 06:54:42,355] INFO : Step 23500/50000 rank4 | Train Loss: 1.0349 | Train Loss Unweighted: 0.5234 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1083 | Classification: 0.0851 | Classification_binary: 0.1870 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1235 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.2011 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0640 | Regression_eta: 0.0420 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0714 | LR: 5.92e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7745s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7924 | max_loss: 0.2020 | max_backward: 1.9787 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.7987 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6728.4 | uss_MB: 6111.2 | child_rss_MB: 5026.6 | allocated_MB: 446.9 | reserved_MB: 31774.0 | inactive_split_MB: 1042.0 +rank4 [2026-10-29 06:59:38,280] INFO : Step 23600/50000 rank4 | Train Loss: 1.1326 | Train Loss Unweighted: 0.5418 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1458 | Classification: 0.0933 | Classification_binary: 0.1900 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1267 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1977 | Regression_cos_phi: 0.0886 | Regression_energy: 0.0619 | Regression_eta: 0.0428 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0892 | LR: 5.90e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9832s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0063 | max_forward: 0.7946 | max_loss: 0.1855 | max_backward: 1.9114 | max_optimizer_step: 0.2664 | max_tensorboard_logging: 0.9547 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6730.1 | uss_MB: 6120.3 | child_rss_MB: 5028.3 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1333.8 +rank4 [2026-10-29 07:04:20,064] INFO : Step 23700/50000 rank4 | Train Loss: 1.0167 | Train Loss Unweighted: 0.4968 | Train Loss Components: Auxiliary: 0.1932 | Cardinality: 0.1158 | Classification: 0.0853 | Classification_binary: 0.1475 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1285 | Regression_eta: 0.0017 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0742 | Regression_energy: 0.0675 | Regression_eta: 0.0515 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0572 | LR: 5.87e-04 | DataLoad Time: 0.0222s | Model Forward Time: 4.7153s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0221 | max_forward: 0.7952 | max_loss: 0.1461 | max_backward: 1.8308 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7718 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6725.6 | uss_MB: 6118.0 | child_rss_MB: 5028.2 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1160.4 +rank4 [2026-10-29 07:08:57,915] INFO : Step 23800/50000 rank4 | Train Loss: 1.0651 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1325 | Classification: 0.0886 | Classification_binary: 0.1616 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1318 | Regression_eta: 0.0015 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0885 | Regression_energy: 0.0778 | Regression_eta: 0.0533 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0701 | LR: 5.84e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.6734s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8562 | max_loss: 0.1426 | max_backward: 1.8700 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6730.2 | uss_MB: 6120.2 | child_rss_MB: 5026.7 | allocated_MB: 449.4 | reserved_MB: 31776.0 | inactive_split_MB: 1046.7 +rank4 [2026-10-29 07:13:41,573] INFO : Step 23900/50000 rank4 | Train Loss: 1.1515 | Train Loss Unweighted: 0.5718 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1646 | Classification: 0.0962 | Classification_binary: 0.2086 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2015 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0672 | Regression_eta: 0.0435 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0642 | LR: 5.81e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.8392s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.8887 | max_loss: 0.2019 | max_backward: 2.1185 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6224 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6714.4 | uss_MB: 6108.3 | child_rss_MB: 5032.9 | allocated_MB: 447.1 | reserved_MB: 31776.0 | inactive_split_MB: 1168.8 +rank4 [2026-10-29 07:18:20,711] INFO : Step 24000/50000 rank4 | Train Loss: 1.0174 | Train Loss Unweighted: 0.5187 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1133 | Classification: 0.0873 | Classification_binary: 0.1674 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0014 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1656 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0950 | Regression_eta: 0.0368 | Regression_pt: 0.1043 | Regression_sin_phi: 0.0580 | LR: 5.78e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.6302s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 1.3237 | max_loss: 0.2403 | max_backward: 1.8319 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5068 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6721.2 | uss_MB: 6114.9 | child_rss_MB: 5029.4 | allocated_MB: 446.4 | reserved_MB: 31776.0 | inactive_split_MB: 957.1 +rank4 [2026-10-29 07:18:21,624] INFO : Running validation on rank4 +rank4 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 07:19:13,460] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:19:13,969] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 07:19:40,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 07:19:42,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 07:19:43,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 07:19:44,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 07:19:54,311] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 07:19:55,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:19:55,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 07:20:12,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 07:20:13,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 07:20:13,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 07:20:14,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 07:20:24,939] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 07:20:25,040] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 07:20:25,608] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 07:20:40,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 07:20:41,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 07:20:42,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 07:20:43,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 07:20:53,336] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 07:26:06,616] INFO : Step 24100/50000 rank4 | Train Loss: 1.0669 | Train Loss Unweighted: 0.5331 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1324 | Classification: 0.0928 | Classification_binary: 0.1578 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1390 | Regression_eta: 0.0017 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1507 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0789 | Regression_eta: 0.0397 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0626 | LR: 5.75e-04 | DataLoad Time: 0.0172s | Model Forward Time: 7.5197s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.7836 | max_loss: 0.1442 | max_backward: 1.8747 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 3.6030 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0626 | Memory: rss_MB: 7857.5 | uss_MB: 7243.1 | child_rss_MB: 5051.0 | allocated_MB: 448.1 | reserved_MB: 24490.0 | inactive_split_MB: 1198.5 +rank4 [2026-10-29 07:30:59,926] INFO : Step 24200/50000 rank4 | Train Loss: 1.1691 | Train Loss Unweighted: 0.5645 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1546 | Classification: 0.1031 | Classification_binary: 0.1915 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1325 | Regression_eta: 0.0019 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1932 | Regression_cos_phi: 0.1004 | Regression_energy: 0.0715 | Regression_eta: 0.0414 | Regression_pt: 0.0759 | Regression_sin_phi: 0.0609 | LR: 5.73e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.2346s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.9207 | max_loss: 0.1533 | max_backward: 2.2288 | max_optimizer_step: 0.2436 | max_tensorboard_logging: 0.8522 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7858.2 | uss_MB: 7247.2 | child_rss_MB: 5039.0 | allocated_MB: 450.2 | reserved_MB: 26490.0 | inactive_split_MB: 976.9 +rank4 [2026-10-29 07:35:42,203] INFO : Step 24300/50000 rank4 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5336 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1380 | Classification: 0.0927 | Classification_binary: 0.1863 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1247 | Regression_eta: 0.0020 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0624 | Regression_eta: 0.0495 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0622 | LR: 5.70e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.6715s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8648 | max_loss: 0.1481 | max_backward: 1.7152 | max_optimizer_step: 0.2781 | max_tensorboard_logging: 0.7831 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6810.0 | uss_MB: 6199.1 | child_rss_MB: 5043.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 983.3 +rank4 [2026-10-29 07:40:19,145] INFO : Step 24400/50000 rank4 | Train Loss: 0.9707 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0879 | Classification_binary: 0.1454 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0014 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0817 | Classification_binary: 0.1653 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0695 | Regression_eta: 0.0521 | Regression_pt: 0.0745 | Regression_sin_phi: 0.1105 | LR: 5.67e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.8494s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7628 | max_loss: 0.1940 | max_backward: 2.4253 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.9481 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6811.6 | uss_MB: 5803.5 | child_rss_MB: 5042.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 1129.3 +rank4 [2026-10-29 07:45:00,050] INFO : Step 24500/50000 rank4 | Train Loss: 1.0937 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1319 | Classification: 0.0941 | Classification_binary: 0.1676 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1334 | Regression_eta: 0.0016 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0780 | Regression_energy: 0.0901 | Regression_eta: 0.0482 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0590 | LR: 5.64e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4283s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7565 | max_loss: 0.5105 | max_backward: 1.8057 | max_optimizer_step: 0.2534 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6807.0 | uss_MB: 6198.6 | child_rss_MB: 5045.1 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 945.2 +rank4 [2026-10-29 07:49:44,236] INFO : Step 24600/50000 rank4 | Train Loss: 1.0924 | Train Loss Unweighted: 0.5511 | Train Loss Components: Auxiliary: 0.2237 | Cardinality: 0.1233 | Classification: 0.0963 | Classification_binary: 0.1991 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1255 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1975 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0767 | Regression_eta: 0.0401 | Regression_pt: 0.0839 | Regression_sin_phi: 0.0554 | LR: 5.61e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6476s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.9060 | max_loss: 0.1523 | max_backward: 2.0894 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.5102 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6811.1 | uss_MB: 6200.6 | child_rss_MB: 5046.3 | allocated_MB: 449.5 | reserved_MB: 26492.0 | inactive_split_MB: 943.2 +rank4 [2026-10-29 07:54:29,279] INFO : Step 24700/50000 rank4 | Train Loss: 0.9972 | Train Loss Unweighted: 0.4984 | Train Loss Components: Auxiliary: 0.1950 | Cardinality: 0.1160 | Classification: 0.0782 | Classification_binary: 0.1692 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0618 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0760 | Regression_energy: 0.0640 | Regression_eta: 0.0422 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0630 | LR: 5.58e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5126s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0060 | max_forward: 0.7847 | max_loss: 0.1519 | max_backward: 1.7159 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.8325 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6813.0 | uss_MB: 6199.7 | child_rss_MB: 5050.6 | allocated_MB: 447.9 | reserved_MB: 27708.0 | inactive_split_MB: 1169.6 +rank4 [2026-10-29 07:59:09,067] INFO : Step 24800/50000 rank4 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4722 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1202 | Classification: 0.0808 | Classification_binary: 0.1484 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0016 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0725 | Regression_eta: 0.0464 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0657 | LR: 5.56e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7168s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0127 | max_forward: 0.7903 | max_loss: 0.1427 | max_backward: 1.7798 | max_optimizer_step: 0.2260 | max_tensorboard_logging: 0.8943 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6813.0 | uss_MB: 6199.9 | child_rss_MB: 5051.5 | allocated_MB: 447.6 | reserved_MB: 28508.0 | inactive_split_MB: 989.1 +rank4 [2026-10-29 08:03:50,794] INFO : Step 24900/50000 rank4 | Train Loss: 1.0773 | Train Loss Unweighted: 0.5300 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1506 | Classification: 0.0964 | Classification_binary: 0.1776 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1257 | Regression_eta: 0.0016 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1736 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0794 | Regression_eta: 0.0475 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0587 | LR: 5.53e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.3164s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8100 | max_loss: 0.1483 | max_backward: 1.9423 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 1.0834 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6802.2 | uss_MB: 5593.7 | child_rss_MB: 5050.0 | allocated_MB: 449.1 | reserved_MB: 28508.0 | inactive_split_MB: 993.6 +rank4 [2026-10-29 08:08:37,679] INFO : Step 25000/50000 rank4 | Train Loss: 1.1315 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1671 | Classification: 0.0915 | Classification_binary: 0.1899 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1201 | Regression_eta: 0.0020 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0617 | Regression_eta: 0.0580 | Regression_pt: 0.0677 | Regression_sin_phi: 0.0698 | LR: 5.50e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.7073s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8152 | max_loss: 0.2476 | max_backward: 1.7290 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.8750 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6807.0 | uss_MB: 6195.6 | child_rss_MB: 5052.2 | allocated_MB: 448.5 | reserved_MB: 28508.0 | inactive_split_MB: 953.9 +rank4 [2026-10-29 08:08:38,676] INFO : Running validation on rank4 +rank4 [2026-10-29 08:08:38,676] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 08:08:38,677] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 08:09:30,804] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:09:31,356] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 08:09:57,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 08:09:58,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 08:09:59,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 08:10:00,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 08:10:10,944] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 08:10:11,661] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:10:12,184] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 08:10:28,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 08:10:29,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 08:10:30,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 08:10:31,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 08:10:41,105] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 08:10:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 08:10:42,693] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 08:10:58,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 08:10:59,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 08:11:00,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 08:11:01,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 08:11:11,178] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 08:16:19,774] INFO : Step 25100/50000 rank4 | Train Loss: 1.0264 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1288 | Classification: 0.0850 | Classification_binary: 0.1497 | HitGrouping: 0.1664 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1378 | Regression_eta: 0.0016 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0295 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0401 | Regression_energy: 0.0843 | Regression_eta: 0.0454 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0653 | LR: 5.47e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.5435s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0059 | max_forward: 0.7878 | max_loss: 0.1437 | max_backward: 2.2383 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9692 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7903.8 | uss_MB: 7286.7 | child_rss_MB: 5042.9 | allocated_MB: 446.8 | reserved_MB: 24062.0 | inactive_split_MB: 1159.3 +rank4 [2026-10-29 08:21:20,699] INFO : Step 25200/50000 rank4 | Train Loss: 1.0775 | Train Loss Unweighted: 0.5434 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1397 | Classification: 0.0996 | Classification_binary: 0.1706 | HitGrouping: 0.1584 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1340 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1585 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0782 | Regression_eta: 0.0406 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0640 | LR: 5.44e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2320s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8831 | max_loss: 0.1456 | max_backward: 2.1320 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7903.3 | uss_MB: 7287.2 | child_rss_MB: 5042.2 | allocated_MB: 448.0 | reserved_MB: 26042.0 | inactive_split_MB: 1093.3 +rank4 [2026-10-29 08:26:17,219] INFO : Step 25300/50000 rank4 | Train Loss: 1.0761 | Train Loss Unweighted: 0.5109 | Train Loss Components: Auxiliary: 0.2132 | Cardinality: 0.1361 | Classification: 0.0961 | Classification_binary: 0.1821 | HitGrouping: 0.1562 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1138 | Regression_eta: 0.0018 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1591 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0562 | Regression_eta: 0.0461 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0798 | LR: 5.42e-04 | DataLoad Time: 0.0196s | Model Forward Time: 6.1306s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.9122 | max_loss: 0.1821 | max_backward: 2.1499 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.9506 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6897.2 | uss_MB: 5222.8 | child_rss_MB: 5036.8 | allocated_MB: 446.7 | reserved_MB: 28062.0 | inactive_split_MB: 1098.3 +rank4 [2026-10-29 08:31:10,694] INFO : Step 25400/50000 rank4 | Train Loss: 1.0222 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1238 | Classification: 0.0928 | Classification_binary: 0.1653 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1221 | Regression_eta: 0.0015 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0776 | Regression_eta: 0.0423 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0569 | LR: 5.39e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.7083s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1954 | max_backward: 1.8415 | max_optimizer_step: 0.3277 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6896.2 | uss_MB: 6278.8 | child_rss_MB: 5052.2 | allocated_MB: 449.3 | reserved_MB: 28062.0 | inactive_split_MB: 945.6 +rank4 [2026-10-29 08:36:13,579] INFO : Step 25500/50000 rank4 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4596 | Train Loss Components: Auxiliary: 0.1817 | Cardinality: 0.1001 | Classification: 0.0953 | Classification_binary: 0.1334 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0016 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1203 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0635 | Regression_eta: 0.0539 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0948 | LR: 5.36e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7860s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7884 | max_loss: 0.1475 | max_backward: 2.0272 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.7862 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0483 | Memory: rss_MB: 6898.2 | uss_MB: 6283.2 | child_rss_MB: 5050.7 | allocated_MB: 450.7 | reserved_MB: 28062.0 | inactive_split_MB: 1166.9 +rank4 [2026-10-29 08:41:08,951] INFO : Step 25600/50000 rank4 | Train Loss: 1.0724 | Train Loss Unweighted: 0.5330 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1190 | Classification: 0.0985 | Classification_binary: 0.1843 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1231 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0616 | Regression_eta: 0.0361 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0602 | LR: 5.33e-04 | DataLoad Time: 0.0184s | Model Forward Time: 6.1578s | Phase Times: max_pre_step_barrier: 0.0194 | max_batch_to_device: 0.0083 | max_forward: 1.0622 | max_loss: 0.3311 | max_backward: 2.5999 | max_optimizer_step: 0.3019 | max_tensorboard_logging: 0.8123 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6895.2 | uss_MB: 5585.2 | child_rss_MB: 5023.7 | allocated_MB: 450.4 | reserved_MB: 28064.0 | inactive_split_MB: 1163.5 +rank4 [2026-10-29 08:46:09,676] INFO : Step 25700/50000 rank4 | Train Loss: 1.0595 | Train Loss Unweighted: 0.5120 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.1394 | Classification: 0.0920 | Classification_binary: 0.1853 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1821 | Regression_cos_phi: 0.0403 | Regression_energy: 0.0573 | Regression_eta: 0.0344 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0568 | LR: 5.30e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.8065s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.8320 | max_loss: 0.2520 | max_backward: 2.6962 | max_optimizer_step: 0.2699 | max_tensorboard_logging: 0.7057 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6885.0 | uss_MB: 6272.6 | child_rss_MB: 5051.6 | allocated_MB: 448.6 | reserved_MB: 29942.0 | inactive_split_MB: 1165.2 +rank4 [2026-10-29 08:51:40,160] INFO : Step 25800/50000 rank4 | Train Loss: 0.9592 | Train Loss Unweighted: 0.4733 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1038 | Classification: 0.0787 | Classification_binary: 0.1531 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1182 | Regression_eta: 0.0016 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1362 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0612 | Regression_eta: 0.0365 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0427 | LR: 5.27e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4835s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7549 | max_loss: 0.1480 | max_backward: 1.5296 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 0.8796 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1361 | Memory: rss_MB: 6889.1 | uss_MB: 6277.5 | child_rss_MB: 5023.6 | allocated_MB: 445.9 | reserved_MB: 30100.0 | inactive_split_MB: 961.8 +rank4 [2026-10-29 08:56:50,110] INFO : Step 25900/50000 rank4 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4613 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1142 | Classification: 0.0768 | Classification_binary: 0.1501 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1145 | Regression_eta: 0.0015 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1444 | Regression_cos_phi: 0.1068 | Regression_energy: 0.0602 | Regression_eta: 0.0349 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0647 | LR: 5.25e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8322s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0124 | max_forward: 0.7901 | max_loss: 0.1430 | max_backward: 1.9362 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8933 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 6880.5 | uss_MB: 6281.6 | child_rss_MB: 5008.5 | allocated_MB: 447.7 | reserved_MB: 30100.0 | inactive_split_MB: 1167.6 +rank4 [2026-10-29 09:01:47,509] INFO : Step 26000/50000 rank4 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5169 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1347 | Classification: 0.0896 | Classification_binary: 0.1938 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0015 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0560 | Regression_eta: 0.0327 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0703 | LR: 5.22e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.4849s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8125 | max_loss: 0.2612 | max_backward: 1.8771 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.8972 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6845.2 | uss_MB: 5700.0 | child_rss_MB: 4882.0 | allocated_MB: 450.3 | reserved_MB: 30108.0 | inactive_split_MB: 1161.3 +rank4 [2026-10-29 09:01:48,415] INFO : Running validation on rank4 +rank4 [2026-10-29 09:01:48,415] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 09:01:48,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 09:02:42,436] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:02:42,920] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 09:03:09,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 09:03:09,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 09:03:10,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 09:03:12,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 09:03:22,022] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 09:03:22,720] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:03:23,246] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 09:03:39,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 09:03:40,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 09:03:41,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 09:03:42,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 09:03:52,146] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 09:03:52,917] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:03:53,429] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 09:04:09,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 09:04:10,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 09:04:10,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 09:04:11,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 09:04:21,754] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 09:09:37,593] INFO : Step 26100/50000 rank4 | Train Loss: 0.9519 | Train Loss Unweighted: 0.4738 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1091 | Classification: 0.0871 | Classification_binary: 0.1547 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1147 | Regression_eta: 0.0013 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1754 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0646 | Regression_eta: 0.0355 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0441 | LR: 5.19e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.2076s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0321 | max_forward: 0.8143 | max_loss: 0.1540 | max_backward: 1.7669 | max_optimizer_step: 0.2382 | max_tensorboard_logging: 1.1185 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 8060.0 | uss_MB: 7433.5 | child_rss_MB: 4850.5 | allocated_MB: 445.1 | reserved_MB: 25850.0 | inactive_split_MB: 1376.9 +rank4 [2026-10-29 09:14:36,072] INFO : Step 26200/50000 rank4 | Train Loss: 0.9758 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.0970 | Classification: 0.0858 | Classification_binary: 0.1501 | HitGrouping: 0.1665 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0628 | Regression_eta: 0.0385 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0687 | LR: 5.16e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6736s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7728 | max_loss: 0.1384 | max_backward: 1.8195 | max_optimizer_step: 0.3036 | max_tensorboard_logging: 0.7010 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0602 | Memory: rss_MB: 8060.1 | uss_MB: 7488.7 | child_rss_MB: 4848.5 | allocated_MB: 447.4 | reserved_MB: 27830.0 | inactive_split_MB: 1117.7 +rank4 [2026-10-29 09:19:51,634] INFO : Step 26300/50000 rank4 | Train Loss: 1.0568 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2146 | Cardinality: 0.1416 | Classification: 0.0935 | Classification_binary: 0.1772 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1831 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0690 | Regression_eta: 0.0643 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0848 | LR: 5.13e-04 | DataLoad Time: 0.0194s | Model Forward Time: 6.1510s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.8205 | max_loss: 0.1808 | max_backward: 2.8872 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 1.1024 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6927.6 | uss_MB: 6354.4 | child_rss_MB: 4842.0 | allocated_MB: 450.8 | reserved_MB: 29830.0 | inactive_split_MB: 1114.5 +rank4 [2026-10-29 09:24:55,631] INFO : Step 26400/50000 rank4 | Train Loss: 1.0536 | Train Loss Unweighted: 0.5239 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1323 | Classification: 0.0937 | Classification_binary: 0.1845 | HitGrouping: 0.1559 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1209 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0619 | Regression_eta: 0.0363 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0651 | LR: 5.10e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.5445s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8639 | max_loss: 0.1569 | max_backward: 2.4710 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 1.0049 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6924.7 | uss_MB: 6353.2 | child_rss_MB: 4853.6 | allocated_MB: 447.6 | reserved_MB: 29832.0 | inactive_split_MB: 1186.4 +rank4 [2026-10-29 09:29:59,513] INFO : Step 26500/50000 rank4 | Train Loss: 0.9812 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1299 | Classification: 0.0807 | Classification_binary: 0.1400 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1217 | Regression_eta: 0.0017 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1284 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0583 | Regression_eta: 0.0501 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0648 | LR: 5.08e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.8055s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7436 | max_loss: 0.1579 | max_backward: 1.8417 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.9265 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6921.4 | uss_MB: 6354.1 | child_rss_MB: 4851.7 | allocated_MB: 444.4 | reserved_MB: 29832.0 | inactive_split_MB: 1229.5 +rank4 [2026-10-29 09:34:58,506] INFO : Step 26600/50000 rank4 | Train Loss: 1.0108 | Train Loss Unweighted: 0.5066 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.1182 | Classification: 0.0957 | Classification_binary: 0.1588 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1242 | Regression_eta: 0.0014 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1610 | Regression_cos_phi: 0.1150 | Regression_energy: 0.0642 | Regression_eta: 0.0248 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0427 | LR: 5.05e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6962s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7561 | max_loss: 0.1430 | max_backward: 1.8427 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6921.4 | uss_MB: 6356.0 | child_rss_MB: 4827.0 | allocated_MB: 447.9 | reserved_MB: 29832.0 | inactive_split_MB: 1037.2 +rank4 [2026-10-29 09:40:07,140] INFO : Step 26700/50000 rank4 | Train Loss: 1.0878 | Train Loss Unweighted: 0.5266 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1269 | Classification: 0.0956 | Classification_binary: 0.1961 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1144 | Regression_eta: 0.0018 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1712 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0651 | Regression_eta: 0.0467 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0768 | LR: 5.02e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1899s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.9788 | max_loss: 0.1540 | max_backward: 1.9407 | max_optimizer_step: 0.2295 | max_tensorboard_logging: 1.0076 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6911.8 | uss_MB: 6344.9 | child_rss_MB: 4859.9 | allocated_MB: 450.1 | reserved_MB: 29832.0 | inactive_split_MB: 1190.4 +rank4 [2026-10-29 09:48:41,511] INFO : Step 26800/50000 rank4 | Train Loss: 1.0077 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1102 | Classification: 0.0787 | Classification_binary: 0.1553 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1359 | Regression_eta: 0.0015 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1355 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0758 | Regression_eta: 0.0535 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0504 | LR: 4.99e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.8128s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.7774 | max_loss: 0.2523 | max_backward: 1.8741 | max_optimizer_step: 0.2873 | max_tensorboard_logging: 0.8206 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6909.2 | uss_MB: 6351.0 | child_rss_MB: 4816.3 | allocated_MB: 444.7 | reserved_MB: 29832.0 | inactive_split_MB: 1121.3 +rank4 [2026-10-29 09:53:40,041] INFO : Step 26900/50000 rank4 | Train Loss: 0.9524 | Train Loss Unweighted: 0.4793 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1099 | Classification: 0.0833 | Classification_binary: 0.1305 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1313 | Regression_eta: 0.0016 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1081 | Regression_cos_phi: 0.0665 | Regression_energy: 0.0516 | Regression_eta: 0.0326 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0335 | LR: 4.96e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.4174s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7292 | max_loss: 0.1373 | max_backward: 1.7442 | max_optimizer_step: 0.3152 | max_tensorboard_logging: 0.6764 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 6905.2 | uss_MB: 5520.6 | child_rss_MB: 4788.1 | allocated_MB: 446.9 | reserved_MB: 29832.0 | inactive_split_MB: 1016.7 +rank4 [2026-10-29 09:58:37,162] INFO : Step 27000/50000 rank4 | Train Loss: 1.0285 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1243 | Classification: 0.0862 | Classification_binary: 0.1888 | HitGrouping: 0.1539 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0016 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1871 | Regression_cos_phi: 0.0868 | Regression_energy: 0.0511 | Regression_eta: 0.0399 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0707 | LR: 4.94e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.1607s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.9134 | max_loss: 0.1497 | max_backward: 2.1803 | max_optimizer_step: 0.2254 | max_tensorboard_logging: 0.9490 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6904.1 | uss_MB: 6354.5 | child_rss_MB: 4772.3 | allocated_MB: 449.7 | reserved_MB: 29832.0 | inactive_split_MB: 1024.3 +rank4 [2026-10-29 09:58:38,108] INFO : Running validation on rank4 +rank4 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 09:59:32,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 09:59:33,284] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 09:59:59,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 10:00:00,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 10:00:02,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 10:00:03,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 10:00:13,131] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 10:00:14,009] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:00:14,537] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 10:00:30,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 10:00:33,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 10:00:34,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 10:00:35,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 10:00:45,365] INFO : Time taken to make predictions on device 4 is: 0.28 min +rank4 [2026-10-29 10:00:46,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:00:46,865] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 10:01:02,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 10:01:03,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 10:01:03,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 10:01:04,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 10:01:14,794] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 10:06:37,478] INFO : Step 27100/50000 rank4 | Train Loss: 1.0614 | Train Loss Unweighted: 0.4989 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1449 | Classification: 0.0752 | Classification_binary: 0.1877 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0406 | Classification_binary: 0.1709 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0739 | Regression_eta: 0.0514 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0594 | LR: 4.91e-04 | DataLoad Time: 0.0195s | Model Forward Time: 7.1187s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0966 | max_forward: 2.1724 | max_loss: 0.6705 | max_backward: 3.1723 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 1.5888 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 8047.0 | uss_MB: 7496.7 | child_rss_MB: 4778.7 | allocated_MB: 447.0 | reserved_MB: 27292.0 | inactive_split_MB: 983.4 +rank4 [2026-10-29 10:11:19,567] INFO : Step 27200/50000 rank4 | Train Loss: 0.9045 | Train Loss Unweighted: 0.4389 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1028 | Classification: 0.0770 | Classification_binary: 0.1287 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0343 | Regression_energy: 0.0622 | Regression_eta: 0.0368 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0612 | LR: 4.88e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7913s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7725 | max_loss: 0.5032 | max_backward: 1.6788 | max_optimizer_step: 0.2815 | max_tensorboard_logging: 0.9686 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8047.0 | uss_MB: 7493.7 | child_rss_MB: 4786.7 | allocated_MB: 445.7 | reserved_MB: 29236.0 | inactive_split_MB: 1232.8 +rank4 [2026-10-29 10:15:58,638] INFO : Step 27300/50000 rank4 | Train Loss: 1.0726 | Train Loss Unweighted: 0.5095 | Train Loss Components: Auxiliary: 0.2138 | Cardinality: 0.1337 | Classification: 0.1081 | Classification_binary: 0.1640 | HitGrouping: 0.1596 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1159 | Regression_eta: 0.0019 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1636 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0564 | Regression_eta: 0.0571 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0635 | LR: 4.85e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1435s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8896 | max_loss: 0.1460 | max_backward: 2.0153 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6974.0 | uss_MB: 6424.6 | child_rss_MB: 4784.6 | allocated_MB: 450.9 | reserved_MB: 29330.0 | inactive_split_MB: 1103.1 +rank4 [2026-10-29 10:20:31,815] INFO : Step 27400/50000 rank4 | Train Loss: 1.0758 | Train Loss Unweighted: 0.5282 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1344 | Classification: 0.0930 | Classification_binary: 0.1899 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1207 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0568 | Regression_eta: 0.0453 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0648 | LR: 4.82e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.0211s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8446 | max_loss: 0.1488 | max_backward: 2.0519 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.8725 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6977.0 | uss_MB: 5627.2 | child_rss_MB: 4777.8 | allocated_MB: 448.4 | reserved_MB: 31310.0 | inactive_split_MB: 895.7 +rank4 [2026-10-29 10:25:07,332] INFO : Step 27500/50000 rank4 | Train Loss: 0.9697 | Train Loss Unweighted: 0.4574 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1111 | Classification: 0.0831 | Classification_binary: 0.1473 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1114 | Regression_eta: 0.0018 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1415 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0558 | Regression_eta: 0.0684 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0519 | LR: 4.80e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4613s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.7904 | max_loss: 0.1563 | max_backward: 1.6291 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.7457 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6966.1 | uss_MB: 4710.9 | child_rss_MB: 4769.8 | allocated_MB: 447.3 | reserved_MB: 31310.0 | inactive_split_MB: 986.8 +rank4 [2026-10-29 10:30:03,960] INFO : Step 27600/50000 rank4 | Train Loss: 0.9616 | Train Loss Unweighted: 0.4618 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1092 | Classification: 0.0865 | Classification_binary: 0.1284 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0019 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1268 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0663 | Regression_eta: 0.1329 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0563 | LR: 4.77e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5709s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0380 | max_forward: 0.7674 | max_loss: 0.1463 | max_backward: 1.9164 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.5145 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6974.1 | uss_MB: 6422.9 | child_rss_MB: 4778.4 | allocated_MB: 446.7 | reserved_MB: 31310.0 | inactive_split_MB: 1182.0 +rank4 [2026-10-29 10:34:46,993] INFO : Step 27700/50000 rank4 | Train Loss: 1.0693 | Train Loss Unweighted: 0.5190 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1414 | Classification: 0.0939 | Classification_binary: 0.1783 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0015 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.2158 | Regression_cos_phi: 0.1180 | Regression_energy: 0.0699 | Regression_eta: 0.0429 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0867 | LR: 4.74e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.6751s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.8151 | max_loss: 0.1750 | max_backward: 2.0398 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.5213 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0659 | Memory: rss_MB: 6974.1 | uss_MB: 6424.8 | child_rss_MB: 4756.0 | allocated_MB: 448.4 | reserved_MB: 33312.0 | inactive_split_MB: 986.3 +rank4 [2026-10-29 10:39:29,303] INFO : Step 27800/50000 rank4 | Train Loss: 1.0875 | Train Loss Unweighted: 0.5294 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1226 | Classification: 0.0924 | Classification_binary: 0.1842 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1237 | Regression_eta: 0.0019 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2033 | Regression_cos_phi: 0.1107 | Regression_energy: 0.0763 | Regression_eta: 0.0597 | Regression_pt: 0.0837 | Regression_sin_phi: 0.1162 | LR: 4.71e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8103s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0152 | max_forward: 0.8463 | max_loss: 0.1744 | max_backward: 1.7379 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.9397 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 6971.2 | uss_MB: 6421.2 | child_rss_MB: 4773.7 | allocated_MB: 452.2 | reserved_MB: 33312.0 | inactive_split_MB: 1320.1 +rank4 [2026-10-29 10:44:11,938] INFO : Step 27900/50000 rank4 | Train Loss: 0.9483 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0988 | Classification: 0.0859 | Classification_binary: 0.1439 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1244 | Regression_eta: 0.0014 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1182 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0567 | Regression_eta: 0.0448 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0733 | LR: 4.68e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.4738s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.7626 | max_loss: 0.1370 | max_backward: 1.8424 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6972.2 | uss_MB: 6423.3 | child_rss_MB: 4773.1 | allocated_MB: 447.1 | reserved_MB: 33312.0 | inactive_split_MB: 1112.6 +rank4 [2026-10-29 10:48:55,971] INFO : Step 28000/50000 rank4 | Train Loss: 0.9840 | Train Loss Unweighted: 0.4736 | Train Loss Components: Auxiliary: 0.1949 | Cardinality: 0.1114 | Classification: 0.0927 | Classification_binary: 0.1564 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1099 | Regression_eta: 0.0015 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0552 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0626 | Regression_eta: 0.0463 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0696 | LR: 4.66e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9426s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8192 | max_loss: 0.2552 | max_backward: 2.1299 | max_optimizer_step: 0.2920 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 6969.4 | uss_MB: 6424.2 | child_rss_MB: 4776.8 | allocated_MB: 448.2 | reserved_MB: 33312.0 | inactive_split_MB: 1178.0 +rank4 [2026-10-29 10:48:56,836] INFO : Running validation on rank4 +rank4 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 10:49:51,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:49:52,004] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 10:50:18,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 10:50:19,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 10:50:20,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 10:50:21,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 10:50:31,618] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 10:50:32,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:50:33,003] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 10:50:49,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 10:50:50,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 10:50:50,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 10:50:51,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 10:51:01,997] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 10:51:02,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 10:51:03,446] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 10:51:18,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 10:51:19,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 10:51:20,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 10:51:21,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 10:51:31,406] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 10:56:32,746] INFO : Step 28100/50000 rank4 | Train Loss: 1.0915 | Train Loss Unweighted: 0.5085 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1649 | Classification: 0.0833 | Classification_binary: 0.1834 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1187 | Regression_eta: 0.0017 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1920 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0579 | Regression_eta: 0.0433 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0750 | LR: 4.63e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8673s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8571 | max_loss: 0.1603 | max_backward: 1.9112 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 0.7670 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0838 | Memory: rss_MB: 8288.3 | uss_MB: 7733.7 | child_rss_MB: 4773.7 | allocated_MB: 448.9 | reserved_MB: 25946.0 | inactive_split_MB: 1313.1 +rank4 [2026-10-29 11:01:18,174] INFO : Step 28200/50000 rank4 | Train Loss: 0.9241 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1144 | Classification: 0.0770 | Classification_binary: 0.1441 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1163 | Regression_eta: 0.0014 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0639 | Regression_eta: 0.0415 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0571 | LR: 4.60e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8985s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0751 | max_forward: 0.8691 | max_loss: 0.5236 | max_backward: 1.8483 | max_optimizer_step: 0.2480 | max_tensorboard_logging: 0.8546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 8289.0 | uss_MB: 7733.3 | child_rss_MB: 4770.2 | allocated_MB: 445.0 | reserved_MB: 25948.0 | inactive_split_MB: 976.4 +rank4 [2026-10-29 11:07:40,703] INFO : Step 28300/50000 rank4 | Train Loss: 0.9513 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1028 | Classification: 0.0882 | Classification_binary: 0.1368 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1147 | Regression_eta: 0.0016 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0524 | Regression_eta: 0.0319 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0572 | LR: 4.57e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.9464s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.7916 | max_loss: 0.1449 | max_backward: 2.0850 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.8583 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7055.4 | uss_MB: 6496.0 | child_rss_MB: 4774.9 | allocated_MB: 447.2 | reserved_MB: 25952.0 | inactive_split_MB: 1165.3 +rank4 [2026-10-29 11:12:27,614] INFO : Step 28400/50000 rank4 | Train Loss: 1.0604 | Train Loss Unweighted: 0.4982 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1382 | Classification: 0.0847 | Classification_binary: 0.1862 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1111 | Regression_eta: 0.0018 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.2105 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0631 | Regression_eta: 0.0703 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0549 | LR: 4.55e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.0291s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9139 | max_loss: 0.1474 | max_backward: 2.1323 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.6846 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0517 | Memory: rss_MB: 7055.5 | uss_MB: 6496.2 | child_rss_MB: 4770.0 | allocated_MB: 448.2 | reserved_MB: 26968.0 | inactive_split_MB: 835.9 +rank4 [2026-10-29 11:17:11,504] INFO : Step 28500/50000 rank4 | Train Loss: 1.0282 | Train Loss Unweighted: 0.5005 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1237 | Classification: 0.0911 | Classification_binary: 0.1756 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0013 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0372 | Regression_energy: 0.0697 | Regression_eta: 0.0261 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0536 | LR: 4.52e-04 | DataLoad Time: 0.0216s | Model Forward Time: 9.4586s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0886 | max_forward: 0.9758 | max_loss: 0.5123 | max_backward: 2.0811 | max_optimizer_step: 0.2797 | max_tensorboard_logging: 1.5644 | max_tensorboard_flush: 0.0047 | max_post_step_barrier: 0.0889 | Memory: rss_MB: 7042.2 | uss_MB: 4349.9 | child_rss_MB: 4774.7 | allocated_MB: 452.0 | reserved_MB: 27968.0 | inactive_split_MB: 923.2 +rank4 [2026-10-29 11:21:56,249] INFO : Step 28600/50000 rank4 | Train Loss: 0.9753 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1202 | Classification: 0.0804 | Classification_binary: 0.1446 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1132 | Regression_eta: 0.0017 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1591 | Regression_cos_phi: 0.0858 | Regression_energy: 0.0510 | Regression_eta: 0.0482 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0554 | LR: 4.49e-04 | DataLoad Time: 0.0230s | Model Forward Time: 4.9427s | Phase Times: max_pre_step_barrier: 0.1127 | max_batch_to_device: 0.0146 | max_forward: 0.7731 | max_loss: 0.2404 | max_backward: 1.6352 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 1.0006 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7048.8 | uss_MB: 5469.7 | child_rss_MB: 4776.7 | allocated_MB: 446.8 | reserved_MB: 30056.0 | inactive_split_MB: 896.4 +rank4 [2026-10-29 11:26:34,428] INFO : Step 28700/50000 rank4 | Train Loss: 1.0558 | Train Loss Unweighted: 0.5173 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1082 | Classification: 0.1065 | Classification_binary: 0.1612 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1217 | Regression_eta: 0.0024 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0941 | Regression_energy: 0.0671 | Regression_eta: 0.0747 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0614 | LR: 4.46e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.9558s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0686 | max_forward: 0.8423 | max_loss: 0.1475 | max_backward: 2.0170 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.8830 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 7053.8 | uss_MB: 6496.4 | child_rss_MB: 4774.2 | allocated_MB: 447.9 | reserved_MB: 30056.0 | inactive_split_MB: 925.2 +rank4 [2026-10-29 11:31:18,244] INFO : Step 28800/50000 rank4 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5167 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1474 | Classification: 0.0969 | Classification_binary: 0.1939 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.2065 | Regression_cos_phi: 0.0779 | Regression_energy: 0.0506 | Regression_eta: 0.0514 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0837 | LR: 4.44e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.8750s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0075 | max_forward: 0.8427 | max_loss: 0.2661 | max_backward: 2.8464 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7050.1 | uss_MB: 4315.7 | child_rss_MB: 4755.7 | allocated_MB: 448.2 | reserved_MB: 30058.0 | inactive_split_MB: 976.8 +rank4 [2026-10-29 11:35:53,801] INFO : Step 28900/50000 rank4 | Train Loss: 0.9389 | Train Loss Unweighted: 0.4547 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.1078 | Classification: 0.0733 | Classification_binary: 0.1521 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0294 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0436 | Regression_energy: 0.0523 | Regression_eta: 0.0312 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0598 | LR: 4.41e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.5670s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0062 | max_forward: 0.7499 | max_loss: 0.2426 | max_backward: 1.8556 | max_optimizer_step: 0.3135 | max_tensorboard_logging: 0.5831 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7048.4 | uss_MB: 6494.0 | child_rss_MB: 4774.7 | allocated_MB: 444.3 | reserved_MB: 30058.0 | inactive_split_MB: 1133.2 +rank4 [2026-10-29 11:40:53,319] INFO : Step 29000/50000 rank4 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4535 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0939 | Classification: 0.0855 | Classification_binary: 0.1308 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0015 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1554 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0689 | Regression_eta: 0.0375 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0626 | LR: 4.38e-04 | DataLoad Time: 0.0223s | Model Forward Time: 4.8470s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0062 | max_forward: 0.9270 | max_loss: 0.2073 | max_backward: 2.1896 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6044 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7047.7 | uss_MB: 6496.6 | child_rss_MB: 4759.3 | allocated_MB: 450.2 | reserved_MB: 30060.0 | inactive_split_MB: 1402.9 +rank4 [2026-10-29 11:40:54,336] INFO : Running validation on rank4 +rank4 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 11:41:46,665] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:41:47,173] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 11:42:15,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 11:42:17,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 11:42:18,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 11:42:19,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 11:42:29,905] INFO : Time taken to make predictions on device 4 is: 0.28 min +rank4 [2026-10-29 11:42:31,032] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:42:31,573] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 11:42:47,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 11:42:48,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 11:42:49,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 11:42:50,067] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 11:43:00,078] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 11:43:00,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 11:43:01,399] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 11:43:16,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 11:43:17,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 11:43:18,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 11:43:19,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 11:43:29,353] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 11:48:35,108] INFO : Step 29100/50000 rank4 | Train Loss: 1.0559 | Train Loss Unweighted: 0.5040 | Train Loss Components: Auxiliary: 0.2142 | Cardinality: 0.1307 | Classification: 0.0939 | Classification_binary: 0.1851 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1098 | Regression_eta: 0.0015 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0513 | Regression_eta: 0.0402 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0726 | LR: 4.35e-04 | DataLoad Time: 0.0154s | Model Forward Time: 7.6736s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.6779 | max_loss: 0.3815 | max_backward: 4.2661 | max_optimizer_step: 0.3470 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8253.0 | uss_MB: 6395.3 | child_rss_MB: 4770.6 | allocated_MB: 448.8 | reserved_MB: 24188.0 | inactive_split_MB: 1162.1 +rank4 [2026-10-29 11:53:34,049] INFO : Step 29200/50000 rank4 | Train Loss: 0.9909 | Train Loss Unweighted: 0.4801 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1204 | Classification: 0.0923 | Classification_binary: 0.1746 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1054 | Regression_eta: 0.0014 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.1980 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0547 | Regression_eta: 0.0468 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0955 | LR: 4.33e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7761s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8354 | max_loss: 0.1558 | max_backward: 2.1201 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0677 | Memory: rss_MB: 8251.2 | uss_MB: 7692.4 | child_rss_MB: 4763.1 | allocated_MB: 447.9 | reserved_MB: 24190.0 | inactive_split_MB: 1166.5 +rank4 [2026-10-29 11:58:27,986] INFO : Step 29300/50000 rank4 | Train Loss: 0.9337 | Train Loss Unweighted: 0.4462 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1121 | Classification: 0.0773 | Classification_binary: 0.1359 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1146 | Regression_eta: 0.0015 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1146 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0531 | Regression_eta: 0.0337 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0432 | LR: 4.30e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.9132s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0277 | max_forward: 0.8212 | max_loss: 0.1457 | max_backward: 1.8414 | max_optimizer_step: 0.2550 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7135.1 | uss_MB: 5997.0 | child_rss_MB: 4761.5 | allocated_MB: 444.9 | reserved_MB: 24190.0 | inactive_split_MB: 1038.6 +rank4 [2026-10-29 12:03:07,470] INFO : Step 29400/50000 rank4 | Train Loss: 1.0186 | Train Loss Unweighted: 0.4675 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1372 | Classification: 0.0948 | Classification_binary: 0.1561 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0020 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1624 | Regression_cos_phi: 0.0708 | Regression_energy: 0.0579 | Regression_eta: 0.1570 | Regression_pt: 0.0657 | Regression_sin_phi: 0.0595 | LR: 4.27e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8032s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8495 | max_loss: 0.1434 | max_backward: 1.9359 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7145.4 | uss_MB: 6588.5 | child_rss_MB: 4762.3 | allocated_MB: 447.9 | reserved_MB: 26208.0 | inactive_split_MB: 957.3 +rank4 [2026-10-29 12:08:16,204] INFO : Step 29500/50000 rank4 | Train Loss: 1.0909 | Train Loss Unweighted: 0.5362 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.1071 | Classification_binary: 0.1896 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1176 | Regression_eta: 0.0019 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.2093 | Regression_cos_phi: 0.0851 | Regression_energy: 0.0688 | Regression_eta: 0.0528 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0752 | LR: 4.24e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9672s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0876 | max_forward: 0.8925 | max_loss: 0.1527 | max_backward: 1.9621 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.8162 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7137.1 | uss_MB: 6580.0 | child_rss_MB: 4760.9 | allocated_MB: 448.9 | reserved_MB: 26214.0 | inactive_split_MB: 1166.5 +rank4 [2026-10-29 12:13:12,653] INFO : Step 29600/50000 rank4 | Train Loss: 0.9608 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.1935 | Cardinality: 0.1093 | Classification: 0.0850 | Classification_binary: 0.1542 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1663 | Regression_cos_phi: 0.0564 | Regression_energy: 0.0643 | Regression_eta: 0.0364 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0683 | LR: 4.22e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.8687s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7848 | max_loss: 0.1495 | max_backward: 1.8042 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.0178 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7115.2 | uss_MB: 6585.9 | child_rss_MB: 4743.4 | allocated_MB: 445.5 | reserved_MB: 26214.0 | inactive_split_MB: 1171.9 +rank4 [2026-10-29 12:18:07,442] INFO : Step 29700/50000 rank4 | Train Loss: 0.9511 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1260 | Classification: 0.0829 | Classification_binary: 0.1357 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1183 | Regression_eta: 0.0014 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0838 | Regression_eta: 0.0445 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0857 | LR: 4.19e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.2357s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7531 | max_loss: 0.2308 | max_backward: 2.2313 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.9708 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7117.3 | uss_MB: 6587.7 | child_rss_MB: 4741.9 | allocated_MB: 446.6 | reserved_MB: 26214.0 | inactive_split_MB: 1160.6 +rank4 [2026-10-29 12:23:08,925] INFO : Step 29800/50000 rank4 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4915 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1104 | Classification: 0.0864 | Classification_binary: 0.1759 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1495 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0563 | Regression_eta: 0.0296 | Regression_pt: 0.0641 | Regression_sin_phi: 0.0588 | LR: 4.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.4242s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8210 | max_loss: 0.1528 | max_backward: 2.4753 | max_optimizer_step: 0.3576 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0038 | max_post_step_barrier: 0.1798 | Memory: rss_MB: 7115.5 | uss_MB: 5536.1 | child_rss_MB: 4741.9 | allocated_MB: 450.3 | reserved_MB: 26214.0 | inactive_split_MB: 938.8 +rank4 [2026-10-29 12:28:10,143] INFO : Step 29900/50000 rank4 | Train Loss: 0.9604 | Train Loss Unweighted: 0.4559 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1243 | Classification: 0.0825 | Classification_binary: 0.1616 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1040 | Regression_eta: 0.0015 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1387 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0551 | Regression_eta: 0.0471 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0490 | LR: 4.14e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9482s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1142 | max_forward: 0.9271 | max_loss: 0.1531 | max_backward: 1.8698 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.8620 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7091.0 | uss_MB: 6437.5 | child_rss_MB: 4745.0 | allocated_MB: 450.0 | reserved_MB: 26214.0 | inactive_split_MB: 1166.0 +rank4 [2026-10-29 12:32:58,888] INFO : Step 30000/50000 rank4 | Train Loss: 0.9450 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.1132 | Classification: 0.0847 | Classification_binary: 0.1394 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0522 | Regression_eta: 0.0361 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0498 | LR: 4.11e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1876s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 1.0327 | max_loss: 0.1419 | max_backward: 1.6503 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.4878 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7039.0 | uss_MB: 6586.5 | child_rss_MB: 4745.4 | allocated_MB: 443.8 | reserved_MB: 26214.0 | inactive_split_MB: 1152.2 +rank4 [2026-10-29 12:47:12,790] INFO : Running validation on rank4 +rank4 [2026-10-29 12:47:12,790] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 12:47:12,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 14:49:00,124] INFO : Initializing model with seed=12345; process seed=12349 +rank4 [2026-10-29 14:49:00,130] INFO : configured dtype=torch.bfloat16 for autocast +rank4 [2026-10-29 14:49:20,929] INFO : Instantiating model +rank4 [2026-10-29 14:49:20,930] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank4 [2026-10-29 14:49:20,930] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank4 [2026-10-29 14:49:20,948] INFO : Input encoding initialization took 0.02s +rank4 [2026-10-29 14:49:20,948] INFO : nn0 parameters: 171776 +rank4 [2026-10-29 14:49:20,948] INFO : Initializing shared backbone layers, num_convs=6 +rank4 [2026-10-29 14:49:20,948] INFO : Initializing backbone layer 0 +rank4 [2026-10-29 14:49:20,983] INFO : Initializing backbone layer 1 +rank4 [2026-10-29 14:49:20,991] INFO : Initializing backbone layer 2 +rank4 [2026-10-29 14:49:21,000] INFO : Initializing backbone layer 3 +rank4 [2026-10-29 14:49:21,008] INFO : Initializing backbone layer 4 +rank4 [2026-10-29 14:49:21,016] INFO : Initializing backbone layer 5 +rank4 [2026-10-29 14:49:21,025] INFO : Backbone initialization took 0.08s +rank4 [2026-10-29 14:49:21,025] INFO : backbone parameters: 7901376 +rank4 [2026-10-29 14:49:21,025] INFO : Initializing output DNNs +rank4 [2026-10-29 14:49:21,062] INFO : Output DNNs initialization took 0.04s +rank4 [2026-10-29 14:49:21,062] INFO : backbone_mode=BackboneMode.SHARED +rank4 [2026-10-29 14:49:21,062] INFO : jagged_attention enabled=False +rank4 [2026-10-29 14:49:21,062] INFO : flash_attn_varlen enabled=False +rank4 [2026-10-29 14:49:21,062] INFO : task_queries enabled=False +rank4 [2026-10-29 14:49:21,062] INFO : classification_norm parameters: 0 +rank4 [2026-10-29 14:49:21,062] INFO : regression_norm parameters: 0 +rank4 [2026-10-29 14:49:21,062] INFO : classification_query parameters: 0 +rank4 [2026-10-29 14:49:21,062] INFO : regression_query parameters: 0 +rank4 [2026-10-29 14:49:21,062] INFO : classification_readout parameters: 0 +rank4 [2026-10-29 14:49:21,062] INFO : regression_readout parameters: 0 +rank4 [2026-10-29 14:49:21,062] INFO : output_mode=set +rank4 [2026-10-29 14:49:21,063] INFO : set_decoder parameters: 4713812 +rank4 [2026-10-29 14:49:21,063] INFO : nn_binary_particle parameters: 0 +rank4 [2026-10-29 14:49:21,063] INFO : nn_pid parameters: 0 +rank4 [2026-10-29 14:49:21,063] INFO : nn_pt parameters: 0 +rank4 [2026-10-29 14:49:21,063] INFO : nn_eta parameters: 0 +rank4 [2026-10-29 14:49:21,063] INFO : nn_sin_phi parameters: 0 +rank4 [2026-10-29 14:49:21,063] INFO : nn_cos_phi parameters: 0 +rank4 [2026-10-29 14:49:21,063] INFO : nn_energy parameters: 0 +rank4 [2026-10-29 14:49:21,064] INFO : Total MLPF parameters: 12786964 +rank4 [2026-10-29 14:49:21,064] INFO : MLPF __init__ done +rank4 [2026-10-29 14:49:21,064] INFO : Instantiated model +rank4 [2026-10-29 14:49:21,064] INFO : Moving model to device rank=4 +rank4 [2026-10-29 14:49:21,085] INFO : Moved model to device rank=4 +rank4 [2026-10-29 14:49:21,087] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank4 [2026-10-29 14:49:21,560] INFO : Restoring training from step 30001 +rank4 [2026-10-29 14:49:21,567] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth +rank4 [2026-10-29 14:49:21,568] INFO : Configured model for SyncBatchNorm rank=4 +rank4 [2026-10-29 14:49:22,134] INFO : Configured model for DistributedDataParallel rank=4 +rank4 [2026-10-29 14:49:22,286] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank4 [2026-10-29 14:49:22,287] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank4 [2026-10-29 14:49:22,288] INFO : Trainable parameters: 12786964 +rank4 [2026-10-29 14:49:22,288] INFO : Non-trainable parameters: 23268 +rank4 [2026-10-29 14:49:22,288] INFO : Total parameters: 12810232 +rank4 [2026-10-29 14:49:22,291] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank4 [2026-10-29 14:49:22,292] INFO : Getting dataloaders +rank4 [2026-10-29 14:49:22,292] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:49:28,710] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:49:29,732] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:49:30,956] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank4 [2026-10-29 14:49:30,957] INFO : Creating WeightedMixIterator with 1 data loaders. +rank4 [2026-10-29 14:49:30,957] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:49:31,235] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:49:31,492] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 14:49:31,741] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank4 [2026-10-29 14:49:31,741] INFO : Creating InterleavedIterator with 1 data loaders. +rank4 [2026-10-29 14:49:45,486] INFO : Got dataloaders +rank4 [2026-10-29 14:49:45,486] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [8888] of epochs [1] +rank4 [2026-10-29 14:49:45,486] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank4 [2026-10-29 14:49:45,486] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank4 [2026-10-29 14:49:45,486] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank4 [2026-10-29 14:49:47,200] INFO : loader split=train rank=4 len=21112 +rank4 [2026-10-29 14:49:47,201] INFO : loader split=valid rank=4 len=12 +rank4 [2026-10-29 14:49:47,201] INFO : Starting training from step 30001 to 50000 on rank 4 of 8 +rank4 [2026-10-29 14:49:47,201] INFO : Creating train_iterator +rank4 [2026-10-29 14:49:47,201] INFO : ResumableSampler physical:train loading state: start_index=142208 +rank4 [2026-10-29 14:50:01,570] INFO : Created train_iterator +rank4 [2026-10-29 14:54:42,004] INFO : Step 30100/50000 rank4 | Train Loss: 0.9854 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.2024 | Cardinality: 0.1295 | Classification: 0.0965 | Classification_binary: 0.1561 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0711 | Regression_eta: 0.0705 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0618 | LR: 4.08e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.6362s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7011 | max_loss: 0.1473 | max_backward: 1.5273 | max_optimizer_step: 0.1574 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0061 | Memory: rss_MB: 4775.9 | uss_MB: 3472.5 | child_rss_MB: 6376.2 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 850.1 +rank4 [2026-10-29 14:59:12,793] INFO : Step 30200/50000 rank4 | Train Loss: 1.0165 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2044 | Cardinality: 0.1520 | Classification: 0.0856 | Classification_binary: 0.1718 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1918 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0608 | Regression_eta: 0.0614 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0733 | LR: 4.06e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9886s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.7915 | max_loss: 0.2568 | max_backward: 1.8926 | max_optimizer_step: 0.1551 | max_tensorboard_logging: 0.4604 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4776.3 | uss_MB: 3476.7 | child_rss_MB: 6367.6 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 842.0 +rank4 [2026-10-29 15:03:42,967] INFO : Step 30300/50000 rank4 | Train Loss: 0.9183 | Train Loss Unweighted: 0.4449 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1232 | Classification: 0.0820 | Classification_binary: 0.1459 | HitGrouping: 0.1516 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0012 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0270 | Regression_energy: 0.0642 | Regression_eta: 0.0242 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0381 | LR: 4.03e-04 | DataLoad Time: 0.0171s | Model Forward Time: 3.8590s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7525 | max_loss: 0.2476 | max_backward: 1.8262 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.4574 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0039 | Memory: rss_MB: 4773.5 | uss_MB: 3475.1 | child_rss_MB: 6385.6 | allocated_MB: 495.8 | reserved_MB: 28760.0 | inactive_split_MB: 848.2 +rank4 [2026-10-29 15:08:13,476] INFO : Step 30400/50000 rank4 | Train Loss: 0.9035 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1137 | Classification: 0.0848 | Classification_binary: 0.1358 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0494 | Regression_eta: 0.0280 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0339 | LR: 4.00e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9726s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7769 | max_loss: 0.1398 | max_backward: 1.9002 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.4925 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4770.8 | uss_MB: 3475.5 | child_rss_MB: 6366.2 | allocated_MB: 498.8 | reserved_MB: 28760.0 | inactive_split_MB: 878.9 +rank4 [2026-10-29 15:12:45,608] INFO : Step 30500/50000 rank4 | Train Loss: 1.0399 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1239 | Classification: 0.0901 | Classification_binary: 0.1841 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0018 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1775 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0497 | Regression_eta: 0.0408 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0721 | LR: 3.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3869s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9688 | max_loss: 0.1627 | max_backward: 2.3054 | max_optimizer_step: 0.1555 | max_tensorboard_logging: 0.4908 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4768.4 | uss_MB: 3475.6 | child_rss_MB: 6405.5 | allocated_MB: 501.7 | reserved_MB: 28760.0 | inactive_split_MB: 944.3 +rank4 [2026-10-29 15:17:13,866] INFO : Step 30600/50000 rank4 | Train Loss: 0.9987 | Train Loss Unweighted: 0.4959 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1112 | Classification: 0.0932 | Classification_binary: 0.1720 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1137 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1750 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0594 | Regression_eta: 0.0347 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0597 | LR: 3.95e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2231s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7736 | max_loss: 0.1566 | max_backward: 1.7922 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.6994 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4770.8 | uss_MB: 3476.1 | child_rss_MB: 6399.5 | allocated_MB: 500.1 | reserved_MB: 28760.0 | inactive_split_MB: 845.9 +rank4 [2026-10-29 15:21:45,752] INFO : Step 30700/50000 rank4 | Train Loss: 0.9490 | Train Loss Unweighted: 0.4575 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1237 | Classification: 0.0844 | Classification_binary: 0.1399 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0014 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1259 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0822 | Regression_eta: 0.0397 | Regression_pt: 0.0890 | Regression_sin_phi: 0.0592 | LR: 3.92e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1844s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7215 | max_loss: 0.5560 | max_backward: 2.0906 | max_optimizer_step: 0.1553 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4768.7 | uss_MB: 3474.9 | child_rss_MB: 6379.2 | allocated_MB: 493.8 | reserved_MB: 28760.0 | inactive_split_MB: 827.5 +rank4 [2026-10-29 15:26:16,185] INFO : Step 30800/50000 rank4 | Train Loss: 1.0320 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1331 | Classification: 0.0916 | Classification_binary: 0.1632 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1173 | Regression_eta: 0.0015 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1929 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0639 | Regression_eta: 0.0408 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0828 | LR: 3.90e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6900s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1016 | max_forward: 0.8041 | max_loss: 0.1462 | max_backward: 1.9351 | max_optimizer_step: 0.1552 | max_tensorboard_logging: 1.1007 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4769.5 | uss_MB: 3475.6 | child_rss_MB: 6407.8 | allocated_MB: 497.9 | reserved_MB: 28760.0 | inactive_split_MB: 842.0 +rank4 [2026-10-29 15:30:49,188] INFO : Step 30900/50000 rank4 | Train Loss: 1.0226 | Train Loss Unweighted: 0.4952 | Train Loss Components: Auxiliary: 0.2139 | Cardinality: 0.1339 | Classification: 0.0875 | Classification_binary: 0.1859 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1092 | Regression_eta: 0.0013 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1925 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0657 | Regression_eta: 0.0361 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0542 | LR: 3.87e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4017s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7610 | max_loss: 0.2067 | max_backward: 1.7028 | max_optimizer_step: 0.1599 | max_tensorboard_logging: 1.0327 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4771.1 | uss_MB: 3476.3 | child_rss_MB: 6385.8 | allocated_MB: 497.2 | reserved_MB: 28760.0 | inactive_split_MB: 838.0 +rank4 [2026-10-29 15:35:26,545] INFO : Step 31000/50000 rank4 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4630 | Train Loss Components: Auxiliary: 0.2006 | Cardinality: 0.1217 | Classification: 0.0821 | Classification_binary: 0.1592 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1085 | Regression_eta: 0.0018 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0279 | Classification_binary: 0.1619 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0613 | Regression_eta: 0.0318 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0550 | LR: 3.84e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4973s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7780 | max_loss: 0.2475 | max_backward: 1.8803 | max_optimizer_step: 0.1588 | max_tensorboard_logging: 0.9947 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4772.1 | uss_MB: 3477.0 | child_rss_MB: 6402.3 | allocated_MB: 498.2 | reserved_MB: 28760.0 | inactive_split_MB: 820.4 +rank4 [2026-10-29 15:35:27,414] INFO : Running validation on rank4 +rank4 [2026-10-29 15:35:27,414] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 15:35:27,414] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 15:36:19,715] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:36:20,266] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 15:36:45,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 15:36:46,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 15:36:47,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 15:36:48,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 15:36:58,964] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 15:36:59,566] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:37:00,107] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 15:37:15,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 15:37:16,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 15:37:17,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 15:37:18,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 15:37:28,698] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 15:37:29,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 15:37:29,837] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 15:37:45,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 15:37:46,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 15:37:46,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 15:37:47,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 15:37:57,722] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 15:42:50,284] INFO : Step 31100/50000 rank4 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4264 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1054 | Classification: 0.0853 | Classification_binary: 0.1288 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1342 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0564 | Regression_eta: 0.0262 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0594 | LR: 3.82e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5242s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7353 | max_loss: 0.1410 | max_backward: 1.7959 | max_optimizer_step: 0.2754 | max_tensorboard_logging: 0.9599 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 6463.6 | uss_MB: 5188.4 | child_rss_MB: 6394.5 | allocated_MB: 495.8 | reserved_MB: 27508.0 | inactive_split_MB: 851.2 +rank4 [2026-10-29 15:47:34,510] INFO : Step 31200/50000 rank4 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4709 | Train Loss Components: Auxiliary: 0.2045 | Cardinality: 0.1286 | Classification: 0.0862 | Classification_binary: 0.1646 | HitGrouping: 0.1524 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1077 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1695 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0540 | Regression_eta: 0.0353 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0602 | LR: 3.79e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3222s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8441 | max_loss: 0.1469 | max_backward: 2.0813 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 6462.4 | uss_MB: 5189.4 | child_rss_MB: 6422.3 | allocated_MB: 496.9 | reserved_MB: 27508.0 | inactive_split_MB: 935.6 +rank4 [2026-10-29 15:52:19,963] INFO : Step 31300/50000 rank4 | Train Loss: 1.0631 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1394 | Classification: 0.0917 | Classification_binary: 0.1731 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0562 | Regression_eta: 0.0299 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0611 | LR: 3.76e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2502s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7950 | max_loss: 0.1940 | max_backward: 1.8810 | max_optimizer_step: 0.2519 | max_tensorboard_logging: 0.5719 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5602.7 | uss_MB: 4330.1 | child_rss_MB: 6421.1 | allocated_MB: 498.6 | reserved_MB: 29580.0 | inactive_split_MB: 773.5 +rank4 [2026-10-29 15:56:52,522] INFO : Step 31400/50000 rank4 | Train Loss: 0.9019 | Train Loss Unweighted: 0.4258 | Train Loss Components: Auxiliary: 0.1823 | Cardinality: 0.1101 | Classification: 0.0746 | Classification_binary: 0.1337 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1070 | Regression_eta: 0.0014 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0609 | Regression_eta: 0.0262 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0409 | LR: 3.74e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2771s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8478 | max_loss: 0.2460 | max_backward: 2.0403 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.4770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5595.4 | uss_MB: 4331.0 | child_rss_MB: 6422.8 | allocated_MB: 495.2 | reserved_MB: 29580.0 | inactive_split_MB: 986.6 +rank4 [2026-10-29 16:01:34,248] INFO : Step 31500/50000 rank4 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4434 | Train Loss Components: Auxiliary: 0.1867 | Cardinality: 0.1036 | Classification: 0.0834 | Classification_binary: 0.1437 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1064 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0558 | Regression_eta: 0.0322 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0453 | LR: 3.71e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.7497s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7930 | max_loss: 0.1458 | max_backward: 1.7855 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 2.2800 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5572.8 | uss_MB: 4310.0 | child_rss_MB: 6426.4 | allocated_MB: 496.8 | reserved_MB: 29580.0 | inactive_split_MB: 850.8 +rank4 [2026-10-29 16:06:05,475] INFO : Step 31600/50000 rank4 | Train Loss: 1.0288 | Train Loss Unweighted: 0.4730 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1317 | Classification: 0.0906 | Classification_binary: 0.1743 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1012 | Regression_eta: 0.0017 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1751 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0432 | Regression_eta: 0.0447 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0640 | LR: 3.69e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1930s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1021 | max_forward: 0.8041 | max_loss: 0.5000 | max_backward: 1.8881 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0133 | Memory: rss_MB: 5567.6 | uss_MB: 4330.2 | child_rss_MB: 6421.3 | allocated_MB: 494.5 | reserved_MB: 29580.0 | inactive_split_MB: 850.5 +rank4 [2026-10-29 16:10:48,123] INFO : Step 31700/50000 rank4 | Train Loss: 0.9206 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1106 | Classification: 0.0729 | Classification_binary: 0.1465 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0821 | Regression_energy: 0.0572 | Regression_eta: 0.0417 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0441 | LR: 3.66e-04 | DataLoad Time: 0.0178s | Model Forward Time: 3.7511s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0062 | max_forward: 0.7097 | max_loss: 0.1806 | max_backward: 1.6117 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0100 | Memory: rss_MB: 5567.0 | uss_MB: 4328.6 | child_rss_MB: 6424.1 | allocated_MB: 495.3 | reserved_MB: 29580.0 | inactive_split_MB: 969.6 +rank4 [2026-10-29 16:15:56,265] INFO : Step 31800/50000 rank4 | Train Loss: 0.9090 | Train Loss Unweighted: 0.4310 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1140 | Classification: 0.0748 | Classification_binary: 0.1332 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1094 | Regression_eta: 0.0013 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.1462 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0693 | Regression_eta: 0.0392 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0495 | LR: 3.64e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.8749s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7102 | max_loss: 0.1436 | max_backward: 1.6093 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5562.0 | uss_MB: 4326.9 | child_rss_MB: 6424.4 | allocated_MB: 499.4 | reserved_MB: 29584.0 | inactive_split_MB: 1068.5 +rank4 [2026-10-29 16:20:39,712] INFO : Step 31900/50000 rank4 | Train Loss: 0.9919 | Train Loss Unweighted: 0.4645 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1269 | Classification: 0.0911 | Classification_binary: 0.1745 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0975 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0517 | Regression_eta: 0.0335 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0597 | LR: 3.61e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3411s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.2538 | max_backward: 1.9768 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.5597 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5568.8 | uss_MB: 4333.1 | child_rss_MB: 6433.0 | allocated_MB: 496.2 | reserved_MB: 29584.0 | inactive_split_MB: 829.8 +rank4 [2026-10-29 16:25:21,506] INFO : Step 32000/50000 rank4 | Train Loss: 0.9662 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1172 | Classification: 0.0861 | Classification_binary: 0.1650 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1010 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1937 | Regression_cos_phi: 0.1298 | Regression_energy: 0.0689 | Regression_eta: 0.1049 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0895 | LR: 3.58e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2267s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7801 | max_loss: 0.1534 | max_backward: 1.8397 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5561.3 | uss_MB: 4329.3 | child_rss_MB: 6422.4 | allocated_MB: 497.7 | reserved_MB: 29584.0 | inactive_split_MB: 927.6 +rank4 [2026-10-29 16:25:22,457] INFO : Running validation on rank4 +rank4 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 16:26:16,177] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:26:16,710] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 16:26:42,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 16:26:43,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 16:26:44,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 16:26:45,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 16:26:55,697] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 16:26:56,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:26:57,013] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 16:27:13,285] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 16:27:14,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 16:27:15,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 16:27:16,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 16:27:26,156] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 16:27:26,912] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 16:27:27,382] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 16:27:43,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 16:27:44,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 16:27:44,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 16:27:45,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 16:27:55,757] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 16:32:59,848] INFO : Step 32100/50000 rank4 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4572 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1125 | Classification: 0.0829 | Classification_binary: 0.1389 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1159 | Regression_eta: 0.0016 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0337 | Classification_binary: 0.1501 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0681 | Regression_eta: 0.0334 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0636 | LR: 3.56e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8034s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0431 | max_forward: 0.7694 | max_loss: 0.1379 | max_backward: 1.5662 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 1.3675 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6985.8 | uss_MB: 5757.8 | child_rss_MB: 6447.8 | allocated_MB: 499.0 | reserved_MB: 25862.0 | inactive_split_MB: 892.0 +rank4 [2026-10-29 16:37:33,177] INFO : Step 32200/50000 rank4 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1193 | Classification: 0.0958 | Classification_binary: 0.1508 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1092 | Regression_eta: 0.0016 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0657 | Regression_eta: 0.0388 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0411 | LR: 3.53e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.1403s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8207 | max_loss: 0.1504 | max_backward: 2.0613 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.9148 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6989.2 | uss_MB: 5763.2 | child_rss_MB: 6457.2 | allocated_MB: 499.4 | reserved_MB: 25864.0 | inactive_split_MB: 778.6 +rank4 [2026-10-29 16:42:04,508] INFO : Step 32300/50000 rank4 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4909 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1295 | Classification: 0.0875 | Classification_binary: 0.1834 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0017 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1978 | Regression_cos_phi: 0.1116 | Regression_energy: 0.0554 | Regression_eta: 0.0610 | Regression_pt: 0.0616 | Regression_sin_phi: 0.1100 | LR: 3.51e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7862s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8382 | max_loss: 0.2412 | max_backward: 2.0637 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.9540 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5664.5 | uss_MB: 4450.0 | child_rss_MB: 6462.2 | allocated_MB: 499.0 | reserved_MB: 27716.0 | inactive_split_MB: 775.0 +rank4 [2026-10-29 16:46:37,461] INFO : Step 32400/50000 rank4 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1089 | Classification: 0.0860 | Classification_binary: 0.1494 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1059 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0493 | Regression_eta: 0.0303 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0542 | LR: 3.48e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5416s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1137 | max_forward: 0.8338 | max_loss: 0.1463 | max_backward: 1.8285 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 0.8970 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5664.1 | uss_MB: 4449.3 | child_rss_MB: 6459.6 | allocated_MB: 497.5 | reserved_MB: 27716.0 | inactive_split_MB: 964.6 +rank4 [2026-10-29 16:51:11,723] INFO : Step 32500/50000 rank4 | Train Loss: 0.9126 | Train Loss Unweighted: 0.4568 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.0989 | Classification: 0.0980 | Classification_binary: 0.1302 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1128 | Regression_eta: 0.0017 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0689 | Regression_eta: 0.0441 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0706 | LR: 3.46e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0953s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7619 | max_loss: 0.1497 | max_backward: 1.5269 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.7665 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5643.1 | uss_MB: 4429.2 | child_rss_MB: 6454.9 | allocated_MB: 495.6 | reserved_MB: 27718.0 | inactive_split_MB: 1084.2 +rank4 [2026-10-29 16:55:48,603] INFO : Step 32600/50000 rank4 | Train Loss: 0.9821 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2025 | Cardinality: 0.1270 | Classification: 0.0917 | Classification_binary: 0.1687 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1000 | Regression_eta: 0.0014 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1691 | Regression_cos_phi: 0.0717 | Regression_energy: 0.0527 | Regression_eta: 0.0470 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0437 | LR: 3.43e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2683s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1505 | max_backward: 1.8823 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 5658.5 | uss_MB: 4448.5 | child_rss_MB: 6453.0 | allocated_MB: 496.5 | reserved_MB: 27844.0 | inactive_split_MB: 835.5 +rank4 [2026-10-29 17:00:16,963] INFO : Step 32700/50000 rank4 | Train Loss: 1.0218 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2103 | Cardinality: 0.1308 | Classification: 0.0908 | Classification_binary: 0.1761 | HitGrouping: 0.1579 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0545 | Regression_eta: 0.0325 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0720 | LR: 3.41e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2039s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7654 | max_loss: 0.2523 | max_backward: 1.8639 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4904 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.1101 | Memory: rss_MB: 5657.8 | uss_MB: 4448.3 | child_rss_MB: 6459.1 | allocated_MB: 495.8 | reserved_MB: 27844.0 | inactive_split_MB: 1177.8 +rank4 [2026-10-29 17:04:50,647] INFO : Step 32800/50000 rank4 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4798 | Train Loss Components: Auxiliary: 0.1977 | Cardinality: 0.1139 | Classification: 0.0957 | Classification_binary: 0.1480 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1163 | Regression_eta: 0.0019 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0545 | Regression_eta: 0.0680 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0652 | LR: 3.38e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0494s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7437 | max_loss: 0.1443 | max_backward: 1.7603 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.5989 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5618.6 | uss_MB: 4447.1 | child_rss_MB: 6328.3 | allocated_MB: 497.6 | reserved_MB: 27846.0 | inactive_split_MB: 1089.8 +rank4 [2026-10-29 17:09:25,013] INFO : Step 32900/50000 rank4 | Train Loss: 1.0585 | Train Loss Unweighted: 0.4917 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1472 | Classification: 0.0991 | Classification_binary: 0.1566 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0020 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1779 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0603 | Regression_eta: 0.0539 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0795 | LR: 3.36e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2376s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0141 | max_forward: 0.7853 | max_loss: 0.1442 | max_backward: 1.9716 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.5450 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5622.7 | uss_MB: 4451.0 | child_rss_MB: 6346.8 | allocated_MB: 498.5 | reserved_MB: 27846.0 | inactive_split_MB: 844.8 +rank4 [2026-10-29 17:13:57,544] INFO : Step 33000/50000 rank4 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4896 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1109 | Classification: 0.0902 | Classification_binary: 0.1757 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1096 | Regression_eta: 0.0015 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.1751 | Regression_cos_phi: 0.0888 | Regression_energy: 0.0521 | Regression_eta: 0.0511 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0694 | LR: 3.33e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1400s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0372 | max_forward: 0.7798 | max_loss: 0.1504 | max_backward: 1.8342 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5614.6 | uss_MB: 4448.7 | child_rss_MB: 6348.5 | allocated_MB: 497.7 | reserved_MB: 27846.0 | inactive_split_MB: 857.6 +rank4 [2026-10-29 17:13:58,425] INFO : Running validation on rank4 +rank4 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 17:14:49,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:14:50,068] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 17:15:17,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 17:15:18,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 17:15:19,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 17:15:20,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 17:15:30,120] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 17:15:31,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:15:31,507] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 17:15:47,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 17:15:48,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 17:15:49,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 17:15:50,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 17:16:00,028] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 17:16:01,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 17:16:01,833] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 17:16:17,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 17:16:17,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 17:16:18,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 17:16:19,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 17:16:29,521] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 17:21:40,219] INFO : Step 33100/50000 rank4 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4666 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1297 | Classification: 0.0842 | Classification_binary: 0.1537 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1125 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1339 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0498 | Regression_eta: 0.0522 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0919 | LR: 3.31e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1423s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7607 | max_loss: 0.1464 | max_backward: 1.8352 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6888.0 | uss_MB: 5749.0 | child_rss_MB: 6329.7 | allocated_MB: 496.1 | reserved_MB: 24572.0 | inactive_split_MB: 941.9 +rank4 [2026-10-29 17:26:33,027] INFO : Step 33200/50000 rank4 | Train Loss: 0.9345 | Train Loss Unweighted: 0.4417 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.1121 | Classification: 0.0825 | Classification_binary: 0.1333 | HitGrouping: 0.1549 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0017 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.1113 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0664 | Regression_eta: 0.0303 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0513 | LR: 3.28e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2178s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0447 | max_forward: 0.7619 | max_loss: 0.4589 | max_backward: 1.9181 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.4922 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6887.2 | uss_MB: 5750.0 | child_rss_MB: 6329.3 | allocated_MB: 497.6 | reserved_MB: 24578.0 | inactive_split_MB: 835.2 +rank4 [2026-10-29 17:31:31,555] INFO : Step 33300/50000 rank4 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1270 | Classification: 0.0780 | Classification_binary: 0.1518 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0985 | Regression_eta: 0.0012 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0475 | Regression_eta: 0.0320 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0389 | LR: 3.26e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.6834s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0334 | max_forward: 0.7851 | max_loss: 0.1434 | max_backward: 2.0284 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.7975 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5654.9 | uss_MB: 4170.1 | child_rss_MB: 6324.3 | allocated_MB: 496.9 | reserved_MB: 24578.0 | inactive_split_MB: 903.1 +rank4 [2026-10-29 17:36:22,674] INFO : Step 33400/50000 rank4 | Train Loss: 0.9537 | Train Loss Unweighted: 0.4634 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1239 | Classification: 0.0872 | Classification_binary: 0.1563 | HitGrouping: 0.1500 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0013 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0608 | Regression_eta: 0.0351 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0555 | LR: 3.23e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2725s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7951 | max_loss: 0.2522 | max_backward: 1.9349 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0134 | Memory: rss_MB: 5664.7 | uss_MB: 4533.0 | child_rss_MB: 6325.6 | allocated_MB: 500.7 | reserved_MB: 26578.0 | inactive_split_MB: 843.2 +rank4 [2026-10-29 17:41:05,304] INFO : Step 33500/50000 rank4 | Train Loss: 0.9039 | Train Loss Unweighted: 0.4494 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.1057 | Classification: 0.0795 | Classification_binary: 0.1278 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0011 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0846 | Regression_eta: 0.0349 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0632 | LR: 3.21e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1841s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0177 | max_forward: 0.7696 | max_loss: 0.1421 | max_backward: 1.9122 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 0.5562 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5654.7 | uss_MB: 4519.8 | child_rss_MB: 6324.9 | allocated_MB: 494.9 | reserved_MB: 26578.0 | inactive_split_MB: 839.3 +rank4 [2026-10-29 17:45:56,712] INFO : Step 33600/50000 rank4 | Train Loss: 0.9179 | Train Loss Unweighted: 0.4415 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.0997 | Classification: 0.0812 | Classification_binary: 0.1440 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1673 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0695 | Regression_eta: 0.0417 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0586 | LR: 3.19e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4052s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7783 | max_loss: 0.2418 | max_backward: 2.0392 | max_optimizer_step: 0.2787 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5667.7 | uss_MB: 4535.9 | child_rss_MB: 6324.1 | allocated_MB: 497.9 | reserved_MB: 28594.0 | inactive_split_MB: 1127.0 +rank4 [2026-10-29 17:50:43,454] INFO : Step 33700/50000 rank4 | Train Loss: 1.0564 | Train Loss Unweighted: 0.4962 | Train Loss Components: Auxiliary: 0.2235 | Cardinality: 0.1517 | Classification: 0.0917 | Classification_binary: 0.1786 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1695 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0521 | Regression_eta: 0.0281 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0519 | LR: 3.16e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.1460s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0121 | max_forward: 0.7981 | max_loss: 0.1554 | max_backward: 1.9054 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5667.6 | uss_MB: 4535.1 | child_rss_MB: 6300.3 | allocated_MB: 501.8 | reserved_MB: 28594.0 | inactive_split_MB: 1103.2 +rank4 [2026-10-29 17:55:24,637] INFO : Step 33800/50000 rank4 | Train Loss: 0.9305 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1894 | Cardinality: 0.1080 | Classification: 0.0751 | Classification_binary: 0.1369 | HitGrouping: 0.1527 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1201 | Regression_eta: 0.0014 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0670 | Regression_energy: 0.0605 | Regression_eta: 0.0358 | Regression_pt: 0.0681 | Regression_sin_phi: 0.0465 | LR: 3.14e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.1046s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8033 | max_loss: 0.1441 | max_backward: 1.7197 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0176 | Memory: rss_MB: 5662.9 | uss_MB: 4531.8 | child_rss_MB: 6320.7 | allocated_MB: 496.9 | reserved_MB: 28594.0 | inactive_split_MB: 909.5 +rank4 [2026-10-29 18:00:01,587] INFO : Step 33900/50000 rank4 | Train Loss: 0.9360 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.1942 | Cardinality: 0.1035 | Classification: 0.0805 | Classification_binary: 0.1361 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1124 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0901 | Regression_energy: 0.0710 | Regression_eta: 0.0742 | Regression_pt: 0.0752 | Regression_sin_phi: 0.1046 | LR: 3.11e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2978s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7557 | max_loss: 0.2439 | max_backward: 1.8923 | max_optimizer_step: 0.2623 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5665.6 | uss_MB: 4533.9 | child_rss_MB: 6306.4 | allocated_MB: 496.5 | reserved_MB: 28598.0 | inactive_split_MB: 941.5 +rank4 [2026-10-29 18:04:39,053] INFO : Step 34000/50000 rank4 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0930 | Classification: 0.0875 | Classification_binary: 0.1606 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0506 | Regression_eta: 0.0341 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0393 | LR: 3.09e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3492s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0943 | max_forward: 0.8289 | max_loss: 0.4616 | max_backward: 2.0219 | max_optimizer_step: 0.1767 | max_tensorboard_logging: 0.4773 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5666.0 | uss_MB: 4534.2 | child_rss_MB: 6326.4 | allocated_MB: 496.0 | reserved_MB: 28598.0 | inactive_split_MB: 1123.0 +rank4 [2026-10-29 18:04:39,969] INFO : Running validation on rank4 +rank4 [2026-10-29 18:04:39,969] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 18:04:39,969] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 18:05:30,627] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:05:31,102] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 18:05:54,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 18:05:55,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 18:05:56,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 18:05:57,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 18:06:07,841] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 18:06:08,708] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:06:09,213] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 18:06:25,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 18:06:26,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 18:06:27,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 18:06:28,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 18:06:38,234] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 18:06:39,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:06:39,483] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 18:06:54,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 18:06:55,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 18:06:56,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 18:06:57,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 18:07:07,300] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 18:12:12,108] INFO : Step 34100/50000 rank4 | Train Loss: 1.0240 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2153 | Cardinality: 0.1202 | Classification: 0.0979 | Classification_binary: 0.1668 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1210 | Regression_eta: 0.0013 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0636 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0536 | Regression_eta: 0.0313 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0488 | LR: 3.06e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5012s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0078 | max_forward: 0.9085 | max_loss: 0.1531 | max_backward: 2.1727 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.4680 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6900.4 | uss_MB: 5761.6 | child_rss_MB: 6326.4 | allocated_MB: 496.4 | reserved_MB: 24560.0 | inactive_split_MB: 1172.2 +rank4 [2026-10-29 18:17:11,666] INFO : Step 34200/50000 rank4 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4225 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.1030 | Classification: 0.0694 | Classification_binary: 0.1279 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0016 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1146 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0561 | Regression_eta: 0.0293 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0740 | LR: 3.04e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.8736s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 1.1401 | max_loss: 1.0887 | max_backward: 3.3565 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5402 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6897.3 | uss_MB: 5759.3 | child_rss_MB: 6328.4 | allocated_MB: 493.1 | reserved_MB: 27668.0 | inactive_split_MB: 842.0 +rank4 [2026-10-29 18:21:53,342] INFO : Step 34300/50000 rank4 | Train Loss: 0.8999 | Train Loss Unweighted: 0.4291 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.1046 | Classification: 0.0782 | Classification_binary: 0.1421 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0015 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1426 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0650 | Regression_eta: 0.0572 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0528 | LR: 3.02e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5611s | Phase Times: max_pre_step_barrier: 0.1089 | max_batch_to_device: 0.0070 | max_forward: 0.7738 | max_loss: 0.4754 | max_backward: 1.8888 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5721.4 | uss_MB: 4584.0 | child_rss_MB: 6315.8 | allocated_MB: 498.9 | reserved_MB: 29668.0 | inactive_split_MB: 777.1 +rank4 [2026-10-29 18:26:42,338] INFO : Step 34400/50000 rank4 | Train Loss: 0.9382 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2026 | Cardinality: 0.1181 | Classification: 0.0846 | Classification_binary: 0.1681 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.2015 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0639 | Regression_eta: 0.0389 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0700 | LR: 2.99e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6572s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.8411 | max_loss: 0.2739 | max_backward: 2.1385 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5734.9 | uss_MB: 4598.3 | child_rss_MB: 6315.2 | allocated_MB: 496.8 | reserved_MB: 29668.0 | inactive_split_MB: 858.8 +rank4 [2026-10-29 18:31:40,822] INFO : Step 34500/50000 rank4 | Train Loss: 0.9189 | Train Loss Unweighted: 0.4456 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1063 | Classification: 0.0767 | Classification_binary: 0.1421 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0900 | Regression_energy: 0.0618 | Regression_eta: 0.0434 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0674 | LR: 2.97e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2127s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.7430 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5727.7 | uss_MB: 4595.8 | child_rss_MB: 6313.8 | allocated_MB: 494.3 | reserved_MB: 29668.0 | inactive_split_MB: 926.0 +rank4 [2026-10-29 18:36:27,181] INFO : Step 34600/50000 rank4 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4340 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1046 | Classification: 0.0794 | Classification_binary: 0.1340 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0016 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1254 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0602 | Regression_eta: 0.0593 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0779 | LR: 2.95e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.6958s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0170 | max_forward: 0.9895 | max_loss: 0.4622 | max_backward: 2.2541 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5731.7 | uss_MB: 4597.0 | child_rss_MB: 6311.8 | allocated_MB: 498.6 | reserved_MB: 29672.0 | inactive_split_MB: 765.1 +rank4 [2026-10-29 18:40:58,824] INFO : Step 34700/50000 rank4 | Train Loss: 0.9586 | Train Loss Unweighted: 0.4539 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1185 | Classification: 0.0862 | Classification_binary: 0.1624 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1745 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0758 | Regression_eta: 0.0354 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0572 | LR: 2.92e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3654s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7862 | max_loss: 0.2628 | max_backward: 1.9231 | max_optimizer_step: 0.2860 | max_tensorboard_logging: 0.4599 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5730.7 | uss_MB: 4596.4 | child_rss_MB: 6303.0 | allocated_MB: 499.0 | reserved_MB: 29672.0 | inactive_split_MB: 1074.3 +rank4 [2026-10-29 18:45:38,651] INFO : Step 34800/50000 rank4 | Train Loss: 0.9346 | Train Loss Unweighted: 0.4373 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1276 | Classification: 0.0867 | Classification_binary: 0.1551 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0959 | Regression_eta: 0.0013 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1497 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0485 | Regression_eta: 0.0305 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0402 | LR: 2.90e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2162s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0770 | max_forward: 0.8404 | max_loss: 0.1556 | max_backward: 1.8035 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.5351 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5727.0 | uss_MB: 4594.2 | child_rss_MB: 6312.4 | allocated_MB: 497.4 | reserved_MB: 29672.0 | inactive_split_MB: 1113.6 +rank4 [2026-10-29 18:50:17,935] INFO : Step 34900/50000 rank4 | Train Loss: 0.8719 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1154 | Classification: 0.0681 | Classification_binary: 0.1220 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1093 | Regression_eta: 0.0013 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1106 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0468 | Regression_eta: 0.0298 | Regression_pt: 0.0518 | Regression_sin_phi: 0.0619 | LR: 2.88e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.1454s | Phase Times: max_pre_step_barrier: 0.0080 | max_batch_to_device: 0.0693 | max_forward: 1.2127 | max_loss: 0.2834 | max_backward: 3.0996 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.6125 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 5724.6 | uss_MB: 4576.6 | child_rss_MB: 6292.2 | allocated_MB: 495.3 | reserved_MB: 29672.0 | inactive_split_MB: 864.7 +rank4 [2026-10-29 18:55:19,194] INFO : Step 35000/50000 rank4 | Train Loss: 0.9157 | Train Loss Unweighted: 0.4220 | Train Loss Components: Auxiliary: 0.1897 | Cardinality: 0.1230 | Classification: 0.0771 | Classification_binary: 0.1455 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0980 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0309 | Classification_binary: 0.1412 | Regression_cos_phi: 0.0407 | Regression_energy: 0.0425 | Regression_eta: 0.0315 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0648 | LR: 2.85e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5123s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.8243 | max_loss: 0.4759 | max_backward: 2.1181 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5728.2 | uss_MB: 4598.3 | child_rss_MB: 6307.6 | allocated_MB: 498.3 | reserved_MB: 29672.0 | inactive_split_MB: 766.0 +rank4 [2026-10-29 18:55:20,127] INFO : Running validation on rank4 +rank4 [2026-10-29 18:55:20,127] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 18:55:20,127] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 18:56:11,604] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:56:12,115] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 18:56:36,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 18:56:37,272] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 18:56:38,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 18:56:39,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 18:56:49,303] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 18:56:49,876] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:56:50,324] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 18:57:07,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 18:57:08,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 18:57:09,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 18:57:10,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 18:57:20,431] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-29 18:57:20,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 18:57:20,903] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 18:57:36,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 18:57:36,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 18:57:37,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 18:57:38,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 18:57:48,562] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 19:02:56,670] INFO : Step 35100/50000 rank4 | Train Loss: 0.9622 | Train Loss Unweighted: 0.4512 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1150 | Classification: 0.0857 | Classification_binary: 0.1724 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0948 | Regression_eta: 0.0014 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0508 | Regression_eta: 0.0242 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0377 | LR: 2.83e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8771s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1018 | max_forward: 0.8921 | max_loss: 0.1518 | max_backward: 2.0328 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8633 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5815.2 | uss_MB: 4686.8 | child_rss_MB: 6305.0 | allocated_MB: 499.4 | reserved_MB: 23366.0 | inactive_split_MB: 1243.8 +rank4 [2026-10-29 19:07:43,277] INFO : Step 35200/50000 rank4 | Train Loss: 0.9331 | Train Loss Unweighted: 0.4385 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.1234 | Classification: 0.0814 | Classification_binary: 0.1441 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1781 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0610 | Regression_eta: 0.0628 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0802 | LR: 2.81e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1468s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0577 | max_forward: 0.7693 | max_loss: 0.1423 | max_backward: 1.7209 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5822.3 | uss_MB: 4693.9 | child_rss_MB: 6304.7 | allocated_MB: 494.4 | reserved_MB: 25346.0 | inactive_split_MB: 1054.2 +rank4 [2026-10-29 19:12:30,668] INFO : Step 35300/50000 rank4 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4075 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1105 | Classification: 0.0819 | Classification_binary: 0.1245 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0989 | Regression_eta: 0.0014 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1164 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0592 | Regression_eta: 0.0389 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0468 | LR: 2.79e-04 | DataLoad Time: 0.0174s | Model Forward Time: 6.8653s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 1.2856 | max_loss: 0.4079 | max_backward: 4.4969 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5749 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5814.5 | uss_MB: 4685.2 | child_rss_MB: 6307.8 | allocated_MB: 498.3 | reserved_MB: 25346.0 | inactive_split_MB: 1140.5 +rank4 [2026-10-29 19:17:14,808] INFO : Step 35400/50000 rank4 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4888 | Train Loss Components: Auxiliary: 0.2130 | Cardinality: 0.1296 | Classification: 0.0908 | Classification_binary: 0.1718 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0014 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.1977 | Regression_cos_phi: 0.0822 | Regression_energy: 0.0496 | Regression_eta: 0.0327 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0601 | LR: 2.76e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3430s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8312 | max_loss: 0.1490 | max_backward: 2.0287 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.4708 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5820.8 | uss_MB: 4696.7 | child_rss_MB: 6308.5 | allocated_MB: 497.8 | reserved_MB: 25346.0 | inactive_split_MB: 1132.0 +rank4 [2026-10-29 19:21:58,250] INFO : Step 35500/50000 rank4 | Train Loss: 0.9444 | Train Loss Unweighted: 0.4557 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1201 | Classification: 0.0924 | Classification_binary: 0.1605 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1465 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0482 | Regression_eta: 0.0257 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0524 | LR: 2.74e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.3607s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7670 | max_loss: 0.2318 | max_backward: 1.8405 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5820.9 | uss_MB: 4694.9 | child_rss_MB: 6312.3 | allocated_MB: 497.5 | reserved_MB: 26942.0 | inactive_split_MB: 1130.5 +rank4 [2026-10-29 19:26:41,940] INFO : Step 35600/50000 rank4 | Train Loss: 0.9280 | Train Loss Unweighted: 0.4451 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1095 | Classification: 0.0861 | Classification_binary: 0.1334 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0538 | Regression_eta: 0.0333 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0690 | LR: 2.72e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7402s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7460 | max_loss: 0.1440 | max_backward: 1.8206 | max_optimizer_step: 0.2911 | max_tensorboard_logging: 0.6036 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 5820.5 | uss_MB: 3919.7 | child_rss_MB: 6316.0 | allocated_MB: 496.1 | reserved_MB: 26942.0 | inactive_split_MB: 907.6 +rank4 [2026-10-29 19:31:29,665] INFO : Step 35700/50000 rank4 | Train Loss: 0.9116 | Train Loss Unweighted: 0.4301 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1148 | Classification: 0.0829 | Classification_binary: 0.1432 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0998 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1386 | Regression_cos_phi: 0.0338 | Regression_energy: 0.0591 | Regression_eta: 0.0317 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0557 | LR: 2.70e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6310s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8312 | max_loss: 0.2568 | max_backward: 2.2622 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5823.2 | uss_MB: 4699.5 | child_rss_MB: 6323.6 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1241.0 +rank4 [2026-10-29 19:36:11,896] INFO : Step 35800/50000 rank4 | Train Loss: 1.0095 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1252 | Classification: 0.0857 | Classification_binary: 0.1786 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1066 | Regression_eta: 0.0014 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.2160 | Regression_cos_phi: 0.0397 | Regression_energy: 0.0557 | Regression_eta: 0.0421 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0796 | LR: 2.68e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.1754s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7644 | max_loss: 0.5090 | max_backward: 1.8855 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4861 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5816.8 | uss_MB: 4694.8 | child_rss_MB: 6319.7 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1129.0 +rank4 [2026-10-29 19:40:48,762] INFO : Step 35900/50000 rank4 | Train Loss: 0.9596 | Train Loss Unweighted: 0.4571 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1173 | Classification: 0.0913 | Classification_binary: 0.1468 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0014 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0364 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0446 | Regression_eta: 0.0448 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0496 | LR: 2.65e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.0313s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7217 | max_loss: 0.1443 | max_backward: 1.6894 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.4586 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5816.9 | uss_MB: 4695.3 | child_rss_MB: 6319.9 | allocated_MB: 495.4 | reserved_MB: 29024.0 | inactive_split_MB: 934.6 +rank4 [2026-10-29 19:45:19,092] INFO : Step 36000/50000 rank4 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0912 | Classification: 0.0938 | Classification_binary: 0.1314 | HitGrouping: 0.1587 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1135 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1471 | Regression_cos_phi: 0.0392 | Regression_energy: 0.0631 | Regression_eta: 0.0356 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0526 | LR: 2.63e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3007s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0073 | max_forward: 0.7630 | max_loss: 0.4769 | max_backward: 1.8869 | max_optimizer_step: 0.2587 | max_tensorboard_logging: 0.5842 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5815.2 | uss_MB: 4695.9 | child_rss_MB: 6324.3 | allocated_MB: 495.0 | reserved_MB: 29024.0 | inactive_split_MB: 937.5 +rank4 [2026-10-29 19:45:19,981] INFO : Running validation on rank4 +rank4 [2026-10-29 19:45:19,982] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 19:45:19,982] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 19:46:10,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:46:11,093] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 19:46:35,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 19:46:36,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 19:46:37,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 19:46:38,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 19:46:48,806] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 19:46:49,454] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:46:49,932] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 19:47:05,880] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 19:47:06,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 19:47:07,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 19:47:08,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 19:47:18,746] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 19:47:19,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 19:47:19,885] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 19:47:35,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 19:47:36,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 19:47:37,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 19:47:37,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 19:47:47,893] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 19:53:12,007] INFO : Step 36100/50000 rank4 | Train Loss: 0.9292 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1100 | Classification: 0.0906 | Classification_binary: 0.1490 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.1522 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0550 | Regression_eta: 0.0377 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0752 | LR: 2.61e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7838s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7768 | max_loss: 0.2455 | max_backward: 1.8077 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 1.0653 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5880.7 | uss_MB: 4756.1 | child_rss_MB: 6293.3 | allocated_MB: 499.2 | reserved_MB: 26012.0 | inactive_split_MB: 1080.2 +rank4 [2026-10-29 19:58:21,469] INFO : Step 36200/50000 rank4 | Train Loss: 0.9030 | Train Loss Unweighted: 0.4308 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1163 | Classification: 0.0762 | Classification_binary: 0.1596 | HitGrouping: 0.1506 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0559 | Regression_eta: 0.0307 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0453 | LR: 2.59e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5357s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0777 | max_forward: 0.8125 | max_loss: 0.1541 | max_backward: 1.8492 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 5884.8 | uss_MB: 4767.2 | child_rss_MB: 6290.6 | allocated_MB: 494.4 | reserved_MB: 26016.0 | inactive_split_MB: 1377.4 +rank4 [2026-10-29 20:03:20,658] INFO : Step 36300/50000 rank4 | Train Loss: 0.9576 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1236 | Classification: 0.0876 | Classification_binary: 0.1347 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1100 | Regression_eta: 0.0017 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1432 | Regression_cos_phi: 0.0779 | Regression_energy: 0.0626 | Regression_eta: 0.0408 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0692 | LR: 2.57e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.4526s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.8065 | max_loss: 0.1484 | max_backward: 1.9315 | max_optimizer_step: 0.1930 | max_tensorboard_logging: 0.6801 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5878.7 | uss_MB: 4768.0 | child_rss_MB: 6273.7 | allocated_MB: 496.0 | reserved_MB: 28008.0 | inactive_split_MB: 1156.5 +rank4 [2026-10-29 20:08:10,865] INFO : Step 36400/50000 rank4 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4147 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.0999 | Classification: 0.0796 | Classification_binary: 0.1372 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1389 | Regression_cos_phi: 0.0837 | Regression_energy: 0.0529 | Regression_eta: 0.0803 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0788 | LR: 2.55e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.8729s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0192 | max_forward: 1.1829 | max_loss: 0.3790 | max_backward: 3.0917 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 5880.4 | uss_MB: 4768.9 | child_rss_MB: 6270.8 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 845.2 +rank4 [2026-10-29 20:12:41,929] INFO : Step 36500/50000 rank4 | Train Loss: 0.9762 | Train Loss Unweighted: 0.4543 | Train Loss Components: Auxiliary: 0.2088 | Cardinality: 0.1239 | Classification: 0.0805 | Classification_binary: 0.1682 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1644 | Regression_cos_phi: 0.0361 | Regression_energy: 0.0495 | Regression_eta: 0.0239 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0420 | LR: 2.52e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2736s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7782 | max_loss: 0.1554 | max_backward: 1.9378 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.5103 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5877.6 | uss_MB: 4769.9 | child_rss_MB: 6273.0 | allocated_MB: 499.0 | reserved_MB: 30008.0 | inactive_split_MB: 913.8 +rank4 [2026-10-29 20:17:47,229] INFO : Step 36600/50000 rank4 | Train Loss: 0.8699 | Train Loss Unweighted: 0.4190 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.1138 | Classification: 0.0734 | Classification_binary: 0.1317 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0010 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1192 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0549 | Regression_eta: 0.0305 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0597 | LR: 2.50e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.2970s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7561 | max_loss: 0.1454 | max_backward: 1.9017 | max_optimizer_step: 0.5115 | max_tensorboard_logging: 0.5672 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5872.6 | uss_MB: 4766.8 | child_rss_MB: 6277.6 | allocated_MB: 494.4 | reserved_MB: 30008.0 | inactive_split_MB: 856.0 +rank4 [2026-10-29 20:22:56,452] INFO : Step 36700/50000 rank4 | Train Loss: 0.9065 | Train Loss Unweighted: 0.4327 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.1102 | Classification: 0.0849 | Classification_binary: 0.1262 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1087 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0610 | Regression_eta: 0.0763 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0775 | LR: 2.48e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.0793s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.7169 | max_loss: 0.1507 | max_backward: 1.7104 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5871.4 | uss_MB: 4766.4 | child_rss_MB: 6270.7 | allocated_MB: 496.5 | reserved_MB: 30008.0 | inactive_split_MB: 1154.6 +rank4 [2026-10-29 20:28:04,049] INFO : Step 36800/50000 rank4 | Train Loss: 0.9068 | Train Loss Unweighted: 0.4227 | Train Loss Components: Auxiliary: 0.1925 | Cardinality: 0.1100 | Classification: 0.0739 | Classification_binary: 0.1571 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0945 | Regression_eta: 0.0013 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0514 | Regression_eta: 0.0343 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0523 | LR: 2.46e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4665s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8487 | max_loss: 0.1525 | max_backward: 2.1121 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5877.1 | uss_MB: 4770.1 | child_rss_MB: 6274.6 | allocated_MB: 499.3 | reserved_MB: 30008.0 | inactive_split_MB: 939.6 +rank4 [2026-10-29 20:33:09,546] INFO : Step 36900/50000 rank4 | Train Loss: 0.8934 | Train Loss Unweighted: 0.4126 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1194 | Classification: 0.0721 | Classification_binary: 0.1507 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0013 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1695 | Regression_cos_phi: 0.0453 | Regression_energy: 0.0551 | Regression_eta: 0.0522 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0467 | LR: 2.44e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.5621s | Phase Times: max_pre_step_barrier: 0.0139 | max_batch_to_device: 0.0068 | max_forward: 1.0975 | max_loss: 0.2241 | max_backward: 2.7278 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.6216 | max_tensorboard_flush: 0.0085 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5871.7 | uss_MB: 4750.7 | child_rss_MB: 6260.8 | allocated_MB: 498.9 | reserved_MB: 30008.0 | inactive_split_MB: 771.1 +rank4 [2026-10-29 20:38:05,959] INFO : Step 37000/50000 rank4 | Train Loss: 0.9040 | Train Loss Unweighted: 0.4182 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1192 | Classification: 0.0807 | Classification_binary: 0.1318 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1240 | Regression_cos_phi: 0.1137 | Regression_energy: 0.0638 | Regression_eta: 0.0830 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0688 | LR: 2.42e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.8499s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7055 | max_loss: 0.1505 | max_backward: 1.5664 | max_optimizer_step: 0.1900 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5866.9 | uss_MB: 4763.2 | child_rss_MB: 6273.0 | allocated_MB: 493.9 | reserved_MB: 30008.0 | inactive_split_MB: 1169.8 +rank4 [2026-10-29 20:38:06,932] INFO : Running validation on rank4 +rank4 [2026-10-29 20:38:06,932] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 20:38:06,932] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 20:39:05,980] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:39:06,532] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 20:39:31,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 20:39:32,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 20:39:33,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 20:39:34,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 20:39:44,461] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 20:39:45,043] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:39:45,533] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 20:40:02,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 20:40:03,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 20:40:04,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 20:40:05,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 20:40:15,440] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 20:40:16,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 20:40:16,887] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 20:40:32,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 20:40:33,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 20:40:34,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 20:40:35,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 20:40:45,074] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 20:45:59,177] INFO : Step 37100/50000 rank4 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4614 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1255 | Classification: 0.0999 | Classification_binary: 0.1492 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1041 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0550 | Regression_eta: 0.0296 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0499 | LR: 2.40e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8441s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0061 | max_forward: 0.8553 | max_loss: 0.1461 | max_backward: 2.1723 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5959.4 | uss_MB: 4852.0 | child_rss_MB: 6311.9 | allocated_MB: 499.3 | reserved_MB: 25668.0 | inactive_split_MB: 852.4 +rank4 [2026-10-29 20:50:39,579] INFO : Step 37200/50000 rank4 | Train Loss: 0.9288 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.1928 | Cardinality: 0.1108 | Classification: 0.0836 | Classification_binary: 0.1643 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0015 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1655 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0553 | Regression_eta: 0.0548 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0696 | LR: 2.38e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6399s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8460 | max_loss: 0.2603 | max_backward: 2.0959 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5966.0 | uss_MB: 4861.5 | child_rss_MB: 6312.1 | allocated_MB: 497.9 | reserved_MB: 25674.0 | inactive_split_MB: 846.0 +rank4 [2026-10-29 20:55:11,586] INFO : Step 37300/50000 rank4 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1275 | Classification: 0.0781 | Classification_binary: 0.1424 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0982 | Regression_eta: 0.0013 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1277 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0553 | Regression_eta: 0.0263 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0571 | LR: 2.36e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.0202s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7653 | max_loss: 0.1472 | max_backward: 1.7105 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.4778 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0692 | Memory: rss_MB: 5962.2 | uss_MB: 4859.4 | child_rss_MB: 6302.2 | allocated_MB: 497.3 | reserved_MB: 27616.0 | inactive_split_MB: 974.4 +rank4 [2026-10-29 21:00:09,947] INFO : Step 37400/50000 rank4 | Train Loss: 0.8836 | Train Loss Unweighted: 0.4228 | Train Loss Components: Auxiliary: 0.1825 | Cardinality: 0.0983 | Classification: 0.0851 | Classification_binary: 0.1289 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1025 | Regression_eta: 0.0015 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0386 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0683 | Regression_eta: 0.0249 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0609 | LR: 2.34e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5374s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7611 | max_loss: 0.1956 | max_backward: 1.9276 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7505 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5960.0 | uss_MB: 4860.1 | child_rss_MB: 6301.8 | allocated_MB: 495.0 | reserved_MB: 27690.0 | inactive_split_MB: 915.5 +rank4 [2026-10-29 21:05:01,763] INFO : Step 37500/50000 rank4 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1260 | Classification: 0.0804 | Classification_binary: 0.1577 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1005 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1746 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0529 | Regression_eta: 0.0317 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0654 | LR: 2.32e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4450s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.9030 | max_loss: 0.1494 | max_backward: 1.9871 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5938.4 | uss_MB: 4859.4 | child_rss_MB: 6224.4 | allocated_MB: 496.1 | reserved_MB: 27690.0 | inactive_split_MB: 923.5 +rank4 [2026-10-29 21:09:54,197] INFO : Step 37600/50000 rank4 | Train Loss: 0.9091 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.0995 | Classification: 0.0683 | Classification_binary: 0.1534 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0957 | Regression_eta: 0.0015 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0323 | Classification_binary: 0.1621 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0594 | Regression_eta: 0.0401 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0597 | LR: 2.30e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3487s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0070 | max_forward: 0.8280 | max_loss: 0.1484 | max_backward: 1.8328 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5933.4 | uss_MB: 4856.7 | child_rss_MB: 6230.2 | allocated_MB: 497.1 | reserved_MB: 27690.0 | inactive_split_MB: 927.5 +rank4 [2026-10-29 21:14:45,920] INFO : Step 37700/50000 rank4 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3995 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.0864 | Classification: 0.0813 | Classification_binary: 0.1231 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0012 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1086 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0548 | Regression_eta: 0.0321 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0643 | LR: 2.28e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.4210s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7582 | max_loss: 0.2486 | max_backward: 1.9197 | max_optimizer_step: 0.3005 | max_tensorboard_logging: 0.5767 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5802.8 | uss_MB: 4857.6 | child_rss_MB: 5767.0 | allocated_MB: 495.2 | reserved_MB: 29766.0 | inactive_split_MB: 925.5 +rank4 [2026-10-29 21:19:50,095] INFO : Step 37800/50000 rank4 | Train Loss: 0.8717 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1057 | Classification: 0.0822 | Classification_binary: 0.1363 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0014 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0519 | Regression_eta: 0.0290 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0364 | LR: 2.26e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6242s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 1.2105 | max_loss: 0.2449 | max_backward: 2.0574 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 5800.5 | uss_MB: 4856.8 | child_rss_MB: 5763.9 | allocated_MB: 498.3 | reserved_MB: 29766.0 | inactive_split_MB: 930.0 +rank4 [2026-10-29 21:24:49,255] INFO : Step 37900/50000 rank4 | Train Loss: 0.9843 | Train Loss Unweighted: 0.4684 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1353 | Classification: 0.0912 | Classification_binary: 0.1705 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0014 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.1788 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0515 | Regression_eta: 0.0343 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0551 | LR: 2.24e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2091s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0297 | max_forward: 0.8081 | max_loss: 0.1547 | max_backward: 2.6708 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5096 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5782.9 | uss_MB: 4859.7 | child_rss_MB: 5732.7 | allocated_MB: 495.6 | reserved_MB: 29766.0 | inactive_split_MB: 912.0 +rank4 [2026-10-29 21:29:44,506] INFO : Step 38000/50000 rank4 | Train Loss: 0.8556 | Train Loss Unweighted: 0.4002 | Train Loss Components: Auxiliary: 0.1833 | Cardinality: 0.1090 | Classification: 0.0730 | Classification_binary: 0.1293 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0972 | Regression_eta: 0.0011 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1128 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0432 | Regression_eta: 0.0266 | Regression_pt: 0.0460 | Regression_sin_phi: 0.0478 | LR: 2.22e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1651s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7330 | max_loss: 0.2284 | max_backward: 1.8082 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5778.8 | uss_MB: 4857.4 | child_rss_MB: 5727.0 | allocated_MB: 494.5 | reserved_MB: 29766.0 | inactive_split_MB: 847.3 +rank4 [2026-10-29 21:29:45,441] INFO : Running validation on rank4 +rank4 [2026-10-29 21:29:45,441] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 21:29:45,441] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 21:30:41,619] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:30:42,096] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 21:31:09,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 21:31:10,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 21:31:11,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 21:31:12,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 21:31:22,454] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 21:31:23,195] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:31:23,629] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 21:31:39,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 21:31:40,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 21:31:41,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 21:31:42,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 21:31:52,642] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 21:31:52,734] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 21:31:53,378] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 21:32:09,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 21:32:10,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 21:32:10,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 21:32:11,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 21:32:21,679] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 21:37:39,070] INFO : Step 38100/50000 rank4 | Train Loss: 0.9020 | Train Loss Unweighted: 0.4337 | Train Loss Components: Auxiliary: 0.1907 | Cardinality: 0.1231 | Classification: 0.0888 | Classification_binary: 0.1339 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0014 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0544 | Regression_eta: 0.0371 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0512 | LR: 2.20e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8730s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0963 | max_forward: 0.8445 | max_loss: 0.1406 | max_backward: 1.9197 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.9697 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7028.7 | uss_MB: 6098.3 | child_rss_MB: 5734.5 | allocated_MB: 496.0 | reserved_MB: 27516.0 | inactive_split_MB: 970.5 +rank4 [2026-10-29 21:42:37,686] INFO : Step 38200/50000 rank4 | Train Loss: 0.9174 | Train Loss Unweighted: 0.4351 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.0971 | Classification: 0.0888 | Classification_binary: 0.1526 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0948 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1368 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0526 | Regression_eta: 0.0240 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0449 | LR: 2.18e-04 | DataLoad Time: 0.0176s | Model Forward Time: 6.4455s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.8480 | max_loss: 0.2651 | max_backward: 3.1905 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.9151 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5881.2 | uss_MB: 4342.7 | child_rss_MB: 5733.1 | allocated_MB: 499.7 | reserved_MB: 27516.0 | inactive_split_MB: 848.5 +rank4 [2026-10-29 21:47:30,432] INFO : Step 38300/50000 rank4 | Train Loss: 0.9342 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1998 | Cardinality: 0.1099 | Classification: 0.0869 | Classification_binary: 0.1470 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0013 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1638 | Regression_cos_phi: 0.1025 | Regression_energy: 0.0543 | Regression_eta: 0.0586 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0626 | LR: 2.16e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.6846s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0664 | max_forward: 0.8187 | max_loss: 0.1464 | max_backward: 1.8154 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.9517 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5875.2 | uss_MB: 4949.5 | child_rss_MB: 5725.9 | allocated_MB: 495.5 | reserved_MB: 27516.0 | inactive_split_MB: 1085.5 +rank4 [2026-10-29 21:52:14,429] INFO : Step 38400/50000 rank4 | Train Loss: 0.8281 | Train Loss Unweighted: 0.4036 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0915 | Classification: 0.0723 | Classification_binary: 0.1160 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1069 | Regression_eta: 0.0012 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1071 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0584 | Regression_eta: 0.0316 | Regression_pt: 0.0618 | Regression_sin_phi: 0.0583 | LR: 2.14e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.7622s | Phase Times: max_pre_step_barrier: 0.0868 | max_batch_to_device: 0.0062 | max_forward: 0.9937 | max_loss: 0.2500 | max_backward: 1.9690 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5872.2 | uss_MB: 4950.8 | child_rss_MB: 5730.9 | allocated_MB: 496.1 | reserved_MB: 27516.0 | inactive_split_MB: 972.6 +rank4 [2026-10-29 21:57:09,317] INFO : Step 38500/50000 rank4 | Train Loss: 0.9386 | Train Loss Unweighted: 0.4484 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1248 | Classification: 0.1044 | Classification_binary: 0.1389 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1277 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0383 | Regression_eta: 0.0259 | Regression_pt: 0.0419 | Regression_sin_phi: 0.0373 | LR: 2.12e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2040s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.7200 | max_loss: 0.1386 | max_backward: 1.5323 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5868.3 | uss_MB: 4952.0 | child_rss_MB: 5707.9 | allocated_MB: 496.0 | reserved_MB: 27520.0 | inactive_split_MB: 776.0 +rank4 [2026-10-29 22:01:43,605] INFO : Step 38600/50000 rank4 | Train Loss: 0.9227 | Train Loss Unweighted: 0.4436 | Train Loss Components: Auxiliary: 0.1984 | Cardinality: 0.1056 | Classification: 0.0788 | Classification_binary: 0.1660 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0979 | Regression_eta: 0.0013 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0500 | Regression_eta: 0.0349 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0645 | LR: 2.11e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4277s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8785 | max_loss: 0.1473 | max_backward: 2.0276 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5861.9 | uss_MB: 4951.4 | child_rss_MB: 5677.1 | allocated_MB: 499.0 | reserved_MB: 29592.0 | inactive_split_MB: 827.5 +rank4 [2026-10-29 22:06:19,439] INFO : Step 38700/50000 rank4 | Train Loss: 0.8153 | Train Loss Unweighted: 0.3795 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0947 | Classification: 0.0722 | Classification_binary: 0.1256 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0893 | Regression_eta: 0.0013 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1320 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0600 | Regression_eta: 0.0408 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0439 | LR: 2.09e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2120s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7251 | max_loss: 0.2433 | max_backward: 1.8091 | max_optimizer_step: 0.1965 | max_tensorboard_logging: 0.4744 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5858.3 | uss_MB: 4950.6 | child_rss_MB: 5664.7 | allocated_MB: 493.1 | reserved_MB: 29592.0 | inactive_split_MB: 1080.0 +rank4 [2026-10-29 22:11:06,083] INFO : Step 38800/50000 rank4 | Train Loss: 0.8386 | Train Loss Unweighted: 0.4077 | Train Loss Components: Auxiliary: 0.1776 | Cardinality: 0.0905 | Classification: 0.0849 | Classification_binary: 0.1308 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0012 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1331 | Regression_cos_phi: 0.0402 | Regression_energy: 0.0464 | Regression_eta: 0.0375 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0418 | LR: 2.07e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1580s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.7346 | max_loss: 0.1426 | max_backward: 1.8320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5857.6 | uss_MB: 4949.0 | child_rss_MB: 5646.6 | allocated_MB: 497.9 | reserved_MB: 29592.0 | inactive_split_MB: 1086.4 +rank4 [2026-10-29 22:15:54,682] INFO : Step 38900/50000 rank4 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1916 | Cardinality: 0.1035 | Classification: 0.0839 | Classification_binary: 0.1479 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0014 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0603 | Regression_energy: 0.0501 | Regression_eta: 0.0449 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0614 | LR: 2.05e-04 | DataLoad Time: 0.0212s | Model Forward Time: 5.8769s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0144 | max_forward: 2.1610 | max_loss: 0.2966 | max_backward: 2.8203 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4876 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 5858.5 | uss_MB: 4951.7 | child_rss_MB: 5647.2 | allocated_MB: 500.0 | reserved_MB: 29592.0 | inactive_split_MB: 849.6 +rank4 [2026-10-29 22:20:36,724] INFO : Step 39000/50000 rank4 | Train Loss: 0.8505 | Train Loss Unweighted: 0.4196 | Train Loss Components: Auxiliary: 0.1813 | Cardinality: 0.0941 | Classification: 0.0808 | Classification_binary: 0.1368 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0993 | Regression_eta: 0.0015 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1238 | Regression_cos_phi: 0.0456 | Regression_energy: 0.0643 | Regression_eta: 0.0263 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0269 | LR: 2.03e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2854s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7392 | max_loss: 0.2445 | max_backward: 1.8584 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5844.6 | uss_MB: 4939.8 | child_rss_MB: 5638.4 | allocated_MB: 496.8 | reserved_MB: 29592.0 | inactive_split_MB: 1086.0 +rank4 [2026-10-29 22:20:37,630] INFO : Running validation on rank4 +rank4 [2026-10-29 22:20:37,630] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 22:20:37,630] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 22:21:31,240] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:21:31,760] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 22:21:58,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 22:21:59,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 22:22:00,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 22:22:01,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 22:22:11,605] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 22:22:12,679] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:22:13,176] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 22:22:29,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 22:22:30,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 22:22:31,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 22:22:32,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 22:22:42,393] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 22:22:43,329] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 22:22:43,800] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 22:22:59,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 22:23:00,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 22:23:01,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 22:23:02,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 22:23:12,036] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-29 22:28:30,319] INFO : Step 39100/50000 rank4 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1110 | Classification: 0.0839 | Classification_binary: 0.1198 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.1226 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0469 | Regression_eta: 0.0383 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0535 | LR: 2.01e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5488s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7800 | max_loss: 0.2457 | max_backward: 1.9719 | max_optimizer_step: 0.3248 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 7010.9 | uss_MB: 6103.0 | child_rss_MB: 5608.2 | allocated_MB: 497.5 | reserved_MB: 24374.0 | inactive_split_MB: 923.6 +rank4 [2026-10-29 22:33:29,807] INFO : Step 39200/50000 rank4 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4242 | Train Loss Components: Auxiliary: 0.1863 | Cardinality: 0.1210 | Classification: 0.0912 | Classification_binary: 0.1382 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0614 | Regression_eta: 0.0423 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0569 | LR: 2.00e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2593s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0324 | max_forward: 0.7817 | max_loss: 0.1452 | max_backward: 1.9286 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5940.0 | uss_MB: 5036.4 | child_rss_MB: 5609.8 | allocated_MB: 497.9 | reserved_MB: 26338.0 | inactive_split_MB: 912.4 +rank4 [2026-10-29 22:38:28,992] INFO : Step 39300/50000 rank4 | Train Loss: 0.9298 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1080 | Classification: 0.0882 | Classification_binary: 0.1594 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0951 | Regression_eta: 0.0013 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1475 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0709 | Regression_eta: 0.0380 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0673 | LR: 1.98e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.1940s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7922 | max_loss: 0.2572 | max_backward: 1.8558 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 5927.1 | uss_MB: 5036.1 | child_rss_MB: 5613.3 | allocated_MB: 499.4 | reserved_MB: 26338.0 | inactive_split_MB: 1408.0 +rank4 [2026-10-29 22:43:21,461] INFO : Step 39400/50000 rank4 | Train Loss: 0.8421 | Train Loss Unweighted: 0.3964 | Train Loss Components: Auxiliary: 0.1755 | Cardinality: 0.1006 | Classification: 0.0679 | Classification_binary: 0.1346 | HitGrouping: 0.1534 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0395 | Classification_binary: 0.1085 | Regression_cos_phi: 0.0328 | Regression_energy: 0.0390 | Regression_eta: 0.0317 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0284 | LR: 1.96e-04 | DataLoad Time: 0.0192s | Model Forward Time: 6.2892s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1084 | max_forward: 1.1740 | max_loss: 0.3046 | max_backward: 2.6778 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8613 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5928.1 | uss_MB: 4351.2 | child_rss_MB: 5609.1 | allocated_MB: 495.0 | reserved_MB: 26338.0 | inactive_split_MB: 1095.6 +rank4 [2026-10-29 22:48:17,290] INFO : Step 39500/50000 rank4 | Train Loss: 0.8539 | Train Loss Unweighted: 0.4179 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0879 | Classification: 0.0889 | Classification_binary: 0.1243 | HitGrouping: 0.1565 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1216 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0442 | Regression_eta: 0.0309 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0456 | LR: 1.94e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6151s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7373 | max_loss: 0.1406 | max_backward: 1.8719 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.8502 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5922.7 | uss_MB: 5037.5 | child_rss_MB: 5598.5 | allocated_MB: 492.8 | reserved_MB: 28318.0 | inactive_split_MB: 1078.3 +rank4 [2026-10-29 22:53:41,206] INFO : Step 39600/50000 rank4 | Train Loss: 0.9371 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1161 | Classification: 0.0813 | Classification_binary: 0.1596 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1039 | Regression_eta: 0.0013 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0413 | Regression_energy: 0.0560 | Regression_eta: 0.0293 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0387 | LR: 1.93e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.3245s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0173 | max_forward: 1.4879 | max_loss: 0.2520 | max_backward: 1.9863 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.8346 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5921.1 | uss_MB: 5038.4 | child_rss_MB: 5602.9 | allocated_MB: 496.0 | reserved_MB: 28318.0 | inactive_split_MB: 858.4 +rank4 [2026-10-29 22:58:48,059] INFO : Step 39700/50000 rank4 | Train Loss: 0.9204 | Train Loss Unweighted: 0.4427 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1101 | Classification: 0.0841 | Classification_binary: 0.1567 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0997 | Regression_eta: 0.0013 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0442 | Regression_eta: 0.0401 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0508 | LR: 1.91e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2755s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.7571 | max_loss: 0.4946 | max_backward: 1.9147 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 5909.3 | uss_MB: 5037.4 | child_rss_MB: 5602.2 | allocated_MB: 497.0 | reserved_MB: 30324.0 | inactive_split_MB: 1175.5 +rank4 [2026-10-29 23:04:38,179] INFO : Step 39800/50000 rank4 | Train Loss: 0.8155 | Train Loss Unweighted: 0.3780 | Train Loss Components: Auxiliary: 0.1674 | Cardinality: 0.0963 | Classification: 0.0643 | Classification_binary: 0.1275 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1211 | Regression_cos_phi: 0.0409 | Regression_energy: 0.0472 | Regression_eta: 0.0247 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0580 | LR: 1.89e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.0216s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7197 | max_loss: 0.1489 | max_backward: 1.6690 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.4760 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1068 | Memory: rss_MB: 5897.7 | uss_MB: 5034.7 | child_rss_MB: 5594.3 | allocated_MB: 494.4 | reserved_MB: 30324.0 | inactive_split_MB: 897.3 +rank4 [2026-10-29 23:09:43,445] INFO : Step 39900/50000 rank4 | Train Loss: 0.8844 | Train Loss Unweighted: 0.3998 | Train Loss Components: Auxiliary: 0.1871 | Cardinality: 0.1077 | Classification: 0.0758 | Classification_binary: 0.1443 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0883 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1511 | Regression_cos_phi: 0.1012 | Regression_energy: 0.0521 | Regression_eta: 0.0516 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0848 | LR: 1.88e-04 | DataLoad Time: 0.0101s | Model Forward Time: 4.9798s | Phase Times: max_pre_step_barrier: 0.0268 | max_batch_to_device: 0.0727 | max_forward: 1.7016 | max_loss: 0.4968 | max_backward: 2.0829 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.4776 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5898.5 | uss_MB: 5037.5 | child_rss_MB: 5601.6 | allocated_MB: 496.1 | reserved_MB: 30324.0 | inactive_split_MB: 1077.6 +rank4 [2026-10-29 23:14:38,167] INFO : Step 40000/50000 rank4 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1170 | Classification: 0.0871 | Classification_binary: 0.1550 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0460 | Regression_eta: 0.0402 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0562 | LR: 1.86e-04 | DataLoad Time: 0.0124s | Model Forward Time: 5.4159s | Phase Times: max_pre_step_barrier: 0.1084 | max_batch_to_device: 0.0070 | max_forward: 1.1725 | max_loss: 0.2555 | max_backward: 2.4297 | max_optimizer_step: 0.2969 | max_tensorboard_logging: 0.6818 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5889.3 | uss_MB: 5028.2 | child_rss_MB: 5597.1 | allocated_MB: 501.7 | reserved_MB: 30324.0 | inactive_split_MB: 1368.0 +rank4 [2026-10-29 23:14:39,118] INFO : Running validation on rank4 +rank4 [2026-10-29 23:14:39,118] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-29 23:14:39,119] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-29 23:15:31,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:15:32,369] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-29 23:16:03,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-29 23:16:04,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-29 23:16:05,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-29 23:16:07,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-29 23:16:17,115] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-29 23:16:28,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:16:28,867] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-29 23:16:45,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-29 23:16:45,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-29 23:16:46,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-29 23:16:47,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-29 23:16:57,913] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-29 23:16:58,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-29 23:16:59,026] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-29 23:17:14,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-29 23:17:15,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-29 23:17:16,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-29 23:17:17,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-29 23:17:27,568] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-29 23:22:41,682] INFO : Step 40100/50000 rank4 | Train Loss: 0.8115 | Train Loss Unweighted: 0.3794 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.1028 | Classification: 0.0708 | Classification_binary: 0.1239 | HitGrouping: 0.1460 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0909 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1373 | Regression_cos_phi: 0.0419 | Regression_energy: 0.0568 | Regression_eta: 0.0371 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0602 | LR: 1.84e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.0780s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.1076 | max_forward: 0.8426 | max_loss: 0.1437 | max_backward: 1.8847 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 7273.7 | uss_MB: 5477.4 | child_rss_MB: 5617.3 | allocated_MB: 493.6 | reserved_MB: 28004.0 | inactive_split_MB: 975.8 +rank4 [2026-10-29 23:27:53,247] INFO : Step 40200/50000 rank4 | Train Loss: 0.8264 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1679 | Cardinality: 0.0984 | Classification: 0.0703 | Classification_binary: 0.1210 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0913 | Regression_eta: 0.0013 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1133 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0452 | Regression_eta: 0.0336 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0468 | LR: 1.83e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.8932s | Phase Times: max_pre_step_barrier: 0.0270 | max_batch_to_device: 0.0062 | max_forward: 0.8324 | max_loss: 0.1802 | max_backward: 2.0569 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0152 | max_post_step_barrier: 0.0867 | Memory: rss_MB: 5955.5 | uss_MB: 5094.1 | child_rss_MB: 5625.1 | allocated_MB: 497.1 | reserved_MB: 28008.0 | inactive_split_MB: 935.5 +rank4 [2026-10-29 23:33:22,818] INFO : Step 40300/50000 rank4 | Train Loss: 0.9351 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1181 | Classification: 0.0959 | Classification_binary: 0.1652 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0014 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0425 | Regression_eta: 0.0520 | Regression_pt: 0.0440 | Regression_sin_phi: 0.0698 | LR: 1.81e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.3111s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8191 | max_loss: 0.1494 | max_backward: 1.9817 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 0.9693 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 5955.1 | uss_MB: 5096.4 | child_rss_MB: 5610.7 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 927.5 +rank4 [2026-10-29 23:38:38,678] INFO : Step 40400/50000 rank4 | Train Loss: 0.9379 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1359 | Classification: 0.0771 | Classification_binary: 0.1554 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0944 | Regression_eta: 0.0014 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0530 | Regression_eta: 0.0270 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0425 | LR: 1.79e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6747s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0500 | max_forward: 0.8111 | max_loss: 0.5067 | max_backward: 1.8144 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5951.4 | uss_MB: 5092.9 | child_rss_MB: 5616.0 | allocated_MB: 497.0 | reserved_MB: 28008.0 | inactive_split_MB: 931.6 +rank4 [2026-10-29 23:43:43,799] INFO : Step 40500/50000 rank4 | Train Loss: 0.8053 | Train Loss Unweighted: 0.3782 | Train Loss Components: Auxiliary: 0.1649 | Cardinality: 0.0788 | Classification: 0.0749 | Classification_binary: 0.1246 | HitGrouping: 0.1502 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1281 | Regression_cos_phi: 0.0680 | Regression_energy: 0.0455 | Regression_eta: 0.0803 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0745 | LR: 1.78e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3978s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.8420 | max_loss: 0.1414 | max_backward: 1.5396 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.9215 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5948.4 | uss_MB: 5092.0 | child_rss_MB: 5597.4 | allocated_MB: 494.5 | reserved_MB: 28012.0 | inactive_split_MB: 979.3 +rank4 [2026-10-29 23:48:43,695] INFO : Step 40600/50000 rank4 | Train Loss: 0.8980 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1108 | Classification: 0.0752 | Classification_binary: 0.1417 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0955 | Regression_eta: 0.0015 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1718 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0556 | Regression_eta: 0.0500 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0703 | LR: 1.76e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3173s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7855 | max_loss: 0.1488 | max_backward: 1.8742 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 5948.2 | uss_MB: 5094.3 | child_rss_MB: 5593.1 | allocated_MB: 499.7 | reserved_MB: 28012.0 | inactive_split_MB: 1102.0 +rank4 [2026-10-29 23:53:39,581] INFO : Step 40700/50000 rank4 | Train Loss: 0.9220 | Train Loss Unweighted: 0.4304 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1148 | Classification: 0.0767 | Classification_binary: 0.1645 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1745 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0428 | Regression_eta: 0.0354 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0552 | LR: 1.75e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.5471s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.8091 | max_loss: 0.2475 | max_backward: 2.0011 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.5972 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5949.6 | uss_MB: 5096.1 | child_rss_MB: 5593.6 | allocated_MB: 498.9 | reserved_MB: 28012.0 | inactive_split_MB: 776.0 +rank4 [2026-10-30 00:01:00,042] INFO : Step 40800/50000 rank4 | Train Loss: 0.8642 | Train Loss Unweighted: 0.4082 | Train Loss Components: Auxiliary: 0.1815 | Cardinality: 0.1013 | Classification: 0.0834 | Classification_binary: 0.1260 | HitGrouping: 0.1507 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0356 | Classification_binary: 0.1285 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0524 | Regression_eta: 0.0370 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0596 | LR: 1.73e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1144s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7033 | max_loss: 0.1373 | max_backward: 1.5042 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1068 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5935.0 | uss_MB: 4327.0 | child_rss_MB: 5586.6 | allocated_MB: 493.1 | reserved_MB: 28012.0 | inactive_split_MB: 897.3 +rank4 [2026-10-30 00:06:14,591] INFO : Step 40900/50000 rank4 | Train Loss: 0.8749 | Train Loss Unweighted: 0.4038 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1070 | Classification: 0.0854 | Classification_binary: 0.1300 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0923 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1017 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0462 | Regression_eta: 0.0297 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0565 | LR: 1.72e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5706s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7515 | max_loss: 0.2142 | max_backward: 1.7950 | max_optimizer_step: 0.4960 | max_tensorboard_logging: 0.9322 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5926.9 | uss_MB: 5094.2 | child_rss_MB: 5563.3 | allocated_MB: 497.9 | reserved_MB: 30012.0 | inactive_split_MB: 893.0 +rank4 [2026-10-30 00:11:07,452] INFO : Step 41000/50000 rank4 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.0956 | Classification: 0.0893 | Classification_binary: 0.1497 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0014 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0607 | Regression_eta: 0.0296 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0548 | LR: 1.70e-04 | DataLoad Time: 0.0118s | Model Forward Time: 5.0878s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0071 | max_forward: 1.1963 | max_loss: 0.1519 | max_backward: 2.0474 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.0118 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 5918.6 | uss_MB: 5085.3 | child_rss_MB: 5547.4 | allocated_MB: 498.7 | reserved_MB: 30012.0 | inactive_split_MB: 1108.0 +rank4 [2026-10-30 00:11:08,373] INFO : Running validation on rank4 +rank4 [2026-10-30 00:11:08,374] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 00:11:08,374] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 00:11:59,245] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:12:00,024] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 00:12:25,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 00:12:26,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 00:12:27,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 00:12:28,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 00:12:38,824] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 00:12:39,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:12:40,281] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 00:12:56,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 00:12:56,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 00:12:57,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 00:12:59,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 00:13:09,199] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 00:13:09,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 00:13:10,648] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 00:13:26,776] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 00:13:27,740] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 00:13:28,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 00:13:29,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 00:13:39,379] INFO : Time taken to make predictions on device 4 is: 0.24 min +rank4 [2026-10-30 00:19:15,373] INFO : Step 41100/50000 rank4 | Train Loss: 0.9430 | Train Loss Unweighted: 0.4319 | Train Loss Components: Auxiliary: 0.1995 | Cardinality: 0.1330 | Classification: 0.0721 | Classification_binary: 0.1579 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0013 | Regression_pt: 0.0998 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0542 | Regression_eta: 0.0359 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0717 | LR: 1.69e-04 | DataLoad Time: 0.0149s | Model Forward Time: 6.3662s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8221 | max_loss: 0.4800 | max_backward: 2.2615 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7242.0 | uss_MB: 5759.2 | child_rss_MB: 5549.6 | allocated_MB: 497.6 | reserved_MB: 23400.0 | inactive_split_MB: 991.8 +rank4 [2026-10-30 00:24:27,079] INFO : Step 41200/50000 rank4 | Train Loss: 0.8544 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1097 | Classification: 0.0734 | Classification_binary: 0.1131 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1015 | Regression_eta: 0.0015 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.0988 | Regression_cos_phi: 0.0305 | Regression_energy: 0.0515 | Regression_eta: 0.0249 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0330 | LR: 1.67e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2540s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7030 | max_loss: 0.2408 | max_backward: 1.7750 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5316 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6039.9 | uss_MB: 5208.8 | child_rss_MB: 5554.6 | allocated_MB: 496.1 | reserved_MB: 27308.0 | inactive_split_MB: 984.0 +rank4 [2026-10-30 00:29:20,952] INFO : Step 41300/50000 rank4 | Train Loss: 0.8221 | Train Loss Unweighted: 0.3852 | Train Loss Components: Auxiliary: 0.1744 | Cardinality: 0.0906 | Classification: 0.0754 | Classification_binary: 0.1369 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0849 | Regression_eta: 0.0012 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1304 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0427 | Regression_eta: 0.0418 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0609 | LR: 1.66e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4687s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8083 | max_loss: 0.1794 | max_backward: 1.9993 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.5906 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6040.0 | uss_MB: 5213.3 | child_rss_MB: 5551.2 | allocated_MB: 497.0 | reserved_MB: 27308.0 | inactive_split_MB: 1229.0 +rank4 [2026-10-30 00:34:25,619] INFO : Step 41400/50000 rank4 | Train Loss: 0.9216 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1171 | Classification: 0.0901 | Classification_binary: 0.1523 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1010 | Regression_eta: 0.0011 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1494 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0538 | Regression_eta: 0.0261 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0655 | LR: 1.64e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6352s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0065 | max_forward: 0.7831 | max_loss: 0.4937 | max_backward: 1.9271 | max_optimizer_step: 0.3071 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6033.3 | uss_MB: 5208.0 | child_rss_MB: 5553.7 | allocated_MB: 498.4 | reserved_MB: 29288.0 | inactive_split_MB: 1180.4 +rank4 [2026-10-30 00:39:29,959] INFO : Step 41500/50000 rank4 | Train Loss: 0.8584 | Train Loss Unweighted: 0.4111 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1030 | Classification: 0.0769 | Classification_binary: 0.1210 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1092 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0557 | Regression_eta: 0.0327 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0388 | LR: 1.63e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.0418s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.1908 | max_loss: 0.3017 | max_backward: 3.1310 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 6009.7 | uss_MB: 5150.0 | child_rss_MB: 5552.7 | allocated_MB: 494.5 | reserved_MB: 29288.0 | inactive_split_MB: 976.9 +rank4 [2026-10-30 00:44:19,404] INFO : Step 41600/50000 rank4 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4026 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1032 | Classification: 0.0762 | Classification_binary: 0.1246 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0992 | Regression_eta: 0.0013 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0326 | Classification_binary: 0.1106 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0403 | Regression_eta: 0.0291 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0417 | LR: 1.61e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9950s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0065 | max_forward: 0.7279 | max_loss: 0.2339 | max_backward: 2.5338 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.4866 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1404 | Memory: rss_MB: 6001.6 | uss_MB: 5206.5 | child_rss_MB: 5546.1 | allocated_MB: 496.4 | reserved_MB: 29288.0 | inactive_split_MB: 761.6 +rank4 [2026-10-30 00:49:10,876] INFO : Step 41700/50000 rank4 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1941 | Cardinality: 0.1112 | Classification: 0.0767 | Classification_binary: 0.1493 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0988 | Regression_eta: 0.0013 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0442 | Regression_eta: 0.0471 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0551 | LR: 1.60e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5068s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7998 | max_loss: 0.1827 | max_backward: 1.9774 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6004.5 | uss_MB: 5210.8 | child_rss_MB: 5548.7 | allocated_MB: 497.0 | reserved_MB: 29288.0 | inactive_split_MB: 1220.0 +rank4 [2026-10-30 00:53:58,971] INFO : Step 41800/50000 rank4 | Train Loss: 0.9125 | Train Loss Unweighted: 0.4285 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.1148 | Classification: 0.0784 | Classification_binary: 0.1537 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0628 | Regression_energy: 0.0415 | Regression_eta: 0.0417 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0685 | LR: 1.58e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8253s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7952 | max_loss: 0.1582 | max_backward: 1.8899 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9392 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1143 | Memory: rss_MB: 5984.7 | uss_MB: 5200.7 | child_rss_MB: 5542.4 | allocated_MB: 499.3 | reserved_MB: 31288.0 | inactive_split_MB: 932.9 +rank4 [2026-10-30 00:59:00,784] INFO : Step 41900/50000 rank4 | Train Loss: 0.8303 | Train Loss Unweighted: 0.3927 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1132 | Classification: 0.0782 | Classification_binary: 0.1167 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0010 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0383 | Classification_binary: 0.1069 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0389 | Regression_eta: 0.0234 | Regression_pt: 0.0411 | Regression_sin_phi: 0.0324 | LR: 1.57e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1524s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7413 | max_loss: 0.1970 | max_backward: 1.7614 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5208 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5992.5 | uss_MB: 5207.2 | child_rss_MB: 5522.2 | allocated_MB: 497.1 | reserved_MB: 31288.0 | inactive_split_MB: 1188.0 +rank4 [2026-10-30 01:03:51,615] INFO : Step 42000/50000 rank4 | Train Loss: 0.8724 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1847 | Cardinality: 0.0893 | Classification: 0.0874 | Classification_binary: 0.1377 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0958 | Regression_eta: 0.0012 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1581 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0422 | Regression_eta: 0.0330 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0507 | LR: 1.56e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7566s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1147 | max_forward: 0.8308 | max_loss: 0.1392 | max_backward: 1.8336 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5996.6 | uss_MB: 5209.4 | child_rss_MB: 5540.2 | allocated_MB: 496.3 | reserved_MB: 31288.0 | inactive_split_MB: 930.5 +rank4 [2026-10-30 01:03:52,597] INFO : Running validation on rank4 +rank4 [2026-10-30 01:03:52,597] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 01:03:52,597] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 01:04:46,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:04:46,897] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 01:05:12,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 01:05:14,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 01:05:15,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 01:05:16,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 01:05:26,666] INFO : Time taken to make predictions on device 4 is: 0.26 min +rank4 [2026-10-30 01:05:27,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:05:27,884] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 01:05:44,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 01:05:44,960] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 01:05:45,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 01:05:46,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 01:05:56,859] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 01:05:57,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:05:57,664] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 01:06:13,793] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 01:06:15,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 01:06:16,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 01:06:17,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 01:06:27,197] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 01:11:42,847] INFO : Step 42100/50000 rank4 | Train Loss: 0.9390 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1200 | Classification: 0.0884 | Classification_binary: 0.1619 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0926 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1452 | Regression_cos_phi: 0.0327 | Regression_energy: 0.0432 | Regression_eta: 0.0395 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0559 | LR: 1.54e-04 | DataLoad Time: 0.0176s | Model Forward Time: 6.0858s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0976 | max_forward: 1.0189 | max_loss: 0.3859 | max_backward: 2.5922 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 1.0368 | max_tensorboard_flush: 0.0149 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7501.8 | uss_MB: 4866.0 | child_rss_MB: 5544.9 | allocated_MB: 497.0 | reserved_MB: 25850.0 | inactive_split_MB: 892.8 +rank4 [2026-10-30 01:16:53,956] INFO : Step 42200/50000 rank4 | Train Loss: 0.9057 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1144 | Classification: 0.0879 | Classification_binary: 0.1381 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1483 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0566 | Regression_eta: 0.0412 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0582 | LR: 1.53e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.1537s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7457 | max_loss: 0.1999 | max_backward: 1.6824 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.5916 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6096.0 | uss_MB: 5306.7 | child_rss_MB: 5537.7 | allocated_MB: 497.3 | reserved_MB: 27866.0 | inactive_split_MB: 959.8 +rank4 [2026-10-30 01:18:06,041] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank4 [2026-10-30 01:18:06,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank4 [2026-10-30 01:21:57,451] INFO : Step 42300/50000 rank4 | Train Loss: 0.9052 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1933 | Cardinality: 0.1127 | Classification: 0.0787 | Classification_binary: 0.1569 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0868 | Regression_eta: 0.0013 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0561 | Regression_eta: 0.0365 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0609 | LR: 1.52e-04 | DataLoad Time: 0.0158s | Model Forward Time: 6.1192s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.9764 | max_loss: 0.1904 | max_backward: 2.7671 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.9997 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6118.1 | uss_MB: 5325.9 | child_rss_MB: 5544.6 | allocated_MB: 499.1 | reserved_MB: 27872.0 | inactive_split_MB: 1033.6 +rank4 [2026-10-30 01:26:55,612] INFO : Step 42400/50000 rank4 | Train Loss: 0.9358 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.1976 | Cardinality: 0.1220 | Classification: 0.0815 | Classification_binary: 0.1533 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1698 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0526 | Regression_eta: 0.0375 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0612 | LR: 1.50e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8880s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8701 | max_loss: 0.1465 | max_backward: 1.9416 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.8192 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6099.3 | uss_MB: 5308.3 | child_rss_MB: 5546.3 | allocated_MB: 499.8 | reserved_MB: 29908.0 | inactive_split_MB: 939.2 +rank4 [2026-10-30 01:31:49,012] INFO : Step 42500/50000 rank4 | Train Loss: 0.8126 | Train Loss Unweighted: 0.3791 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0975 | Classification: 0.0765 | Classification_binary: 0.1172 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0918 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0313 | Classification_binary: 0.0908 | Regression_cos_phi: 0.0441 | Regression_energy: 0.0441 | Regression_eta: 0.0224 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0363 | LR: 1.49e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.5918s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0858 | max_forward: 0.8156 | max_loss: 0.1407 | max_backward: 1.8570 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7997 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6101.2 | uss_MB: 5308.6 | child_rss_MB: 5538.8 | allocated_MB: 497.5 | reserved_MB: 29908.0 | inactive_split_MB: 943.4 +rank4 [2026-10-30 01:36:44,519] INFO : Step 42600/50000 rank4 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0772 | Classification_binary: 0.1347 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0012 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1608 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0575 | Regression_eta: 0.0302 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0526 | LR: 1.48e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0404s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7519 | max_loss: 0.1393 | max_backward: 1.8021 | max_optimizer_step: 0.3023 | max_tensorboard_logging: 0.7973 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6101.5 | uss_MB: 4501.2 | child_rss_MB: 5544.2 | allocated_MB: 497.0 | reserved_MB: 29908.0 | inactive_split_MB: 877.2 +rank4 [2026-10-30 01:41:48,354] INFO : Step 42700/50000 rank4 | Train Loss: 0.9129 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1178 | Classification: 0.0751 | Classification_binary: 0.1632 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0858 | Regression_eta: 0.0014 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1336 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0389 | Regression_eta: 0.0278 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0491 | LR: 1.47e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6161s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 1.0869 | max_loss: 0.1547 | max_backward: 1.6854 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7949 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6101.6 | uss_MB: 5308.9 | child_rss_MB: 5545.8 | allocated_MB: 497.3 | reserved_MB: 29908.0 | inactive_split_MB: 953.4 +rank4 [2026-10-30 01:46:39,116] INFO : Step 42800/50000 rank4 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1179 | Classification: 0.0767 | Classification_binary: 0.1362 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0013 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0304 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0483 | Regression_eta: 0.0510 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0461 | LR: 1.45e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3540s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8438 | max_loss: 0.1478 | max_backward: 1.6710 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.6821 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6089.4 | uss_MB: 5295.4 | child_rss_MB: 5546.6 | allocated_MB: 499.0 | reserved_MB: 30126.0 | inactive_split_MB: 875.7 +rank4 [2026-10-30 01:51:40,828] INFO : Step 42900/50000 rank4 | Train Loss: 0.7857 | Train Loss Unweighted: 0.3758 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0951 | Classification: 0.0820 | Classification_binary: 0.1113 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0898 | Regression_eta: 0.0010 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0290 | Classification_binary: 0.0996 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0398 | Regression_eta: 0.0217 | Regression_pt: 0.0421 | Regression_sin_phi: 0.0270 | LR: 1.44e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9239s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8792 | max_loss: 0.1461 | max_backward: 1.9928 | max_optimizer_step: 0.1903 | max_tensorboard_logging: 0.9695 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6098.2 | uss_MB: 5308.5 | child_rss_MB: 5544.6 | allocated_MB: 496.5 | reserved_MB: 30126.0 | inactive_split_MB: 951.8 +rank4 [2026-10-30 01:56:37,111] INFO : Step 43000/50000 rank4 | Train Loss: 0.8899 | Train Loss Unweighted: 0.4207 | Train Loss Components: Auxiliary: 0.1911 | Cardinality: 0.1035 | Classification: 0.0847 | Classification_binary: 0.1529 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1712 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0526 | Regression_eta: 0.0353 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0783 | LR: 1.43e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7374s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1951 | max_backward: 1.9694 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.8418 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6086.1 | uss_MB: 5311.0 | child_rss_MB: 5508.6 | allocated_MB: 495.5 | reserved_MB: 30126.0 | inactive_split_MB: 1120.4 +rank4 [2026-10-30 01:56:38,137] INFO : Running validation on rank4 +rank4 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 01:57:33,181] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:57:33,675] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 01:57:59,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 01:58:00,331] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 01:58:01,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 01:58:02,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 01:58:12,417] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 01:58:13,038] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:58:13,512] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 01:58:29,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 01:58:30,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 01:58:31,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 01:58:32,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 01:58:42,449] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 01:58:43,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 01:58:43,755] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 01:58:59,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 01:59:00,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 01:59:01,061] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 01:59:01,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 01:59:11,912] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 02:04:24,630] INFO : Step 43100/50000 rank4 | Train Loss: 0.8965 | Train Loss Unweighted: 0.4137 | Train Loss Components: Auxiliary: 0.1901 | Cardinality: 0.1074 | Classification: 0.0705 | Classification_binary: 0.1472 | HitGrouping: 0.1435 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0439 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0669 | Regression_eta: 0.0312 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0611 | LR: 1.42e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2199s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.2396 | max_backward: 1.9954 | max_optimizer_step: 0.3209 | max_tensorboard_logging: 1.0803 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7631.1 | uss_MB: 6853.7 | child_rss_MB: 5519.2 | allocated_MB: 496.0 | reserved_MB: 26194.0 | inactive_split_MB: 843.4 +rank4 [2026-10-30 02:09:06,731] INFO : Step 43200/50000 rank4 | Train Loss: 0.8164 | Train Loss Unweighted: 0.3744 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0957 | Classification: 0.0763 | Classification_binary: 0.1197 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1288 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0448 | Regression_eta: 0.0343 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0819 | LR: 1.40e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4237s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7823 | max_loss: 0.4822 | max_backward: 1.6766 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6123.7 | uss_MB: 5349.6 | child_rss_MB: 5511.0 | allocated_MB: 495.8 | reserved_MB: 26686.0 | inactive_split_MB: 848.9 +rank4 [2026-10-30 02:13:48,742] INFO : Step 43300/50000 rank4 | Train Loss: 0.9151 | Train Loss Unweighted: 0.4260 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1202 | Classification: 0.0906 | Classification_binary: 0.1467 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0013 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0521 | Regression_eta: 0.0435 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0476 | LR: 1.39e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.7169s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0155 | max_forward: 0.8938 | max_loss: 0.1464 | max_backward: 2.1010 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.5477 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1243 | Memory: rss_MB: 6097.4 | uss_MB: 5352.9 | child_rss_MB: 5427.7 | allocated_MB: 497.9 | reserved_MB: 28278.0 | inactive_split_MB: 1309.2 +rank4 [2026-10-30 02:18:30,112] INFO : Step 43400/50000 rank4 | Train Loss: 0.9723 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2082 | Cardinality: 0.1222 | Classification: 0.0880 | Classification_binary: 0.1684 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0015 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1758 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0626 | Regression_eta: 0.0368 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0782 | LR: 1.38e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4356s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.5043 | max_backward: 2.0661 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.4824 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6077.9 | uss_MB: 5350.3 | child_rss_MB: 5335.1 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 851.9 +rank4 [2026-10-30 02:23:12,891] INFO : Step 43500/50000 rank4 | Train Loss: 0.8402 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.0947 | Classification: 0.0776 | Classification_binary: 0.1278 | HitGrouping: 0.1501 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0999 | Regression_eta: 0.0012 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0446 | Regression_eta: 0.0271 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0568 | LR: 1.37e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.8639s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8086 | max_loss: 0.1489 | max_backward: 2.0887 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.7978 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1094 | Memory: rss_MB: 6077.6 | uss_MB: 5352.8 | child_rss_MB: 5340.6 | allocated_MB: 498.6 | reserved_MB: 28278.0 | inactive_split_MB: 909.7 +rank4 [2026-10-30 02:28:09,613] INFO : Step 43600/50000 rank4 | Train Loss: 0.7985 | Train Loss Unweighted: 0.3653 | Train Loss Components: Auxiliary: 0.1621 | Cardinality: 0.0887 | Classification: 0.0728 | Classification_binary: 0.1184 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0851 | Regression_eta: 0.0013 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1229 | Regression_cos_phi: 0.0260 | Regression_energy: 0.0468 | Regression_eta: 0.0275 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0373 | LR: 1.36e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.2498s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0170 | max_forward: 1.2777 | max_loss: 0.2064 | max_backward: 1.9835 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 1.1084 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6044.0 | uss_MB: 4957.3 | child_rss_MB: 5260.0 | allocated_MB: 493.5 | reserved_MB: 28278.0 | inactive_split_MB: 1313.2 +rank4 [2026-10-30 02:33:07,295] INFO : Step 43700/50000 rank4 | Train Loss: 0.8599 | Train Loss Unweighted: 0.3884 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1089 | Classification: 0.0797 | Classification_binary: 0.1434 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0812 | Regression_eta: 0.0013 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1495 | Regression_cos_phi: 0.0430 | Regression_energy: 0.0418 | Regression_eta: 0.0384 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0406 | LR: 1.35e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6048s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7902 | max_loss: 0.1752 | max_backward: 1.8774 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8044 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6046.0 | uss_MB: 5350.4 | child_rss_MB: 5241.5 | allocated_MB: 498.0 | reserved_MB: 28278.0 | inactive_split_MB: 1046.0 +rank4 [2026-10-30 02:37:45,726] INFO : Step 43800/50000 rank4 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1875 | Cardinality: 0.1049 | Classification: 0.0814 | Classification_binary: 0.1423 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0907 | Regression_eta: 0.0012 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0472 | Regression_eta: 0.0523 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0547 | LR: 1.34e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5743s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7940 | max_loss: 0.4805 | max_backward: 1.8159 | max_optimizer_step: 0.2941 | max_tensorboard_logging: 0.7850 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6040.0 | uss_MB: 5349.3 | child_rss_MB: 5218.8 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 839.4 +rank4 [2026-10-30 02:42:41,735] INFO : Step 43900/50000 rank4 | Train Loss: 0.7987 | Train Loss Unweighted: 0.3885 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.0999 | Classification: 0.0753 | Classification_binary: 0.1166 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0968 | Regression_eta: 0.0010 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1247 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0486 | Regression_eta: 0.0400 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0535 | LR: 1.33e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5942s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0846 | max_forward: 0.8891 | max_loss: 0.1768 | max_backward: 1.7357 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7516 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6030.6 | uss_MB: 5347.7 | child_rss_MB: 5179.6 | allocated_MB: 494.5 | reserved_MB: 28278.0 | inactive_split_MB: 1129.5 +rank4 [2026-10-30 02:47:33,826] INFO : Step 44000/50000 rank4 | Train Loss: 0.8263 | Train Loss Unweighted: 0.3926 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.1027 | Classification: 0.0840 | Classification_binary: 0.1318 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0868 | Regression_eta: 0.0011 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1352 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0389 | Regression_eta: 0.0342 | Regression_pt: 0.0421 | Regression_sin_phi: 0.0484 | LR: 1.32e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4530s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8816 | max_loss: 0.1426 | max_backward: 1.9616 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.4846 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6020.6 | uss_MB: 5352.2 | child_rss_MB: 5134.9 | allocated_MB: 496.3 | reserved_MB: 28762.0 | inactive_split_MB: 1307.2 +rank4 [2026-10-30 02:47:34,860] INFO : Running validation on rank4 +rank4 [2026-10-30 02:47:34,861] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 02:47:34,861] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 02:48:28,154] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 02:48:28,669] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 02:48:55,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 02:48:56,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 02:48:57,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 02:48:58,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 02:49:08,876] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 02:49:09,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 02:49:10,103] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 02:49:25,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 02:49:26,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 02:49:27,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 02:49:28,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 02:49:38,625] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 02:49:39,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 02:49:40,185] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 02:49:55,677] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 02:49:56,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 02:49:57,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 02:49:58,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 02:50:08,146] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 02:55:20,176] INFO : Step 44100/50000 rank4 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1986 | Cardinality: 0.1334 | Classification: 0.0764 | Classification_binary: 0.1516 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0523 | Regression_eta: 0.0340 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0417 | LR: 1.31e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.1237s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0079 | max_forward: 0.8221 | max_loss: 0.1579 | max_backward: 2.0192 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 1.0889 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7539.1 | uss_MB: 6869.4 | child_rss_MB: 5111.3 | allocated_MB: 498.6 | reserved_MB: 25920.0 | inactive_split_MB: 1073.8 +rank4 [2026-10-30 03:00:12,362] INFO : Step 44200/50000 rank4 | Train Loss: 0.9018 | Train Loss Unweighted: 0.4273 | Train Loss Components: Auxiliary: 0.1909 | Cardinality: 0.1109 | Classification: 0.0847 | Classification_binary: 0.1354 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0012 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0511 | Regression_eta: 0.0480 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0554 | LR: 1.30e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3582s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.1013 | max_forward: 0.8608 | max_loss: 0.1485 | max_backward: 1.6010 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 6119.2 | uss_MB: 5452.1 | child_rss_MB: 5083.8 | allocated_MB: 496.1 | reserved_MB: 27956.0 | inactive_split_MB: 1248.1 +rank4 [2026-10-30 03:04:47,145] INFO : Step 44300/50000 rank4 | Train Loss: 0.8239 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1722 | Cardinality: 0.1007 | Classification: 0.0789 | Classification_binary: 0.1118 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1086 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0682 | Regression_eta: 0.0340 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0509 | LR: 1.29e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3179s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8120 | max_loss: 0.1377 | max_backward: 1.4700 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9055 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6110.2 | uss_MB: 5452.8 | child_rss_MB: 5031.6 | allocated_MB: 492.8 | reserved_MB: 27956.0 | inactive_split_MB: 1035.8 +rank4 [2026-10-30 03:09:18,055] INFO : Step 44400/50000 rank4 | Train Loss: 0.8683 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.0849 | Classification: 0.0836 | Classification_binary: 0.1500 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1208 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0464 | Regression_eta: 0.0274 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0341 | LR: 1.28e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9259s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8907 | max_loss: 0.1476 | max_backward: 2.0623 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6106.5 | uss_MB: 5454.6 | child_rss_MB: 5002.7 | allocated_MB: 499.5 | reserved_MB: 27956.0 | inactive_split_MB: 926.0 +rank4 [2026-10-30 03:14:01,912] INFO : Step 44500/50000 rank4 | Train Loss: 0.8672 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.0989 | Classification: 0.0738 | Classification_binary: 0.1455 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1433 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0388 | Regression_eta: 0.0281 | Regression_pt: 0.0427 | Regression_sin_phi: 0.0625 | LR: 1.27e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2931s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0597 | max_forward: 0.8669 | max_loss: 0.1547 | max_backward: 1.7327 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.5628 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6105.7 | uss_MB: 5452.5 | child_rss_MB: 5004.4 | allocated_MB: 498.3 | reserved_MB: 27956.0 | inactive_split_MB: 860.5 +rank4 [2026-10-30 03:18:59,362] INFO : Step 44600/50000 rank4 | Train Loss: 0.7900 | Train Loss Unweighted: 0.3743 | Train Loss Components: Auxiliary: 0.1688 | Cardinality: 0.0956 | Classification: 0.0724 | Classification_binary: 0.1157 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0919 | Regression_eta: 0.0010 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1295 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0465 | Regression_eta: 0.0297 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0495 | LR: 1.26e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5438s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0745 | max_forward: 0.7797 | max_loss: 0.1409 | max_backward: 1.9334 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6022 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6098.6 | uss_MB: 5447.3 | child_rss_MB: 4994.6 | allocated_MB: 494.5 | reserved_MB: 27958.0 | inactive_split_MB: 927.6 +rank4 [2026-10-30 03:23:44,790] INFO : Step 44700/50000 rank4 | Train Loss: 0.8669 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1878 | Cardinality: 0.1160 | Classification: 0.0768 | Classification_binary: 0.1380 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0863 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1355 | Regression_cos_phi: 0.0285 | Regression_energy: 0.0475 | Regression_eta: 0.0363 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0398 | LR: 1.25e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4958s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8629 | max_loss: 0.1449 | max_backward: 2.0281 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.5242 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6099.2 | uss_MB: 5434.3 | child_rss_MB: 5003.0 | allocated_MB: 498.4 | reserved_MB: 27958.0 | inactive_split_MB: 781.6 +rank4 [2026-10-30 03:28:42,238] INFO : Step 44800/50000 rank4 | Train Loss: 0.8585 | Train Loss Unweighted: 0.4003 | Train Loss Components: Auxiliary: 0.1869 | Cardinality: 0.1024 | Classification: 0.0769 | Classification_binary: 0.1550 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0823 | Regression_eta: 0.0011 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1291 | Regression_cos_phi: 0.0377 | Regression_energy: 0.0356 | Regression_eta: 0.0269 | Regression_pt: 0.0386 | Regression_sin_phi: 0.0388 | LR: 1.24e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8030s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0072 | max_forward: 0.7883 | max_loss: 0.5033 | max_backward: 1.9727 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6090.2 | uss_MB: 5455.2 | child_rss_MB: 4956.4 | allocated_MB: 496.5 | reserved_MB: 27958.0 | inactive_split_MB: 778.0 +rank4 [2026-10-30 03:33:12,493] INFO : Step 44900/50000 rank4 | Train Loss: 0.8358 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.1024 | Classification: 0.0797 | Classification_binary: 0.1294 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0859 | Regression_eta: 0.0011 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1306 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0459 | Regression_eta: 0.0361 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0621 | LR: 1.23e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3696s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7360 | max_loss: 0.1482 | max_backward: 1.6886 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7825 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6084.1 | uss_MB: 5452.9 | child_rss_MB: 4956.9 | allocated_MB: 494.3 | reserved_MB: 27958.0 | inactive_split_MB: 922.0 +rank4 [2026-10-30 03:37:54,705] INFO : Step 45000/50000 rank4 | Train Loss: 0.8678 | Train Loss Unweighted: 0.3957 | Train Loss Components: Auxiliary: 0.1790 | Cardinality: 0.1036 | Classification: 0.0801 | Classification_binary: 0.1237 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0941 | Regression_eta: 0.0013 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1143 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0461 | Regression_eta: 0.0394 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0695 | LR: 1.22e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5707s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7189 | max_loss: 0.2183 | max_backward: 1.8172 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.8812 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6058.9 | uss_MB: 5454.9 | child_rss_MB: 4844.5 | allocated_MB: 495.0 | reserved_MB: 27960.0 | inactive_split_MB: 978.5 +rank4 [2026-10-30 03:37:55,714] INFO : Running validation on rank4 +rank4 [2026-10-30 03:37:55,714] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 03:37:55,715] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 03:38:46,291] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 03:38:46,808] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 03:39:14,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 03:39:15,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 03:39:16,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 03:39:18,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 03:39:28,365] INFO : Time taken to make predictions on device 4 is: 0.27 min +rank4 [2026-10-30 03:39:29,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 03:39:30,321] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 03:39:46,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 03:39:47,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 03:39:48,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 03:39:49,352] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 03:39:59,363] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 03:40:00,144] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 03:40:00,673] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 03:40:16,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 03:40:17,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 03:40:18,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 03:40:18,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 03:40:28,860] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 03:45:39,355] INFO : Step 45100/50000 rank4 | Train Loss: 0.8712 | Train Loss Unweighted: 0.4129 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0997 | Classification: 0.0837 | Classification_binary: 0.1445 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0910 | Regression_eta: 0.0012 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1394 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0468 | Regression_eta: 0.0289 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0345 | LR: 1.21e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.1918s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8448 | max_loss: 0.1523 | max_backward: 2.1445 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 1.0003 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7489.1 | uss_MB: 6883.8 | child_rss_MB: 4866.9 | allocated_MB: 499.4 | reserved_MB: 24746.0 | inactive_split_MB: 784.5 +rank4 [2026-10-30 03:50:46,382] INFO : Step 45200/50000 rank4 | Train Loss: 0.9130 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1058 | Classification: 0.0776 | Classification_binary: 0.1464 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0963 | Regression_eta: 0.0015 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0445 | Regression_eta: 0.0728 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0425 | LR: 1.20e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.5415s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1574 | max_backward: 1.6440 | max_optimizer_step: 0.2111 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1112 | Memory: rss_MB: 6132.8 | uss_MB: 4597.7 | child_rss_MB: 4866.1 | allocated_MB: 499.7 | reserved_MB: 26730.0 | inactive_split_MB: 1073.5 +rank4 [2026-10-30 03:55:44,004] INFO : Step 45300/50000 rank4 | Train Loss: 0.8299 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.1126 | Classification: 0.0744 | Classification_binary: 0.1207 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0521 | Regression_eta: 0.0270 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0404 | LR: 1.19e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1814s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0827 | max_forward: 0.7966 | max_loss: 0.1432 | max_backward: 1.5141 | max_optimizer_step: 0.2646 | max_tensorboard_logging: 0.6448 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6128.3 | uss_MB: 5538.9 | child_rss_MB: 4864.4 | allocated_MB: 495.4 | reserved_MB: 27170.0 | inactive_split_MB: 881.8 +rank4 [2026-10-30 04:00:47,346] INFO : Step 45400/50000 rank4 | Train Loss: 0.8255 | Train Loss Unweighted: 0.4011 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.0984 | Classification: 0.0907 | Classification_binary: 0.1338 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0870 | Regression_eta: 0.0011 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0453 | Regression_eta: 0.0223 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0417 | LR: 1.19e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.3449s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7846 | max_loss: 0.1934 | max_backward: 1.9857 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 0.7880 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6124.9 | uss_MB: 4519.9 | child_rss_MB: 4862.5 | allocated_MB: 499.2 | reserved_MB: 27170.0 | inactive_split_MB: 989.3 +rank4 [2026-10-30 04:05:44,853] INFO : Step 45500/50000 rank4 | Train Loss: 0.8945 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1090 | Classification: 0.0825 | Classification_binary: 0.1567 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0012 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0491 | Regression_eta: 0.0285 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0452 | LR: 1.18e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4575s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8686 | max_loss: 0.1508 | max_backward: 1.9581 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.4909 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6130.5 | uss_MB: 5543.1 | child_rss_MB: 4864.9 | allocated_MB: 498.4 | reserved_MB: 28766.0 | inactive_split_MB: 883.2 +rank4 [2026-10-30 04:10:39,744] INFO : Step 45600/50000 rank4 | Train Loss: 0.8704 | Train Loss Unweighted: 0.4121 | Train Loss Components: Auxiliary: 0.1840 | Cardinality: 0.1126 | Classification: 0.0802 | Classification_binary: 0.1324 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0983 | Regression_eta: 0.0010 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1146 | Regression_cos_phi: 0.0528 | Regression_energy: 0.0434 | Regression_eta: 0.0211 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0526 | LR: 1.17e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.8859s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0163 | max_forward: 0.8161 | max_loss: 0.2729 | max_backward: 2.1846 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6122.7 | uss_MB: 5488.3 | child_rss_MB: 4871.5 | allocated_MB: 495.6 | reserved_MB: 28766.0 | inactive_split_MB: 1080.4 +rank4 [2026-10-30 04:15:37,194] INFO : Step 45700/50000 rank4 | Train Loss: 0.8006 | Train Loss Unweighted: 0.3784 | Train Loss Components: Auxiliary: 0.1681 | Cardinality: 0.0873 | Classification: 0.0773 | Classification_binary: 0.1111 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0012 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0614 | Regression_eta: 0.0317 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0480 | LR: 1.16e-04 | DataLoad Time: 0.0101s | Model Forward Time: 4.8297s | Phase Times: max_pre_step_barrier: 0.0327 | max_batch_to_device: 0.0149 | max_forward: 1.1858 | max_loss: 0.1429 | max_backward: 1.6968 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1366 | Memory: rss_MB: 6126.5 | uss_MB: 5540.3 | child_rss_MB: 4873.0 | allocated_MB: 496.6 | reserved_MB: 28766.0 | inactive_split_MB: 903.7 +rank4 [2026-10-30 04:20:28,800] INFO : Step 45800/50000 rank4 | Train Loss: 0.8566 | Train Loss Unweighted: 0.3982 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1097 | Classification: 0.0794 | Classification_binary: 0.1398 | HitGrouping: 0.1406 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1190 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0441 | Regression_eta: 0.0328 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0552 | LR: 1.16e-04 | DataLoad Time: 0.0147s | Model Forward Time: 5.1219s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.9805 | max_loss: 0.5234 | max_backward: 2.3872 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.8153 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6131.1 | uss_MB: 5547.8 | child_rss_MB: 4874.9 | allocated_MB: 501.7 | reserved_MB: 28766.0 | inactive_split_MB: 1064.3 +rank4 [2026-10-30 04:25:16,811] INFO : Step 45900/50000 rank4 | Train Loss: 0.8754 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1042 | Classification: 0.0696 | Classification_binary: 0.1484 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0394 | Regression_eta: 0.0362 | Regression_pt: 0.0437 | Regression_sin_phi: 0.0427 | LR: 1.15e-04 | DataLoad Time: 0.0143s | Model Forward Time: 5.8464s | Phase Times: max_pre_step_barrier: 0.0308 | max_batch_to_device: 0.0265 | max_forward: 1.7626 | max_loss: 0.1678 | max_backward: 2.6283 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 6124.4 | uss_MB: 5544.6 | child_rss_MB: 4869.8 | allocated_MB: 496.0 | reserved_MB: 29170.0 | inactive_split_MB: 983.4 +rank4 [2026-10-30 04:29:51,934] INFO : Step 46000/50000 rank4 | Train Loss: 0.7531 | Train Loss Unweighted: 0.3556 | Train Loss Components: Auxiliary: 0.1535 | Cardinality: 0.0870 | Classification: 0.0690 | Classification_binary: 0.1026 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0903 | Regression_eta: 0.0011 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0294 | Classification_binary: 0.1007 | Regression_cos_phi: 0.0350 | Regression_energy: 0.0541 | Regression_eta: 0.0251 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0528 | LR: 1.14e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3115s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0329 | max_forward: 0.7497 | max_loss: 0.1389 | max_backward: 1.9140 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5104 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6122.4 | uss_MB: 5543.2 | child_rss_MB: 4869.8 | allocated_MB: 492.4 | reserved_MB: 29170.0 | inactive_split_MB: 973.8 +rank4 [2026-10-30 04:29:53,062] INFO : Running validation on rank4 +rank4 [2026-10-30 04:29:53,062] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 04:29:53,063] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 04:30:44,348] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 04:30:44,927] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 04:31:11,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 04:31:12,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 04:31:13,437] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 04:31:14,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 04:31:24,506] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 04:31:25,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 04:31:25,580] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 04:31:41,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 04:31:42,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 04:31:43,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 04:31:44,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 04:31:54,438] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 04:31:54,979] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 04:31:55,464] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 04:32:11,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 04:32:11,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 04:32:12,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 04:32:13,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 04:32:23,599] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 04:37:27,738] INFO : Step 46100/50000 rank4 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3769 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.0999 | Classification: 0.0743 | Classification_binary: 0.1253 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0010 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1122 | Regression_cos_phi: 0.0350 | Regression_energy: 0.0429 | Regression_eta: 0.0234 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0515 | LR: 1.13e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8719s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.9293 | max_loss: 0.1452 | max_backward: 2.3008 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.4968 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1219 | Memory: rss_MB: 7472.5 | uss_MB: 6889.9 | child_rss_MB: 4882.8 | allocated_MB: 498.3 | reserved_MB: 23298.0 | inactive_split_MB: 776.1 +rank4 [2026-10-30 04:42:19,614] INFO : Step 46200/50000 rank4 | Train Loss: 0.8661 | Train Loss Unweighted: 0.3994 | Train Loss Components: Auxiliary: 0.1947 | Cardinality: 0.1164 | Classification: 0.0793 | Classification_binary: 0.1511 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0833 | Regression_eta: 0.0011 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1587 | Regression_cos_phi: 0.0500 | Regression_energy: 0.0510 | Regression_eta: 0.0316 | Regression_pt: 0.0557 | Regression_sin_phi: 0.0368 | LR: 1.13e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7264s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8070 | max_loss: 0.2423 | max_backward: 1.9860 | max_optimizer_step: 0.2482 | max_tensorboard_logging: 0.7539 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6199.1 | uss_MB: 5617.8 | child_rss_MB: 4859.8 | allocated_MB: 499.3 | reserved_MB: 25288.0 | inactive_split_MB: 1325.5 +rank4 [2026-10-30 04:46:59,453] INFO : Step 46300/50000 rank4 | Train Loss: 0.8634 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0916 | Classification: 0.0803 | Classification_binary: 0.1320 | HitGrouping: 0.1536 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1031 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1270 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0454 | Regression_eta: 0.0245 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0632 | LR: 1.12e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5155s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7891 | max_loss: 0.1470 | max_backward: 1.7900 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.7709 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6201.3 | uss_MB: 5618.4 | child_rss_MB: 4887.4 | allocated_MB: 498.4 | reserved_MB: 25290.0 | inactive_split_MB: 977.6 +rank4 [2026-10-30 04:51:36,900] INFO : Step 46400/50000 rank4 | Train Loss: 0.8499 | Train Loss Unweighted: 0.4102 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.0956 | Classification: 0.0755 | Classification_binary: 0.1172 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1310 | Regression_cos_phi: 0.0443 | Regression_energy: 0.0595 | Regression_eta: 0.0287 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0589 | LR: 1.11e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9773s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7847 | max_loss: 0.2487 | max_backward: 1.9181 | max_optimizer_step: 0.3148 | max_tensorboard_logging: 0.9408 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6188.6 | uss_MB: 5608.9 | child_rss_MB: 4878.3 | allocated_MB: 501.6 | reserved_MB: 25290.0 | inactive_split_MB: 992.4 +rank4 [2026-10-30 04:56:10,901] INFO : Step 46500/50000 rank4 | Train Loss: 0.8687 | Train Loss Unweighted: 0.4055 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1002 | Classification: 0.0822 | Classification_binary: 0.1448 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0872 | Regression_eta: 0.0013 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0421 | Regression_eta: 0.0396 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0732 | LR: 1.11e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5403s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1839 | max_backward: 1.9629 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6193.9 | uss_MB: 5618.8 | child_rss_MB: 4880.0 | allocated_MB: 498.8 | reserved_MB: 29306.0 | inactive_split_MB: 924.4 +rank4 [2026-10-30 05:00:44,087] INFO : Step 46600/50000 rank4 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4049 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1082 | Classification: 0.0757 | Classification_binary: 0.1508 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0479 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0477 | Regression_eta: 0.0396 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0554 | LR: 1.10e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2113s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7939 | max_loss: 0.2001 | max_backward: 1.6859 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6188.5 | uss_MB: 5615.4 | child_rss_MB: 4846.0 | allocated_MB: 496.0 | reserved_MB: 29306.0 | inactive_split_MB: 901.7 +rank4 [2026-10-30 05:05:18,398] INFO : Step 46700/50000 rank4 | Train Loss: 0.8417 | Train Loss Unweighted: 0.3855 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0982 | Classification: 0.0807 | Classification_binary: 0.1184 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0916 | Regression_eta: 0.0015 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1337 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0542 | Regression_eta: 0.0360 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0559 | LR: 1.10e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7619s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8551 | max_loss: 0.5124 | max_backward: 1.9004 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.7762 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1375 | Memory: rss_MB: 6185.4 | uss_MB: 5616.8 | child_rss_MB: 4868.3 | allocated_MB: 495.0 | reserved_MB: 29308.0 | inactive_split_MB: 1000.4 +rank4 [2026-10-30 05:09:48,388] INFO : Step 46800/50000 rank4 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3911 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0920 | Classification: 0.0920 | Classification_binary: 0.1296 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0828 | Regression_eta: 0.0013 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.0999 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0470 | Regression_eta: 0.0363 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0605 | LR: 1.09e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4612s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7934 | max_loss: 0.1440 | max_backward: 1.7630 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.7219 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6183.1 | uss_MB: 5617.4 | child_rss_MB: 4871.4 | allocated_MB: 495.6 | reserved_MB: 29308.0 | inactive_split_MB: 1332.0 +rank4 [2026-10-30 05:14:19,737] INFO : Step 46900/50000 rank4 | Train Loss: 0.8496 | Train Loss Unweighted: 0.4142 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0976 | Classification: 0.0776 | Classification_binary: 0.1448 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0946 | Regression_eta: 0.0010 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1326 | Regression_cos_phi: 0.0443 | Regression_energy: 0.0532 | Regression_eta: 0.0302 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0353 | LR: 1.09e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.4525s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.9007 | max_loss: 0.1576 | max_backward: 1.8470 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6183.1 | uss_MB: 5616.1 | child_rss_MB: 4872.4 | allocated_MB: 495.4 | reserved_MB: 29310.0 | inactive_split_MB: 850.5 +rank4 [2026-10-30 05:18:49,738] INFO : Step 47000/50000 rank4 | Train Loss: 0.7980 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0982 | Classification: 0.0774 | Classification_binary: 0.1196 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0318 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0479 | Regression_eta: 0.0294 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0445 | LR: 1.08e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.8524s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 1.2405 | max_loss: 0.1502 | max_backward: 2.1273 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.7105 | max_tensorboard_flush: 0.0082 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6184.1 | uss_MB: 5617.4 | child_rss_MB: 4877.5 | allocated_MB: 494.9 | reserved_MB: 29310.0 | inactive_split_MB: 852.6 +rank4 [2026-10-30 05:18:50,808] INFO : Running validation on rank4 +rank4 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 05:19:41,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 05:19:42,340] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 05:20:07,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 05:20:08,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 05:20:09,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 05:20:10,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 05:20:20,939] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 05:20:21,505] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 05:20:22,014] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 05:20:38,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 05:20:39,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 05:20:39,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 05:20:40,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 05:20:50,946] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 05:20:51,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 05:20:52,370] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 05:21:07,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 05:21:08,809] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 05:21:09,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 05:21:10,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 05:21:20,459] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 05:26:12,382] INFO : Step 47100/50000 rank4 | Train Loss: 0.8046 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0851 | Classification: 0.0882 | Classification_binary: 0.1121 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0936 | Regression_eta: 0.0010 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1033 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0603 | Regression_eta: 0.0302 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0500 | LR: 1.07e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5751s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8250 | max_loss: 0.1453 | max_backward: 1.6267 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8108 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 7449.0 | uss_MB: 6878.6 | child_rss_MB: 4837.4 | allocated_MB: 494.1 | reserved_MB: 25416.0 | inactive_split_MB: 894.5 +rank4 [2026-10-30 05:30:44,527] INFO : Step 47200/50000 rank4 | Train Loss: 0.8884 | Train Loss Unweighted: 0.4191 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1138 | Classification: 0.0793 | Classification_binary: 0.1489 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0010 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0620 | Regression_eta: 0.0233 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0450 | LR: 1.07e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9175s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7960 | max_loss: 0.1482 | max_backward: 1.9261 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.9435 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0661 | Memory: rss_MB: 6249.9 | uss_MB: 5682.5 | child_rss_MB: 4852.8 | allocated_MB: 496.8 | reserved_MB: 26534.0 | inactive_split_MB: 847.6 +rank4 [2026-10-30 05:35:15,255] INFO : Step 47300/50000 rank4 | Train Loss: 0.8547 | Train Loss Unweighted: 0.3920 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1049 | Classification: 0.0798 | Classification_binary: 0.1368 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0012 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0497 | Regression_eta: 0.0259 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0589 | LR: 1.06e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5889s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7649 | max_loss: 0.2313 | max_backward: 1.8030 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8438 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6253.9 | uss_MB: 5688.5 | child_rss_MB: 4855.9 | allocated_MB: 496.9 | reserved_MB: 28534.0 | inactive_split_MB: 933.0 +rank4 [2026-10-30 05:39:47,014] INFO : Step 47400/50000 rank4 | Train Loss: 0.7606 | Train Loss Unweighted: 0.3755 | Train Loss Components: Auxiliary: 0.1640 | Cardinality: 0.0704 | Classification: 0.0737 | Classification_binary: 0.1072 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0960 | Regression_eta: 0.0010 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1216 | Regression_cos_phi: 0.0470 | Regression_energy: 0.0582 | Regression_eta: 0.0330 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0442 | LR: 1.06e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5309s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0275 | max_forward: 0.8465 | max_loss: 0.1482 | max_backward: 1.7103 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.8388 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 6241.7 | uss_MB: 5678.8 | child_rss_MB: 4860.9 | allocated_MB: 496.4 | reserved_MB: 28534.0 | inactive_split_MB: 913.7 +rank4 [2026-10-30 05:44:18,077] INFO : Step 47500/50000 rank4 | Train Loss: 0.8134 | Train Loss Unweighted: 0.3681 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1057 | Classification: 0.0739 | Classification_binary: 0.1303 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0805 | Regression_eta: 0.0011 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1471 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0440 | Regression_eta: 0.0355 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0551 | LR: 1.06e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.7798s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1751 | max_backward: 1.9857 | max_optimizer_step: 0.3263 | max_tensorboard_logging: 0.7363 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6249.7 | uss_MB: 5690.9 | child_rss_MB: 4845.9 | allocated_MB: 498.3 | reserved_MB: 28534.0 | inactive_split_MB: 851.5 +rank4 [2026-10-30 05:48:48,525] INFO : Step 47600/50000 rank4 | Train Loss: 0.8663 | Train Loss Unweighted: 0.4076 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1090 | Classification: 0.0820 | Classification_binary: 0.1491 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0865 | Regression_eta: 0.0011 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1755 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0449 | Regression_eta: 0.0320 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0753 | LR: 1.05e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.6248s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.9385 | max_loss: 0.1530 | max_backward: 2.1678 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6249.7 | uss_MB: 5691.7 | child_rss_MB: 4853.8 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 843.5 +rank4 [2026-10-30 05:53:19,839] INFO : Step 47700/50000 rank4 | Train Loss: 0.8615 | Train Loss Unweighted: 0.4088 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0954 | Classification: 0.0764 | Classification_binary: 0.1291 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1002 | Regression_eta: 0.0011 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0549 | Regression_eta: 0.0313 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0892 | LR: 1.05e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2197s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7406 | max_loss: 0.2391 | max_backward: 1.7059 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1444 | Memory: rss_MB: 6248.7 | uss_MB: 5687.5 | child_rss_MB: 4851.8 | allocated_MB: 493.9 | reserved_MB: 28534.0 | inactive_split_MB: 895.3 +rank4 [2026-10-30 05:57:52,842] INFO : Step 47800/50000 rank4 | Train Loss: 0.8267 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.0942 | Classification: 0.0882 | Classification_binary: 0.1186 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0870 | Regression_eta: 0.0012 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.1033 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0542 | Regression_eta: 0.0349 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0717 | LR: 1.04e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.0870s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.7869 | max_loss: 0.5030 | max_backward: 2.0522 | max_optimizer_step: 0.2267 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6249.7 | uss_MB: 5692.3 | child_rss_MB: 4857.3 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 916.9 +rank4 [2026-10-30 06:02:25,107] INFO : Step 47900/50000 rank4 | Train Loss: 0.8423 | Train Loss Unweighted: 0.4000 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.0845 | Classification: 0.0794 | Classification_binary: 0.1416 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1495 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0460 | Regression_eta: 0.0268 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0485 | LR: 1.04e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5092s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7994 | max_loss: 0.1511 | max_backward: 1.9354 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.6685 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6247.4 | uss_MB: 5692.2 | child_rss_MB: 4860.2 | allocated_MB: 496.8 | reserved_MB: 30642.0 | inactive_split_MB: 779.5 +rank4 [2026-10-30 06:06:56,645] INFO : Step 48000/50000 rank4 | Train Loss: 0.8641 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0972 | Classification: 0.0768 | Classification_binary: 0.1396 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0890 | Regression_eta: 0.0015 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0578 | Regression_eta: 0.0356 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0584 | LR: 1.04e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4076s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0057 | max_forward: 0.8017 | max_loss: 0.1550 | max_backward: 1.6990 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.7498 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6247.1 | uss_MB: 5690.2 | child_rss_MB: 4855.9 | allocated_MB: 495.1 | reserved_MB: 30642.0 | inactive_split_MB: 896.2 +rank4 [2026-10-30 06:06:57,654] INFO : Running validation on rank4 +rank4 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 06:07:49,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:07:49,560] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 06:08:14,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 06:08:15,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 06:08:16,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 06:08:17,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 06:08:27,174] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 06:08:28,016] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:08:28,620] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 06:08:44,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 06:08:45,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 06:08:45,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 06:08:46,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 06:08:56,952] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 06:08:58,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:08:58,715] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 06:09:14,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 06:09:15,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 06:09:15,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 06:09:16,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 06:09:26,786] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 06:14:16,583] INFO : Step 48100/50000 rank4 | Train Loss: 0.8226 | Train Loss Unweighted: 0.3895 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.1043 | Classification: 0.0717 | Classification_binary: 0.1116 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1023 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1065 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0461 | Regression_eta: 0.0964 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0535 | LR: 1.03e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4964s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7477 | max_loss: 0.1426 | max_backward: 1.7972 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.6718 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1303 | Memory: rss_MB: 7752.6 | uss_MB: 7191.3 | child_rss_MB: 4880.7 | allocated_MB: 494.1 | reserved_MB: 23078.0 | inactive_split_MB: 1096.3 +rank4 [2026-10-30 06:18:47,187] INFO : Step 48200/50000 rank4 | Train Loss: 0.8500 | Train Loss Unweighted: 0.3835 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1027 | Classification: 0.0771 | Classification_binary: 0.1374 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0827 | Regression_eta: 0.0012 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1324 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0386 | Regression_eta: 0.0512 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0557 | LR: 1.03e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.2144s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8132 | max_loss: 0.2545 | max_backward: 2.1167 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1241 | Memory: rss_MB: 6310.4 | uss_MB: 5751.6 | child_rss_MB: 4867.3 | allocated_MB: 496.6 | reserved_MB: 25094.0 | inactive_split_MB: 913.4 +rank4 [2026-10-30 06:23:18,411] INFO : Step 48300/50000 rank4 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4303 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1089 | Classification: 0.0832 | Classification_binary: 0.1567 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0013 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0566 | Regression_eta: 0.0482 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0585 | LR: 1.03e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4478s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8079 | max_loss: 0.1526 | max_backward: 1.9733 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6317.4 | uss_MB: 5760.9 | child_rss_MB: 4872.1 | allocated_MB: 498.5 | reserved_MB: 25098.0 | inactive_split_MB: 771.0 +rank4 [2026-10-30 06:27:49,027] INFO : Step 48400/50000 rank4 | Train Loss: 0.7961 | Train Loss Unweighted: 0.3733 | Train Loss Components: Auxiliary: 0.1682 | Cardinality: 0.0877 | Classification: 0.0685 | Classification_binary: 0.1191 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0012 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1369 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0458 | Regression_eta: 0.0396 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0572 | LR: 1.02e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2382s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7916 | max_loss: 0.1471 | max_backward: 1.6943 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5940 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6314.4 | uss_MB: 5759.4 | child_rss_MB: 4842.4 | allocated_MB: 496.3 | reserved_MB: 25098.0 | inactive_split_MB: 1087.5 +rank4 [2026-10-30 06:32:21,028] INFO : Step 48500/50000 rank4 | Train Loss: 0.8668 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0816 | Classification: 0.0832 | Classification_binary: 0.1257 | HitGrouping: 0.1542 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1210 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0670 | Regression_eta: 0.0389 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0555 | LR: 1.02e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.3827s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7587 | max_loss: 0.1606 | max_backward: 1.7573 | max_optimizer_step: 0.3024 | max_tensorboard_logging: 0.5851 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6309.3 | uss_MB: 5759.8 | child_rss_MB: 4851.4 | allocated_MB: 498.9 | reserved_MB: 25098.0 | inactive_split_MB: 1192.4 +rank4 [2026-10-30 06:36:52,986] INFO : Step 48600/50000 rank4 | Train Loss: 0.8434 | Train Loss Unweighted: 0.4010 | Train Loss Components: Auxiliary: 0.1850 | Cardinality: 0.0964 | Classification: 0.0791 | Classification_binary: 0.1425 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0373 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0675 | Regression_energy: 0.0446 | Regression_eta: 0.0270 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0516 | LR: 1.02e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4853s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7760 | max_loss: 0.1505 | max_backward: 1.8991 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0654 | Memory: rss_MB: 6311.3 | uss_MB: 5762.6 | child_rss_MB: 4856.0 | allocated_MB: 497.0 | reserved_MB: 26848.0 | inactive_split_MB: 777.5 +rank4 [2026-10-30 06:41:29,684] INFO : Step 48700/50000 rank4 | Train Loss: 0.9079 | Train Loss Unweighted: 0.4266 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1195 | Classification: 0.0920 | Classification_binary: 0.1456 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0419 | Regression_eta: 0.0374 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0525 | LR: 1.02e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5440s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7797 | max_loss: 0.1578 | max_backward: 1.7107 | max_optimizer_step: 0.5280 | max_tensorboard_logging: 0.7650 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6308.9 | uss_MB: 5759.1 | child_rss_MB: 4857.7 | allocated_MB: 496.9 | reserved_MB: 26848.0 | inactive_split_MB: 891.8 +rank4 [2026-10-30 06:46:02,598] INFO : Step 48800/50000 rank4 | Train Loss: 0.8011 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1685 | Cardinality: 0.0863 | Classification: 0.0837 | Classification_binary: 0.1116 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0965 | Regression_eta: 0.0012 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0716 | Classification_binary: 0.1046 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0399 | Regression_eta: 0.0311 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0473 | LR: 1.01e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4998s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7374 | max_loss: 0.2432 | max_backward: 1.8459 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.7427 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6306.9 | uss_MB: 5759.1 | child_rss_MB: 4857.0 | allocated_MB: 493.9 | reserved_MB: 26848.0 | inactive_split_MB: 991.0 +rank4 [2026-10-30 06:50:45,084] INFO : Step 48900/50000 rank4 | Train Loss: 0.8551 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.1109 | Classification: 0.0760 | Classification_binary: 0.1310 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0369 | Classification_binary: 0.1444 | Regression_cos_phi: 0.0590 | Regression_energy: 0.0393 | Regression_eta: 0.0312 | Regression_pt: 0.0415 | Regression_sin_phi: 0.0463 | LR: 1.01e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5416s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8768 | max_loss: 0.1470 | max_backward: 2.0816 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6306.9 | uss_MB: 5761.7 | child_rss_MB: 4858.9 | allocated_MB: 499.5 | reserved_MB: 26852.0 | inactive_split_MB: 1398.0 +rank4 [2026-10-30 06:55:20,573] INFO : Step 49000/50000 rank4 | Train Loss: 0.9005 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.2010 | Cardinality: 0.1248 | Classification: 0.0800 | Classification_binary: 0.1499 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0939 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0427 | Regression_eta: 0.0233 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0452 | LR: 1.01e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9016s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0439 | max_forward: 0.8365 | max_loss: 0.1536 | max_backward: 1.9893 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8890 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6309.3 | uss_MB: 5762.6 | child_rss_MB: 4865.2 | allocated_MB: 497.5 | reserved_MB: 26852.0 | inactive_split_MB: 943.6 +rank4 [2026-10-30 06:55:21,669] INFO : Running validation on rank4 +rank4 [2026-10-30 06:55:21,669] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 06:55:21,669] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 06:56:13,351] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:56:13,862] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 06:56:39,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 06:56:40,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 06:56:41,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 06:56:42,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 06:56:52,058] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 06:56:52,550] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:56:53,029] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 06:57:09,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 06:57:10,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 06:57:10,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 06:57:11,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 06:57:21,955] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 06:57:22,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 06:57:23,149] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 06:57:38,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 06:57:39,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 06:57:40,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 06:57:41,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 06:57:51,197] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 07:02:45,630] INFO : Step 49100/50000 rank4 | Train Loss: 0.8219 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.0880 | Classification: 0.0841 | Classification_binary: 0.1303 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0855 | Regression_eta: 0.0012 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0742 | Classification_binary: 0.1290 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0403 | Regression_eta: 0.0441 | Regression_pt: 0.0430 | Regression_sin_phi: 0.0743 | LR: 1.01e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3716s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7738 | max_loss: 0.1431 | max_backward: 1.6939 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6276 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 7720.3 | uss_MB: 7170.2 | child_rss_MB: 4829.0 | allocated_MB: 495.9 | reserved_MB: 24730.0 | inactive_split_MB: 936.0 +rank4 [2026-10-30 07:07:18,678] INFO : Step 49200/50000 rank4 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3843 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0856 | Classification: 0.0829 | Classification_binary: 0.1188 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0892 | Regression_eta: 0.0011 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0386 | Regression_energy: 0.0417 | Regression_eta: 0.0308 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0423 | LR: 1.01e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.7731s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7590 | max_loss: 0.5032 | max_backward: 1.9826 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.7926 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6365.4 | uss_MB: 5817.0 | child_rss_MB: 4830.4 | allocated_MB: 494.5 | reserved_MB: 26752.0 | inactive_split_MB: 1117.5 +rank4 [2026-10-30 07:11:50,862] INFO : Step 49300/50000 rank4 | Train Loss: 0.9397 | Train Loss Unweighted: 0.4282 | Train Loss Components: Auxiliary: 0.2065 | Cardinality: 0.1351 | Classification: 0.0889 | Classification_binary: 0.1552 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0901 | Regression_eta: 0.0012 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0467 | Regression_eta: 0.0432 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0374 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4926s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0070 | max_forward: 0.7807 | max_loss: 0.1520 | max_backward: 1.8619 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6076 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6376.6 | uss_MB: 5827.0 | child_rss_MB: 4814.1 | allocated_MB: 496.4 | reserved_MB: 26752.0 | inactive_split_MB: 1120.6 +rank4 [2026-10-30 07:16:22,252] INFO : Step 49400/50000 rank4 | Train Loss: 0.8870 | Train Loss Unweighted: 0.4213 | Train Loss Components: Auxiliary: 0.1937 | Cardinality: 0.1005 | Classification: 0.0815 | Classification_binary: 0.1465 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1888 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0506 | Regression_eta: 0.0451 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0597 | LR: 1.00e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2705s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8046 | max_loss: 0.1540 | max_backward: 1.7244 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.6030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6373.7 | uss_MB: 5825.4 | child_rss_MB: 4813.8 | allocated_MB: 498.5 | reserved_MB: 28804.0 | inactive_split_MB: 886.2 +rank4 [2026-10-30 07:20:55,284] INFO : Step 49500/50000 rank4 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3949 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0991 | Classification: 0.0800 | Classification_binary: 0.1101 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0013 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1242 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0408 | Regression_eta: 0.0654 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0418 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.2167s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0074 | max_forward: 0.9924 | max_loss: 0.1865 | max_backward: 2.5853 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.6095 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0429 | Memory: rss_MB: 6377.1 | uss_MB: 5695.1 | child_rss_MB: 4830.9 | allocated_MB: 495.9 | reserved_MB: 28804.0 | inactive_split_MB: 849.6 +rank4 [2026-10-30 07:25:48,132] INFO : Step 49600/50000 rank4 | Train Loss: 0.9075 | Train Loss Unweighted: 0.4421 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1021 | Classification: 0.0939 | Classification_binary: 0.1401 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0737 | Classification_binary: 0.1788 | Regression_cos_phi: 0.0576 | Regression_energy: 0.0591 | Regression_eta: 0.0397 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0540 | LR: 1.00e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4395s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7852 | max_loss: 0.1445 | max_backward: 2.0967 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4754 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6378.7 | uss_MB: 5829.2 | child_rss_MB: 4808.2 | allocated_MB: 497.8 | reserved_MB: 28804.0 | inactive_split_MB: 845.2 +rank4 [2026-10-30 07:30:32,223] INFO : Step 49700/50000 rank4 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4307 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1074 | Classification: 0.0931 | Classification_binary: 0.1611 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0865 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.1842 | Regression_cos_phi: 0.0828 | Regression_energy: 0.0492 | Regression_eta: 0.0430 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0750 | LR: 1.00e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6420s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.8169 | max_loss: 0.1522 | max_backward: 2.0553 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.4937 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1402 | Memory: rss_MB: 6379.2 | uss_MB: 5828.6 | child_rss_MB: 4829.3 | allocated_MB: 499.3 | reserved_MB: 30718.0 | inactive_split_MB: 838.8 +rank4 [2026-10-30 07:35:17,054] INFO : Step 49800/50000 rank4 | Train Loss: 0.8342 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.1036 | Classification: 0.0769 | Classification_binary: 0.1292 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1259 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0493 | Regression_eta: 0.0287 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0314 | LR: 1.00e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1032s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0329 | max_forward: 0.7490 | max_loss: 0.1889 | max_backward: 2.2249 | max_optimizer_step: 0.2452 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6374.9 | uss_MB: 5812.4 | child_rss_MB: 4828.7 | allocated_MB: 494.4 | reserved_MB: 30718.0 | inactive_split_MB: 1134.4 +rank4 [2026-10-30 07:39:58,302] INFO : Step 49900/50000 rank4 | Train Loss: 0.8436 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.0947 | Classification: 0.0943 | Classification_binary: 0.1179 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0982 | Regression_eta: 0.0012 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0492 | Regression_eta: 0.0286 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0358 | LR: 1.00e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.4161s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7496 | max_loss: 0.5142 | max_backward: 1.9365 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6373.9 | uss_MB: 5825.7 | child_rss_MB: 4831.8 | allocated_MB: 494.8 | reserved_MB: 30718.0 | inactive_split_MB: 1090.4 +rank4 [2026-10-30 07:44:45,887] INFO : Step 50000/50000 rank4 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4040 | Train Loss Components: Auxiliary: 0.1921 | Cardinality: 0.1113 | Classification: 0.0818 | Classification_binary: 0.1436 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0872 | Regression_eta: 0.0014 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1642 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0474 | Regression_eta: 0.0361 | Regression_pt: 0.0505 | Regression_sin_phi: 0.0561 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.6364s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 1.3997 | max_loss: 0.5678 | max_backward: 3.0655 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6373.9 | uss_MB: 5825.4 | child_rss_MB: 4827.6 | allocated_MB: 497.9 | reserved_MB: 30720.0 | inactive_split_MB: 906.1 +rank4 [2026-10-30 07:44:46,920] INFO : Running validation on rank4 +rank4 [2026-10-30 07:44:46,920] INFO : ResumableSampler physical:valid resetting +rank4 [2026-10-30 07:44:46,920] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank4 [2026-10-30 07:45:38,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 07:45:38,997] INFO : Running predictions on cld_edm_ttbar_hits +rank4 [2026-10-30 07:46:04,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_4_0.parquet +rank4 [2026-10-30 07:46:05,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_4_1.parquet +rank4 [2026-10-30 07:46:06,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_4_2.parquet +rank4 [2026-10-30 07:46:07,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_4_3.parquet +rank4 [2026-10-30 07:46:17,045] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 07:46:17,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 07:46:17,995] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank4 [2026-10-30 07:46:33,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_4_0.parquet +rank4 [2026-10-30 07:46:34,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_4_1.parquet +rank4 [2026-10-30 07:46:35,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_4_2.parquet +rank4 [2026-10-30 07:46:36,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_4_3.parquet +rank4 [2026-10-30 07:46:46,588] INFO : Time taken to make predictions on device 4 is: 0.25 min +rank4 [2026-10-30 07:46:47,615] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank4 [2026-10-30 07:46:48,507] INFO : Running predictions on cld_edm_qq_hits +rank4 [2026-10-30 07:47:03,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_4_0.parquet +rank4 [2026-10-30 07:47:04,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_4_1.parquet +rank4 [2026-10-30 07:47:05,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_4_2.parquet +rank4 [2026-10-30 07:47:06,433] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_4_3.parquet +rank4 [2026-10-30 07:47:16,443] INFO : Time taken to make predictions on device 4 is: 0.23 min +rank4 [2026-10-30 07:47:17,260] INFO : Training completed. Total time on device 4: 1017.501min diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.5 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.5 new file mode 100644 index 0000000000000000000000000000000000000000..e03d47dd8881b104914b55440dfe7ae55de3f7c3 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.5 @@ -0,0 +1,2631 @@ +rank5 [2026-10-28 11:12:32,416] INFO : Initializing model with seed=12345; process seed=12350 +rank5 [2026-10-28 11:12:32,417] INFO : configured dtype=torch.bfloat16 for autocast +rank5 [2026-10-28 11:12:54,158] INFO : Instantiating model +rank5 [2026-10-28 11:12:54,158] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank5 [2026-10-28 11:12:54,158] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank5 [2026-10-28 11:12:54,177] INFO : Input encoding initialization took 0.02s +rank5 [2026-10-28 11:12:54,177] INFO : nn0 parameters: 171776 +rank5 [2026-10-28 11:12:54,177] INFO : Initializing shared backbone layers, num_convs=6 +rank5 [2026-10-28 11:12:54,177] INFO : Initializing backbone layer 0 +rank5 [2026-10-28 11:12:54,215] INFO : Initializing backbone layer 1 +rank5 [2026-10-28 11:12:54,228] INFO : Initializing backbone layer 2 +rank5 [2026-10-28 11:12:54,237] INFO : Initializing backbone layer 3 +rank5 [2026-10-28 11:12:54,246] INFO : Initializing backbone layer 4 +rank5 [2026-10-28 11:12:54,254] INFO : Initializing backbone layer 5 +rank5 [2026-10-28 11:12:54,263] INFO : Backbone initialization took 0.09s +rank5 [2026-10-28 11:12:54,263] INFO : backbone parameters: 7901376 +rank5 [2026-10-28 11:12:54,263] INFO : Initializing output DNNs +rank5 [2026-10-28 11:12:54,296] INFO : Output DNNs initialization took 0.03s +rank5 [2026-10-28 11:12:54,296] INFO : backbone_mode=BackboneMode.SHARED +rank5 [2026-10-28 11:12:54,296] INFO : jagged_attention enabled=False +rank5 [2026-10-28 11:12:54,296] INFO : flash_attn_varlen enabled=False +rank5 [2026-10-28 11:12:54,296] INFO : task_queries enabled=False +rank5 [2026-10-28 11:12:54,296] INFO : classification_norm parameters: 0 +rank5 [2026-10-28 11:12:54,296] INFO : regression_norm parameters: 0 +rank5 [2026-10-28 11:12:54,296] INFO : classification_query parameters: 0 +rank5 [2026-10-28 11:12:54,296] INFO : regression_query parameters: 0 +rank5 [2026-10-28 11:12:54,296] INFO : classification_readout parameters: 0 +rank5 [2026-10-28 11:12:54,296] INFO : regression_readout parameters: 0 +rank5 [2026-10-28 11:12:54,297] INFO : output_mode=set +rank5 [2026-10-28 11:12:54,297] INFO : set_decoder parameters: 4713812 +rank5 [2026-10-28 11:12:54,297] INFO : nn_binary_particle parameters: 0 +rank5 [2026-10-28 11:12:54,297] INFO : nn_pid parameters: 0 +rank5 [2026-10-28 11:12:54,297] INFO : nn_pt parameters: 0 +rank5 [2026-10-28 11:12:54,297] INFO : nn_eta parameters: 0 +rank5 [2026-10-28 11:12:54,297] INFO : nn_sin_phi parameters: 0 +rank5 [2026-10-28 11:12:54,297] INFO : nn_cos_phi parameters: 0 +rank5 [2026-10-28 11:12:54,297] INFO : nn_energy parameters: 0 +rank5 [2026-10-28 11:12:54,298] INFO : Total MLPF parameters: 12786964 +rank5 [2026-10-28 11:12:54,298] INFO : MLPF __init__ done +rank5 [2026-10-28 11:12:54,298] INFO : Instantiated model +rank5 [2026-10-28 11:12:54,298] INFO : Moving model to device rank=5 +rank5 [2026-10-28 11:12:54,340] INFO : Moved model to device rank=5 +rank5 [2026-10-28 11:12:54,342] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank5 [2026-10-28 11:12:54,343] INFO : Configured model for SyncBatchNorm rank=5 +rank5 [2026-10-28 11:12:54,648] INFO : Configured model for DistributedDataParallel rank=5 +rank5 [2026-10-28 11:12:54,808] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank5 [2026-10-28 11:12:54,809] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank5 [2026-10-28 11:12:54,810] INFO : Trainable parameters: 12786964 +rank5 [2026-10-28 11:12:54,810] INFO : Non-trainable parameters: 23268 +rank5 [2026-10-28 11:12:54,810] INFO : Total parameters: 12810232 +rank5 [2026-10-28 11:12:54,813] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank5 [2026-10-28 11:12:54,813] INFO : Getting dataloaders +rank5 [2026-10-28 11:12:54,813] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:03,739] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:04,811] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,154] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-28 11:13:06,154] INFO : Creating WeightedMixIterator with 1 data loaders. +rank5 [2026-10-28 11:13:06,154] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,428] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,705] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:13:06,970] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-28 11:13:06,970] INFO : Creating InterleavedIterator with 1 data loaders. +rank5 [2026-10-28 11:13:21,089] INFO : Got dataloaders +rank5 [2026-10-28 11:13:21,089] INFO : loader split=train rank=5 len=21112 +rank5 [2026-10-28 11:13:21,089] INFO : loader split=valid rank=5 len=12 +rank5 [2026-10-28 11:13:21,089] INFO : Starting training from step 1 to 50000 on rank 5 of 8 +rank5 [2026-10-28 11:13:21,090] INFO : Creating train_iterator +rank5 [2026-10-28 11:13:21,090] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-28 11:13:35,649] INFO : Created train_iterator +rank5 [2026-10-28 11:18:11,208] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5408100029081107, 'Classification': 0.8863177470117808, 'Regression_pt': 0.9386234817653895, 'Regression_eta': 0.019238746587652712, 'Regression_sin_phi': 0.006576384851359762, 'Regression_cos_phi': 0.006405904629500583, 'Regression_energy': 1.0145655398070812}, weights={'Classification_binary': 1.0, 'Classification': 0.6101762056350708, 'Regression_pt': 0.5761735439300537, 'Regression_eta': 28.110458374023438, 'Regression_sin_phi': 82.23515319824219, 'Regression_cos_phi': 84.42367553710938, 'Regression_energy': 0.5330458879470825} +rank5 [2026-10-28 11:18:13,719] INFO : Step 100/50000 rank5 | Train Loss: 4.2631 | Train Loss Unweighted: 2.7366 | Train Loss Components: Auxiliary: 0.7941 | Cardinality: 0.3942 | Classification: 0.6312 | Classification_binary: 0.4549 | HitGrouping: 0.3381 | Regression_cos_phi: 0.0047 | Regression_energy: 0.8263 | Regression_eta: 0.0122 | Regression_pt: 0.8026 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.6474 | Classification_binary: 0.4678 | Regression_cos_phi: 0.0052 | Regression_energy: 0.8938 | Regression_eta: 0.0131 | Regression_pt: 0.8558 | Regression_sin_phi: 0.0048 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 3.5378s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6509 | max_loss: 0.1961 | max_backward: 1.4166 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4769.1 | uss_MB: 3475.5 | child_rss_MB: 6372.1 | allocated_MB: 446.7 | reserved_MB: 24804.0 | inactive_split_MB: 1063.6 +rank5 [2026-10-28 11:22:42,538] INFO : Step 200/50000 rank5 | Train Loss: 4.2029 | Train Loss Unweighted: 2.5447 | Train Loss Components: Auxiliary: 0.8074 | Cardinality: 0.4500 | Classification: 0.5373 | Classification_binary: 0.5135 | HitGrouping: 0.3101 | Regression_cos_phi: 0.0042 | Regression_energy: 0.7508 | Regression_eta: 0.0093 | Regression_pt: 0.7252 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.3233 | Classification_binary: 0.4755 | Regression_cos_phi: 0.2888 | Regression_energy: 0.4291 | Regression_eta: 0.1800 | Regression_pt: 0.4429 | Regression_sin_phi: 0.3243 | LR: 1.00e-03 | DataLoad Time: 0.0160s | Model Forward Time: 4.6625s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9214 | max_loss: 0.1502 | max_backward: 2.1220 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 4772.0 | uss_MB: 3480.3 | child_rss_MB: 6374.7 | allocated_MB: 449.0 | reserved_MB: 26252.0 | inactive_split_MB: 1054.4 +rank5 [2026-10-28 11:27:10,390] INFO : Step 300/50000 rank5 | Train Loss: 3.5114 | Train Loss Unweighted: 2.0975 | Train Loss Components: Auxiliary: 0.6791 | Cardinality: 0.3510 | Classification: 0.4201 | Classification_binary: 0.4975 | HitGrouping: 0.2993 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5752 | Regression_eta: 0.0073 | Regression_pt: 0.5905 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2763 | Classification_binary: 0.4898 | Regression_cos_phi: 0.1692 | Regression_energy: 0.3158 | Regression_eta: 0.1520 | Regression_pt: 0.3531 | Regression_sin_phi: 0.2176 | LR: 1.00e-03 | DataLoad Time: 0.0154s | Model Forward Time: 4.2604s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.7369 | max_loss: 0.1526 | max_backward: 1.7330 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.9030 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 4773.0 | uss_MB: 3484.3 | child_rss_MB: 6364.6 | allocated_MB: 446.1 | reserved_MB: 26252.0 | inactive_split_MB: 1068.0 +rank5 [2026-10-28 11:31:37,117] INFO : Step 400/50000 rank5 | Train Loss: 3.3615 | Train Loss Unweighted: 1.9456 | Train Loss Components: Auxiliary: 0.6310 | Cardinality: 0.3555 | Classification: 0.4058 | Classification_binary: 0.4436 | HitGrouping: 0.3006 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5363 | Regression_eta: 0.0080 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2254 | Classification_binary: 0.4551 | Regression_cos_phi: 0.2150 | Regression_energy: 0.2852 | Regression_eta: 0.1609 | Regression_pt: 0.3178 | Regression_sin_phi: 0.2718 | LR: 1.00e-03 | DataLoad Time: 0.0167s | Model Forward Time: 4.5034s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1151 | max_forward: 0.7642 | max_loss: 0.1384 | max_backward: 1.7906 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9359 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0880 | Memory: rss_MB: 4771.8 | uss_MB: 3484.8 | child_rss_MB: 6366.3 | allocated_MB: 446.8 | reserved_MB: 26252.0 | inactive_split_MB: 1064.0 +rank5 [2026-10-28 11:36:08,659] INFO : Step 500/50000 rank5 | Train Loss: 3.8257 | Train Loss Unweighted: 2.0630 | Train Loss Components: Auxiliary: 0.7010 | Cardinality: 0.4740 | Classification: 0.4380 | Classification_binary: 0.4817 | HitGrouping: 0.2961 | Regression_cos_phi: 0.0044 | Regression_energy: 0.5611 | Regression_eta: 0.0087 | Regression_pt: 0.5645 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2385 | Classification_binary: 0.4225 | Regression_cos_phi: 0.2209 | Regression_energy: 0.3254 | Regression_eta: 0.1726 | Regression_pt: 0.3481 | Regression_sin_phi: 0.2004 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 5.1256s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0077 | max_forward: 1.2433 | max_loss: 0.1536 | max_backward: 2.4885 | max_optimizer_step: 0.1562 | max_tensorboard_logging: 1.0374 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0034 | Memory: rss_MB: 4768.9 | uss_MB: 3486.1 | child_rss_MB: 6361.5 | allocated_MB: 449.6 | reserved_MB: 26256.0 | inactive_split_MB: 929.2 +rank5 [2026-10-28 11:40:42,094] INFO : Step 600/50000 rank5 | Train Loss: 3.4702 | Train Loss Unweighted: 1.9733 | Train Loss Components: Auxiliary: 0.6509 | Cardinality: 0.3585 | Classification: 0.3825 | Classification_binary: 0.5061 | HitGrouping: 0.2915 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5321 | Regression_eta: 0.0074 | Regression_pt: 0.5377 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2415 | Classification_binary: 0.5066 | Regression_cos_phi: 0.3819 | Regression_energy: 0.3109 | Regression_eta: 0.2512 | Regression_pt: 0.3381 | Regression_sin_phi: 0.2876 | LR: 1.00e-03 | DataLoad Time: 0.0162s | Model Forward Time: 4.5574s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8098 | max_loss: 0.1544 | max_backward: 1.9541 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 0.9377 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 4772.7 | uss_MB: 3486.1 | child_rss_MB: 6366.7 | allocated_MB: 449.5 | reserved_MB: 27396.0 | inactive_split_MB: 1022.5 +rank5 [2026-10-28 11:45:16,604] INFO : Step 700/50000 rank5 | Train Loss: 2.9693 | Train Loss Unweighted: 1.8072 | Train Loss Components: Auxiliary: 0.5907 | Cardinality: 0.2768 | Classification: 0.3175 | Classification_binary: 0.3969 | HitGrouping: 0.2848 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5321 | Regression_eta: 0.0061 | Regression_pt: 0.5492 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1848 | Classification_binary: 0.3849 | Regression_cos_phi: 0.3318 | Regression_energy: 0.2975 | Regression_eta: 0.1951 | Regression_pt: 0.3369 | Regression_sin_phi: 0.2197 | LR: 1.00e-03 | DataLoad Time: 0.0213s | Model Forward Time: 3.9458s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7910 | max_loss: 0.2436 | max_backward: 1.8672 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.8 | uss_MB: 3484.3 | child_rss_MB: 6369.5 | allocated_MB: 446.1 | reserved_MB: 27396.0 | inactive_split_MB: 1066.2 +rank5 [2026-10-28 11:49:48,175] INFO : Step 800/50000 rank5 | Train Loss: 3.1527 | Train Loss Unweighted: 1.8424 | Train Loss Components: Auxiliary: 0.6214 | Cardinality: 0.3518 | Classification: 0.3758 | Classification_binary: 0.3967 | HitGrouping: 0.2962 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5290 | Regression_eta: 0.0063 | Regression_pt: 0.5287 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2197 | Classification_binary: 0.3849 | Regression_cos_phi: 0.2135 | Regression_energy: 0.3064 | Regression_eta: 0.1623 | Regression_pt: 0.3241 | Regression_sin_phi: 0.2608 | LR: 9.99e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6445s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0093 | max_forward: 0.8507 | max_loss: 0.1493 | max_backward: 1.9706 | max_optimizer_step: 0.1582 | max_tensorboard_logging: 1.0840 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0072 | Memory: rss_MB: 4754.0 | uss_MB: 3484.3 | child_rss_MB: 6293.3 | allocated_MB: 450.9 | reserved_MB: 27396.0 | inactive_split_MB: 920.9 +rank5 [2026-10-28 11:54:30,570] INFO : Step 900/50000 rank5 | Train Loss: 3.2552 | Train Loss Unweighted: 1.9216 | Train Loss Components: Auxiliary: 0.6208 | Cardinality: 0.2901 | Classification: 0.3818 | Classification_binary: 0.4734 | HitGrouping: 0.2881 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5239 | Regression_eta: 0.0075 | Regression_pt: 0.5284 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2374 | Classification_binary: 0.4750 | Regression_cos_phi: 0.3415 | Regression_energy: 0.3024 | Regression_eta: 0.2565 | Regression_pt: 0.3286 | Regression_sin_phi: 0.3279 | LR: 9.99e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6740s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8050 | max_loss: 0.1486 | max_backward: 1.9075 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.9729 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4751.7 | uss_MB: 3485.4 | child_rss_MB: 6307.4 | allocated_MB: 449.7 | reserved_MB: 27406.0 | inactive_split_MB: 927.3 +rank5 [2026-10-28 11:59:03,648] INFO : Step 1000/50000 rank5 | Train Loss: 2.8964 | Train Loss Unweighted: 1.7179 | Train Loss Components: Auxiliary: 0.5694 | Cardinality: 0.2509 | Classification: 0.3299 | Classification_binary: 0.4129 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4804 | Regression_eta: 0.0063 | Regression_pt: 0.4827 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2051 | Classification_binary: 0.4216 | Regression_cos_phi: 0.2394 | Regression_energy: 0.2838 | Regression_eta: 0.2200 | Regression_pt: 0.3050 | Regression_sin_phi: 0.2569 | LR: 9.99e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4516s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7367 | max_loss: 0.2419 | max_backward: 1.8137 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 1.0408 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 4746.9 | uss_MB: 3485.7 | child_rss_MB: 6281.8 | allocated_MB: 449.1 | reserved_MB: 27406.0 | inactive_split_MB: 959.2 +rank5 [2026-10-28 11:59:04,553] INFO : Running validation on rank5 +rank5 [2026-10-28 11:59:04,554] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 11:59:04,554] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 11:59:57,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 11:59:57,578] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 12:00:26,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 12:00:27,966] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 12:00:29,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 12:00:30,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 12:00:40,024] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-28 12:00:40,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:00:41,019] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 12:00:56,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 12:00:57,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 12:00:58,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 12:00:59,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 12:01:09,747] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 12:01:10,610] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:01:11,131] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 12:01:26,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 12:01:27,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 12:01:28,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 12:01:29,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 12:01:39,349] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 12:06:47,467] INFO : Step 1100/50000 rank5 | Train Loss: 2.7351 | Train Loss Unweighted: 1.6531 | Train Loss Components: Auxiliary: 0.5524 | Cardinality: 0.2470 | Classification: 0.3238 | Classification_binary: 0.3516 | HitGrouping: 0.2744 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4831 | Regression_eta: 0.0054 | Regression_pt: 0.4842 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1732 | Classification_binary: 0.2982 | Regression_cos_phi: 0.0966 | Regression_energy: 0.3290 | Regression_eta: 0.1104 | Regression_pt: 0.3604 | Regression_sin_phi: 0.1316 | LR: 9.99e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7315s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7345 | max_loss: 0.2408 | max_backward: 1.8906 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 1.1618 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6205.6 | uss_MB: 4949.9 | child_rss_MB: 6308.4 | allocated_MB: 448.2 | reserved_MB: 24590.0 | inactive_split_MB: 1051.2 +rank5 [2026-10-28 12:11:28,615] INFO : Step 1200/50000 rank5 | Train Loss: 2.9931 | Train Loss Unweighted: 1.7517 | Train Loss Components: Auxiliary: 0.5886 | Cardinality: 0.2847 | Classification: 0.3460 | Classification_binary: 0.4151 | HitGrouping: 0.2795 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4874 | Regression_eta: 0.0065 | Regression_pt: 0.4908 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2088 | Classification_binary: 0.4171 | Regression_cos_phi: 0.2919 | Regression_energy: 0.2517 | Regression_eta: 0.2058 | Regression_pt: 0.2709 | Regression_sin_phi: 0.2143 | LR: 9.99e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.9395s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0061 | max_forward: 0.9910 | max_loss: 0.1662 | max_backward: 2.3256 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.8355 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 6205.6 | uss_MB: 4943.2 | child_rss_MB: 6333.1 | allocated_MB: 449.6 | reserved_MB: 24590.0 | inactive_split_MB: 1155.8 +rank5 [2026-10-28 12:16:08,306] INFO : Step 1300/50000 rank5 | Train Loss: 3.0304 | Train Loss Unweighted: 1.7607 | Train Loss Components: Auxiliary: 0.5900 | Cardinality: 0.2909 | Classification: 0.3397 | Classification_binary: 0.4400 | HitGrouping: 0.2683 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4807 | Regression_eta: 0.0062 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2424 | Classification_binary: 0.4730 | Regression_cos_phi: 0.2588 | Regression_energy: 0.2942 | Regression_eta: 0.2101 | Regression_pt: 0.3243 | Regression_sin_phi: 0.3620 | LR: 9.98e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3301s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8206 | max_loss: 0.1504 | max_backward: 1.9515 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5688 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5595.4 | uss_MB: 4349.7 | child_rss_MB: 6286.2 | allocated_MB: 448.6 | reserved_MB: 25778.0 | inactive_split_MB: 1050.0 +rank5 [2026-10-28 12:20:48,119] INFO : Step 1400/50000 rank5 | Train Loss: 2.5732 | Train Loss Unweighted: 1.5137 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2349 | Classification: 0.2906 | Classification_binary: 0.3573 | HitGrouping: 0.2641 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4252 | Regression_eta: 0.0055 | Regression_pt: 0.4303 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1841 | Classification_binary: 0.3751 | Regression_cos_phi: 0.1730 | Regression_energy: 0.2284 | Regression_eta: 0.1160 | Regression_pt: 0.2514 | Regression_sin_phi: 0.1880 | LR: 9.98e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1775s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.8166 | max_loss: 0.1962 | max_backward: 1.8388 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4964 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5588.2 | uss_MB: 3765.0 | child_rss_MB: 6275.0 | allocated_MB: 454.5 | reserved_MB: 28272.0 | inactive_split_MB: 1143.7 +rank5 [2026-10-28 12:25:37,859] INFO : Step 1500/50000 rank5 | Train Loss: 2.8564 | Train Loss Unweighted: 1.5902 | Train Loss Components: Auxiliary: 0.5576 | Cardinality: 0.3788 | Classification: 0.3167 | Classification_binary: 0.3534 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4517 | Regression_eta: 0.0055 | Regression_pt: 0.4576 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1701 | Classification_binary: 0.3680 | Regression_cos_phi: 0.1852 | Regression_energy: 0.2822 | Regression_eta: 0.1538 | Regression_pt: 0.3077 | Regression_sin_phi: 0.1765 | LR: 9.98e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6181s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.0272 | max_loss: 0.2459 | max_backward: 1.8682 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.0883 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0105 | Memory: rss_MB: 5566.9 | uss_MB: 4329.3 | child_rss_MB: 6244.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 989.3 +rank5 [2026-10-28 12:30:20,921] INFO : Step 1600/50000 rank5 | Train Loss: 2.8293 | Train Loss Unweighted: 1.6754 | Train Loss Components: Auxiliary: 0.5537 | Cardinality: 0.2456 | Classification: 0.3224 | Classification_binary: 0.4168 | HitGrouping: 0.2726 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4572 | Regression_eta: 0.0058 | Regression_pt: 0.4676 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1993 | Classification_binary: 0.4118 | Regression_cos_phi: 0.2484 | Regression_energy: 0.2544 | Regression_eta: 0.1808 | Regression_pt: 0.2833 | Regression_sin_phi: 0.2382 | LR: 9.98e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5692s | Phase Times: max_pre_step_barrier: 0.1094 | max_batch_to_device: 0.0066 | max_forward: 0.7720 | max_loss: 0.1510 | max_backward: 1.7462 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5578.6 | uss_MB: 4348.9 | child_rss_MB: 6226.8 | allocated_MB: 448.1 | reserved_MB: 28272.0 | inactive_split_MB: 989.9 +rank5 [2026-10-28 12:35:04,109] INFO : Step 1700/50000 rank5 | Train Loss: 2.7295 | Train Loss Unweighted: 1.5885 | Train Loss Components: Auxiliary: 0.5261 | Cardinality: 0.2505 | Classification: 0.2942 | Classification_binary: 0.3852 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4488 | Regression_eta: 0.0059 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1880 | Classification_binary: 0.4147 | Regression_cos_phi: 0.2933 | Regression_energy: 0.2272 | Regression_eta: 0.1721 | Regression_pt: 0.2424 | Regression_sin_phi: 0.3025 | LR: 9.97e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9352s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 1.0304 | max_loss: 0.4846 | max_backward: 1.9957 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0278 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5571.8 | uss_MB: 4349.5 | child_rss_MB: 6226.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 999.4 +rank5 [2026-10-28 12:39:46,515] INFO : Step 1800/50000 rank5 | Train Loss: 2.5541 | Train Loss Unweighted: 1.4989 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2504 | Classification: 0.2799 | Classification_binary: 0.3357 | HitGrouping: 0.2658 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4350 | Regression_eta: 0.0052 | Regression_pt: 0.4384 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1487 | Classification_binary: 0.3281 | Regression_cos_phi: 0.1901 | Regression_energy: 0.2581 | Regression_eta: 0.1605 | Regression_pt: 0.2803 | Regression_sin_phi: 0.1845 | LR: 9.97e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.3589s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7316 | max_loss: 0.1436 | max_backward: 1.6570 | max_optimizer_step: 0.2625 | max_tensorboard_logging: 0.9220 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5563.3 | uss_MB: 4348.3 | child_rss_MB: 6170.2 | allocated_MB: 444.7 | reserved_MB: 28272.0 | inactive_split_MB: 991.9 +rank5 [2026-10-28 12:44:29,074] INFO : Step 1900/50000 rank5 | Train Loss: 2.7201 | Train Loss Unweighted: 1.5780 | Train Loss Components: Auxiliary: 0.5257 | Cardinality: 0.2710 | Classification: 0.3152 | Classification_binary: 0.3657 | HitGrouping: 0.2653 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4371 | Regression_eta: 0.0058 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2056 | Classification_binary: 0.3866 | Regression_cos_phi: 0.2310 | Regression_energy: 0.2436 | Regression_eta: 0.1354 | Regression_pt: 0.2736 | Regression_sin_phi: 0.1895 | LR: 9.97e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4051s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.7723 | max_loss: 0.1448 | max_backward: 2.0316 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5561.2 | uss_MB: 4335.4 | child_rss_MB: 6153.7 | allocated_MB: 448.9 | reserved_MB: 28272.0 | inactive_split_MB: 1166.2 +rank5 [2026-10-28 12:49:11,972] INFO : Step 2000/50000 rank5 | Train Loss: 2.7893 | Train Loss Unweighted: 1.5772 | Train Loss Components: Auxiliary: 0.5342 | Cardinality: 0.3149 | Classification: 0.3056 | Classification_binary: 0.4233 | HitGrouping: 0.2623 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4169 | Regression_eta: 0.0058 | Regression_pt: 0.4204 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1794 | Classification_binary: 0.4232 | Regression_cos_phi: 0.2088 | Regression_energy: 0.2017 | Regression_eta: 0.1424 | Regression_pt: 0.2206 | Regression_sin_phi: 0.1676 | LR: 9.96e-04 | DataLoad Time: 0.0144s | Model Forward Time: 5.5230s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0070 | max_forward: 1.0424 | max_loss: 0.2160 | max_backward: 2.3184 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.9849 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0132 | Memory: rss_MB: 5555.4 | uss_MB: 4316.1 | child_rss_MB: 6148.5 | allocated_MB: 451.3 | reserved_MB: 28272.0 | inactive_split_MB: 992.4 +rank5 [2026-10-28 12:49:12,927] INFO : Running validation on rank5 +rank5 [2026-10-28 12:49:12,927] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 12:49:12,927] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 12:50:07,789] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:50:08,359] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 12:50:34,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 12:50:35,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 12:50:36,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 12:50:37,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 12:50:47,302] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 12:50:48,335] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:50:48,900] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 12:51:04,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 12:51:05,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 12:51:06,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 12:51:07,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 12:51:17,336] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 12:51:18,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 12:51:18,824] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 12:51:34,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 12:51:35,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 12:51:36,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 12:51:36,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 12:51:46,878] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 12:57:01,909] INFO : Step 2100/50000 rank5 | Train Loss: 2.3990 | Train Loss Unweighted: 1.4049 | Train Loss Components: Auxiliary: 0.4671 | Cardinality: 0.2199 | Classification: 0.2491 | Classification_binary: 0.3399 | HitGrouping: 0.2605 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4015 | Regression_eta: 0.0046 | Regression_pt: 0.4053 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1558 | Classification_binary: 0.3643 | Regression_cos_phi: 0.2245 | Regression_energy: 0.2036 | Regression_eta: 0.1423 | Regression_pt: 0.2263 | Regression_sin_phi: 0.2607 | LR: 9.96e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.8139s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7238 | max_loss: 0.1490 | max_backward: 1.8098 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.2800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6608.5 | uss_MB: 5401.4 | child_rss_MB: 6187.1 | allocated_MB: 444.8 | reserved_MB: 23654.0 | inactive_split_MB: 964.6 +rank5 [2026-10-28 13:01:41,081] INFO : Step 2200/50000 rank5 | Train Loss: 2.6105 | Train Loss Unweighted: 1.4907 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2489 | Classification: 0.2976 | Classification_binary: 0.3552 | HitGrouping: 0.2519 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4108 | Regression_eta: 0.0054 | Regression_pt: 0.4161 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1418 | Classification_binary: 0.3457 | Regression_cos_phi: 0.2373 | Regression_energy: 0.2223 | Regression_eta: 0.1023 | Regression_pt: 0.2412 | Regression_sin_phi: 0.2208 | LR: 9.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9419s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 1.0006 | max_loss: 0.1446 | max_backward: 1.7671 | max_optimizer_step: 0.2923 | max_tensorboard_logging: 1.2733 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6609.6 | uss_MB: 5403.3 | child_rss_MB: 6178.9 | allocated_MB: 445.7 | reserved_MB: 24780.0 | inactive_split_MB: 1132.5 +rank5 [2026-10-28 13:06:12,758] INFO : Step 2300/50000 rank5 | Train Loss: 2.7462 | Train Loss Unweighted: 1.5643 | Train Loss Components: Auxiliary: 0.5175 | Cardinality: 0.2724 | Classification: 0.3064 | Classification_binary: 0.4106 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4145 | Regression_eta: 0.0056 | Regression_pt: 0.4213 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1814 | Classification_binary: 0.3811 | Regression_cos_phi: 0.2139 | Regression_energy: 0.2278 | Regression_eta: 0.1290 | Regression_pt: 0.2489 | Regression_sin_phi: 0.1909 | LR: 9.95e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7176s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0353 | max_forward: 0.7930 | max_loss: 0.1572 | max_backward: 1.9535 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0140 | Memory: rss_MB: 5702.4 | uss_MB: 4498.0 | child_rss_MB: 6178.9 | allocated_MB: 451.2 | reserved_MB: 26626.0 | inactive_split_MB: 1036.8 +rank5 [2026-10-28 13:10:49,217] INFO : Step 2400/50000 rank5 | Train Loss: 2.5721 | Train Loss Unweighted: 1.4496 | Train Loss Components: Auxiliary: 0.4832 | Cardinality: 0.2667 | Classification: 0.2677 | Classification_binary: 0.3686 | HitGrouping: 0.2521 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3975 | Regression_eta: 0.0053 | Regression_pt: 0.4051 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1262 | Classification_binary: 0.3550 | Regression_cos_phi: 0.1568 | Regression_energy: 0.2157 | Regression_eta: 0.1014 | Regression_pt: 0.2397 | Regression_sin_phi: 0.1627 | LR: 9.95e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.0740s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7367 | max_loss: 0.1564 | max_backward: 1.6902 | max_optimizer_step: 0.2413 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5703.1 | uss_MB: 4496.2 | child_rss_MB: 6167.5 | allocated_MB: 448.4 | reserved_MB: 26626.0 | inactive_split_MB: 1138.1 +rank5 [2026-10-28 13:15:33,529] INFO : Step 2500/50000 rank5 | Train Loss: 2.3709 | Train Loss Unweighted: 1.3478 | Train Loss Components: Auxiliary: 0.4634 | Cardinality: 0.2742 | Classification: 0.2528 | Classification_binary: 0.3100 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3861 | Regression_eta: 0.0042 | Regression_pt: 0.3903 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1643 | Classification_binary: 0.3024 | Regression_cos_phi: 0.1466 | Regression_energy: 0.1904 | Regression_eta: 0.1327 | Regression_pt: 0.2081 | Regression_sin_phi: 0.1885 | LR: 9.94e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3389s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7598 | max_loss: 0.2442 | max_backward: 1.9617 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5783 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5689.7 | uss_MB: 4483.6 | child_rss_MB: 6173.2 | allocated_MB: 447.2 | reserved_MB: 27052.0 | inactive_split_MB: 1262.0 +rank5 [2026-10-28 13:20:26,044] INFO : Step 2600/50000 rank5 | Train Loss: 2.6216 | Train Loss Unweighted: 1.5111 | Train Loss Components: Auxiliary: 0.4988 | Cardinality: 0.2823 | Classification: 0.2863 | Classification_binary: 0.3723 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4174 | Regression_eta: 0.0052 | Regression_pt: 0.4247 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1639 | Classification_binary: 0.3697 | Regression_cos_phi: 0.1961 | Regression_energy: 0.2140 | Regression_eta: 0.1476 | Regression_pt: 0.2336 | Regression_sin_phi: 0.2168 | LR: 9.94e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2879s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8125 | max_loss: 0.1494 | max_backward: 1.9790 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5698.8 | uss_MB: 4500.2 | child_rss_MB: 6171.1 | allocated_MB: 448.2 | reserved_MB: 28734.0 | inactive_split_MB: 1300.2 +rank5 [2026-10-28 13:25:15,883] INFO : Step 2700/50000 rank5 | Train Loss: 2.5735 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4801 | Cardinality: 0.2457 | Classification: 0.2692 | Classification_binary: 0.4021 | HitGrouping: 0.2514 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3948 | Regression_eta: 0.0060 | Regression_pt: 0.3979 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1515 | Classification_binary: 0.4029 | Regression_cos_phi: 0.2411 | Regression_energy: 0.2032 | Regression_eta: 0.1598 | Regression_pt: 0.2234 | Regression_sin_phi: 0.1721 | LR: 9.94e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3201s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0070 | max_forward: 0.8435 | max_loss: 0.1556 | max_backward: 1.9409 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5695.7 | uss_MB: 4500.5 | child_rss_MB: 6182.0 | allocated_MB: 447.4 | reserved_MB: 28734.0 | inactive_split_MB: 980.7 +rank5 [2026-10-28 13:29:49,852] INFO : Step 2800/50000 rank5 | Train Loss: 2.3517 | Train Loss Unweighted: 1.3315 | Train Loss Components: Auxiliary: 0.4396 | Cardinality: 0.2218 | Classification: 0.2269 | Classification_binary: 0.3206 | HitGrouping: 0.2464 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3829 | Regression_eta: 0.0052 | Regression_pt: 0.3909 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1398 | Classification_binary: 0.3435 | Regression_cos_phi: 0.2232 | Regression_energy: 0.2041 | Regression_eta: 0.1668 | Regression_pt: 0.2314 | Regression_sin_phi: 0.2484 | LR: 9.93e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3201s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0062 | max_forward: 0.8898 | max_loss: 0.1952 | max_backward: 1.8882 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0163 | Memory: rss_MB: 5687.7 | uss_MB: 4493.8 | child_rss_MB: 6179.6 | allocated_MB: 445.9 | reserved_MB: 28734.0 | inactive_split_MB: 1208.5 +rank5 [2026-10-28 13:34:34,296] INFO : Step 2900/50000 rank5 | Train Loss: 2.4505 | Train Loss Unweighted: 1.3674 | Train Loss Components: Auxiliary: 0.4603 | Cardinality: 0.2746 | Classification: 0.2611 | Classification_binary: 0.3388 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3750 | Regression_eta: 0.0049 | Regression_pt: 0.3827 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1721 | Classification_binary: 0.3170 | Regression_cos_phi: 0.2283 | Regression_energy: 0.1922 | Regression_eta: 0.1357 | Regression_pt: 0.2136 | Regression_sin_phi: 0.1970 | LR: 9.93e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.3933s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7484 | max_loss: 0.6089 | max_backward: 2.2353 | max_optimizer_step: 0.1775 | max_tensorboard_logging: 2.4313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0155 | Memory: rss_MB: 5691.8 | uss_MB: 4497.3 | child_rss_MB: 6175.9 | allocated_MB: 446.7 | reserved_MB: 28734.0 | inactive_split_MB: 1133.2 +rank5 [2026-10-28 13:39:22,258] INFO : Step 3000/50000 rank5 | Train Loss: 2.6179 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4760 | Cardinality: 0.2600 | Classification: 0.3170 | Classification_binary: 0.3907 | HitGrouping: 0.2509 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3743 | Regression_eta: 0.0054 | Regression_pt: 0.3819 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2347 | Classification_binary: 0.4101 | Regression_cos_phi: 0.3147 | Regression_energy: 0.2158 | Regression_eta: 0.1887 | Regression_pt: 0.2365 | Regression_sin_phi: 0.2809 | LR: 9.92e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.8198s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0842 | max_forward: 0.8203 | max_loss: 0.1511 | max_backward: 1.9714 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.0307 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5691.4 | uss_MB: 4499.3 | child_rss_MB: 6154.8 | allocated_MB: 451.4 | reserved_MB: 28740.0 | inactive_split_MB: 1256.0 +rank5 [2026-10-28 13:39:23,129] INFO : Running validation on rank5 +rank5 [2026-10-28 13:39:23,129] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 13:39:23,130] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 13:40:15,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:40:16,491] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 13:40:44,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 13:40:45,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 13:40:46,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 13:40:47,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 13:40:57,189] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 13:40:57,943] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:40:58,511] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 13:41:14,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 13:41:15,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 13:41:16,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 13:41:17,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 13:41:27,523] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 13:41:28,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 13:41:28,597] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 13:41:44,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 13:41:45,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 13:41:46,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 13:41:46,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 13:41:56,992] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 13:46:51,408] INFO : Step 3100/50000 rank5 | Train Loss: 2.3456 | Train Loss Unweighted: 1.3820 | Train Loss Components: Auxiliary: 0.4397 | Cardinality: 0.2017 | Classification: 0.2549 | Classification_binary: 0.3432 | HitGrouping: 0.2399 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3852 | Regression_eta: 0.0046 | Regression_pt: 0.3892 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1442 | Classification_binary: 0.3330 | Regression_cos_phi: 0.1656 | Regression_energy: 0.2107 | Regression_eta: 0.1137 | Regression_pt: 0.2279 | Regression_sin_phi: 0.1575 | LR: 9.91e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4916s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0425 | max_forward: 0.7824 | max_loss: 0.1439 | max_backward: 1.7183 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.8666 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6905.9 | uss_MB: 5708.9 | child_rss_MB: 6176.0 | allocated_MB: 447.6 | reserved_MB: 24468.0 | inactive_split_MB: 1269.5 +rank5 [2026-10-28 13:51:36,981] INFO : Step 3200/50000 rank5 | Train Loss: 2.2453 | Train Loss Unweighted: 1.2729 | Train Loss Components: Auxiliary: 0.4171 | Cardinality: 0.2143 | Classification: 0.2366 | Classification_binary: 0.3116 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3569 | Regression_eta: 0.0047 | Regression_pt: 0.3584 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1229 | Classification_binary: 0.2926 | Regression_cos_phi: 0.1553 | Regression_energy: 0.1684 | Regression_eta: 0.1212 | Regression_pt: 0.1820 | Regression_sin_phi: 0.1692 | LR: 9.91e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6666s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7985 | max_loss: 0.1379 | max_backward: 1.7895 | max_optimizer_step: 0.2027 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6908.5 | uss_MB: 5710.9 | child_rss_MB: 6167.1 | allocated_MB: 448.6 | reserved_MB: 26448.0 | inactive_split_MB: 1391.4 +rank5 [2026-10-28 13:56:16,618] INFO : Step 3300/50000 rank5 | Train Loss: 2.5474 | Train Loss Unweighted: 1.4472 | Train Loss Components: Auxiliary: 0.4779 | Cardinality: 0.2732 | Classification: 0.2824 | Classification_binary: 0.3640 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3911 | Regression_eta: 0.0054 | Regression_pt: 0.3992 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1861 | Classification_binary: 0.3464 | Regression_cos_phi: 0.2037 | Regression_energy: 0.2018 | Regression_eta: 0.1664 | Regression_pt: 0.2207 | Regression_sin_phi: 0.2344 | LR: 9.90e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6661s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8323 | max_loss: 0.1479 | max_backward: 2.0543 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.7834 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5785.1 | uss_MB: 4587.8 | child_rss_MB: 6143.0 | allocated_MB: 448.7 | reserved_MB: 26460.0 | inactive_split_MB: 1087.4 +rank5 [2026-10-28 14:00:49,247] INFO : Step 3400/50000 rank5 | Train Loss: 2.5743 | Train Loss Unweighted: 1.4358 | Train Loss Components: Auxiliary: 0.4720 | Cardinality: 0.2664 | Classification: 0.2699 | Classification_binary: 0.3915 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3784 | Regression_eta: 0.0053 | Regression_pt: 0.3851 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1592 | Classification_binary: 0.4334 | Regression_cos_phi: 0.2661 | Regression_energy: 0.1779 | Regression_eta: 0.1372 | Regression_pt: 0.1943 | Regression_sin_phi: 0.2159 | LR: 9.90e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2599s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7827 | max_loss: 0.1630 | max_backward: 1.9191 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5606 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5794.1 | uss_MB: 4597.7 | child_rss_MB: 6161.5 | allocated_MB: 450.4 | reserved_MB: 28138.0 | inactive_split_MB: 1261.0 +rank5 [2026-10-28 14:05:23,603] INFO : Step 3500/50000 rank5 | Train Loss: 2.1916 | Train Loss Unweighted: 1.2418 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2204 | Classification: 0.2212 | Classification_binary: 0.3041 | HitGrouping: 0.2426 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3517 | Regression_eta: 0.0047 | Regression_pt: 0.3558 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1370 | Classification_binary: 0.2844 | Regression_cos_phi: 0.1784 | Regression_energy: 0.1898 | Regression_eta: 0.1203 | Regression_pt: 0.2022 | Regression_sin_phi: 0.1719 | LR: 9.89e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2267s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7972 | max_loss: 0.1441 | max_backward: 1.7398 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6845 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5780.0 | uss_MB: 4584.6 | child_rss_MB: 6167.3 | allocated_MB: 443.7 | reserved_MB: 28532.0 | inactive_split_MB: 1176.3 +rank5 [2026-10-28 14:10:06,692] INFO : Step 3600/50000 rank5 | Train Loss: 2.3195 | Train Loss Unweighted: 1.3024 | Train Loss Components: Auxiliary: 0.4248 | Cardinality: 0.2221 | Classification: 0.2518 | Classification_binary: 0.3172 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3610 | Regression_eta: 0.0052 | Regression_pt: 0.3623 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1536 | Classification_binary: 0.3182 | Regression_cos_phi: 0.2157 | Regression_energy: 0.1611 | Regression_eta: 0.1269 | Regression_pt: 0.1777 | Regression_sin_phi: 0.1956 | LR: 9.89e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3064s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8456 | max_loss: 0.1388 | max_backward: 1.9466 | max_optimizer_step: 0.1811 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5787.8 | uss_MB: 4598.8 | child_rss_MB: 6163.2 | allocated_MB: 449.1 | reserved_MB: 28532.0 | inactive_split_MB: 1277.8 +rank5 [2026-10-28 14:14:47,797] INFO : Step 3700/50000 rank5 | Train Loss: 2.5861 | Train Loss Unweighted: 1.4427 | Train Loss Components: Auxiliary: 0.4700 | Cardinality: 0.2678 | Classification: 0.2978 | Classification_binary: 0.3844 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3719 | Regression_eta: 0.0058 | Regression_pt: 0.3773 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1900 | Classification_binary: 0.4262 | Regression_cos_phi: 0.2980 | Regression_energy: 0.2009 | Regression_eta: 0.1825 | Regression_pt: 0.2257 | Regression_sin_phi: 0.3159 | LR: 9.88e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.5017s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.8001 | max_loss: 0.1519 | max_backward: 1.8239 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.7916 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0181 | Memory: rss_MB: 5775.1 | uss_MB: 4600.5 | child_rss_MB: 6165.4 | allocated_MB: 450.6 | reserved_MB: 30118.0 | inactive_split_MB: 1097.6 +rank5 [2026-10-28 14:19:24,598] INFO : Step 3800/50000 rank5 | Train Loss: 2.3269 | Train Loss Unweighted: 1.3016 | Train Loss Components: Auxiliary: 0.4225 | Cardinality: 0.2364 | Classification: 0.2400 | Classification_binary: 0.3389 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3534 | Regression_eta: 0.0054 | Regression_pt: 0.3591 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1547 | Classification_binary: 0.3418 | Regression_cos_phi: 0.1982 | Regression_energy: 0.1932 | Regression_eta: 0.1565 | Regression_pt: 0.2145 | Regression_sin_phi: 0.2109 | LR: 9.87e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.4706s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7555 | max_loss: 0.1601 | max_backward: 1.7612 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.9394 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5772.0 | uss_MB: 4595.7 | child_rss_MB: 6158.5 | allocated_MB: 447.4 | reserved_MB: 30118.0 | inactive_split_MB: 1355.2 +rank5 [2026-10-28 14:24:05,269] INFO : Step 3900/50000 rank5 | Train Loss: 2.0276 | Train Loss Unweighted: 1.1874 | Train Loss Components: Auxiliary: 0.3861 | Cardinality: 0.1825 | Classification: 0.2116 | Classification_binary: 0.2707 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3485 | Regression_eta: 0.0038 | Regression_pt: 0.3489 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1143 | Classification_binary: 0.2611 | Regression_cos_phi: 0.1285 | Regression_energy: 0.1772 | Regression_eta: 0.1024 | Regression_pt: 0.1941 | Regression_sin_phi: 0.1538 | LR: 9.87e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.4473s | Phase Times: max_pre_step_barrier: 0.0095 | max_batch_to_device: 0.0140 | max_forward: 1.1007 | max_loss: 0.1867 | max_backward: 2.0952 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 0.5091 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5772.2 | uss_MB: 4598.5 | child_rss_MB: 6165.7 | allocated_MB: 448.6 | reserved_MB: 30118.0 | inactive_split_MB: 1119.4 +rank5 [2026-10-28 14:28:52,324] INFO : Step 4000/50000 rank5 | Train Loss: 2.3921 | Train Loss Unweighted: 1.3683 | Train Loss Components: Auxiliary: 0.4366 | Cardinality: 0.2254 | Classification: 0.2720 | Classification_binary: 0.3524 | HitGrouping: 0.2405 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3647 | Regression_eta: 0.0048 | Regression_pt: 0.3692 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1655 | Classification_binary: 0.3367 | Regression_cos_phi: 0.2215 | Regression_energy: 0.1801 | Regression_eta: 0.1135 | Regression_pt: 0.1958 | Regression_sin_phi: 0.1649 | LR: 9.86e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8919s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.8532 | max_loss: 0.1476 | max_backward: 2.4362 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.6138 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5773.5 | uss_MB: 4601.5 | child_rss_MB: 6164.9 | allocated_MB: 449.7 | reserved_MB: 30118.0 | inactive_split_MB: 899.8 +rank5 [2026-10-28 14:28:53,308] INFO : Running validation on rank5 +rank5 [2026-10-28 14:28:53,308] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 14:28:53,308] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 14:29:46,118] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:29:46,700] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 14:30:13,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 14:30:14,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 14:30:15,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 14:30:16,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 14:30:26,235] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 14:30:26,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:30:27,511] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 14:30:43,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 14:30:44,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 14:30:45,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 14:30:46,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 14:30:56,278] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 14:30:57,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 14:30:57,800] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 14:31:13,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 14:31:14,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 14:31:14,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 14:31:15,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 14:31:25,756] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 14:36:30,159] INFO : Step 4100/50000 rank5 | Train Loss: 2.5062 | Train Loss Unweighted: 1.3582 | Train Loss Components: Auxiliary: 0.4559 | Cardinality: 0.3208 | Classification: 0.2565 | Classification_binary: 0.3823 | HitGrouping: 0.2386 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3543 | Regression_eta: 0.0046 | Regression_pt: 0.3553 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1598 | Classification_binary: 0.3945 | Regression_cos_phi: 0.1456 | Regression_energy: 0.1867 | Regression_eta: 0.1146 | Regression_pt: 0.2053 | Regression_sin_phi: 0.2189 | LR: 9.85e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.2027s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0069 | max_forward: 0.8943 | max_loss: 0.2518 | max_backward: 1.9865 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 1.1537 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6976.0 | uss_MB: 5798.7 | child_rss_MB: 6190.7 | allocated_MB: 449.4 | reserved_MB: 25148.0 | inactive_split_MB: 1028.6 +rank5 [2026-10-28 14:41:18,155] INFO : Step 4200/50000 rank5 | Train Loss: 2.0961 | Train Loss Unweighted: 1.2232 | Train Loss Components: Auxiliary: 0.3927 | Cardinality: 0.2101 | Classification: 0.2242 | Classification_binary: 0.2902 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3497 | Regression_eta: 0.0042 | Regression_pt: 0.3510 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1424 | Classification_binary: 0.2973 | Regression_cos_phi: 0.1784 | Regression_energy: 0.1539 | Regression_eta: 0.1238 | Regression_pt: 0.1694 | Regression_sin_phi: 0.1741 | LR: 9.84e-04 | DataLoad Time: 0.0197s | Model Forward Time: 5.6527s | Phase Times: max_pre_step_barrier: 0.1131 | max_batch_to_device: 0.0199 | max_forward: 1.5911 | max_loss: 0.3393 | max_backward: 3.0707 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5197 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6973.9 | uss_MB: 5797.7 | child_rss_MB: 6186.7 | allocated_MB: 446.0 | reserved_MB: 25296.0 | inactive_split_MB: 922.1 +rank5 [2026-10-28 14:46:18,702] INFO : Step 4300/50000 rank5 | Train Loss: 2.2792 | Train Loss Unweighted: 1.3065 | Train Loss Components: Auxiliary: 0.4224 | Cardinality: 0.2158 | Classification: 0.2339 | Classification_binary: 0.3250 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3669 | Regression_eta: 0.0044 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1628 | Classification_binary: 0.3192 | Regression_cos_phi: 0.2373 | Regression_energy: 0.1976 | Regression_eta: 0.1434 | Regression_pt: 0.2166 | Regression_sin_phi: 0.3038 | LR: 9.84e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5473s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0938 | max_forward: 0.8451 | max_loss: 0.1396 | max_backward: 1.9669 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.6639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5862.7 | uss_MB: 4685.4 | child_rss_MB: 6195.4 | allocated_MB: 448.4 | reserved_MB: 28580.0 | inactive_split_MB: 1346.0 +rank5 [2026-10-28 14:50:58,885] INFO : Step 4400/50000 rank5 | Train Loss: 2.4955 | Train Loss Unweighted: 1.3385 | Train Loss Components: Auxiliary: 0.4510 | Cardinality: 0.3020 | Classification: 0.2394 | Classification_binary: 0.3842 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3482 | Regression_eta: 0.0051 | Regression_pt: 0.3563 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1485 | Classification_binary: 0.3623 | Regression_cos_phi: 0.3356 | Regression_energy: 0.1686 | Regression_eta: 0.1759 | Regression_pt: 0.1865 | Regression_sin_phi: 0.2411 | LR: 9.83e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2052s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0083 | max_forward: 0.7889 | max_loss: 0.1505 | max_backward: 1.8572 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4868 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5870.1 | uss_MB: 4697.1 | child_rss_MB: 6175.1 | allocated_MB: 449.1 | reserved_MB: 28582.0 | inactive_split_MB: 1105.9 +rank5 [2026-10-28 14:55:40,224] INFO : Step 4500/50000 rank5 | Train Loss: 2.3283 | Train Loss Unweighted: 1.2750 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.3015 | Classification: 0.2318 | Classification_binary: 0.3495 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3409 | Regression_eta: 0.0044 | Regression_pt: 0.3441 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1427 | Classification_binary: 0.3969 | Regression_cos_phi: 0.2209 | Regression_energy: 0.1859 | Regression_eta: 0.1290 | Regression_pt: 0.2051 | Regression_sin_phi: 0.2470 | LR: 9.82e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0178s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7326 | max_loss: 0.2428 | max_backward: 1.7841 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5868.2 | uss_MB: 4696.0 | child_rss_MB: 6178.6 | allocated_MB: 447.8 | reserved_MB: 28582.0 | inactive_split_MB: 1258.0 +rank5 [2026-10-28 15:00:16,448] INFO : Step 4600/50000 rank5 | Train Loss: 2.0353 | Train Loss Unweighted: 1.1777 | Train Loss Components: Auxiliary: 0.3804 | Cardinality: 0.1920 | Classification: 0.2053 | Classification_binary: 0.2642 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3490 | Regression_eta: 0.0039 | Regression_pt: 0.3512 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1139 | Classification_binary: 0.2554 | Regression_cos_phi: 0.1352 | Regression_energy: 0.2188 | Regression_eta: 0.1035 | Regression_pt: 0.2316 | Regression_sin_phi: 0.1537 | LR: 9.81e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0849s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0402 | max_forward: 0.7143 | max_loss: 0.1412 | max_backward: 1.6726 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 5861.4 | uss_MB: 4694.6 | child_rss_MB: 6186.8 | allocated_MB: 446.4 | reserved_MB: 28582.0 | inactive_split_MB: 981.4 +rank5 [2026-10-28 15:05:12,090] INFO : Step 4700/50000 rank5 | Train Loss: 2.2426 | Train Loss Unweighted: 1.2576 | Train Loss Components: Auxiliary: 0.4073 | Cardinality: 0.2222 | Classification: 0.2485 | Classification_binary: 0.3333 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3307 | Regression_eta: 0.0049 | Regression_pt: 0.3356 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1428 | Classification_binary: 0.3185 | Regression_cos_phi: 0.1759 | Regression_energy: 0.1594 | Regression_eta: 0.1275 | Regression_pt: 0.1745 | Regression_sin_phi: 0.1851 | LR: 9.81e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6209s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.1471 | max_backward: 2.0627 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.7062 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0769 | Memory: rss_MB: 5860.5 | uss_MB: 4699.2 | child_rss_MB: 6169.2 | allocated_MB: 447.7 | reserved_MB: 28652.0 | inactive_split_MB: 1164.9 +rank5 [2026-10-28 15:09:59,907] INFO : Step 4800/50000 rank5 | Train Loss: 2.2697 | Train Loss Unweighted: 1.2637 | Train Loss Components: Auxiliary: 0.4132 | Cardinality: 0.2493 | Classification: 0.2262 | Classification_binary: 0.3667 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3287 | Regression_eta: 0.0044 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1645 | Classification_binary: 0.4044 | Regression_cos_phi: 0.2647 | Regression_energy: 0.1873 | Regression_eta: 0.1302 | Regression_pt: 0.2084 | Regression_sin_phi: 0.2222 | LR: 9.80e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.0700s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0065 | max_forward: 0.7626 | max_loss: 0.1571 | max_backward: 1.6465 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5855.3 | uss_MB: 4693.4 | child_rss_MB: 6188.9 | allocated_MB: 450.7 | reserved_MB: 28652.0 | inactive_split_MB: 1088.6 +rank5 [2026-10-28 15:14:51,790] INFO : Step 4900/50000 rank5 | Train Loss: 2.0529 | Train Loss Unweighted: 1.1598 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.1942 | Classification: 0.2269 | Classification_binary: 0.2874 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3172 | Regression_eta: 0.0041 | Regression_pt: 0.3199 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1577 | Classification_binary: 0.2647 | Regression_cos_phi: 0.2350 | Regression_energy: 0.1620 | Regression_eta: 0.1117 | Regression_pt: 0.1751 | Regression_sin_phi: 0.1888 | LR: 9.79e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3247s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7293 | max_loss: 0.1974 | max_backward: 1.8633 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5858.6 | uss_MB: 4697.0 | child_rss_MB: 6189.0 | allocated_MB: 447.4 | reserved_MB: 28652.0 | inactive_split_MB: 1266.1 +rank5 [2026-10-28 15:19:59,465] INFO : Step 5000/50000 rank5 | Train Loss: 2.0827 | Train Loss Unweighted: 1.1872 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2154 | Classification: 0.2462 | Classification_binary: 0.2941 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3169 | Regression_eta: 0.0042 | Regression_pt: 0.3219 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1390 | Classification_binary: 0.3053 | Regression_cos_phi: 0.1858 | Regression_energy: 0.1699 | Regression_eta: 0.1255 | Regression_pt: 0.1886 | Regression_sin_phi: 0.1177 | LR: 9.78e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3260s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.8382 | max_loss: 0.1450 | max_backward: 1.9472 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0203 | Memory: rss_MB: 5858.9 | uss_MB: 4697.5 | child_rss_MB: 6177.0 | allocated_MB: 446.9 | reserved_MB: 29316.0 | inactive_split_MB: 980.8 +rank5 [2026-10-28 15:20:00,329] INFO : Running validation on rank5 +rank5 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 15:20:55,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:20:56,481] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 15:21:22,780] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 15:21:23,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 15:21:24,952] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 15:21:25,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 15:21:35,977] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 15:21:36,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:21:36,967] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 15:21:53,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 15:21:54,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 15:21:55,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 15:21:56,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 15:22:06,125] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 15:22:06,552] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 15:22:07,053] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 15:22:23,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 15:22:24,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 15:22:25,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 15:22:26,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 15:22:36,278] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 15:28:09,086] INFO : Step 5100/50000 rank5 | Train Loss: 2.4098 | Train Loss Unweighted: 1.3457 | Train Loss Components: Auxiliary: 0.4318 | Cardinality: 0.2397 | Classification: 0.2711 | Classification_binary: 0.3798 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3391 | Regression_eta: 0.0051 | Regression_pt: 0.3453 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1619 | Classification_binary: 0.3806 | Regression_cos_phi: 0.1944 | Regression_energy: 0.1733 | Regression_eta: 0.1312 | Regression_pt: 0.1918 | Regression_sin_phi: 0.1751 | LR: 9.77e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.8395s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1511 | max_backward: 1.7914 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 1.0544 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5949.7 | uss_MB: 4786.0 | child_rss_MB: 6159.3 | allocated_MB: 447.6 | reserved_MB: 25650.0 | inactive_split_MB: 1122.4 +rank5 [2026-10-28 15:32:58,661] INFO : Step 5200/50000 rank5 | Train Loss: 2.0929 | Train Loss Unweighted: 1.1974 | Train Loss Components: Auxiliary: 0.3875 | Cardinality: 0.2007 | Classification: 0.2193 | Classification_binary: 0.3278 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3194 | Regression_eta: 0.0041 | Regression_pt: 0.3227 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1699 | Classification_binary: 0.3443 | Regression_cos_phi: 0.1791 | Regression_energy: 0.1618 | Regression_eta: 0.1331 | Regression_pt: 0.1770 | Regression_sin_phi: 0.2032 | LR: 9.76e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4925s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7458 | max_loss: 0.2423 | max_backward: 1.7123 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9648 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5957.0 | uss_MB: 4796.7 | child_rss_MB: 6179.0 | allocated_MB: 450.7 | reserved_MB: 28708.0 | inactive_split_MB: 1139.3 +rank5 [2026-10-28 15:37:52,596] INFO : Step 5300/50000 rank5 | Train Loss: 1.9905 | Train Loss Unweighted: 1.1352 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.2083 | Classification: 0.2239 | Classification_binary: 0.2818 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0037 | Regression_pt: 0.3135 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1371 | Classification_binary: 0.2991 | Regression_cos_phi: 0.2002 | Regression_energy: 0.1590 | Regression_eta: 0.0954 | Regression_pt: 0.1751 | Regression_sin_phi: 0.2011 | LR: 9.75e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.1046s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 1.2453 | max_loss: 0.2326 | max_backward: 2.1392 | max_optimizer_step: 0.1883 | max_tensorboard_logging: 1.0947 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5945.0 | uss_MB: 4784.6 | child_rss_MB: 6156.3 | allocated_MB: 445.9 | reserved_MB: 30688.0 | inactive_split_MB: 925.3 +rank5 [2026-10-28 15:42:39,663] INFO : Step 5400/50000 rank5 | Train Loss: 2.2932 | Train Loss Unweighted: 1.2957 | Train Loss Components: Auxiliary: 0.4189 | Cardinality: 0.2411 | Classification: 0.2689 | Classification_binary: 0.3420 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3351 | Regression_eta: 0.0045 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1803 | Classification_binary: 0.3347 | Regression_cos_phi: 0.1773 | Regression_energy: 0.1825 | Regression_eta: 0.1231 | Regression_pt: 0.2010 | Regression_sin_phi: 0.2397 | LR: 9.74e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5706s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7688 | max_loss: 0.1697 | max_backward: 1.8189 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8641 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5955.2 | uss_MB: 4796.8 | child_rss_MB: 6163.2 | allocated_MB: 450.8 | reserved_MB: 30688.0 | inactive_split_MB: 915.1 +rank5 [2026-10-28 15:47:42,229] INFO : Step 5500/50000 rank5 | Train Loss: 2.1734 | Train Loss Unweighted: 1.2295 | Train Loss Components: Auxiliary: 0.3970 | Cardinality: 0.2269 | Classification: 0.2218 | Classification_binary: 0.3571 | HitGrouping: 0.2200 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3188 | Regression_eta: 0.0041 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1189 | Classification_binary: 0.3488 | Regression_cos_phi: 0.1950 | Regression_energy: 0.1737 | Regression_eta: 0.1220 | Regression_pt: 0.1921 | Regression_sin_phi: 0.1779 | LR: 9.73e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.9292s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0153 | max_forward: 1.1913 | max_loss: 0.6135 | max_backward: 3.0599 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.6982 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5952.5 | uss_MB: 4797.4 | child_rss_MB: 6170.1 | allocated_MB: 446.9 | reserved_MB: 32804.0 | inactive_split_MB: 913.2 +rank5 [2026-10-28 15:52:30,289] INFO : Step 5600/50000 rank5 | Train Loss: 2.0307 | Train Loss Unweighted: 1.1169 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.1966 | Classification: 0.2085 | Classification_binary: 0.3031 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2975 | Regression_eta: 0.0047 | Regression_pt: 0.2988 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1301 | Classification_binary: 0.2746 | Regression_cos_phi: 0.1498 | Regression_energy: 0.1994 | Regression_eta: 0.1242 | Regression_pt: 0.2148 | Regression_sin_phi: 0.1593 | LR: 9.72e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.7953s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7794 | max_loss: 0.1497 | max_backward: 1.9410 | max_optimizer_step: 0.5154 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0061 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5920.9 | uss_MB: 4797.0 | child_rss_MB: 6174.4 | allocated_MB: 445.4 | reserved_MB: 32804.0 | inactive_split_MB: 1164.2 +rank5 [2026-10-28 15:57:16,862] INFO : Step 5700/50000 rank5 | Train Loss: 2.2064 | Train Loss Unweighted: 1.2224 | Train Loss Components: Auxiliary: 0.4023 | Cardinality: 0.2320 | Classification: 0.2526 | Classification_binary: 0.3174 | HitGrouping: 0.2343 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3173 | Regression_eta: 0.0045 | Regression_pt: 0.3259 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1610 | Classification_binary: 0.3554 | Regression_cos_phi: 0.1705 | Regression_energy: 0.1573 | Regression_eta: 0.1385 | Regression_pt: 0.1773 | Regression_sin_phi: 0.2293 | LR: 9.71e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.6664s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7746 | max_loss: 0.4928 | max_backward: 1.9590 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8719 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 5893.1 | uss_MB: 3755.9 | child_rss_MB: 6168.2 | allocated_MB: 448.4 | reserved_MB: 32804.0 | inactive_split_MB: 913.0 +rank5 [2026-10-28 16:02:01,175] INFO : Step 5800/50000 rank5 | Train Loss: 2.1438 | Train Loss Unweighted: 1.2123 | Train Loss Components: Auxiliary: 0.3898 | Cardinality: 0.2335 | Classification: 0.2404 | Classification_binary: 0.3575 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3006 | Regression_eta: 0.0042 | Regression_pt: 0.3054 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1387 | Classification_binary: 0.3540 | Regression_cos_phi: 0.1442 | Regression_energy: 0.1539 | Regression_eta: 0.1124 | Regression_pt: 0.1716 | Regression_sin_phi: 0.1830 | LR: 9.70e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.5885s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8028 | max_loss: 0.1540 | max_backward: 1.8645 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.8379 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5883.5 | uss_MB: 4797.1 | child_rss_MB: 6169.4 | allocated_MB: 448.3 | reserved_MB: 32804.0 | inactive_split_MB: 916.5 +rank5 [2026-10-28 16:06:46,106] INFO : Step 5900/50000 rank5 | Train Loss: 2.2094 | Train Loss Unweighted: 1.1453 | Train Loss Components: Auxiliary: 0.3936 | Cardinality: 0.2891 | Classification: 0.2198 | Classification_binary: 0.3337 | HitGrouping: 0.2238 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2890 | Regression_eta: 0.0046 | Regression_pt: 0.2935 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1188 | Classification_binary: 0.3203 | Regression_cos_phi: 0.1861 | Regression_energy: 0.1519 | Regression_eta: 0.1120 | Regression_pt: 0.1678 | Regression_sin_phi: 0.1579 | LR: 9.69e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.5896s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0065 | max_forward: 1.0922 | max_loss: 0.1477 | max_backward: 1.8468 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8251 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0991 | Memory: rss_MB: 5881.4 | uss_MB: 4798.1 | child_rss_MB: 6148.2 | allocated_MB: 446.5 | reserved_MB: 32804.0 | inactive_split_MB: 927.1 +rank5 [2026-10-28 16:11:31,999] INFO : Step 6000/50000 rank5 | Train Loss: 1.8458 | Train Loss Unweighted: 1.0363 | Train Loss Components: Auxiliary: 0.3398 | Cardinality: 0.1796 | Classification: 0.1819 | Classification_binary: 0.2710 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2865 | Regression_eta: 0.0032 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0998 | Classification_binary: 0.2545 | Regression_cos_phi: 0.1403 | Regression_energy: 0.1603 | Regression_eta: 0.0788 | Regression_pt: 0.1753 | Regression_sin_phi: 0.1496 | LR: 9.68e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5092s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7380 | max_loss: 0.2483 | max_backward: 1.8094 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.9243 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5850.9 | uss_MB: 4795.2 | child_rss_MB: 6173.4 | allocated_MB: 448.1 | reserved_MB: 32804.0 | inactive_split_MB: 1232.3 +rank5 [2026-10-28 16:11:32,849] INFO : Running validation on rank5 +rank5 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 16:12:26,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:12:26,924] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 16:12:53,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 16:12:55,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 16:12:56,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 16:12:57,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 16:13:07,653] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 16:13:07,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:13:08,323] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 16:13:24,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 16:13:25,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 16:13:26,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 16:13:27,223] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 16:13:37,233] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 16:13:38,169] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 16:13:38,687] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 16:13:54,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 16:13:55,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 16:13:55,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 16:13:56,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 16:14:06,709] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 16:19:13,934] INFO : Step 6100/50000 rank5 | Train Loss: 2.0818 | Train Loss Unweighted: 1.1297 | Train Loss Components: Auxiliary: 0.3734 | Cardinality: 0.2287 | Classification: 0.2079 | Classification_binary: 0.3249 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2929 | Regression_eta: 0.0044 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1368 | Classification_binary: 0.3425 | Regression_cos_phi: 0.2053 | Regression_energy: 0.1552 | Regression_eta: 0.1531 | Regression_pt: 0.1702 | Regression_sin_phi: 0.1943 | LR: 9.67e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.5162s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8130 | max_loss: 0.1524 | max_backward: 3.1921 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 1.4719 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5941.1 | uss_MB: 4883.2 | child_rss_MB: 6189.9 | allocated_MB: 448.4 | reserved_MB: 24434.0 | inactive_split_MB: 1287.8 +rank5 [2026-10-28 16:23:58,090] INFO : Step 6200/50000 rank5 | Train Loss: 2.3716 | Train Loss Unweighted: 1.2065 | Train Loss Components: Auxiliary: 0.4163 | Cardinality: 0.3927 | Classification: 0.2343 | Classification_binary: 0.3679 | HitGrouping: 0.2165 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2973 | Regression_eta: 0.0043 | Regression_pt: 0.2981 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1368 | Classification_binary: 0.3616 | Regression_cos_phi: 0.1566 | Regression_energy: 0.1425 | Regression_eta: 0.1118 | Regression_pt: 0.1519 | Regression_sin_phi: 0.1722 | LR: 9.66e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.5592s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8556 | max_loss: 0.1996 | max_backward: 1.9593 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.5823 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5946.6 | uss_MB: 3767.3 | child_rss_MB: 6181.8 | allocated_MB: 447.9 | reserved_MB: 27020.0 | inactive_split_MB: 1185.4 +rank5 [2026-10-28 16:28:38,484] INFO : Step 6300/50000 rank5 | Train Loss: 1.9346 | Train Loss Unweighted: 1.0816 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.2071 | Classification: 0.2053 | Classification_binary: 0.2781 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2941 | Regression_eta: 0.0037 | Regression_pt: 0.2966 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1236 | Classification_binary: 0.2759 | Regression_cos_phi: 0.1741 | Regression_energy: 0.1525 | Regression_eta: 0.1098 | Regression_pt: 0.1668 | Regression_sin_phi: 0.1497 | LR: 9.65e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5880s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0068 | max_forward: 0.7412 | max_loss: 0.1444 | max_backward: 1.7431 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.9045 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5941.6 | uss_MB: 4896.5 | child_rss_MB: 6164.9 | allocated_MB: 444.6 | reserved_MB: 27022.0 | inactive_split_MB: 1185.4 +rank5 [2026-10-28 16:33:23,144] INFO : Step 6400/50000 rank5 | Train Loss: 2.0123 | Train Loss Unweighted: 1.0992 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2082 | Classification: 0.2291 | Classification_binary: 0.2885 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2842 | Regression_eta: 0.0040 | Regression_pt: 0.2891 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1410 | Classification_binary: 0.2941 | Regression_cos_phi: 0.1614 | Regression_energy: 0.1435 | Regression_eta: 0.1471 | Regression_pt: 0.1576 | Regression_sin_phi: 0.1921 | LR: 9.64e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7583s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0069 | max_forward: 1.1063 | max_loss: 0.2471 | max_backward: 1.9949 | max_optimizer_step: 0.2726 | max_tensorboard_logging: 0.8233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5936.1 | uss_MB: 4896.3 | child_rss_MB: 6176.5 | allocated_MB: 451.6 | reserved_MB: 27022.0 | inactive_split_MB: 1113.0 +rank5 [2026-10-28 16:37:57,186] INFO : Step 6500/50000 rank5 | Train Loss: 2.1791 | Train Loss Unweighted: 1.2220 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2499 | Classification: 0.2271 | Classification_binary: 0.3508 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3177 | Regression_eta: 0.0042 | Regression_pt: 0.3181 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1743 | Classification_binary: 0.3492 | Regression_cos_phi: 0.1886 | Regression_energy: 0.1660 | Regression_eta: 0.1305 | Regression_pt: 0.1781 | Regression_sin_phi: 0.1821 | LR: 9.63e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.9965s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0548 | max_forward: 0.8069 | max_loss: 0.1518 | max_backward: 2.0692 | max_optimizer_step: 0.1860 | max_tensorboard_logging: 0.9414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5926.6 | uss_MB: 4895.5 | child_rss_MB: 6160.0 | allocated_MB: 449.6 | reserved_MB: 28336.0 | inactive_split_MB: 1199.2 +rank5 [2026-10-28 16:42:33,043] INFO : Step 6600/50000 rank5 | Train Loss: 1.9690 | Train Loss Unweighted: 1.0745 | Train Loss Components: Auxiliary: 0.3537 | Cardinality: 0.2132 | Classification: 0.1982 | Classification_binary: 0.3095 | HitGrouping: 0.2136 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2771 | Regression_eta: 0.0040 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1153 | Classification_binary: 0.2933 | Regression_cos_phi: 0.1582 | Regression_energy: 0.1462 | Regression_eta: 0.0933 | Regression_pt: 0.1610 | Regression_sin_phi: 0.1755 | LR: 9.62e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8255s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7382 | max_loss: 0.1467 | max_backward: 1.8335 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 1.1664 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5923.1 | uss_MB: 4893.3 | child_rss_MB: 6174.8 | allocated_MB: 447.4 | reserved_MB: 28340.0 | inactive_split_MB: 1012.8 +rank5 [2026-10-28 16:47:06,903] INFO : Step 6700/50000 rank5 | Train Loss: 1.9470 | Train Loss Unweighted: 1.0781 | Train Loss Components: Auxiliary: 0.3616 | Cardinality: 0.2363 | Classification: 0.1923 | Classification_binary: 0.2661 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3028 | Regression_eta: 0.0035 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1545 | Classification_binary: 0.2947 | Regression_cos_phi: 0.1910 | Regression_energy: 0.1726 | Regression_eta: 0.1205 | Regression_pt: 0.1888 | Regression_sin_phi: 0.2028 | LR: 9.61e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6587s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7926 | max_loss: 0.2421 | max_backward: 2.0577 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.7621 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5925.5 | uss_MB: 4898.4 | child_rss_MB: 6176.2 | allocated_MB: 450.1 | reserved_MB: 29806.0 | inactive_split_MB: 1002.0 +rank5 [2026-10-28 16:51:50,913] INFO : Step 6800/50000 rank5 | Train Loss: 2.1609 | Train Loss Unweighted: 1.1507 | Train Loss Components: Auxiliary: 0.3771 | Cardinality: 0.2180 | Classification: 0.2315 | Classification_binary: 0.3225 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0025 | Regression_energy: 0.2916 | Regression_eta: 0.0048 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1349 | Classification_binary: 0.2590 | Regression_cos_phi: 0.1756 | Regression_energy: 0.1418 | Regression_eta: 0.1169 | Regression_pt: 0.1551 | Regression_sin_phi: 0.2241 | LR: 9.60e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.1895s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0070 | max_forward: 0.9144 | max_loss: 0.4866 | max_backward: 2.1484 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 1.2438 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5920.8 | uss_MB: 4896.0 | child_rss_MB: 6174.1 | allocated_MB: 451.3 | reserved_MB: 29806.0 | inactive_split_MB: 1098.9 +rank5 [2026-10-28 16:56:32,287] INFO : Step 6900/50000 rank5 | Train Loss: 1.9936 | Train Loss Unweighted: 1.1069 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.1991 | Classification: 0.2085 | Classification_binary: 0.3385 | HitGrouping: 0.2125 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2742 | Regression_eta: 0.0041 | Regression_pt: 0.2777 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1317 | Classification_binary: 0.3586 | Regression_cos_phi: 0.1634 | Regression_energy: 0.1422 | Regression_eta: 0.1186 | Regression_pt: 0.1548 | Regression_sin_phi: 0.1612 | LR: 9.58e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6484s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7764 | max_loss: 0.2225 | max_backward: 1.6637 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1184 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5918.2 | uss_MB: 4895.4 | child_rss_MB: 6170.7 | allocated_MB: 450.8 | reserved_MB: 29806.0 | inactive_split_MB: 1193.4 +rank5 [2026-10-28 17:01:16,519] INFO : Step 7000/50000 rank5 | Train Loss: 1.9256 | Train Loss Unweighted: 1.0803 | Train Loss Components: Auxiliary: 0.3608 | Cardinality: 0.2036 | Classification: 0.1946 | Classification_binary: 0.2738 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2998 | Regression_eta: 0.0039 | Regression_pt: 0.3044 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1209 | Classification_binary: 0.2981 | Regression_cos_phi: 0.1471 | Regression_energy: 0.1632 | Regression_eta: 0.1269 | Regression_pt: 0.1754 | Regression_sin_phi: 0.1427 | LR: 9.57e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.7628s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7896 | max_loss: 0.1557 | max_backward: 1.7415 | max_optimizer_step: 0.2946 | max_tensorboard_logging: 1.0959 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 5913.7 | uss_MB: 4893.6 | child_rss_MB: 6162.0 | allocated_MB: 450.3 | reserved_MB: 29806.0 | inactive_split_MB: 1283.4 +rank5 [2026-10-28 17:01:17,465] INFO : Running validation on rank5 +rank5 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 17:02:11,957] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:02:12,537] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 17:02:38,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 17:02:40,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 17:02:41,088] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 17:02:42,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 17:02:52,093] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 17:02:52,592] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:02:53,109] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 17:03:09,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 17:03:10,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 17:03:11,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 17:03:12,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 17:03:22,255] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 17:03:22,841] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:03:23,346] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 17:03:39,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 17:03:39,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 17:03:40,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 17:03:41,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 17:03:51,659] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 17:08:58,934] INFO : Step 7100/50000 rank5 | Train Loss: 1.8251 | Train Loss Unweighted: 1.0448 | Train Loss Components: Auxiliary: 0.3403 | Cardinality: 0.1731 | Classification: 0.2040 | Classification_binary: 0.2675 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2809 | Regression_eta: 0.0035 | Regression_pt: 0.2853 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1278 | Classification_binary: 0.2490 | Regression_cos_phi: 0.1568 | Regression_energy: 0.1329 | Regression_eta: 0.1091 | Regression_pt: 0.1451 | Regression_sin_phi: 0.1874 | LR: 9.56e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.2544s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0622 | max_forward: 0.8282 | max_loss: 0.1540 | max_backward: 1.9352 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.3584 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6015.2 | uss_MB: 4988.2 | child_rss_MB: 6155.9 | allocated_MB: 449.9 | reserved_MB: 22498.0 | inactive_split_MB: 1193.4 +rank5 [2026-10-28 17:13:31,724] INFO : Step 7200/50000 rank5 | Train Loss: 2.1146 | Train Loss Unweighted: 1.1506 | Train Loss Components: Auxiliary: 0.3887 | Cardinality: 0.2411 | Classification: 0.2243 | Classification_binary: 0.3469 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2844 | Regression_eta: 0.0044 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1468 | Classification_binary: 0.3239 | Regression_cos_phi: 0.1586 | Regression_energy: 0.1672 | Regression_eta: 0.1644 | Regression_pt: 0.1835 | Regression_sin_phi: 0.1455 | LR: 9.55e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.5733s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0369 | max_forward: 0.7969 | max_loss: 0.1517 | max_backward: 1.9380 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6030.2 | uss_MB: 5003.5 | child_rss_MB: 6152.6 | allocated_MB: 448.6 | reserved_MB: 22906.0 | inactive_split_MB: 1087.6 +rank5 [2026-10-28 17:18:11,974] INFO : Step 7300/50000 rank5 | Train Loss: 1.8657 | Train Loss Unweighted: 1.0129 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2046 | Classification: 0.1759 | Classification_binary: 0.2802 | HitGrouping: 0.2112 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2746 | Regression_eta: 0.0037 | Regression_pt: 0.2747 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1265 | Classification_binary: 0.3226 | Regression_cos_phi: 0.1652 | Regression_energy: 0.1290 | Regression_eta: 0.1183 | Regression_pt: 0.1396 | Regression_sin_phi: 0.1713 | LR: 9.53e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4748s | Phase Times: max_pre_step_barrier: 0.0082 | max_batch_to_device: 0.0150 | max_forward: 0.7713 | max_loss: 0.1507 | max_backward: 1.8303 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.6113 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6029.9 | uss_MB: 5003.9 | child_rss_MB: 6151.8 | allocated_MB: 449.9 | reserved_MB: 25044.0 | inactive_split_MB: 918.5 +rank5 [2026-10-28 17:22:47,474] INFO : Step 7400/50000 rank5 | Train Loss: 1.8326 | Train Loss Unweighted: 1.0006 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2227 | Classification: 0.1795 | Classification_binary: 0.2563 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2767 | Regression_eta: 0.0034 | Regression_pt: 0.2813 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.2581 | Regression_cos_phi: 0.1125 | Regression_energy: 0.1581 | Regression_eta: 0.0898 | Regression_pt: 0.1721 | Regression_sin_phi: 0.1242 | LR: 9.52e-04 | DataLoad Time: 0.0221s | Model Forward Time: 4.2039s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0684 | max_forward: 0.7762 | max_loss: 0.1384 | max_backward: 1.7881 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6026.9 | uss_MB: 5000.5 | child_rss_MB: 6151.8 | allocated_MB: 447.4 | reserved_MB: 25044.0 | inactive_split_MB: 1152.4 +rank5 [2026-10-28 17:27:28,847] INFO : Step 7500/50000 rank5 | Train Loss: 2.1479 | Train Loss Unweighted: 1.1510 | Train Loss Components: Auxiliary: 0.3847 | Cardinality: 0.2329 | Classification: 0.2181 | Classification_binary: 0.3323 | HitGrouping: 0.2143 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2947 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1488 | Classification_binary: 0.3671 | Regression_cos_phi: 0.1800 | Regression_energy: 0.1659 | Regression_eta: 0.1685 | Regression_pt: 0.1822 | Regression_sin_phi: 0.2140 | LR: 9.51e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2514s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7726 | max_loss: 0.2518 | max_backward: 1.8458 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1040 | Memory: rss_MB: 6026.8 | uss_MB: 5002.5 | child_rss_MB: 6153.2 | allocated_MB: 452.3 | reserved_MB: 26846.0 | inactive_split_MB: 1117.1 +rank5 [2026-10-28 17:32:29,655] INFO : Step 7600/50000 rank5 | Train Loss: 1.9984 | Train Loss Unweighted: 1.0873 | Train Loss Components: Auxiliary: 0.3667 | Cardinality: 0.2088 | Classification: 0.2053 | Classification_binary: 0.3405 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2647 | Regression_eta: 0.0038 | Regression_pt: 0.2689 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1152 | Classification_binary: 0.3302 | Regression_cos_phi: 0.1342 | Regression_energy: 0.1376 | Regression_eta: 0.0874 | Regression_pt: 0.1509 | Regression_sin_phi: 0.1623 | LR: 9.50e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5784s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7834 | max_loss: 0.1641 | max_backward: 1.8407 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.8255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6029.1 | uss_MB: 5003.6 | child_rss_MB: 6150.1 | allocated_MB: 448.9 | reserved_MB: 26846.0 | inactive_split_MB: 1232.3 +rank5 [2026-10-28 17:37:19,852] INFO : Step 7700/50000 rank5 | Train Loss: 1.7524 | Train Loss Unweighted: 0.9761 | Train Loss Components: Auxiliary: 0.3200 | Cardinality: 0.1783 | Classification: 0.1756 | Classification_binary: 0.2521 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2705 | Regression_eta: 0.0035 | Regression_pt: 0.2709 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1107 | Classification_binary: 0.2565 | Regression_cos_phi: 0.1386 | Regression_energy: 0.1545 | Regression_eta: 0.0893 | Regression_pt: 0.1691 | Regression_sin_phi: 0.1598 | LR: 9.48e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4864s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7659 | max_loss: 0.1551 | max_backward: 1.8103 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6027.2 | uss_MB: 5004.4 | child_rss_MB: 6154.6 | allocated_MB: 448.8 | reserved_MB: 26846.0 | inactive_split_MB: 1042.2 +rank5 [2026-10-28 17:42:10,708] INFO : Step 7800/50000 rank5 | Train Loss: 1.8039 | Train Loss Unweighted: 1.0226 | Train Loss Components: Auxiliary: 0.3378 | Cardinality: 0.1686 | Classification: 0.1822 | Classification_binary: 0.2556 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2863 | Regression_eta: 0.0034 | Regression_pt: 0.2914 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0824 | Classification_binary: 0.2360 | Regression_cos_phi: 0.1321 | Regression_energy: 0.1553 | Regression_eta: 0.0727 | Regression_pt: 0.1699 | Regression_sin_phi: 0.1215 | LR: 9.47e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.6884s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.8680 | max_loss: 0.1424 | max_backward: 1.9949 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.8163 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 6024.0 | uss_MB: 5003.5 | child_rss_MB: 6156.5 | allocated_MB: 448.6 | reserved_MB: 26846.0 | inactive_split_MB: 1396.9 +rank5 [2026-10-28 17:47:05,103] INFO : Step 7900/50000 rank5 | Train Loss: 2.0641 | Train Loss Unweighted: 1.0960 | Train Loss Components: Auxiliary: 0.3855 | Cardinality: 0.2401 | Classification: 0.2031 | Classification_binary: 0.3367 | HitGrouping: 0.2085 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2721 | Regression_eta: 0.0041 | Regression_pt: 0.2759 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1460 | Classification_binary: 0.3635 | Regression_cos_phi: 0.2017 | Regression_energy: 0.1392 | Regression_eta: 0.1342 | Regression_pt: 0.1515 | Regression_sin_phi: 0.1751 | LR: 9.46e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6372s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7971 | max_loss: 0.2430 | max_backward: 1.9131 | max_optimizer_step: 0.2681 | max_tensorboard_logging: 0.7971 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6025.3 | uss_MB: 5004.3 | child_rss_MB: 6161.5 | allocated_MB: 451.8 | reserved_MB: 27044.0 | inactive_split_MB: 827.4 +rank5 [2026-10-28 17:51:56,121] INFO : Step 8000/50000 rank5 | Train Loss: 1.8391 | Train Loss Unweighted: 0.9813 | Train Loss Components: Auxiliary: 0.3355 | Cardinality: 0.2153 | Classification: 0.1625 | Classification_binary: 0.2896 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2605 | Regression_eta: 0.0035 | Regression_pt: 0.2616 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1185 | Classification_binary: 0.3397 | Regression_cos_phi: 0.1587 | Regression_energy: 0.1432 | Regression_eta: 0.0925 | Regression_pt: 0.1524 | Regression_sin_phi: 0.1540 | LR: 9.44e-04 | DataLoad Time: 0.0179s | Model Forward Time: 6.5538s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 1.2188 | max_loss: 0.4570 | max_backward: 3.9408 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.5566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6013.9 | uss_MB: 4999.4 | child_rss_MB: 6155.0 | allocated_MB: 448.4 | reserved_MB: 27044.0 | inactive_split_MB: 1156.9 +rank5 [2026-10-28 17:51:56,979] INFO : Running validation on rank5 +rank5 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 17:52:50,097] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:52:50,645] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 17:53:16,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 17:53:18,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 17:53:19,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 17:53:20,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 17:53:30,089] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 17:53:30,251] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:53:30,807] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 17:53:47,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 17:53:48,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 17:53:49,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 17:53:50,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 17:54:00,463] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-28 17:54:01,631] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 17:54:02,177] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 17:54:17,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 17:54:18,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 17:54:18,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 17:54:19,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 17:54:29,726] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 17:59:43,745] INFO : Step 8100/50000 rank5 | Train Loss: 1.7101 | Train Loss Unweighted: 0.9276 | Train Loss Components: Auxiliary: 0.3166 | Cardinality: 0.1787 | Classification: 0.1664 | Classification_binary: 0.2381 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2564 | Regression_eta: 0.0032 | Regression_pt: 0.2599 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1128 | Classification_binary: 0.2544 | Regression_cos_phi: 0.1622 | Regression_energy: 0.1508 | Regression_eta: 0.0956 | Regression_pt: 0.1664 | Regression_sin_phi: 0.1211 | LR: 9.43e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6992s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7628 | max_loss: 0.1353 | max_backward: 2.0123 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.8300 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6970.2 | uss_MB: 5951.6 | child_rss_MB: 6190.8 | allocated_MB: 447.2 | reserved_MB: 25306.0 | inactive_split_MB: 1261.3 +rank5 [2026-10-28 18:04:56,111] INFO : Step 8200/50000 rank5 | Train Loss: 1.8745 | Train Loss Unweighted: 1.0293 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.1946 | Classification: 0.1986 | Classification_binary: 0.2966 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2609 | Regression_eta: 0.0035 | Regression_pt: 0.2657 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1291 | Classification_binary: 0.2981 | Regression_cos_phi: 0.1597 | Regression_energy: 0.1368 | Regression_eta: 0.1032 | Regression_pt: 0.1486 | Regression_sin_phi: 0.2007 | LR: 9.42e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3073s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7790 | max_loss: 0.1535 | max_backward: 1.9181 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.5352 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6080.6 | uss_MB: 5064.4 | child_rss_MB: 6184.5 | allocated_MB: 448.2 | reserved_MB: 25360.0 | inactive_split_MB: 1159.7 +rank5 [2026-10-28 18:09:37,296] INFO : Step 8300/50000 rank5 | Train Loss: 1.8727 | Train Loss Unweighted: 1.0119 | Train Loss Components: Auxiliary: 0.3401 | Cardinality: 0.1867 | Classification: 0.1728 | Classification_binary: 0.3272 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2513 | Regression_eta: 0.0033 | Regression_pt: 0.2533 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1127 | Classification_binary: 0.3361 | Regression_cos_phi: 0.1563 | Regression_energy: 0.1359 | Regression_eta: 0.0885 | Regression_pt: 0.1486 | Regression_sin_phi: 0.1606 | LR: 9.40e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5427s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0436 | max_forward: 1.1303 | max_loss: 0.1503 | max_backward: 2.0213 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6082.4 | uss_MB: 5067.6 | child_rss_MB: 6182.5 | allocated_MB: 449.6 | reserved_MB: 27362.0 | inactive_split_MB: 1355.2 +rank5 [2026-10-28 18:14:10,521] INFO : Step 8400/50000 rank5 | Train Loss: 1.6705 | Train Loss Unweighted: 0.9205 | Train Loss Components: Auxiliary: 0.3113 | Cardinality: 0.1938 | Classification: 0.1656 | Classification_binary: 0.2512 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2485 | Regression_eta: 0.0028 | Regression_pt: 0.2494 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0872 | Classification_binary: 0.2603 | Regression_cos_phi: 0.0939 | Regression_energy: 0.1397 | Regression_eta: 0.0605 | Regression_pt: 0.1551 | Regression_sin_phi: 0.1215 | LR: 9.39e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4482s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8079 | max_loss: 0.1514 | max_backward: 1.6942 | max_optimizer_step: 0.2670 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6074.8 | uss_MB: 5060.4 | child_rss_MB: 6188.9 | allocated_MB: 446.4 | reserved_MB: 27366.0 | inactive_split_MB: 1386.9 +rank5 [2026-10-28 18:18:45,093] INFO : Step 8500/50000 rank5 | Train Loss: 1.8447 | Train Loss Unweighted: 0.9684 | Train Loss Components: Auxiliary: 0.3299 | Cardinality: 0.2197 | Classification: 0.1758 | Classification_binary: 0.2823 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2478 | Regression_eta: 0.0040 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1526 | Classification_binary: 0.3351 | Regression_cos_phi: 0.2268 | Regression_energy: 0.1411 | Regression_eta: 0.1238 | Regression_pt: 0.1556 | Regression_sin_phi: 0.1698 | LR: 9.37e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.1688s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0066 | max_forward: 0.7185 | max_loss: 0.1465 | max_backward: 1.6783 | max_optimizer_step: 0.2239 | max_tensorboard_logging: 0.6269 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 6077.8 | uss_MB: 5064.0 | child_rss_MB: 6185.3 | allocated_MB: 448.4 | reserved_MB: 27368.0 | inactive_split_MB: 1257.8 +rank5 [2026-10-28 18:23:30,153] INFO : Step 8600/50000 rank5 | Train Loss: 1.9431 | Train Loss Unweighted: 1.0237 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.2483 | Classification: 0.1814 | Classification_binary: 0.3328 | HitGrouping: 0.2024 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2495 | Regression_eta: 0.0034 | Regression_pt: 0.2527 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1022 | Classification_binary: 0.3266 | Regression_cos_phi: 0.1389 | Regression_energy: 0.1332 | Regression_eta: 0.1022 | Regression_pt: 0.1442 | Regression_sin_phi: 0.1462 | LR: 9.36e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.5573s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1011 | max_forward: 0.9340 | max_loss: 0.1508 | max_backward: 1.7704 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.7686 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 6082.0 | uss_MB: 5062.1 | child_rss_MB: 6191.5 | allocated_MB: 448.9 | reserved_MB: 27368.0 | inactive_split_MB: 1268.3 +rank5 [2026-10-28 18:28:08,444] INFO : Step 8700/50000 rank5 | Train Loss: 1.9739 | Train Loss Unweighted: 1.0707 | Train Loss Components: Auxiliary: 0.3725 | Cardinality: 0.2322 | Classification: 0.1854 | Classification_binary: 0.2934 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2900 | Regression_eta: 0.0037 | Regression_pt: 0.2944 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0978 | Classification_binary: 0.2960 | Regression_cos_phi: 0.1555 | Regression_energy: 0.1414 | Regression_eta: 0.0977 | Regression_pt: 0.1555 | Regression_sin_phi: 0.1200 | LR: 9.34e-04 | DataLoad Time: 0.0199s | Model Forward Time: 6.1622s | Phase Times: max_pre_step_barrier: 0.0890 | max_batch_to_device: 0.0067 | max_forward: 0.7726 | max_loss: 0.1574 | max_backward: 1.6332 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 2.5948 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 6082.7 | uss_MB: 5062.6 | child_rss_MB: 6185.4 | allocated_MB: 448.0 | reserved_MB: 29420.0 | inactive_split_MB: 876.0 +rank5 [2026-10-28 18:32:55,587] INFO : Step 8800/50000 rank5 | Train Loss: 1.5247 | Train Loss Unweighted: 0.8421 | Train Loss Components: Auxiliary: 0.2808 | Cardinality: 0.1524 | Classification: 0.1581 | Classification_binary: 0.2323 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2217 | Regression_eta: 0.0029 | Regression_pt: 0.2242 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2252 | Regression_cos_phi: 0.1053 | Regression_energy: 0.1057 | Regression_eta: 0.0746 | Regression_pt: 0.1140 | Regression_sin_phi: 0.1116 | LR: 9.33e-04 | DataLoad Time: 0.0154s | Model Forward Time: 3.9931s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7382 | max_loss: 0.2414 | max_backward: 1.7741 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 6078.3 | uss_MB: 5061.0 | child_rss_MB: 6166.0 | allocated_MB: 447.9 | reserved_MB: 29420.0 | inactive_split_MB: 1260.3 +rank5 [2026-10-28 18:37:36,890] INFO : Step 8900/50000 rank5 | Train Loss: 1.7540 | Train Loss Unweighted: 0.9436 | Train Loss Components: Auxiliary: 0.3227 | Cardinality: 0.2072 | Classification: 0.1816 | Classification_binary: 0.2735 | HitGrouping: 0.1904 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2393 | Regression_eta: 0.0031 | Regression_pt: 0.2426 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0991 | Classification_binary: 0.2498 | Regression_cos_phi: 0.0937 | Regression_energy: 0.1308 | Regression_eta: 0.0579 | Regression_pt: 0.1446 | Regression_sin_phi: 0.1177 | LR: 9.31e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4485s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8077 | max_loss: 0.2089 | max_backward: 1.9976 | max_optimizer_step: 0.2211 | max_tensorboard_logging: 0.5747 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 6080.9 | uss_MB: 5000.4 | child_rss_MB: 6188.7 | allocated_MB: 451.8 | reserved_MB: 29420.0 | inactive_split_MB: 1133.8 +rank5 [2026-10-28 18:42:20,745] INFO : Step 9000/50000 rank5 | Train Loss: 1.7807 | Train Loss Unweighted: 0.9795 | Train Loss Components: Auxiliary: 0.3326 | Cardinality: 0.1942 | Classification: 0.1600 | Classification_binary: 0.3157 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2461 | Regression_eta: 0.0033 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1161 | Classification_binary: 0.3715 | Regression_cos_phi: 0.1367 | Regression_energy: 0.1298 | Regression_eta: 0.1075 | Regression_pt: 0.1427 | Regression_sin_phi: 0.1757 | LR: 9.30e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8566s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.8696 | max_loss: 0.1559 | max_backward: 1.8245 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.9520 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 6064.6 | uss_MB: 5051.1 | child_rss_MB: 6187.5 | allocated_MB: 448.9 | reserved_MB: 29420.0 | inactive_split_MB: 1051.6 +rank5 [2026-10-28 18:42:21,625] INFO : Running validation on rank5 +rank5 [2026-10-28 18:42:21,626] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 18:42:21,626] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 18:43:19,670] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:43:20,246] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 18:43:47,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 18:43:48,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 18:43:49,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 18:43:50,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 18:44:00,332] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 18:44:00,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:44:01,306] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 18:44:17,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 18:44:17,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 18:44:18,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 18:44:19,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 18:44:29,922] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 18:44:31,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 18:44:31,605] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 18:44:47,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 18:44:48,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 18:44:49,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 18:44:50,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 18:45:00,288] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-28 18:50:14,978] INFO : Step 9100/50000 rank5 | Train Loss: 1.6002 | Train Loss Unweighted: 0.8685 | Train Loss Components: Auxiliary: 0.3013 | Cardinality: 0.1975 | Classification: 0.1658 | Classification_binary: 0.2349 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2304 | Regression_eta: 0.0027 | Regression_pt: 0.2318 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1150 | Classification_binary: 0.2157 | Regression_cos_phi: 0.1070 | Regression_energy: 0.1080 | Regression_eta: 0.0729 | Regression_pt: 0.1173 | Regression_sin_phi: 0.1509 | LR: 9.28e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5181s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7981 | max_loss: 0.1460 | max_backward: 1.6475 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.9178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7065.6 | uss_MB: 6052.1 | child_rss_MB: 6167.4 | allocated_MB: 449.7 | reserved_MB: 25788.0 | inactive_split_MB: 957.3 +rank5 [2026-10-28 18:55:02,311] INFO : Step 9200/50000 rank5 | Train Loss: 1.6859 | Train Loss Unweighted: 0.8867 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.1906 | Classification: 0.1447 | Classification_binary: 0.2542 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2389 | Regression_eta: 0.0033 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0907 | Classification_binary: 0.2510 | Regression_cos_phi: 0.1366 | Regression_energy: 0.1297 | Regression_eta: 0.0981 | Regression_pt: 0.1409 | Regression_sin_phi: 0.1187 | LR: 9.27e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3385s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7865 | max_loss: 0.1441 | max_backward: 1.8922 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6128.5 | uss_MB: 5123.6 | child_rss_MB: 6161.7 | allocated_MB: 450.8 | reserved_MB: 25788.0 | inactive_split_MB: 1040.1 +rank5 [2026-10-28 18:59:49,908] INFO : Step 9300/50000 rank5 | Train Loss: 1.8980 | Train Loss Unweighted: 0.9669 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.3104 | Classification: 0.1729 | Classification_binary: 0.3073 | HitGrouping: 0.1936 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2397 | Regression_eta: 0.0030 | Regression_pt: 0.2407 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1027 | Classification_binary: 0.3028 | Regression_cos_phi: 0.1096 | Regression_energy: 0.1253 | Regression_eta: 0.0724 | Regression_pt: 0.1355 | Regression_sin_phi: 0.1298 | LR: 9.25e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.1852s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0143 | max_forward: 0.7819 | max_loss: 0.1515 | max_backward: 1.8209 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6071.1 | uss_MB: 5121.9 | child_rss_MB: 5951.4 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1031.1 +rank5 [2026-10-28 19:04:46,838] INFO : Step 9400/50000 rank5 | Train Loss: 1.5878 | Train Loss Unweighted: 0.8563 | Train Loss Components: Auxiliary: 0.2880 | Cardinality: 0.1552 | Classification: 0.1580 | Classification_binary: 0.2625 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2141 | Regression_eta: 0.0030 | Regression_pt: 0.2153 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2515 | Regression_cos_phi: 0.1372 | Regression_energy: 0.1105 | Regression_eta: 0.0829 | Regression_pt: 0.1189 | Regression_sin_phi: 0.1181 | LR: 9.24e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.0175s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7342 | max_loss: 0.2471 | max_backward: 1.7320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.4671 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6067.3 | uss_MB: 5118.7 | child_rss_MB: 5945.7 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1121.1 +rank5 [2026-10-28 19:09:47,342] INFO : Step 9500/50000 rank5 | Train Loss: 1.5519 | Train Loss Unweighted: 0.8141 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1719 | Classification: 0.1393 | Classification_binary: 0.2430 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2129 | Regression_eta: 0.0030 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0946 | Classification_binary: 0.2499 | Regression_cos_phi: 0.1491 | Regression_energy: 0.1108 | Regression_eta: 0.1251 | Regression_pt: 0.1198 | Regression_sin_phi: 0.1489 | LR: 9.22e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.1309s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0076 | max_forward: 0.7261 | max_loss: 0.1376 | max_backward: 1.8267 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6069.1 | uss_MB: 5122.8 | child_rss_MB: 5952.7 | allocated_MB: 446.7 | reserved_MB: 27752.0 | inactive_split_MB: 1128.4 +rank5 [2026-10-28 19:14:55,454] INFO : Step 9600/50000 rank5 | Train Loss: 1.7379 | Train Loss Unweighted: 0.9171 | Train Loss Components: Auxiliary: 0.3221 | Cardinality: 0.2199 | Classification: 0.1679 | Classification_binary: 0.2886 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2245 | Regression_eta: 0.0032 | Regression_pt: 0.2297 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0972 | Classification_binary: 0.2888 | Regression_cos_phi: 0.1318 | Regression_energy: 0.1138 | Regression_eta: 0.0894 | Regression_pt: 0.1245 | Regression_sin_phi: 0.1153 | LR: 9.21e-04 | DataLoad Time: 0.0180s | Model Forward Time: 7.7253s | Phase Times: max_pre_step_barrier: 0.0081 | max_batch_to_device: 0.0183 | max_forward: 3.1116 | max_loss: 0.7194 | max_backward: 4.4829 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6068.7 | uss_MB: 5120.4 | child_rss_MB: 5924.4 | allocated_MB: 449.4 | reserved_MB: 29732.0 | inactive_split_MB: 1322.6 +rank5 [2026-10-28 19:20:02,940] INFO : Step 9700/50000 rank5 | Train Loss: 1.8683 | Train Loss Unweighted: 0.9602 | Train Loss Components: Auxiliary: 0.3449 | Cardinality: 0.2461 | Classification: 0.1512 | Classification_binary: 0.3271 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2364 | Regression_eta: 0.0037 | Regression_pt: 0.2383 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1159 | Classification_binary: 0.3608 | Regression_cos_phi: 0.1586 | Regression_energy: 0.1262 | Regression_eta: 0.1327 | Regression_pt: 0.1368 | Regression_sin_phi: 0.1356 | LR: 9.19e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.0317s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0069 | max_forward: 0.8299 | max_loss: 0.7776 | max_backward: 2.5571 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5726 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6067.5 | uss_MB: 5123.7 | child_rss_MB: 5943.1 | allocated_MB: 450.3 | reserved_MB: 29734.0 | inactive_split_MB: 1128.1 +rank5 [2026-10-28 19:25:00,572] INFO : Step 9800/50000 rank5 | Train Loss: 1.6119 | Train Loss Unweighted: 0.8200 | Train Loss Components: Auxiliary: 0.3036 | Cardinality: 0.2266 | Classification: 0.1371 | Classification_binary: 0.2410 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2176 | Regression_eta: 0.0027 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1078 | Classification_binary: 0.2341 | Regression_cos_phi: 0.1282 | Regression_energy: 0.1053 | Regression_eta: 0.0832 | Regression_pt: 0.1159 | Regression_sin_phi: 0.1107 | LR: 9.17e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.5939s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7430 | max_loss: 0.2467 | max_backward: 1.8961 | max_optimizer_step: 0.2008 | max_tensorboard_logging: 0.9056 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6064.8 | uss_MB: 5122.0 | child_rss_MB: 5925.9 | allocated_MB: 446.9 | reserved_MB: 29734.0 | inactive_split_MB: 905.1 +rank5 [2026-10-28 19:29:40,210] INFO : Step 9900/50000 rank5 | Train Loss: 1.5408 | Train Loss Unweighted: 0.8391 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1631 | Classification: 0.1532 | Classification_binary: 0.2416 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2178 | Regression_eta: 0.0029 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0959 | Classification_binary: 0.2783 | Regression_cos_phi: 0.1118 | Regression_energy: 0.1074 | Regression_eta: 0.0814 | Regression_pt: 0.1171 | Regression_sin_phi: 0.1355 | LR: 9.16e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6041s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7611 | max_loss: 0.1425 | max_backward: 1.8994 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6066.1 | uss_MB: 5122.8 | child_rss_MB: 5945.5 | allocated_MB: 448.3 | reserved_MB: 29734.0 | inactive_split_MB: 1119.7 +rank5 [2026-10-28 19:34:23,378] INFO : Step 10000/50000 rank5 | Train Loss: 1.7147 | Train Loss Unweighted: 0.9225 | Train Loss Components: Auxiliary: 0.3261 | Cardinality: 0.1969 | Classification: 0.1590 | Classification_binary: 0.2912 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2305 | Regression_eta: 0.0029 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1006 | Classification_binary: 0.2726 | Regression_cos_phi: 0.1445 | Regression_energy: 0.1326 | Regression_eta: 0.0763 | Regression_pt: 0.1480 | Regression_sin_phi: 0.1319 | LR: 9.14e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8784s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1140 | max_forward: 0.8951 | max_loss: 0.1695 | max_backward: 1.8837 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 1.0339 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 6055.8 | uss_MB: 5112.2 | child_rss_MB: 5922.2 | allocated_MB: 449.4 | reserved_MB: 29734.0 | inactive_split_MB: 1320.6 +rank5 [2026-10-28 19:34:24,226] INFO : Running validation on rank5 +rank5 [2026-10-28 19:34:24,226] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 19:34:24,226] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 19:35:21,693] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:35:22,179] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 19:35:48,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 19:35:49,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 19:35:50,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 19:35:51,810] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 19:36:01,817] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 19:36:02,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:36:02,985] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 19:36:18,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 19:36:20,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 19:36:21,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 19:36:22,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 19:36:32,115] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 19:36:36,471] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 19:36:36,972] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 19:36:52,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 19:36:53,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 19:36:53,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 19:36:54,743] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 19:37:04,754] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 19:42:23,379] INFO : Step 10100/50000 rank5 | Train Loss: 1.6697 | Train Loss Unweighted: 0.8627 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.2287 | Classification: 0.1283 | Classification_binary: 0.2543 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2359 | Regression_eta: 0.0032 | Regression_pt: 0.2378 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.2616 | Regression_cos_phi: 0.1146 | Regression_energy: 0.1312 | Regression_eta: 0.0684 | Regression_pt: 0.1402 | Regression_sin_phi: 0.1382 | LR: 9.12e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.7389s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7652 | max_loss: 0.1496 | max_backward: 1.8791 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 0.9326 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 7065.4 | uss_MB: 6119.6 | child_rss_MB: 5927.2 | allocated_MB: 448.1 | reserved_MB: 26360.0 | inactive_split_MB: 1080.0 +rank5 [2026-10-28 19:47:10,586] INFO : Step 10200/50000 rank5 | Train Loss: 1.3968 | Train Loss Unweighted: 0.7619 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1483 | Classification: 0.1293 | Classification_binary: 0.2070 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2090 | Regression_eta: 0.0026 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0808 | Classification_binary: 0.1863 | Regression_cos_phi: 0.1324 | Regression_energy: 0.0923 | Regression_eta: 0.0723 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0983 | LR: 9.11e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.4829s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0330 | max_forward: 0.8960 | max_loss: 0.5080 | max_backward: 1.9040 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.6816 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 6137.8 | uss_MB: 5194.1 | child_rss_MB: 5912.5 | allocated_MB: 450.3 | reserved_MB: 27130.0 | inactive_split_MB: 1088.5 +rank5 [2026-10-28 19:51:56,122] INFO : Step 10300/50000 rank5 | Train Loss: 1.6039 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.2943 | Cardinality: 0.1656 | Classification: 0.1487 | Classification_binary: 0.2679 | HitGrouping: 0.1881 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2132 | Regression_eta: 0.0030 | Regression_pt: 0.2158 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1001 | Classification_binary: 0.2931 | Regression_cos_phi: 0.1892 | Regression_energy: 0.1124 | Regression_eta: 0.1410 | Regression_pt: 0.1247 | Regression_sin_phi: 0.2104 | LR: 9.09e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.7939s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8152 | max_loss: 0.2478 | max_backward: 2.0483 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6143.7 | uss_MB: 5198.5 | child_rss_MB: 5930.4 | allocated_MB: 448.7 | reserved_MB: 28404.0 | inactive_split_MB: 991.9 +rank5 [2026-10-28 19:56:39,452] INFO : Step 10400/50000 rank5 | Train Loss: 1.7188 | Train Loss Unweighted: 0.8798 | Train Loss Components: Auxiliary: 0.3209 | Cardinality: 0.2578 | Classification: 0.1544 | Classification_binary: 0.2910 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2139 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0994 | Classification_binary: 0.3015 | Regression_cos_phi: 0.1117 | Regression_energy: 0.1125 | Regression_eta: 0.0816 | Regression_pt: 0.1218 | Regression_sin_phi: 0.1197 | LR: 9.07e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6762s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7821 | max_loss: 0.2448 | max_backward: 1.8782 | max_optimizer_step: 0.2932 | max_tensorboard_logging: 0.8061 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 6134.3 | uss_MB: 5193.2 | child_rss_MB: 5938.4 | allocated_MB: 448.9 | reserved_MB: 28404.0 | inactive_split_MB: 983.3 +rank5 [2026-10-28 20:01:29,563] INFO : Step 10500/50000 rank5 | Train Loss: 1.4652 | Train Loss Unweighted: 0.7986 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1697 | Classification: 0.1280 | Classification_binary: 0.2159 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2232 | Regression_eta: 0.0026 | Regression_pt: 0.2263 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.2490 | Regression_cos_phi: 0.1047 | Regression_energy: 0.1338 | Regression_eta: 0.0746 | Regression_pt: 0.1452 | Regression_sin_phi: 0.1070 | LR: 9.06e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.2142s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.7113 | max_loss: 0.1455 | max_backward: 1.6308 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.7760 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6134.1 | uss_MB: 5192.0 | child_rss_MB: 5916.0 | allocated_MB: 444.7 | reserved_MB: 28404.0 | inactive_split_MB: 1307.3 +rank5 [2026-10-28 20:06:14,521] INFO : Step 10600/50000 rank5 | Train Loss: 1.5657 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2802 | Cardinality: 0.1297 | Classification: 0.1617 | Classification_binary: 0.2471 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2084 | Regression_eta: 0.0037 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0717 | Classification_binary: 0.2463 | Regression_cos_phi: 0.1402 | Regression_energy: 0.1259 | Regression_eta: 0.0600 | Regression_pt: 0.1377 | Regression_sin_phi: 0.1242 | LR: 9.04e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.9409s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0173 | max_forward: 0.7875 | max_loss: 0.2211 | max_backward: 1.9019 | max_optimizer_step: 0.2814 | max_tensorboard_logging: 0.8518 | max_tensorboard_flush: 0.0072 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6133.2 | uss_MB: 5193.0 | child_rss_MB: 5934.7 | allocated_MB: 449.8 | reserved_MB: 28404.0 | inactive_split_MB: 1130.2 +rank5 [2026-10-28 20:10:59,504] INFO : Step 10700/50000 rank5 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8676 | Train Loss Components: Auxiliary: 0.3003 | Cardinality: 0.1749 | Classification: 0.1453 | Classification_binary: 0.2956 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2085 | Regression_eta: 0.0033 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0806 | Classification_binary: 0.2790 | Regression_cos_phi: 0.1213 | Regression_energy: 0.1090 | Regression_eta: 0.0792 | Regression_pt: 0.1182 | Regression_sin_phi: 0.1043 | LR: 9.02e-04 | DataLoad Time: 0.0118s | Model Forward Time: 5.1530s | Phase Times: max_pre_step_barrier: 0.0305 | max_batch_to_device: 0.0154 | max_forward: 2.0246 | max_loss: 0.2864 | max_backward: 2.7424 | max_optimizer_step: 0.1980 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6136.9 | uss_MB: 5195.8 | child_rss_MB: 5906.4 | allocated_MB: 446.2 | reserved_MB: 29148.0 | inactive_split_MB: 1134.5 +rank5 [2026-10-28 20:15:41,623] INFO : Step 10800/50000 rank5 | Train Loss: 1.5849 | Train Loss Unweighted: 0.7936 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1942 | Classification: 0.1383 | Classification_binary: 0.2480 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2007 | Regression_eta: 0.0032 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1029 | Classification_binary: 0.2730 | Regression_cos_phi: 0.1553 | Regression_energy: 0.0993 | Regression_eta: 0.1269 | Regression_pt: 0.1080 | Regression_sin_phi: 0.1125 | LR: 9.00e-04 | DataLoad Time: 0.0176s | Model Forward Time: 8.1980s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2408 | max_backward: 1.7977 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 1.7470 | max_tensorboard_flush: 0.0028 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 6126.4 | uss_MB: 4363.9 | child_rss_MB: 5910.3 | allocated_MB: 448.6 | reserved_MB: 30628.0 | inactive_split_MB: 1292.6 +rank5 [2026-10-28 20:20:32,832] INFO : Step 10900/50000 rank5 | Train Loss: 1.3552 | Train Loss Unweighted: 0.7247 | Train Loss Components: Auxiliary: 0.2531 | Cardinality: 0.1449 | Classification: 0.1275 | Classification_binary: 0.2074 | HitGrouping: 0.1874 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1917 | Regression_eta: 0.0024 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.2100 | Regression_cos_phi: 0.0730 | Regression_energy: 0.1074 | Regression_eta: 0.0668 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0923 | LR: 8.99e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4547s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7076 | max_loss: 0.2425 | max_backward: 1.6536 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6131.8 | uss_MB: 5191.3 | child_rss_MB: 5932.5 | allocated_MB: 446.4 | reserved_MB: 30628.0 | inactive_split_MB: 984.9 +rank5 [2026-10-28 20:25:36,607] INFO : Step 11000/50000 rank5 | Train Loss: 1.5999 | Train Loss Unweighted: 0.8322 | Train Loss Components: Auxiliary: 0.2959 | Cardinality: 0.1737 | Classification: 0.1477 | Classification_binary: 0.2877 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0017 | Regression_energy: 0.1943 | Regression_eta: 0.0030 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0911 | Classification_binary: 0.2992 | Regression_cos_phi: 0.1376 | Regression_energy: 0.0964 | Regression_eta: 0.1027 | Regression_pt: 0.1049 | Regression_sin_phi: 0.1215 | LR: 8.97e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9318s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8260 | max_loss: 0.1503 | max_backward: 1.9603 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6124.9 | uss_MB: 5185.7 | child_rss_MB: 5918.3 | allocated_MB: 451.8 | reserved_MB: 30628.0 | inactive_split_MB: 996.6 +rank5 [2026-10-28 20:25:37,479] INFO : Running validation on rank5 +rank5 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 20:25:37,480] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 20:26:32,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:26:33,360] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 20:27:00,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 20:27:01,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 20:27:02,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 20:27:03,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 20:27:13,399] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 20:27:14,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:27:14,833] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 20:27:31,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 20:27:32,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 20:27:33,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 20:27:35,393] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 20:27:45,402] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-28 20:27:46,111] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 20:27:46,649] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 20:28:02,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 20:28:02,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 20:28:03,550] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 20:28:04,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 20:28:14,425] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 20:33:29,189] INFO : Step 11100/50000 rank5 | Train Loss: 1.6214 | Train Loss Unweighted: 0.8328 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.2186 | Classification: 0.1350 | Classification_binary: 0.2802 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2042 | Regression_eta: 0.0030 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0889 | Classification_binary: 0.3160 | Regression_cos_phi: 0.1297 | Regression_energy: 0.1013 | Regression_eta: 0.0936 | Regression_pt: 0.1112 | Regression_sin_phi: 0.1303 | LR: 8.95e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.8756s | Phase Times: max_pre_step_barrier: 0.0088 | max_batch_to_device: 0.0068 | max_forward: 0.8488 | max_loss: 0.1548 | max_backward: 1.7645 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 1.1046 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7069.9 | uss_MB: 6125.7 | child_rss_MB: 5940.1 | allocated_MB: 448.1 | reserved_MB: 24340.0 | inactive_split_MB: 1081.9 +rank5 [2026-10-28 20:38:18,706] INFO : Step 11200/50000 rank5 | Train Loss: 1.7050 | Train Loss Unweighted: 0.8927 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2640 | Classification: 0.1534 | Classification_binary: 0.2523 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2404 | Regression_eta: 0.0030 | Regression_pt: 0.2410 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1094 | Classification_binary: 0.2607 | Regression_cos_phi: 0.1000 | Regression_energy: 0.1323 | Regression_eta: 0.0867 | Regression_pt: 0.1407 | Regression_sin_phi: 0.1301 | LR: 8.93e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.8972s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8707 | max_loss: 0.2001 | max_backward: 2.2037 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.7912 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 6163.4 | uss_MB: 5218.5 | child_rss_MB: 5949.2 | allocated_MB: 447.8 | reserved_MB: 26392.0 | inactive_split_MB: 938.7 +rank5 [2026-10-28 20:43:01,723] INFO : Step 11300/50000 rank5 | Train Loss: 1.3885 | Train Loss Unweighted: 0.7511 | Train Loss Components: Auxiliary: 0.2594 | Cardinality: 0.1357 | Classification: 0.1316 | Classification_binary: 0.2357 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1874 | Regression_eta: 0.0024 | Regression_pt: 0.1915 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.2472 | Regression_cos_phi: 0.1113 | Regression_energy: 0.1023 | Regression_eta: 0.0667 | Regression_pt: 0.1142 | Regression_sin_phi: 0.1275 | LR: 8.91e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4299s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8053 | max_loss: 0.1469 | max_backward: 2.0062 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.4618 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1345 | Memory: rss_MB: 6165.0 | uss_MB: 5222.0 | child_rss_MB: 5948.8 | allocated_MB: 450.6 | reserved_MB: 26400.0 | inactive_split_MB: 1092.9 +rank5 [2026-10-28 20:48:05,522] INFO : Step 11400/50000 rank5 | Train Loss: 1.5865 | Train Loss Unweighted: 0.8323 | Train Loss Components: Auxiliary: 0.2981 | Cardinality: 0.1866 | Classification: 0.1338 | Classification_binary: 0.2899 | HitGrouping: 0.1840 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1999 | Regression_eta: 0.0028 | Regression_pt: 0.2030 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2506 | Regression_cos_phi: 0.1072 | Regression_energy: 0.1190 | Regression_eta: 0.0620 | Regression_pt: 0.1287 | Regression_sin_phi: 0.1264 | LR: 8.89e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.7651s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0769 | max_forward: 1.3800 | max_loss: 0.6715 | max_backward: 3.0339 | max_optimizer_step: 0.2660 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6163.4 | uss_MB: 5219.6 | child_rss_MB: 5946.4 | allocated_MB: 448.1 | reserved_MB: 27144.0 | inactive_split_MB: 1081.9 +rank5 [2026-10-28 20:52:57,975] INFO : Step 11500/50000 rank5 | Train Loss: 1.4825 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2779 | Cardinality: 0.1999 | Classification: 0.1273 | Classification_binary: 0.2463 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1956 | Regression_eta: 0.0026 | Regression_pt: 0.1972 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0847 | Classification_binary: 0.2667 | Regression_cos_phi: 0.1114 | Regression_energy: 0.1078 | Regression_eta: 0.0799 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0959 | LR: 8.88e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0558s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.7822 | max_loss: 0.1471 | max_backward: 1.7308 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 1.4583 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6164.2 | uss_MB: 5219.4 | child_rss_MB: 5944.4 | allocated_MB: 450.8 | reserved_MB: 27144.0 | inactive_split_MB: 1342.3 +rank5 [2026-10-28 20:57:41,224] INFO : Step 11600/50000 rank5 | Train Loss: 1.4219 | Train Loss Unweighted: 0.7444 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1679 | Classification: 0.1415 | Classification_binary: 0.2106 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1945 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1111 | Classification_binary: 0.2224 | Regression_cos_phi: 0.1319 | Regression_energy: 0.1197 | Regression_eta: 0.0712 | Regression_pt: 0.1294 | Regression_sin_phi: 0.1751 | LR: 8.86e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.7003s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8362 | max_loss: 0.1428 | max_backward: 2.0109 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6156.9 | uss_MB: 5220.2 | child_rss_MB: 5947.3 | allocated_MB: 448.6 | reserved_MB: 27146.0 | inactive_split_MB: 1315.4 +rank5 [2026-10-28 21:02:35,117] INFO : Step 11700/50000 rank5 | Train Loss: 1.5203 | Train Loss Unweighted: 0.8017 | Train Loss Components: Auxiliary: 0.2866 | Cardinality: 0.1889 | Classification: 0.1302 | Classification_binary: 0.2577 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2024 | Regression_eta: 0.0026 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2603 | Regression_cos_phi: 0.1055 | Regression_energy: 0.1062 | Regression_eta: 0.0834 | Regression_pt: 0.1170 | Regression_sin_phi: 0.1135 | LR: 8.84e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.3175s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 1.0014 | max_loss: 0.1975 | max_backward: 2.5347 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0688 | Memory: rss_MB: 6157.0 | uss_MB: 5144.6 | child_rss_MB: 5922.6 | allocated_MB: 448.7 | reserved_MB: 27266.0 | inactive_split_MB: 1090.9 +rank5 [2026-10-28 21:07:42,591] INFO : Step 11800/50000 rank5 | Train Loss: 1.5591 | Train Loss Unweighted: 0.7907 | Train Loss Components: Auxiliary: 0.2873 | Cardinality: 0.2051 | Classification: 0.1346 | Classification_binary: 0.2655 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1918 | Regression_eta: 0.0030 | Regression_pt: 0.1929 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.2475 | Regression_cos_phi: 0.0948 | Regression_energy: 0.1114 | Regression_eta: 0.0815 | Regression_pt: 0.1219 | Regression_sin_phi: 0.1268 | LR: 8.82e-04 | DataLoad Time: 0.0192s | Model Forward Time: 7.4772s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.0966 | max_forward: 1.7907 | max_loss: 0.6496 | max_backward: 2.7103 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 2.3587 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6136.9 | uss_MB: 5209.7 | child_rss_MB: 5946.4 | allocated_MB: 450.4 | reserved_MB: 27266.0 | inactive_split_MB: 1187.2 +rank5 [2026-10-28 21:12:51,258] INFO : Step 11900/50000 rank5 | Train Loss: 1.3283 | Train Loss Unweighted: 0.7113 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1476 | Classification: 0.1193 | Classification_binary: 0.2094 | HitGrouping: 0.1824 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0021 | Regression_pt: 0.1901 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0823 | Classification_binary: 0.2580 | Regression_cos_phi: 0.1135 | Regression_energy: 0.1196 | Regression_eta: 0.0711 | Regression_pt: 0.1297 | Regression_sin_phi: 0.1125 | LR: 8.80e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.1697s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0785 | max_forward: 0.7917 | max_loss: 0.1465 | max_backward: 1.5595 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6137.9 | uss_MB: 5216.4 | child_rss_MB: 5947.1 | allocated_MB: 443.9 | reserved_MB: 27266.0 | inactive_split_MB: 955.3 +rank5 [2026-10-28 21:17:48,734] INFO : Step 12000/50000 rank5 | Train Loss: 1.4074 | Train Loss Unweighted: 0.7415 | Train Loss Components: Auxiliary: 0.2683 | Cardinality: 0.1719 | Classification: 0.1235 | Classification_binary: 0.2283 | HitGrouping: 0.1812 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1909 | Regression_eta: 0.0022 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.2369 | Regression_cos_phi: 0.1354 | Regression_energy: 0.0997 | Regression_eta: 0.0713 | Regression_pt: 0.1091 | Regression_sin_phi: 0.1247 | LR: 8.78e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2513s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0303 | max_forward: 0.7589 | max_loss: 0.1459 | max_backward: 1.9073 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6136.6 | uss_MB: 5219.4 | child_rss_MB: 5952.0 | allocated_MB: 447.6 | reserved_MB: 27266.0 | inactive_split_MB: 1310.4 +rank5 [2026-10-28 21:17:49,719] INFO : Running validation on rank5 +rank5 [2026-10-28 21:17:49,719] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 21:17:49,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 21:18:44,923] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:18:45,601] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 21:19:11,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 21:19:12,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 21:19:13,772] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 21:19:14,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 21:19:24,769] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 21:19:25,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:19:25,829] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 21:19:41,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 21:19:42,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 21:19:43,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 21:19:44,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 21:19:54,826] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 21:19:55,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 21:19:56,383] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 21:20:12,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 21:20:13,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 21:20:13,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 21:20:14,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 21:20:24,808] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 21:25:33,195] INFO : Step 12100/50000 rank5 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8521 | Train Loss Components: Auxiliary: 0.3093 | Cardinality: 0.2088 | Classification: 0.1337 | Classification_binary: 0.2971 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0029 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0895 | Classification_binary: 0.3082 | Regression_cos_phi: 0.1294 | Regression_energy: 0.1063 | Regression_eta: 0.1311 | Regression_pt: 0.1157 | Regression_sin_phi: 0.1326 | LR: 8.76e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9390s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8139 | max_loss: 0.1586 | max_backward: 1.9999 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9439 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7027.3 | uss_MB: 6104.0 | child_rss_MB: 5925.8 | allocated_MB: 450.8 | reserved_MB: 28008.0 | inactive_split_MB: 1125.6 +rank5 [2026-10-28 21:30:25,390] INFO : Step 12200/50000 rank5 | Train Loss: 1.4655 | Train Loss Unweighted: 0.7431 | Train Loss Components: Auxiliary: 0.2757 | Cardinality: 0.1845 | Classification: 0.1184 | Classification_binary: 0.2450 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1862 | Regression_eta: 0.0026 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0683 | Classification_binary: 0.2271 | Regression_cos_phi: 0.0877 | Regression_energy: 0.0895 | Regression_eta: 0.0529 | Regression_pt: 0.0985 | Regression_sin_phi: 0.1215 | LR: 8.74e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4992s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0364 | max_forward: 0.7684 | max_loss: 0.1438 | max_backward: 1.8240 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.7243 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0697 | Memory: rss_MB: 6148.5 | uss_MB: 5233.4 | child_rss_MB: 5949.1 | allocated_MB: 445.9 | reserved_MB: 28008.0 | inactive_split_MB: 995.3 +rank5 [2026-10-28 21:35:16,327] INFO : Step 12300/50000 rank5 | Train Loss: 1.3845 | Train Loss Unweighted: 0.7270 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1523 | Classification: 0.1341 | Classification_binary: 0.2109 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1869 | Regression_eta: 0.0026 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1000 | Classification_binary: 0.2040 | Regression_cos_phi: 0.1108 | Regression_energy: 0.0959 | Regression_eta: 0.0749 | Regression_pt: 0.1052 | Regression_sin_phi: 0.1211 | LR: 8.72e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.5854s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7264 | max_loss: 0.1976 | max_backward: 1.8534 | max_optimizer_step: 0.2993 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6147.3 | uss_MB: 5233.4 | child_rss_MB: 5944.1 | allocated_MB: 448.1 | reserved_MB: 29988.0 | inactive_split_MB: 1298.2 +rank5 [2026-10-28 21:40:15,077] INFO : Step 12400/50000 rank5 | Train Loss: 1.4006 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.2654 | Cardinality: 0.1602 | Classification: 0.1310 | Classification_binary: 0.2363 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0804 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0913 | Regression_eta: 0.0552 | Regression_pt: 0.0991 | Regression_sin_phi: 0.1090 | LR: 8.70e-04 | DataLoad Time: 0.0178s | Model Forward Time: 7.1201s | Phase Times: max_pre_step_barrier: 0.0107 | max_batch_to_device: 0.0207 | max_forward: 1.2284 | max_loss: 0.2740 | max_backward: 2.6428 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8714 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 6147.7 | uss_MB: 5022.1 | child_rss_MB: 5936.9 | allocated_MB: 448.8 | reserved_MB: 29988.0 | inactive_split_MB: 1332.2 +rank5 [2026-10-28 21:45:07,596] INFO : Step 12500/50000 rank5 | Train Loss: 1.4931 | Train Loss Unweighted: 0.7691 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1768 | Classification: 0.1197 | Classification_binary: 0.2587 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1910 | Regression_eta: 0.0024 | Regression_pt: 0.1944 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.2674 | Regression_cos_phi: 0.1082 | Regression_energy: 0.1128 | Regression_eta: 0.0578 | Regression_pt: 0.1255 | Regression_sin_phi: 0.1099 | LR: 8.68e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7425s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0333 | max_forward: 0.8806 | max_loss: 0.1518 | max_backward: 2.1554 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.5983 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6144.5 | uss_MB: 5232.3 | child_rss_MB: 5946.8 | allocated_MB: 452.8 | reserved_MB: 29990.0 | inactive_split_MB: 1046.7 +rank5 [2026-10-28 21:50:03,651] INFO : Step 12600/50000 rank5 | Train Loss: 1.3547 | Train Loss Unweighted: 0.7028 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1629 | Classification: 0.1033 | Classification_binary: 0.2112 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1912 | Regression_eta: 0.0024 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.2062 | Regression_cos_phi: 0.1187 | Regression_energy: 0.0908 | Regression_eta: 0.0625 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0953 | LR: 8.66e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5164s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.7639 | max_loss: 0.2516 | max_backward: 1.8947 | max_optimizer_step: 0.2049 | max_tensorboard_logging: 0.7350 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6141.1 | uss_MB: 5233.0 | child_rss_MB: 5920.5 | allocated_MB: 446.9 | reserved_MB: 29990.0 | inactive_split_MB: 1034.9 +rank5 [2026-10-28 21:54:54,644] INFO : Step 12700/50000 rank5 | Train Loss: 1.3557 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1573 | Classification: 0.1262 | Classification_binary: 0.2132 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1863 | Regression_eta: 0.0022 | Regression_pt: 0.1887 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.2375 | Regression_cos_phi: 0.1136 | Regression_energy: 0.1032 | Regression_eta: 0.0715 | Regression_pt: 0.1115 | Regression_sin_phi: 0.1073 | LR: 8.64e-04 | DataLoad Time: 0.0155s | Model Forward Time: 6.1271s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7499 | max_loss: 0.2453 | max_backward: 3.4334 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6139.6 | uss_MB: 5232.1 | child_rss_MB: 5948.4 | allocated_MB: 450.2 | reserved_MB: 29990.0 | inactive_split_MB: 1245.6 +rank5 [2026-10-28 21:59:39,371] INFO : Step 12800/50000 rank5 | Train Loss: 1.5298 | Train Loss Unweighted: 0.7754 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1939 | Classification: 0.1237 | Classification_binary: 0.2821 | HitGrouping: 0.1822 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1810 | Regression_eta: 0.0027 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.2828 | Regression_cos_phi: 0.1251 | Regression_energy: 0.0956 | Regression_eta: 0.0638 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0897 | LR: 8.62e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.2941s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.5299 | max_backward: 2.3185 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6133.4 | uss_MB: 4813.1 | child_rss_MB: 5927.2 | allocated_MB: 449.9 | reserved_MB: 32006.0 | inactive_split_MB: 889.3 +rank5 [2026-10-28 22:04:39,764] INFO : Step 12900/50000 rank5 | Train Loss: 1.3302 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1576 | Classification: 0.1054 | Classification_binary: 0.2343 | HitGrouping: 0.1811 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1758 | Regression_eta: 0.0020 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.2598 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0909 | Regression_eta: 0.0643 | Regression_pt: 0.0989 | Regression_sin_phi: 0.1256 | LR: 8.60e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.6346s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0137 | max_forward: 0.9836 | max_loss: 0.2293 | max_backward: 2.2888 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.9302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6139.6 | uss_MB: 5231.4 | child_rss_MB: 5950.5 | allocated_MB: 445.9 | reserved_MB: 32006.0 | inactive_split_MB: 1247.6 +rank5 [2026-10-28 22:09:25,338] INFO : Step 13000/50000 rank5 | Train Loss: 1.3124 | Train Loss Unweighted: 0.7073 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1406 | Classification: 0.1147 | Classification_binary: 0.1973 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1944 | Regression_eta: 0.0023 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.1938 | Regression_cos_phi: 0.1142 | Regression_energy: 0.0951 | Regression_eta: 0.0544 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0844 | LR: 8.58e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2803s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0065 | max_forward: 0.7321 | max_loss: 0.1616 | max_backward: 1.7374 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6140.2 | uss_MB: 5231.6 | child_rss_MB: 5949.6 | allocated_MB: 444.7 | reserved_MB: 32006.0 | inactive_split_MB: 1300.2 +rank5 [2026-10-28 22:09:26,330] INFO : Running validation on rank5 +rank5 [2026-10-28 22:09:26,330] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 22:09:26,330] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 22:10:21,901] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:10:22,474] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 22:10:48,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 22:10:49,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 22:10:50,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 22:10:51,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 22:11:01,839] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 22:11:02,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:11:02,875] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 22:11:18,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 22:11:19,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 22:11:20,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 22:11:21,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 22:11:31,928] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 22:11:32,690] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 22:11:33,217] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 22:11:48,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 22:11:49,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 22:11:50,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 22:11:51,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 22:12:01,455] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 22:17:08,486] INFO : Step 13100/50000 rank5 | Train Loss: 1.6165 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3082 | Cardinality: 0.2635 | Classification: 0.1364 | Classification_binary: 0.2628 | HitGrouping: 0.1835 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0026 | Regression_pt: 0.1957 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2795 | Regression_cos_phi: 0.0974 | Regression_energy: 0.0962 | Regression_eta: 0.0735 | Regression_pt: 0.1068 | Regression_sin_phi: 0.1261 | LR: 8.56e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2705s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0479 | max_forward: 0.8994 | max_loss: 0.2807 | max_backward: 2.1108 | max_optimizer_step: 0.2760 | max_tensorboard_logging: 0.9690 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 7057.8 | uss_MB: 6143.5 | child_rss_MB: 5948.1 | allocated_MB: 446.5 | reserved_MB: 23120.0 | inactive_split_MB: 920.0 +rank5 [2026-10-28 22:22:01,650] INFO : Step 13200/50000 rank5 | Train Loss: 1.4928 | Train Loss Unweighted: 0.7469 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.2211 | Classification: 0.1278 | Classification_binary: 0.2505 | HitGrouping: 0.1746 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1808 | Regression_eta: 0.0025 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.2592 | Regression_cos_phi: 0.1027 | Regression_energy: 0.0979 | Regression_eta: 0.0719 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0782 | LR: 8.54e-04 | DataLoad Time: 0.0196s | Model Forward Time: 6.5986s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.7755 | max_loss: 0.2864 | max_backward: 3.4589 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6222.1 | uss_MB: 5074.0 | child_rss_MB: 5943.3 | allocated_MB: 448.3 | reserved_MB: 25156.0 | inactive_split_MB: 978.0 +rank5 [2026-10-28 22:26:52,973] INFO : Step 13300/50000 rank5 | Train Loss: 1.3251 | Train Loss Unweighted: 0.6741 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1607 | Classification: 0.1132 | Classification_binary: 0.1927 | HitGrouping: 0.1773 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1805 | Regression_eta: 0.0024 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0865 | Classification_binary: 0.2177 | Regression_cos_phi: 0.0931 | Regression_energy: 0.0993 | Regression_eta: 0.0682 | Regression_pt: 0.1079 | Regression_sin_phi: 0.1097 | LR: 8.52e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3792s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8118 | max_loss: 0.1467 | max_backward: 1.5503 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.9039 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6221.5 | uss_MB: 5307.8 | child_rss_MB: 5952.8 | allocated_MB: 446.0 | reserved_MB: 26736.0 | inactive_split_MB: 965.8 +rank5 [2026-10-28 22:31:44,118] INFO : Step 13400/50000 rank5 | Train Loss: 1.2635 | Train Loss Unweighted: 0.6770 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1436 | Classification: 0.1317 | Classification_binary: 0.2016 | HitGrouping: 0.1790 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1692 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.1788 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0886 | Regression_eta: 0.0441 | Regression_pt: 0.0961 | Regression_sin_phi: 0.0777 | LR: 8.50e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7355s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8396 | max_loss: 0.1412 | max_backward: 2.0130 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.8047 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6225.0 | uss_MB: 5310.9 | child_rss_MB: 5925.4 | allocated_MB: 445.8 | reserved_MB: 26736.0 | inactive_split_MB: 1014.4 +rank5 [2026-10-28 22:36:27,627] INFO : Step 13500/50000 rank5 | Train Loss: 1.3935 | Train Loss Unweighted: 0.7401 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1713 | Classification: 0.1245 | Classification_binary: 0.2541 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0021 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0894 | Classification_binary: 0.2738 | Regression_cos_phi: 0.0982 | Regression_energy: 0.1042 | Regression_eta: 0.0642 | Regression_pt: 0.1141 | Regression_sin_phi: 0.0996 | LR: 8.48e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5759s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0859 | max_forward: 0.8721 | max_loss: 0.1498 | max_backward: 1.7581 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.7855 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6221.6 | uss_MB: 5308.9 | child_rss_MB: 5951.2 | allocated_MB: 449.1 | reserved_MB: 26736.0 | inactive_split_MB: 922.5 +rank5 [2026-10-28 22:41:19,303] INFO : Step 13600/50000 rank5 | Train Loss: 1.3997 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1870 | Classification: 0.1206 | Classification_binary: 0.2286 | HitGrouping: 0.1765 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1702 | Regression_eta: 0.0026 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2057 | Regression_cos_phi: 0.0893 | Regression_energy: 0.0743 | Regression_eta: 0.0660 | Regression_pt: 0.0808 | Regression_sin_phi: 0.0935 | LR: 8.45e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.2259s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7703 | max_loss: 0.2484 | max_backward: 1.6943 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 6210.7 | uss_MB: 5299.9 | child_rss_MB: 5953.4 | allocated_MB: 450.3 | reserved_MB: 26736.0 | inactive_split_MB: 922.0 +rank5 [2026-10-28 22:46:12,326] INFO : Step 13700/50000 rank5 | Train Loss: 1.2648 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2420 | Cardinality: 0.1315 | Classification: 0.1136 | Classification_binary: 0.1964 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1793 | Regression_eta: 0.0020 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1945 | Regression_cos_phi: 0.0846 | Regression_energy: 0.1024 | Regression_eta: 0.0578 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0857 | LR: 8.43e-04 | DataLoad Time: 0.0169s | Model Forward Time: 6.2731s | Phase Times: max_pre_step_barrier: 0.0406 | max_batch_to_device: 0.0119 | max_forward: 1.2281 | max_loss: 0.2480 | max_backward: 3.6472 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.6460 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0767 | Memory: rss_MB: 6216.0 | uss_MB: 5307.4 | child_rss_MB: 5948.4 | allocated_MB: 446.9 | reserved_MB: 26742.0 | inactive_split_MB: 1058.7 +rank5 [2026-10-28 22:51:09,234] INFO : Step 13800/50000 rank5 | Train Loss: 1.3469 | Train Loss Unweighted: 0.7266 | Train Loss Components: Auxiliary: 0.2604 | Cardinality: 0.1436 | Classification: 0.1184 | Classification_binary: 0.2382 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1812 | Regression_eta: 0.0021 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0771 | Classification_binary: 0.2165 | Regression_cos_phi: 0.0758 | Regression_energy: 0.1027 | Regression_eta: 0.0431 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0930 | LR: 8.41e-04 | DataLoad Time: 0.0149s | Model Forward Time: 5.8545s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7995 | max_loss: 0.1458 | max_backward: 3.4735 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6221.0 | uss_MB: 5310.1 | child_rss_MB: 5919.7 | allocated_MB: 446.4 | reserved_MB: 27082.0 | inactive_split_MB: 1370.3 +rank5 [2026-10-28 22:56:04,349] INFO : Step 13900/50000 rank5 | Train Loss: 1.4882 | Train Loss Unweighted: 0.7682 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1631 | Classification: 0.1199 | Classification_binary: 0.2624 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1900 | Regression_eta: 0.0029 | Regression_pt: 0.1902 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2992 | Regression_cos_phi: 0.1348 | Regression_energy: 0.1116 | Regression_eta: 0.1638 | Regression_pt: 0.1191 | Regression_sin_phi: 0.2829 | LR: 8.39e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4529s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.7750 | max_loss: 0.1563 | max_backward: 1.5716 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.8868 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6217.9 | uss_MB: 5307.2 | child_rss_MB: 5946.5 | allocated_MB: 449.9 | reserved_MB: 27316.0 | inactive_split_MB: 918.0 +rank5 [2026-10-28 23:00:54,752] INFO : Step 14000/50000 rank5 | Train Loss: 1.2239 | Train Loss Unweighted: 0.6384 | Train Loss Components: Auxiliary: 0.2362 | Cardinality: 0.1573 | Classification: 0.0898 | Classification_binary: 0.1853 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0021 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1654 | Regression_cos_phi: 0.0435 | Regression_energy: 0.1029 | Regression_eta: 0.0427 | Regression_pt: 0.1136 | Regression_sin_phi: 0.0847 | LR: 8.37e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5039s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7611 | max_loss: 0.1442 | max_backward: 1.7887 | max_optimizer_step: 0.2976 | max_tensorboard_logging: 0.7418 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6219.2 | uss_MB: 5309.9 | child_rss_MB: 5948.9 | allocated_MB: 446.8 | reserved_MB: 29264.0 | inactive_split_MB: 1150.9 +rank5 [2026-10-28 23:00:55,745] INFO : Running validation on rank5 +rank5 [2026-10-28 23:00:55,746] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 23:00:55,746] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 23:01:50,596] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:01:51,151] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 23:02:17,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 23:02:18,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 23:02:20,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 23:02:21,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 23:02:31,992] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-28 23:02:33,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:02:33,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 23:02:49,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 23:02:50,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 23:02:51,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 23:02:52,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 23:03:02,720] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 23:03:03,407] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:03:04,128] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 23:03:19,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 23:03:20,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 23:03:21,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 23:03:22,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 23:03:32,212] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 23:08:22,314] INFO : Step 14100/50000 rank5 | Train Loss: 1.3472 | Train Loss Unweighted: 0.6964 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1511 | Classification: 0.1164 | Classification_binary: 0.2090 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1819 | Regression_eta: 0.0023 | Regression_pt: 0.1843 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.2273 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0924 | Regression_eta: 0.0507 | Regression_pt: 0.1010 | Regression_sin_phi: 0.0758 | LR: 8.35e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.2189s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8885 | max_loss: 0.1430 | max_backward: 2.1784 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 1.1214 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7149.4 | uss_MB: 6238.2 | child_rss_MB: 5960.3 | allocated_MB: 451.8 | reserved_MB: 25710.0 | inactive_split_MB: 1103.7 +rank5 [2026-10-28 23:12:52,862] INFO : Step 14200/50000 rank5 | Train Loss: 1.4358 | Train Loss Unweighted: 0.7354 | Train Loss Components: Auxiliary: 0.2744 | Cardinality: 0.1685 | Classification: 0.1248 | Classification_binary: 0.2623 | HitGrouping: 0.1814 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1709 | Regression_eta: 0.0023 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2832 | Regression_cos_phi: 0.1240 | Regression_energy: 0.0969 | Regression_eta: 0.0716 | Regression_pt: 0.1047 | Regression_sin_phi: 0.1350 | LR: 8.32e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3437s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0875 | max_forward: 0.8579 | max_loss: 0.1577 | max_backward: 1.8303 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6333.7 | uss_MB: 5422.0 | child_rss_MB: 5960.5 | allocated_MB: 449.2 | reserved_MB: 27638.0 | inactive_split_MB: 1319.6 +rank5 [2026-10-28 23:17:26,477] INFO : Step 14300/50000 rank5 | Train Loss: 1.2642 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2452 | Cardinality: 0.1652 | Classification: 0.1069 | Classification_binary: 0.2108 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0018 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0724 | Classification_binary: 0.2138 | Regression_cos_phi: 0.1176 | Regression_energy: 0.0938 | Regression_eta: 0.0591 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0644 | LR: 8.30e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.3466s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7599 | max_loss: 0.1477 | max_backward: 1.7369 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6331.9 | uss_MB: 5419.4 | child_rss_MB: 5959.7 | allocated_MB: 446.3 | reserved_MB: 29582.0 | inactive_split_MB: 949.7 +rank5 [2026-10-28 23:22:05,870] INFO : Step 14400/50000 rank5 | Train Loss: 1.2457 | Train Loss Unweighted: 0.6253 | Train Loss Components: Auxiliary: 0.2300 | Cardinality: 0.1408 | Classification: 0.1072 | Classification_binary: 0.1847 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1629 | Regression_eta: 0.0022 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0693 | Classification_binary: 0.1830 | Regression_cos_phi: 0.1059 | Regression_energy: 0.1057 | Regression_eta: 0.0614 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0936 | LR: 8.28e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.9719s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.9667 | max_loss: 0.2306 | max_backward: 2.7434 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6331.8 | uss_MB: 4969.8 | child_rss_MB: 5956.2 | allocated_MB: 450.7 | reserved_MB: 29584.0 | inactive_split_MB: 1252.8 +rank5 [2026-10-28 23:26:50,540] INFO : Step 14500/50000 rank5 | Train Loss: 1.4093 | Train Loss Unweighted: 0.7194 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1803 | Classification: 0.1211 | Classification_binary: 0.2406 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.2131 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0975 | Regression_eta: 0.0480 | Regression_pt: 0.1062 | Regression_sin_phi: 0.0784 | LR: 8.26e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8485s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8204 | max_loss: 0.1455 | max_backward: 2.0838 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7592 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1066 | Memory: rss_MB: 6329.0 | uss_MB: 5423.5 | child_rss_MB: 5932.6 | allocated_MB: 449.1 | reserved_MB: 29584.0 | inactive_split_MB: 1312.1 +rank5 [2026-10-28 23:31:26,129] INFO : Step 14600/50000 rank5 | Train Loss: 1.4466 | Train Loss Unweighted: 0.7628 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1607 | Classification: 0.1257 | Classification_binary: 0.2761 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1767 | Regression_eta: 0.0023 | Regression_pt: 0.1796 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.2697 | Regression_cos_phi: 0.1063 | Regression_energy: 0.0965 | Regression_eta: 0.0623 | Regression_pt: 0.1050 | Regression_sin_phi: 0.1278 | LR: 8.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8556s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0622 | max_forward: 0.8658 | max_loss: 0.1496 | max_backward: 1.9094 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.9894 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6320.2 | uss_MB: 5414.6 | child_rss_MB: 5962.0 | allocated_MB: 450.2 | reserved_MB: 29584.0 | inactive_split_MB: 939.8 +rank5 [2026-10-28 23:35:59,711] INFO : Step 14700/50000 rank5 | Train Loss: 1.2298 | Train Loss Unweighted: 0.6544 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1321 | Classification: 0.0995 | Classification_binary: 0.1882 | HitGrouping: 0.1788 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1813 | Regression_eta: 0.0021 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2134 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0916 | Regression_eta: 0.1129 | Regression_pt: 0.0983 | Regression_sin_phi: 0.1203 | LR: 8.21e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9560s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8361 | max_loss: 0.2462 | max_backward: 1.9700 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.8314 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6323.9 | uss_MB: 5421.4 | child_rss_MB: 5964.3 | allocated_MB: 445.8 | reserved_MB: 31548.0 | inactive_split_MB: 1127.1 +rank5 [2026-10-28 23:40:34,505] INFO : Step 14800/50000 rank5 | Train Loss: 1.2682 | Train Loss Unweighted: 0.6574 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1516 | Classification: 0.1063 | Classification_binary: 0.2120 | HitGrouping: 0.1767 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1664 | Regression_eta: 0.0020 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.2395 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0895 | Regression_eta: 0.0570 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0801 | LR: 8.19e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.6946s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7530 | max_loss: 0.1476 | max_backward: 1.8229 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6323.4 | uss_MB: 5420.5 | child_rss_MB: 5957.9 | allocated_MB: 449.4 | reserved_MB: 33554.0 | inactive_split_MB: 1332.6 +rank5 [2026-10-28 23:45:11,571] INFO : Step 14900/50000 rank5 | Train Loss: 1.2753 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2409 | Cardinality: 0.1305 | Classification: 0.1126 | Classification_binary: 0.2387 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1497 | Regression_eta: 0.0022 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.2650 | Regression_cos_phi: 0.1217 | Regression_energy: 0.0841 | Regression_eta: 0.0705 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0961 | LR: 8.17e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8751s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.7955 | max_loss: 0.1633 | max_backward: 1.8376 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.9623 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1206 | Memory: rss_MB: 6322.3 | uss_MB: 5420.8 | child_rss_MB: 5961.8 | allocated_MB: 449.7 | reserved_MB: 33554.0 | inactive_split_MB: 1310.1 +rank5 [2026-10-28 23:49:45,306] INFO : Step 15000/50000 rank5 | Train Loss: 1.2701 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1516 | Classification: 0.1012 | Classification_binary: 0.2121 | HitGrouping: 0.1755 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.2113 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0938 | Regression_eta: 0.0488 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0962 | LR: 8.15e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1007s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7505 | max_loss: 0.2449 | max_backward: 1.7335 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.4870 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6318.2 | uss_MB: 5419.0 | child_rss_MB: 5953.6 | allocated_MB: 447.3 | reserved_MB: 33554.0 | inactive_split_MB: 1038.0 +rank5 [2026-10-28 23:49:46,286] INFO : Running validation on rank5 +rank5 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-28 23:49:46,287] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-28 23:50:37,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:50:38,005] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-28 23:51:04,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-28 23:51:05,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-28 23:51:06,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-28 23:51:07,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-28 23:51:17,537] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 23:51:18,492] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:51:18,970] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-28 23:51:35,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-28 23:51:36,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-28 23:51:37,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-28 23:51:37,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-28 23:51:47,990] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-28 23:51:48,549] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-28 23:51:49,025] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-28 23:52:04,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-28 23:52:05,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-28 23:52:06,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-28 23:52:07,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-28 23:52:17,117] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-28 23:57:18,881] INFO : Step 15100/50000 rank5 | Train Loss: 1.2388 | Train Loss Unweighted: 0.6590 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1366 | Classification: 0.1178 | Classification_binary: 0.1841 | HitGrouping: 0.1810 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1743 | Regression_eta: 0.0018 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2081 | Regression_cos_phi: 0.1055 | Regression_energy: 0.0895 | Regression_eta: 0.0470 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0972 | LR: 8.12e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8274s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.8010 | max_loss: 0.1660 | max_backward: 1.9832 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.8565 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7422.7 | uss_MB: 6518.8 | child_rss_MB: 5938.1 | allocated_MB: 446.2 | reserved_MB: 25230.0 | inactive_split_MB: 1087.3 +rank5 [2026-10-29 00:02:05,559] INFO : Step 15200/50000 rank5 | Train Loss: 1.3362 | Train Loss Unweighted: 0.6611 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1596 | Classification: 0.1086 | Classification_binary: 0.2256 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1600 | Regression_eta: 0.0022 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.2343 | Regression_cos_phi: 0.1106 | Regression_energy: 0.0822 | Regression_eta: 0.0464 | Regression_pt: 0.0907 | Regression_sin_phi: 0.1099 | LR: 8.10e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.6181s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1077 | max_forward: 0.8572 | max_loss: 0.1505 | max_backward: 1.8951 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7089 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6402.3 | uss_MB: 5500.4 | child_rss_MB: 5939.7 | allocated_MB: 449.9 | reserved_MB: 25448.0 | inactive_split_MB: 1183.4 +rank5 [2026-10-29 00:06:54,749] INFO : Step 15300/50000 rank5 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6783 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1400 | Classification: 0.1062 | Classification_binary: 0.2357 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1646 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0745 | Classification_binary: 0.2078 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0817 | Regression_eta: 0.0464 | Regression_pt: 0.0912 | Regression_sin_phi: 0.1179 | LR: 8.08e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.7474s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7962 | max_loss: 0.2606 | max_backward: 1.9831 | max_optimizer_step: 0.2491 | max_tensorboard_logging: 0.7840 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6402.1 | uss_MB: 5500.3 | child_rss_MB: 5940.3 | allocated_MB: 446.7 | reserved_MB: 27282.0 | inactive_split_MB: 1189.4 +rank5 [2026-10-29 00:11:34,440] INFO : Step 15400/50000 rank5 | Train Loss: 1.2329 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1583 | Classification: 0.1127 | Classification_binary: 0.1891 | HitGrouping: 0.1750 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1648 | Regression_eta: 0.0018 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1665 | Regression_cos_phi: 0.0808 | Regression_energy: 0.1013 | Regression_eta: 0.0448 | Regression_pt: 0.1092 | Regression_sin_phi: 0.0847 | LR: 8.05e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2516s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.7153 | max_loss: 0.1508 | max_backward: 1.6501 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.5965 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6395.9 | uss_MB: 5491.2 | child_rss_MB: 5941.0 | allocated_MB: 444.7 | reserved_MB: 27520.0 | inactive_split_MB: 1102.9 +rank5 [2026-10-29 00:16:23,332] INFO : Step 15500/50000 rank5 | Train Loss: 1.2944 | Train Loss Unweighted: 0.6657 | Train Loss Components: Auxiliary: 0.2480 | Cardinality: 0.1645 | Classification: 0.1174 | Classification_binary: 0.2036 | HitGrouping: 0.1739 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1691 | Regression_eta: 0.0021 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0957 | Regression_energy: 0.0886 | Regression_eta: 0.0615 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0947 | LR: 8.03e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.2294s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.9970 | max_loss: 0.1740 | max_backward: 2.3916 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.8759 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 6406.1 | uss_MB: 5472.3 | child_rss_MB: 5938.5 | allocated_MB: 449.9 | reserved_MB: 27520.0 | inactive_split_MB: 1174.5 +rank5 [2026-10-29 00:21:07,253] INFO : Step 15600/50000 rank5 | Train Loss: 1.4613 | Train Loss Unweighted: 0.7617 | Train Loss Components: Auxiliary: 0.2872 | Cardinality: 0.1801 | Classification: 0.1276 | Classification_binary: 0.2641 | HitGrouping: 0.1785 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1795 | Regression_eta: 0.0025 | Regression_pt: 0.1856 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2941 | Regression_cos_phi: 0.1381 | Regression_energy: 0.0974 | Regression_eta: 0.0788 | Regression_pt: 0.1094 | Regression_sin_phi: 0.1004 | LR: 8.01e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.7223s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.9893 | max_loss: 0.1680 | max_backward: 2.1731 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.0419 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6396.7 | uss_MB: 5034.1 | child_rss_MB: 5938.2 | allocated_MB: 448.1 | reserved_MB: 27520.0 | inactive_split_MB: 1080.9 +rank5 [2026-10-29 00:25:54,170] INFO : Step 15700/50000 rank5 | Train Loss: 1.3299 | Train Loss Unweighted: 0.6547 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1880 | Classification: 0.1192 | Classification_binary: 0.2203 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1547 | Regression_eta: 0.0020 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2343 | Regression_cos_phi: 0.1142 | Regression_energy: 0.0873 | Regression_eta: 0.0500 | Regression_pt: 0.0953 | Regression_sin_phi: 0.1347 | LR: 7.98e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.0338s | Phase Times: max_pre_step_barrier: 0.0061 | max_batch_to_device: 0.0068 | max_forward: 0.9761 | max_loss: 0.2543 | max_backward: 2.0225 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8016 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.1020 | Memory: rss_MB: 6402.6 | uss_MB: 5496.7 | child_rss_MB: 5941.3 | allocated_MB: 448.3 | reserved_MB: 27520.0 | inactive_split_MB: 1390.5 +rank5 [2026-10-29 00:30:33,526] INFO : Step 15800/50000 rank5 | Train Loss: 1.2079 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1572 | Classification: 0.0909 | Classification_binary: 0.1814 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1704 | Regression_eta: 0.0019 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0833 | Regression_energy: 0.0809 | Regression_eta: 0.0529 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0698 | LR: 7.96e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3429s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1066 | max_forward: 0.8154 | max_loss: 0.1440 | max_backward: 1.6020 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.7740 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6405.3 | uss_MB: 5497.2 | child_rss_MB: 5941.3 | allocated_MB: 447.1 | reserved_MB: 27520.0 | inactive_split_MB: 1038.3 +rank5 [2026-10-29 00:35:19,264] INFO : Step 15900/50000 rank5 | Train Loss: 1.4055 | Train Loss Unweighted: 0.7313 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1808 | Classification: 0.1304 | Classification_binary: 0.2426 | HitGrouping: 0.1763 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1752 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0713 | Classification_binary: 0.2425 | Regression_cos_phi: 0.0727 | Regression_energy: 0.0982 | Regression_eta: 0.0474 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0885 | LR: 7.94e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7721s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 1.0653 | max_loss: 0.1515 | max_backward: 2.0024 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.6957 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6402.5 | uss_MB: 5500.0 | child_rss_MB: 5940.1 | allocated_MB: 447.4 | reserved_MB: 27520.0 | inactive_split_MB: 1172.5 +rank5 [2026-10-29 00:40:00,683] INFO : Step 16000/50000 rank5 | Train Loss: 1.3107 | Train Loss Unweighted: 0.6580 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1760 | Classification: 0.1019 | Classification_binary: 0.2391 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1548 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.2299 | Regression_cos_phi: 0.0611 | Regression_energy: 0.0830 | Regression_eta: 0.0434 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0745 | LR: 7.91e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.7638s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.1053 | max_forward: 1.1703 | max_loss: 0.1516 | max_backward: 1.8974 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6403.0 | uss_MB: 5502.7 | child_rss_MB: 5941.0 | allocated_MB: 447.2 | reserved_MB: 27520.0 | inactive_split_MB: 1195.6 +rank5 [2026-10-29 00:40:01,696] INFO : Running validation on rank5 +rank5 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 00:40:58,073] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:40:58,626] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 00:41:23,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 00:41:24,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 00:41:25,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 00:41:26,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 00:41:36,575] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 00:41:36,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:41:37,530] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 00:41:53,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 00:41:54,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 00:41:55,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 00:41:56,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 00:42:06,896] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 00:42:07,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 00:42:08,022] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 00:42:23,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 00:42:24,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 00:42:25,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 00:42:26,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 00:42:36,049] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 00:47:34,083] INFO : Step 16100/50000 rank5 | Train Loss: 1.2037 | Train Loss Unweighted: 0.6025 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1467 | Classification: 0.1080 | Classification_binary: 0.1878 | HitGrouping: 0.1747 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1512 | Regression_eta: 0.0018 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.2176 | Regression_cos_phi: 0.1109 | Regression_energy: 0.0828 | Regression_eta: 0.0487 | Regression_pt: 0.0910 | Regression_sin_phi: 0.0717 | LR: 7.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0447s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7665 | max_loss: 0.1405 | max_backward: 1.4602 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7547.3 | uss_MB: 6643.5 | child_rss_MB: 5941.5 | allocated_MB: 446.1 | reserved_MB: 27680.0 | inactive_split_MB: 1038.2 +rank5 [2026-10-29 00:52:15,156] INFO : Step 16200/50000 rank5 | Train Loss: 1.2083 | Train Loss Unweighted: 0.6192 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1499 | Classification: 0.1107 | Classification_binary: 0.2017 | HitGrouping: 0.1711 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1506 | Regression_eta: 0.0018 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.2070 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0745 | Regression_eta: 0.0654 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0684 | LR: 7.86e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9521s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0473 | max_forward: 0.8343 | max_loss: 0.2530 | max_backward: 2.1136 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6458.0 | uss_MB: 5555.6 | child_rss_MB: 5961.3 | allocated_MB: 446.7 | reserved_MB: 27682.0 | inactive_split_MB: 1157.2 +rank5 [2026-10-29 00:56:51,718] INFO : Step 16300/50000 rank5 | Train Loss: 1.2745 | Train Loss Unweighted: 0.6291 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1604 | Classification: 0.1062 | Classification_binary: 0.2417 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1373 | Regression_eta: 0.0022 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.2374 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0738 | Regression_eta: 0.0927 | Regression_pt: 0.0801 | Regression_sin_phi: 0.0760 | LR: 7.84e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.8116s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.8059 | max_loss: 0.2506 | max_backward: 2.1763 | max_optimizer_step: 0.5213 | max_tensorboard_logging: 0.5923 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6459.9 | uss_MB: 5555.3 | child_rss_MB: 5936.4 | allocated_MB: 450.9 | reserved_MB: 27682.0 | inactive_split_MB: 950.8 +rank5 [2026-10-29 01:01:26,582] INFO : Step 16400/50000 rank5 | Train Loss: 1.2654 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2380 | Cardinality: 0.1554 | Classification: 0.1095 | Classification_binary: 0.2018 | HitGrouping: 0.1699 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1561 | Regression_eta: 0.0023 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0924 | Classification_binary: 0.2530 | Regression_cos_phi: 0.1391 | Regression_energy: 0.0796 | Regression_eta: 0.0778 | Regression_pt: 0.0869 | Regression_sin_phi: 0.1034 | LR: 7.81e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.2788s | Phase Times: max_pre_step_barrier: 0.0533 | max_batch_to_device: 0.0064 | max_forward: 0.7459 | max_loss: 0.1486 | max_backward: 1.5352 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.6677 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6452.2 | uss_MB: 5545.1 | child_rss_MB: 5962.3 | allocated_MB: 446.4 | reserved_MB: 27682.0 | inactive_split_MB: 949.1 +rank5 [2026-10-29 01:06:05,050] INFO : Step 16500/50000 rank5 | Train Loss: 1.1728 | Train Loss Unweighted: 0.5918 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1444 | Classification: 0.0927 | Classification_binary: 0.1686 | HitGrouping: 0.1723 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0021 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1754 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0906 | Regression_eta: 0.0465 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0502 | LR: 7.79e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4922s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8043 | max_loss: 0.1772 | max_backward: 2.0250 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6458.1 | uss_MB: 5554.8 | child_rss_MB: 5927.7 | allocated_MB: 448.6 | reserved_MB: 29698.0 | inactive_split_MB: 1167.6 +rank5 [2026-10-29 01:10:53,703] INFO : Step 16600/50000 rank5 | Train Loss: 1.2920 | Train Loss Unweighted: 0.6427 | Train Loss Components: Auxiliary: 0.2455 | Cardinality: 0.1794 | Classification: 0.1148 | Classification_binary: 0.2098 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1561 | Regression_eta: 0.0021 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2103 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0803 | Regression_eta: 0.0441 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0804 | LR: 7.77e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2538s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0377 | max_forward: 1.0830 | max_loss: 0.1480 | max_backward: 1.7919 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6457.2 | uss_MB: 5552.9 | child_rss_MB: 5951.6 | allocated_MB: 449.3 | reserved_MB: 29698.0 | inactive_split_MB: 1014.7 +rank5 [2026-10-29 01:15:45,846] INFO : Step 16700/50000 rank5 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6325 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1731 | Classification: 0.1072 | Classification_binary: 0.2252 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1469 | Regression_eta: 0.0019 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0612 | Classification_binary: 0.2166 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0659 | Regression_eta: 0.0484 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0589 | LR: 7.74e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4543s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0722 | max_loss: 0.1506 | max_backward: 2.0365 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5067 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6456.8 | uss_MB: 5555.5 | child_rss_MB: 5918.8 | allocated_MB: 448.3 | reserved_MB: 29698.0 | inactive_split_MB: 893.3 +rank5 [2026-10-29 01:20:20,823] INFO : Step 16800/50000 rank5 | Train Loss: 1.1947 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1542 | Classification: 0.1036 | Classification_binary: 0.1800 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0023 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1711 | Regression_cos_phi: 0.0867 | Regression_energy: 0.0794 | Regression_eta: 0.0548 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0669 | LR: 7.72e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2736s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0166 | max_forward: 0.8224 | max_loss: 0.1466 | max_backward: 1.6608 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6428.0 | uss_MB: 5552.9 | child_rss_MB: 5812.3 | allocated_MB: 446.4 | reserved_MB: 29698.0 | inactive_split_MB: 1161.6 +rank5 [2026-10-29 01:25:06,863] INFO : Step 16900/50000 rank5 | Train Loss: 1.2025 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1406 | Classification: 0.1120 | Classification_binary: 0.1884 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1446 | Regression_eta: 0.0023 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.1952 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0794 | Regression_eta: 0.0798 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0991 | LR: 7.69e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.3235s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7445 | max_loss: 0.1453 | max_backward: 1.8211 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6418.8 | uss_MB: 5553.4 | child_rss_MB: 5783.4 | allocated_MB: 448.9 | reserved_MB: 29698.0 | inactive_split_MB: 901.3 +rank5 [2026-10-29 01:29:52,082] INFO : Step 17000/50000 rank5 | Train Loss: 1.2049 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2349 | Cardinality: 0.1467 | Classification: 0.1018 | Classification_binary: 0.2228 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.2248 | Regression_cos_phi: 0.1225 | Regression_energy: 0.0710 | Regression_eta: 0.0613 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0957 | LR: 7.67e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.1428s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0141 | max_forward: 1.3980 | max_loss: 0.5129 | max_backward: 2.5664 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.5255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1218 | Memory: rss_MB: 6382.7 | uss_MB: 5556.2 | child_rss_MB: 5576.1 | allocated_MB: 448.2 | reserved_MB: 29698.0 | inactive_split_MB: 955.8 +rank5 [2026-10-29 01:29:53,075] INFO : Running validation on rank5 +rank5 [2026-10-29 01:29:53,075] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 01:29:53,075] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 01:30:46,067] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:30:46,579] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 01:31:13,807] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 01:31:14,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 01:31:16,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 01:31:17,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 01:31:27,094] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 01:31:28,037] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:31:28,589] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 01:31:44,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 01:31:45,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 01:31:46,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 01:31:47,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 01:31:57,061] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 01:31:58,252] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 01:31:58,743] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 01:32:14,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 01:32:15,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 01:32:16,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 01:32:17,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 01:32:27,201] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 01:37:36,198] INFO : Step 17100/50000 rank5 | Train Loss: 1.2386 | Train Loss Unweighted: 0.5919 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1931 | Classification: 0.0995 | Classification_binary: 0.1969 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.2248 | Regression_cos_phi: 0.1308 | Regression_energy: 0.0758 | Regression_eta: 0.0625 | Regression_pt: 0.0825 | Regression_sin_phi: 0.0845 | LR: 7.64e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.3607s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0067 | max_forward: 1.0304 | max_loss: 0.1869 | max_backward: 2.3467 | max_optimizer_step: 0.3175 | max_tensorboard_logging: 0.7405 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 7631.0 | uss_MB: 6798.9 | child_rss_MB: 5579.4 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 1373.9 +rank5 [2026-10-29 01:42:27,135] INFO : Step 17200/50000 rank5 | Train Loss: 1.1810 | Train Loss Unweighted: 0.5844 | Train Loss Components: Auxiliary: 0.2270 | Cardinality: 0.1495 | Classification: 0.1085 | Classification_binary: 0.1683 | HitGrouping: 0.1731 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0022 | Regression_pt: 0.1515 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1907 | Regression_cos_phi: 0.0926 | Regression_energy: 0.0767 | Regression_eta: 0.0446 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0604 | LR: 7.62e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.1056s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7638 | max_loss: 0.2424 | max_backward: 1.9393 | max_optimizer_step: 0.2921 | max_tensorboard_logging: 2.1267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6444.8 | uss_MB: 5633.5 | child_rss_MB: 5577.8 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 988.1 +rank5 [2026-10-29 01:47:11,312] INFO : Step 17300/50000 rank5 | Train Loss: 1.2271 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2397 | Cardinality: 0.1372 | Classification: 0.1102 | Classification_binary: 0.2127 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1512 | Regression_eta: 0.0022 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2074 | Regression_cos_phi: 0.0977 | Regression_energy: 0.0839 | Regression_eta: 0.0688 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0968 | LR: 7.59e-04 | DataLoad Time: 0.0206s | Model Forward Time: 6.1198s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0243 | max_forward: 1.4273 | max_loss: 0.6053 | max_backward: 3.4810 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6426.5 | uss_MB: 5175.0 | child_rss_MB: 5586.3 | allocated_MB: 449.6 | reserved_MB: 26752.0 | inactive_split_MB: 1120.1 +rank5 [2026-10-29 01:52:04,203] INFO : Step 17400/50000 rank5 | Train Loss: 1.2297 | Train Loss Unweighted: 0.6013 | Train Loss Components: Auxiliary: 0.2404 | Cardinality: 0.1709 | Classification: 0.1017 | Classification_binary: 0.2232 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1354 | Regression_eta: 0.0019 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2200 | Regression_cos_phi: 0.0893 | Regression_energy: 0.0668 | Regression_eta: 0.0543 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0717 | LR: 7.57e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.6398s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0185 | max_forward: 1.0000 | max_loss: 0.1504 | max_backward: 1.9673 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6331 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6409.7 | uss_MB: 5631.7 | child_rss_MB: 5583.3 | allocated_MB: 447.1 | reserved_MB: 28732.0 | inactive_split_MB: 1125.6 +rank5 [2026-10-29 01:56:48,791] INFO : Step 17500/50000 rank5 | Train Loss: 1.2152 | Train Loss Unweighted: 0.6044 | Train Loss Components: Auxiliary: 0.2290 | Cardinality: 0.1485 | Classification: 0.1023 | Classification_binary: 0.1789 | HitGrouping: 0.1757 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1590 | Regression_eta: 0.0026 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1882 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0983 | Regression_eta: 0.0491 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0851 | LR: 7.54e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.2350s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0167 | max_forward: 0.7711 | max_loss: 0.1449 | max_backward: 1.8153 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.4947 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6388.7 | uss_MB: 5638.8 | child_rss_MB: 5515.3 | allocated_MB: 450.1 | reserved_MB: 28732.0 | inactive_split_MB: 1375.8 +rank5 [2026-10-29 02:01:39,791] INFO : Step 17600/50000 rank5 | Train Loss: 1.1560 | Train Loss Unweighted: 0.6042 | Train Loss Components: Auxiliary: 0.2286 | Cardinality: 0.1294 | Classification: 0.1057 | Classification_binary: 0.1993 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1471 | Regression_eta: 0.0018 | Regression_pt: 0.1484 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1923 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0667 | Regression_eta: 0.0442 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0633 | LR: 7.52e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8132s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.8868 | max_loss: 0.1705 | max_backward: 2.0183 | max_optimizer_step: 0.2066 | max_tensorboard_logging: 0.7222 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1349 | Memory: rss_MB: 6384.6 | uss_MB: 5639.3 | child_rss_MB: 5506.2 | allocated_MB: 448.3 | reserved_MB: 28732.0 | inactive_split_MB: 1319.7 +rank5 [2026-10-29 02:06:28,891] INFO : Step 17700/50000 rank5 | Train Loss: 1.7848 | Train Loss Unweighted: 0.7433 | Train Loss Components: Auxiliary: 0.3415 | Cardinality: 0.4306 | Classification: 0.1200 | Classification_binary: 0.2341 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1920 | Regression_eta: 0.0037 | Regression_pt: 0.1910 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.2355 | Regression_cos_phi: 0.0910 | Regression_energy: 0.1046 | Regression_eta: 0.0959 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0902 | LR: 7.49e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5776s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8788 | max_loss: 0.4738 | max_backward: 1.9859 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.5305 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6359.7 | uss_MB: 5640.4 | child_rss_MB: 5400.4 | allocated_MB: 450.1 | reserved_MB: 28738.0 | inactive_split_MB: 1125.6 +rank5 [2026-10-29 02:11:14,773] INFO : Step 17800/50000 rank5 | Train Loss: 1.1828 | Train Loss Unweighted: 0.6014 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1456 | Classification: 0.0982 | Classification_binary: 0.1996 | HitGrouping: 0.1749 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1503 | Regression_eta: 0.0019 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1878 | Regression_cos_phi: 0.0747 | Regression_energy: 0.0731 | Regression_eta: 0.0363 | Regression_pt: 0.0777 | Regression_sin_phi: 0.0549 | LR: 7.47e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.1932s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.3167 | max_backward: 2.2751 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 6343.6 | uss_MB: 5261.8 | child_rss_MB: 5331.5 | allocated_MB: 444.9 | reserved_MB: 28738.0 | inactive_split_MB: 1155.9 +rank5 [2026-10-29 02:16:10,311] INFO : Step 17900/50000 rank5 | Train Loss: 1.0765 | Train Loss Unweighted: 0.5646 | Train Loss Components: Auxiliary: 0.2108 | Cardinality: 0.1097 | Classification: 0.0932 | Classification_binary: 0.1585 | HitGrouping: 0.1686 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1536 | Regression_eta: 0.0016 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0782 | Regression_energy: 0.0729 | Regression_eta: 0.0339 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0610 | LR: 7.44e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2960s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7593 | max_loss: 0.4918 | max_backward: 1.7987 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.6052 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6290.9 | uss_MB: 5640.2 | child_rss_MB: 5109.0 | allocated_MB: 447.1 | reserved_MB: 28738.0 | inactive_split_MB: 1154.4 +rank5 [2026-10-29 02:20:55,876] INFO : Step 18000/50000 rank5 | Train Loss: 1.2227 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2303 | Cardinality: 0.1450 | Classification: 0.1088 | Classification_binary: 0.2099 | HitGrouping: 0.1650 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1447 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1887 | Regression_cos_phi: 0.0754 | Regression_energy: 0.0713 | Regression_eta: 0.0405 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0961 | LR: 7.42e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.7019s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9293 | max_loss: 0.1567 | max_backward: 2.2476 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6283.0 | uss_MB: 5642.9 | child_rss_MB: 5063.5 | allocated_MB: 449.4 | reserved_MB: 28738.0 | inactive_split_MB: 1324.2 +rank5 [2026-10-29 02:20:56,967] INFO : Running validation on rank5 +rank5 [2026-10-29 02:20:56,967] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 02:20:56,967] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 02:21:57,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:21:58,045] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 02:22:25,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 02:22:26,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 02:22:27,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 02:22:28,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 02:22:38,202] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 02:22:38,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:22:39,417] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 02:22:55,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 02:22:56,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 02:22:57,208] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 02:22:58,150] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 02:23:08,158] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 02:23:08,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 02:23:09,543] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 02:23:25,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 02:23:26,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 02:23:26,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 02:23:27,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 02:23:37,830] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 02:28:43,002] INFO : Step 18100/50000 rank5 | Train Loss: 1.2922 | Train Loss Unweighted: 0.6330 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1763 | Classification: 0.1061 | Classification_binary: 0.2257 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1467 | Regression_eta: 0.0022 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.2274 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0755 | Regression_eta: 0.0742 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0771 | LR: 7.39e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0681s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.9150 | max_loss: 0.1573 | max_backward: 2.1915 | max_optimizer_step: 0.2256 | max_tensorboard_logging: 0.8426 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7551.6 | uss_MB: 6907.4 | child_rss_MB: 5063.5 | allocated_MB: 448.8 | reserved_MB: 25396.0 | inactive_split_MB: 1192.4 +rank5 [2026-10-29 02:33:21,589] INFO : Step 18200/50000 rank5 | Train Loss: 1.1403 | Train Loss Unweighted: 0.5693 | Train Loss Components: Auxiliary: 0.2140 | Cardinality: 0.1362 | Classification: 0.1099 | Classification_binary: 0.1658 | HitGrouping: 0.1698 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1444 | Regression_eta: 0.0021 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0653 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0952 | Regression_energy: 0.0657 | Regression_eta: 0.0773 | Regression_pt: 0.0702 | Regression_sin_phi: 0.1001 | LR: 7.36e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.5284s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0059 | max_forward: 0.7843 | max_loss: 0.1427 | max_backward: 1.4948 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 1.0509 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6418.0 | uss_MB: 5773.9 | child_rss_MB: 5038.4 | allocated_MB: 446.0 | reserved_MB: 27340.0 | inactive_split_MB: 922.1 +rank5 [2026-10-29 02:37:53,622] INFO : Step 18300/50000 rank5 | Train Loss: 1.1938 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2339 | Cardinality: 0.1526 | Classification: 0.1133 | Classification_binary: 0.1878 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1493 | Regression_eta: 0.0019 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.1741 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0739 | Regression_eta: 0.0655 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0811 | LR: 7.34e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.9193s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.9058 | max_loss: 0.1527 | max_backward: 2.1689 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6422.1 | uss_MB: 5784.1 | child_rss_MB: 5032.3 | allocated_MB: 448.4 | reserved_MB: 27340.0 | inactive_split_MB: 1185.8 +rank5 [2026-10-29 02:42:24,375] INFO : Step 18400/50000 rank5 | Train Loss: 1.1959 | Train Loss Unweighted: 0.6109 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1130 | Classification: 0.1042 | Classification_binary: 0.2213 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1400 | Regression_eta: 0.0022 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.2138 | Regression_cos_phi: 0.0971 | Regression_energy: 0.0763 | Regression_eta: 0.0758 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0951 | LR: 7.31e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.7074s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8434 | max_loss: 0.1474 | max_backward: 1.9441 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6423.0 | uss_MB: 5783.9 | child_rss_MB: 5008.1 | allocated_MB: 448.8 | reserved_MB: 29306.0 | inactive_split_MB: 1091.4 +rank5 [2026-10-29 02:47:17,191] INFO : Step 18500/50000 rank5 | Train Loss: 1.2719 | Train Loss Unweighted: 0.6036 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1990 | Classification: 0.1087 | Classification_binary: 0.1793 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1538 | Regression_eta: 0.0024 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0720 | Classification_binary: 0.1831 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0845 | Regression_eta: 0.0667 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0807 | LR: 7.29e-04 | DataLoad Time: 0.0163s | Model Forward Time: 6.4227s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 1.6245 | max_loss: 0.3527 | max_backward: 3.4112 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.7932 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 6423.2 | uss_MB: 5781.7 | child_rss_MB: 5035.9 | allocated_MB: 447.9 | reserved_MB: 29306.0 | inactive_split_MB: 1193.4 +rank5 [2026-10-29 02:52:01,281] INFO : Step 18600/50000 rank5 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5476 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1351 | Classification: 0.0870 | Classification_binary: 0.1697 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1433 | Regression_eta: 0.0017 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0769 | Regression_eta: 0.0422 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0756 | LR: 7.26e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2408s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0527 | max_forward: 0.7639 | max_loss: 0.1373 | max_backward: 1.5273 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.7424 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6420.8 | uss_MB: 5779.1 | child_rss_MB: 5038.0 | allocated_MB: 445.6 | reserved_MB: 29306.0 | inactive_split_MB: 1194.4 +rank5 [2026-10-29 02:56:35,339] INFO : Step 18700/50000 rank5 | Train Loss: 1.2376 | Train Loss Unweighted: 0.5906 | Train Loss Components: Auxiliary: 0.2405 | Cardinality: 0.1841 | Classification: 0.1039 | Classification_binary: 0.2012 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1400 | Regression_eta: 0.0020 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1716 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0692 | Regression_eta: 0.0474 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0851 | LR: 7.24e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7565s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8903 | max_loss: 0.1469 | max_backward: 2.1433 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4822 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1128 | Memory: rss_MB: 6419.7 | uss_MB: 5784.3 | child_rss_MB: 5013.1 | allocated_MB: 449.2 | reserved_MB: 31324.0 | inactive_split_MB: 1093.4 +rank5 [2026-10-29 03:01:09,006] INFO : Step 18800/50000 rank5 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1585 | Classification: 0.0943 | Classification_binary: 0.2038 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1342 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0663 | Classification_binary: 0.2101 | Regression_cos_phi: 0.0977 | Regression_energy: 0.0725 | Regression_eta: 0.0466 | Regression_pt: 0.0804 | Regression_sin_phi: 0.1211 | LR: 7.21e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4217s | Phase Times: max_pre_step_barrier: 0.1142 | max_batch_to_device: 0.0068 | max_forward: 0.7808 | max_loss: 0.2502 | max_backward: 1.8229 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0996 | Memory: rss_MB: 6421.2 | uss_MB: 5781.7 | child_rss_MB: 5036.2 | allocated_MB: 450.4 | reserved_MB: 31336.0 | inactive_split_MB: 1098.4 +rank5 [2026-10-29 03:05:45,769] INFO : Step 18900/50000 rank5 | Train Loss: 1.0747 | Train Loss Unweighted: 0.5385 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.1306 | Classification: 0.0887 | Classification_binary: 0.1613 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1414 | Regression_eta: 0.0017 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1836 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0754 | Regression_eta: 0.0481 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0936 | LR: 7.18e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.3931s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7400 | max_loss: 0.1656 | max_backward: 1.8003 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.7596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6421.5 | uss_MB: 5781.2 | child_rss_MB: 5039.7 | allocated_MB: 443.5 | reserved_MB: 31336.0 | inactive_split_MB: 928.5 +rank5 [2026-10-29 03:10:20,956] INFO : Step 19000/50000 rank5 | Train Loss: 1.1726 | Train Loss Unweighted: 0.5966 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1418 | Classification: 0.1102 | Classification_binary: 0.1870 | HitGrouping: 0.1705 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1463 | Regression_eta: 0.0019 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0900 | Regression_energy: 0.0727 | Regression_eta: 0.0498 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0607 | LR: 7.16e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8914s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7637 | max_loss: 0.5203 | max_backward: 1.8720 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 1.1055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6422.5 | uss_MB: 5783.8 | child_rss_MB: 5041.8 | allocated_MB: 446.6 | reserved_MB: 31336.0 | inactive_split_MB: 1097.4 +rank5 [2026-10-29 03:10:21,983] INFO : Running validation on rank5 +rank5 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 03:11:18,711] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:11:19,247] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 03:11:45,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 03:11:46,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 03:11:48,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 03:11:49,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 03:11:59,833] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 03:12:01,384] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:12:01,927] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 03:12:18,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 03:12:19,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 03:12:20,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 03:12:20,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 03:12:31,002] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 03:12:31,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 03:12:31,972] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 03:12:47,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 03:12:48,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 03:12:48,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 03:12:49,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 03:12:59,677] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 03:17:58,239] INFO : Step 19100/50000 rank5 | Train Loss: 1.2150 | Train Loss Unweighted: 0.6107 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1439 | Classification: 0.1037 | Classification_binary: 0.2230 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1395 | Regression_eta: 0.0020 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.2331 | Regression_cos_phi: 0.1030 | Regression_energy: 0.0637 | Regression_eta: 0.0602 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0915 | LR: 7.13e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7823s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8630 | max_loss: 0.1532 | max_backward: 1.9622 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.7474 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7697.6 | uss_MB: 7053.9 | child_rss_MB: 5049.6 | allocated_MB: 449.4 | reserved_MB: 23972.0 | inactive_split_MB: 1371.8 +rank5 [2026-10-29 03:22:35,411] INFO : Step 19200/50000 rank5 | Train Loss: 1.1075 | Train Loss Unweighted: 0.5303 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1382 | Classification: 0.0958 | Classification_binary: 0.1758 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1270 | Regression_eta: 0.0019 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1949 | Regression_cos_phi: 0.0960 | Regression_energy: 0.0544 | Regression_eta: 0.0439 | Regression_pt: 0.0607 | Regression_sin_phi: 0.1175 | LR: 7.10e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0046s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8287 | max_loss: 0.1556 | max_backward: 1.8127 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6504.0 | uss_MB: 5863.3 | child_rss_MB: 5048.7 | allocated_MB: 445.3 | reserved_MB: 27640.0 | inactive_split_MB: 940.7 +rank5 [2026-10-29 03:27:09,997] INFO : Step 19300/50000 rank5 | Train Loss: 1.0792 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2111 | Cardinality: 0.1254 | Classification: 0.0911 | Classification_binary: 0.1561 | HitGrouping: 0.1662 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0777 | Regression_eta: 0.0523 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0707 | LR: 7.08e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.5150s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8891 | max_loss: 0.1513 | max_backward: 1.7637 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6508.5 | uss_MB: 5869.2 | child_rss_MB: 5044.9 | allocated_MB: 445.1 | reserved_MB: 27640.0 | inactive_split_MB: 1366.9 +rank5 [2026-10-29 03:31:44,570] INFO : Step 19400/50000 rank5 | Train Loss: 1.2411 | Train Loss Unweighted: 0.6183 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1555 | Classification: 0.1055 | Classification_binary: 0.2145 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1463 | Regression_eta: 0.0020 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.2194 | Regression_cos_phi: 0.0962 | Regression_energy: 0.0854 | Regression_eta: 0.0509 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0697 | LR: 7.05e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.6745s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0092 | max_forward: 0.8079 | max_loss: 0.1550 | max_backward: 1.7710 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 2.8282 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6511.0 | uss_MB: 5872.2 | child_rss_MB: 5045.1 | allocated_MB: 452.5 | reserved_MB: 28010.0 | inactive_split_MB: 1195.9 +rank5 [2026-10-29 03:36:21,473] INFO : Step 19500/50000 rank5 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6008 | Train Loss Components: Auxiliary: 0.2448 | Cardinality: 0.1836 | Classification: 0.1087 | Classification_binary: 0.2123 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1374 | Regression_eta: 0.0023 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2045 | Regression_cos_phi: 0.1063 | Regression_energy: 0.0680 | Regression_eta: 0.0488 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0657 | LR: 7.02e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6199s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7880 | max_loss: 0.1506 | max_backward: 1.7760 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.7620 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6510.7 | uss_MB: 5870.2 | child_rss_MB: 5041.2 | allocated_MB: 449.4 | reserved_MB: 29680.0 | inactive_split_MB: 1368.8 +rank5 [2026-10-29 03:40:54,623] INFO : Step 19600/50000 rank5 | Train Loss: 1.1451 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1464 | Classification: 0.0933 | Classification_binary: 0.1736 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1533 | Regression_eta: 0.0018 | Regression_pt: 0.1557 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1883 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0845 | Regression_eta: 0.0476 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0453 | LR: 7.00e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.3552s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8082 | max_loss: 0.1437 | max_backward: 1.6649 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.6567 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6510.9 | uss_MB: 5869.4 | child_rss_MB: 5045.0 | allocated_MB: 444.5 | reserved_MB: 29680.0 | inactive_split_MB: 1016.8 +rank5 [2026-10-29 03:45:30,310] INFO : Step 19700/50000 rank5 | Train Loss: 1.1424 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2236 | Cardinality: 0.1385 | Classification: 0.1093 | Classification_binary: 0.1797 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1379 | Regression_eta: 0.0020 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.1870 | Regression_cos_phi: 0.1263 | Regression_energy: 0.0654 | Regression_eta: 0.0533 | Regression_pt: 0.0706 | Regression_sin_phi: 0.1002 | LR: 6.97e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4880s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7612 | max_loss: 0.2297 | max_backward: 1.9668 | max_optimizer_step: 0.5229 | max_tensorboard_logging: 0.5756 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6508.5 | uss_MB: 5869.4 | child_rss_MB: 5047.4 | allocated_MB: 449.1 | reserved_MB: 29680.0 | inactive_split_MB: 1100.9 +rank5 [2026-10-29 03:50:08,780] INFO : Step 19800/50000 rank5 | Train Loss: 1.2806 | Train Loss Unweighted: 0.6390 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1432 | Classification: 0.1110 | Classification_binary: 0.2282 | HitGrouping: 0.1668 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0030 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2324 | Regression_cos_phi: 0.0909 | Regression_energy: 0.0902 | Regression_eta: 0.0901 | Regression_pt: 0.0985 | Regression_sin_phi: 0.0738 | LR: 6.94e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5065s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8034 | max_loss: 0.1812 | max_backward: 1.9502 | max_optimizer_step: 0.2119 | max_tensorboard_logging: 0.5857 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6510.7 | uss_MB: 5873.6 | child_rss_MB: 5045.7 | allocated_MB: 452.3 | reserved_MB: 29680.0 | inactive_split_MB: 1110.9 +rank5 [2026-10-29 03:54:46,662] INFO : Step 19900/50000 rank5 | Train Loss: 1.0692 | Train Loss Unweighted: 0.5407 | Train Loss Components: Auxiliary: 0.2077 | Cardinality: 0.1171 | Classification: 0.0922 | Classification_binary: 0.1812 | HitGrouping: 0.1610 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2149 | Regression_cos_phi: 0.0924 | Regression_energy: 0.0670 | Regression_eta: 0.0520 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0846 | LR: 6.92e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6118s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7533 | max_loss: 0.2412 | max_backward: 2.0720 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5848 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6505.8 | uss_MB: 5870.0 | child_rss_MB: 5057.5 | allocated_MB: 444.7 | reserved_MB: 29682.0 | inactive_split_MB: 1000.0 +rank5 [2026-10-29 03:59:25,168] INFO : Step 20000/50000 rank5 | Train Loss: 1.0304 | Train Loss Unweighted: 0.5275 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1060 | Classification: 0.0940 | Classification_binary: 0.1604 | HitGrouping: 0.1656 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1344 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1613 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0761 | Regression_eta: 0.0414 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0664 | LR: 6.89e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.8375s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.8721 | max_loss: 0.1507 | max_backward: 2.1861 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6505.7 | uss_MB: 5870.7 | child_rss_MB: 5049.5 | allocated_MB: 445.9 | reserved_MB: 30966.0 | inactive_split_MB: 1179.2 +rank5 [2026-10-29 03:59:26,239] INFO : Running validation on rank5 +rank5 [2026-10-29 03:59:26,240] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 03:59:26,240] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 04:00:19,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:00:19,530] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 04:00:44,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 04:00:46,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 04:00:47,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 04:00:48,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 04:00:58,257] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 04:00:59,045] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:00:59,501] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 04:01:15,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 04:01:16,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 04:01:17,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 04:01:18,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 04:01:28,382] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 04:01:28,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:01:29,372] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 04:01:44,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 04:01:45,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 04:01:46,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 04:01:47,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 04:01:57,317] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 04:06:53,361] INFO : Step 20100/50000 rank5 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2437 | Cardinality: 0.1586 | Classification: 0.1055 | Classification_binary: 0.2238 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1360 | Regression_eta: 0.0021 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2151 | Regression_cos_phi: 0.0995 | Regression_energy: 0.0791 | Regression_eta: 0.0557 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0908 | LR: 6.86e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6021s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0372 | max_forward: 0.8293 | max_loss: 0.1584 | max_backward: 1.7738 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.7139 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7674.6 | uss_MB: 7034.7 | child_rss_MB: 5034.6 | allocated_MB: 448.4 | reserved_MB: 29328.0 | inactive_split_MB: 969.2 +rank5 [2026-10-29 04:11:25,835] INFO : Step 20200/50000 rank5 | Train Loss: 1.3163 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.1695 | Classification: 0.1000 | Classification_binary: 0.2250 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1433 | Regression_eta: 0.0029 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.2414 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0800 | Regression_eta: 0.0746 | Regression_pt: 0.0885 | Regression_sin_phi: 0.1156 | LR: 6.84e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6427s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8823 | max_loss: 0.1595 | max_backward: 2.1030 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5136 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7674.6 | uss_MB: 7036.8 | child_rss_MB: 5027.8 | allocated_MB: 449.6 | reserved_MB: 29340.0 | inactive_split_MB: 938.7 +rank5 [2026-10-29 04:16:03,708] INFO : Step 20300/50000 rank5 | Train Loss: 1.1420 | Train Loss Unweighted: 0.5734 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1487 | Classification: 0.1087 | Classification_binary: 0.1713 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1438 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0802 | Classification_binary: 0.1664 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0691 | Regression_eta: 0.0413 | Regression_pt: 0.0751 | Regression_sin_phi: 0.0522 | LR: 6.81e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.9963s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.9650 | max_loss: 0.1904 | max_backward: 2.0801 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.8679 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6555.4 | uss_MB: 5913.2 | child_rss_MB: 5021.5 | allocated_MB: 444.8 | reserved_MB: 29344.0 | inactive_split_MB: 986.2 +rank5 [2026-10-29 04:20:44,976] INFO : Step 20400/50000 rank5 | Train Loss: 1.1926 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1642 | Classification: 0.1091 | Classification_binary: 0.1922 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1329 | Regression_eta: 0.0018 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0824 | Regression_eta: 0.0497 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0767 | LR: 6.78e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.2536s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0167 | max_forward: 1.2639 | max_loss: 0.1873 | max_backward: 2.4261 | max_optimizer_step: 0.3105 | max_tensorboard_logging: 0.7562 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6556.4 | uss_MB: 5916.7 | child_rss_MB: 5031.4 | allocated_MB: 447.7 | reserved_MB: 29344.0 | inactive_split_MB: 985.2 +rank5 [2026-10-29 04:25:27,116] INFO : Step 20500/50000 rank5 | Train Loss: 1.2110 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1591 | Classification: 0.1022 | Classification_binary: 0.2117 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1333 | Regression_eta: 0.0017 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.1932 | Regression_cos_phi: 0.0868 | Regression_energy: 0.0701 | Regression_eta: 0.0413 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0611 | LR: 6.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5482s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7705 | max_loss: 0.1525 | max_backward: 1.8060 | max_optimizer_step: 0.2622 | max_tensorboard_logging: 0.7595 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6553.9 | uss_MB: 5915.2 | child_rss_MB: 5013.6 | allocated_MB: 450.6 | reserved_MB: 29344.0 | inactive_split_MB: 1123.4 +rank5 [2026-10-29 04:30:07,866] INFO : Step 20600/50000 rank5 | Train Loss: 1.1161 | Train Loss Unweighted: 0.5564 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1436 | Classification: 0.0978 | Classification_binary: 0.1795 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1920 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0667 | Regression_eta: 0.0432 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0683 | LR: 6.73e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4382s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8358 | max_loss: 0.5148 | max_backward: 1.8393 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6556.9 | uss_MB: 5916.8 | child_rss_MB: 5024.7 | allocated_MB: 445.4 | reserved_MB: 31380.0 | inactive_split_MB: 917.0 +rank5 [2026-10-29 04:34:51,728] INFO : Step 20700/50000 rank5 | Train Loss: 1.0740 | Train Loss Unweighted: 0.5459 | Train Loss Components: Auxiliary: 0.2086 | Cardinality: 0.1252 | Classification: 0.0962 | Classification_binary: 0.1560 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1886 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0887 | Regression_eta: 0.0416 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0830 | LR: 6.70e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.0710s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7111 | max_loss: 0.2396 | max_backward: 1.6829 | max_optimizer_step: 0.2121 | max_tensorboard_logging: 0.4996 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6553.0 | uss_MB: 5912.9 | child_rss_MB: 5017.8 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1185.7 +rank5 [2026-10-29 04:39:29,326] INFO : Step 20800/50000 rank5 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5426 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1263 | Classification: 0.0965 | Classification_binary: 0.1999 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1207 | Regression_eta: 0.0017 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1879 | Regression_cos_phi: 0.0882 | Regression_energy: 0.0669 | Regression_eta: 0.0495 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0882 | LR: 6.67e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7620s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.9079 | max_loss: 0.1550 | max_backward: 2.1083 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.5897 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6553.9 | uss_MB: 5918.9 | child_rss_MB: 5012.5 | allocated_MB: 448.2 | reserved_MB: 31380.0 | inactive_split_MB: 1157.8 +rank5 [2026-10-29 04:44:09,834] INFO : Step 20900/50000 rank5 | Train Loss: 1.2188 | Train Loss Unweighted: 0.5970 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1525 | Classification: 0.1062 | Classification_binary: 0.2022 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0022 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1884 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0698 | Regression_eta: 0.0555 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0761 | LR: 6.65e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.7874s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0138 | max_forward: 0.8922 | max_loss: 0.1535 | max_backward: 2.0483 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6550.7 | uss_MB: 5916.4 | child_rss_MB: 5027.7 | allocated_MB: 447.8 | reserved_MB: 31380.0 | inactive_split_MB: 1126.3 +rank5 [2026-10-29 04:48:48,418] INFO : Step 21000/50000 rank5 | Train Loss: 1.0503 | Train Loss Unweighted: 0.5338 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1267 | Classification: 0.0900 | Classification_binary: 0.1573 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1405 | Regression_eta: 0.0016 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1725 | Regression_cos_phi: 0.0883 | Regression_energy: 0.0842 | Regression_eta: 0.0668 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0745 | LR: 6.62e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8543s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7596 | max_loss: 0.2536 | max_backward: 1.8713 | max_optimizer_step: 0.3177 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6551.7 | uss_MB: 5915.9 | child_rss_MB: 5032.5 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1148.0 +rank5 [2026-10-29 04:48:49,285] INFO : Running validation on rank5 +rank5 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 04:49:44,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:49:44,825] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 04:50:11,501] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 04:50:12,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 04:50:13,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 04:50:14,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 04:50:24,602] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 04:50:25,193] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:50:25,767] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 04:50:42,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 04:50:42,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 04:50:44,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 04:50:45,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 04:50:55,014] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 04:50:55,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 04:50:55,860] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 04:51:11,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 04:51:12,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 04:51:12,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 04:51:13,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 04:51:23,832] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 04:56:25,805] INFO : Step 21100/50000 rank5 | Train Loss: 1.1646 | Train Loss Unweighted: 0.5785 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1401 | Classification: 0.1095 | Classification_binary: 0.1927 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1344 | Regression_eta: 0.0022 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.2143 | Regression_cos_phi: 0.0847 | Regression_energy: 0.0671 | Regression_eta: 0.0452 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0879 | LR: 6.59e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.4037s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.9258 | max_loss: 0.1506 | max_backward: 2.4347 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.8145 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7847.0 | uss_MB: 5890.3 | child_rss_MB: 5018.3 | allocated_MB: 448.7 | reserved_MB: 23438.0 | inactive_split_MB: 911.7 +rank5 [2026-10-29 04:56:59,983] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank5 [2026-10-29 04:56:59,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-29 05:01:22,613] INFO : Step 21200/50000 rank5 | Train Loss: 1.1437 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1413 | Classification: 0.0985 | Classification_binary: 0.1768 | HitGrouping: 0.1669 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0020 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1989 | Regression_cos_phi: 0.1111 | Regression_energy: 0.0760 | Regression_eta: 0.0686 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0785 | LR: 6.56e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5653s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7993 | max_loss: 0.1450 | max_backward: 1.7427 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7842.9 | uss_MB: 7208.8 | child_rss_MB: 5012.9 | allocated_MB: 446.9 | reserved_MB: 23590.0 | inactive_split_MB: 1086.9 +rank5 [2026-10-29 05:06:02,214] INFO : Step 21300/50000 rank5 | Train Loss: 1.1586 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1648 | Classification: 0.1118 | Classification_binary: 0.1760 | HitGrouping: 0.1681 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1392 | Regression_eta: 0.0021 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1094 | Classification_binary: 0.1820 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0641 | Regression_eta: 0.0488 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0718 | LR: 6.54e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.2673s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0558 | max_forward: 0.9498 | max_loss: 0.1699 | max_backward: 2.4288 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.7788 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1270 | Memory: rss_MB: 6605.4 | uss_MB: 5978.0 | child_rss_MB: 5012.8 | allocated_MB: 447.4 | reserved_MB: 25440.0 | inactive_split_MB: 1003.9 +rank5 [2026-10-29 05:10:41,726] INFO : Step 21400/50000 rank5 | Train Loss: 1.1843 | Train Loss Unweighted: 0.5823 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1516 | Classification: 0.1038 | Classification_binary: 0.2065 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1335 | Regression_eta: 0.0020 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0892 | Classification_binary: 0.2064 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0716 | Regression_eta: 0.0662 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0737 | LR: 6.51e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.0959s | Phase Times: max_pre_step_barrier: 0.0085 | max_batch_to_device: 0.0640 | max_forward: 0.9920 | max_loss: 0.5036 | max_backward: 2.0711 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.8200 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6606.3 | uss_MB: 5978.8 | child_rss_MB: 5018.5 | allocated_MB: 448.9 | reserved_MB: 25440.0 | inactive_split_MB: 994.4 +rank5 [2026-10-29 05:15:15,088] INFO : Step 21500/50000 rank5 | Train Loss: 1.1484 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1478 | Classification: 0.0953 | Classification_binary: 0.2062 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1299 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1903 | Regression_cos_phi: 0.0462 | Regression_energy: 0.0718 | Regression_eta: 0.0383 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0537 | LR: 6.48e-04 | DataLoad Time: 0.0190s | Model Forward Time: 5.9382s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8371 | max_loss: 0.1537 | max_backward: 2.1150 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.8633 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6603.8 | uss_MB: 5979.6 | child_rss_MB: 5017.0 | allocated_MB: 447.2 | reserved_MB: 27056.0 | inactive_split_MB: 1468.9 +rank5 [2026-10-29 05:19:51,738] INFO : Step 21600/50000 rank5 | Train Loss: 1.0171 | Train Loss Unweighted: 0.5098 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1167 | Classification: 0.0973 | Classification_binary: 0.1559 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1265 | Regression_eta: 0.0014 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0554 | Regression_eta: 0.0351 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0490 | LR: 6.45e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5909s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0062 | max_forward: 0.7285 | max_loss: 0.1940 | max_backward: 1.8148 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.8819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6602.2 | uss_MB: 5978.2 | child_rss_MB: 5014.0 | allocated_MB: 445.7 | reserved_MB: 27056.0 | inactive_split_MB: 1196.5 +rank5 [2026-10-29 05:24:30,476] INFO : Step 21700/50000 rank5 | Train Loss: 1.1623 | Train Loss Unweighted: 0.5535 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1498 | Classification: 0.1025 | Classification_binary: 0.1819 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1695 | Regression_cos_phi: 0.1024 | Regression_energy: 0.0738 | Regression_eta: 0.0502 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0607 | LR: 6.43e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4938s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7947 | max_loss: 0.1379 | max_backward: 1.9808 | max_optimizer_step: 0.2157 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6600.2 | uss_MB: 5978.2 | child_rss_MB: 5010.8 | allocated_MB: 448.2 | reserved_MB: 27056.0 | inactive_split_MB: 1045.2 +rank5 [2026-10-29 05:29:08,471] INFO : Step 21800/50000 rank5 | Train Loss: 1.0898 | Train Loss Unweighted: 0.5518 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1233 | Classification: 0.0977 | Classification_binary: 0.1967 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1269 | Regression_eta: 0.0015 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.2100 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0811 | Regression_eta: 0.0398 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0644 | LR: 6.40e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.7238s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8952 | max_loss: 0.1765 | max_backward: 2.1628 | max_optimizer_step: 0.2138 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6602.0 | uss_MB: 5872.4 | child_rss_MB: 5012.0 | allocated_MB: 448.4 | reserved_MB: 27056.0 | inactive_split_MB: 1001.4 +rank5 [2026-10-29 05:33:46,455] INFO : Step 21900/50000 rank5 | Train Loss: 1.0699 | Train Loss Unweighted: 0.5306 | Train Loss Components: Auxiliary: 0.2121 | Cardinality: 0.1436 | Classification: 0.0991 | Classification_binary: 0.1704 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1276 | Regression_eta: 0.0015 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0719 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0701 | Regression_eta: 0.0409 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0685 | LR: 6.37e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4513s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7813 | max_loss: 0.1495 | max_backward: 1.8703 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 6600.0 | uss_MB: 5976.9 | child_rss_MB: 5013.0 | allocated_MB: 447.4 | reserved_MB: 27056.0 | inactive_split_MB: 1002.4 +rank5 [2026-10-29 05:38:23,934] INFO : Step 22000/50000 rank5 | Train Loss: 1.0659 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2100 | Cardinality: 0.1294 | Classification: 0.0955 | Classification_binary: 0.1521 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1360 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0761 | Regression_eta: 0.0421 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0593 | LR: 6.34e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4514s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7855 | max_loss: 0.1383 | max_backward: 1.9648 | max_optimizer_step: 0.2132 | max_tensorboard_logging: 0.5884 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6605.4 | uss_MB: 5979.9 | child_rss_MB: 5015.2 | allocated_MB: 449.1 | reserved_MB: 27056.0 | inactive_split_MB: 1180.9 +rank5 [2026-10-29 05:38:24,835] INFO : Running validation on rank5 +rank5 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 05:39:16,673] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:39:17,216] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 05:39:42,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 05:39:43,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 05:39:44,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 05:39:45,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 05:39:55,926] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 05:39:56,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:39:57,074] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 05:40:13,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 05:40:14,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 05:40:15,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 05:40:17,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 05:40:27,054] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 05:40:27,479] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 05:40:27,968] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 05:40:43,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 05:40:44,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 05:40:45,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 05:40:45,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 05:40:55,948] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 05:45:56,687] INFO : Step 22100/50000 rank5 | Train Loss: 1.1235 | Train Loss Unweighted: 0.5530 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1528 | Classification: 0.0997 | Classification_binary: 0.1905 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1304 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1725 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0690 | Regression_eta: 0.0329 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0588 | LR: 6.32e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5234s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8135 | max_loss: 0.1488 | max_backward: 1.7877 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5979 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7900.1 | uss_MB: 7271.0 | child_rss_MB: 5019.5 | allocated_MB: 451.8 | reserved_MB: 29136.0 | inactive_split_MB: 1136.2 +rank5 [2026-10-29 05:50:45,341] INFO : Step 22200/50000 rank5 | Train Loss: 1.1922 | Train Loss Unweighted: 0.5684 | Train Loss Components: Auxiliary: 0.2361 | Cardinality: 0.1699 | Classification: 0.0906 | Classification_binary: 0.2085 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2019 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0688 | Regression_eta: 0.0469 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0872 | LR: 6.29e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5684s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8616 | max_loss: 0.1545 | max_backward: 1.6969 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 0.7120 | max_tensorboard_flush: 0.0150 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7899.1 | uss_MB: 7269.8 | child_rss_MB: 5025.9 | allocated_MB: 448.1 | reserved_MB: 29136.0 | inactive_split_MB: 1371.0 +rank5 [2026-10-29 05:55:21,941] INFO : Step 22300/50000 rank5 | Train Loss: 1.0221 | Train Loss Unweighted: 0.5006 | Train Loss Components: Auxiliary: 0.1973 | Cardinality: 0.1169 | Classification: 0.0876 | Classification_binary: 0.1576 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1252 | Regression_eta: 0.0021 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1570 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0650 | Regression_eta: 0.0490 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0618 | LR: 6.26e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5037s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7903 | max_loss: 0.1446 | max_backward: 1.7078 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 0.8083 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6690.1 | uss_MB: 6062.0 | child_rss_MB: 5023.7 | allocated_MB: 446.4 | reserved_MB: 29136.0 | inactive_split_MB: 1159.3 +rank5 [2026-10-29 06:00:08,367] INFO : Step 22400/50000 rank5 | Train Loss: 1.1170 | Train Loss Unweighted: 0.5609 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1391 | Classification: 0.1065 | Classification_binary: 0.1807 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1348 | Regression_eta: 0.0019 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0613 | Classification_binary: 0.1831 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0770 | Regression_eta: 0.0408 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0584 | LR: 6.23e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9525s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7658 | max_loss: 0.2479 | max_backward: 1.9808 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9928 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6691.0 | uss_MB: 6065.0 | child_rss_MB: 5027.0 | allocated_MB: 447.1 | reserved_MB: 31152.0 | inactive_split_MB: 925.7 +rank5 [2026-10-29 06:04:41,559] INFO : Step 22500/50000 rank5 | Train Loss: 1.1245 | Train Loss Unweighted: 0.5482 | Train Loss Components: Auxiliary: 0.2231 | Cardinality: 0.1463 | Classification: 0.0856 | Classification_binary: 0.2083 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1244 | Regression_eta: 0.0016 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0543 | Regression_eta: 0.0352 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0702 | LR: 6.20e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.0097s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 1.1916 | max_loss: 0.1613 | max_backward: 2.1225 | max_optimizer_step: 0.2176 | max_tensorboard_logging: 0.7664 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6685.2 | uss_MB: 6063.6 | child_rss_MB: 5003.5 | allocated_MB: 447.4 | reserved_MB: 31152.0 | inactive_split_MB: 959.7 +rank5 [2026-10-29 06:09:13,551] INFO : Step 22600/50000 rank5 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5177 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1260 | Classification: 0.0928 | Classification_binary: 0.1789 | HitGrouping: 0.1619 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1208 | Regression_eta: 0.0019 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1728 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0630 | Regression_eta: 0.0360 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0518 | LR: 6.18e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.8019s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0651 | max_forward: 1.1064 | max_loss: 0.1531 | max_backward: 1.9236 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.8515 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6676.3 | uss_MB: 6063.0 | child_rss_MB: 5024.2 | allocated_MB: 448.3 | reserved_MB: 31154.0 | inactive_split_MB: 961.5 +rank5 [2026-10-29 06:13:51,719] INFO : Step 22700/50000 rank5 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5090 | Train Loss Components: Auxiliary: 0.1959 | Cardinality: 0.1101 | Classification: 0.0877 | Classification_binary: 0.1479 | HitGrouping: 0.1634 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1339 | Regression_eta: 0.0015 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0890 | Regression_eta: 0.0451 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0509 | LR: 6.15e-04 | DataLoad Time: 0.0213s | Model Forward Time: 4.4584s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 1.0743 | max_loss: 0.1383 | max_backward: 1.7235 | max_optimizer_step: 0.2204 | max_tensorboard_logging: 0.5170 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 6672.9 | uss_MB: 6061.8 | child_rss_MB: 5030.8 | allocated_MB: 446.5 | reserved_MB: 31154.0 | inactive_split_MB: 972.7 +rank5 [2026-10-29 06:18:32,627] INFO : Step 22800/50000 rank5 | Train Loss: 1.0730 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2215 | Cardinality: 0.1332 | Classification: 0.1095 | Classification_binary: 0.1782 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1289 | Regression_eta: 0.0014 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.1933 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0540 | Regression_eta: 0.0458 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0651 | LR: 6.12e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.7950s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.2500 | max_backward: 2.0107 | max_optimizer_step: 0.2761 | max_tensorboard_logging: 0.6168 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 6673.9 | uss_MB: 6062.6 | child_rss_MB: 5029.6 | allocated_MB: 447.2 | reserved_MB: 31154.0 | inactive_split_MB: 922.1 +rank5 [2026-10-29 06:23:13,211] INFO : Step 22900/50000 rank5 | Train Loss: 1.1830 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2389 | Cardinality: 0.1788 | Classification: 0.0948 | Classification_binary: 0.2023 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1393 | Regression_eta: 0.0018 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0761 | Regression_eta: 0.0365 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0519 | LR: 6.09e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.9933s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0073 | max_forward: 0.7992 | max_loss: 0.1478 | max_backward: 1.9051 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.9858 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 6664.2 | uss_MB: 6055.9 | child_rss_MB: 5028.8 | allocated_MB: 448.8 | reserved_MB: 31154.0 | inactive_split_MB: 1109.2 +rank5 [2026-10-29 06:27:51,864] INFO : Step 23000/50000 rank5 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5014 | Train Loss Components: Auxiliary: 0.1951 | Cardinality: 0.1184 | Classification: 0.0781 | Classification_binary: 0.1455 | HitGrouping: 0.1653 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1742 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0703 | Regression_eta: 0.0508 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0793 | LR: 6.06e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.6099s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7528 | max_loss: 0.2292 | max_backward: 1.7818 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.8884 | max_tensorboard_flush: 0.0060 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6672.0 | uss_MB: 6063.2 | child_rss_MB: 5028.5 | allocated_MB: 443.9 | reserved_MB: 31154.0 | inactive_split_MB: 985.1 +rank5 [2026-10-29 06:27:52,751] INFO : Running validation on rank5 +rank5 [2026-10-29 06:27:52,751] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 06:27:52,751] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 06:28:45,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:28:45,725] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 06:29:12,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 06:29:13,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 06:29:14,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 06:29:15,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 06:29:25,213] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 06:29:26,163] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:29:26,631] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 06:29:42,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 06:29:43,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 06:29:44,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 06:29:45,503] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 06:29:55,514] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 06:29:56,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 06:29:56,643] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 06:30:12,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 06:30:13,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 06:30:14,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 06:30:15,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 06:30:25,064] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 06:35:29,455] INFO : Step 23100/50000 rank5 | Train Loss: 1.1240 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1320 | Classification: 0.1122 | Classification_binary: 0.1745 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1274 | Regression_eta: 0.0021 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0913 | Classification_binary: 0.1944 | Regression_cos_phi: 0.1000 | Regression_energy: 0.0666 | Regression_eta: 0.0731 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0981 | LR: 6.04e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8779s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7914 | max_loss: 0.1430 | max_backward: 1.7255 | max_optimizer_step: 0.3295 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0695 | Memory: rss_MB: 7843.1 | uss_MB: 7233.5 | child_rss_MB: 5030.9 | allocated_MB: 449.4 | reserved_MB: 25680.0 | inactive_split_MB: 1046.0 +rank5 [2026-10-29 06:40:11,955] INFO : Step 23200/50000 rank5 | Train Loss: 1.1736 | Train Loss Unweighted: 0.5745 | Train Loss Components: Auxiliary: 0.2341 | Cardinality: 0.1586 | Classification: 0.1057 | Classification_binary: 0.2033 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1304 | Regression_eta: 0.0018 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0662 | Regression_eta: 0.0511 | Regression_pt: 0.0709 | Regression_sin_phi: 0.0806 | LR: 6.01e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.1118s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9381 | max_loss: 0.1692 | max_backward: 2.2545 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6161 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 7846.1 | uss_MB: 5861.1 | child_rss_MB: 5025.2 | allocated_MB: 447.8 | reserved_MB: 25680.0 | inactive_split_MB: 1169.7 +rank5 [2026-10-29 06:44:52,828] INFO : Step 23300/50000 rank5 | Train Loss: 1.0647 | Train Loss Unweighted: 0.5262 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1268 | Classification: 0.0958 | Classification_binary: 0.1665 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1301 | Regression_eta: 0.0017 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2084 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0741 | Regression_eta: 0.0452 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0884 | LR: 5.98e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.6091s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8613 | max_loss: 0.1501 | max_backward: 2.0035 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 0.6274 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6729.0 | uss_MB: 6118.5 | child_rss_MB: 5031.7 | allocated_MB: 447.3 | reserved_MB: 29702.0 | inactive_split_MB: 1252.4 +rank5 [2026-10-29 06:49:32,125] INFO : Step 23400/50000 rank5 | Train Loss: 0.9916 | Train Loss Unweighted: 0.5023 | Train Loss Components: Auxiliary: 0.1957 | Cardinality: 0.1164 | Classification: 0.1006 | Classification_binary: 0.1374 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1170 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0739 | Regression_eta: 0.0443 | Regression_pt: 0.0775 | Regression_sin_phi: 0.0425 | LR: 5.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3950s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7229 | max_loss: 0.2412 | max_backward: 1.6798 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.4992 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1172 | Memory: rss_MB: 6727.8 | uss_MB: 6117.0 | child_rss_MB: 5031.9 | allocated_MB: 445.7 | reserved_MB: 29702.0 | inactive_split_MB: 1158.3 +rank5 [2026-10-29 06:54:42,356] INFO : Step 23500/50000 rank5 | Train Loss: 1.0349 | Train Loss Unweighted: 0.5234 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1083 | Classification: 0.0851 | Classification_binary: 0.1870 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1235 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.1857 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0735 | Regression_eta: 0.0346 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0525 | LR: 5.92e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7768s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7924 | max_loss: 0.2020 | max_backward: 1.9787 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.7987 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6728.4 | uss_MB: 6111.2 | child_rss_MB: 5026.6 | allocated_MB: 446.9 | reserved_MB: 31774.0 | inactive_split_MB: 1042.0 +rank5 [2026-10-29 06:59:38,279] INFO : Step 23600/50000 rank5 | Train Loss: 1.1326 | Train Loss Unweighted: 0.5418 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1458 | Classification: 0.0933 | Classification_binary: 0.1900 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1267 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1865 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0551 | Regression_eta: 0.0426 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0922 | LR: 5.90e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9837s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0063 | max_forward: 0.7946 | max_loss: 0.1855 | max_backward: 1.9114 | max_optimizer_step: 0.2664 | max_tensorboard_logging: 0.9547 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6730.1 | uss_MB: 6120.3 | child_rss_MB: 5028.3 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1333.8 +rank5 [2026-10-29 07:04:20,063] INFO : Step 23700/50000 rank5 | Train Loss: 1.0167 | Train Loss Unweighted: 0.4968 | Train Loss Components: Auxiliary: 0.1932 | Cardinality: 0.1158 | Classification: 0.0853 | Classification_binary: 0.1475 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1285 | Regression_eta: 0.0017 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0641 | Regression_eta: 0.0311 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0469 | LR: 5.87e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.7163s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0221 | max_forward: 0.7952 | max_loss: 0.1461 | max_backward: 1.8308 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7718 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6725.6 | uss_MB: 6118.0 | child_rss_MB: 5028.2 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1160.4 +rank5 [2026-10-29 07:08:57,915] INFO : Step 23800/50000 rank5 | Train Loss: 1.0651 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1325 | Classification: 0.0886 | Classification_binary: 0.1616 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1318 | Regression_eta: 0.0015 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1712 | Regression_cos_phi: 0.0715 | Regression_energy: 0.0640 | Regression_eta: 0.0447 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0745 | LR: 5.84e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.6715s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8562 | max_loss: 0.1426 | max_backward: 1.8700 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6730.2 | uss_MB: 6120.2 | child_rss_MB: 5026.7 | allocated_MB: 449.4 | reserved_MB: 31776.0 | inactive_split_MB: 1046.7 +rank5 [2026-10-29 07:13:41,573] INFO : Step 23900/50000 rank5 | Train Loss: 1.1515 | Train Loss Unweighted: 0.5718 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1646 | Classification: 0.0962 | Classification_binary: 0.2086 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.2135 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0660 | Regression_eta: 0.0532 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0949 | LR: 5.81e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.8389s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.8887 | max_loss: 0.2019 | max_backward: 2.1185 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6224 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6714.4 | uss_MB: 6108.3 | child_rss_MB: 5032.9 | allocated_MB: 447.1 | reserved_MB: 31776.0 | inactive_split_MB: 1168.8 +rank5 [2026-10-29 07:18:20,710] INFO : Step 24000/50000 rank5 | Train Loss: 1.0174 | Train Loss Unweighted: 0.5187 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1133 | Classification: 0.0873 | Classification_binary: 0.1674 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0014 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1942 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0706 | Regression_eta: 0.0479 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0609 | LR: 5.78e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6305s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 1.3237 | max_loss: 0.2403 | max_backward: 1.8319 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5068 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6721.2 | uss_MB: 6114.9 | child_rss_MB: 5029.4 | allocated_MB: 446.4 | reserved_MB: 31776.0 | inactive_split_MB: 957.1 +rank5 [2026-10-29 07:18:21,624] INFO : Running validation on rank5 +rank5 [2026-10-29 07:18:21,625] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 07:18:21,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 07:19:13,461] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:19:13,969] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 07:19:41,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 07:19:42,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 07:19:43,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 07:19:44,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 07:19:54,572] INFO : Time taken to make predictions on device 5 is: 0.26 min +rank5 [2026-10-29 07:19:55,149] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:19:55,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 07:20:11,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 07:20:12,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 07:20:13,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 07:20:14,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 07:20:24,882] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 07:20:25,040] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 07:20:25,605] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 07:20:41,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 07:20:41,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 07:20:42,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 07:20:43,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 07:20:53,551] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 07:26:06,615] INFO : Step 24100/50000 rank5 | Train Loss: 1.0669 | Train Loss Unweighted: 0.5331 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1324 | Classification: 0.0928 | Classification_binary: 0.1578 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1390 | Regression_eta: 0.0017 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1301 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0775 | Regression_eta: 0.0376 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0817 | LR: 5.75e-04 | DataLoad Time: 0.0177s | Model Forward Time: 7.5186s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.7836 | max_loss: 0.1442 | max_backward: 1.8747 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 3.6030 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0626 | Memory: rss_MB: 7857.5 | uss_MB: 7243.1 | child_rss_MB: 5051.0 | allocated_MB: 448.1 | reserved_MB: 24490.0 | inactive_split_MB: 1198.5 +rank5 [2026-10-29 07:30:59,926] INFO : Step 24200/50000 rank5 | Train Loss: 1.1691 | Train Loss Unweighted: 0.5645 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1546 | Classification: 0.1031 | Classification_binary: 0.1915 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1325 | Regression_eta: 0.0019 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0951 | Regression_energy: 0.0831 | Regression_eta: 0.0661 | Regression_pt: 0.0908 | Regression_sin_phi: 0.1055 | LR: 5.73e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.2335s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.9207 | max_loss: 0.1533 | max_backward: 2.2288 | max_optimizer_step: 0.2436 | max_tensorboard_logging: 0.8522 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7858.2 | uss_MB: 7247.2 | child_rss_MB: 5039.0 | allocated_MB: 450.2 | reserved_MB: 26490.0 | inactive_split_MB: 976.9 +rank5 [2026-10-29 07:35:42,203] INFO : Step 24300/50000 rank5 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5336 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1380 | Classification: 0.0927 | Classification_binary: 0.1863 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1247 | Regression_eta: 0.0020 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1996 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0633 | Regression_eta: 0.0510 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0656 | LR: 5.70e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.6729s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8648 | max_loss: 0.1481 | max_backward: 1.7152 | max_optimizer_step: 0.2781 | max_tensorboard_logging: 0.7831 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6810.0 | uss_MB: 6199.1 | child_rss_MB: 5043.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 983.3 +rank5 [2026-10-29 07:40:19,152] INFO : Step 24400/50000 rank5 | Train Loss: 0.9707 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0879 | Classification_binary: 0.1454 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0014 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1456 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0673 | Regression_eta: 0.0482 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0707 | LR: 5.67e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.8574s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7628 | max_loss: 0.1940 | max_backward: 2.4253 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.9481 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6811.6 | uss_MB: 5803.5 | child_rss_MB: 5042.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 1129.3 +rank5 [2026-10-29 07:45:00,049] INFO : Step 24500/50000 rank5 | Train Loss: 1.0937 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1319 | Classification: 0.0941 | Classification_binary: 0.1676 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1334 | Regression_eta: 0.0016 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1795 | Regression_cos_phi: 0.1158 | Regression_energy: 0.0793 | Regression_eta: 0.0502 | Regression_pt: 0.0894 | Regression_sin_phi: 0.1000 | LR: 5.64e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4271s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7565 | max_loss: 0.5105 | max_backward: 1.8057 | max_optimizer_step: 0.2534 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6807.0 | uss_MB: 6198.6 | child_rss_MB: 5045.1 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 945.2 +rank5 [2026-10-29 07:49:44,236] INFO : Step 24600/50000 rank5 | Train Loss: 1.0924 | Train Loss Unweighted: 0.5511 | Train Loss Components: Auxiliary: 0.2237 | Cardinality: 0.1233 | Classification: 0.0963 | Classification_binary: 0.1991 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1255 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.2114 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0691 | Regression_eta: 0.0323 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0733 | LR: 5.61e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6477s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.9060 | max_loss: 0.1523 | max_backward: 2.0894 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.5102 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6811.1 | uss_MB: 6200.6 | child_rss_MB: 5046.3 | allocated_MB: 449.5 | reserved_MB: 26492.0 | inactive_split_MB: 943.2 +rank5 [2026-10-29 07:54:29,279] INFO : Step 24700/50000 rank5 | Train Loss: 0.9972 | Train Loss Unweighted: 0.4984 | Train Loss Components: Auxiliary: 0.1950 | Cardinality: 0.1160 | Classification: 0.0782 | Classification_binary: 0.1692 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1760 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0613 | Regression_eta: 0.0282 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0628 | LR: 5.58e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5126s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0060 | max_forward: 0.7847 | max_loss: 0.1519 | max_backward: 1.7159 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.8325 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6813.0 | uss_MB: 6199.7 | child_rss_MB: 5050.6 | allocated_MB: 447.9 | reserved_MB: 27708.0 | inactive_split_MB: 1169.6 +rank5 [2026-10-29 07:59:09,067] INFO : Step 24800/50000 rank5 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4722 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1202 | Classification: 0.0808 | Classification_binary: 0.1484 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0016 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1452 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0486 | Regression_eta: 0.0425 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0712 | LR: 5.56e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7178s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0127 | max_forward: 0.7903 | max_loss: 0.1427 | max_backward: 1.7798 | max_optimizer_step: 0.2260 | max_tensorboard_logging: 0.8943 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6813.0 | uss_MB: 6199.9 | child_rss_MB: 5051.5 | allocated_MB: 447.6 | reserved_MB: 28508.0 | inactive_split_MB: 989.1 +rank5 [2026-10-29 08:03:50,800] INFO : Step 24900/50000 rank5 | Train Loss: 1.0773 | Train Loss Unweighted: 0.5300 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1506 | Classification: 0.0964 | Classification_binary: 0.1776 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1257 | Regression_eta: 0.0016 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1753 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0636 | Regression_eta: 0.0347 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0435 | LR: 5.53e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.3163s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8100 | max_loss: 0.1483 | max_backward: 1.9423 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 1.0834 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6802.2 | uss_MB: 5593.7 | child_rss_MB: 5050.0 | allocated_MB: 449.1 | reserved_MB: 28508.0 | inactive_split_MB: 993.6 +rank5 [2026-10-29 08:08:37,678] INFO : Step 25000/50000 rank5 | Train Loss: 1.1315 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1671 | Classification: 0.0915 | Classification_binary: 0.1899 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1201 | Regression_eta: 0.0020 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1960 | Regression_cos_phi: 0.1182 | Regression_energy: 0.0789 | Regression_eta: 0.0554 | Regression_pt: 0.0858 | Regression_sin_phi: 0.1101 | LR: 5.50e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.7046s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8152 | max_loss: 0.2476 | max_backward: 1.7290 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.8750 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6807.0 | uss_MB: 6195.6 | child_rss_MB: 5052.2 | allocated_MB: 448.5 | reserved_MB: 28508.0 | inactive_split_MB: 953.9 +rank5 [2026-10-29 08:08:38,677] INFO : Running validation on rank5 +rank5 [2026-10-29 08:08:38,677] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 08:08:38,677] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 08:09:30,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:09:31,483] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 08:09:58,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 08:09:59,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 08:10:00,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 08:10:01,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 08:10:11,210] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 08:10:11,662] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:10:12,184] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 08:10:28,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 08:10:29,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 08:10:30,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 08:10:31,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 08:10:41,058] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 08:10:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 08:10:42,694] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 08:10:58,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 08:10:59,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 08:11:00,097] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 08:11:00,963] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 08:11:10,974] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 08:16:19,774] INFO : Step 25100/50000 rank5 | Train Loss: 1.0264 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1288 | Classification: 0.0850 | Classification_binary: 0.1497 | HitGrouping: 0.1664 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1378 | Regression_eta: 0.0016 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1512 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0765 | Regression_eta: 0.0359 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0499 | LR: 5.47e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.5436s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0059 | max_forward: 0.7878 | max_loss: 0.1437 | max_backward: 2.2383 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9692 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7903.8 | uss_MB: 7286.7 | child_rss_MB: 5042.9 | allocated_MB: 446.8 | reserved_MB: 24062.0 | inactive_split_MB: 1159.3 +rank5 [2026-10-29 08:21:20,700] INFO : Step 25200/50000 rank5 | Train Loss: 1.0775 | Train Loss Unweighted: 0.5434 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1397 | Classification: 0.0996 | Classification_binary: 0.1706 | HitGrouping: 0.1584 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1340 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0587 | Regression_eta: 0.0625 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0671 | LR: 5.44e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.2325s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8831 | max_loss: 0.1456 | max_backward: 2.1320 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7903.3 | uss_MB: 7287.2 | child_rss_MB: 5042.2 | allocated_MB: 448.0 | reserved_MB: 26042.0 | inactive_split_MB: 1093.3 +rank5 [2026-10-29 08:26:17,211] INFO : Step 25300/50000 rank5 | Train Loss: 1.0761 | Train Loss Unweighted: 0.5109 | Train Loss Components: Auxiliary: 0.2132 | Cardinality: 0.1361 | Classification: 0.0961 | Classification_binary: 0.1821 | HitGrouping: 0.1562 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1138 | Regression_eta: 0.0018 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0565 | Regression_eta: 0.0324 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0480 | LR: 5.42e-04 | DataLoad Time: 0.0190s | Model Forward Time: 6.1302s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.9122 | max_loss: 0.1821 | max_backward: 2.1499 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.9506 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6897.2 | uss_MB: 5222.8 | child_rss_MB: 5036.8 | allocated_MB: 446.7 | reserved_MB: 28062.0 | inactive_split_MB: 1098.3 +rank5 [2026-10-29 08:31:10,694] INFO : Step 25400/50000 rank5 | Train Loss: 1.0222 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1238 | Classification: 0.0928 | Classification_binary: 0.1653 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1221 | Regression_eta: 0.0015 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0628 | Regression_eta: 0.0535 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0866 | LR: 5.39e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.7077s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1954 | max_backward: 1.8415 | max_optimizer_step: 0.3277 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6896.2 | uss_MB: 6278.8 | child_rss_MB: 5052.2 | allocated_MB: 449.3 | reserved_MB: 28062.0 | inactive_split_MB: 945.6 +rank5 [2026-10-29 08:36:13,578] INFO : Step 25500/50000 rank5 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4596 | Train Loss Components: Auxiliary: 0.1817 | Cardinality: 0.1001 | Classification: 0.0953 | Classification_binary: 0.1334 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0016 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1603 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0550 | Regression_eta: 0.0438 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0673 | LR: 5.36e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7841s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7884 | max_loss: 0.1475 | max_backward: 2.0272 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.7862 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0483 | Memory: rss_MB: 6898.2 | uss_MB: 6283.2 | child_rss_MB: 5050.7 | allocated_MB: 450.7 | reserved_MB: 28062.0 | inactive_split_MB: 1166.9 +rank5 [2026-10-29 08:41:08,951] INFO : Step 25600/50000 rank5 | Train Loss: 1.0724 | Train Loss Unweighted: 0.5330 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1190 | Classification: 0.0985 | Classification_binary: 0.1843 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1231 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.1815 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0657 | Regression_eta: 0.0460 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0689 | LR: 5.33e-04 | DataLoad Time: 0.0154s | Model Forward Time: 6.1560s | Phase Times: max_pre_step_barrier: 0.0194 | max_batch_to_device: 0.0083 | max_forward: 1.0622 | max_loss: 0.3311 | max_backward: 2.5999 | max_optimizer_step: 0.3019 | max_tensorboard_logging: 0.8123 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6895.2 | uss_MB: 5585.2 | child_rss_MB: 5023.7 | allocated_MB: 450.4 | reserved_MB: 28064.0 | inactive_split_MB: 1163.5 +rank5 [2026-10-29 08:46:09,676] INFO : Step 25700/50000 rank5 | Train Loss: 1.0595 | Train Loss Unweighted: 0.5120 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.1394 | Classification: 0.0920 | Classification_binary: 0.1853 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0604 | Regression_eta: 0.0331 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0583 | LR: 5.30e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.8066s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.8320 | max_loss: 0.2520 | max_backward: 2.6962 | max_optimizer_step: 0.2699 | max_tensorboard_logging: 0.7057 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6885.0 | uss_MB: 6272.6 | child_rss_MB: 5051.6 | allocated_MB: 448.6 | reserved_MB: 29942.0 | inactive_split_MB: 1165.2 +rank5 [2026-10-29 08:51:40,159] INFO : Step 25800/50000 rank5 | Train Loss: 0.9592 | Train Loss Unweighted: 0.4733 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1038 | Classification: 0.0787 | Classification_binary: 0.1531 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1182 | Regression_eta: 0.0016 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1675 | Regression_cos_phi: 0.0624 | Regression_energy: 0.0602 | Regression_eta: 0.0362 | Regression_pt: 0.0662 | Regression_sin_phi: 0.0614 | LR: 5.27e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4827s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7549 | max_loss: 0.1480 | max_backward: 1.5296 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 0.8796 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1361 | Memory: rss_MB: 6889.1 | uss_MB: 6277.5 | child_rss_MB: 5023.6 | allocated_MB: 445.9 | reserved_MB: 30100.0 | inactive_split_MB: 961.8 +rank5 [2026-10-29 08:56:50,110] INFO : Step 25900/50000 rank5 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4613 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1142 | Classification: 0.0768 | Classification_binary: 0.1501 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1145 | Regression_eta: 0.0015 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1549 | Regression_cos_phi: 0.1226 | Regression_energy: 0.0627 | Regression_eta: 0.0429 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0978 | LR: 5.25e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8322s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0124 | max_forward: 0.7901 | max_loss: 0.1430 | max_backward: 1.9362 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8933 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 6880.5 | uss_MB: 6281.6 | child_rss_MB: 5008.5 | allocated_MB: 447.7 | reserved_MB: 30100.0 | inactive_split_MB: 1167.6 +rank5 [2026-10-29 09:01:47,509] INFO : Step 26000/50000 rank5 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5169 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1347 | Classification: 0.0896 | Classification_binary: 0.1938 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0015 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1901 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0498 | Regression_eta: 0.0464 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0735 | LR: 5.22e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.4844s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8125 | max_loss: 0.2612 | max_backward: 1.8771 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.8972 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6845.2 | uss_MB: 5700.0 | child_rss_MB: 4882.0 | allocated_MB: 450.3 | reserved_MB: 30108.0 | inactive_split_MB: 1161.3 +rank5 [2026-10-29 09:01:48,415] INFO : Running validation on rank5 +rank5 [2026-10-29 09:01:48,415] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 09:01:48,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 09:02:42,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:02:42,921] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 09:03:09,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 09:03:10,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 09:03:11,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 09:03:12,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 09:03:22,352] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 09:03:22,720] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:03:23,247] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 09:03:38,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 09:03:39,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 09:03:40,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 09:03:41,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 09:03:51,886] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 09:03:52,917] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:03:53,429] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 09:04:08,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 09:04:09,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 09:04:10,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 09:04:11,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 09:04:21,054] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 09:09:37,594] INFO : Step 26100/50000 rank5 | Train Loss: 0.9519 | Train Loss Unweighted: 0.4738 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1091 | Classification: 0.0871 | Classification_binary: 0.1547 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1147 | Regression_eta: 0.0013 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0540 | Classification_binary: 0.1409 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0555 | Regression_eta: 0.0330 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0688 | LR: 5.19e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.2102s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0321 | max_forward: 0.8143 | max_loss: 0.1540 | max_backward: 1.7669 | max_optimizer_step: 0.2382 | max_tensorboard_logging: 1.1185 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 8060.0 | uss_MB: 7433.5 | child_rss_MB: 4850.5 | allocated_MB: 445.1 | reserved_MB: 25850.0 | inactive_split_MB: 1376.9 +rank5 [2026-10-29 09:14:36,072] INFO : Step 26200/50000 rank5 | Train Loss: 0.9758 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.0970 | Classification: 0.0858 | Classification_binary: 0.1501 | HitGrouping: 0.1665 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1302 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0702 | Regression_eta: 0.0506 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0488 | LR: 5.16e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6754s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7728 | max_loss: 0.1384 | max_backward: 1.8195 | max_optimizer_step: 0.3036 | max_tensorboard_logging: 0.7010 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0602 | Memory: rss_MB: 8060.1 | uss_MB: 7488.7 | child_rss_MB: 4848.5 | allocated_MB: 447.4 | reserved_MB: 27830.0 | inactive_split_MB: 1117.7 +rank5 [2026-10-29 09:19:51,626] INFO : Step 26300/50000 rank5 | Train Loss: 1.0568 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2146 | Cardinality: 0.1416 | Classification: 0.0935 | Classification_binary: 0.1772 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0718 | Regression_energy: 0.0637 | Regression_eta: 0.0346 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0446 | LR: 5.13e-04 | DataLoad Time: 0.0187s | Model Forward Time: 6.1516s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.8205 | max_loss: 0.1808 | max_backward: 2.8872 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 1.1024 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6927.6 | uss_MB: 6354.4 | child_rss_MB: 4842.0 | allocated_MB: 450.8 | reserved_MB: 29830.0 | inactive_split_MB: 1114.5 +rank5 [2026-10-29 09:24:55,631] INFO : Step 26400/50000 rank5 | Train Loss: 1.0536 | Train Loss Unweighted: 0.5239 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1323 | Classification: 0.0937 | Classification_binary: 0.1845 | HitGrouping: 0.1559 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1209 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1979 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0712 | Regression_eta: 0.0381 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0494 | LR: 5.10e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.5419s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8639 | max_loss: 0.1569 | max_backward: 2.4710 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 1.0049 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6924.7 | uss_MB: 6353.2 | child_rss_MB: 4853.6 | allocated_MB: 447.6 | reserved_MB: 29832.0 | inactive_split_MB: 1186.4 +rank5 [2026-10-29 09:29:59,513] INFO : Step 26500/50000 rank5 | Train Loss: 0.9812 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1299 | Classification: 0.0807 | Classification_binary: 0.1400 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1217 | Regression_eta: 0.0017 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0535 | Regression_eta: 0.0423 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0472 | LR: 5.08e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.8054s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7436 | max_loss: 0.1579 | max_backward: 1.8417 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.9265 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6921.4 | uss_MB: 6354.1 | child_rss_MB: 4851.7 | allocated_MB: 444.4 | reserved_MB: 29832.0 | inactive_split_MB: 1229.5 +rank5 [2026-10-29 09:34:58,505] INFO : Step 26600/50000 rank5 | Train Loss: 1.0108 | Train Loss Unweighted: 0.5066 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.1182 | Classification: 0.0957 | Classification_binary: 0.1588 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1242 | Regression_eta: 0.0014 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0849 | Regression_eta: 0.0461 | Regression_pt: 0.0908 | Regression_sin_phi: 0.0505 | LR: 5.05e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6966s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7561 | max_loss: 0.1430 | max_backward: 1.8427 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6921.4 | uss_MB: 6356.0 | child_rss_MB: 4827.0 | allocated_MB: 447.9 | reserved_MB: 29832.0 | inactive_split_MB: 1037.2 +rank5 [2026-10-29 09:40:07,140] INFO : Step 26700/50000 rank5 | Train Loss: 1.0878 | Train Loss Unweighted: 0.5266 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1269 | Classification: 0.0956 | Classification_binary: 0.1961 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1144 | Regression_eta: 0.0018 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.2332 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0629 | Regression_eta: 0.0500 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0873 | LR: 5.02e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1889s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.9788 | max_loss: 0.1540 | max_backward: 1.9407 | max_optimizer_step: 0.2295 | max_tensorboard_logging: 1.0076 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6911.8 | uss_MB: 6344.9 | child_rss_MB: 4859.9 | allocated_MB: 450.1 | reserved_MB: 29832.0 | inactive_split_MB: 1190.4 +rank5 [2026-10-29 09:48:41,512] INFO : Step 26800/50000 rank5 | Train Loss: 1.0077 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1102 | Classification: 0.0787 | Classification_binary: 0.1553 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1359 | Regression_eta: 0.0015 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1757 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0668 | Regression_eta: 0.0376 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0527 | LR: 4.99e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.8134s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.7774 | max_loss: 0.2523 | max_backward: 1.8741 | max_optimizer_step: 0.2873 | max_tensorboard_logging: 0.8206 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6909.2 | uss_MB: 6351.0 | child_rss_MB: 4816.3 | allocated_MB: 444.7 | reserved_MB: 29832.0 | inactive_split_MB: 1121.3 +rank5 [2026-10-29 09:53:40,042] INFO : Step 26900/50000 rank5 | Train Loss: 0.9524 | Train Loss Unweighted: 0.4793 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1099 | Classification: 0.0833 | Classification_binary: 0.1305 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1313 | Regression_eta: 0.0016 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1198 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0613 | Regression_eta: 0.0327 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0537 | LR: 4.96e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.4179s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7292 | max_loss: 0.1373 | max_backward: 1.7442 | max_optimizer_step: 0.3152 | max_tensorboard_logging: 0.6764 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 6905.2 | uss_MB: 5520.6 | child_rss_MB: 4788.1 | allocated_MB: 446.9 | reserved_MB: 29832.0 | inactive_split_MB: 1016.7 +rank5 [2026-10-29 09:58:37,161] INFO : Step 27000/50000 rank5 | Train Loss: 1.0285 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1243 | Classification: 0.0862 | Classification_binary: 0.1888 | HitGrouping: 0.1539 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0016 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0596 | Regression_eta: 0.0354 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0748 | LR: 4.94e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.1614s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.9134 | max_loss: 0.1497 | max_backward: 2.1803 | max_optimizer_step: 0.2254 | max_tensorboard_logging: 0.9490 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6904.1 | uss_MB: 6354.5 | child_rss_MB: 4772.3 | allocated_MB: 449.7 | reserved_MB: 29832.0 | inactive_split_MB: 1024.3 +rank5 [2026-10-29 09:58:38,108] INFO : Running validation on rank5 +rank5 [2026-10-29 09:58:38,109] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 09:58:38,109] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 09:59:32,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 09:59:33,284] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 09:59:59,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 10:00:00,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 10:00:02,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 10:00:03,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 10:00:13,087] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 10:00:14,009] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:00:14,539] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 10:00:31,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 10:00:33,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 10:00:34,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 10:00:35,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 10:00:45,298] INFO : Time taken to make predictions on device 5 is: 0.28 min +rank5 [2026-10-29 10:00:46,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:00:46,864] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 10:01:02,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 10:01:03,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 10:01:03,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 10:01:04,778] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 10:01:14,789] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 10:06:37,478] INFO : Step 27100/50000 rank5 | Train Loss: 1.0614 | Train Loss Unweighted: 0.4989 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1449 | Classification: 0.0752 | Classification_binary: 0.1877 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2294 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0602 | Regression_eta: 0.0550 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0670 | LR: 4.91e-04 | DataLoad Time: 0.0208s | Model Forward Time: 7.1171s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0966 | max_forward: 2.1724 | max_loss: 0.6705 | max_backward: 3.1723 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 1.5888 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 8047.0 | uss_MB: 7496.7 | child_rss_MB: 4778.7 | allocated_MB: 447.0 | reserved_MB: 27292.0 | inactive_split_MB: 983.4 +rank5 [2026-10-29 10:11:19,567] INFO : Step 27200/50000 rank5 | Train Loss: 0.9045 | Train Loss Unweighted: 0.4389 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1028 | Classification: 0.0770 | Classification_binary: 0.1287 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1539 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0545 | Regression_eta: 0.0453 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0638 | LR: 4.88e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.7907s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7725 | max_loss: 0.5032 | max_backward: 1.6788 | max_optimizer_step: 0.2815 | max_tensorboard_logging: 0.9686 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8047.0 | uss_MB: 7493.7 | child_rss_MB: 4786.7 | allocated_MB: 445.7 | reserved_MB: 29236.0 | inactive_split_MB: 1232.8 +rank5 [2026-10-29 10:15:58,639] INFO : Step 27300/50000 rank5 | Train Loss: 1.0726 | Train Loss Unweighted: 0.5095 | Train Loss Components: Auxiliary: 0.2138 | Cardinality: 0.1337 | Classification: 0.1081 | Classification_binary: 0.1640 | HitGrouping: 0.1596 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1159 | Regression_eta: 0.0019 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0753 | Classification_binary: 0.1501 | Regression_cos_phi: 0.0796 | Regression_energy: 0.0525 | Regression_eta: 0.0374 | Regression_pt: 0.0566 | Regression_sin_phi: 0.1135 | LR: 4.85e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.1438s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8896 | max_loss: 0.1460 | max_backward: 2.0153 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6974.0 | uss_MB: 6424.6 | child_rss_MB: 4784.6 | allocated_MB: 450.9 | reserved_MB: 29330.0 | inactive_split_MB: 1103.1 +rank5 [2026-10-29 10:20:31,815] INFO : Step 27400/50000 rank5 | Train Loss: 1.0758 | Train Loss Unweighted: 0.5282 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1344 | Classification: 0.0930 | Classification_binary: 0.1899 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1207 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.2010 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0688 | Regression_eta: 0.0501 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0550 | LR: 4.82e-04 | DataLoad Time: 0.0195s | Model Forward Time: 5.0227s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8446 | max_loss: 0.1488 | max_backward: 2.0519 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.8725 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6977.0 | uss_MB: 5627.2 | child_rss_MB: 4777.8 | allocated_MB: 448.4 | reserved_MB: 31310.0 | inactive_split_MB: 895.7 +rank5 [2026-10-29 10:25:07,332] INFO : Step 27500/50000 rank5 | Train Loss: 0.9697 | Train Loss Unweighted: 0.4574 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1111 | Classification: 0.0831 | Classification_binary: 0.1473 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1114 | Regression_eta: 0.0018 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0362 | Regression_energy: 0.0532 | Regression_eta: 0.0331 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0393 | LR: 4.80e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4615s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.7904 | max_loss: 0.1563 | max_backward: 1.6291 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.7457 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6966.1 | uss_MB: 4710.9 | child_rss_MB: 4769.8 | allocated_MB: 447.3 | reserved_MB: 31310.0 | inactive_split_MB: 986.8 +rank5 [2026-10-29 10:30:03,960] INFO : Step 27600/50000 rank5 | Train Loss: 0.9616 | Train Loss Unweighted: 0.4618 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1092 | Classification: 0.0865 | Classification_binary: 0.1284 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0019 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0712 | Regression_eta: 0.0342 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0648 | LR: 4.77e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5702s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0380 | max_forward: 0.7674 | max_loss: 0.1463 | max_backward: 1.9164 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.5145 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6974.1 | uss_MB: 6422.9 | child_rss_MB: 4778.4 | allocated_MB: 446.7 | reserved_MB: 31310.0 | inactive_split_MB: 1182.0 +rank5 [2026-10-29 10:34:46,993] INFO : Step 27700/50000 rank5 | Train Loss: 1.0693 | Train Loss Unweighted: 0.5190 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1414 | Classification: 0.0939 | Classification_binary: 0.1783 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0015 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1856 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0678 | Regression_eta: 0.0619 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0500 | LR: 4.74e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.6752s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.8151 | max_loss: 0.1750 | max_backward: 2.0398 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.5213 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0659 | Memory: rss_MB: 6974.1 | uss_MB: 6424.8 | child_rss_MB: 4756.0 | allocated_MB: 448.4 | reserved_MB: 33312.0 | inactive_split_MB: 986.3 +rank5 [2026-10-29 10:39:29,302] INFO : Step 27800/50000 rank5 | Train Loss: 1.0875 | Train Loss Unweighted: 0.5294 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1226 | Classification: 0.0924 | Classification_binary: 0.1842 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1237 | Regression_eta: 0.0019 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1831 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0669 | Regression_eta: 0.0558 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0755 | LR: 4.71e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.8099s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0152 | max_forward: 0.8463 | max_loss: 0.1744 | max_backward: 1.7379 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.9397 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 6971.2 | uss_MB: 6421.2 | child_rss_MB: 4773.7 | allocated_MB: 452.2 | reserved_MB: 33312.0 | inactive_split_MB: 1320.1 +rank5 [2026-10-29 10:44:11,939] INFO : Step 27900/50000 rank5 | Train Loss: 0.9483 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0988 | Classification: 0.0859 | Classification_binary: 0.1439 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1244 | Regression_eta: 0.0014 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1657 | Regression_cos_phi: 0.0419 | Regression_energy: 0.0596 | Regression_eta: 0.0336 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0617 | LR: 4.68e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.4741s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.7626 | max_loss: 0.1370 | max_backward: 1.8424 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6972.2 | uss_MB: 6423.3 | child_rss_MB: 4773.1 | allocated_MB: 447.1 | reserved_MB: 33312.0 | inactive_split_MB: 1112.6 +rank5 [2026-10-29 10:48:55,972] INFO : Step 28000/50000 rank5 | Train Loss: 0.9840 | Train Loss Unweighted: 0.4736 | Train Loss Components: Auxiliary: 0.1949 | Cardinality: 0.1114 | Classification: 0.0927 | Classification_binary: 0.1564 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1099 | Regression_eta: 0.0015 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0609 | Regression_eta: 0.0340 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0975 | LR: 4.66e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9423s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8192 | max_loss: 0.2552 | max_backward: 2.1299 | max_optimizer_step: 0.2920 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 6969.4 | uss_MB: 6424.2 | child_rss_MB: 4776.8 | allocated_MB: 448.2 | reserved_MB: 33312.0 | inactive_split_MB: 1178.0 +rank5 [2026-10-29 10:48:56,836] INFO : Running validation on rank5 +rank5 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 10:49:51,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:49:52,005] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 10:50:18,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 10:50:19,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 10:50:20,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 10:50:21,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 10:50:31,583] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 10:50:32,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:50:32,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 10:50:49,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 10:50:50,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 10:50:51,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 10:50:52,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 10:51:02,369] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 10:51:02,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 10:51:03,444] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 10:51:18,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 10:51:19,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 10:51:20,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 10:51:21,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 10:51:31,425] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 10:56:32,745] INFO : Step 28100/50000 rank5 | Train Loss: 1.0915 | Train Loss Unweighted: 0.5085 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1649 | Classification: 0.0833 | Classification_binary: 0.1834 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1187 | Regression_eta: 0.0017 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1813 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0744 | Regression_eta: 0.0446 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0722 | LR: 4.63e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8678s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8571 | max_loss: 0.1603 | max_backward: 1.9112 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 0.7670 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0838 | Memory: rss_MB: 8288.3 | uss_MB: 7733.7 | child_rss_MB: 4773.7 | allocated_MB: 448.9 | reserved_MB: 25946.0 | inactive_split_MB: 1313.1 +rank5 [2026-10-29 11:01:18,173] INFO : Step 28200/50000 rank5 | Train Loss: 0.9241 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1144 | Classification: 0.0770 | Classification_binary: 0.1441 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1163 | Regression_eta: 0.0014 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1333 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0633 | Regression_eta: 0.0535 | Regression_pt: 0.0691 | Regression_sin_phi: 0.0517 | LR: 4.60e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8982s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0751 | max_forward: 0.8691 | max_loss: 0.5236 | max_backward: 1.8483 | max_optimizer_step: 0.2480 | max_tensorboard_logging: 0.8546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 8289.0 | uss_MB: 7733.3 | child_rss_MB: 4770.2 | allocated_MB: 445.0 | reserved_MB: 25948.0 | inactive_split_MB: 976.4 +rank5 [2026-10-29 11:07:40,703] INFO : Step 28300/50000 rank5 | Train Loss: 0.9513 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1028 | Classification: 0.0882 | Classification_binary: 0.1368 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1147 | Regression_eta: 0.0016 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0382 | Classification_binary: 0.1191 | Regression_cos_phi: 0.0414 | Regression_energy: 0.0550 | Regression_eta: 0.0339 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0648 | LR: 4.57e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.9466s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.7916 | max_loss: 0.1449 | max_backward: 2.0850 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.8583 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7055.4 | uss_MB: 6496.0 | child_rss_MB: 4774.9 | allocated_MB: 447.2 | reserved_MB: 25952.0 | inactive_split_MB: 1165.3 +rank5 [2026-10-29 11:12:27,615] INFO : Step 28400/50000 rank5 | Train Loss: 1.0604 | Train Loss Unweighted: 0.4982 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1382 | Classification: 0.0847 | Classification_binary: 0.1862 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1111 | Regression_eta: 0.0018 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1657 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0574 | Regression_eta: 0.0474 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0812 | LR: 4.55e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.0303s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9139 | max_loss: 0.1474 | max_backward: 2.1323 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.6846 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0517 | Memory: rss_MB: 7055.5 | uss_MB: 6496.2 | child_rss_MB: 4770.0 | allocated_MB: 448.2 | reserved_MB: 26968.0 | inactive_split_MB: 835.9 +rank5 [2026-10-29 11:17:11,504] INFO : Step 28500/50000 rank5 | Train Loss: 1.0282 | Train Loss Unweighted: 0.5005 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1237 | Classification: 0.0911 | Classification_binary: 0.1756 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0013 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0596 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0758 | Regression_energy: 0.0688 | Regression_eta: 0.0385 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0536 | LR: 4.52e-04 | DataLoad Time: 0.0222s | Model Forward Time: 9.4584s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0886 | max_forward: 0.9758 | max_loss: 0.5123 | max_backward: 2.0811 | max_optimizer_step: 0.2797 | max_tensorboard_logging: 1.5644 | max_tensorboard_flush: 0.0047 | max_post_step_barrier: 0.0889 | Memory: rss_MB: 7042.2 | uss_MB: 4349.9 | child_rss_MB: 4774.7 | allocated_MB: 452.0 | reserved_MB: 27968.0 | inactive_split_MB: 923.2 +rank5 [2026-10-29 11:21:56,249] INFO : Step 28600/50000 rank5 | Train Loss: 0.9753 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1202 | Classification: 0.0804 | Classification_binary: 0.1446 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1132 | Regression_eta: 0.0017 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0316 | Classification_binary: 0.1466 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0594 | Regression_eta: 0.0495 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0492 | LR: 4.49e-04 | DataLoad Time: 0.0233s | Model Forward Time: 4.9426s | Phase Times: max_pre_step_barrier: 0.1127 | max_batch_to_device: 0.0146 | max_forward: 0.7731 | max_loss: 0.2404 | max_backward: 1.6352 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 1.0006 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7048.8 | uss_MB: 5469.7 | child_rss_MB: 4776.7 | allocated_MB: 446.8 | reserved_MB: 30056.0 | inactive_split_MB: 896.4 +rank5 [2026-10-29 11:26:34,429] INFO : Step 28700/50000 rank5 | Train Loss: 1.0558 | Train Loss Unweighted: 0.5173 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1082 | Classification: 0.1065 | Classification_binary: 0.1612 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1217 | Regression_eta: 0.0024 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0851 | Classification_binary: 0.1904 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0768 | Regression_eta: 0.0413 | Regression_pt: 0.0860 | Regression_sin_phi: 0.0821 | LR: 4.46e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.9575s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0686 | max_forward: 0.8423 | max_loss: 0.1475 | max_backward: 2.0170 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.8830 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 7053.8 | uss_MB: 6496.4 | child_rss_MB: 4774.2 | allocated_MB: 447.9 | reserved_MB: 30056.0 | inactive_split_MB: 925.2 +rank5 [2026-10-29 11:31:18,244] INFO : Step 28800/50000 rank5 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5167 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1474 | Classification: 0.0969 | Classification_binary: 0.1939 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0560 | Classification_binary: 0.2103 | Regression_cos_phi: 0.0563 | Regression_energy: 0.0561 | Regression_eta: 0.0466 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0871 | LR: 4.44e-04 | DataLoad Time: 0.0193s | Model Forward Time: 5.8765s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0075 | max_forward: 0.8427 | max_loss: 0.2661 | max_backward: 2.8464 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7050.1 | uss_MB: 4315.7 | child_rss_MB: 4755.7 | allocated_MB: 448.2 | reserved_MB: 30058.0 | inactive_split_MB: 976.8 +rank5 [2026-10-29 11:35:53,801] INFO : Step 28900/50000 rank5 | Train Loss: 0.9389 | Train Loss Unweighted: 0.4547 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.1078 | Classification: 0.0733 | Classification_binary: 0.1521 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0638 | Regression_eta: 0.0467 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0745 | LR: 4.41e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.5662s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0062 | max_forward: 0.7499 | max_loss: 0.2426 | max_backward: 1.8556 | max_optimizer_step: 0.3135 | max_tensorboard_logging: 0.5831 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7048.4 | uss_MB: 6494.0 | child_rss_MB: 4774.7 | allocated_MB: 444.3 | reserved_MB: 30058.0 | inactive_split_MB: 1133.2 +rank5 [2026-10-29 11:40:53,318] INFO : Step 29000/50000 rank5 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4535 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0939 | Classification: 0.0855 | Classification_binary: 0.1308 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0015 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0686 | Classification_binary: 0.1017 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0551 | Regression_eta: 0.0474 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0509 | LR: 4.38e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.8494s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0062 | max_forward: 0.9270 | max_loss: 0.2073 | max_backward: 2.1896 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6044 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7047.7 | uss_MB: 6496.6 | child_rss_MB: 4759.3 | allocated_MB: 450.2 | reserved_MB: 30060.0 | inactive_split_MB: 1402.9 +rank5 [2026-10-29 11:40:54,336] INFO : Running validation on rank5 +rank5 [2026-10-29 11:40:54,337] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 11:40:54,337] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 11:41:46,665] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:41:47,177] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 11:42:15,909] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 11:42:18,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 11:42:19,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 11:42:20,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 11:42:30,366] INFO : Time taken to make predictions on device 5 is: 0.29 min +rank5 [2026-10-29 11:42:31,033] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:42:31,573] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 11:42:47,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 11:42:48,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 11:42:49,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 11:42:50,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 11:43:00,145] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 11:43:00,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 11:43:01,398] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 11:43:16,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 11:43:17,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 11:43:18,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 11:43:19,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 11:43:29,171] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 11:48:35,108] INFO : Step 29100/50000 rank5 | Train Loss: 1.0559 | Train Loss Unweighted: 0.5040 | Train Loss Components: Auxiliary: 0.2142 | Cardinality: 0.1307 | Classification: 0.0939 | Classification_binary: 0.1851 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1098 | Regression_eta: 0.0015 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1928 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0666 | Regression_eta: 0.0389 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0702 | LR: 4.35e-04 | DataLoad Time: 0.0161s | Model Forward Time: 7.6729s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.6779 | max_loss: 0.3815 | max_backward: 4.2661 | max_optimizer_step: 0.3470 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8253.0 | uss_MB: 6395.3 | child_rss_MB: 4770.6 | allocated_MB: 448.8 | reserved_MB: 24188.0 | inactive_split_MB: 1162.1 +rank5 [2026-10-29 11:53:34,050] INFO : Step 29200/50000 rank5 | Train Loss: 0.9909 | Train Loss Unweighted: 0.4801 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1204 | Classification: 0.0923 | Classification_binary: 0.1746 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1054 | Regression_eta: 0.0014 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1966 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0624 | Regression_eta: 0.0477 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0674 | LR: 4.33e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7779s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8354 | max_loss: 0.1558 | max_backward: 2.1201 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0677 | Memory: rss_MB: 8251.2 | uss_MB: 7692.4 | child_rss_MB: 4763.1 | allocated_MB: 447.9 | reserved_MB: 24190.0 | inactive_split_MB: 1166.5 +rank5 [2026-10-29 11:58:27,981] INFO : Step 29300/50000 rank5 | Train Loss: 0.9337 | Train Loss Unweighted: 0.4462 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1121 | Classification: 0.0773 | Classification_binary: 0.1359 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1146 | Regression_eta: 0.0015 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0874 | Regression_energy: 0.0646 | Regression_eta: 0.0547 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0669 | LR: 4.30e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.9130s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0277 | max_forward: 0.8212 | max_loss: 0.1457 | max_backward: 1.8414 | max_optimizer_step: 0.2550 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7135.1 | uss_MB: 5997.0 | child_rss_MB: 4761.5 | allocated_MB: 444.9 | reserved_MB: 24190.0 | inactive_split_MB: 1038.6 +rank5 [2026-10-29 12:03:07,470] INFO : Step 29400/50000 rank5 | Train Loss: 1.0186 | Train Loss Unweighted: 0.4675 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1372 | Classification: 0.0948 | Classification_binary: 0.1561 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0020 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.1987 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0592 | Regression_eta: 0.0473 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0684 | LR: 4.27e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8024s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8495 | max_loss: 0.1434 | max_backward: 1.9359 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7145.4 | uss_MB: 6588.5 | child_rss_MB: 4762.3 | allocated_MB: 447.9 | reserved_MB: 26208.0 | inactive_split_MB: 957.3 +rank5 [2026-10-29 12:08:16,205] INFO : Step 29500/50000 rank5 | Train Loss: 1.0909 | Train Loss Unweighted: 0.5362 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.1071 | Classification_binary: 0.1896 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1176 | Regression_eta: 0.0019 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1846 | Regression_cos_phi: 0.0992 | Regression_energy: 0.0565 | Regression_eta: 0.0687 | Regression_pt: 0.0596 | Regression_sin_phi: 0.1063 | LR: 4.24e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9681s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0876 | max_forward: 0.8925 | max_loss: 0.1527 | max_backward: 1.9621 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.8162 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7137.1 | uss_MB: 6580.0 | child_rss_MB: 4760.9 | allocated_MB: 448.9 | reserved_MB: 26214.0 | inactive_split_MB: 1166.5 +rank5 [2026-10-29 12:13:12,652] INFO : Step 29600/50000 rank5 | Train Loss: 0.9608 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.1935 | Cardinality: 0.1093 | Classification: 0.0850 | Classification_binary: 0.1542 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1599 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0506 | Regression_eta: 0.0464 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0516 | LR: 4.22e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.8682s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7848 | max_loss: 0.1495 | max_backward: 1.8042 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.0178 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7115.2 | uss_MB: 6585.9 | child_rss_MB: 4743.4 | allocated_MB: 445.5 | reserved_MB: 26214.0 | inactive_split_MB: 1171.9 +rank5 [2026-10-29 12:18:07,441] INFO : Step 29700/50000 rank5 | Train Loss: 0.9511 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1260 | Classification: 0.0829 | Classification_binary: 0.1357 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1183 | Regression_eta: 0.0014 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1209 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0568 | Regression_eta: 0.0388 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0387 | LR: 4.19e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.2347s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7531 | max_loss: 0.2308 | max_backward: 2.2313 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.9708 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7117.3 | uss_MB: 6587.7 | child_rss_MB: 4741.9 | allocated_MB: 446.6 | reserved_MB: 26214.0 | inactive_split_MB: 1160.6 +rank5 [2026-10-29 12:23:08,925] INFO : Step 29800/50000 rank5 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4915 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1104 | Classification: 0.0864 | Classification_binary: 0.1759 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1742 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0565 | Regression_eta: 0.0433 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0659 | LR: 4.16e-04 | DataLoad Time: 0.0156s | Model Forward Time: 6.4242s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8210 | max_loss: 0.1528 | max_backward: 2.4753 | max_optimizer_step: 0.3576 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0038 | max_post_step_barrier: 0.1798 | Memory: rss_MB: 7115.5 | uss_MB: 5536.1 | child_rss_MB: 4741.9 | allocated_MB: 450.3 | reserved_MB: 26214.0 | inactive_split_MB: 938.8 +rank5 [2026-10-29 12:28:10,142] INFO : Step 29900/50000 rank5 | Train Loss: 0.9604 | Train Loss Unweighted: 0.4559 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1243 | Classification: 0.0825 | Classification_binary: 0.1616 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1040 | Regression_eta: 0.0015 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1511 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0524 | Regression_eta: 0.0442 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0551 | LR: 4.14e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.9484s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1142 | max_forward: 0.9271 | max_loss: 0.1531 | max_backward: 1.8698 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.8620 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7091.0 | uss_MB: 6437.5 | child_rss_MB: 4745.0 | allocated_MB: 450.0 | reserved_MB: 26214.0 | inactive_split_MB: 1166.0 +rank5 [2026-10-29 12:32:58,888] INFO : Step 30000/50000 rank5 | Train Loss: 0.9450 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.1132 | Classification: 0.0847 | Classification_binary: 0.1394 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0589 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0721 | Regression_eta: 0.0338 | Regression_pt: 0.0796 | Regression_sin_phi: 0.0576 | LR: 4.11e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1882s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 1.0327 | max_loss: 0.1419 | max_backward: 1.6503 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.4878 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7039.0 | uss_MB: 6586.5 | child_rss_MB: 4745.4 | allocated_MB: 443.8 | reserved_MB: 26214.0 | inactive_split_MB: 1152.2 +rank5 [2026-10-29 12:47:12,790] INFO : Running validation on rank5 +rank5 [2026-10-29 12:47:12,791] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 12:47:12,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 14:48:59,888] INFO : Initializing model with seed=12345; process seed=12350 +rank5 [2026-10-29 14:48:59,903] INFO : configured dtype=torch.bfloat16 for autocast +rank5 [2026-10-29 14:49:20,930] INFO : Instantiating model +rank5 [2026-10-29 14:49:20,930] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank5 [2026-10-29 14:49:20,930] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank5 [2026-10-29 14:49:20,948] INFO : Input encoding initialization took 0.02s +rank5 [2026-10-29 14:49:20,948] INFO : nn0 parameters: 171776 +rank5 [2026-10-29 14:49:20,948] INFO : Initializing shared backbone layers, num_convs=6 +rank5 [2026-10-29 14:49:20,948] INFO : Initializing backbone layer 0 +rank5 [2026-10-29 14:49:20,983] INFO : Initializing backbone layer 1 +rank5 [2026-10-29 14:49:20,992] INFO : Initializing backbone layer 2 +rank5 [2026-10-29 14:49:21,000] INFO : Initializing backbone layer 3 +rank5 [2026-10-29 14:49:21,009] INFO : Initializing backbone layer 4 +rank5 [2026-10-29 14:49:21,017] INFO : Initializing backbone layer 5 +rank5 [2026-10-29 14:49:21,026] INFO : Backbone initialization took 0.08s +rank5 [2026-10-29 14:49:21,026] INFO : backbone parameters: 7901376 +rank5 [2026-10-29 14:49:21,026] INFO : Initializing output DNNs +rank5 [2026-10-29 14:49:21,063] INFO : Output DNNs initialization took 0.04s +rank5 [2026-10-29 14:49:21,063] INFO : backbone_mode=BackboneMode.SHARED +rank5 [2026-10-29 14:49:21,063] INFO : jagged_attention enabled=False +rank5 [2026-10-29 14:49:21,063] INFO : flash_attn_varlen enabled=False +rank5 [2026-10-29 14:49:21,063] INFO : task_queries enabled=False +rank5 [2026-10-29 14:49:21,063] INFO : classification_norm parameters: 0 +rank5 [2026-10-29 14:49:21,063] INFO : regression_norm parameters: 0 +rank5 [2026-10-29 14:49:21,063] INFO : classification_query parameters: 0 +rank5 [2026-10-29 14:49:21,063] INFO : regression_query parameters: 0 +rank5 [2026-10-29 14:49:21,063] INFO : classification_readout parameters: 0 +rank5 [2026-10-29 14:49:21,063] INFO : regression_readout parameters: 0 +rank5 [2026-10-29 14:49:21,063] INFO : output_mode=set +rank5 [2026-10-29 14:49:21,064] INFO : set_decoder parameters: 4713812 +rank5 [2026-10-29 14:49:21,064] INFO : nn_binary_particle parameters: 0 +rank5 [2026-10-29 14:49:21,064] INFO : nn_pid parameters: 0 +rank5 [2026-10-29 14:49:21,064] INFO : nn_pt parameters: 0 +rank5 [2026-10-29 14:49:21,064] INFO : nn_eta parameters: 0 +rank5 [2026-10-29 14:49:21,064] INFO : nn_sin_phi parameters: 0 +rank5 [2026-10-29 14:49:21,064] INFO : nn_cos_phi parameters: 0 +rank5 [2026-10-29 14:49:21,064] INFO : nn_energy parameters: 0 +rank5 [2026-10-29 14:49:21,065] INFO : Total MLPF parameters: 12786964 +rank5 [2026-10-29 14:49:21,065] INFO : MLPF __init__ done +rank5 [2026-10-29 14:49:21,065] INFO : Instantiated model +rank5 [2026-10-29 14:49:21,065] INFO : Moving model to device rank=5 +rank5 [2026-10-29 14:49:21,086] INFO : Moved model to device rank=5 +rank5 [2026-10-29 14:49:21,087] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank5 [2026-10-29 14:49:21,561] INFO : Restoring training from step 30001 +rank5 [2026-10-29 14:49:21,568] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth +rank5 [2026-10-29 14:49:21,569] INFO : Configured model for SyncBatchNorm rank=5 +rank5 [2026-10-29 14:49:22,135] INFO : Configured model for DistributedDataParallel rank=5 +rank5 [2026-10-29 14:49:22,287] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank5 [2026-10-29 14:49:22,287] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank5 [2026-10-29 14:49:22,288] INFO : Trainable parameters: 12786964 +rank5 [2026-10-29 14:49:22,288] INFO : Non-trainable parameters: 23268 +rank5 [2026-10-29 14:49:22,288] INFO : Total parameters: 12810232 +rank5 [2026-10-29 14:49:22,291] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank5 [2026-10-29 14:49:22,292] INFO : Getting dataloaders +rank5 [2026-10-29 14:49:22,292] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:49:28,714] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:49:29,733] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:49:30,958] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank5 [2026-10-29 14:49:30,958] INFO : Creating WeightedMixIterator with 1 data loaders. +rank5 [2026-10-29 14:49:30,958] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:49:31,227] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:49:31,479] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 14:49:31,733] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank5 [2026-10-29 14:49:31,733] INFO : Creating InterleavedIterator with 1 data loaders. +rank5 [2026-10-29 14:49:45,373] INFO : Got dataloaders +rank5 [2026-10-29 14:49:45,373] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [8888] of epochs [1] +rank5 [2026-10-29 14:49:45,373] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank5 [2026-10-29 14:49:45,373] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank5 [2026-10-29 14:49:45,373] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank5 [2026-10-29 14:49:47,131] INFO : loader split=train rank=5 len=21112 +rank5 [2026-10-29 14:49:47,131] INFO : loader split=valid rank=5 len=12 +rank5 [2026-10-29 14:49:47,131] INFO : Starting training from step 30001 to 50000 on rank 5 of 8 +rank5 [2026-10-29 14:49:47,131] INFO : Creating train_iterator +rank5 [2026-10-29 14:49:47,131] INFO : ResumableSampler physical:train loading state: start_index=142208 +rank5 [2026-10-29 14:50:01,524] INFO : Created train_iterator +rank5 [2026-10-29 14:54:42,003] INFO : Step 30100/50000 rank5 | Train Loss: 0.9854 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.2024 | Cardinality: 0.1295 | Classification: 0.0965 | Classification_binary: 0.1561 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0515 | Regression_eta: 0.0275 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0355 | LR: 4.08e-04 | DataLoad Time: 0.0182s | Model Forward Time: 3.6351s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7011 | max_loss: 0.1473 | max_backward: 1.5273 | max_optimizer_step: 0.1574 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0061 | Memory: rss_MB: 4775.9 | uss_MB: 3472.5 | child_rss_MB: 6376.2 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 850.1 +rank5 [2026-10-29 14:59:12,793] INFO : Step 30200/50000 rank5 | Train Loss: 1.0165 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2044 | Cardinality: 0.1520 | Classification: 0.0856 | Classification_binary: 0.1718 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1910 | Regression_cos_phi: 0.0820 | Regression_energy: 0.0625 | Regression_eta: 0.0399 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0583 | LR: 4.06e-04 | DataLoad Time: 0.0156s | Model Forward Time: 3.9890s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.7915 | max_loss: 0.2568 | max_backward: 1.8926 | max_optimizer_step: 0.1551 | max_tensorboard_logging: 0.4604 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4776.3 | uss_MB: 3476.7 | child_rss_MB: 6367.6 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 842.0 +rank5 [2026-10-29 15:03:42,965] INFO : Step 30300/50000 rank5 | Train Loss: 0.9183 | Train Loss Unweighted: 0.4449 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1232 | Classification: 0.0820 | Classification_binary: 0.1459 | HitGrouping: 0.1516 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0012 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1547 | Regression_cos_phi: 0.0808 | Regression_energy: 0.0704 | Regression_eta: 0.0296 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0678 | LR: 4.03e-04 | DataLoad Time: 0.0159s | Model Forward Time: 3.8583s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7525 | max_loss: 0.2476 | max_backward: 1.8262 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.4574 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0039 | Memory: rss_MB: 4773.5 | uss_MB: 3475.1 | child_rss_MB: 6385.6 | allocated_MB: 495.8 | reserved_MB: 28760.0 | inactive_split_MB: 848.2 +rank5 [2026-10-29 15:08:13,476] INFO : Step 30400/50000 rank5 | Train Loss: 0.9035 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1137 | Classification: 0.0848 | Classification_binary: 0.1358 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1074 | Regression_cos_phi: 0.0393 | Regression_energy: 0.0627 | Regression_eta: 0.0249 | Regression_pt: 0.0664 | Regression_sin_phi: 0.0432 | LR: 4.00e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.9731s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7769 | max_loss: 0.1398 | max_backward: 1.9002 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.4925 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4770.8 | uss_MB: 3475.5 | child_rss_MB: 6366.2 | allocated_MB: 498.8 | reserved_MB: 28760.0 | inactive_split_MB: 878.9 +rank5 [2026-10-29 15:12:45,607] INFO : Step 30500/50000 rank5 | Train Loss: 1.0399 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1239 | Classification: 0.0901 | Classification_binary: 0.1841 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0018 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0690 | Classification_binary: 0.1826 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0604 | Regression_eta: 0.0513 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0588 | LR: 3.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3860s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9688 | max_loss: 0.1627 | max_backward: 2.3054 | max_optimizer_step: 0.1555 | max_tensorboard_logging: 0.4908 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4768.4 | uss_MB: 3475.6 | child_rss_MB: 6405.5 | allocated_MB: 501.7 | reserved_MB: 28760.0 | inactive_split_MB: 944.3 +rank5 [2026-10-29 15:17:13,866] INFO : Step 30600/50000 rank5 | Train Loss: 0.9987 | Train Loss Unweighted: 0.4959 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1112 | Classification: 0.0932 | Classification_binary: 0.1720 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1137 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1758 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0538 | Regression_eta: 0.0477 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0699 | LR: 3.95e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2252s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7736 | max_loss: 0.1566 | max_backward: 1.7922 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.6994 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4770.8 | uss_MB: 3476.1 | child_rss_MB: 6399.5 | allocated_MB: 500.1 | reserved_MB: 28760.0 | inactive_split_MB: 845.9 +rank5 [2026-10-29 15:21:45,752] INFO : Step 30700/50000 rank5 | Train Loss: 0.9490 | Train Loss Unweighted: 0.4575 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1237 | Classification: 0.0844 | Classification_binary: 0.1399 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0014 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1321 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0666 | Regression_eta: 0.0338 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0667 | LR: 3.92e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1848s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7215 | max_loss: 0.5560 | max_backward: 2.0906 | max_optimizer_step: 0.1553 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4768.7 | uss_MB: 3474.9 | child_rss_MB: 6379.2 | allocated_MB: 493.8 | reserved_MB: 28760.0 | inactive_split_MB: 827.5 +rank5 [2026-10-29 15:26:16,185] INFO : Step 30800/50000 rank5 | Train Loss: 1.0320 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1331 | Classification: 0.0916 | Classification_binary: 0.1632 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1173 | Regression_eta: 0.0015 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0404 | Regression_energy: 0.0603 | Regression_eta: 0.0457 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0573 | LR: 3.90e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6902s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1016 | max_forward: 0.8041 | max_loss: 0.1462 | max_backward: 1.9351 | max_optimizer_step: 0.1552 | max_tensorboard_logging: 1.1007 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4769.5 | uss_MB: 3475.6 | child_rss_MB: 6407.8 | allocated_MB: 497.9 | reserved_MB: 28760.0 | inactive_split_MB: 842.0 +rank5 [2026-10-29 15:30:49,188] INFO : Step 30900/50000 rank5 | Train Loss: 1.0226 | Train Loss Unweighted: 0.4952 | Train Loss Components: Auxiliary: 0.2139 | Cardinality: 0.1339 | Classification: 0.0875 | Classification_binary: 0.1859 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1092 | Regression_eta: 0.0013 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1831 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0538 | Regression_eta: 0.0362 | Regression_pt: 0.0592 | Regression_sin_phi: 0.0576 | LR: 3.87e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4009s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7610 | max_loss: 0.2067 | max_backward: 1.7028 | max_optimizer_step: 0.1599 | max_tensorboard_logging: 1.0327 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4771.1 | uss_MB: 3476.3 | child_rss_MB: 6385.8 | allocated_MB: 497.2 | reserved_MB: 28760.0 | inactive_split_MB: 838.0 +rank5 [2026-10-29 15:35:26,546] INFO : Step 31000/50000 rank5 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4630 | Train Loss Components: Auxiliary: 0.2006 | Cardinality: 0.1217 | Classification: 0.0821 | Classification_binary: 0.1592 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1085 | Regression_eta: 0.0018 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0771 | Regression_energy: 0.0519 | Regression_eta: 0.0404 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0740 | LR: 3.84e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4983s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7780 | max_loss: 0.2475 | max_backward: 1.8803 | max_optimizer_step: 0.1588 | max_tensorboard_logging: 0.9947 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4772.1 | uss_MB: 3477.0 | child_rss_MB: 6402.3 | allocated_MB: 498.2 | reserved_MB: 28760.0 | inactive_split_MB: 820.4 +rank5 [2026-10-29 15:35:27,414] INFO : Running validation on rank5 +rank5 [2026-10-29 15:35:27,415] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 15:35:27,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 15:36:19,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:36:20,266] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 15:36:45,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 15:36:46,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 15:36:48,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 15:36:48,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 15:36:58,998] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 15:36:59,566] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:37:00,105] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 15:37:15,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 15:37:16,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 15:37:17,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 15:37:18,490] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 15:37:28,500] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 15:37:29,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 15:37:29,837] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 15:37:44,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 15:37:45,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 15:37:46,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 15:37:47,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 15:37:57,290] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 15:42:50,283] INFO : Step 31100/50000 rank5 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4264 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1054 | Classification: 0.0853 | Classification_binary: 0.1288 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1107 | Regression_cos_phi: 0.0367 | Regression_energy: 0.0536 | Regression_eta: 0.0230 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0390 | LR: 3.82e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5228s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7353 | max_loss: 0.1410 | max_backward: 1.7959 | max_optimizer_step: 0.2754 | max_tensorboard_logging: 0.9599 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 6463.6 | uss_MB: 5188.4 | child_rss_MB: 6394.5 | allocated_MB: 495.8 | reserved_MB: 27508.0 | inactive_split_MB: 851.2 +rank5 [2026-10-29 15:47:34,509] INFO : Step 31200/50000 rank5 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4709 | Train Loss Components: Auxiliary: 0.2045 | Cardinality: 0.1286 | Classification: 0.0862 | Classification_binary: 0.1646 | HitGrouping: 0.1524 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1077 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0812 | Regression_energy: 0.0604 | Regression_eta: 0.0452 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0546 | LR: 3.79e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3220s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8441 | max_loss: 0.1469 | max_backward: 2.0813 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 6462.4 | uss_MB: 5189.4 | child_rss_MB: 6422.3 | allocated_MB: 496.9 | reserved_MB: 27508.0 | inactive_split_MB: 935.6 +rank5 [2026-10-29 15:52:19,963] INFO : Step 31300/50000 rank5 | Train Loss: 1.0631 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1394 | Classification: 0.0917 | Classification_binary: 0.1731 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0746 | Regression_energy: 0.0561 | Regression_eta: 0.0399 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0721 | LR: 3.76e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2512s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7950 | max_loss: 0.1940 | max_backward: 1.8810 | max_optimizer_step: 0.2519 | max_tensorboard_logging: 0.5719 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5602.7 | uss_MB: 4330.1 | child_rss_MB: 6421.1 | allocated_MB: 498.6 | reserved_MB: 29580.0 | inactive_split_MB: 773.5 +rank5 [2026-10-29 15:56:52,523] INFO : Step 31400/50000 rank5 | Train Loss: 0.9019 | Train Loss Unweighted: 0.4258 | Train Loss Components: Auxiliary: 0.1823 | Cardinality: 0.1101 | Classification: 0.0746 | Classification_binary: 0.1337 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1070 | Regression_eta: 0.0014 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1344 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0456 | Regression_eta: 0.0423 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0573 | LR: 3.74e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2770s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8478 | max_loss: 0.2460 | max_backward: 2.0403 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.4770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5595.4 | uss_MB: 4331.0 | child_rss_MB: 6422.8 | allocated_MB: 495.2 | reserved_MB: 29580.0 | inactive_split_MB: 986.6 +rank5 [2026-10-29 16:01:34,248] INFO : Step 31500/50000 rank5 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4434 | Train Loss Components: Auxiliary: 0.1867 | Cardinality: 0.1036 | Classification: 0.0834 | Classification_binary: 0.1437 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1064 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1457 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0616 | Regression_eta: 0.0298 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0582 | LR: 3.71e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.7495s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7930 | max_loss: 0.1458 | max_backward: 1.7855 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 2.2800 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5572.8 | uss_MB: 4310.0 | child_rss_MB: 6426.4 | allocated_MB: 496.8 | reserved_MB: 29580.0 | inactive_split_MB: 850.8 +rank5 [2026-10-29 16:06:05,474] INFO : Step 31600/50000 rank5 | Train Loss: 1.0288 | Train Loss Unweighted: 0.4730 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1317 | Classification: 0.0906 | Classification_binary: 0.1743 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1012 | Regression_eta: 0.0017 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1644 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0581 | Regression_eta: 0.0303 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0646 | LR: 3.69e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1923s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1021 | max_forward: 0.8041 | max_loss: 0.5000 | max_backward: 1.8881 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0133 | Memory: rss_MB: 5567.6 | uss_MB: 4330.2 | child_rss_MB: 6421.3 | allocated_MB: 494.5 | reserved_MB: 29580.0 | inactive_split_MB: 850.5 +rank5 [2026-10-29 16:10:48,122] INFO : Step 31700/50000 rank5 | Train Loss: 0.9206 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1106 | Classification: 0.0729 | Classification_binary: 0.1465 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0439 | Regression_energy: 0.0513 | Regression_eta: 0.0436 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0530 | LR: 3.66e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.7509s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0062 | max_forward: 0.7097 | max_loss: 0.1806 | max_backward: 1.6117 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0100 | Memory: rss_MB: 5567.0 | uss_MB: 4328.6 | child_rss_MB: 6424.1 | allocated_MB: 495.3 | reserved_MB: 29580.0 | inactive_split_MB: 969.6 +rank5 [2026-10-29 16:15:56,265] INFO : Step 31800/50000 rank5 | Train Loss: 0.9090 | Train Loss Unweighted: 0.4310 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1140 | Classification: 0.0748 | Classification_binary: 0.1332 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1094 | Regression_eta: 0.0013 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0504 | Regression_eta: 0.0332 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0552 | LR: 3.64e-04 | DataLoad Time: 0.0161s | Model Forward Time: 3.8756s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7102 | max_loss: 0.1436 | max_backward: 1.6093 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5562.0 | uss_MB: 4326.9 | child_rss_MB: 6424.4 | allocated_MB: 499.4 | reserved_MB: 29584.0 | inactive_split_MB: 1068.5 +rank5 [2026-10-29 16:20:39,712] INFO : Step 31900/50000 rank5 | Train Loss: 0.9919 | Train Loss Unweighted: 0.4645 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1269 | Classification: 0.0911 | Classification_binary: 0.1745 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0975 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1858 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0615 | Regression_eta: 0.0333 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0756 | LR: 3.61e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3401s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.2538 | max_backward: 1.9768 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.5597 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5568.8 | uss_MB: 4333.1 | child_rss_MB: 6433.0 | allocated_MB: 496.2 | reserved_MB: 29584.0 | inactive_split_MB: 829.8 +rank5 [2026-10-29 16:25:21,506] INFO : Step 32000/50000 rank5 | Train Loss: 0.9662 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1172 | Classification: 0.0861 | Classification_binary: 0.1650 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1010 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0408 | Regression_energy: 0.0484 | Regression_eta: 0.0293 | Regression_pt: 0.0530 | Regression_sin_phi: 0.0558 | LR: 3.58e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2272s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7801 | max_loss: 0.1534 | max_backward: 1.8397 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5561.3 | uss_MB: 4329.3 | child_rss_MB: 6422.4 | allocated_MB: 497.7 | reserved_MB: 29584.0 | inactive_split_MB: 927.6 +rank5 [2026-10-29 16:25:22,457] INFO : Running validation on rank5 +rank5 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 16:26:16,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:26:16,710] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 16:26:42,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 16:26:43,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 16:26:44,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 16:26:45,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 16:26:55,840] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 16:26:56,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:26:57,010] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 16:27:12,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 16:27:13,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 16:27:14,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 16:27:15,888] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 16:27:25,899] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 16:27:26,913] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 16:27:27,382] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 16:27:42,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 16:27:43,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 16:27:44,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 16:27:45,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 16:27:55,157] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 16:32:59,848] INFO : Step 32100/50000 rank5 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4572 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1125 | Classification: 0.0829 | Classification_binary: 0.1389 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1159 | Regression_eta: 0.0016 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1416 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0456 | Regression_eta: 0.0407 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0480 | LR: 3.56e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8024s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0431 | max_forward: 0.7694 | max_loss: 0.1379 | max_backward: 1.5662 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 1.3675 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6985.8 | uss_MB: 5757.8 | child_rss_MB: 6447.8 | allocated_MB: 499.0 | reserved_MB: 25862.0 | inactive_split_MB: 892.0 +rank5 [2026-10-29 16:37:33,176] INFO : Step 32200/50000 rank5 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1193 | Classification: 0.0958 | Classification_binary: 0.1508 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1092 | Regression_eta: 0.0016 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0605 | Regression_eta: 0.0531 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0670 | LR: 3.53e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1398s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8207 | max_loss: 0.1504 | max_backward: 2.0613 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.9148 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6989.2 | uss_MB: 5763.2 | child_rss_MB: 6457.2 | allocated_MB: 499.4 | reserved_MB: 25864.0 | inactive_split_MB: 778.6 +rank5 [2026-10-29 16:42:04,508] INFO : Step 32300/50000 rank5 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4909 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1295 | Classification: 0.0875 | Classification_binary: 0.1834 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0017 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0556 | Regression_eta: 0.0508 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0488 | LR: 3.51e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.7859s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8382 | max_loss: 0.2412 | max_backward: 2.0637 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.9540 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5664.5 | uss_MB: 4450.0 | child_rss_MB: 6462.2 | allocated_MB: 499.0 | reserved_MB: 27716.0 | inactive_split_MB: 775.0 +rank5 [2026-10-29 16:46:37,461] INFO : Step 32400/50000 rank5 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1089 | Classification: 0.0860 | Classification_binary: 0.1494 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1059 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0657 | Regression_eta: 0.0291 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0637 | LR: 3.48e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5413s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1137 | max_forward: 0.8338 | max_loss: 0.1463 | max_backward: 1.8285 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 0.8970 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5664.1 | uss_MB: 4449.3 | child_rss_MB: 6459.6 | allocated_MB: 497.5 | reserved_MB: 27716.0 | inactive_split_MB: 964.6 +rank5 [2026-10-29 16:51:11,722] INFO : Step 32500/50000 rank5 | Train Loss: 0.9126 | Train Loss Unweighted: 0.4568 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.0989 | Classification: 0.0980 | Classification_binary: 0.1302 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1128 | Regression_eta: 0.0017 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0713 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0580 | Regression_eta: 0.0620 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0612 | LR: 3.46e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0944s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7619 | max_loss: 0.1497 | max_backward: 1.5269 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.7665 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5643.1 | uss_MB: 4429.2 | child_rss_MB: 6454.9 | allocated_MB: 495.6 | reserved_MB: 27718.0 | inactive_split_MB: 1084.2 +rank5 [2026-10-29 16:55:48,602] INFO : Step 32600/50000 rank5 | Train Loss: 0.9821 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2025 | Cardinality: 0.1270 | Classification: 0.0917 | Classification_binary: 0.1687 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1000 | Regression_eta: 0.0014 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0499 | Regression_eta: 0.0507 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0928 | LR: 3.43e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.2678s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1505 | max_backward: 1.8823 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 5658.5 | uss_MB: 4448.5 | child_rss_MB: 6453.0 | allocated_MB: 496.5 | reserved_MB: 27844.0 | inactive_split_MB: 835.5 +rank5 [2026-10-29 17:00:16,962] INFO : Step 32700/50000 rank5 | Train Loss: 1.0218 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2103 | Cardinality: 0.1308 | Classification: 0.0908 | Classification_binary: 0.1761 | HitGrouping: 0.1579 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0585 | Classification_binary: 0.1901 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0634 | Regression_eta: 0.0575 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0533 | LR: 3.41e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.2012s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7654 | max_loss: 0.2523 | max_backward: 1.8639 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4904 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.1101 | Memory: rss_MB: 5657.8 | uss_MB: 4448.3 | child_rss_MB: 6459.1 | allocated_MB: 495.8 | reserved_MB: 27844.0 | inactive_split_MB: 1177.8 +rank5 [2026-10-29 17:04:50,646] INFO : Step 32800/50000 rank5 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4798 | Train Loss Components: Auxiliary: 0.1977 | Cardinality: 0.1139 | Classification: 0.0957 | Classification_binary: 0.1480 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1163 | Regression_eta: 0.0019 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0451 | Regression_eta: 0.0626 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0822 | LR: 3.38e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0488s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7437 | max_loss: 0.1443 | max_backward: 1.7603 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.5989 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5618.6 | uss_MB: 4447.1 | child_rss_MB: 6328.3 | allocated_MB: 497.6 | reserved_MB: 27846.0 | inactive_split_MB: 1089.8 +rank5 [2026-10-29 17:09:25,011] INFO : Step 32900/50000 rank5 | Train Loss: 1.0585 | Train Loss Unweighted: 0.4917 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1472 | Classification: 0.0991 | Classification_binary: 0.1566 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0020 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1455 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0695 | Regression_eta: 0.0638 | Regression_pt: 0.0755 | Regression_sin_phi: 0.0496 | LR: 3.36e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2349s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0141 | max_forward: 0.7853 | max_loss: 0.1442 | max_backward: 1.9716 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.5450 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5622.7 | uss_MB: 4451.0 | child_rss_MB: 6346.8 | allocated_MB: 498.5 | reserved_MB: 27846.0 | inactive_split_MB: 844.8 +rank5 [2026-10-29 17:13:57,543] INFO : Step 33000/50000 rank5 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4896 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1109 | Classification: 0.0902 | Classification_binary: 0.1757 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1096 | Regression_eta: 0.0015 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1930 | Regression_cos_phi: 0.0799 | Regression_energy: 0.0611 | Regression_eta: 0.0504 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0782 | LR: 3.33e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.1385s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0372 | max_forward: 0.7798 | max_loss: 0.1504 | max_backward: 1.8342 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5614.6 | uss_MB: 4448.7 | child_rss_MB: 6348.5 | allocated_MB: 497.7 | reserved_MB: 27846.0 | inactive_split_MB: 857.6 +rank5 [2026-10-29 17:13:58,425] INFO : Running validation on rank5 +rank5 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 17:13:58,426] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 17:14:49,526] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:14:50,068] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 17:15:17,120] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 17:15:18,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 17:15:19,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 17:15:20,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 17:15:30,207] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 17:15:31,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:15:31,508] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 17:15:47,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 17:15:48,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 17:15:49,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 17:15:50,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 17:16:00,056] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 17:16:01,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 17:16:01,832] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 17:16:17,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 17:16:17,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 17:16:18,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 17:16:19,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 17:16:29,481] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 17:21:40,218] INFO : Step 33100/50000 rank5 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4666 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1297 | Classification: 0.0842 | Classification_binary: 0.1537 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1125 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1839 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0813 | Regression_eta: 0.0382 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0491 | LR: 3.31e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1407s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7607 | max_loss: 0.1464 | max_backward: 1.8352 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6888.0 | uss_MB: 5749.0 | child_rss_MB: 6329.7 | allocated_MB: 496.1 | reserved_MB: 24572.0 | inactive_split_MB: 941.9 +rank5 [2026-10-29 17:26:33,026] INFO : Step 33200/50000 rank5 | Train Loss: 0.9345 | Train Loss Unweighted: 0.4417 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.1121 | Classification: 0.0825 | Classification_binary: 0.1333 | HitGrouping: 0.1549 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0017 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0624 | Regression_eta: 0.0289 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0613 | LR: 3.28e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2180s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0447 | max_forward: 0.7619 | max_loss: 0.4589 | max_backward: 1.9181 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.4922 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6887.2 | uss_MB: 5750.0 | child_rss_MB: 6329.3 | allocated_MB: 497.6 | reserved_MB: 24578.0 | inactive_split_MB: 835.2 +rank5 [2026-10-29 17:31:31,566] INFO : Step 33300/50000 rank5 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1270 | Classification: 0.0780 | Classification_binary: 0.1518 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0985 | Regression_eta: 0.0012 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0446 | Regression_eta: 0.0269 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0451 | LR: 3.26e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.6823s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0334 | max_forward: 0.7851 | max_loss: 0.1434 | max_backward: 2.0284 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.7975 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5654.9 | uss_MB: 4170.1 | child_rss_MB: 6324.3 | allocated_MB: 496.9 | reserved_MB: 24578.0 | inactive_split_MB: 903.1 +rank5 [2026-10-29 17:36:22,673] INFO : Step 33400/50000 rank5 | Train Loss: 0.9537 | Train Loss Unweighted: 0.4634 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1239 | Classification: 0.0872 | Classification_binary: 0.1563 | HitGrouping: 0.1500 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0013 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0693 | Regression_eta: 0.0403 | Regression_pt: 0.0773 | Regression_sin_phi: 0.0685 | LR: 3.23e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2724s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7951 | max_loss: 0.2522 | max_backward: 1.9349 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0134 | Memory: rss_MB: 5664.7 | uss_MB: 4533.0 | child_rss_MB: 6325.6 | allocated_MB: 500.7 | reserved_MB: 26578.0 | inactive_split_MB: 843.2 +rank5 [2026-10-29 17:41:05,303] INFO : Step 33500/50000 rank5 | Train Loss: 0.9039 | Train Loss Unweighted: 0.4494 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.1057 | Classification: 0.0795 | Classification_binary: 0.1278 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0011 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1117 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0424 | Regression_eta: 0.0266 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0617 | LR: 3.21e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1832s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0177 | max_forward: 0.7696 | max_loss: 0.1421 | max_backward: 1.9122 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 0.5562 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5654.7 | uss_MB: 4519.8 | child_rss_MB: 6324.9 | allocated_MB: 494.9 | reserved_MB: 26578.0 | inactive_split_MB: 839.3 +rank5 [2026-10-29 17:45:56,711] INFO : Step 33600/50000 rank5 | Train Loss: 0.9179 | Train Loss Unweighted: 0.4415 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.0997 | Classification: 0.0812 | Classification_binary: 0.1440 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0608 | Regression_eta: 0.0278 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0461 | LR: 3.19e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4037s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7783 | max_loss: 0.2418 | max_backward: 2.0392 | max_optimizer_step: 0.2787 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5667.7 | uss_MB: 4535.9 | child_rss_MB: 6324.1 | allocated_MB: 497.9 | reserved_MB: 28594.0 | inactive_split_MB: 1127.0 +rank5 [2026-10-29 17:50:43,454] INFO : Step 33700/50000 rank5 | Train Loss: 1.0564 | Train Loss Unweighted: 0.4962 | Train Loss Components: Auxiliary: 0.2235 | Cardinality: 0.1517 | Classification: 0.0917 | Classification_binary: 0.1786 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0659 | Regression_eta: 0.0311 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0729 | LR: 3.16e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1462s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0121 | max_forward: 0.7981 | max_loss: 0.1554 | max_backward: 1.9054 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5667.6 | uss_MB: 4535.1 | child_rss_MB: 6300.3 | allocated_MB: 501.8 | reserved_MB: 28594.0 | inactive_split_MB: 1103.2 +rank5 [2026-10-29 17:55:24,637] INFO : Step 33800/50000 rank5 | Train Loss: 0.9305 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1894 | Cardinality: 0.1080 | Classification: 0.0751 | Classification_binary: 0.1369 | HitGrouping: 0.1527 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1201 | Regression_eta: 0.0014 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1397 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0484 | Regression_eta: 0.0451 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0794 | LR: 3.14e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.1045s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8033 | max_loss: 0.1441 | max_backward: 1.7197 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0176 | Memory: rss_MB: 5662.9 | uss_MB: 4531.8 | child_rss_MB: 6320.7 | allocated_MB: 496.9 | reserved_MB: 28594.0 | inactive_split_MB: 909.5 +rank5 [2026-10-29 18:00:01,586] INFO : Step 33900/50000 rank5 | Train Loss: 0.9360 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.1942 | Cardinality: 0.1035 | Classification: 0.0805 | Classification_binary: 0.1361 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1124 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0520 | Regression_eta: 0.0316 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0372 | LR: 3.11e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.2968s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7557 | max_loss: 0.2439 | max_backward: 1.8923 | max_optimizer_step: 0.2623 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5665.6 | uss_MB: 4533.9 | child_rss_MB: 6306.4 | allocated_MB: 496.5 | reserved_MB: 28598.0 | inactive_split_MB: 941.5 +rank5 [2026-10-29 18:04:39,053] INFO : Step 34000/50000 rank5 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0930 | Classification: 0.0875 | Classification_binary: 0.1606 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1634 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0504 | Regression_eta: 0.0559 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0527 | LR: 3.09e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3488s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0943 | max_forward: 0.8289 | max_loss: 0.4616 | max_backward: 2.0219 | max_optimizer_step: 0.1767 | max_tensorboard_logging: 0.4773 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5666.0 | uss_MB: 4534.2 | child_rss_MB: 6326.4 | allocated_MB: 496.0 | reserved_MB: 28598.0 | inactive_split_MB: 1123.0 +rank5 [2026-10-29 18:04:39,969] INFO : Running validation on rank5 +rank5 [2026-10-29 18:04:39,969] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 18:04:39,969] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 18:05:30,627] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:05:31,101] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 18:05:54,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 18:05:55,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 18:05:56,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 18:05:57,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 18:06:07,854] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:06:08,708] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:06:09,213] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 18:06:25,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 18:06:26,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 18:06:27,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 18:06:27,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 18:06:37,981] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:06:39,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:06:39,482] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 18:06:54,925] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 18:06:55,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 18:06:56,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 18:06:57,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 18:07:07,317] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 18:12:12,107] INFO : Step 34100/50000 rank5 | Train Loss: 1.0240 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2153 | Cardinality: 0.1202 | Classification: 0.0979 | Classification_binary: 0.1668 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1210 | Regression_eta: 0.0013 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0442 | Regression_eta: 0.0324 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0520 | LR: 3.06e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5011s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0078 | max_forward: 0.9085 | max_loss: 0.1531 | max_backward: 2.1727 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.4680 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6900.4 | uss_MB: 5761.6 | child_rss_MB: 6326.4 | allocated_MB: 496.4 | reserved_MB: 24560.0 | inactive_split_MB: 1172.2 +rank5 [2026-10-29 18:17:11,665] INFO : Step 34200/50000 rank5 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4225 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.1030 | Classification: 0.0694 | Classification_binary: 0.1279 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0016 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1183 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0573 | Regression_eta: 0.0308 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0401 | LR: 3.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.8730s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 1.1401 | max_loss: 1.0887 | max_backward: 3.3565 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5402 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6897.3 | uss_MB: 5759.3 | child_rss_MB: 6328.4 | allocated_MB: 493.1 | reserved_MB: 27668.0 | inactive_split_MB: 842.0 +rank5 [2026-10-29 18:21:53,342] INFO : Step 34300/50000 rank5 | Train Loss: 0.8999 | Train Loss Unweighted: 0.4291 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.1046 | Classification: 0.0782 | Classification_binary: 0.1421 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0015 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0556 | Classification_binary: 0.1405 | Regression_cos_phi: 0.0905 | Regression_energy: 0.0519 | Regression_eta: 0.0553 | Regression_pt: 0.0578 | Regression_sin_phi: 0.0643 | LR: 3.02e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5615s | Phase Times: max_pre_step_barrier: 0.1089 | max_batch_to_device: 0.0070 | max_forward: 0.7738 | max_loss: 0.4754 | max_backward: 1.8888 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5721.4 | uss_MB: 4584.0 | child_rss_MB: 6315.8 | allocated_MB: 498.9 | reserved_MB: 29668.0 | inactive_split_MB: 777.1 +rank5 [2026-10-29 18:26:42,330] INFO : Step 34400/50000 rank5 | Train Loss: 0.9382 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2026 | Cardinality: 0.1181 | Classification: 0.0846 | Classification_binary: 0.1681 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0488 | Regression_eta: 0.0263 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0513 | LR: 2.99e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.6560s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.8411 | max_loss: 0.2739 | max_backward: 2.1385 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5734.9 | uss_MB: 4598.3 | child_rss_MB: 6315.2 | allocated_MB: 496.8 | reserved_MB: 29668.0 | inactive_split_MB: 858.8 +rank5 [2026-10-29 18:31:40,822] INFO : Step 34500/50000 rank5 | Train Loss: 0.9189 | Train Loss Unweighted: 0.4456 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1063 | Classification: 0.0767 | Classification_binary: 0.1421 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0595 | Classification_binary: 0.1237 | Regression_cos_phi: 0.0388 | Regression_energy: 0.0610 | Regression_eta: 0.0437 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0539 | LR: 2.97e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2133s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.7430 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5727.7 | uss_MB: 4595.8 | child_rss_MB: 6313.8 | allocated_MB: 494.3 | reserved_MB: 29668.0 | inactive_split_MB: 926.0 +rank5 [2026-10-29 18:36:27,182] INFO : Step 34600/50000 rank5 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4340 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1046 | Classification: 0.0794 | Classification_binary: 0.1340 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0016 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1178 | Regression_cos_phi: 0.0780 | Regression_energy: 0.0579 | Regression_eta: 0.0358 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0761 | LR: 2.95e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6956s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0170 | max_forward: 0.9895 | max_loss: 0.4622 | max_backward: 2.2541 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5731.7 | uss_MB: 4597.0 | child_rss_MB: 6311.8 | allocated_MB: 498.6 | reserved_MB: 29672.0 | inactive_split_MB: 765.1 +rank5 [2026-10-29 18:40:58,825] INFO : Step 34700/50000 rank5 | Train Loss: 0.9586 | Train Loss Unweighted: 0.4539 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1185 | Classification: 0.0862 | Classification_binary: 0.1624 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1380 | Regression_cos_phi: 0.0533 | Regression_energy: 0.0511 | Regression_eta: 0.0326 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0511 | LR: 2.92e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3657s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7862 | max_loss: 0.2628 | max_backward: 1.9231 | max_optimizer_step: 0.2860 | max_tensorboard_logging: 0.4599 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5730.7 | uss_MB: 4596.4 | child_rss_MB: 6303.0 | allocated_MB: 499.0 | reserved_MB: 29672.0 | inactive_split_MB: 1074.3 +rank5 [2026-10-29 18:45:38,651] INFO : Step 34800/50000 rank5 | Train Loss: 0.9346 | Train Loss Unweighted: 0.4373 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1276 | Classification: 0.0867 | Classification_binary: 0.1551 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0959 | Regression_eta: 0.0013 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0707 | Regression_energy: 0.0510 | Regression_eta: 0.0443 | Regression_pt: 0.0548 | Regression_sin_phi: 0.0879 | LR: 2.90e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2159s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0770 | max_forward: 0.8404 | max_loss: 0.1556 | max_backward: 1.8035 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.5351 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5727.0 | uss_MB: 4594.2 | child_rss_MB: 6312.4 | allocated_MB: 497.4 | reserved_MB: 29672.0 | inactive_split_MB: 1113.6 +rank5 [2026-10-29 18:50:17,935] INFO : Step 34900/50000 rank5 | Train Loss: 0.8719 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1154 | Classification: 0.0681 | Classification_binary: 0.1220 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1093 | Regression_eta: 0.0013 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1414 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0610 | Regression_eta: 0.0320 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0369 | LR: 2.88e-04 | DataLoad Time: 0.0198s | Model Forward Time: 6.1443s | Phase Times: max_pre_step_barrier: 0.0080 | max_batch_to_device: 0.0693 | max_forward: 1.2127 | max_loss: 0.2834 | max_backward: 3.0996 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.6125 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 5724.6 | uss_MB: 4576.6 | child_rss_MB: 6292.2 | allocated_MB: 495.3 | reserved_MB: 29672.0 | inactive_split_MB: 864.7 +rank5 [2026-10-29 18:55:19,194] INFO : Step 35000/50000 rank5 | Train Loss: 0.9157 | Train Loss Unweighted: 0.4220 | Train Loss Components: Auxiliary: 0.1897 | Cardinality: 0.1230 | Classification: 0.0771 | Classification_binary: 0.1455 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0980 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0357 | Classification_binary: 0.1507 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0525 | Regression_eta: 0.0310 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0628 | LR: 2.85e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5128s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.8243 | max_loss: 0.4759 | max_backward: 2.1181 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5728.2 | uss_MB: 4598.3 | child_rss_MB: 6307.6 | allocated_MB: 498.3 | reserved_MB: 29672.0 | inactive_split_MB: 766.0 +rank5 [2026-10-29 18:55:20,127] INFO : Running validation on rank5 +rank5 [2026-10-29 18:55:20,128] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 18:55:20,128] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 18:56:11,605] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:56:12,116] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 18:56:36,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 18:56:37,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 18:56:38,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 18:56:39,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 18:56:49,841] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 18:56:49,876] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:56:50,324] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 18:57:07,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 18:57:08,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 18:57:09,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 18:57:10,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 18:57:20,451] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-29 18:57:20,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 18:57:20,900] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 18:57:36,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 18:57:37,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 18:57:38,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 18:57:38,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 18:57:48,901] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 19:02:56,669] INFO : Step 35100/50000 rank5 | Train Loss: 0.9622 | Train Loss Unweighted: 0.4512 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1150 | Classification: 0.0857 | Classification_binary: 0.1724 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0948 | Regression_eta: 0.0014 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1803 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0505 | Regression_eta: 0.0338 | Regression_pt: 0.0549 | Regression_sin_phi: 0.0580 | LR: 2.83e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8766s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1018 | max_forward: 0.8921 | max_loss: 0.1518 | max_backward: 2.0328 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8633 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5815.2 | uss_MB: 4686.8 | child_rss_MB: 6305.0 | allocated_MB: 499.4 | reserved_MB: 23366.0 | inactive_split_MB: 1243.8 +rank5 [2026-10-29 19:07:43,277] INFO : Step 35200/50000 rank5 | Train Loss: 0.9331 | Train Loss Unweighted: 0.4385 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.1234 | Classification: 0.0814 | Classification_binary: 0.1441 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0533 | Regression_eta: 0.0479 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0760 | LR: 2.81e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1467s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0577 | max_forward: 0.7693 | max_loss: 0.1423 | max_backward: 1.7209 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5822.3 | uss_MB: 4693.9 | child_rss_MB: 6304.7 | allocated_MB: 494.4 | reserved_MB: 25346.0 | inactive_split_MB: 1054.2 +rank5 [2026-10-29 19:12:30,667] INFO : Step 35300/50000 rank5 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4075 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1105 | Classification: 0.0819 | Classification_binary: 0.1245 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0989 | Regression_eta: 0.0014 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.0978 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0441 | Regression_eta: 0.0760 | Regression_pt: 0.0473 | Regression_sin_phi: 0.0420 | LR: 2.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.8651s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 1.2856 | max_loss: 0.4079 | max_backward: 4.4969 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5749 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5814.5 | uss_MB: 4685.2 | child_rss_MB: 6307.8 | allocated_MB: 498.3 | reserved_MB: 25346.0 | inactive_split_MB: 1140.5 +rank5 [2026-10-29 19:17:14,808] INFO : Step 35400/50000 rank5 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4888 | Train Loss Components: Auxiliary: 0.2130 | Cardinality: 0.1296 | Classification: 0.0908 | Classification_binary: 0.1718 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0014 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.1525 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0649 | Regression_eta: 0.0321 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0520 | LR: 2.76e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3429s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8312 | max_loss: 0.1490 | max_backward: 2.0287 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.4708 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5820.8 | uss_MB: 4696.7 | child_rss_MB: 6308.5 | allocated_MB: 497.8 | reserved_MB: 25346.0 | inactive_split_MB: 1132.0 +rank5 [2026-10-29 19:21:58,249] INFO : Step 35500/50000 rank5 | Train Loss: 0.9444 | Train Loss Unweighted: 0.4557 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1201 | Classification: 0.0924 | Classification_binary: 0.1605 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1800 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0517 | Regression_eta: 0.0389 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0580 | LR: 2.74e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3607s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7670 | max_loss: 0.2318 | max_backward: 1.8405 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5820.9 | uss_MB: 4694.9 | child_rss_MB: 6312.3 | allocated_MB: 497.5 | reserved_MB: 26942.0 | inactive_split_MB: 1130.5 +rank5 [2026-10-29 19:26:41,946] INFO : Step 35600/50000 rank5 | Train Loss: 0.9280 | Train Loss Unweighted: 0.4451 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1095 | Classification: 0.0861 | Classification_binary: 0.1334 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1165 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0702 | Regression_eta: 0.0283 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0570 | LR: 2.72e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.7415s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7460 | max_loss: 0.1440 | max_backward: 1.8206 | max_optimizer_step: 0.2911 | max_tensorboard_logging: 0.6036 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 5820.5 | uss_MB: 3919.7 | child_rss_MB: 6316.0 | allocated_MB: 496.1 | reserved_MB: 26942.0 | inactive_split_MB: 907.6 +rank5 [2026-10-29 19:31:29,665] INFO : Step 35700/50000 rank5 | Train Loss: 0.9116 | Train Loss Unweighted: 0.4301 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1148 | Classification: 0.0829 | Classification_binary: 0.1432 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0998 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0575 | Regression_eta: 0.0710 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0734 | LR: 2.70e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6307s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8312 | max_loss: 0.2568 | max_backward: 2.2622 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5823.2 | uss_MB: 4699.5 | child_rss_MB: 6323.6 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1241.0 +rank5 [2026-10-29 19:36:11,896] INFO : Step 35800/50000 rank5 | Train Loss: 1.0095 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1252 | Classification: 0.0857 | Classification_binary: 0.1786 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1066 | Regression_eta: 0.0014 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1707 | Regression_cos_phi: 0.0493 | Regression_energy: 0.0454 | Regression_eta: 0.0328 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0544 | LR: 2.68e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1758s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7644 | max_loss: 0.5090 | max_backward: 1.8855 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4861 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5816.8 | uss_MB: 4694.8 | child_rss_MB: 6319.7 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1129.0 +rank5 [2026-10-29 19:40:48,762] INFO : Step 35900/50000 rank5 | Train Loss: 0.9596 | Train Loss Unweighted: 0.4571 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1173 | Classification: 0.0913 | Classification_binary: 0.1468 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0014 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0497 | Regression_eta: 0.0396 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0480 | LR: 2.65e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.0310s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7217 | max_loss: 0.1443 | max_backward: 1.6894 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.4586 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5816.9 | uss_MB: 4695.3 | child_rss_MB: 6319.9 | allocated_MB: 495.4 | reserved_MB: 29024.0 | inactive_split_MB: 934.6 +rank5 [2026-10-29 19:45:19,091] INFO : Step 36000/50000 rank5 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0912 | Classification: 0.0938 | Classification_binary: 0.1314 | HitGrouping: 0.1587 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1135 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1179 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0621 | Regression_eta: 0.0583 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0428 | LR: 2.63e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2999s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0073 | max_forward: 0.7630 | max_loss: 0.4769 | max_backward: 1.8869 | max_optimizer_step: 0.2587 | max_tensorboard_logging: 0.5842 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5815.2 | uss_MB: 4695.9 | child_rss_MB: 6324.3 | allocated_MB: 495.0 | reserved_MB: 29024.0 | inactive_split_MB: 937.5 +rank5 [2026-10-29 19:45:19,982] INFO : Running validation on rank5 +rank5 [2026-10-29 19:45:19,982] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 19:45:19,982] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 19:46:10,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:46:11,093] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 19:46:35,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 19:46:36,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 19:46:37,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 19:46:38,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 19:46:48,909] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 19:46:49,454] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:46:49,932] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 19:47:05,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 19:47:06,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 19:47:07,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 19:47:08,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 19:47:18,376] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 19:47:19,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 19:47:19,885] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 19:47:35,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 19:47:36,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 19:47:36,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 19:47:37,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 19:47:47,737] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 19:53:12,007] INFO : Step 36100/50000 rank5 | Train Loss: 0.9292 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1100 | Classification: 0.0906 | Classification_binary: 0.1490 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1269 | Regression_cos_phi: 0.0302 | Regression_energy: 0.0471 | Regression_eta: 0.0278 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0468 | LR: 2.61e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7839s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7768 | max_loss: 0.2455 | max_backward: 1.8077 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 1.0653 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5880.7 | uss_MB: 4756.1 | child_rss_MB: 6293.3 | allocated_MB: 499.2 | reserved_MB: 26012.0 | inactive_split_MB: 1080.2 +rank5 [2026-10-29 19:58:21,470] INFO : Step 36200/50000 rank5 | Train Loss: 0.9030 | Train Loss Unweighted: 0.4308 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1163 | Classification: 0.0762 | Classification_binary: 0.1596 | HitGrouping: 0.1506 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0384 | Classification_binary: 0.1521 | Regression_cos_phi: 0.0485 | Regression_energy: 0.0519 | Regression_eta: 0.0385 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0535 | LR: 2.59e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5367s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0777 | max_forward: 0.8125 | max_loss: 0.1541 | max_backward: 1.8492 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 5884.8 | uss_MB: 4767.2 | child_rss_MB: 6290.6 | allocated_MB: 494.4 | reserved_MB: 26016.0 | inactive_split_MB: 1377.4 +rank5 [2026-10-29 20:03:20,658] INFO : Step 36300/50000 rank5 | Train Loss: 0.9576 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1236 | Classification: 0.0876 | Classification_binary: 0.1347 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1100 | Regression_eta: 0.0017 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0531 | Regression_eta: 0.0318 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0340 | LR: 2.57e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.4518s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.8065 | max_loss: 0.1484 | max_backward: 1.9315 | max_optimizer_step: 0.1930 | max_tensorboard_logging: 0.6801 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5878.7 | uss_MB: 4768.0 | child_rss_MB: 6273.7 | allocated_MB: 496.0 | reserved_MB: 28008.0 | inactive_split_MB: 1156.5 +rank5 [2026-10-29 20:08:10,865] INFO : Step 36400/50000 rank5 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4147 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.0999 | Classification: 0.0796 | Classification_binary: 0.1372 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0629 | Classification_binary: 0.1448 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0503 | Regression_eta: 0.0374 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0589 | LR: 2.55e-04 | DataLoad Time: 0.0225s | Model Forward Time: 5.8688s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0192 | max_forward: 1.1829 | max_loss: 0.3790 | max_backward: 3.0917 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 5880.4 | uss_MB: 4768.9 | child_rss_MB: 6270.8 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 845.2 +rank5 [2026-10-29 20:12:41,929] INFO : Step 36500/50000 rank5 | Train Loss: 0.9762 | Train Loss Unweighted: 0.4543 | Train Loss Components: Auxiliary: 0.2088 | Cardinality: 0.1239 | Classification: 0.0805 | Classification_binary: 0.1682 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1926 | Regression_cos_phi: 0.0982 | Regression_energy: 0.0639 | Regression_eta: 0.0476 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0801 | LR: 2.52e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2751s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7782 | max_loss: 0.1554 | max_backward: 1.9378 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.5103 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5877.6 | uss_MB: 4769.9 | child_rss_MB: 6273.0 | allocated_MB: 499.0 | reserved_MB: 30008.0 | inactive_split_MB: 913.8 +rank5 [2026-10-29 20:17:47,229] INFO : Step 36600/50000 rank5 | Train Loss: 0.8699 | Train Loss Unweighted: 0.4190 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.1138 | Classification: 0.0734 | Classification_binary: 0.1317 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0010 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0269 | Classification_binary: 0.1360 | Regression_cos_phi: 0.0426 | Regression_energy: 0.0599 | Regression_eta: 0.0241 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0510 | LR: 2.50e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.2971s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7561 | max_loss: 0.1454 | max_backward: 1.9017 | max_optimizer_step: 0.5115 | max_tensorboard_logging: 0.5672 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5872.6 | uss_MB: 4766.8 | child_rss_MB: 6277.6 | allocated_MB: 494.4 | reserved_MB: 30008.0 | inactive_split_MB: 856.0 +rank5 [2026-10-29 20:22:56,451] INFO : Step 36700/50000 rank5 | Train Loss: 0.9065 | Train Loss Unweighted: 0.4327 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.1102 | Classification: 0.0849 | Classification_binary: 0.1262 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1087 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0804 | Classification_binary: 0.1088 | Regression_cos_phi: 0.0473 | Regression_energy: 0.0539 | Regression_eta: 0.0311 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0665 | LR: 2.48e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.0791s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.7169 | max_loss: 0.1507 | max_backward: 1.7104 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5871.4 | uss_MB: 4766.4 | child_rss_MB: 6270.7 | allocated_MB: 496.5 | reserved_MB: 30008.0 | inactive_split_MB: 1154.6 +rank5 [2026-10-29 20:28:04,049] INFO : Step 36800/50000 rank5 | Train Loss: 0.9068 | Train Loss Unweighted: 0.4227 | Train Loss Components: Auxiliary: 0.1925 | Cardinality: 0.1100 | Classification: 0.0739 | Classification_binary: 0.1571 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0945 | Regression_eta: 0.0013 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1659 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0427 | Regression_eta: 0.0375 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0665 | LR: 2.46e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4663s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8487 | max_loss: 0.1525 | max_backward: 2.1121 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5877.1 | uss_MB: 4770.1 | child_rss_MB: 6274.6 | allocated_MB: 499.3 | reserved_MB: 30008.0 | inactive_split_MB: 939.6 +rank5 [2026-10-29 20:33:09,554] INFO : Step 36900/50000 rank5 | Train Loss: 0.8934 | Train Loss Unweighted: 0.4126 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1194 | Classification: 0.0721 | Classification_binary: 0.1507 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0013 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0517 | Regression_eta: 0.0247 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0616 | LR: 2.44e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.5633s | Phase Times: max_pre_step_barrier: 0.0139 | max_batch_to_device: 0.0068 | max_forward: 1.0975 | max_loss: 0.2241 | max_backward: 2.7278 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.6216 | max_tensorboard_flush: 0.0085 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5871.7 | uss_MB: 4750.7 | child_rss_MB: 6260.8 | allocated_MB: 498.9 | reserved_MB: 30008.0 | inactive_split_MB: 771.1 +rank5 [2026-10-29 20:38:05,959] INFO : Step 37000/50000 rank5 | Train Loss: 0.9040 | Train Loss Unweighted: 0.4182 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1192 | Classification: 0.0807 | Classification_binary: 0.1318 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1130 | Regression_cos_phi: 0.0627 | Regression_energy: 0.0442 | Regression_eta: 0.0227 | Regression_pt: 0.0489 | Regression_sin_phi: 0.0366 | LR: 2.42e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8501s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7055 | max_loss: 0.1505 | max_backward: 1.5664 | max_optimizer_step: 0.1900 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5866.9 | uss_MB: 4763.2 | child_rss_MB: 6273.0 | allocated_MB: 493.9 | reserved_MB: 30008.0 | inactive_split_MB: 1169.8 +rank5 [2026-10-29 20:38:06,932] INFO : Running validation on rank5 +rank5 [2026-10-29 20:38:06,932] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 20:38:06,932] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 20:39:05,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:39:06,530] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 20:39:31,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 20:39:32,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 20:39:33,418] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 20:39:34,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 20:39:44,421] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 20:39:45,043] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:39:45,533] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 20:40:02,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 20:40:03,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 20:40:04,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 20:40:05,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 20:40:15,233] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 20:40:16,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 20:40:16,608] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 20:40:32,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 20:40:32,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 20:40:33,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 20:40:34,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 20:40:44,888] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 20:45:59,178] INFO : Step 37100/50000 rank5 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4614 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1255 | Classification: 0.0999 | Classification_binary: 0.1492 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1041 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.1458 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0571 | Regression_eta: 0.0430 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0967 | LR: 2.40e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.8455s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0061 | max_forward: 0.8553 | max_loss: 0.1461 | max_backward: 2.1723 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5959.4 | uss_MB: 4852.0 | child_rss_MB: 6311.9 | allocated_MB: 499.3 | reserved_MB: 25668.0 | inactive_split_MB: 852.4 +rank5 [2026-10-29 20:50:39,579] INFO : Step 37200/50000 rank5 | Train Loss: 0.9288 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.1928 | Cardinality: 0.1108 | Classification: 0.0836 | Classification_binary: 0.1643 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0015 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0468 | Regression_eta: 0.0227 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0467 | LR: 2.38e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6405s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8460 | max_loss: 0.2603 | max_backward: 2.0959 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5966.0 | uss_MB: 4861.5 | child_rss_MB: 6312.1 | allocated_MB: 497.9 | reserved_MB: 25674.0 | inactive_split_MB: 846.0 +rank5 [2026-10-29 20:55:11,586] INFO : Step 37300/50000 rank5 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1275 | Classification: 0.0781 | Classification_binary: 0.1424 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0982 | Regression_eta: 0.0013 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0340 | Classification_binary: 0.1556 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0452 | Regression_eta: 0.0250 | Regression_pt: 0.0494 | Regression_sin_phi: 0.0372 | LR: 2.36e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.0193s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7653 | max_loss: 0.1472 | max_backward: 1.7105 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.4778 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0692 | Memory: rss_MB: 5962.2 | uss_MB: 4859.4 | child_rss_MB: 6302.2 | allocated_MB: 497.3 | reserved_MB: 27616.0 | inactive_split_MB: 974.4 +rank5 [2026-10-29 21:00:09,946] INFO : Step 37400/50000 rank5 | Train Loss: 0.8836 | Train Loss Unweighted: 0.4228 | Train Loss Components: Auxiliary: 0.1825 | Cardinality: 0.0983 | Classification: 0.0851 | Classification_binary: 0.1289 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1025 | Regression_eta: 0.0015 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1390 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0454 | Regression_eta: 0.0306 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0451 | LR: 2.34e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5373s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7611 | max_loss: 0.1956 | max_backward: 1.9276 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7505 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5960.0 | uss_MB: 4860.1 | child_rss_MB: 6301.8 | allocated_MB: 495.0 | reserved_MB: 27690.0 | inactive_split_MB: 915.5 +rank5 [2026-10-29 21:05:01,763] INFO : Step 37500/50000 rank5 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1260 | Classification: 0.0804 | Classification_binary: 0.1577 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1005 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1506 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0547 | Regression_eta: 0.0363 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0538 | LR: 2.32e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4452s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.9030 | max_loss: 0.1494 | max_backward: 1.9871 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5938.4 | uss_MB: 4859.4 | child_rss_MB: 6224.4 | allocated_MB: 496.1 | reserved_MB: 27690.0 | inactive_split_MB: 923.5 +rank5 [2026-10-29 21:09:54,197] INFO : Step 37600/50000 rank5 | Train Loss: 0.9091 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.0995 | Classification: 0.0683 | Classification_binary: 0.1534 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0957 | Regression_eta: 0.0015 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1450 | Regression_cos_phi: 0.0502 | Regression_energy: 0.0455 | Regression_eta: 0.0341 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0819 | LR: 2.30e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3496s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0070 | max_forward: 0.8280 | max_loss: 0.1484 | max_backward: 1.8328 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5933.4 | uss_MB: 4856.7 | child_rss_MB: 6230.2 | allocated_MB: 497.1 | reserved_MB: 27690.0 | inactive_split_MB: 927.5 +rank5 [2026-10-29 21:14:45,920] INFO : Step 37700/50000 rank5 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3995 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.0864 | Classification: 0.0813 | Classification_binary: 0.1231 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0012 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1076 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0442 | Regression_eta: 0.0198 | Regression_pt: 0.0478 | Regression_sin_phi: 0.0461 | LR: 2.28e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.4209s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7582 | max_loss: 0.2486 | max_backward: 1.9197 | max_optimizer_step: 0.3005 | max_tensorboard_logging: 0.5767 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5802.8 | uss_MB: 4857.6 | child_rss_MB: 5767.0 | allocated_MB: 495.2 | reserved_MB: 29766.0 | inactive_split_MB: 925.5 +rank5 [2026-10-29 21:19:50,095] INFO : Step 37800/50000 rank5 | Train Loss: 0.8717 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1057 | Classification: 0.0822 | Classification_binary: 0.1363 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0014 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0573 | Regression_energy: 0.0575 | Regression_eta: 0.0587 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0668 | LR: 2.26e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.6252s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 1.2105 | max_loss: 0.2449 | max_backward: 2.0574 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 5800.5 | uss_MB: 4856.8 | child_rss_MB: 5763.9 | allocated_MB: 498.3 | reserved_MB: 29766.0 | inactive_split_MB: 930.0 +rank5 [2026-10-29 21:24:49,254] INFO : Step 37900/50000 rank5 | Train Loss: 0.9843 | Train Loss Unweighted: 0.4684 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1353 | Classification: 0.0912 | Classification_binary: 0.1705 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0014 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1516 | Regression_cos_phi: 0.0425 | Regression_energy: 0.0683 | Regression_eta: 0.0468 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0431 | LR: 2.24e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.2088s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0297 | max_forward: 0.8081 | max_loss: 0.1547 | max_backward: 2.6708 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5096 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5782.9 | uss_MB: 4859.7 | child_rss_MB: 5732.7 | allocated_MB: 495.6 | reserved_MB: 29766.0 | inactive_split_MB: 912.0 +rank5 [2026-10-29 21:29:44,506] INFO : Step 38000/50000 rank5 | Train Loss: 0.8556 | Train Loss Unweighted: 0.4002 | Train Loss Components: Auxiliary: 0.1833 | Cardinality: 0.1090 | Classification: 0.0730 | Classification_binary: 0.1293 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0972 | Regression_eta: 0.0011 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0520 | Regression_eta: 0.0285 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0610 | LR: 2.22e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1650s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7330 | max_loss: 0.2284 | max_backward: 1.8082 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5778.8 | uss_MB: 4857.4 | child_rss_MB: 5727.0 | allocated_MB: 494.5 | reserved_MB: 29766.0 | inactive_split_MB: 847.3 +rank5 [2026-10-29 21:29:45,441] INFO : Running validation on rank5 +rank5 [2026-10-29 21:29:45,441] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 21:29:45,442] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 21:30:41,619] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:30:42,096] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 21:31:09,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 21:31:10,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 21:31:11,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 21:31:12,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 21:31:22,656] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 21:31:23,195] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:31:23,630] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 21:31:39,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 21:31:40,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 21:31:41,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 21:31:42,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 21:31:52,519] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 21:31:52,734] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 21:31:53,234] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 21:32:08,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 21:32:09,612] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 21:32:10,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 21:32:11,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 21:32:21,225] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 21:37:39,070] INFO : Step 38100/50000 rank5 | Train Loss: 0.9020 | Train Loss Unweighted: 0.4337 | Train Loss Components: Auxiliary: 0.1907 | Cardinality: 0.1231 | Classification: 0.0888 | Classification_binary: 0.1339 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0014 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0599 | Regression_energy: 0.0503 | Regression_eta: 0.0409 | Regression_pt: 0.0544 | Regression_sin_phi: 0.0578 | LR: 2.20e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8729s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0963 | max_forward: 0.8445 | max_loss: 0.1406 | max_backward: 1.9197 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.9697 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7028.7 | uss_MB: 6098.3 | child_rss_MB: 5734.5 | allocated_MB: 496.0 | reserved_MB: 27516.0 | inactive_split_MB: 970.5 +rank5 [2026-10-29 21:42:37,695] INFO : Step 38200/50000 rank5 | Train Loss: 0.9174 | Train Loss Unweighted: 0.4351 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.0971 | Classification: 0.0888 | Classification_binary: 0.1526 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0948 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1280 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0495 | Regression_eta: 0.0282 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0455 | LR: 2.18e-04 | DataLoad Time: 0.0171s | Model Forward Time: 6.4471s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.8480 | max_loss: 0.2651 | max_backward: 3.1905 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.9151 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5881.2 | uss_MB: 4342.7 | child_rss_MB: 5733.1 | allocated_MB: 499.7 | reserved_MB: 27516.0 | inactive_split_MB: 848.5 +rank5 [2026-10-29 21:47:30,432] INFO : Step 38300/50000 rank5 | Train Loss: 0.9342 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1998 | Cardinality: 0.1099 | Classification: 0.0869 | Classification_binary: 0.1470 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0013 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0559 | Regression_energy: 0.0532 | Regression_eta: 0.0275 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0644 | LR: 2.16e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.6868s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0664 | max_forward: 0.8187 | max_loss: 0.1464 | max_backward: 1.8154 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.9517 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5875.2 | uss_MB: 4949.5 | child_rss_MB: 5725.9 | allocated_MB: 495.5 | reserved_MB: 27516.0 | inactive_split_MB: 1085.5 +rank5 [2026-10-29 21:52:14,429] INFO : Step 38400/50000 rank5 | Train Loss: 0.8281 | Train Loss Unweighted: 0.4036 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0915 | Classification: 0.0723 | Classification_binary: 0.1160 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1069 | Regression_eta: 0.0012 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1372 | Regression_cos_phi: 0.0341 | Regression_energy: 0.0779 | Regression_eta: 0.0410 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0565 | LR: 2.14e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.7621s | Phase Times: max_pre_step_barrier: 0.0868 | max_batch_to_device: 0.0062 | max_forward: 0.9937 | max_loss: 0.2500 | max_backward: 1.9690 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5872.2 | uss_MB: 4950.8 | child_rss_MB: 5730.9 | allocated_MB: 496.1 | reserved_MB: 27516.0 | inactive_split_MB: 972.6 +rank5 [2026-10-29 21:57:09,317] INFO : Step 38500/50000 rank5 | Train Loss: 0.9386 | Train Loss Unweighted: 0.4484 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1248 | Classification: 0.1044 | Classification_binary: 0.1389 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0558 | Regression_eta: 0.0385 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0436 | LR: 2.12e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2029s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.7200 | max_loss: 0.1386 | max_backward: 1.5323 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5868.3 | uss_MB: 4952.0 | child_rss_MB: 5707.9 | allocated_MB: 496.0 | reserved_MB: 27520.0 | inactive_split_MB: 776.0 +rank5 [2026-10-29 22:01:43,606] INFO : Step 38600/50000 rank5 | Train Loss: 0.9227 | Train Loss Unweighted: 0.4436 | Train Loss Components: Auxiliary: 0.1984 | Cardinality: 0.1056 | Classification: 0.0788 | Classification_binary: 0.1660 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0979 | Regression_eta: 0.0013 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1747 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0474 | Regression_eta: 0.0334 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0568 | LR: 2.11e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4276s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8785 | max_loss: 0.1473 | max_backward: 2.0276 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5861.9 | uss_MB: 4951.4 | child_rss_MB: 5677.1 | allocated_MB: 499.0 | reserved_MB: 29592.0 | inactive_split_MB: 827.5 +rank5 [2026-10-29 22:06:19,437] INFO : Step 38700/50000 rank5 | Train Loss: 0.8153 | Train Loss Unweighted: 0.3795 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0947 | Classification: 0.0722 | Classification_binary: 0.1256 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0893 | Regression_eta: 0.0013 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1460 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0417 | Regression_eta: 0.0446 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0491 | LR: 2.09e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2100s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7251 | max_loss: 0.2433 | max_backward: 1.8091 | max_optimizer_step: 0.1965 | max_tensorboard_logging: 0.4744 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5858.3 | uss_MB: 4950.6 | child_rss_MB: 5664.7 | allocated_MB: 493.1 | reserved_MB: 29592.0 | inactive_split_MB: 1080.0 +rank5 [2026-10-29 22:11:06,082] INFO : Step 38800/50000 rank5 | Train Loss: 0.8386 | Train Loss Unweighted: 0.4077 | Train Loss Components: Auxiliary: 0.1776 | Cardinality: 0.0905 | Classification: 0.0849 | Classification_binary: 0.1308 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0012 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0358 | Regression_energy: 0.0529 | Regression_eta: 0.0248 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0402 | LR: 2.07e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1578s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.7346 | max_loss: 0.1426 | max_backward: 1.8320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5857.6 | uss_MB: 4949.0 | child_rss_MB: 5646.6 | allocated_MB: 497.9 | reserved_MB: 29592.0 | inactive_split_MB: 1086.4 +rank5 [2026-10-29 22:15:54,682] INFO : Step 38900/50000 rank5 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1916 | Cardinality: 0.1035 | Classification: 0.0839 | Classification_binary: 0.1479 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0014 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1333 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0488 | Regression_eta: 0.0449 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0667 | LR: 2.05e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.8765s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0144 | max_forward: 2.1610 | max_loss: 0.2966 | max_backward: 2.8203 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4876 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 5858.5 | uss_MB: 4951.7 | child_rss_MB: 5647.2 | allocated_MB: 500.0 | reserved_MB: 29592.0 | inactive_split_MB: 849.6 +rank5 [2026-10-29 22:20:36,723] INFO : Step 39000/50000 rank5 | Train Loss: 0.8505 | Train Loss Unweighted: 0.4196 | Train Loss Components: Auxiliary: 0.1813 | Cardinality: 0.0941 | Classification: 0.0808 | Classification_binary: 0.1368 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0993 | Regression_eta: 0.0015 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0320 | Classification_binary: 0.0922 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0418 | Regression_eta: 0.0340 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0377 | LR: 2.03e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2833s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7392 | max_loss: 0.2445 | max_backward: 1.8584 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5844.6 | uss_MB: 4939.8 | child_rss_MB: 5638.4 | allocated_MB: 496.8 | reserved_MB: 29592.0 | inactive_split_MB: 1086.0 +rank5 [2026-10-29 22:20:37,631] INFO : Running validation on rank5 +rank5 [2026-10-29 22:20:37,631] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 22:20:37,631] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 22:21:31,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:21:31,758] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 22:21:58,992] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 22:22:00,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 22:22:01,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 22:22:02,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 22:22:12,053] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 22:22:12,680] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:22:13,174] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 22:22:29,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 22:22:30,425] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 22:22:31,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 22:22:32,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 22:22:42,385] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 22:22:43,329] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 22:22:43,799] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 22:22:59,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 22:22:59,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 22:23:00,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 22:23:01,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 22:23:11,621] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-29 22:28:30,320] INFO : Step 39100/50000 rank5 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1110 | Classification: 0.0839 | Classification_binary: 0.1198 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1329 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0649 | Regression_eta: 0.0596 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0480 | LR: 2.01e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5495s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7800 | max_loss: 0.2457 | max_backward: 1.9719 | max_optimizer_step: 0.3248 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 7010.9 | uss_MB: 6103.0 | child_rss_MB: 5608.2 | allocated_MB: 497.5 | reserved_MB: 24374.0 | inactive_split_MB: 923.6 +rank5 [2026-10-29 22:33:29,806] INFO : Step 39200/50000 rank5 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4242 | Train Loss Components: Auxiliary: 0.1863 | Cardinality: 0.1210 | Classification: 0.0912 | Classification_binary: 0.1382 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1142 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0472 | Regression_eta: 0.0261 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0520 | LR: 2.00e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2588s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0324 | max_forward: 0.7817 | max_loss: 0.1452 | max_backward: 1.9286 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5940.0 | uss_MB: 5036.4 | child_rss_MB: 5609.8 | allocated_MB: 497.9 | reserved_MB: 26338.0 | inactive_split_MB: 912.4 +rank5 [2026-10-29 22:38:28,991] INFO : Step 39300/50000 rank5 | Train Loss: 0.9298 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1080 | Classification: 0.0882 | Classification_binary: 0.1594 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0951 | Regression_eta: 0.0013 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1693 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0523 | Regression_eta: 0.0428 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0595 | LR: 1.98e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.1930s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7922 | max_loss: 0.2572 | max_backward: 1.8558 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 5927.1 | uss_MB: 5036.1 | child_rss_MB: 5613.3 | allocated_MB: 499.4 | reserved_MB: 26338.0 | inactive_split_MB: 1408.0 +rank5 [2026-10-29 22:43:21,467] INFO : Step 39400/50000 rank5 | Train Loss: 0.8421 | Train Loss Unweighted: 0.3964 | Train Loss Components: Auxiliary: 0.1755 | Cardinality: 0.1006 | Classification: 0.0679 | Classification_binary: 0.1346 | HitGrouping: 0.1534 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0731 | Regression_energy: 0.0605 | Regression_eta: 0.0481 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0595 | LR: 1.96e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.2884s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1084 | max_forward: 1.1740 | max_loss: 0.3046 | max_backward: 2.6778 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8613 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5928.1 | uss_MB: 4351.2 | child_rss_MB: 5609.1 | allocated_MB: 495.0 | reserved_MB: 26338.0 | inactive_split_MB: 1095.6 +rank5 [2026-10-29 22:48:17,291] INFO : Step 39500/50000 rank5 | Train Loss: 0.8539 | Train Loss Unweighted: 0.4179 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0879 | Classification: 0.0889 | Classification_binary: 0.1243 | HitGrouping: 0.1565 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1206 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0610 | Regression_eta: 0.0274 | Regression_pt: 0.0658 | Regression_sin_phi: 0.0458 | LR: 1.94e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6156s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7373 | max_loss: 0.1406 | max_backward: 1.8719 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.8502 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5922.7 | uss_MB: 5037.5 | child_rss_MB: 5598.5 | allocated_MB: 492.8 | reserved_MB: 28318.0 | inactive_split_MB: 1078.3 +rank5 [2026-10-29 22:53:41,208] INFO : Step 39600/50000 rank5 | Train Loss: 0.9371 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1161 | Classification: 0.0813 | Classification_binary: 0.1596 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1039 | Regression_eta: 0.0013 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1566 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0638 | Regression_eta: 0.0285 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0669 | LR: 1.93e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.3256s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0173 | max_forward: 1.4879 | max_loss: 0.2520 | max_backward: 1.9863 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.8346 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5921.1 | uss_MB: 5038.4 | child_rss_MB: 5602.9 | allocated_MB: 496.0 | reserved_MB: 28318.0 | inactive_split_MB: 858.4 +rank5 [2026-10-29 22:58:48,059] INFO : Step 39700/50000 rank5 | Train Loss: 0.9204 | Train Loss Unweighted: 0.4427 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1101 | Classification: 0.0841 | Classification_binary: 0.1567 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0997 | Regression_eta: 0.0013 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1735 | Regression_cos_phi: 0.0714 | Regression_energy: 0.0516 | Regression_eta: 0.0449 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0897 | LR: 1.91e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2766s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.7571 | max_loss: 0.4946 | max_backward: 1.9147 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 5909.3 | uss_MB: 5037.4 | child_rss_MB: 5602.2 | allocated_MB: 497.0 | reserved_MB: 30324.0 | inactive_split_MB: 1175.5 +rank5 [2026-10-29 23:04:38,179] INFO : Step 39800/50000 rank5 | Train Loss: 0.8155 | Train Loss Unweighted: 0.3780 | Train Loss Components: Auxiliary: 0.1674 | Cardinality: 0.0963 | Classification: 0.0643 | Classification_binary: 0.1275 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0550 | Regression_eta: 0.0653 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0744 | LR: 1.89e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0223s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7197 | max_loss: 0.1489 | max_backward: 1.6690 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.4760 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1068 | Memory: rss_MB: 5897.7 | uss_MB: 5034.7 | child_rss_MB: 5594.3 | allocated_MB: 494.4 | reserved_MB: 30324.0 | inactive_split_MB: 897.3 +rank5 [2026-10-29 23:09:43,446] INFO : Step 39900/50000 rank5 | Train Loss: 0.8844 | Train Loss Unweighted: 0.3998 | Train Loss Components: Auxiliary: 0.1871 | Cardinality: 0.1077 | Classification: 0.0758 | Classification_binary: 0.1443 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0883 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1525 | Regression_cos_phi: 0.1019 | Regression_energy: 0.0417 | Regression_eta: 0.0500 | Regression_pt: 0.0454 | Regression_sin_phi: 0.1014 | LR: 1.88e-04 | DataLoad Time: 0.0129s | Model Forward Time: 4.9662s | Phase Times: max_pre_step_barrier: 0.0268 | max_batch_to_device: 0.0727 | max_forward: 1.7016 | max_loss: 0.4968 | max_backward: 2.0829 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.4776 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5898.5 | uss_MB: 5037.5 | child_rss_MB: 5601.6 | allocated_MB: 496.1 | reserved_MB: 30324.0 | inactive_split_MB: 1077.6 +rank5 [2026-10-29 23:14:38,166] INFO : Step 40000/50000 rank5 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1170 | Classification: 0.0871 | Classification_binary: 0.1550 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0303 | Regression_energy: 0.0669 | Regression_eta: 0.0266 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0440 | LR: 1.86e-04 | DataLoad Time: 0.0138s | Model Forward Time: 5.4140s | Phase Times: max_pre_step_barrier: 0.1084 | max_batch_to_device: 0.0070 | max_forward: 1.1725 | max_loss: 0.2555 | max_backward: 2.4297 | max_optimizer_step: 0.2969 | max_tensorboard_logging: 0.6818 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5889.3 | uss_MB: 5028.2 | child_rss_MB: 5597.1 | allocated_MB: 501.7 | reserved_MB: 30324.0 | inactive_split_MB: 1368.0 +rank5 [2026-10-29 23:14:39,118] INFO : Running validation on rank5 +rank5 [2026-10-29 23:14:39,119] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-29 23:14:39,119] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-29 23:15:31,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:15:32,370] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-29 23:16:03,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-29 23:16:04,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-29 23:16:06,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-29 23:16:07,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-29 23:16:17,888] INFO : Time taken to make predictions on device 5 is: 0.28 min +rank5 [2026-10-29 23:16:28,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:16:28,893] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-29 23:16:44,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-29 23:16:45,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-29 23:16:46,822] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-29 23:16:47,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-29 23:16:57,785] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-29 23:16:58,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-29 23:16:59,025] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-29 23:17:14,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-29 23:17:15,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-29 23:17:16,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-29 23:17:17,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-29 23:17:27,366] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-29 23:22:41,683] INFO : Step 40100/50000 rank5 | Train Loss: 0.8115 | Train Loss Unweighted: 0.3794 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.1028 | Classification: 0.0708 | Classification_binary: 0.1239 | HitGrouping: 0.1460 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0909 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0405 | Classification_binary: 0.1204 | Regression_cos_phi: 0.0331 | Regression_energy: 0.0464 | Regression_eta: 0.0286 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0350 | LR: 1.84e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.0769s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.1076 | max_forward: 0.8426 | max_loss: 0.1437 | max_backward: 1.8847 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 7273.7 | uss_MB: 5477.4 | child_rss_MB: 5617.3 | allocated_MB: 493.6 | reserved_MB: 28004.0 | inactive_split_MB: 975.8 +rank5 [2026-10-29 23:27:53,247] INFO : Step 40200/50000 rank5 | Train Loss: 0.8264 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1679 | Cardinality: 0.0984 | Classification: 0.0703 | Classification_binary: 0.1210 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0913 | Regression_eta: 0.0013 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1166 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0535 | Regression_eta: 0.0301 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0408 | LR: 1.83e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.8930s | Phase Times: max_pre_step_barrier: 0.0270 | max_batch_to_device: 0.0062 | max_forward: 0.8324 | max_loss: 0.1802 | max_backward: 2.0569 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0152 | max_post_step_barrier: 0.0867 | Memory: rss_MB: 5955.5 | uss_MB: 5094.1 | child_rss_MB: 5625.1 | allocated_MB: 497.1 | reserved_MB: 28008.0 | inactive_split_MB: 935.5 +rank5 [2026-10-29 23:33:22,808] INFO : Step 40300/50000 rank5 | Train Loss: 0.9351 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1181 | Classification: 0.0959 | Classification_binary: 0.1652 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0014 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1691 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0464 | Regression_eta: 0.0415 | Regression_pt: 0.0506 | Regression_sin_phi: 0.0514 | LR: 1.81e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.3106s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8191 | max_loss: 0.1494 | max_backward: 1.9817 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 0.9693 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 5955.1 | uss_MB: 5096.4 | child_rss_MB: 5610.7 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 927.5 +rank5 [2026-10-29 23:38:38,678] INFO : Step 40400/50000 rank5 | Train Loss: 0.9379 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1359 | Classification: 0.0771 | Classification_binary: 0.1554 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0944 | Regression_eta: 0.0014 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1479 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0417 | Regression_eta: 0.0358 | Regression_pt: 0.0449 | Regression_sin_phi: 0.0485 | LR: 1.79e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6741s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0500 | max_forward: 0.8111 | max_loss: 0.5067 | max_backward: 1.8144 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5951.4 | uss_MB: 5092.9 | child_rss_MB: 5616.0 | allocated_MB: 497.0 | reserved_MB: 28008.0 | inactive_split_MB: 931.6 +rank5 [2026-10-29 23:43:43,800] INFO : Step 40500/50000 rank5 | Train Loss: 0.8053 | Train Loss Unweighted: 0.3782 | Train Loss Components: Auxiliary: 0.1649 | Cardinality: 0.0788 | Classification: 0.0749 | Classification_binary: 0.1246 | HitGrouping: 0.1502 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0503 | Regression_eta: 0.0411 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0503 | LR: 1.78e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3983s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.8420 | max_loss: 0.1414 | max_backward: 1.5396 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.9215 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5948.4 | uss_MB: 5092.0 | child_rss_MB: 5597.4 | allocated_MB: 494.5 | reserved_MB: 28012.0 | inactive_split_MB: 979.3 +rank5 [2026-10-29 23:48:43,695] INFO : Step 40600/50000 rank5 | Train Loss: 0.8980 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1108 | Classification: 0.0752 | Classification_binary: 0.1417 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0955 | Regression_eta: 0.0015 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1228 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0536 | Regression_eta: 0.0265 | Regression_pt: 0.0584 | Regression_sin_phi: 0.0237 | LR: 1.76e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3180s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7855 | max_loss: 0.1488 | max_backward: 1.8742 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 5948.2 | uss_MB: 5094.3 | child_rss_MB: 5593.1 | allocated_MB: 499.7 | reserved_MB: 28012.0 | inactive_split_MB: 1102.0 +rank5 [2026-10-29 23:53:39,581] INFO : Step 40700/50000 rank5 | Train Loss: 0.9220 | Train Loss Unweighted: 0.4304 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1148 | Classification: 0.0767 | Classification_binary: 0.1645 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0592 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0567 | Regression_eta: 0.0360 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0565 | LR: 1.75e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5453s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.8091 | max_loss: 0.2475 | max_backward: 2.0011 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.5972 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5949.6 | uss_MB: 5096.1 | child_rss_MB: 5593.6 | allocated_MB: 498.9 | reserved_MB: 28012.0 | inactive_split_MB: 776.0 +rank5 [2026-10-30 00:01:00,043] INFO : Step 40800/50000 rank5 | Train Loss: 0.8642 | Train Loss Unweighted: 0.4082 | Train Loss Components: Auxiliary: 0.1815 | Cardinality: 0.1013 | Classification: 0.0834 | Classification_binary: 0.1260 | HitGrouping: 0.1507 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1200 | Regression_cos_phi: 0.0349 | Regression_energy: 0.0542 | Regression_eta: 0.0417 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0343 | LR: 1.73e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.1156s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7033 | max_loss: 0.1373 | max_backward: 1.5042 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1068 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5935.0 | uss_MB: 4327.0 | child_rss_MB: 5586.6 | allocated_MB: 493.1 | reserved_MB: 28012.0 | inactive_split_MB: 897.3 +rank5 [2026-10-30 00:06:14,591] INFO : Step 40900/50000 rank5 | Train Loss: 0.8749 | Train Loss Unweighted: 0.4038 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1070 | Classification: 0.0854 | Classification_binary: 0.1300 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0923 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1237 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0465 | Regression_eta: 0.0419 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0581 | LR: 1.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5710s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7515 | max_loss: 0.2142 | max_backward: 1.7950 | max_optimizer_step: 0.4960 | max_tensorboard_logging: 0.9322 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5926.9 | uss_MB: 5094.2 | child_rss_MB: 5563.3 | allocated_MB: 497.9 | reserved_MB: 30012.0 | inactive_split_MB: 893.0 +rank5 [2026-10-30 00:11:07,451] INFO : Step 41000/50000 rank5 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.0956 | Classification: 0.0893 | Classification_binary: 0.1497 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0014 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.1735 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0562 | Regression_eta: 0.0492 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0894 | LR: 1.70e-04 | DataLoad Time: 0.0109s | Model Forward Time: 5.0886s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0071 | max_forward: 1.1963 | max_loss: 0.1519 | max_backward: 2.0474 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.0118 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 5918.6 | uss_MB: 5085.3 | child_rss_MB: 5547.4 | allocated_MB: 498.7 | reserved_MB: 30012.0 | inactive_split_MB: 1108.0 +rank5 [2026-10-30 00:11:08,374] INFO : Running validation on rank5 +rank5 [2026-10-30 00:11:08,374] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 00:11:08,374] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 00:11:59,246] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:12:00,027] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 00:12:25,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 00:12:26,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 00:12:27,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 00:12:28,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 00:12:38,921] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 00:12:39,785] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:12:40,281] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 00:12:56,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 00:12:57,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 00:12:58,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 00:12:58,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 00:13:09,008] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 00:13:09,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 00:13:10,647] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 00:13:26,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 00:13:27,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 00:13:28,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 00:13:29,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 00:13:39,139] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 00:19:15,373] INFO : Step 41100/50000 rank5 | Train Loss: 0.9430 | Train Loss Unweighted: 0.4319 | Train Loss Components: Auxiliary: 0.1995 | Cardinality: 0.1330 | Classification: 0.0721 | Classification_binary: 0.1579 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0013 | Regression_pt: 0.0998 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1637 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0528 | Regression_eta: 0.0313 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0413 | LR: 1.69e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.3668s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8221 | max_loss: 0.4800 | max_backward: 2.2615 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7242.0 | uss_MB: 5759.2 | child_rss_MB: 5549.6 | allocated_MB: 497.6 | reserved_MB: 23400.0 | inactive_split_MB: 991.8 +rank5 [2026-10-30 00:24:27,080] INFO : Step 41200/50000 rank5 | Train Loss: 0.8544 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1097 | Classification: 0.0734 | Classification_binary: 0.1131 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1015 | Regression_eta: 0.0015 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1321 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0578 | Regression_eta: 0.0329 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0775 | LR: 1.67e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2547s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7030 | max_loss: 0.2408 | max_backward: 1.7750 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5316 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6039.9 | uss_MB: 5208.8 | child_rss_MB: 5554.6 | allocated_MB: 496.1 | reserved_MB: 27308.0 | inactive_split_MB: 984.0 +rank5 [2026-10-30 00:29:20,951] INFO : Step 41300/50000 rank5 | Train Loss: 0.8221 | Train Loss Unweighted: 0.3852 | Train Loss Components: Auxiliary: 0.1744 | Cardinality: 0.0906 | Classification: 0.0754 | Classification_binary: 0.1369 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0849 | Regression_eta: 0.0012 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0421 | Regression_eta: 0.0226 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0477 | LR: 1.66e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4690s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8083 | max_loss: 0.1794 | max_backward: 1.9993 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.5906 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6040.0 | uss_MB: 5213.3 | child_rss_MB: 5551.2 | allocated_MB: 497.0 | reserved_MB: 27308.0 | inactive_split_MB: 1229.0 +rank5 [2026-10-30 00:34:25,619] INFO : Step 41400/50000 rank5 | Train Loss: 0.9216 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1171 | Classification: 0.0901 | Classification_binary: 0.1523 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1010 | Regression_eta: 0.0011 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1381 | Regression_cos_phi: 0.0608 | Regression_energy: 0.0563 | Regression_eta: 0.0230 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0413 | LR: 1.64e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6346s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0065 | max_forward: 0.7831 | max_loss: 0.4937 | max_backward: 1.9271 | max_optimizer_step: 0.3071 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6033.3 | uss_MB: 5208.0 | child_rss_MB: 5553.7 | allocated_MB: 498.4 | reserved_MB: 29288.0 | inactive_split_MB: 1180.4 +rank5 [2026-10-30 00:39:29,959] INFO : Step 41500/50000 rank5 | Train Loss: 0.8584 | Train Loss Unweighted: 0.4111 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1030 | Classification: 0.0769 | Classification_binary: 0.1210 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0691 | Regression_eta: 0.0457 | Regression_pt: 0.0733 | Regression_sin_phi: 0.0587 | LR: 1.63e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.0416s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.1908 | max_loss: 0.3017 | max_backward: 3.1310 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 6009.7 | uss_MB: 5150.0 | child_rss_MB: 5552.7 | allocated_MB: 494.5 | reserved_MB: 29288.0 | inactive_split_MB: 976.9 +rank5 [2026-10-30 00:44:19,404] INFO : Step 41600/50000 rank5 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4026 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1032 | Classification: 0.0762 | Classification_binary: 0.1246 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0992 | Regression_eta: 0.0013 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1273 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0522 | Regression_eta: 0.0351 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0567 | LR: 1.61e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9945s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0065 | max_forward: 0.7279 | max_loss: 0.2339 | max_backward: 2.5338 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.4866 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1404 | Memory: rss_MB: 6001.6 | uss_MB: 5206.5 | child_rss_MB: 5546.1 | allocated_MB: 496.4 | reserved_MB: 29288.0 | inactive_split_MB: 761.6 +rank5 [2026-10-30 00:49:10,876] INFO : Step 41700/50000 rank5 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1941 | Cardinality: 0.1112 | Classification: 0.0767 | Classification_binary: 0.1493 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0988 | Regression_eta: 0.0013 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0805 | Regression_eta: 0.0484 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0606 | LR: 1.60e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.5075s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7998 | max_loss: 0.1827 | max_backward: 1.9774 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6004.5 | uss_MB: 5210.8 | child_rss_MB: 5548.7 | allocated_MB: 497.0 | reserved_MB: 29288.0 | inactive_split_MB: 1220.0 +rank5 [2026-10-30 00:53:58,971] INFO : Step 41800/50000 rank5 | Train Loss: 0.9125 | Train Loss Unweighted: 0.4285 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.1148 | Classification: 0.0784 | Classification_binary: 0.1537 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1391 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0606 | Regression_eta: 0.0276 | Regression_pt: 0.0652 | Regression_sin_phi: 0.0390 | LR: 1.58e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8255s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7952 | max_loss: 0.1582 | max_backward: 1.8899 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9392 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1143 | Memory: rss_MB: 5984.7 | uss_MB: 5200.7 | child_rss_MB: 5542.4 | allocated_MB: 499.3 | reserved_MB: 31288.0 | inactive_split_MB: 932.9 +rank5 [2026-10-30 00:59:00,784] INFO : Step 41900/50000 rank5 | Train Loss: 0.8303 | Train Loss Unweighted: 0.3927 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1132 | Classification: 0.0782 | Classification_binary: 0.1167 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0010 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.1017 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0448 | Regression_eta: 0.0362 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0414 | LR: 1.57e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.1516s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7413 | max_loss: 0.1970 | max_backward: 1.7614 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5208 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5992.5 | uss_MB: 5207.2 | child_rss_MB: 5522.2 | allocated_MB: 497.1 | reserved_MB: 31288.0 | inactive_split_MB: 1188.0 +rank5 [2026-10-30 01:03:51,615] INFO : Step 42000/50000 rank5 | Train Loss: 0.8724 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1847 | Cardinality: 0.0893 | Classification: 0.0874 | Classification_binary: 0.1377 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0958 | Regression_eta: 0.0012 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0676 | Classification_binary: 0.1496 | Regression_cos_phi: 0.0550 | Regression_energy: 0.0480 | Regression_eta: 0.0371 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0750 | LR: 1.56e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7584s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1147 | max_forward: 0.8308 | max_loss: 0.1392 | max_backward: 1.8336 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5996.6 | uss_MB: 5209.4 | child_rss_MB: 5540.2 | allocated_MB: 496.3 | reserved_MB: 31288.0 | inactive_split_MB: 930.5 +rank5 [2026-10-30 01:03:52,597] INFO : Running validation on rank5 +rank5 [2026-10-30 01:03:52,597] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 01:03:52,598] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 01:04:46,128] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:04:46,670] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 01:05:13,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 01:05:14,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 01:05:15,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 01:05:16,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 01:05:26,749] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-30 01:05:27,369] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:05:27,884] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 01:05:44,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 01:05:45,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 01:05:46,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 01:05:47,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 01:05:57,178] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 01:05:57,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:05:57,663] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 01:06:13,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 01:06:14,329] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 01:06:15,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 01:06:16,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 01:06:26,478] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 01:11:42,846] INFO : Step 42100/50000 rank5 | Train Loss: 0.9390 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1200 | Classification: 0.0884 | Classification_binary: 0.1619 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0926 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1722 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0540 | Regression_eta: 0.0376 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0523 | LR: 1.54e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.0857s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0976 | max_forward: 1.0189 | max_loss: 0.3859 | max_backward: 2.5922 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 1.0368 | max_tensorboard_flush: 0.0149 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7501.8 | uss_MB: 4866.0 | child_rss_MB: 5544.9 | allocated_MB: 497.0 | reserved_MB: 25850.0 | inactive_split_MB: 892.8 +rank5 [2026-10-30 01:16:53,956] INFO : Step 42200/50000 rank5 | Train Loss: 0.9057 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1144 | Classification: 0.0879 | Classification_binary: 0.1381 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0388 | Classification_binary: 0.1504 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0425 | Regression_eta: 0.0324 | Regression_pt: 0.0459 | Regression_sin_phi: 0.0425 | LR: 1.53e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.1530s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7457 | max_loss: 0.1999 | max_backward: 1.6824 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.5916 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6096.0 | uss_MB: 5306.7 | child_rss_MB: 5537.7 | allocated_MB: 497.3 | reserved_MB: 27866.0 | inactive_split_MB: 959.8 +rank5 [2026-10-30 01:18:06,041] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank5 [2026-10-30 01:18:06,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank5 [2026-10-30 01:21:57,449] INFO : Step 42300/50000 rank5 | Train Loss: 0.9052 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1933 | Cardinality: 0.1127 | Classification: 0.0787 | Classification_binary: 0.1569 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0868 | Regression_eta: 0.0013 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0572 | Regression_energy: 0.0482 | Regression_eta: 0.0372 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0438 | LR: 1.52e-04 | DataLoad Time: 0.0154s | Model Forward Time: 6.1179s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.9764 | max_loss: 0.1904 | max_backward: 2.7671 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.9997 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6118.1 | uss_MB: 5325.9 | child_rss_MB: 5544.6 | allocated_MB: 499.1 | reserved_MB: 27872.0 | inactive_split_MB: 1033.6 +rank5 [2026-10-30 01:26:55,611] INFO : Step 42400/50000 rank5 | Train Loss: 0.9358 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.1976 | Cardinality: 0.1220 | Classification: 0.0815 | Classification_binary: 0.1533 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1595 | Regression_cos_phi: 0.0678 | Regression_energy: 0.0521 | Regression_eta: 0.0401 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0476 | LR: 1.50e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.8854s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8701 | max_loss: 0.1465 | max_backward: 1.9416 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.8192 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6099.3 | uss_MB: 5308.3 | child_rss_MB: 5546.3 | allocated_MB: 499.8 | reserved_MB: 29908.0 | inactive_split_MB: 939.2 +rank5 [2026-10-30 01:31:49,012] INFO : Step 42500/50000 rank5 | Train Loss: 0.8126 | Train Loss Unweighted: 0.3791 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0975 | Classification: 0.0765 | Classification_binary: 0.1172 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0918 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0331 | Classification_binary: 0.0952 | Regression_cos_phi: 0.0503 | Regression_energy: 0.0401 | Regression_eta: 0.0361 | Regression_pt: 0.0452 | Regression_sin_phi: 0.0443 | LR: 1.49e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5920s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0858 | max_forward: 0.8156 | max_loss: 0.1407 | max_backward: 1.8570 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7997 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6101.2 | uss_MB: 5308.6 | child_rss_MB: 5538.8 | allocated_MB: 497.5 | reserved_MB: 29908.0 | inactive_split_MB: 943.4 +rank5 [2026-10-30 01:36:44,528] INFO : Step 42600/50000 rank5 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0772 | Classification_binary: 0.1347 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0012 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1288 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0438 | Regression_eta: 0.0256 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0443 | LR: 1.48e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0477s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7519 | max_loss: 0.1393 | max_backward: 1.8021 | max_optimizer_step: 0.3023 | max_tensorboard_logging: 0.7973 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6101.5 | uss_MB: 4501.2 | child_rss_MB: 5544.2 | allocated_MB: 497.0 | reserved_MB: 29908.0 | inactive_split_MB: 877.2 +rank5 [2026-10-30 01:41:48,354] INFO : Step 42700/50000 rank5 | Train Loss: 0.9129 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1178 | Classification: 0.0751 | Classification_binary: 0.1632 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0858 | Regression_eta: 0.0014 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1767 | Regression_cos_phi: 0.0520 | Regression_energy: 0.0452 | Regression_eta: 0.0294 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0668 | LR: 1.47e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6153s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 1.0869 | max_loss: 0.1547 | max_backward: 1.6854 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7949 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6101.6 | uss_MB: 5308.9 | child_rss_MB: 5545.8 | allocated_MB: 497.3 | reserved_MB: 29908.0 | inactive_split_MB: 953.4 +rank5 [2026-10-30 01:46:39,116] INFO : Step 42800/50000 rank5 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1179 | Classification: 0.0767 | Classification_binary: 0.1362 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0013 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1602 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0587 | Regression_eta: 0.0333 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0603 | LR: 1.45e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3534s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8438 | max_loss: 0.1478 | max_backward: 1.6710 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.6821 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6089.4 | uss_MB: 5295.4 | child_rss_MB: 5546.6 | allocated_MB: 499.0 | reserved_MB: 30126.0 | inactive_split_MB: 875.7 +rank5 [2026-10-30 01:51:40,828] INFO : Step 42900/50000 rank5 | Train Loss: 0.7857 | Train Loss Unweighted: 0.3758 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0951 | Classification: 0.0820 | Classification_binary: 0.1113 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0898 | Regression_eta: 0.0010 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1004 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0449 | Regression_eta: 0.0236 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0534 | LR: 1.44e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.9243s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8792 | max_loss: 0.1461 | max_backward: 1.9928 | max_optimizer_step: 0.1903 | max_tensorboard_logging: 0.9695 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6098.2 | uss_MB: 5308.5 | child_rss_MB: 5544.6 | allocated_MB: 496.5 | reserved_MB: 30126.0 | inactive_split_MB: 951.8 +rank5 [2026-10-30 01:56:37,111] INFO : Step 43000/50000 rank5 | Train Loss: 0.8899 | Train Loss Unweighted: 0.4207 | Train Loss Components: Auxiliary: 0.1911 | Cardinality: 0.1035 | Classification: 0.0847 | Classification_binary: 0.1529 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0540 | Regression_eta: 0.0338 | Regression_pt: 0.0581 | Regression_sin_phi: 0.0586 | LR: 1.43e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7367s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1951 | max_backward: 1.9694 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.8418 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6086.1 | uss_MB: 5311.0 | child_rss_MB: 5508.6 | allocated_MB: 495.5 | reserved_MB: 30126.0 | inactive_split_MB: 1120.4 +rank5 [2026-10-30 01:56:38,137] INFO : Running validation on rank5 +rank5 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 01:57:33,188] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:57:33,675] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 01:57:59,402] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 01:58:00,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 01:58:01,484] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 01:58:02,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 01:58:12,462] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 01:58:13,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:58:13,515] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 01:58:29,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 01:58:30,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 01:58:31,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 01:58:32,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 01:58:42,556] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 01:58:43,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 01:58:43,755] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 01:58:59,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 01:58:59,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 01:59:00,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 01:59:01,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 01:59:11,506] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 02:04:24,630] INFO : Step 43100/50000 rank5 | Train Loss: 0.8965 | Train Loss Unweighted: 0.4137 | Train Loss Components: Auxiliary: 0.1901 | Cardinality: 0.1074 | Classification: 0.0705 | Classification_binary: 0.1472 | HitGrouping: 0.1435 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0477 | Regression_eta: 0.0295 | Regression_pt: 0.0511 | Regression_sin_phi: 0.0571 | LR: 1.42e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2192s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.2396 | max_backward: 1.9954 | max_optimizer_step: 0.3209 | max_tensorboard_logging: 1.0803 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7631.1 | uss_MB: 6853.7 | child_rss_MB: 5519.2 | allocated_MB: 496.0 | reserved_MB: 26194.0 | inactive_split_MB: 843.4 +rank5 [2026-10-30 02:09:06,730] INFO : Step 43200/50000 rank5 | Train Loss: 0.8164 | Train Loss Unweighted: 0.3744 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0957 | Classification: 0.0763 | Classification_binary: 0.1197 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0421 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0300 | Regression_energy: 0.0352 | Regression_eta: 0.0289 | Regression_pt: 0.0378 | Regression_sin_phi: 0.0445 | LR: 1.40e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4246s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7823 | max_loss: 0.4822 | max_backward: 1.6766 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6123.7 | uss_MB: 5349.6 | child_rss_MB: 5511.0 | allocated_MB: 495.8 | reserved_MB: 26686.0 | inactive_split_MB: 848.9 +rank5 [2026-10-30 02:13:48,742] INFO : Step 43300/50000 rank5 | Train Loss: 0.9151 | Train Loss Unweighted: 0.4260 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1202 | Classification: 0.0906 | Classification_binary: 0.1467 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0013 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1138 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0561 | Regression_eta: 0.0239 | Regression_pt: 0.0621 | Regression_sin_phi: 0.0378 | LR: 1.39e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.7154s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0155 | max_forward: 0.8938 | max_loss: 0.1464 | max_backward: 2.1010 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.5477 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1243 | Memory: rss_MB: 6097.4 | uss_MB: 5352.9 | child_rss_MB: 5427.7 | allocated_MB: 497.9 | reserved_MB: 28278.0 | inactive_split_MB: 1309.2 +rank5 [2026-10-30 02:18:30,112] INFO : Step 43400/50000 rank5 | Train Loss: 0.9723 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2082 | Cardinality: 0.1222 | Classification: 0.0880 | Classification_binary: 0.1684 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0015 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0516 | Regression_eta: 0.0459 | Regression_pt: 0.0571 | Regression_sin_phi: 0.0532 | LR: 1.38e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.4362s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.5043 | max_backward: 2.0661 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.4824 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6077.9 | uss_MB: 5350.3 | child_rss_MB: 5335.1 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 851.9 +rank5 [2026-10-30 02:23:12,892] INFO : Step 43500/50000 rank5 | Train Loss: 0.8402 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.0947 | Classification: 0.0776 | Classification_binary: 0.1278 | HitGrouping: 0.1501 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0999 | Regression_eta: 0.0012 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1253 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0406 | Regression_eta: 0.0274 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0362 | LR: 1.37e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8630s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8086 | max_loss: 0.1489 | max_backward: 2.0887 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.7978 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1094 | Memory: rss_MB: 6077.6 | uss_MB: 5352.8 | child_rss_MB: 5340.6 | allocated_MB: 498.6 | reserved_MB: 28278.0 | inactive_split_MB: 909.7 +rank5 [2026-10-30 02:28:09,613] INFO : Step 43600/50000 rank5 | Train Loss: 0.7985 | Train Loss Unweighted: 0.3653 | Train Loss Components: Auxiliary: 0.1621 | Cardinality: 0.0887 | Classification: 0.0728 | Classification_binary: 0.1184 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0851 | Regression_eta: 0.0013 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1108 | Regression_cos_phi: 0.0422 | Regression_energy: 0.0375 | Regression_eta: 0.0391 | Regression_pt: 0.0406 | Regression_sin_phi: 0.0726 | LR: 1.36e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.2496s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0170 | max_forward: 1.2777 | max_loss: 0.2064 | max_backward: 1.9835 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 1.1084 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6044.0 | uss_MB: 4957.3 | child_rss_MB: 5260.0 | allocated_MB: 493.5 | reserved_MB: 28278.0 | inactive_split_MB: 1313.2 +rank5 [2026-10-30 02:33:07,295] INFO : Step 43700/50000 rank5 | Train Loss: 0.8599 | Train Loss Unweighted: 0.3884 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1089 | Classification: 0.0797 | Classification_binary: 0.1434 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0812 | Regression_eta: 0.0013 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0396 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0573 | Regression_eta: 0.0501 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0470 | LR: 1.35e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6049s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7902 | max_loss: 0.1752 | max_backward: 1.8774 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8044 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6046.0 | uss_MB: 5350.4 | child_rss_MB: 5241.5 | allocated_MB: 498.0 | reserved_MB: 28278.0 | inactive_split_MB: 1046.0 +rank5 [2026-10-30 02:37:45,726] INFO : Step 43800/50000 rank5 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1875 | Cardinality: 0.1049 | Classification: 0.0814 | Classification_binary: 0.1423 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0907 | Regression_eta: 0.0012 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1597 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0421 | Regression_eta: 0.0327 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0731 | LR: 1.34e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5741s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7940 | max_loss: 0.4805 | max_backward: 1.8159 | max_optimizer_step: 0.2941 | max_tensorboard_logging: 0.7850 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6040.0 | uss_MB: 5349.3 | child_rss_MB: 5218.8 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 839.4 +rank5 [2026-10-30 02:42:41,735] INFO : Step 43900/50000 rank5 | Train Loss: 0.7987 | Train Loss Unweighted: 0.3885 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.0999 | Classification: 0.0753 | Classification_binary: 0.1166 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0968 | Regression_eta: 0.0010 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0401 | Classification_binary: 0.1205 | Regression_cos_phi: 0.0302 | Regression_energy: 0.0650 | Regression_eta: 0.0272 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0506 | LR: 1.33e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.5927s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0846 | max_forward: 0.8891 | max_loss: 0.1768 | max_backward: 1.7357 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7516 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6030.6 | uss_MB: 5347.7 | child_rss_MB: 5179.6 | allocated_MB: 494.5 | reserved_MB: 28278.0 | inactive_split_MB: 1129.5 +rank5 [2026-10-30 02:47:33,825] INFO : Step 44000/50000 rank5 | Train Loss: 0.8263 | Train Loss Unweighted: 0.3926 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.1027 | Classification: 0.0840 | Classification_binary: 0.1318 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0868 | Regression_eta: 0.0011 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1128 | Regression_cos_phi: 0.0381 | Regression_energy: 0.0465 | Regression_eta: 0.0224 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0480 | LR: 1.32e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4521s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8816 | max_loss: 0.1426 | max_backward: 1.9616 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.4846 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6020.6 | uss_MB: 5352.2 | child_rss_MB: 5134.9 | allocated_MB: 496.3 | reserved_MB: 28762.0 | inactive_split_MB: 1307.2 +rank5 [2026-10-30 02:47:34,861] INFO : Running validation on rank5 +rank5 [2026-10-30 02:47:34,861] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 02:47:34,861] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 02:48:28,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 02:48:28,670] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 02:48:55,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 02:48:56,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 02:48:58,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 02:48:58,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 02:49:08,996] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 02:49:09,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 02:49:10,104] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 02:49:25,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 02:49:26,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 02:49:27,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 02:49:28,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 02:49:38,580] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 02:49:39,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 02:49:40,181] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 02:49:55,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 02:49:56,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 02:49:57,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 02:49:58,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 02:50:08,055] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 02:55:20,174] INFO : Step 44100/50000 rank5 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1986 | Cardinality: 0.1334 | Classification: 0.0764 | Classification_binary: 0.1516 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0522 | Regression_eta: 0.0346 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0644 | LR: 1.31e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.1230s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0079 | max_forward: 0.8221 | max_loss: 0.1579 | max_backward: 2.0192 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 1.0889 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7539.1 | uss_MB: 6869.4 | child_rss_MB: 5111.3 | allocated_MB: 498.6 | reserved_MB: 25920.0 | inactive_split_MB: 1073.8 +rank5 [2026-10-30 03:00:12,362] INFO : Step 44200/50000 rank5 | Train Loss: 0.9018 | Train Loss Unweighted: 0.4273 | Train Loss Components: Auxiliary: 0.1909 | Cardinality: 0.1109 | Classification: 0.0847 | Classification_binary: 0.1354 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0012 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0443 | Regression_eta: 0.0270 | Regression_pt: 0.0491 | Regression_sin_phi: 0.0644 | LR: 1.30e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3575s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.1013 | max_forward: 0.8608 | max_loss: 0.1485 | max_backward: 1.6010 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 6119.2 | uss_MB: 5452.1 | child_rss_MB: 5083.8 | allocated_MB: 496.1 | reserved_MB: 27956.0 | inactive_split_MB: 1248.1 +rank5 [2026-10-30 03:04:47,144] INFO : Step 44300/50000 rank5 | Train Loss: 0.8239 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1722 | Cardinality: 0.1007 | Classification: 0.0789 | Classification_binary: 0.1118 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0359 | Classification_binary: 0.1087 | Regression_cos_phi: 0.0601 | Regression_energy: 0.0448 | Regression_eta: 0.0231 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0444 | LR: 1.29e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3174s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8120 | max_loss: 0.1377 | max_backward: 1.4700 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9055 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6110.2 | uss_MB: 5452.8 | child_rss_MB: 5031.6 | allocated_MB: 492.8 | reserved_MB: 27956.0 | inactive_split_MB: 1035.8 +rank5 [2026-10-30 03:09:18,055] INFO : Step 44400/50000 rank5 | Train Loss: 0.8683 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.0849 | Classification: 0.0836 | Classification_binary: 0.1500 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1737 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0474 | Regression_eta: 0.0408 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0699 | LR: 1.28e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.9254s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8907 | max_loss: 0.1476 | max_backward: 2.0623 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6106.5 | uss_MB: 5454.6 | child_rss_MB: 5002.7 | allocated_MB: 499.5 | reserved_MB: 27956.0 | inactive_split_MB: 926.0 +rank5 [2026-10-30 03:14:01,913] INFO : Step 44500/50000 rank5 | Train Loss: 0.8672 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.0989 | Classification: 0.0738 | Classification_binary: 0.1455 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1347 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0569 | Regression_eta: 0.0240 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0623 | LR: 1.27e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2931s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0597 | max_forward: 0.8669 | max_loss: 0.1547 | max_backward: 1.7327 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.5628 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6105.7 | uss_MB: 5452.5 | child_rss_MB: 5004.4 | allocated_MB: 498.3 | reserved_MB: 27956.0 | inactive_split_MB: 860.5 +rank5 [2026-10-30 03:18:59,361] INFO : Step 44600/50000 rank5 | Train Loss: 0.7900 | Train Loss Unweighted: 0.3743 | Train Loss Components: Auxiliary: 0.1688 | Cardinality: 0.0956 | Classification: 0.0724 | Classification_binary: 0.1157 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0919 | Regression_eta: 0.0010 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0365 | Classification_binary: 0.0988 | Regression_cos_phi: 0.0362 | Regression_energy: 0.0415 | Regression_eta: 0.0242 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0385 | LR: 1.26e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5451s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0745 | max_forward: 0.7797 | max_loss: 0.1409 | max_backward: 1.9334 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6022 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6098.6 | uss_MB: 5447.3 | child_rss_MB: 4994.6 | allocated_MB: 494.5 | reserved_MB: 27958.0 | inactive_split_MB: 927.6 +rank5 [2026-10-30 03:23:44,790] INFO : Step 44700/50000 rank5 | Train Loss: 0.8669 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1878 | Cardinality: 0.1160 | Classification: 0.0768 | Classification_binary: 0.1380 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0863 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1120 | Regression_cos_phi: 0.0237 | Regression_energy: 0.0498 | Regression_eta: 0.0324 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0731 | LR: 1.25e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4958s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8629 | max_loss: 0.1449 | max_backward: 2.0281 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.5242 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6099.2 | uss_MB: 5434.3 | child_rss_MB: 5003.0 | allocated_MB: 498.4 | reserved_MB: 27958.0 | inactive_split_MB: 781.6 +rank5 [2026-10-30 03:28:42,238] INFO : Step 44800/50000 rank5 | Train Loss: 0.8585 | Train Loss Unweighted: 0.4003 | Train Loss Components: Auxiliary: 0.1869 | Cardinality: 0.1024 | Classification: 0.0769 | Classification_binary: 0.1550 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0823 | Regression_eta: 0.0011 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1597 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0478 | Regression_eta: 0.0290 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0476 | LR: 1.24e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8028s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0072 | max_forward: 0.7883 | max_loss: 0.5033 | max_backward: 1.9727 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6090.2 | uss_MB: 5455.2 | child_rss_MB: 4956.4 | allocated_MB: 496.5 | reserved_MB: 27958.0 | inactive_split_MB: 778.0 +rank5 [2026-10-30 03:33:12,492] INFO : Step 44900/50000 rank5 | Train Loss: 0.8358 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.1024 | Classification: 0.0797 | Classification_binary: 0.1294 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0859 | Regression_eta: 0.0011 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1141 | Regression_cos_phi: 0.0416 | Regression_energy: 0.0392 | Regression_eta: 0.0328 | Regression_pt: 0.0428 | Regression_sin_phi: 0.0577 | LR: 1.23e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3691s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7360 | max_loss: 0.1482 | max_backward: 1.6886 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7825 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6084.1 | uss_MB: 5452.9 | child_rss_MB: 4956.9 | allocated_MB: 494.3 | reserved_MB: 27958.0 | inactive_split_MB: 922.0 +rank5 [2026-10-30 03:37:54,705] INFO : Step 45000/50000 rank5 | Train Loss: 0.8678 | Train Loss Unweighted: 0.3957 | Train Loss Components: Auxiliary: 0.1790 | Cardinality: 0.1036 | Classification: 0.0801 | Classification_binary: 0.1237 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0941 | Regression_eta: 0.0013 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0390 | Regression_energy: 0.0544 | Regression_eta: 0.0335 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0599 | LR: 1.22e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5696s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7189 | max_loss: 0.2183 | max_backward: 1.8172 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.8812 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6058.9 | uss_MB: 5454.9 | child_rss_MB: 4844.5 | allocated_MB: 495.0 | reserved_MB: 27960.0 | inactive_split_MB: 978.5 +rank5 [2026-10-30 03:37:55,714] INFO : Running validation on rank5 +rank5 [2026-10-30 03:37:55,714] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 03:37:55,715] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 03:38:46,292] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 03:38:46,807] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 03:39:14,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 03:39:15,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 03:39:16,906] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 03:39:18,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 03:39:28,554] INFO : Time taken to make predictions on device 5 is: 0.27 min +rank5 [2026-10-30 03:39:29,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 03:39:30,322] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 03:39:46,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 03:39:47,015] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 03:39:47,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 03:39:48,914] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 03:39:58,925] INFO : Time taken to make predictions on device 5 is: 0.24 min +rank5 [2026-10-30 03:40:00,144] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 03:40:00,674] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 03:40:16,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 03:40:17,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 03:40:18,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 03:40:19,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 03:40:29,293] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 03:45:39,356] INFO : Step 45100/50000 rank5 | Train Loss: 0.8712 | Train Loss Unweighted: 0.4129 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0997 | Classification: 0.0837 | Classification_binary: 0.1445 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0910 | Regression_eta: 0.0012 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0548 | Regression_energy: 0.0379 | Regression_eta: 0.0310 | Regression_pt: 0.0405 | Regression_sin_phi: 0.0585 | LR: 1.21e-04 | DataLoad Time: 0.0143s | Model Forward Time: 5.1935s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8448 | max_loss: 0.1523 | max_backward: 2.1445 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 1.0003 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7489.1 | uss_MB: 6883.8 | child_rss_MB: 4866.9 | allocated_MB: 499.4 | reserved_MB: 24746.0 | inactive_split_MB: 784.5 +rank5 [2026-10-30 03:50:46,383] INFO : Step 45200/50000 rank5 | Train Loss: 0.9130 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1058 | Classification: 0.0776 | Classification_binary: 0.1464 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0963 | Regression_eta: 0.0015 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0547 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0457 | Regression_eta: 0.0432 | Regression_pt: 0.0499 | Regression_sin_phi: 0.0896 | LR: 1.20e-04 | DataLoad Time: 0.0152s | Model Forward Time: 6.5431s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1574 | max_backward: 1.6440 | max_optimizer_step: 0.2111 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1112 | Memory: rss_MB: 6132.8 | uss_MB: 4597.7 | child_rss_MB: 4866.1 | allocated_MB: 499.7 | reserved_MB: 26730.0 | inactive_split_MB: 1073.5 +rank5 [2026-10-30 03:55:44,004] INFO : Step 45300/50000 rank5 | Train Loss: 0.8299 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.1126 | Classification: 0.0744 | Classification_binary: 0.1207 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0424 | Regression_eta: 0.0223 | Regression_pt: 0.0468 | Regression_sin_phi: 0.0564 | LR: 1.19e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1829s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0827 | max_forward: 0.7966 | max_loss: 0.1432 | max_backward: 1.5141 | max_optimizer_step: 0.2646 | max_tensorboard_logging: 0.6448 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6128.3 | uss_MB: 5538.9 | child_rss_MB: 4864.4 | allocated_MB: 495.4 | reserved_MB: 27170.0 | inactive_split_MB: 881.8 +rank5 [2026-10-30 04:00:47,339] INFO : Step 45400/50000 rank5 | Train Loss: 0.8255 | Train Loss Unweighted: 0.4011 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.0984 | Classification: 0.0907 | Classification_binary: 0.1338 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0870 | Regression_eta: 0.0011 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0633 | Classification_binary: 0.1245 | Regression_cos_phi: 0.0328 | Regression_energy: 0.0529 | Regression_eta: 0.0338 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0405 | LR: 1.19e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.3438s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7846 | max_loss: 0.1934 | max_backward: 1.9857 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 0.7880 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6124.9 | uss_MB: 4519.9 | child_rss_MB: 4862.5 | allocated_MB: 499.2 | reserved_MB: 27170.0 | inactive_split_MB: 989.3 +rank5 [2026-10-30 04:05:44,853] INFO : Step 45500/50000 rank5 | Train Loss: 0.8945 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1090 | Classification: 0.0825 | Classification_binary: 0.1567 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0012 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1368 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0442 | Regression_eta: 0.0314 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0444 | LR: 1.18e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4579s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8686 | max_loss: 0.1508 | max_backward: 1.9581 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.4909 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6130.5 | uss_MB: 5543.1 | child_rss_MB: 4864.9 | allocated_MB: 498.4 | reserved_MB: 28766.0 | inactive_split_MB: 883.2 +rank5 [2026-10-30 04:10:39,743] INFO : Step 45600/50000 rank5 | Train Loss: 0.8704 | Train Loss Unweighted: 0.4121 | Train Loss Components: Auxiliary: 0.1840 | Cardinality: 0.1126 | Classification: 0.0802 | Classification_binary: 0.1324 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0983 | Regression_eta: 0.0010 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1406 | Regression_cos_phi: 0.0496 | Regression_energy: 0.0587 | Regression_eta: 0.0292 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0504 | LR: 1.17e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.8852s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0163 | max_forward: 0.8161 | max_loss: 0.2729 | max_backward: 2.1846 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6122.7 | uss_MB: 5488.3 | child_rss_MB: 4871.5 | allocated_MB: 495.6 | reserved_MB: 28766.0 | inactive_split_MB: 1080.4 +rank5 [2026-10-30 04:15:37,194] INFO : Step 45700/50000 rank5 | Train Loss: 0.8006 | Train Loss Unweighted: 0.3784 | Train Loss Components: Auxiliary: 0.1681 | Cardinality: 0.0873 | Classification: 0.0773 | Classification_binary: 0.1111 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0012 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0594 | Classification_binary: 0.0997 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0508 | Regression_eta: 0.0337 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0646 | LR: 1.16e-04 | DataLoad Time: 0.0129s | Model Forward Time: 4.8145s | Phase Times: max_pre_step_barrier: 0.0327 | max_batch_to_device: 0.0149 | max_forward: 1.1858 | max_loss: 0.1429 | max_backward: 1.6968 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1366 | Memory: rss_MB: 6126.5 | uss_MB: 5540.3 | child_rss_MB: 4873.0 | allocated_MB: 496.6 | reserved_MB: 28766.0 | inactive_split_MB: 903.7 +rank5 [2026-10-30 04:20:28,801] INFO : Step 45800/50000 rank5 | Train Loss: 0.8566 | Train Loss Unweighted: 0.3982 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1097 | Classification: 0.0794 | Classification_binary: 0.1398 | HitGrouping: 0.1406 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1041 | Regression_cos_phi: 0.0331 | Regression_energy: 0.0352 | Regression_eta: 0.0221 | Regression_pt: 0.0380 | Regression_sin_phi: 0.0426 | LR: 1.16e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.1216s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.9805 | max_loss: 0.5234 | max_backward: 2.3872 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.8153 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6131.1 | uss_MB: 5547.8 | child_rss_MB: 4874.9 | allocated_MB: 501.7 | reserved_MB: 28766.0 | inactive_split_MB: 1064.3 +rank5 [2026-10-30 04:25:16,811] INFO : Step 45900/50000 rank5 | Train Loss: 0.8754 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1042 | Classification: 0.0696 | Classification_binary: 0.1484 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1603 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0453 | Regression_eta: 0.0327 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0542 | LR: 1.15e-04 | DataLoad Time: 0.0106s | Model Forward Time: 5.8424s | Phase Times: max_pre_step_barrier: 0.0308 | max_batch_to_device: 0.0265 | max_forward: 1.7626 | max_loss: 0.1678 | max_backward: 2.6283 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 6124.4 | uss_MB: 5544.6 | child_rss_MB: 4869.8 | allocated_MB: 496.0 | reserved_MB: 29170.0 | inactive_split_MB: 983.4 +rank5 [2026-10-30 04:29:51,934] INFO : Step 46000/50000 rank5 | Train Loss: 0.7531 | Train Loss Unweighted: 0.3556 | Train Loss Components: Auxiliary: 0.1535 | Cardinality: 0.0870 | Classification: 0.0690 | Classification_binary: 0.1026 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0903 | Regression_eta: 0.0011 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0372 | Classification_binary: 0.1032 | Regression_cos_phi: 0.0363 | Regression_energy: 0.0353 | Regression_eta: 0.0293 | Regression_pt: 0.0411 | Regression_sin_phi: 0.0290 | LR: 1.14e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3119s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0329 | max_forward: 0.7497 | max_loss: 0.1389 | max_backward: 1.9140 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5104 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6122.4 | uss_MB: 5543.2 | child_rss_MB: 4869.8 | allocated_MB: 492.4 | reserved_MB: 29170.0 | inactive_split_MB: 973.8 +rank5 [2026-10-30 04:29:53,063] INFO : Running validation on rank5 +rank5 [2026-10-30 04:29:53,063] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 04:29:53,063] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 04:30:44,349] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 04:30:44,926] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 04:31:11,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 04:31:12,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 04:31:13,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 04:31:14,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 04:31:24,722] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 04:31:25,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 04:31:25,580] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 04:31:41,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 04:31:42,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 04:31:43,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 04:31:44,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 04:31:54,389] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 04:31:54,979] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 04:31:55,463] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 04:32:10,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 04:32:11,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 04:32:12,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 04:32:13,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 04:32:23,240] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 04:37:27,738] INFO : Step 46100/50000 rank5 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3769 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.0999 | Classification: 0.0743 | Classification_binary: 0.1253 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0010 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1328 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0496 | Regression_eta: 0.0262 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0501 | LR: 1.13e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8716s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.9293 | max_loss: 0.1452 | max_backward: 2.3008 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.4968 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1219 | Memory: rss_MB: 7472.5 | uss_MB: 6889.9 | child_rss_MB: 4882.8 | allocated_MB: 498.3 | reserved_MB: 23298.0 | inactive_split_MB: 776.1 +rank5 [2026-10-30 04:42:19,615] INFO : Step 46200/50000 rank5 | Train Loss: 0.8661 | Train Loss Unweighted: 0.3994 | Train Loss Components: Auxiliary: 0.1947 | Cardinality: 0.1164 | Classification: 0.0793 | Classification_binary: 0.1511 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0833 | Regression_eta: 0.0011 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0309 | Regression_energy: 0.0448 | Regression_eta: 0.0282 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0440 | LR: 1.13e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.7278s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8070 | max_loss: 0.2423 | max_backward: 1.9860 | max_optimizer_step: 0.2482 | max_tensorboard_logging: 0.7539 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6199.1 | uss_MB: 5617.8 | child_rss_MB: 4859.8 | allocated_MB: 499.3 | reserved_MB: 25288.0 | inactive_split_MB: 1325.5 +rank5 [2026-10-30 04:46:59,453] INFO : Step 46300/50000 rank5 | Train Loss: 0.8634 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0916 | Classification: 0.0803 | Classification_binary: 0.1320 | HitGrouping: 0.1536 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1031 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1284 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0488 | Regression_eta: 0.0428 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0479 | LR: 1.12e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5165s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7891 | max_loss: 0.1470 | max_backward: 1.7900 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.7709 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6201.3 | uss_MB: 5618.4 | child_rss_MB: 4887.4 | allocated_MB: 498.4 | reserved_MB: 25290.0 | inactive_split_MB: 977.6 +rank5 [2026-10-30 04:51:36,900] INFO : Step 46400/50000 rank5 | Train Loss: 0.8499 | Train Loss Unweighted: 0.4102 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.0956 | Classification: 0.0755 | Classification_binary: 0.1172 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1203 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0492 | Regression_eta: 0.0656 | Regression_pt: 0.0532 | Regression_sin_phi: 0.0704 | LR: 1.11e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9766s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7847 | max_loss: 0.2487 | max_backward: 1.9181 | max_optimizer_step: 0.3148 | max_tensorboard_logging: 0.9408 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6188.6 | uss_MB: 5608.9 | child_rss_MB: 4878.3 | allocated_MB: 501.6 | reserved_MB: 25290.0 | inactive_split_MB: 992.4 +rank5 [2026-10-30 04:56:10,901] INFO : Step 46500/50000 rank5 | Train Loss: 0.8687 | Train Loss Unweighted: 0.4055 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1002 | Classification: 0.0822 | Classification_binary: 0.1448 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0872 | Regression_eta: 0.0013 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0465 | Regression_eta: 0.0435 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0554 | LR: 1.11e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5413s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1839 | max_backward: 1.9629 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6193.9 | uss_MB: 5618.8 | child_rss_MB: 4880.0 | allocated_MB: 498.8 | reserved_MB: 29306.0 | inactive_split_MB: 924.4 +rank5 [2026-10-30 05:00:44,088] INFO : Step 46600/50000 rank5 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4049 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1082 | Classification: 0.0757 | Classification_binary: 0.1508 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1508 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0485 | Regression_eta: 0.0259 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0460 | LR: 1.10e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2106s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7939 | max_loss: 0.2001 | max_backward: 1.6859 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6188.5 | uss_MB: 5615.4 | child_rss_MB: 4846.0 | allocated_MB: 496.0 | reserved_MB: 29306.0 | inactive_split_MB: 901.7 +rank5 [2026-10-30 05:05:18,398] INFO : Step 46700/50000 rank5 | Train Loss: 0.8417 | Train Loss Unweighted: 0.3855 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0982 | Classification: 0.0807 | Classification_binary: 0.1184 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0916 | Regression_eta: 0.0015 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0517 | Regression_eta: 0.0476 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0992 | LR: 1.10e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7623s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8551 | max_loss: 0.5124 | max_backward: 1.9004 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.7762 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1375 | Memory: rss_MB: 6185.4 | uss_MB: 5616.8 | child_rss_MB: 4868.3 | allocated_MB: 495.0 | reserved_MB: 29308.0 | inactive_split_MB: 1000.4 +rank5 [2026-10-30 05:09:48,388] INFO : Step 46800/50000 rank5 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3911 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0920 | Classification: 0.0920 | Classification_binary: 0.1296 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0828 | Regression_eta: 0.0013 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0774 | Classification_binary: 0.1293 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0460 | Regression_eta: 0.0695 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0509 | LR: 1.09e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4628s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7934 | max_loss: 0.1440 | max_backward: 1.7630 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.7219 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6183.1 | uss_MB: 5617.4 | child_rss_MB: 4871.4 | allocated_MB: 495.6 | reserved_MB: 29308.0 | inactive_split_MB: 1332.0 +rank5 [2026-10-30 05:14:19,737] INFO : Step 46900/50000 rank5 | Train Loss: 0.8496 | Train Loss Unweighted: 0.4142 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0976 | Classification: 0.0776 | Classification_binary: 0.1448 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0946 | Regression_eta: 0.0010 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1372 | Regression_cos_phi: 0.0412 | Regression_energy: 0.0467 | Regression_eta: 0.0207 | Regression_pt: 0.0503 | Regression_sin_phi: 0.0650 | LR: 1.09e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4512s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.9007 | max_loss: 0.1576 | max_backward: 1.8470 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6183.1 | uss_MB: 5616.1 | child_rss_MB: 4872.4 | allocated_MB: 495.4 | reserved_MB: 29310.0 | inactive_split_MB: 850.5 +rank5 [2026-10-30 05:18:49,739] INFO : Step 47000/50000 rank5 | Train Loss: 0.7980 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0982 | Classification: 0.0774 | Classification_binary: 0.1196 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.1484 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0489 | Regression_eta: 0.0332 | Regression_pt: 0.0524 | Regression_sin_phi: 0.0491 | LR: 1.08e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8522s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 1.2405 | max_loss: 0.1502 | max_backward: 2.1273 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.7105 | max_tensorboard_flush: 0.0082 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6184.1 | uss_MB: 5617.4 | child_rss_MB: 4877.5 | allocated_MB: 494.9 | reserved_MB: 29310.0 | inactive_split_MB: 852.6 +rank5 [2026-10-30 05:18:50,808] INFO : Running validation on rank5 +rank5 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 05:19:41,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 05:19:42,338] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 05:20:07,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 05:20:08,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 05:20:10,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 05:20:11,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 05:20:21,029] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 05:20:21,505] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 05:20:22,014] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 05:20:37,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 05:20:38,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 05:20:39,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 05:20:40,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 05:20:50,572] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 05:20:51,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 05:20:52,199] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 05:21:07,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 05:21:08,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 05:21:09,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 05:21:09,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 05:21:19,954] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 05:26:12,382] INFO : Step 47100/50000 rank5 | Train Loss: 0.8046 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0851 | Classification: 0.0882 | Classification_binary: 0.1121 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0936 | Regression_eta: 0.0010 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1028 | Regression_cos_phi: 0.0541 | Regression_energy: 0.0488 | Regression_eta: 0.0284 | Regression_pt: 0.0541 | Regression_sin_phi: 0.0420 | LR: 1.07e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5748s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8250 | max_loss: 0.1453 | max_backward: 1.6267 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8108 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 7449.0 | uss_MB: 6878.6 | child_rss_MB: 4837.4 | allocated_MB: 494.1 | reserved_MB: 25416.0 | inactive_split_MB: 894.5 +rank5 [2026-10-30 05:30:44,527] INFO : Step 47200/50000 rank5 | Train Loss: 0.8884 | Train Loss Unweighted: 0.4191 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1138 | Classification: 0.0793 | Classification_binary: 0.1489 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0010 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0510 | Regression_eta: 0.0366 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0696 | LR: 1.07e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.9183s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7960 | max_loss: 0.1482 | max_backward: 1.9261 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.9435 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0661 | Memory: rss_MB: 6249.9 | uss_MB: 5682.5 | child_rss_MB: 4852.8 | allocated_MB: 496.8 | reserved_MB: 26534.0 | inactive_split_MB: 847.6 +rank5 [2026-10-30 05:35:15,255] INFO : Step 47300/50000 rank5 | Train Loss: 0.8547 | Train Loss Unweighted: 0.3920 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1049 | Classification: 0.0798 | Classification_binary: 0.1368 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0012 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1138 | Regression_cos_phi: 0.0366 | Regression_energy: 0.0411 | Regression_eta: 0.0256 | Regression_pt: 0.0435 | Regression_sin_phi: 0.0534 | LR: 1.06e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5889s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7649 | max_loss: 0.2313 | max_backward: 1.8030 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8438 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6253.9 | uss_MB: 5688.5 | child_rss_MB: 4855.9 | allocated_MB: 496.9 | reserved_MB: 28534.0 | inactive_split_MB: 933.0 +rank5 [2026-10-30 05:39:47,014] INFO : Step 47400/50000 rank5 | Train Loss: 0.7606 | Train Loss Unweighted: 0.3755 | Train Loss Components: Auxiliary: 0.1640 | Cardinality: 0.0704 | Classification: 0.0737 | Classification_binary: 0.1072 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0960 | Regression_eta: 0.0010 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.0829 | Regression_cos_phi: 0.0275 | Regression_energy: 0.0499 | Regression_eta: 0.0292 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0313 | LR: 1.06e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5301s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0275 | max_forward: 0.8465 | max_loss: 0.1482 | max_backward: 1.7103 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.8388 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 6241.7 | uss_MB: 5678.8 | child_rss_MB: 4860.9 | allocated_MB: 496.4 | reserved_MB: 28534.0 | inactive_split_MB: 913.7 +rank5 [2026-10-30 05:44:18,076] INFO : Step 47500/50000 rank5 | Train Loss: 0.8134 | Train Loss Unweighted: 0.3681 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1057 | Classification: 0.0739 | Classification_binary: 0.1303 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0805 | Regression_eta: 0.0011 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1071 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0359 | Regression_eta: 0.0237 | Regression_pt: 0.0398 | Regression_sin_phi: 0.0351 | LR: 1.06e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7777s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1751 | max_backward: 1.9857 | max_optimizer_step: 0.3263 | max_tensorboard_logging: 0.7363 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6249.7 | uss_MB: 5690.9 | child_rss_MB: 4845.9 | allocated_MB: 498.3 | reserved_MB: 28534.0 | inactive_split_MB: 851.5 +rank5 [2026-10-30 05:48:48,523] INFO : Step 47600/50000 rank5 | Train Loss: 0.8663 | Train Loss Unweighted: 0.4076 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1090 | Classification: 0.0820 | Classification_binary: 0.1491 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0865 | Regression_eta: 0.0011 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1272 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0380 | Regression_eta: 0.0364 | Regression_pt: 0.0416 | Regression_sin_phi: 0.0425 | LR: 1.05e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6224s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.9385 | max_loss: 0.1530 | max_backward: 2.1678 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6249.7 | uss_MB: 5691.7 | child_rss_MB: 4853.8 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 843.5 +rank5 [2026-10-30 05:53:19,839] INFO : Step 47700/50000 rank5 | Train Loss: 0.8615 | Train Loss Unweighted: 0.4088 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0954 | Classification: 0.0764 | Classification_binary: 0.1291 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1002 | Regression_eta: 0.0011 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1165 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0437 | Regression_eta: 0.0229 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0449 | LR: 1.05e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2181s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7406 | max_loss: 0.2391 | max_backward: 1.7059 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1444 | Memory: rss_MB: 6248.7 | uss_MB: 5687.5 | child_rss_MB: 4851.8 | allocated_MB: 493.9 | reserved_MB: 28534.0 | inactive_split_MB: 895.3 +rank5 [2026-10-30 05:57:52,842] INFO : Step 47800/50000 rank5 | Train Loss: 0.8267 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.0942 | Classification: 0.0882 | Classification_binary: 0.1186 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0870 | Regression_eta: 0.0012 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1151 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0373 | Regression_eta: 0.0364 | Regression_pt: 0.0421 | Regression_sin_phi: 0.0780 | LR: 1.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.0877s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.7869 | max_loss: 0.5030 | max_backward: 2.0522 | max_optimizer_step: 0.2267 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6249.7 | uss_MB: 5692.3 | child_rss_MB: 4857.3 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 916.9 +rank5 [2026-10-30 06:02:25,107] INFO : Step 47900/50000 rank5 | Train Loss: 0.8423 | Train Loss Unweighted: 0.4000 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.0845 | Classification: 0.0794 | Classification_binary: 0.1416 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1238 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0421 | Regression_eta: 0.0266 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0575 | LR: 1.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5091s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7994 | max_loss: 0.1511 | max_backward: 1.9354 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.6685 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6247.4 | uss_MB: 5692.2 | child_rss_MB: 4860.2 | allocated_MB: 496.8 | reserved_MB: 30642.0 | inactive_split_MB: 779.5 +rank5 [2026-10-30 06:06:56,644] INFO : Step 48000/50000 rank5 | Train Loss: 0.8641 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0972 | Classification: 0.0768 | Classification_binary: 0.1396 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0890 | Regression_eta: 0.0015 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0438 | Regression_eta: 0.0368 | Regression_pt: 0.0476 | Regression_sin_phi: 0.0563 | LR: 1.04e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4074s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0057 | max_forward: 0.8017 | max_loss: 0.1550 | max_backward: 1.6990 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.7498 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6247.1 | uss_MB: 5690.2 | child_rss_MB: 4855.9 | allocated_MB: 495.1 | reserved_MB: 30642.0 | inactive_split_MB: 896.2 +rank5 [2026-10-30 06:06:57,654] INFO : Running validation on rank5 +rank5 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 06:07:49,054] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:07:49,560] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 06:08:14,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 06:08:15,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 06:08:16,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 06:08:17,164] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 06:08:27,174] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 06:08:28,016] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:08:28,620] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 06:08:44,345] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 06:08:45,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 06:08:46,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 06:08:47,232] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 06:08:57,242] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 06:08:58,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:08:58,714] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 06:09:14,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 06:09:15,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 06:09:15,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 06:09:16,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 06:09:26,722] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 06:14:16,584] INFO : Step 48100/50000 rank5 | Train Loss: 0.8226 | Train Loss Unweighted: 0.3895 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.1043 | Classification: 0.0717 | Classification_binary: 0.1116 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1023 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1284 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0486 | Regression_eta: 0.0273 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0517 | LR: 1.03e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4981s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7477 | max_loss: 0.1426 | max_backward: 1.7972 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.6718 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1303 | Memory: rss_MB: 7752.6 | uss_MB: 7191.3 | child_rss_MB: 4880.7 | allocated_MB: 494.1 | reserved_MB: 23078.0 | inactive_split_MB: 1096.3 +rank5 [2026-10-30 06:18:47,187] INFO : Step 48200/50000 rank5 | Train Loss: 0.8500 | Train Loss Unweighted: 0.3835 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1027 | Classification: 0.0771 | Classification_binary: 0.1374 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0827 | Regression_eta: 0.0012 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0436 | Regression_eta: 0.0314 | Regression_pt: 0.0481 | Regression_sin_phi: 0.0367 | LR: 1.03e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2124s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8132 | max_loss: 0.2545 | max_backward: 2.1167 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1241 | Memory: rss_MB: 6310.4 | uss_MB: 5751.6 | child_rss_MB: 4867.3 | allocated_MB: 496.6 | reserved_MB: 25094.0 | inactive_split_MB: 913.4 +rank5 [2026-10-30 06:23:18,412] INFO : Step 48300/50000 rank5 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4303 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1089 | Classification: 0.0832 | Classification_binary: 0.1567 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0013 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0527 | Regression_eta: 0.0383 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0484 | LR: 1.03e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4482s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8079 | max_loss: 0.1526 | max_backward: 1.9733 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6317.4 | uss_MB: 5760.9 | child_rss_MB: 4872.1 | allocated_MB: 498.5 | reserved_MB: 25098.0 | inactive_split_MB: 771.0 +rank5 [2026-10-30 06:27:49,028] INFO : Step 48400/50000 rank5 | Train Loss: 0.7961 | Train Loss Unweighted: 0.3733 | Train Loss Components: Auxiliary: 0.1682 | Cardinality: 0.0877 | Classification: 0.0685 | Classification_binary: 0.1191 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0012 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1203 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0564 | Regression_eta: 0.0212 | Regression_pt: 0.0609 | Regression_sin_phi: 0.0350 | LR: 1.02e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.2396s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7916 | max_loss: 0.1471 | max_backward: 1.6943 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5940 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6314.4 | uss_MB: 5759.4 | child_rss_MB: 4842.4 | allocated_MB: 496.3 | reserved_MB: 25098.0 | inactive_split_MB: 1087.5 +rank5 [2026-10-30 06:32:21,027] INFO : Step 48500/50000 rank5 | Train Loss: 0.8668 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0816 | Classification: 0.0832 | Classification_binary: 0.1257 | HitGrouping: 0.1542 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1508 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0593 | Regression_eta: 0.0502 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0539 | LR: 1.02e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3826s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7587 | max_loss: 0.1606 | max_backward: 1.7573 | max_optimizer_step: 0.3024 | max_tensorboard_logging: 0.5851 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6309.3 | uss_MB: 5759.8 | child_rss_MB: 4851.4 | allocated_MB: 498.9 | reserved_MB: 25098.0 | inactive_split_MB: 1192.4 +rank5 [2026-10-30 06:36:52,986] INFO : Step 48600/50000 rank5 | Train Loss: 0.8434 | Train Loss Unweighted: 0.4010 | Train Loss Components: Auxiliary: 0.1850 | Cardinality: 0.0964 | Classification: 0.0791 | Classification_binary: 0.1425 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1452 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0527 | Regression_eta: 0.0365 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0544 | LR: 1.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4846s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7760 | max_loss: 0.1505 | max_backward: 1.8991 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0654 | Memory: rss_MB: 6311.3 | uss_MB: 5762.6 | child_rss_MB: 4856.0 | allocated_MB: 497.0 | reserved_MB: 26848.0 | inactive_split_MB: 777.5 +rank5 [2026-10-30 06:41:29,683] INFO : Step 48700/50000 rank5 | Train Loss: 0.9079 | Train Loss Unweighted: 0.4266 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1195 | Classification: 0.0920 | Classification_binary: 0.1456 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.1739 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0587 | Regression_eta: 0.0501 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0795 | LR: 1.02e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5446s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7797 | max_loss: 0.1578 | max_backward: 1.7107 | max_optimizer_step: 0.5280 | max_tensorboard_logging: 0.7650 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6308.9 | uss_MB: 5759.1 | child_rss_MB: 4857.7 | allocated_MB: 496.9 | reserved_MB: 26848.0 | inactive_split_MB: 891.8 +rank5 [2026-10-30 06:46:02,599] INFO : Step 48800/50000 rank5 | Train Loss: 0.8011 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1685 | Cardinality: 0.0863 | Classification: 0.0837 | Classification_binary: 0.1116 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0965 | Regression_eta: 0.0012 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1289 | Regression_cos_phi: 0.0428 | Regression_energy: 0.0686 | Regression_eta: 0.0392 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0543 | LR: 1.01e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5001s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7374 | max_loss: 0.2432 | max_backward: 1.8459 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.7427 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6306.9 | uss_MB: 5759.1 | child_rss_MB: 4857.0 | allocated_MB: 493.9 | reserved_MB: 26848.0 | inactive_split_MB: 991.0 +rank5 [2026-10-30 06:50:45,084] INFO : Step 48900/50000 rank5 | Train Loss: 0.8551 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.1109 | Classification: 0.0760 | Classification_binary: 0.1310 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0597 | Regression_eta: 0.0372 | Regression_pt: 0.0649 | Regression_sin_phi: 0.0508 | LR: 1.01e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5409s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8768 | max_loss: 0.1470 | max_backward: 2.0816 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6306.9 | uss_MB: 5761.7 | child_rss_MB: 4858.9 | allocated_MB: 499.5 | reserved_MB: 26852.0 | inactive_split_MB: 1398.0 +rank5 [2026-10-30 06:55:20,573] INFO : Step 49000/50000 rank5 | Train Loss: 0.9005 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.2010 | Cardinality: 0.1248 | Classification: 0.0800 | Classification_binary: 0.1499 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0939 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1485 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0407 | Regression_eta: 0.0376 | Regression_pt: 0.0454 | Regression_sin_phi: 0.0538 | LR: 1.01e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.9025s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0439 | max_forward: 0.8365 | max_loss: 0.1536 | max_backward: 1.9893 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8890 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6309.3 | uss_MB: 5762.6 | child_rss_MB: 4865.2 | allocated_MB: 497.5 | reserved_MB: 26852.0 | inactive_split_MB: 943.6 +rank5 [2026-10-30 06:55:21,669] INFO : Running validation on rank5 +rank5 [2026-10-30 06:55:21,669] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 06:55:21,670] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 06:56:13,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:56:13,863] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 06:56:39,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 06:56:40,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 06:56:41,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 06:56:42,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 06:56:52,256] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 06:56:52,550] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:56:53,028] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 06:57:08,493] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 06:57:09,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 06:57:10,423] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 06:57:11,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 06:57:21,373] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 06:57:22,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 06:57:23,150] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 06:57:38,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 06:57:39,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 06:57:39,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 06:57:40,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 06:57:50,712] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 07:02:45,630] INFO : Step 49100/50000 rank5 | Train Loss: 0.8219 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.0880 | Classification: 0.0841 | Classification_binary: 0.1303 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0855 | Regression_eta: 0.0012 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0494 | Regression_eta: 0.0336 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0524 | LR: 1.01e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3716s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7738 | max_loss: 0.1431 | max_backward: 1.6939 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6276 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 7720.3 | uss_MB: 7170.2 | child_rss_MB: 4829.0 | allocated_MB: 495.9 | reserved_MB: 24730.0 | inactive_split_MB: 936.0 +rank5 [2026-10-30 07:07:18,677] INFO : Step 49200/50000 rank5 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3843 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0856 | Classification: 0.0829 | Classification_binary: 0.1188 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0892 | Regression_eta: 0.0011 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1162 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0559 | Regression_eta: 0.0299 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0546 | LR: 1.01e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7726s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7590 | max_loss: 0.5032 | max_backward: 1.9826 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.7926 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6365.4 | uss_MB: 5817.0 | child_rss_MB: 4830.4 | allocated_MB: 494.5 | reserved_MB: 26752.0 | inactive_split_MB: 1117.5 +rank5 [2026-10-30 07:11:50,862] INFO : Step 49300/50000 rank5 | Train Loss: 0.9397 | Train Loss Unweighted: 0.4282 | Train Loss Components: Auxiliary: 0.2065 | Cardinality: 0.1351 | Classification: 0.0889 | Classification_binary: 0.1552 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0901 | Regression_eta: 0.0012 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0396 | Regression_energy: 0.0443 | Regression_eta: 0.0313 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0494 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4927s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0070 | max_forward: 0.7807 | max_loss: 0.1520 | max_backward: 1.8619 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6076 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6376.6 | uss_MB: 5827.0 | child_rss_MB: 4814.1 | allocated_MB: 496.4 | reserved_MB: 26752.0 | inactive_split_MB: 1120.6 +rank5 [2026-10-30 07:16:22,252] INFO : Step 49400/50000 rank5 | Train Loss: 0.8870 | Train Loss Unweighted: 0.4213 | Train Loss Components: Auxiliary: 0.1937 | Cardinality: 0.1005 | Classification: 0.0815 | Classification_binary: 0.1465 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1596 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0541 | Regression_eta: 0.0465 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0611 | LR: 1.00e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2720s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8046 | max_loss: 0.1540 | max_backward: 1.7244 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.6030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6373.7 | uss_MB: 5825.4 | child_rss_MB: 4813.8 | allocated_MB: 498.5 | reserved_MB: 28804.0 | inactive_split_MB: 886.2 +rank5 [2026-10-30 07:20:55,283] INFO : Step 49500/50000 rank5 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3949 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0991 | Classification: 0.0800 | Classification_binary: 0.1101 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0013 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0367 | Classification_binary: 0.0964 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0462 | Regression_eta: 0.0416 | Regression_pt: 0.0501 | Regression_sin_phi: 0.0635 | LR: 1.00e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.2138s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0074 | max_forward: 0.9924 | max_loss: 0.1865 | max_backward: 2.5853 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.6095 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0429 | Memory: rss_MB: 6377.1 | uss_MB: 5695.1 | child_rss_MB: 4830.9 | allocated_MB: 495.9 | reserved_MB: 28804.0 | inactive_split_MB: 849.6 +rank5 [2026-10-30 07:25:48,132] INFO : Step 49600/50000 rank5 | Train Loss: 0.9075 | Train Loss Unweighted: 0.4421 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1021 | Classification: 0.0939 | Classification_binary: 0.1401 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1214 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0508 | Regression_eta: 0.0298 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0616 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4381s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7852 | max_loss: 0.1445 | max_backward: 2.0967 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4754 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6378.7 | uss_MB: 5829.2 | child_rss_MB: 4808.2 | allocated_MB: 497.8 | reserved_MB: 28804.0 | inactive_split_MB: 845.2 +rank5 [2026-10-30 07:30:32,223] INFO : Step 49700/50000 rank5 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4307 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1074 | Classification: 0.0931 | Classification_binary: 0.1611 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0865 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0426 | Classification_binary: 0.1567 | Regression_cos_phi: 0.0836 | Regression_energy: 0.0416 | Regression_eta: 0.0476 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0586 | LR: 1.00e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6425s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.8169 | max_loss: 0.1522 | max_backward: 2.0553 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.4937 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1402 | Memory: rss_MB: 6379.2 | uss_MB: 5828.6 | child_rss_MB: 4829.3 | allocated_MB: 499.3 | reserved_MB: 30718.0 | inactive_split_MB: 838.8 +rank5 [2026-10-30 07:35:17,062] INFO : Step 49800/50000 rank5 | Train Loss: 0.8342 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.1036 | Classification: 0.0769 | Classification_binary: 0.1292 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1420 | Regression_cos_phi: 0.0811 | Regression_energy: 0.0383 | Regression_eta: 0.0388 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0378 | LR: 1.00e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.1025s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0329 | max_forward: 0.7490 | max_loss: 0.1889 | max_backward: 2.2249 | max_optimizer_step: 0.2452 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6374.9 | uss_MB: 5812.4 | child_rss_MB: 4828.7 | allocated_MB: 494.4 | reserved_MB: 30718.0 | inactive_split_MB: 1134.4 +rank5 [2026-10-30 07:39:58,302] INFO : Step 49900/50000 rank5 | Train Loss: 0.8436 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.0947 | Classification: 0.0943 | Classification_binary: 0.1179 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0982 | Regression_eta: 0.0012 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1361 | Regression_cos_phi: 0.0377 | Regression_energy: 0.0682 | Regression_eta: 0.0245 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0674 | LR: 1.00e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.4159s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7496 | max_loss: 0.5142 | max_backward: 1.9365 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6373.9 | uss_MB: 5825.7 | child_rss_MB: 4831.8 | allocated_MB: 494.8 | reserved_MB: 30718.0 | inactive_split_MB: 1090.4 +rank5 [2026-10-30 07:44:45,887] INFO : Step 50000/50000 rank5 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4040 | Train Loss Components: Auxiliary: 0.1921 | Cardinality: 0.1113 | Classification: 0.0818 | Classification_binary: 0.1436 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0872 | Regression_eta: 0.0014 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0529 | Regression_eta: 0.0358 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0600 | LR: 1.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.6370s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 1.3997 | max_loss: 0.5678 | max_backward: 3.0655 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6373.9 | uss_MB: 5825.4 | child_rss_MB: 4827.6 | allocated_MB: 497.9 | reserved_MB: 30720.0 | inactive_split_MB: 906.1 +rank5 [2026-10-30 07:44:46,920] INFO : Running validation on rank5 +rank5 [2026-10-30 07:44:46,920] INFO : ResumableSampler physical:valid resetting +rank5 [2026-10-30 07:44:46,920] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank5 [2026-10-30 07:45:38,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 07:45:38,998] INFO : Running predictions on cld_edm_ttbar_hits +rank5 [2026-10-30 07:46:04,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_5_0.parquet +rank5 [2026-10-30 07:46:05,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_5_1.parquet +rank5 [2026-10-30 07:46:06,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_5_2.parquet +rank5 [2026-10-30 07:46:07,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_5_3.parquet +rank5 [2026-10-30 07:46:17,383] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 07:46:17,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 07:46:17,995] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank5 [2026-10-30 07:46:33,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_5_0.parquet +rank5 [2026-10-30 07:46:34,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_5_1.parquet +rank5 [2026-10-30 07:46:35,630] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_5_2.parquet +rank5 [2026-10-30 07:46:36,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_5_3.parquet +rank5 [2026-10-30 07:46:46,601] INFO : Time taken to make predictions on device 5 is: 0.25 min +rank5 [2026-10-30 07:46:47,615] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank5 [2026-10-30 07:46:48,154] INFO : Running predictions on cld_edm_qq_hits +rank5 [2026-10-30 07:47:03,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_5_0.parquet +rank5 [2026-10-30 07:47:04,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_5_1.parquet +rank5 [2026-10-30 07:47:04,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_5_2.parquet +rank5 [2026-10-30 07:47:05,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_5_3.parquet +rank5 [2026-10-30 07:47:15,768] INFO : Time taken to make predictions on device 5 is: 0.23 min +rank5 [2026-10-30 07:47:17,249] INFO : Training completed. Total time on device 5: 1017.502min diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.6 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.6 new file mode 100644 index 0000000000000000000000000000000000000000..ecc8bfaec4d05c49141923217eb5a87513a4c316 --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.6 @@ -0,0 +1,2631 @@ +rank6 [2026-10-28 11:12:32,369] INFO : Initializing model with seed=12345; process seed=12351 +rank6 [2026-10-28 11:12:32,370] INFO : configured dtype=torch.bfloat16 for autocast +rank6 [2026-10-28 11:12:54,158] INFO : Instantiating model +rank6 [2026-10-28 11:12:54,158] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank6 [2026-10-28 11:12:54,158] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank6 [2026-10-28 11:12:54,177] INFO : Input encoding initialization took 0.02s +rank6 [2026-10-28 11:12:54,177] INFO : nn0 parameters: 171776 +rank6 [2026-10-28 11:12:54,177] INFO : Initializing shared backbone layers, num_convs=6 +rank6 [2026-10-28 11:12:54,177] INFO : Initializing backbone layer 0 +rank6 [2026-10-28 11:12:54,212] INFO : Initializing backbone layer 1 +rank6 [2026-10-28 11:12:54,221] INFO : Initializing backbone layer 2 +rank6 [2026-10-28 11:12:54,229] INFO : Initializing backbone layer 3 +rank6 [2026-10-28 11:12:54,238] INFO : Initializing backbone layer 4 +rank6 [2026-10-28 11:12:54,246] INFO : Initializing backbone layer 5 +rank6 [2026-10-28 11:12:54,255] INFO : Backbone initialization took 0.08s +rank6 [2026-10-28 11:12:54,255] INFO : backbone parameters: 7901376 +rank6 [2026-10-28 11:12:54,255] INFO : Initializing output DNNs +rank6 [2026-10-28 11:12:54,291] INFO : Output DNNs initialization took 0.04s +rank6 [2026-10-28 11:12:54,292] INFO : backbone_mode=BackboneMode.SHARED +rank6 [2026-10-28 11:12:54,292] INFO : jagged_attention enabled=False +rank6 [2026-10-28 11:12:54,292] INFO : flash_attn_varlen enabled=False +rank6 [2026-10-28 11:12:54,292] INFO : task_queries enabled=False +rank6 [2026-10-28 11:12:54,292] INFO : classification_norm parameters: 0 +rank6 [2026-10-28 11:12:54,292] INFO : regression_norm parameters: 0 +rank6 [2026-10-28 11:12:54,292] INFO : classification_query parameters: 0 +rank6 [2026-10-28 11:12:54,292] INFO : regression_query parameters: 0 +rank6 [2026-10-28 11:12:54,292] INFO : classification_readout parameters: 0 +rank6 [2026-10-28 11:12:54,292] INFO : regression_readout parameters: 0 +rank6 [2026-10-28 11:12:54,292] INFO : output_mode=set +rank6 [2026-10-28 11:12:54,292] INFO : set_decoder parameters: 4713812 +rank6 [2026-10-28 11:12:54,292] INFO : nn_binary_particle parameters: 0 +rank6 [2026-10-28 11:12:54,292] INFO : nn_pid parameters: 0 +rank6 [2026-10-28 11:12:54,293] INFO : nn_pt parameters: 0 +rank6 [2026-10-28 11:12:54,293] INFO : nn_eta parameters: 0 +rank6 [2026-10-28 11:12:54,293] INFO : nn_sin_phi parameters: 0 +rank6 [2026-10-28 11:12:54,293] INFO : nn_cos_phi parameters: 0 +rank6 [2026-10-28 11:12:54,293] INFO : nn_energy parameters: 0 +rank6 [2026-10-28 11:12:54,293] INFO : Total MLPF parameters: 12786964 +rank6 [2026-10-28 11:12:54,293] INFO : MLPF __init__ done +rank6 [2026-10-28 11:12:54,294] INFO : Instantiated model +rank6 [2026-10-28 11:12:54,294] INFO : Moving model to device rank=6 +rank6 [2026-10-28 11:12:54,335] INFO : Moved model to device rank=6 +rank6 [2026-10-28 11:12:54,339] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank6 [2026-10-28 11:12:54,340] INFO : Configured model for SyncBatchNorm rank=6 +rank6 [2026-10-28 11:12:54,649] INFO : Configured model for DistributedDataParallel rank=6 +rank6 [2026-10-28 11:12:54,810] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank6 [2026-10-28 11:12:54,810] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank6 [2026-10-28 11:12:54,811] INFO : Trainable parameters: 12786964 +rank6 [2026-10-28 11:12:54,811] INFO : Non-trainable parameters: 23268 +rank6 [2026-10-28 11:12:54,811] INFO : Total parameters: 12810232 +rank6 [2026-10-28 11:12:54,814] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank6 [2026-10-28 11:12:54,814] INFO : Getting dataloaders +rank6 [2026-10-28 11:12:54,814] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:03,737] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:04,811] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,156] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-28 11:13:06,157] INFO : Creating WeightedMixIterator with 1 data loaders. +rank6 [2026-10-28 11:13:06,157] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,428] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,686] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:13:06,943] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-28 11:13:06,944] INFO : Creating InterleavedIterator with 1 data loaders. +rank6 [2026-10-28 11:13:21,087] INFO : Got dataloaders +rank6 [2026-10-28 11:13:21,087] INFO : loader split=train rank=6 len=21112 +rank6 [2026-10-28 11:13:21,087] INFO : loader split=valid rank=6 len=12 +rank6 [2026-10-28 11:13:21,087] INFO : Starting training from step 1 to 50000 on rank 6 of 8 +rank6 [2026-10-28 11:13:21,088] INFO : Creating train_iterator +rank6 [2026-10-28 11:13:21,088] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-28 11:13:35,322] INFO : Created train_iterator +rank6 [2026-10-28 11:18:11,208] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5408100029081107, 'Classification': 0.8863177470117808, 'Regression_pt': 0.9386234817653895, 'Regression_eta': 0.019238746587652712, 'Regression_sin_phi': 0.006576384851359762, 'Regression_cos_phi': 0.006405904629500583, 'Regression_energy': 1.0145655398070812}, weights={'Classification_binary': 1.0, 'Classification': 0.6101762056350708, 'Regression_pt': 0.5761735439300537, 'Regression_eta': 28.110458374023438, 'Regression_sin_phi': 82.23515319824219, 'Regression_cos_phi': 84.42367553710938, 'Regression_energy': 0.5330458879470825} +rank6 [2026-10-28 11:18:13,718] INFO : Step 100/50000 rank6 | Train Loss: 4.2631 | Train Loss Unweighted: 2.7366 | Train Loss Components: Auxiliary: 0.7941 | Cardinality: 0.3942 | Classification: 0.6312 | Classification_binary: 0.4549 | HitGrouping: 0.3381 | Regression_cos_phi: 0.0047 | Regression_energy: 0.8263 | Regression_eta: 0.0122 | Regression_pt: 0.8026 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.6182 | Classification_binary: 0.4487 | Regression_cos_phi: 0.0052 | Regression_energy: 0.7958 | Regression_eta: 0.0156 | Regression_pt: 0.7844 | Regression_sin_phi: 0.0049 | LR: 1.00e-03 | DataLoad Time: 0.0165s | Model Forward Time: 3.5368s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6509 | max_loss: 0.1961 | max_backward: 1.4166 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4769.1 | uss_MB: 3475.5 | child_rss_MB: 6372.1 | allocated_MB: 446.7 | reserved_MB: 24804.0 | inactive_split_MB: 1063.6 +rank6 [2026-10-28 11:22:42,537] INFO : Step 200/50000 rank6 | Train Loss: 4.2029 | Train Loss Unweighted: 2.5447 | Train Loss Components: Auxiliary: 0.8074 | Cardinality: 0.4500 | Classification: 0.5373 | Classification_binary: 0.5135 | HitGrouping: 0.3101 | Regression_cos_phi: 0.0042 | Regression_energy: 0.7508 | Regression_eta: 0.0093 | Regression_pt: 0.7252 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.3404 | Classification_binary: 0.5128 | Regression_cos_phi: 0.3447 | Regression_energy: 0.3770 | Regression_eta: 0.2966 | Regression_pt: 0.3910 | Regression_sin_phi: 0.3244 | LR: 1.00e-03 | DataLoad Time: 0.0164s | Model Forward Time: 4.6614s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9214 | max_loss: 0.1502 | max_backward: 2.1220 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 4772.0 | uss_MB: 3480.3 | child_rss_MB: 6374.7 | allocated_MB: 449.0 | reserved_MB: 26252.0 | inactive_split_MB: 1054.4 +rank6 [2026-10-28 11:27:10,390] INFO : Step 300/50000 rank6 | Train Loss: 3.5114 | Train Loss Unweighted: 2.0975 | Train Loss Components: Auxiliary: 0.6791 | Cardinality: 0.3510 | Classification: 0.4201 | Classification_binary: 0.4975 | HitGrouping: 0.2993 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5752 | Regression_eta: 0.0073 | Regression_pt: 0.5905 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2140 | Classification_binary: 0.5492 | Regression_cos_phi: 0.2950 | Regression_energy: 0.3380 | Regression_eta: 0.2240 | Regression_pt: 0.3755 | Regression_sin_phi: 0.3089 | LR: 1.00e-03 | DataLoad Time: 0.0163s | Model Forward Time: 4.2596s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.7369 | max_loss: 0.1526 | max_backward: 1.7330 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.9030 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 4773.0 | uss_MB: 3484.3 | child_rss_MB: 6364.6 | allocated_MB: 446.1 | reserved_MB: 26252.0 | inactive_split_MB: 1068.0 +rank6 [2026-10-28 11:31:37,117] INFO : Step 400/50000 rank6 | Train Loss: 3.3615 | Train Loss Unweighted: 1.9456 | Train Loss Components: Auxiliary: 0.6310 | Cardinality: 0.3555 | Classification: 0.4058 | Classification_binary: 0.4436 | HitGrouping: 0.3006 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5363 | Regression_eta: 0.0080 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2505 | Classification_binary: 0.4475 | Regression_cos_phi: 0.2683 | Regression_energy: 0.2533 | Regression_eta: 0.2292 | Regression_pt: 0.2852 | Regression_sin_phi: 0.2526 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 4.5041s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1151 | max_forward: 0.7642 | max_loss: 0.1384 | max_backward: 1.7906 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9359 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0880 | Memory: rss_MB: 4771.8 | uss_MB: 3484.8 | child_rss_MB: 6366.3 | allocated_MB: 446.8 | reserved_MB: 26252.0 | inactive_split_MB: 1064.0 +rank6 [2026-10-28 11:36:08,659] INFO : Step 500/50000 rank6 | Train Loss: 3.8257 | Train Loss Unweighted: 2.0630 | Train Loss Components: Auxiliary: 0.7010 | Cardinality: 0.4740 | Classification: 0.4380 | Classification_binary: 0.4817 | HitGrouping: 0.2961 | Regression_cos_phi: 0.0044 | Regression_energy: 0.5611 | Regression_eta: 0.0087 | Regression_pt: 0.5645 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2671 | Classification_binary: 0.4535 | Regression_cos_phi: 0.3786 | Regression_energy: 0.2873 | Regression_eta: 0.1953 | Regression_pt: 0.3175 | Regression_sin_phi: 0.2746 | LR: 1.00e-03 | DataLoad Time: 0.0168s | Model Forward Time: 5.1247s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0077 | max_forward: 1.2433 | max_loss: 0.1536 | max_backward: 2.4885 | max_optimizer_step: 0.1562 | max_tensorboard_logging: 1.0374 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0034 | Memory: rss_MB: 4768.9 | uss_MB: 3486.1 | child_rss_MB: 6361.5 | allocated_MB: 449.6 | reserved_MB: 26256.0 | inactive_split_MB: 929.2 +rank6 [2026-10-28 11:40:42,093] INFO : Step 600/50000 rank6 | Train Loss: 3.4702 | Train Loss Unweighted: 1.9733 | Train Loss Components: Auxiliary: 0.6509 | Cardinality: 0.3585 | Classification: 0.3825 | Classification_binary: 0.5061 | HitGrouping: 0.2915 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5321 | Regression_eta: 0.0074 | Regression_pt: 0.5377 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2150 | Classification_binary: 0.4980 | Regression_cos_phi: 0.2716 | Regression_energy: 0.2747 | Regression_eta: 0.1957 | Regression_pt: 0.3026 | Regression_sin_phi: 0.2064 | LR: 1.00e-03 | DataLoad Time: 0.0139s | Model Forward Time: 4.5594s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8098 | max_loss: 0.1544 | max_backward: 1.9541 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 0.9377 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 4772.7 | uss_MB: 3486.1 | child_rss_MB: 6366.7 | allocated_MB: 449.5 | reserved_MB: 27396.0 | inactive_split_MB: 1022.5 +rank6 [2026-10-28 11:45:16,605] INFO : Step 700/50000 rank6 | Train Loss: 2.9693 | Train Loss Unweighted: 1.8072 | Train Loss Components: Auxiliary: 0.5907 | Cardinality: 0.2768 | Classification: 0.3175 | Classification_binary: 0.3969 | HitGrouping: 0.2848 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5321 | Regression_eta: 0.0061 | Regression_pt: 0.5492 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2107 | Classification_binary: 0.4134 | Regression_cos_phi: 0.2420 | Regression_energy: 0.2644 | Regression_eta: 0.1861 | Regression_pt: 0.2865 | Regression_sin_phi: 0.2708 | LR: 1.00e-03 | DataLoad Time: 0.0219s | Model Forward Time: 3.9465s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7910 | max_loss: 0.2436 | max_backward: 1.8672 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.8 | uss_MB: 3484.3 | child_rss_MB: 6369.5 | allocated_MB: 446.1 | reserved_MB: 27396.0 | inactive_split_MB: 1066.2 +rank6 [2026-10-28 11:49:48,175] INFO : Step 800/50000 rank6 | Train Loss: 3.1527 | Train Loss Unweighted: 1.8424 | Train Loss Components: Auxiliary: 0.6214 | Cardinality: 0.3518 | Classification: 0.3758 | Classification_binary: 0.3967 | HitGrouping: 0.2962 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5290 | Regression_eta: 0.0063 | Regression_pt: 0.5287 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2423 | Classification_binary: 0.3955 | Regression_cos_phi: 0.2345 | Regression_energy: 0.2750 | Regression_eta: 0.1894 | Regression_pt: 0.2882 | Regression_sin_phi: 0.2693 | LR: 9.99e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.6442s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0093 | max_forward: 0.8507 | max_loss: 0.1493 | max_backward: 1.9706 | max_optimizer_step: 0.1582 | max_tensorboard_logging: 1.0840 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0072 | Memory: rss_MB: 4754.0 | uss_MB: 3484.3 | child_rss_MB: 6293.3 | allocated_MB: 450.9 | reserved_MB: 27396.0 | inactive_split_MB: 920.9 +rank6 [2026-10-28 11:54:30,570] INFO : Step 900/50000 rank6 | Train Loss: 3.2552 | Train Loss Unweighted: 1.9216 | Train Loss Components: Auxiliary: 0.6208 | Cardinality: 0.2901 | Classification: 0.3818 | Classification_binary: 0.4734 | HitGrouping: 0.2881 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5239 | Regression_eta: 0.0075 | Regression_pt: 0.5284 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2400 | Classification_binary: 0.4653 | Regression_cos_phi: 0.3004 | Regression_energy: 0.3018 | Regression_eta: 0.1950 | Regression_pt: 0.3316 | Regression_sin_phi: 0.2576 | LR: 9.99e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6737s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8050 | max_loss: 0.1486 | max_backward: 1.9075 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.9729 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4751.7 | uss_MB: 3485.4 | child_rss_MB: 6307.4 | allocated_MB: 449.7 | reserved_MB: 27406.0 | inactive_split_MB: 927.3 +rank6 [2026-10-28 11:59:03,649] INFO : Step 1000/50000 rank6 | Train Loss: 2.8964 | Train Loss Unweighted: 1.7179 | Train Loss Components: Auxiliary: 0.5694 | Cardinality: 0.2509 | Classification: 0.3299 | Classification_binary: 0.4129 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4804 | Regression_eta: 0.0063 | Regression_pt: 0.4827 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2104 | Classification_binary: 0.3872 | Regression_cos_phi: 0.1708 | Regression_energy: 0.2505 | Regression_eta: 0.1242 | Regression_pt: 0.2740 | Regression_sin_phi: 0.1943 | LR: 9.99e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4509s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7367 | max_loss: 0.2419 | max_backward: 1.8137 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 1.0408 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 4746.9 | uss_MB: 3485.7 | child_rss_MB: 6281.8 | allocated_MB: 449.1 | reserved_MB: 27406.0 | inactive_split_MB: 959.2 +rank6 [2026-10-28 11:59:04,553] INFO : Running validation on rank6 +rank6 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 11:59:57,025] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 11:59:57,577] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 12:00:26,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 12:00:27,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 12:00:29,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 12:00:30,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 12:00:40,041] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 12:00:40,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:00:41,018] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 12:00:57,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 12:00:58,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 12:00:59,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 12:01:00,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 12:01:10,315] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 12:01:10,611] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:01:11,132] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 12:01:26,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 12:01:27,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 12:01:28,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 12:01:29,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 12:01:39,065] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 12:06:47,466] INFO : Step 1100/50000 rank6 | Train Loss: 2.7351 | Train Loss Unweighted: 1.6531 | Train Loss Components: Auxiliary: 0.5524 | Cardinality: 0.2470 | Classification: 0.3238 | Classification_binary: 0.3516 | HitGrouping: 0.2744 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4831 | Regression_eta: 0.0054 | Regression_pt: 0.4842 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1862 | Classification_binary: 0.3867 | Regression_cos_phi: 0.2282 | Regression_energy: 0.2537 | Regression_eta: 0.1794 | Regression_pt: 0.2707 | Regression_sin_phi: 0.2201 | LR: 9.99e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7305s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7345 | max_loss: 0.2408 | max_backward: 1.8906 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 1.1618 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6205.6 | uss_MB: 4949.9 | child_rss_MB: 6308.4 | allocated_MB: 448.2 | reserved_MB: 24590.0 | inactive_split_MB: 1051.2 +rank6 [2026-10-28 12:11:28,614] INFO : Step 1200/50000 rank6 | Train Loss: 2.9931 | Train Loss Unweighted: 1.7517 | Train Loss Components: Auxiliary: 0.5886 | Cardinality: 0.2847 | Classification: 0.3460 | Classification_binary: 0.4151 | HitGrouping: 0.2795 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4874 | Regression_eta: 0.0065 | Regression_pt: 0.4908 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2492 | Classification_binary: 0.4077 | Regression_cos_phi: 0.2913 | Regression_energy: 0.2422 | Regression_eta: 0.2121 | Regression_pt: 0.2602 | Regression_sin_phi: 0.2510 | LR: 9.99e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.9388s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0061 | max_forward: 0.9910 | max_loss: 0.1662 | max_backward: 2.3256 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.8355 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 6205.6 | uss_MB: 4943.2 | child_rss_MB: 6333.1 | allocated_MB: 449.6 | reserved_MB: 24590.0 | inactive_split_MB: 1155.8 +rank6 [2026-10-28 12:16:08,306] INFO : Step 1300/50000 rank6 | Train Loss: 3.0304 | Train Loss Unweighted: 1.7607 | Train Loss Components: Auxiliary: 0.5900 | Cardinality: 0.2909 | Classification: 0.3397 | Classification_binary: 0.4400 | HitGrouping: 0.2683 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4807 | Regression_eta: 0.0062 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2231 | Classification_binary: 0.4677 | Regression_cos_phi: 0.2736 | Regression_energy: 0.2674 | Regression_eta: 0.1771 | Regression_pt: 0.2947 | Regression_sin_phi: 0.3165 | LR: 9.98e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3301s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8206 | max_loss: 0.1504 | max_backward: 1.9515 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5688 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5595.4 | uss_MB: 4349.7 | child_rss_MB: 6286.2 | allocated_MB: 448.6 | reserved_MB: 25778.0 | inactive_split_MB: 1050.0 +rank6 [2026-10-28 12:20:48,119] INFO : Step 1400/50000 rank6 | Train Loss: 2.5732 | Train Loss Unweighted: 1.5137 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2349 | Classification: 0.2906 | Classification_binary: 0.3573 | HitGrouping: 0.2641 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4252 | Regression_eta: 0.0055 | Regression_pt: 0.4303 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1745 | Classification_binary: 0.4134 | Regression_cos_phi: 0.2258 | Regression_energy: 0.2258 | Regression_eta: 0.1541 | Regression_pt: 0.2477 | Regression_sin_phi: 0.1956 | LR: 9.98e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.1758s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.8166 | max_loss: 0.1962 | max_backward: 1.8388 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4964 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5588.2 | uss_MB: 3765.0 | child_rss_MB: 6275.0 | allocated_MB: 454.5 | reserved_MB: 28272.0 | inactive_split_MB: 1143.7 +rank6 [2026-10-28 12:25:37,857] INFO : Step 1500/50000 rank6 | Train Loss: 2.8564 | Train Loss Unweighted: 1.5902 | Train Loss Components: Auxiliary: 0.5576 | Cardinality: 0.3788 | Classification: 0.3167 | Classification_binary: 0.3534 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4517 | Regression_eta: 0.0055 | Regression_pt: 0.4576 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2081 | Classification_binary: 0.3478 | Regression_cos_phi: 0.1927 | Regression_energy: 0.2522 | Regression_eta: 0.1676 | Regression_pt: 0.2723 | Regression_sin_phi: 0.1978 | LR: 9.98e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6155s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.0272 | max_loss: 0.2459 | max_backward: 1.8682 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.0883 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0105 | Memory: rss_MB: 5566.9 | uss_MB: 4329.3 | child_rss_MB: 6244.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 989.3 +rank6 [2026-10-28 12:30:20,921] INFO : Step 1600/50000 rank6 | Train Loss: 2.8293 | Train Loss Unweighted: 1.6754 | Train Loss Components: Auxiliary: 0.5537 | Cardinality: 0.2456 | Classification: 0.3224 | Classification_binary: 0.4168 | HitGrouping: 0.2726 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4572 | Regression_eta: 0.0058 | Regression_pt: 0.4676 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2060 | Classification_binary: 0.3956 | Regression_cos_phi: 0.2143 | Regression_energy: 0.2596 | Regression_eta: 0.1650 | Regression_pt: 0.2885 | Regression_sin_phi: 0.2112 | LR: 9.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5688s | Phase Times: max_pre_step_barrier: 0.1094 | max_batch_to_device: 0.0066 | max_forward: 0.7720 | max_loss: 0.1510 | max_backward: 1.7462 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5578.6 | uss_MB: 4348.9 | child_rss_MB: 6226.8 | allocated_MB: 448.1 | reserved_MB: 28272.0 | inactive_split_MB: 989.9 +rank6 [2026-10-28 12:35:04,110] INFO : Step 1700/50000 rank6 | Train Loss: 2.7295 | Train Loss Unweighted: 1.5885 | Train Loss Components: Auxiliary: 0.5261 | Cardinality: 0.2505 | Classification: 0.2942 | Classification_binary: 0.3852 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4488 | Regression_eta: 0.0059 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1697 | Classification_binary: 0.3679 | Regression_cos_phi: 0.1828 | Regression_energy: 0.2583 | Regression_eta: 0.1396 | Regression_pt: 0.2774 | Regression_sin_phi: 0.1531 | LR: 9.97e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9345s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 1.0304 | max_loss: 0.4846 | max_backward: 1.9957 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0278 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5571.8 | uss_MB: 4349.5 | child_rss_MB: 6226.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 999.4 +rank6 [2026-10-28 12:39:46,515] INFO : Step 1800/50000 rank6 | Train Loss: 2.5541 | Train Loss Unweighted: 1.4989 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2504 | Classification: 0.2799 | Classification_binary: 0.3357 | HitGrouping: 0.2658 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4350 | Regression_eta: 0.0052 | Regression_pt: 0.4384 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1751 | Classification_binary: 0.3288 | Regression_cos_phi: 0.2747 | Regression_energy: 0.2313 | Regression_eta: 0.1513 | Regression_pt: 0.2589 | Regression_sin_phi: 0.2186 | LR: 9.97e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3583s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7316 | max_loss: 0.1436 | max_backward: 1.6570 | max_optimizer_step: 0.2625 | max_tensorboard_logging: 0.9220 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5563.3 | uss_MB: 4348.3 | child_rss_MB: 6170.2 | allocated_MB: 444.7 | reserved_MB: 28272.0 | inactive_split_MB: 991.9 +rank6 [2026-10-28 12:44:29,073] INFO : Step 1900/50000 rank6 | Train Loss: 2.7201 | Train Loss Unweighted: 1.5780 | Train Loss Components: Auxiliary: 0.5257 | Cardinality: 0.2710 | Classification: 0.3152 | Classification_binary: 0.3657 | HitGrouping: 0.2653 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4371 | Regression_eta: 0.0058 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1733 | Classification_binary: 0.3688 | Regression_cos_phi: 0.1976 | Regression_energy: 0.2630 | Regression_eta: 0.1513 | Regression_pt: 0.2924 | Regression_sin_phi: 0.2208 | LR: 9.97e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4050s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.7723 | max_loss: 0.1448 | max_backward: 2.0316 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5561.2 | uss_MB: 4335.4 | child_rss_MB: 6153.7 | allocated_MB: 448.9 | reserved_MB: 28272.0 | inactive_split_MB: 1166.2 +rank6 [2026-10-28 12:49:11,971] INFO : Step 2000/50000 rank6 | Train Loss: 2.7893 | Train Loss Unweighted: 1.5772 | Train Loss Components: Auxiliary: 0.5342 | Cardinality: 0.3149 | Classification: 0.3056 | Classification_binary: 0.4233 | HitGrouping: 0.2623 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4169 | Regression_eta: 0.0058 | Regression_pt: 0.4204 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2207 | Classification_binary: 0.4517 | Regression_cos_phi: 0.2607 | Regression_energy: 0.2168 | Regression_eta: 0.2144 | Regression_pt: 0.2321 | Regression_sin_phi: 0.2295 | LR: 9.96e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.5190s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0070 | max_forward: 1.0424 | max_loss: 0.2160 | max_backward: 2.3184 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.9849 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0132 | Memory: rss_MB: 5555.4 | uss_MB: 4316.1 | child_rss_MB: 6148.5 | allocated_MB: 451.3 | reserved_MB: 28272.0 | inactive_split_MB: 992.4 +rank6 [2026-10-28 12:49:12,930] INFO : Running validation on rank6 +rank6 [2026-10-28 12:49:12,930] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 12:49:12,930] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 12:50:07,790] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:50:08,359] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 12:50:34,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 12:50:35,283] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 12:50:36,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 12:50:37,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 12:50:47,445] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 12:50:48,335] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:50:48,904] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 12:51:04,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 12:51:05,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 12:51:06,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 12:51:07,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 12:51:17,493] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 12:51:18,315] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 12:51:18,824] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 12:51:35,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 12:51:35,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 12:51:36,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 12:51:37,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 12:51:47,486] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 12:57:01,909] INFO : Step 2100/50000 rank6 | Train Loss: 2.3990 | Train Loss Unweighted: 1.4049 | Train Loss Components: Auxiliary: 0.4671 | Cardinality: 0.2199 | Classification: 0.2491 | Classification_binary: 0.3399 | HitGrouping: 0.2605 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4015 | Regression_eta: 0.0046 | Regression_pt: 0.4053 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1580 | Classification_binary: 0.3703 | Regression_cos_phi: 0.1683 | Regression_energy: 0.2104 | Regression_eta: 0.1257 | Regression_pt: 0.2283 | Regression_sin_phi: 0.1713 | LR: 9.96e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8160s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7238 | max_loss: 0.1490 | max_backward: 1.8098 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.2800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6608.5 | uss_MB: 5401.4 | child_rss_MB: 6187.1 | allocated_MB: 444.8 | reserved_MB: 23654.0 | inactive_split_MB: 964.6 +rank6 [2026-10-28 13:01:41,080] INFO : Step 2200/50000 rank6 | Train Loss: 2.6105 | Train Loss Unweighted: 1.4907 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2489 | Classification: 0.2976 | Classification_binary: 0.3552 | HitGrouping: 0.2519 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4108 | Regression_eta: 0.0054 | Regression_pt: 0.4161 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1905 | Classification_binary: 0.3598 | Regression_cos_phi: 0.2345 | Regression_energy: 0.1839 | Regression_eta: 0.2120 | Regression_pt: 0.1997 | Regression_sin_phi: 0.2344 | LR: 9.96e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9417s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 1.0006 | max_loss: 0.1446 | max_backward: 1.7671 | max_optimizer_step: 0.2923 | max_tensorboard_logging: 1.2733 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6609.6 | uss_MB: 5403.3 | child_rss_MB: 6178.9 | allocated_MB: 445.7 | reserved_MB: 24780.0 | inactive_split_MB: 1132.5 +rank6 [2026-10-28 13:06:12,757] INFO : Step 2300/50000 rank6 | Train Loss: 2.7462 | Train Loss Unweighted: 1.5643 | Train Loss Components: Auxiliary: 0.5175 | Cardinality: 0.2724 | Classification: 0.3064 | Classification_binary: 0.4106 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4145 | Regression_eta: 0.0056 | Regression_pt: 0.4213 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2218 | Classification_binary: 0.4009 | Regression_cos_phi: 0.3557 | Regression_energy: 0.2376 | Regression_eta: 0.1743 | Regression_pt: 0.2619 | Regression_sin_phi: 0.2587 | LR: 9.95e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7163s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0353 | max_forward: 0.7930 | max_loss: 0.1572 | max_backward: 1.9535 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0140 | Memory: rss_MB: 5702.4 | uss_MB: 4498.0 | child_rss_MB: 6178.9 | allocated_MB: 451.2 | reserved_MB: 26626.0 | inactive_split_MB: 1036.8 +rank6 [2026-10-28 13:10:49,216] INFO : Step 2400/50000 rank6 | Train Loss: 2.5721 | Train Loss Unweighted: 1.4496 | Train Loss Components: Auxiliary: 0.4832 | Cardinality: 0.2667 | Classification: 0.2677 | Classification_binary: 0.3686 | HitGrouping: 0.2521 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3975 | Regression_eta: 0.0053 | Regression_pt: 0.4051 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1868 | Classification_binary: 0.3996 | Regression_cos_phi: 0.2711 | Regression_energy: 0.2316 | Regression_eta: 0.1652 | Regression_pt: 0.2556 | Regression_sin_phi: 0.2575 | LR: 9.95e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.0725s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7367 | max_loss: 0.1564 | max_backward: 1.6902 | max_optimizer_step: 0.2413 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5703.1 | uss_MB: 4496.2 | child_rss_MB: 6167.5 | allocated_MB: 448.4 | reserved_MB: 26626.0 | inactive_split_MB: 1138.1 +rank6 [2026-10-28 13:15:33,528] INFO : Step 2500/50000 rank6 | Train Loss: 2.3709 | Train Loss Unweighted: 1.3478 | Train Loss Components: Auxiliary: 0.4634 | Cardinality: 0.2742 | Classification: 0.2528 | Classification_binary: 0.3100 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3861 | Regression_eta: 0.0042 | Regression_pt: 0.3903 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1604 | Classification_binary: 0.3253 | Regression_cos_phi: 0.2918 | Regression_energy: 0.2102 | Regression_eta: 0.1372 | Regression_pt: 0.2337 | Regression_sin_phi: 0.2440 | LR: 9.94e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.3385s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7598 | max_loss: 0.2442 | max_backward: 1.9617 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5783 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5689.7 | uss_MB: 4483.6 | child_rss_MB: 6173.2 | allocated_MB: 447.2 | reserved_MB: 27052.0 | inactive_split_MB: 1262.0 +rank6 [2026-10-28 13:20:26,045] INFO : Step 2600/50000 rank6 | Train Loss: 2.6216 | Train Loss Unweighted: 1.5111 | Train Loss Components: Auxiliary: 0.4988 | Cardinality: 0.2823 | Classification: 0.2863 | Classification_binary: 0.3723 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4174 | Regression_eta: 0.0052 | Regression_pt: 0.4247 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1623 | Classification_binary: 0.3563 | Regression_cos_phi: 0.2270 | Regression_energy: 0.2109 | Regression_eta: 0.1506 | Regression_pt: 0.2327 | Regression_sin_phi: 0.2069 | LR: 9.94e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2876s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8125 | max_loss: 0.1494 | max_backward: 1.9790 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5698.8 | uss_MB: 4500.2 | child_rss_MB: 6171.1 | allocated_MB: 448.2 | reserved_MB: 28734.0 | inactive_split_MB: 1300.2 +rank6 [2026-10-28 13:25:15,883] INFO : Step 2700/50000 rank6 | Train Loss: 2.5735 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4801 | Cardinality: 0.2457 | Classification: 0.2692 | Classification_binary: 0.4021 | HitGrouping: 0.2514 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3948 | Regression_eta: 0.0060 | Regression_pt: 0.3979 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1859 | Classification_binary: 0.4311 | Regression_cos_phi: 0.2376 | Regression_energy: 0.2159 | Regression_eta: 0.1850 | Regression_pt: 0.2349 | Regression_sin_phi: 0.2496 | LR: 9.94e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.3221s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0070 | max_forward: 0.8435 | max_loss: 0.1556 | max_backward: 1.9409 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5695.7 | uss_MB: 4500.5 | child_rss_MB: 6182.0 | allocated_MB: 447.4 | reserved_MB: 28734.0 | inactive_split_MB: 980.7 +rank6 [2026-10-28 13:29:49,852] INFO : Step 2800/50000 rank6 | Train Loss: 2.3517 | Train Loss Unweighted: 1.3315 | Train Loss Components: Auxiliary: 0.4396 | Cardinality: 0.2218 | Classification: 0.2269 | Classification_binary: 0.3206 | HitGrouping: 0.2464 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3829 | Regression_eta: 0.0052 | Regression_pt: 0.3909 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1582 | Classification_binary: 0.3759 | Regression_cos_phi: 0.2302 | Regression_energy: 0.2087 | Regression_eta: 0.2090 | Regression_pt: 0.2288 | Regression_sin_phi: 0.2288 | LR: 9.93e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.3199s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0062 | max_forward: 0.8898 | max_loss: 0.1952 | max_backward: 1.8882 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0163 | Memory: rss_MB: 5687.7 | uss_MB: 4493.8 | child_rss_MB: 6179.6 | allocated_MB: 445.9 | reserved_MB: 28734.0 | inactive_split_MB: 1208.5 +rank6 [2026-10-28 13:34:34,297] INFO : Step 2900/50000 rank6 | Train Loss: 2.4505 | Train Loss Unweighted: 1.3674 | Train Loss Components: Auxiliary: 0.4603 | Cardinality: 0.2746 | Classification: 0.2611 | Classification_binary: 0.3388 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3750 | Regression_eta: 0.0049 | Regression_pt: 0.3827 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1794 | Classification_binary: 0.3447 | Regression_cos_phi: 0.2470 | Regression_energy: 0.1811 | Regression_eta: 0.1415 | Regression_pt: 0.2005 | Regression_sin_phi: 0.1990 | LR: 9.93e-04 | DataLoad Time: 0.0177s | Model Forward Time: 6.3928s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7484 | max_loss: 0.6089 | max_backward: 2.2353 | max_optimizer_step: 0.1775 | max_tensorboard_logging: 2.4313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0155 | Memory: rss_MB: 5691.8 | uss_MB: 4497.3 | child_rss_MB: 6175.9 | allocated_MB: 446.7 | reserved_MB: 28734.0 | inactive_split_MB: 1133.2 +rank6 [2026-10-28 13:39:22,258] INFO : Step 3000/50000 rank6 | Train Loss: 2.6179 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4760 | Cardinality: 0.2600 | Classification: 0.3170 | Classification_binary: 0.3907 | HitGrouping: 0.2509 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3743 | Regression_eta: 0.0054 | Regression_pt: 0.3819 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1725 | Classification_binary: 0.3503 | Regression_cos_phi: 0.2231 | Regression_energy: 0.2129 | Regression_eta: 0.1443 | Regression_pt: 0.2372 | Regression_sin_phi: 0.2408 | LR: 9.92e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8193s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0842 | max_forward: 0.8203 | max_loss: 0.1511 | max_backward: 1.9714 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.0307 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5691.4 | uss_MB: 4499.3 | child_rss_MB: 6154.8 | allocated_MB: 451.4 | reserved_MB: 28740.0 | inactive_split_MB: 1256.0 +rank6 [2026-10-28 13:39:23,129] INFO : Running validation on rank6 +rank6 [2026-10-28 13:39:23,129] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 13:39:23,129] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 13:40:15,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:40:16,490] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 13:40:44,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 13:40:45,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 13:40:46,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 13:40:47,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 13:40:57,300] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:40:57,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:40:58,516] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 13:41:14,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 13:41:15,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 13:41:16,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 13:41:17,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 13:41:27,392] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 13:41:28,082] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 13:41:28,596] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 13:41:44,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 13:41:45,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 13:41:46,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 13:41:46,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 13:41:56,879] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 13:46:51,408] INFO : Step 3100/50000 rank6 | Train Loss: 2.3456 | Train Loss Unweighted: 1.3820 | Train Loss Components: Auxiliary: 0.4397 | Cardinality: 0.2017 | Classification: 0.2549 | Classification_binary: 0.3432 | HitGrouping: 0.2399 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3852 | Regression_eta: 0.0046 | Regression_pt: 0.3892 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1608 | Classification_binary: 0.3608 | Regression_cos_phi: 0.1845 | Regression_energy: 0.1891 | Regression_eta: 0.1073 | Regression_pt: 0.2093 | Regression_sin_phi: 0.1352 | LR: 9.91e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4911s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0425 | max_forward: 0.7824 | max_loss: 0.1439 | max_backward: 1.7183 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.8666 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6905.9 | uss_MB: 5708.9 | child_rss_MB: 6176.0 | allocated_MB: 447.6 | reserved_MB: 24468.0 | inactive_split_MB: 1269.5 +rank6 [2026-10-28 13:51:36,980] INFO : Step 3200/50000 rank6 | Train Loss: 2.2453 | Train Loss Unweighted: 1.2729 | Train Loss Components: Auxiliary: 0.4171 | Cardinality: 0.2143 | Classification: 0.2366 | Classification_binary: 0.3116 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3569 | Regression_eta: 0.0047 | Regression_pt: 0.3584 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1369 | Classification_binary: 0.2849 | Regression_cos_phi: 0.1481 | Regression_energy: 0.1811 | Regression_eta: 0.0898 | Regression_pt: 0.1985 | Regression_sin_phi: 0.1224 | LR: 9.91e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6648s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7985 | max_loss: 0.1379 | max_backward: 1.7895 | max_optimizer_step: 0.2027 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6908.5 | uss_MB: 5710.9 | child_rss_MB: 6167.1 | allocated_MB: 448.6 | reserved_MB: 26448.0 | inactive_split_MB: 1391.4 +rank6 [2026-10-28 13:56:16,618] INFO : Step 3300/50000 rank6 | Train Loss: 2.5474 | Train Loss Unweighted: 1.4472 | Train Loss Components: Auxiliary: 0.4779 | Cardinality: 0.2732 | Classification: 0.2824 | Classification_binary: 0.3640 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3911 | Regression_eta: 0.0054 | Regression_pt: 0.3992 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1552 | Classification_binary: 0.3220 | Regression_cos_phi: 0.1697 | Regression_energy: 0.2111 | Regression_eta: 0.1156 | Regression_pt: 0.2320 | Regression_sin_phi: 0.1796 | LR: 9.90e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6658s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8323 | max_loss: 0.1479 | max_backward: 2.0543 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.7834 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5785.1 | uss_MB: 4587.8 | child_rss_MB: 6143.0 | allocated_MB: 448.7 | reserved_MB: 26460.0 | inactive_split_MB: 1087.4 +rank6 [2026-10-28 14:00:49,246] INFO : Step 3400/50000 rank6 | Train Loss: 2.5743 | Train Loss Unweighted: 1.4358 | Train Loss Components: Auxiliary: 0.4720 | Cardinality: 0.2664 | Classification: 0.2699 | Classification_binary: 0.3915 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3784 | Regression_eta: 0.0053 | Regression_pt: 0.3851 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1517 | Classification_binary: 0.3791 | Regression_cos_phi: 0.1664 | Regression_energy: 0.2114 | Regression_eta: 0.1204 | Regression_pt: 0.2352 | Regression_sin_phi: 0.2092 | LR: 9.90e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.2600s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7827 | max_loss: 0.1630 | max_backward: 1.9191 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5606 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5794.1 | uss_MB: 4597.7 | child_rss_MB: 6161.5 | allocated_MB: 450.4 | reserved_MB: 28138.0 | inactive_split_MB: 1261.0 +rank6 [2026-10-28 14:05:23,603] INFO : Step 3500/50000 rank6 | Train Loss: 2.1916 | Train Loss Unweighted: 1.2418 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2204 | Classification: 0.2212 | Classification_binary: 0.3041 | HitGrouping: 0.2426 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3517 | Regression_eta: 0.0047 | Regression_pt: 0.3558 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1410 | Classification_binary: 0.3381 | Regression_cos_phi: 0.1551 | Regression_energy: 0.1551 | Regression_eta: 0.0920 | Regression_pt: 0.1711 | Regression_sin_phi: 0.2120 | LR: 9.89e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2267s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7972 | max_loss: 0.1441 | max_backward: 1.7398 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6845 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5780.0 | uss_MB: 4584.6 | child_rss_MB: 6167.3 | allocated_MB: 443.7 | reserved_MB: 28532.0 | inactive_split_MB: 1176.3 +rank6 [2026-10-28 14:10:06,692] INFO : Step 3600/50000 rank6 | Train Loss: 2.3195 | Train Loss Unweighted: 1.3024 | Train Loss Components: Auxiliary: 0.4248 | Cardinality: 0.2221 | Classification: 0.2518 | Classification_binary: 0.3172 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3610 | Regression_eta: 0.0052 | Regression_pt: 0.3623 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1663 | Classification_binary: 0.2816 | Regression_cos_phi: 0.2131 | Regression_energy: 0.1979 | Regression_eta: 0.1475 | Regression_pt: 0.2121 | Regression_sin_phi: 0.1854 | LR: 9.89e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3066s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8456 | max_loss: 0.1388 | max_backward: 1.9466 | max_optimizer_step: 0.1811 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5787.8 | uss_MB: 4598.8 | child_rss_MB: 6163.2 | allocated_MB: 449.1 | reserved_MB: 28532.0 | inactive_split_MB: 1277.8 +rank6 [2026-10-28 14:14:47,797] INFO : Step 3700/50000 rank6 | Train Loss: 2.5861 | Train Loss Unweighted: 1.4427 | Train Loss Components: Auxiliary: 0.4700 | Cardinality: 0.2678 | Classification: 0.2978 | Classification_binary: 0.3844 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3719 | Regression_eta: 0.0058 | Regression_pt: 0.3773 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2056 | Classification_binary: 0.3827 | Regression_cos_phi: 0.2342 | Regression_energy: 0.1685 | Regression_eta: 0.1613 | Regression_pt: 0.1843 | Regression_sin_phi: 0.2104 | LR: 9.88e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5010s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.8001 | max_loss: 0.1519 | max_backward: 1.8239 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.7916 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0181 | Memory: rss_MB: 5775.1 | uss_MB: 4600.5 | child_rss_MB: 6165.4 | allocated_MB: 450.6 | reserved_MB: 30118.0 | inactive_split_MB: 1097.6 +rank6 [2026-10-28 14:19:24,597] INFO : Step 3800/50000 rank6 | Train Loss: 2.3269 | Train Loss Unweighted: 1.3016 | Train Loss Components: Auxiliary: 0.4225 | Cardinality: 0.2364 | Classification: 0.2400 | Classification_binary: 0.3389 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3534 | Regression_eta: 0.0054 | Regression_pt: 0.3591 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1480 | Classification_binary: 0.3534 | Regression_cos_phi: 0.2149 | Regression_energy: 0.1953 | Regression_eta: 0.1347 | Regression_pt: 0.2138 | Regression_sin_phi: 0.2232 | LR: 9.87e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.4701s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7555 | max_loss: 0.1601 | max_backward: 1.7612 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.9394 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5772.0 | uss_MB: 4595.7 | child_rss_MB: 6158.5 | allocated_MB: 447.4 | reserved_MB: 30118.0 | inactive_split_MB: 1355.2 +rank6 [2026-10-28 14:24:05,270] INFO : Step 3900/50000 rank6 | Train Loss: 2.0276 | Train Loss Unweighted: 1.1874 | Train Loss Components: Auxiliary: 0.3861 | Cardinality: 0.1825 | Classification: 0.2116 | Classification_binary: 0.2707 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3485 | Regression_eta: 0.0038 | Regression_pt: 0.3489 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1297 | Classification_binary: 0.3045 | Regression_cos_phi: 0.1808 | Regression_energy: 0.1836 | Regression_eta: 0.1318 | Regression_pt: 0.2006 | Regression_sin_phi: 0.2098 | LR: 9.87e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.4481s | Phase Times: max_pre_step_barrier: 0.0095 | max_batch_to_device: 0.0140 | max_forward: 1.1007 | max_loss: 0.1867 | max_backward: 2.0952 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 0.5091 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5772.2 | uss_MB: 4598.5 | child_rss_MB: 6165.7 | allocated_MB: 448.6 | reserved_MB: 30118.0 | inactive_split_MB: 1119.4 +rank6 [2026-10-28 14:28:52,324] INFO : Step 4000/50000 rank6 | Train Loss: 2.3921 | Train Loss Unweighted: 1.3683 | Train Loss Components: Auxiliary: 0.4366 | Cardinality: 0.2254 | Classification: 0.2720 | Classification_binary: 0.3524 | HitGrouping: 0.2405 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3647 | Regression_eta: 0.0048 | Regression_pt: 0.3692 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1657 | Classification_binary: 0.3473 | Regression_cos_phi: 0.2124 | Regression_energy: 0.1743 | Regression_eta: 0.1198 | Regression_pt: 0.1888 | Regression_sin_phi: 0.2276 | LR: 9.86e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8905s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.8532 | max_loss: 0.1476 | max_backward: 2.4362 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.6138 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5773.5 | uss_MB: 4601.5 | child_rss_MB: 6164.9 | allocated_MB: 449.7 | reserved_MB: 30118.0 | inactive_split_MB: 899.8 +rank6 [2026-10-28 14:28:53,310] INFO : Running validation on rank6 +rank6 [2026-10-28 14:28:53,310] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 14:28:53,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 14:29:46,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:29:46,701] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 14:30:12,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 14:30:14,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 14:30:15,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 14:30:16,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 14:30:26,118] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 14:30:26,980] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:30:27,512] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 14:30:43,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 14:30:44,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 14:30:45,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 14:30:46,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 14:30:56,111] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 14:30:57,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 14:30:57,800] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 14:31:13,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 14:31:14,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 14:31:15,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 14:31:16,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 14:31:26,078] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 14:36:30,159] INFO : Step 4100/50000 rank6 | Train Loss: 2.5062 | Train Loss Unweighted: 1.3582 | Train Loss Components: Auxiliary: 0.4559 | Cardinality: 0.3208 | Classification: 0.2565 | Classification_binary: 0.3823 | HitGrouping: 0.2386 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3543 | Regression_eta: 0.0046 | Regression_pt: 0.3553 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1843 | Classification_binary: 0.4016 | Regression_cos_phi: 0.2821 | Regression_energy: 0.1794 | Regression_eta: 0.1579 | Regression_pt: 0.1958 | Regression_sin_phi: 0.2847 | LR: 9.85e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.2039s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0069 | max_forward: 0.8943 | max_loss: 0.2518 | max_backward: 1.9865 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 1.1537 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6976.0 | uss_MB: 5798.7 | child_rss_MB: 6190.7 | allocated_MB: 449.4 | reserved_MB: 25148.0 | inactive_split_MB: 1028.6 +rank6 [2026-10-28 14:41:18,155] INFO : Step 4200/50000 rank6 | Train Loss: 2.0961 | Train Loss Unweighted: 1.2232 | Train Loss Components: Auxiliary: 0.3927 | Cardinality: 0.2101 | Classification: 0.2242 | Classification_binary: 0.2902 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3497 | Regression_eta: 0.0042 | Regression_pt: 0.3510 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1676 | Classification_binary: 0.3310 | Regression_cos_phi: 0.2215 | Regression_energy: 0.2034 | Regression_eta: 0.1278 | Regression_pt: 0.2217 | Regression_sin_phi: 0.1939 | LR: 9.84e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.6516s | Phase Times: max_pre_step_barrier: 0.1131 | max_batch_to_device: 0.0199 | max_forward: 1.5911 | max_loss: 0.3393 | max_backward: 3.0707 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5197 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6973.9 | uss_MB: 5797.7 | child_rss_MB: 6186.7 | allocated_MB: 446.0 | reserved_MB: 25296.0 | inactive_split_MB: 922.1 +rank6 [2026-10-28 14:46:18,702] INFO : Step 4300/50000 rank6 | Train Loss: 2.2792 | Train Loss Unweighted: 1.3065 | Train Loss Components: Auxiliary: 0.4224 | Cardinality: 0.2158 | Classification: 0.2339 | Classification_binary: 0.3250 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3669 | Regression_eta: 0.0044 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1336 | Classification_binary: 0.2971 | Regression_cos_phi: 0.1421 | Regression_energy: 0.1969 | Regression_eta: 0.1116 | Regression_pt: 0.2139 | Regression_sin_phi: 0.1511 | LR: 9.84e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5463s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0938 | max_forward: 0.8451 | max_loss: 0.1396 | max_backward: 1.9669 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.6639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5862.7 | uss_MB: 4685.4 | child_rss_MB: 6195.4 | allocated_MB: 448.4 | reserved_MB: 28580.0 | inactive_split_MB: 1346.0 +rank6 [2026-10-28 14:50:58,885] INFO : Step 4400/50000 rank6 | Train Loss: 2.4955 | Train Loss Unweighted: 1.3385 | Train Loss Components: Auxiliary: 0.4510 | Cardinality: 0.3020 | Classification: 0.2394 | Classification_binary: 0.3842 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3482 | Regression_eta: 0.0051 | Regression_pt: 0.3563 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1461 | Classification_binary: 0.3598 | Regression_cos_phi: 0.1918 | Regression_energy: 0.1881 | Regression_eta: 0.1492 | Regression_pt: 0.2090 | Regression_sin_phi: 0.2893 | LR: 9.83e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2054s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0083 | max_forward: 0.7889 | max_loss: 0.1505 | max_backward: 1.8572 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4868 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5870.1 | uss_MB: 4697.1 | child_rss_MB: 6175.1 | allocated_MB: 449.1 | reserved_MB: 28582.0 | inactive_split_MB: 1105.9 +rank6 [2026-10-28 14:55:40,224] INFO : Step 4500/50000 rank6 | Train Loss: 2.3283 | Train Loss Unweighted: 1.2750 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.3015 | Classification: 0.2318 | Classification_binary: 0.3495 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3409 | Regression_eta: 0.0044 | Regression_pt: 0.3441 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1503 | Classification_binary: 0.3592 | Regression_cos_phi: 0.1741 | Regression_energy: 0.1681 | Regression_eta: 0.1306 | Regression_pt: 0.1874 | Regression_sin_phi: 0.1736 | LR: 9.82e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.0166s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7326 | max_loss: 0.2428 | max_backward: 1.7841 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5868.2 | uss_MB: 4696.0 | child_rss_MB: 6178.6 | allocated_MB: 447.8 | reserved_MB: 28582.0 | inactive_split_MB: 1258.0 +rank6 [2026-10-28 15:00:16,448] INFO : Step 4600/50000 rank6 | Train Loss: 2.0353 | Train Loss Unweighted: 1.1777 | Train Loss Components: Auxiliary: 0.3804 | Cardinality: 0.1920 | Classification: 0.2053 | Classification_binary: 0.2642 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3490 | Regression_eta: 0.0039 | Regression_pt: 0.3512 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1121 | Classification_binary: 0.2679 | Regression_cos_phi: 0.1830 | Regression_energy: 0.2119 | Regression_eta: 0.1209 | Regression_pt: 0.2286 | Regression_sin_phi: 0.2374 | LR: 9.81e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0860s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0402 | max_forward: 0.7143 | max_loss: 0.1412 | max_backward: 1.6726 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 5861.4 | uss_MB: 4694.6 | child_rss_MB: 6186.8 | allocated_MB: 446.4 | reserved_MB: 28582.0 | inactive_split_MB: 981.4 +rank6 [2026-10-28 15:05:12,090] INFO : Step 4700/50000 rank6 | Train Loss: 2.2426 | Train Loss Unweighted: 1.2576 | Train Loss Components: Auxiliary: 0.4073 | Cardinality: 0.2222 | Classification: 0.2485 | Classification_binary: 0.3333 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3307 | Regression_eta: 0.0049 | Regression_pt: 0.3356 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1738 | Classification_binary: 0.3380 | Regression_cos_phi: 0.2146 | Regression_energy: 0.1874 | Regression_eta: 0.1177 | Regression_pt: 0.2057 | Regression_sin_phi: 0.1917 | LR: 9.81e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6199s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.1471 | max_backward: 2.0627 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.7062 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0769 | Memory: rss_MB: 5860.5 | uss_MB: 4699.2 | child_rss_MB: 6169.2 | allocated_MB: 447.7 | reserved_MB: 28652.0 | inactive_split_MB: 1164.9 +rank6 [2026-10-28 15:09:59,906] INFO : Step 4800/50000 rank6 | Train Loss: 2.2697 | Train Loss Unweighted: 1.2637 | Train Loss Components: Auxiliary: 0.4132 | Cardinality: 0.2493 | Classification: 0.2262 | Classification_binary: 0.3667 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3287 | Regression_eta: 0.0044 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1256 | Classification_binary: 0.3370 | Regression_cos_phi: 0.1678 | Regression_energy: 0.1598 | Regression_eta: 0.1095 | Regression_pt: 0.1758 | Regression_sin_phi: 0.1447 | LR: 9.80e-04 | DataLoad Time: 0.0138s | Model Forward Time: 4.0705s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0065 | max_forward: 0.7626 | max_loss: 0.1571 | max_backward: 1.6465 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5855.3 | uss_MB: 4693.4 | child_rss_MB: 6188.9 | allocated_MB: 450.7 | reserved_MB: 28652.0 | inactive_split_MB: 1088.6 +rank6 [2026-10-28 15:14:51,789] INFO : Step 4900/50000 rank6 | Train Loss: 2.0529 | Train Loss Unweighted: 1.1598 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.1942 | Classification: 0.2269 | Classification_binary: 0.2874 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3172 | Regression_eta: 0.0041 | Regression_pt: 0.3199 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1210 | Classification_binary: 0.2897 | Regression_cos_phi: 0.1126 | Regression_energy: 0.1574 | Regression_eta: 0.0976 | Regression_pt: 0.1709 | Regression_sin_phi: 0.1645 | LR: 9.79e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3248s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7293 | max_loss: 0.1974 | max_backward: 1.8633 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5858.6 | uss_MB: 4697.0 | child_rss_MB: 6189.0 | allocated_MB: 447.4 | reserved_MB: 28652.0 | inactive_split_MB: 1266.1 +rank6 [2026-10-28 15:19:59,465] INFO : Step 5000/50000 rank6 | Train Loss: 2.0827 | Train Loss Unweighted: 1.1872 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2154 | Classification: 0.2462 | Classification_binary: 0.2941 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3169 | Regression_eta: 0.0042 | Regression_pt: 0.3219 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1706 | Classification_binary: 0.2863 | Regression_cos_phi: 0.2009 | Regression_energy: 0.1915 | Regression_eta: 0.1327 | Regression_pt: 0.2090 | Regression_sin_phi: 0.1800 | LR: 9.78e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3264s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.8382 | max_loss: 0.1450 | max_backward: 1.9472 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0203 | Memory: rss_MB: 5858.9 | uss_MB: 4697.5 | child_rss_MB: 6177.0 | allocated_MB: 446.9 | reserved_MB: 29316.0 | inactive_split_MB: 980.8 +rank6 [2026-10-28 15:20:00,329] INFO : Running validation on rank6 +rank6 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 15:20:55,982] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:20:56,480] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 15:21:22,919] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 15:21:23,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 15:21:25,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 15:21:26,077] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 15:21:36,087] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 15:21:36,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:21:36,967] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 15:21:53,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 15:21:54,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 15:21:55,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 15:21:56,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 15:22:06,044] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 15:22:06,552] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 15:22:07,339] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 15:22:23,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 15:22:24,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 15:22:25,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 15:22:26,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 15:22:36,396] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 15:28:09,086] INFO : Step 5100/50000 rank6 | Train Loss: 2.4098 | Train Loss Unweighted: 1.3457 | Train Loss Components: Auxiliary: 0.4318 | Cardinality: 0.2397 | Classification: 0.2711 | Classification_binary: 0.3798 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3391 | Regression_eta: 0.0051 | Regression_pt: 0.3453 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1426 | Classification_binary: 0.3543 | Regression_cos_phi: 0.2406 | Regression_energy: 0.1669 | Regression_eta: 0.1374 | Regression_pt: 0.1826 | Regression_sin_phi: 0.2217 | LR: 9.77e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.8396s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1511 | max_backward: 1.7914 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 1.0544 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5949.7 | uss_MB: 4786.0 | child_rss_MB: 6159.3 | allocated_MB: 447.6 | reserved_MB: 25650.0 | inactive_split_MB: 1122.4 +rank6 [2026-10-28 15:32:58,661] INFO : Step 5200/50000 rank6 | Train Loss: 2.0929 | Train Loss Unweighted: 1.1974 | Train Loss Components: Auxiliary: 0.3875 | Cardinality: 0.2007 | Classification: 0.2193 | Classification_binary: 0.3278 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3194 | Regression_eta: 0.0041 | Regression_pt: 0.3227 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1211 | Classification_binary: 0.3216 | Regression_cos_phi: 0.1553 | Regression_energy: 0.1767 | Regression_eta: 0.0951 | Regression_pt: 0.1958 | Regression_sin_phi: 0.1678 | LR: 9.76e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.4912s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7458 | max_loss: 0.2423 | max_backward: 1.7123 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9648 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5957.0 | uss_MB: 4796.7 | child_rss_MB: 6179.0 | allocated_MB: 450.7 | reserved_MB: 28708.0 | inactive_split_MB: 1139.3 +rank6 [2026-10-28 15:37:52,596] INFO : Step 5300/50000 rank6 | Train Loss: 1.9905 | Train Loss Unweighted: 1.1352 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.2083 | Classification: 0.2239 | Classification_binary: 0.2818 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0037 | Regression_pt: 0.3135 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1214 | Classification_binary: 0.2952 | Regression_cos_phi: 0.1529 | Regression_energy: 0.1718 | Regression_eta: 0.0913 | Regression_pt: 0.1896 | Regression_sin_phi: 0.1416 | LR: 9.75e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.1046s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 1.2453 | max_loss: 0.2326 | max_backward: 2.1392 | max_optimizer_step: 0.1883 | max_tensorboard_logging: 1.0947 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5945.0 | uss_MB: 4784.6 | child_rss_MB: 6156.3 | allocated_MB: 445.9 | reserved_MB: 30688.0 | inactive_split_MB: 925.3 +rank6 [2026-10-28 15:42:39,663] INFO : Step 5400/50000 rank6 | Train Loss: 2.2932 | Train Loss Unweighted: 1.2957 | Train Loss Components: Auxiliary: 0.4189 | Cardinality: 0.2411 | Classification: 0.2689 | Classification_binary: 0.3420 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3351 | Regression_eta: 0.0045 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1930 | Classification_binary: 0.3245 | Regression_cos_phi: 0.1828 | Regression_energy: 0.1967 | Regression_eta: 0.1549 | Regression_pt: 0.2183 | Regression_sin_phi: 0.1787 | LR: 9.74e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5703s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7688 | max_loss: 0.1697 | max_backward: 1.8189 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8641 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5955.2 | uss_MB: 4796.8 | child_rss_MB: 6163.2 | allocated_MB: 450.8 | reserved_MB: 30688.0 | inactive_split_MB: 915.1 +rank6 [2026-10-28 15:47:42,229] INFO : Step 5500/50000 rank6 | Train Loss: 2.1734 | Train Loss Unweighted: 1.2295 | Train Loss Components: Auxiliary: 0.3970 | Cardinality: 0.2269 | Classification: 0.2218 | Classification_binary: 0.3571 | HitGrouping: 0.2200 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3188 | Regression_eta: 0.0041 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1577 | Classification_binary: 0.3759 | Regression_cos_phi: 0.1988 | Regression_energy: 0.1743 | Regression_eta: 0.1067 | Regression_pt: 0.1912 | Regression_sin_phi: 0.1955 | LR: 9.73e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.9286s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0153 | max_forward: 1.1913 | max_loss: 0.6135 | max_backward: 3.0599 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.6982 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5952.5 | uss_MB: 4797.4 | child_rss_MB: 6170.1 | allocated_MB: 446.9 | reserved_MB: 32804.0 | inactive_split_MB: 913.2 +rank6 [2026-10-28 15:52:30,289] INFO : Step 5600/50000 rank6 | Train Loss: 2.0307 | Train Loss Unweighted: 1.1169 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.1966 | Classification: 0.2085 | Classification_binary: 0.3031 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2975 | Regression_eta: 0.0047 | Regression_pt: 0.2988 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1397 | Classification_binary: 0.3312 | Regression_cos_phi: 0.1810 | Regression_energy: 0.1570 | Regression_eta: 0.1291 | Regression_pt: 0.1742 | Regression_sin_phi: 0.1523 | LR: 9.72e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.7953s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7794 | max_loss: 0.1497 | max_backward: 1.9410 | max_optimizer_step: 0.5154 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0061 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5920.9 | uss_MB: 4797.0 | child_rss_MB: 6174.4 | allocated_MB: 445.4 | reserved_MB: 32804.0 | inactive_split_MB: 1164.2 +rank6 [2026-10-28 15:57:16,862] INFO : Step 5700/50000 rank6 | Train Loss: 2.2064 | Train Loss Unweighted: 1.2224 | Train Loss Components: Auxiliary: 0.4023 | Cardinality: 0.2320 | Classification: 0.2526 | Classification_binary: 0.3174 | HitGrouping: 0.2343 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3173 | Regression_eta: 0.0045 | Regression_pt: 0.3259 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1698 | Classification_binary: 0.3049 | Regression_cos_phi: 0.2264 | Regression_energy: 0.1797 | Regression_eta: 0.1249 | Regression_pt: 0.2004 | Regression_sin_phi: 0.2224 | LR: 9.71e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.6663s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7746 | max_loss: 0.4928 | max_backward: 1.9590 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8719 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 5893.1 | uss_MB: 3755.9 | child_rss_MB: 6168.2 | allocated_MB: 448.4 | reserved_MB: 32804.0 | inactive_split_MB: 913.0 +rank6 [2026-10-28 16:02:01,175] INFO : Step 5800/50000 rank6 | Train Loss: 2.1438 | Train Loss Unweighted: 1.2123 | Train Loss Components: Auxiliary: 0.3898 | Cardinality: 0.2335 | Classification: 0.2404 | Classification_binary: 0.3575 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3006 | Regression_eta: 0.0042 | Regression_pt: 0.3054 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1339 | Classification_binary: 0.3596 | Regression_cos_phi: 0.1963 | Regression_energy: 0.1663 | Regression_eta: 0.1177 | Regression_pt: 0.1825 | Regression_sin_phi: 0.1594 | LR: 9.70e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.5878s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8028 | max_loss: 0.1540 | max_backward: 1.8645 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.8379 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5883.5 | uss_MB: 4797.1 | child_rss_MB: 6169.4 | allocated_MB: 448.3 | reserved_MB: 32804.0 | inactive_split_MB: 916.5 +rank6 [2026-10-28 16:06:46,106] INFO : Step 5900/50000 rank6 | Train Loss: 2.2094 | Train Loss Unweighted: 1.1453 | Train Loss Components: Auxiliary: 0.3936 | Cardinality: 0.2891 | Classification: 0.2198 | Classification_binary: 0.3337 | HitGrouping: 0.2238 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2890 | Regression_eta: 0.0046 | Regression_pt: 0.2935 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1566 | Classification_binary: 0.3760 | Regression_cos_phi: 0.1951 | Regression_energy: 0.1484 | Regression_eta: 0.1392 | Regression_pt: 0.1624 | Regression_sin_phi: 0.2667 | LR: 9.69e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.5884s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0065 | max_forward: 1.0922 | max_loss: 0.1477 | max_backward: 1.8468 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8251 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0991 | Memory: rss_MB: 5881.4 | uss_MB: 4798.1 | child_rss_MB: 6148.2 | allocated_MB: 446.5 | reserved_MB: 32804.0 | inactive_split_MB: 927.1 +rank6 [2026-10-28 16:11:31,999] INFO : Step 6000/50000 rank6 | Train Loss: 1.8458 | Train Loss Unweighted: 1.0363 | Train Loss Components: Auxiliary: 0.3398 | Cardinality: 0.1796 | Classification: 0.1819 | Classification_binary: 0.2710 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2865 | Regression_eta: 0.0032 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1174 | Classification_binary: 0.3080 | Regression_cos_phi: 0.1171 | Regression_energy: 0.1606 | Regression_eta: 0.0840 | Regression_pt: 0.1761 | Regression_sin_phi: 0.1676 | LR: 9.68e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5088s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7380 | max_loss: 0.2483 | max_backward: 1.8094 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.9243 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5850.9 | uss_MB: 4795.2 | child_rss_MB: 6173.4 | allocated_MB: 448.1 | reserved_MB: 32804.0 | inactive_split_MB: 1232.3 +rank6 [2026-10-28 16:11:32,849] INFO : Running validation on rank6 +rank6 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 16:12:26,408] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:12:26,924] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 16:12:54,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 16:12:55,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 16:12:56,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 16:12:57,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 16:13:07,433] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 16:13:07,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:13:08,327] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 16:13:24,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 16:13:25,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 16:13:26,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 16:13:27,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 16:13:37,443] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 16:13:38,169] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 16:13:38,686] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 16:13:54,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 16:13:55,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 16:13:56,353] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 16:13:57,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 16:14:07,159] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 16:19:13,933] INFO : Step 6100/50000 rank6 | Train Loss: 2.0818 | Train Loss Unweighted: 1.1297 | Train Loss Components: Auxiliary: 0.3734 | Cardinality: 0.2287 | Classification: 0.2079 | Classification_binary: 0.3249 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2929 | Regression_eta: 0.0044 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1450 | Classification_binary: 0.3561 | Regression_cos_phi: 0.2015 | Regression_energy: 0.1419 | Regression_eta: 0.1264 | Regression_pt: 0.1561 | Regression_sin_phi: 0.2173 | LR: 9.67e-04 | DataLoad Time: 0.0157s | Model Forward Time: 6.5162s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8130 | max_loss: 0.1524 | max_backward: 3.1921 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 1.4719 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5941.1 | uss_MB: 4883.2 | child_rss_MB: 6189.9 | allocated_MB: 448.4 | reserved_MB: 24434.0 | inactive_split_MB: 1287.8 +rank6 [2026-10-28 16:23:58,090] INFO : Step 6200/50000 rank6 | Train Loss: 2.3716 | Train Loss Unweighted: 1.2065 | Train Loss Components: Auxiliary: 0.4163 | Cardinality: 0.3927 | Classification: 0.2343 | Classification_binary: 0.3679 | HitGrouping: 0.2165 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2973 | Regression_eta: 0.0043 | Regression_pt: 0.2981 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1325 | Classification_binary: 0.3757 | Regression_cos_phi: 0.1868 | Regression_energy: 0.1541 | Regression_eta: 0.1070 | Regression_pt: 0.1702 | Regression_sin_phi: 0.1877 | LR: 9.66e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.5592s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8556 | max_loss: 0.1996 | max_backward: 1.9593 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.5823 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5946.6 | uss_MB: 3767.3 | child_rss_MB: 6181.8 | allocated_MB: 447.9 | reserved_MB: 27020.0 | inactive_split_MB: 1185.4 +rank6 [2026-10-28 16:28:38,484] INFO : Step 6300/50000 rank6 | Train Loss: 1.9346 | Train Loss Unweighted: 1.0816 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.2071 | Classification: 0.2053 | Classification_binary: 0.2781 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2941 | Regression_eta: 0.0037 | Regression_pt: 0.2966 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1193 | Classification_binary: 0.2691 | Regression_cos_phi: 0.1454 | Regression_energy: 0.1591 | Regression_eta: 0.1083 | Regression_pt: 0.1783 | Regression_sin_phi: 0.1145 | LR: 9.65e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5892s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0068 | max_forward: 0.7412 | max_loss: 0.1444 | max_backward: 1.7431 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.9045 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5941.6 | uss_MB: 4896.5 | child_rss_MB: 6164.9 | allocated_MB: 444.6 | reserved_MB: 27022.0 | inactive_split_MB: 1185.4 +rank6 [2026-10-28 16:33:23,144] INFO : Step 6400/50000 rank6 | Train Loss: 2.0123 | Train Loss Unweighted: 1.0992 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2082 | Classification: 0.2291 | Classification_binary: 0.2885 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2842 | Regression_eta: 0.0040 | Regression_pt: 0.2891 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1558 | Classification_binary: 0.2711 | Regression_cos_phi: 0.1926 | Regression_energy: 0.1619 | Regression_eta: 0.1121 | Regression_pt: 0.1791 | Regression_sin_phi: 0.2497 | LR: 9.64e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.7577s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0069 | max_forward: 1.1063 | max_loss: 0.2471 | max_backward: 1.9949 | max_optimizer_step: 0.2726 | max_tensorboard_logging: 0.8233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5936.1 | uss_MB: 4896.3 | child_rss_MB: 6176.5 | allocated_MB: 451.6 | reserved_MB: 27022.0 | inactive_split_MB: 1113.0 +rank6 [2026-10-28 16:37:57,182] INFO : Step 6500/50000 rank6 | Train Loss: 2.1791 | Train Loss Unweighted: 1.2220 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2499 | Classification: 0.2271 | Classification_binary: 0.3508 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3177 | Regression_eta: 0.0042 | Regression_pt: 0.3181 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1746 | Classification_binary: 0.3523 | Regression_cos_phi: 0.1764 | Regression_energy: 0.1877 | Regression_eta: 0.1085 | Regression_pt: 0.2062 | Regression_sin_phi: 0.1387 | LR: 9.63e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9957s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0548 | max_forward: 0.8069 | max_loss: 0.1518 | max_backward: 2.0692 | max_optimizer_step: 0.1860 | max_tensorboard_logging: 0.9414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5926.6 | uss_MB: 4895.5 | child_rss_MB: 6160.0 | allocated_MB: 449.6 | reserved_MB: 28336.0 | inactive_split_MB: 1199.2 +rank6 [2026-10-28 16:42:33,043] INFO : Step 6600/50000 rank6 | Train Loss: 1.9690 | Train Loss Unweighted: 1.0745 | Train Loss Components: Auxiliary: 0.3537 | Cardinality: 0.2132 | Classification: 0.1982 | Classification_binary: 0.3095 | HitGrouping: 0.2136 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2771 | Regression_eta: 0.0040 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1518 | Classification_binary: 0.3475 | Regression_cos_phi: 0.1743 | Regression_energy: 0.1653 | Regression_eta: 0.1286 | Regression_pt: 0.1777 | Regression_sin_phi: 0.1962 | LR: 9.62e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8255s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7382 | max_loss: 0.1467 | max_backward: 1.8335 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 1.1664 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5923.1 | uss_MB: 4893.3 | child_rss_MB: 6174.8 | allocated_MB: 447.4 | reserved_MB: 28340.0 | inactive_split_MB: 1012.8 +rank6 [2026-10-28 16:47:06,903] INFO : Step 6700/50000 rank6 | Train Loss: 1.9470 | Train Loss Unweighted: 1.0781 | Train Loss Components: Auxiliary: 0.3616 | Cardinality: 0.2363 | Classification: 0.1923 | Classification_binary: 0.2661 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3028 | Regression_eta: 0.0035 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1376 | Classification_binary: 0.2848 | Regression_cos_phi: 0.1361 | Regression_energy: 0.1411 | Regression_eta: 0.0879 | Regression_pt: 0.1565 | Regression_sin_phi: 0.1258 | LR: 9.61e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6594s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7926 | max_loss: 0.2421 | max_backward: 2.0577 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.7621 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5925.5 | uss_MB: 4898.4 | child_rss_MB: 6176.2 | allocated_MB: 450.1 | reserved_MB: 29806.0 | inactive_split_MB: 1002.0 +rank6 [2026-10-28 16:51:50,914] INFO : Step 6800/50000 rank6 | Train Loss: 2.1609 | Train Loss Unweighted: 1.1507 | Train Loss Components: Auxiliary: 0.3771 | Cardinality: 0.2180 | Classification: 0.2315 | Classification_binary: 0.3225 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0025 | Regression_energy: 0.2916 | Regression_eta: 0.0048 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1404 | Classification_binary: 0.3172 | Regression_cos_phi: 0.1663 | Regression_energy: 0.1605 | Regression_eta: 0.1280 | Regression_pt: 0.1765 | Regression_sin_phi: 0.2253 | LR: 9.60e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.1889s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0070 | max_forward: 0.9144 | max_loss: 0.4866 | max_backward: 2.1484 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 1.2438 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5920.8 | uss_MB: 4896.0 | child_rss_MB: 6174.1 | allocated_MB: 451.3 | reserved_MB: 29806.0 | inactive_split_MB: 1098.9 +rank6 [2026-10-28 16:56:32,288] INFO : Step 6900/50000 rank6 | Train Loss: 1.9936 | Train Loss Unweighted: 1.1069 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.1991 | Classification: 0.2085 | Classification_binary: 0.3385 | HitGrouping: 0.2125 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2742 | Regression_eta: 0.0041 | Regression_pt: 0.2777 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1265 | Classification_binary: 0.3496 | Regression_cos_phi: 0.2196 | Regression_energy: 0.1377 | Regression_eta: 0.1237 | Regression_pt: 0.1522 | Regression_sin_phi: 0.1894 | LR: 9.58e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6489s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7764 | max_loss: 0.2225 | max_backward: 1.6637 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1184 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5918.2 | uss_MB: 4895.4 | child_rss_MB: 6170.7 | allocated_MB: 450.8 | reserved_MB: 29806.0 | inactive_split_MB: 1193.4 +rank6 [2026-10-28 17:01:16,519] INFO : Step 7000/50000 rank6 | Train Loss: 1.9256 | Train Loss Unweighted: 1.0803 | Train Loss Components: Auxiliary: 0.3608 | Cardinality: 0.2036 | Classification: 0.1946 | Classification_binary: 0.2738 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2998 | Regression_eta: 0.0039 | Regression_pt: 0.3044 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1211 | Classification_binary: 0.2859 | Regression_cos_phi: 0.1613 | Regression_energy: 0.1536 | Regression_eta: 0.0976 | Regression_pt: 0.1727 | Regression_sin_phi: 0.1382 | LR: 9.57e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.7627s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7896 | max_loss: 0.1557 | max_backward: 1.7415 | max_optimizer_step: 0.2946 | max_tensorboard_logging: 1.0959 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 5913.7 | uss_MB: 4893.6 | child_rss_MB: 6162.0 | allocated_MB: 450.3 | reserved_MB: 29806.0 | inactive_split_MB: 1283.4 +rank6 [2026-10-28 17:01:17,465] INFO : Running validation on rank6 +rank6 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 17:02:11,957] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:02:12,868] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 17:02:38,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 17:02:40,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 17:02:41,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 17:02:42,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 17:02:52,117] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 17:02:52,592] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:02:53,110] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 17:03:08,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 17:03:09,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 17:03:10,892] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 17:03:11,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 17:03:21,865] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 17:03:22,841] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:03:23,343] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 17:03:39,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 17:03:39,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 17:03:40,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 17:03:41,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 17:03:51,649] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 17:08:58,934] INFO : Step 7100/50000 rank6 | Train Loss: 1.8251 | Train Loss Unweighted: 1.0448 | Train Loss Components: Auxiliary: 0.3403 | Cardinality: 0.1731 | Classification: 0.2040 | Classification_binary: 0.2675 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2809 | Regression_eta: 0.0035 | Regression_pt: 0.2853 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1362 | Classification_binary: 0.2518 | Regression_cos_phi: 0.1727 | Regression_energy: 0.1659 | Regression_eta: 0.0899 | Regression_pt: 0.1821 | Regression_sin_phi: 0.1514 | LR: 9.56e-04 | DataLoad Time: 0.0220s | Model Forward Time: 5.2526s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0622 | max_forward: 0.8282 | max_loss: 0.1540 | max_backward: 1.9352 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.3584 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6015.2 | uss_MB: 4988.2 | child_rss_MB: 6155.9 | allocated_MB: 449.9 | reserved_MB: 22498.0 | inactive_split_MB: 1193.4 +rank6 [2026-10-28 17:13:31,724] INFO : Step 7200/50000 rank6 | Train Loss: 2.1146 | Train Loss Unweighted: 1.1506 | Train Loss Components: Auxiliary: 0.3887 | Cardinality: 0.2411 | Classification: 0.2243 | Classification_binary: 0.3469 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2844 | Regression_eta: 0.0044 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1167 | Classification_binary: 0.3369 | Regression_cos_phi: 0.1473 | Regression_energy: 0.1509 | Regression_eta: 0.1014 | Regression_pt: 0.1624 | Regression_sin_phi: 0.1462 | LR: 9.55e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.5714s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0369 | max_forward: 0.7969 | max_loss: 0.1517 | max_backward: 1.9380 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6030.2 | uss_MB: 5003.5 | child_rss_MB: 6152.6 | allocated_MB: 448.6 | reserved_MB: 22906.0 | inactive_split_MB: 1087.6 +rank6 [2026-10-28 17:18:11,974] INFO : Step 7300/50000 rank6 | Train Loss: 1.8657 | Train Loss Unweighted: 1.0129 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2046 | Classification: 0.1759 | Classification_binary: 0.2802 | HitGrouping: 0.2112 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2746 | Regression_eta: 0.0037 | Regression_pt: 0.2747 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1061 | Classification_binary: 0.2848 | Regression_cos_phi: 0.2019 | Regression_energy: 0.1318 | Regression_eta: 0.1090 | Regression_pt: 0.1426 | Regression_sin_phi: 0.1464 | LR: 9.53e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4746s | Phase Times: max_pre_step_barrier: 0.0082 | max_batch_to_device: 0.0150 | max_forward: 0.7713 | max_loss: 0.1507 | max_backward: 1.8303 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.6113 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6029.9 | uss_MB: 5003.9 | child_rss_MB: 6151.8 | allocated_MB: 449.9 | reserved_MB: 25044.0 | inactive_split_MB: 918.5 +rank6 [2026-10-28 17:22:47,474] INFO : Step 7400/50000 rank6 | Train Loss: 1.8326 | Train Loss Unweighted: 1.0006 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2227 | Classification: 0.1795 | Classification_binary: 0.2563 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2767 | Regression_eta: 0.0034 | Regression_pt: 0.2813 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1302 | Classification_binary: 0.2602 | Regression_cos_phi: 0.1328 | Regression_energy: 0.1387 | Regression_eta: 0.1034 | Regression_pt: 0.1525 | Regression_sin_phi: 0.1361 | LR: 9.52e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.2042s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0684 | max_forward: 0.7762 | max_loss: 0.1384 | max_backward: 1.7881 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6026.9 | uss_MB: 5000.5 | child_rss_MB: 6151.8 | allocated_MB: 447.4 | reserved_MB: 25044.0 | inactive_split_MB: 1152.4 +rank6 [2026-10-28 17:27:28,847] INFO : Step 7500/50000 rank6 | Train Loss: 2.1479 | Train Loss Unweighted: 1.1510 | Train Loss Components: Auxiliary: 0.3847 | Cardinality: 0.2329 | Classification: 0.2181 | Classification_binary: 0.3323 | HitGrouping: 0.2143 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2947 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1495 | Classification_binary: 0.3481 | Regression_cos_phi: 0.2346 | Regression_energy: 0.1468 | Regression_eta: 0.1190 | Regression_pt: 0.1603 | Regression_sin_phi: 0.2126 | LR: 9.51e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2511s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7726 | max_loss: 0.2518 | max_backward: 1.8458 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1040 | Memory: rss_MB: 6026.8 | uss_MB: 5002.5 | child_rss_MB: 6153.2 | allocated_MB: 452.3 | reserved_MB: 26846.0 | inactive_split_MB: 1117.1 +rank6 [2026-10-28 17:32:29,655] INFO : Step 7600/50000 rank6 | Train Loss: 1.9984 | Train Loss Unweighted: 1.0873 | Train Loss Components: Auxiliary: 0.3667 | Cardinality: 0.2088 | Classification: 0.2053 | Classification_binary: 0.3405 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2647 | Regression_eta: 0.0038 | Regression_pt: 0.2689 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1477 | Classification_binary: 0.3653 | Regression_cos_phi: 0.1805 | Regression_energy: 0.1601 | Regression_eta: 0.1166 | Regression_pt: 0.1772 | Regression_sin_phi: 0.1891 | LR: 9.50e-04 | DataLoad Time: 0.0140s | Model Forward Time: 4.5807s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7834 | max_loss: 0.1641 | max_backward: 1.8407 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.8255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6029.1 | uss_MB: 5003.6 | child_rss_MB: 6150.1 | allocated_MB: 448.9 | reserved_MB: 26846.0 | inactive_split_MB: 1232.3 +rank6 [2026-10-28 17:37:19,852] INFO : Step 7700/50000 rank6 | Train Loss: 1.7524 | Train Loss Unweighted: 0.9761 | Train Loss Components: Auxiliary: 0.3200 | Cardinality: 0.1783 | Classification: 0.1756 | Classification_binary: 0.2521 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2705 | Regression_eta: 0.0035 | Regression_pt: 0.2709 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1374 | Classification_binary: 0.2791 | Regression_cos_phi: 0.1596 | Regression_energy: 0.1412 | Regression_eta: 0.1141 | Regression_pt: 0.1536 | Regression_sin_phi: 0.1722 | LR: 9.48e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.4850s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7659 | max_loss: 0.1551 | max_backward: 1.8103 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6027.2 | uss_MB: 5004.4 | child_rss_MB: 6154.6 | allocated_MB: 448.8 | reserved_MB: 26846.0 | inactive_split_MB: 1042.2 +rank6 [2026-10-28 17:42:10,706] INFO : Step 7800/50000 rank6 | Train Loss: 1.8039 | Train Loss Unweighted: 1.0226 | Train Loss Components: Auxiliary: 0.3378 | Cardinality: 0.1686 | Classification: 0.1822 | Classification_binary: 0.2556 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2863 | Regression_eta: 0.0034 | Regression_pt: 0.2914 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1297 | Classification_binary: 0.2598 | Regression_cos_phi: 0.1451 | Regression_energy: 0.1745 | Regression_eta: 0.1194 | Regression_pt: 0.1898 | Regression_sin_phi: 0.1856 | LR: 9.47e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6877s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.8680 | max_loss: 0.1424 | max_backward: 1.9949 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.8163 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 6024.0 | uss_MB: 5003.5 | child_rss_MB: 6156.5 | allocated_MB: 448.6 | reserved_MB: 26846.0 | inactive_split_MB: 1396.9 +rank6 [2026-10-28 17:47:05,102] INFO : Step 7900/50000 rank6 | Train Loss: 2.0641 | Train Loss Unweighted: 1.0960 | Train Loss Components: Auxiliary: 0.3855 | Cardinality: 0.2401 | Classification: 0.2031 | Classification_binary: 0.3367 | HitGrouping: 0.2085 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2721 | Regression_eta: 0.0041 | Regression_pt: 0.2759 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1073 | Classification_binary: 0.3011 | Regression_cos_phi: 0.1091 | Regression_energy: 0.1317 | Regression_eta: 0.0884 | Regression_pt: 0.1447 | Regression_sin_phi: 0.1103 | LR: 9.46e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.6381s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7971 | max_loss: 0.2430 | max_backward: 1.9131 | max_optimizer_step: 0.2681 | max_tensorboard_logging: 0.7971 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6025.3 | uss_MB: 5004.3 | child_rss_MB: 6161.5 | allocated_MB: 451.8 | reserved_MB: 27044.0 | inactive_split_MB: 827.4 +rank6 [2026-10-28 17:51:56,122] INFO : Step 8000/50000 rank6 | Train Loss: 1.8391 | Train Loss Unweighted: 0.9813 | Train Loss Components: Auxiliary: 0.3355 | Cardinality: 0.2153 | Classification: 0.1625 | Classification_binary: 0.2896 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2605 | Regression_eta: 0.0035 | Regression_pt: 0.2616 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1094 | Classification_binary: 0.3094 | Regression_cos_phi: 0.1458 | Regression_energy: 0.1323 | Regression_eta: 0.1055 | Regression_pt: 0.1414 | Regression_sin_phi: 0.1324 | LR: 9.44e-04 | DataLoad Time: 0.0183s | Model Forward Time: 6.5536s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 1.2188 | max_loss: 0.4570 | max_backward: 3.9408 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.5566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6013.9 | uss_MB: 4999.4 | child_rss_MB: 6155.0 | allocated_MB: 448.4 | reserved_MB: 27044.0 | inactive_split_MB: 1156.9 +rank6 [2026-10-28 17:51:56,979] INFO : Running validation on rank6 +rank6 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 17:52:50,097] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:52:50,645] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 17:53:16,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 17:53:17,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 17:53:18,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 17:53:20,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 17:53:30,013] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 17:53:30,251] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:53:30,806] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 17:53:47,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 17:53:48,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 17:53:49,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 17:53:50,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 17:54:00,844] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-28 17:54:01,631] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 17:54:02,176] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 17:54:17,412] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 17:54:18,253] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 17:54:19,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 17:54:19,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 17:54:29,854] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 17:59:43,743] INFO : Step 8100/50000 rank6 | Train Loss: 1.7101 | Train Loss Unweighted: 0.9276 | Train Loss Components: Auxiliary: 0.3166 | Cardinality: 0.1787 | Classification: 0.1664 | Classification_binary: 0.2381 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2564 | Regression_eta: 0.0032 | Regression_pt: 0.2599 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1157 | Classification_binary: 0.2224 | Regression_cos_phi: 0.1562 | Regression_energy: 0.1272 | Regression_eta: 0.1069 | Regression_pt: 0.1409 | Regression_sin_phi: 0.1582 | LR: 9.43e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6981s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7628 | max_loss: 0.1353 | max_backward: 2.0123 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.8300 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6970.2 | uss_MB: 5951.6 | child_rss_MB: 6190.8 | allocated_MB: 447.2 | reserved_MB: 25306.0 | inactive_split_MB: 1261.3 +rank6 [2026-10-28 18:04:56,111] INFO : Step 8200/50000 rank6 | Train Loss: 1.8745 | Train Loss Unweighted: 1.0293 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.1946 | Classification: 0.1986 | Classification_binary: 0.2966 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2609 | Regression_eta: 0.0035 | Regression_pt: 0.2657 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1401 | Classification_binary: 0.2540 | Regression_cos_phi: 0.1726 | Regression_energy: 0.1372 | Regression_eta: 0.0888 | Regression_pt: 0.1544 | Regression_sin_phi: 0.1922 | LR: 9.42e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.3078s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7790 | max_loss: 0.1535 | max_backward: 1.9181 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.5352 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6080.6 | uss_MB: 5064.4 | child_rss_MB: 6184.5 | allocated_MB: 448.2 | reserved_MB: 25360.0 | inactive_split_MB: 1159.7 +rank6 [2026-10-28 18:09:37,296] INFO : Step 8300/50000 rank6 | Train Loss: 1.8727 | Train Loss Unweighted: 1.0119 | Train Loss Components: Auxiliary: 0.3401 | Cardinality: 0.1867 | Classification: 0.1728 | Classification_binary: 0.3272 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2513 | Regression_eta: 0.0033 | Regression_pt: 0.2533 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0963 | Classification_binary: 0.3323 | Regression_cos_phi: 0.1695 | Regression_energy: 0.1336 | Regression_eta: 0.0967 | Regression_pt: 0.1467 | Regression_sin_phi: 0.1729 | LR: 9.40e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.5444s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0436 | max_forward: 1.1303 | max_loss: 0.1503 | max_backward: 2.0213 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6082.4 | uss_MB: 5067.6 | child_rss_MB: 6182.5 | allocated_MB: 449.6 | reserved_MB: 27362.0 | inactive_split_MB: 1355.2 +rank6 [2026-10-28 18:14:10,521] INFO : Step 8400/50000 rank6 | Train Loss: 1.6705 | Train Loss Unweighted: 0.9205 | Train Loss Components: Auxiliary: 0.3113 | Cardinality: 0.1938 | Classification: 0.1656 | Classification_binary: 0.2512 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2485 | Regression_eta: 0.0028 | Regression_pt: 0.2494 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1110 | Classification_binary: 0.2704 | Regression_cos_phi: 0.1462 | Regression_energy: 0.1191 | Regression_eta: 0.0884 | Regression_pt: 0.1255 | Regression_sin_phi: 0.1021 | LR: 9.39e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4487s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8079 | max_loss: 0.1514 | max_backward: 1.6942 | max_optimizer_step: 0.2670 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6074.8 | uss_MB: 5060.4 | child_rss_MB: 6188.9 | allocated_MB: 446.4 | reserved_MB: 27366.0 | inactive_split_MB: 1386.9 +rank6 [2026-10-28 18:18:45,092] INFO : Step 8500/50000 rank6 | Train Loss: 1.8447 | Train Loss Unweighted: 0.9684 | Train Loss Components: Auxiliary: 0.3299 | Cardinality: 0.2197 | Classification: 0.1758 | Classification_binary: 0.2823 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2478 | Regression_eta: 0.0040 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1130 | Classification_binary: 0.2768 | Regression_cos_phi: 0.1241 | Regression_energy: 0.1297 | Regression_eta: 0.1061 | Regression_pt: 0.1433 | Regression_sin_phi: 0.1346 | LR: 9.37e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.1660s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0066 | max_forward: 0.7185 | max_loss: 0.1465 | max_backward: 1.6783 | max_optimizer_step: 0.2239 | max_tensorboard_logging: 0.6269 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 6077.8 | uss_MB: 5064.0 | child_rss_MB: 6185.3 | allocated_MB: 448.4 | reserved_MB: 27368.0 | inactive_split_MB: 1257.8 +rank6 [2026-10-28 18:23:30,152] INFO : Step 8600/50000 rank6 | Train Loss: 1.9431 | Train Loss Unweighted: 1.0237 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.2483 | Classification: 0.1814 | Classification_binary: 0.3328 | HitGrouping: 0.2024 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2495 | Regression_eta: 0.0034 | Regression_pt: 0.2527 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1430 | Classification_binary: 0.3435 | Regression_cos_phi: 0.2110 | Regression_energy: 0.1423 | Regression_eta: 0.1075 | Regression_pt: 0.1539 | Regression_sin_phi: 0.2369 | LR: 9.36e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.5552s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1011 | max_forward: 0.9340 | max_loss: 0.1508 | max_backward: 1.7704 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.7686 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 6082.0 | uss_MB: 5062.1 | child_rss_MB: 6191.5 | allocated_MB: 448.9 | reserved_MB: 27368.0 | inactive_split_MB: 1268.3 +rank6 [2026-10-28 18:28:08,444] INFO : Step 8700/50000 rank6 | Train Loss: 1.9739 | Train Loss Unweighted: 1.0707 | Train Loss Components: Auxiliary: 0.3725 | Cardinality: 0.2322 | Classification: 0.1854 | Classification_binary: 0.2934 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2900 | Regression_eta: 0.0037 | Regression_pt: 0.2944 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1234 | Classification_binary: 0.3188 | Regression_cos_phi: 0.1720 | Regression_energy: 0.1476 | Regression_eta: 0.1109 | Regression_pt: 0.1634 | Regression_sin_phi: 0.1395 | LR: 9.34e-04 | DataLoad Time: 0.0181s | Model Forward Time: 6.1640s | Phase Times: max_pre_step_barrier: 0.0890 | max_batch_to_device: 0.0067 | max_forward: 0.7726 | max_loss: 0.1574 | max_backward: 1.6332 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 2.5948 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 6082.7 | uss_MB: 5062.6 | child_rss_MB: 6185.4 | allocated_MB: 448.0 | reserved_MB: 29420.0 | inactive_split_MB: 876.0 +rank6 [2026-10-28 18:32:55,586] INFO : Step 8800/50000 rank6 | Train Loss: 1.5247 | Train Loss Unweighted: 0.8421 | Train Loss Components: Auxiliary: 0.2808 | Cardinality: 0.1524 | Classification: 0.1581 | Classification_binary: 0.2323 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2217 | Regression_eta: 0.0029 | Regression_pt: 0.2242 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0882 | Classification_binary: 0.2257 | Regression_cos_phi: 0.0954 | Regression_energy: 0.1053 | Regression_eta: 0.0672 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0915 | LR: 9.33e-04 | DataLoad Time: 0.0170s | Model Forward Time: 3.9900s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7382 | max_loss: 0.2414 | max_backward: 1.7741 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 6078.3 | uss_MB: 5061.0 | child_rss_MB: 6166.0 | allocated_MB: 447.9 | reserved_MB: 29420.0 | inactive_split_MB: 1260.3 +rank6 [2026-10-28 18:37:36,890] INFO : Step 8900/50000 rank6 | Train Loss: 1.7540 | Train Loss Unweighted: 0.9436 | Train Loss Components: Auxiliary: 0.3227 | Cardinality: 0.2072 | Classification: 0.1816 | Classification_binary: 0.2735 | HitGrouping: 0.1904 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2393 | Regression_eta: 0.0031 | Regression_pt: 0.2426 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1000 | Classification_binary: 0.2849 | Regression_cos_phi: 0.2042 | Regression_energy: 0.1253 | Regression_eta: 0.1058 | Regression_pt: 0.1360 | Regression_sin_phi: 0.2120 | LR: 9.31e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4468s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8077 | max_loss: 0.2089 | max_backward: 1.9976 | max_optimizer_step: 0.2211 | max_tensorboard_logging: 0.5747 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 6080.9 | uss_MB: 5000.4 | child_rss_MB: 6188.7 | allocated_MB: 451.8 | reserved_MB: 29420.0 | inactive_split_MB: 1133.8 +rank6 [2026-10-28 18:42:20,745] INFO : Step 9000/50000 rank6 | Train Loss: 1.7807 | Train Loss Unweighted: 0.9795 | Train Loss Components: Auxiliary: 0.3326 | Cardinality: 0.1942 | Classification: 0.1600 | Classification_binary: 0.3157 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2461 | Regression_eta: 0.0033 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1237 | Classification_binary: 0.3728 | Regression_cos_phi: 0.1687 | Regression_energy: 0.1359 | Regression_eta: 0.1175 | Regression_pt: 0.1503 | Regression_sin_phi: 0.1471 | LR: 9.30e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8576s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.8696 | max_loss: 0.1559 | max_backward: 1.8245 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.9520 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 6064.6 | uss_MB: 5051.1 | child_rss_MB: 6187.5 | allocated_MB: 448.9 | reserved_MB: 29420.0 | inactive_split_MB: 1051.6 +rank6 [2026-10-28 18:42:21,625] INFO : Running validation on rank6 +rank6 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 18:43:19,670] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:43:20,246] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 18:43:47,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 18:43:48,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 18:43:49,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 18:43:50,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 18:44:00,469] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 18:44:00,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:44:01,305] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 18:44:17,400] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 18:44:18,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 18:44:19,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 18:44:20,342] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 18:44:30,350] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 18:44:31,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 18:44:31,609] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 18:44:47,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 18:44:48,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 18:44:49,578] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 18:44:50,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 18:45:00,388] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 18:50:14,978] INFO : Step 9100/50000 rank6 | Train Loss: 1.6002 | Train Loss Unweighted: 0.8685 | Train Loss Components: Auxiliary: 0.3013 | Cardinality: 0.1975 | Classification: 0.1658 | Classification_binary: 0.2349 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2304 | Regression_eta: 0.0027 | Regression_pt: 0.2318 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1065 | Classification_binary: 0.2502 | Regression_cos_phi: 0.1103 | Regression_energy: 0.1222 | Regression_eta: 0.0797 | Regression_pt: 0.1300 | Regression_sin_phi: 0.1588 | LR: 9.28e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5180s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7981 | max_loss: 0.1460 | max_backward: 1.6475 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.9178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7065.6 | uss_MB: 6052.1 | child_rss_MB: 6167.4 | allocated_MB: 449.7 | reserved_MB: 25788.0 | inactive_split_MB: 957.3 +rank6 [2026-10-28 18:55:02,312] INFO : Step 9200/50000 rank6 | Train Loss: 1.6859 | Train Loss Unweighted: 0.8867 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.1906 | Classification: 0.1447 | Classification_binary: 0.2542 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2389 | Regression_eta: 0.0033 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0812 | Classification_binary: 0.2279 | Regression_cos_phi: 0.1630 | Regression_energy: 0.1384 | Regression_eta: 0.1044 | Regression_pt: 0.1496 | Regression_sin_phi: 0.1193 | LR: 9.27e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3385s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7865 | max_loss: 0.1441 | max_backward: 1.8922 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6128.5 | uss_MB: 5123.6 | child_rss_MB: 6161.7 | allocated_MB: 450.8 | reserved_MB: 25788.0 | inactive_split_MB: 1040.1 +rank6 [2026-10-28 18:59:49,907] INFO : Step 9300/50000 rank6 | Train Loss: 1.8980 | Train Loss Unweighted: 0.9669 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.3104 | Classification: 0.1729 | Classification_binary: 0.3073 | HitGrouping: 0.1936 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2397 | Regression_eta: 0.0030 | Regression_pt: 0.2407 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1165 | Classification_binary: 0.3057 | Regression_cos_phi: 0.1158 | Regression_energy: 0.1201 | Regression_eta: 0.0950 | Regression_pt: 0.1297 | Regression_sin_phi: 0.1449 | LR: 9.25e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.1833s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0143 | max_forward: 0.7819 | max_loss: 0.1515 | max_backward: 1.8209 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6071.1 | uss_MB: 5121.9 | child_rss_MB: 5951.4 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1031.1 +rank6 [2026-10-28 19:04:46,838] INFO : Step 9400/50000 rank6 | Train Loss: 1.5878 | Train Loss Unweighted: 0.8563 | Train Loss Components: Auxiliary: 0.2880 | Cardinality: 0.1552 | Classification: 0.1580 | Classification_binary: 0.2625 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2141 | Regression_eta: 0.0030 | Regression_pt: 0.2153 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0945 | Classification_binary: 0.2983 | Regression_cos_phi: 0.1162 | Regression_energy: 0.1321 | Regression_eta: 0.0778 | Regression_pt: 0.1452 | Regression_sin_phi: 0.1056 | LR: 9.24e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.0173s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7342 | max_loss: 0.2471 | max_backward: 1.7320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.4671 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6067.3 | uss_MB: 5118.7 | child_rss_MB: 5945.7 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1121.1 +rank6 [2026-10-28 19:09:47,342] INFO : Step 9500/50000 rank6 | Train Loss: 1.5519 | Train Loss Unweighted: 0.8141 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1719 | Classification: 0.1393 | Classification_binary: 0.2430 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2129 | Regression_eta: 0.0030 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0963 | Classification_binary: 0.2597 | Regression_cos_phi: 0.2178 | Regression_energy: 0.1153 | Regression_eta: 0.1040 | Regression_pt: 0.1218 | Regression_sin_phi: 0.1354 | LR: 9.22e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.1291s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0076 | max_forward: 0.7261 | max_loss: 0.1376 | max_backward: 1.8267 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6069.1 | uss_MB: 5122.8 | child_rss_MB: 5952.7 | allocated_MB: 446.7 | reserved_MB: 27752.0 | inactive_split_MB: 1128.4 +rank6 [2026-10-28 19:14:55,455] INFO : Step 9600/50000 rank6 | Train Loss: 1.7379 | Train Loss Unweighted: 0.9171 | Train Loss Components: Auxiliary: 0.3221 | Cardinality: 0.2199 | Classification: 0.1679 | Classification_binary: 0.2886 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2245 | Regression_eta: 0.0032 | Regression_pt: 0.2297 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1250 | Classification_binary: 0.2873 | Regression_cos_phi: 0.1690 | Regression_energy: 0.1221 | Regression_eta: 0.0907 | Regression_pt: 0.1335 | Regression_sin_phi: 0.1418 | LR: 9.21e-04 | DataLoad Time: 0.0174s | Model Forward Time: 7.7255s | Phase Times: max_pre_step_barrier: 0.0081 | max_batch_to_device: 0.0183 | max_forward: 3.1116 | max_loss: 0.7194 | max_backward: 4.4829 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6068.7 | uss_MB: 5120.4 | child_rss_MB: 5924.4 | allocated_MB: 449.4 | reserved_MB: 29732.0 | inactive_split_MB: 1322.6 +rank6 [2026-10-28 19:20:02,940] INFO : Step 9700/50000 rank6 | Train Loss: 1.8683 | Train Loss Unweighted: 0.9602 | Train Loss Components: Auxiliary: 0.3449 | Cardinality: 0.2461 | Classification: 0.1512 | Classification_binary: 0.3271 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2364 | Regression_eta: 0.0037 | Regression_pt: 0.2383 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1005 | Classification_binary: 0.3596 | Regression_cos_phi: 0.1240 | Regression_energy: 0.1302 | Regression_eta: 0.1035 | Regression_pt: 0.1381 | Regression_sin_phi: 0.1489 | LR: 9.19e-04 | DataLoad Time: 0.0183s | Model Forward Time: 5.0342s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0069 | max_forward: 0.8299 | max_loss: 0.7776 | max_backward: 2.5571 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5726 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6067.5 | uss_MB: 5123.7 | child_rss_MB: 5943.1 | allocated_MB: 450.3 | reserved_MB: 29734.0 | inactive_split_MB: 1128.1 +rank6 [2026-10-28 19:25:00,574] INFO : Step 9800/50000 rank6 | Train Loss: 1.6119 | Train Loss Unweighted: 0.8200 | Train Loss Components: Auxiliary: 0.3036 | Cardinality: 0.2266 | Classification: 0.1371 | Classification_binary: 0.2410 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2176 | Regression_eta: 0.0027 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0904 | Classification_binary: 0.2753 | Regression_cos_phi: 0.1226 | Regression_energy: 0.1030 | Regression_eta: 0.0774 | Regression_pt: 0.1138 | Regression_sin_phi: 0.1211 | LR: 9.17e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5956s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7430 | max_loss: 0.2467 | max_backward: 1.8961 | max_optimizer_step: 0.2008 | max_tensorboard_logging: 0.9056 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6064.8 | uss_MB: 5122.0 | child_rss_MB: 5925.9 | allocated_MB: 446.9 | reserved_MB: 29734.0 | inactive_split_MB: 905.1 +rank6 [2026-10-28 19:29:40,210] INFO : Step 9900/50000 rank6 | Train Loss: 1.5408 | Train Loss Unweighted: 0.8391 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1631 | Classification: 0.1532 | Classification_binary: 0.2416 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2178 | Regression_eta: 0.0029 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1319 | Classification_binary: 0.2066 | Regression_cos_phi: 0.1208 | Regression_energy: 0.1116 | Regression_eta: 0.0838 | Regression_pt: 0.1243 | Regression_sin_phi: 0.1352 | LR: 9.16e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6036s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7611 | max_loss: 0.1425 | max_backward: 1.8994 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6066.1 | uss_MB: 5122.8 | child_rss_MB: 5945.5 | allocated_MB: 448.3 | reserved_MB: 29734.0 | inactive_split_MB: 1119.7 +rank6 [2026-10-28 19:34:23,378] INFO : Step 10000/50000 rank6 | Train Loss: 1.7147 | Train Loss Unweighted: 0.9225 | Train Loss Components: Auxiliary: 0.3261 | Cardinality: 0.1969 | Classification: 0.1590 | Classification_binary: 0.2912 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2305 | Regression_eta: 0.0029 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0987 | Classification_binary: 0.3075 | Regression_cos_phi: 0.1300 | Regression_energy: 0.1260 | Regression_eta: 0.0760 | Regression_pt: 0.1419 | Regression_sin_phi: 0.1554 | LR: 9.14e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8797s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1140 | max_forward: 0.8951 | max_loss: 0.1695 | max_backward: 1.8837 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 1.0339 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 6055.8 | uss_MB: 5112.2 | child_rss_MB: 5922.2 | allocated_MB: 449.4 | reserved_MB: 29734.0 | inactive_split_MB: 1320.6 +rank6 [2026-10-28 19:34:24,225] INFO : Running validation on rank6 +rank6 [2026-10-28 19:34:24,225] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 19:34:24,226] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 19:35:21,694] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:35:22,180] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 19:35:48,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 19:35:49,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 19:35:50,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 19:35:52,010] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 19:36:02,017] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 19:36:02,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:36:03,005] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 19:36:20,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 19:36:23,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 19:36:24,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 19:36:25,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 19:36:35,868] INFO : Time taken to make predictions on device 6 is: 0.31 min +rank6 [2026-10-28 19:36:36,472] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 19:36:36,973] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 19:36:52,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 19:36:53,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 19:36:54,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 19:36:55,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 19:37:05,090] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 19:42:23,380] INFO : Step 10100/50000 rank6 | Train Loss: 1.6697 | Train Loss Unweighted: 0.8627 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.2287 | Classification: 0.1283 | Classification_binary: 0.2543 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2359 | Regression_eta: 0.0032 | Regression_pt: 0.2378 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0778 | Classification_binary: 0.2705 | Regression_cos_phi: 0.1320 | Regression_energy: 0.1131 | Regression_eta: 0.0865 | Regression_pt: 0.1268 | Regression_sin_phi: 0.1405 | LR: 9.12e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.7411s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7652 | max_loss: 0.1496 | max_backward: 1.8791 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 0.9326 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 7065.4 | uss_MB: 6119.6 | child_rss_MB: 5927.2 | allocated_MB: 448.1 | reserved_MB: 26360.0 | inactive_split_MB: 1080.0 +rank6 [2026-10-28 19:47:10,586] INFO : Step 10200/50000 rank6 | Train Loss: 1.3968 | Train Loss Unweighted: 0.7619 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1483 | Classification: 0.1293 | Classification_binary: 0.2070 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2090 | Regression_eta: 0.0026 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0853 | Classification_binary: 0.1954 | Regression_cos_phi: 0.0998 | Regression_energy: 0.1032 | Regression_eta: 0.0739 | Regression_pt: 0.1104 | Regression_sin_phi: 0.0890 | LR: 9.11e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.4826s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0330 | max_forward: 0.8960 | max_loss: 0.5080 | max_backward: 1.9040 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.6816 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 6137.8 | uss_MB: 5194.1 | child_rss_MB: 5912.5 | allocated_MB: 450.3 | reserved_MB: 27130.0 | inactive_split_MB: 1088.5 +rank6 [2026-10-28 19:51:56,122] INFO : Step 10300/50000 rank6 | Train Loss: 1.6039 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.2943 | Cardinality: 0.1656 | Classification: 0.1487 | Classification_binary: 0.2679 | HitGrouping: 0.1881 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2132 | Regression_eta: 0.0030 | Regression_pt: 0.2158 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0986 | Classification_binary: 0.2698 | Regression_cos_phi: 0.1168 | Regression_energy: 0.1083 | Regression_eta: 0.0693 | Regression_pt: 0.1200 | Regression_sin_phi: 0.1419 | LR: 9.09e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7917s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8152 | max_loss: 0.2478 | max_backward: 2.0483 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6143.7 | uss_MB: 5198.5 | child_rss_MB: 5930.4 | allocated_MB: 448.7 | reserved_MB: 28404.0 | inactive_split_MB: 991.9 +rank6 [2026-10-28 19:56:39,452] INFO : Step 10400/50000 rank6 | Train Loss: 1.7188 | Train Loss Unweighted: 0.8798 | Train Loss Components: Auxiliary: 0.3209 | Cardinality: 0.2578 | Classification: 0.1544 | Classification_binary: 0.2910 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2139 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0995 | Classification_binary: 0.3026 | Regression_cos_phi: 0.1382 | Regression_energy: 0.1174 | Regression_eta: 0.0763 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0869 | LR: 9.07e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.6761s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7821 | max_loss: 0.2448 | max_backward: 1.8782 | max_optimizer_step: 0.2932 | max_tensorboard_logging: 0.8061 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 6134.3 | uss_MB: 5193.2 | child_rss_MB: 5938.4 | allocated_MB: 448.9 | reserved_MB: 28404.0 | inactive_split_MB: 983.3 +rank6 [2026-10-28 20:01:29,564] INFO : Step 10500/50000 rank6 | Train Loss: 1.4652 | Train Loss Unweighted: 0.7986 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1697 | Classification: 0.1280 | Classification_binary: 0.2159 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2232 | Regression_eta: 0.0026 | Regression_pt: 0.2263 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2100 | Regression_cos_phi: 0.1018 | Regression_energy: 0.0970 | Regression_eta: 0.0703 | Regression_pt: 0.1065 | Regression_sin_phi: 0.0978 | LR: 9.06e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.2136s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.7113 | max_loss: 0.1455 | max_backward: 1.6308 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.7760 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6134.1 | uss_MB: 5192.0 | child_rss_MB: 5916.0 | allocated_MB: 444.7 | reserved_MB: 28404.0 | inactive_split_MB: 1307.3 +rank6 [2026-10-28 20:06:14,520] INFO : Step 10600/50000 rank6 | Train Loss: 1.5657 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2802 | Cardinality: 0.1297 | Classification: 0.1617 | Classification_binary: 0.2471 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2084 | Regression_eta: 0.0037 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1178 | Classification_binary: 0.2152 | Regression_cos_phi: 0.1200 | Regression_energy: 0.1237 | Regression_eta: 0.2239 | Regression_pt: 0.1300 | Regression_sin_phi: 0.1060 | LR: 9.04e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.9407s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0173 | max_forward: 0.7875 | max_loss: 0.2211 | max_backward: 1.9019 | max_optimizer_step: 0.2814 | max_tensorboard_logging: 0.8518 | max_tensorboard_flush: 0.0072 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6133.2 | uss_MB: 5193.0 | child_rss_MB: 5934.7 | allocated_MB: 449.8 | reserved_MB: 28404.0 | inactive_split_MB: 1130.2 +rank6 [2026-10-28 20:10:59,504] INFO : Step 10700/50000 rank6 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8676 | Train Loss Components: Auxiliary: 0.3003 | Cardinality: 0.1749 | Classification: 0.1453 | Classification_binary: 0.2956 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2085 | Regression_eta: 0.0033 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1031 | Classification_binary: 0.3362 | Regression_cos_phi: 0.1493 | Regression_energy: 0.1197 | Regression_eta: 0.0914 | Regression_pt: 0.1329 | Regression_sin_phi: 0.1916 | LR: 9.02e-04 | DataLoad Time: 0.0120s | Model Forward Time: 5.1526s | Phase Times: max_pre_step_barrier: 0.0305 | max_batch_to_device: 0.0154 | max_forward: 2.0246 | max_loss: 0.2864 | max_backward: 2.7424 | max_optimizer_step: 0.1980 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6136.9 | uss_MB: 5195.8 | child_rss_MB: 5906.4 | allocated_MB: 446.2 | reserved_MB: 29148.0 | inactive_split_MB: 1134.5 +rank6 [2026-10-28 20:15:41,634] INFO : Step 10800/50000 rank6 | Train Loss: 1.5849 | Train Loss Unweighted: 0.7936 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1942 | Classification: 0.1383 | Classification_binary: 0.2480 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2007 | Regression_eta: 0.0032 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.2445 | Regression_cos_phi: 0.0862 | Regression_energy: 0.0983 | Regression_eta: 0.0865 | Regression_pt: 0.1063 | Regression_sin_phi: 0.1538 | LR: 9.00e-04 | DataLoad Time: 0.0174s | Model Forward Time: 8.1988s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2408 | max_backward: 1.7977 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 1.7470 | max_tensorboard_flush: 0.0028 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 6126.4 | uss_MB: 4363.9 | child_rss_MB: 5910.3 | allocated_MB: 448.6 | reserved_MB: 30628.0 | inactive_split_MB: 1292.6 +rank6 [2026-10-28 20:20:32,831] INFO : Step 10900/50000 rank6 | Train Loss: 1.3552 | Train Loss Unweighted: 0.7247 | Train Loss Components: Auxiliary: 0.2531 | Cardinality: 0.1449 | Classification: 0.1275 | Classification_binary: 0.2074 | HitGrouping: 0.1874 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1917 | Regression_eta: 0.0024 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0874 | Classification_binary: 0.2044 | Regression_cos_phi: 0.1382 | Regression_energy: 0.1051 | Regression_eta: 0.0550 | Regression_pt: 0.1106 | Regression_sin_phi: 0.1047 | LR: 8.99e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.4538s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7076 | max_loss: 0.2425 | max_backward: 1.6536 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6131.8 | uss_MB: 5191.3 | child_rss_MB: 5932.5 | allocated_MB: 446.4 | reserved_MB: 30628.0 | inactive_split_MB: 984.9 +rank6 [2026-10-28 20:25:36,607] INFO : Step 11000/50000 rank6 | Train Loss: 1.5999 | Train Loss Unweighted: 0.8322 | Train Loss Components: Auxiliary: 0.2959 | Cardinality: 0.1737 | Classification: 0.1477 | Classification_binary: 0.2877 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0017 | Regression_energy: 0.1943 | Regression_eta: 0.0030 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0735 | Classification_binary: 0.2731 | Regression_cos_phi: 0.1340 | Regression_energy: 0.1082 | Regression_eta: 0.0702 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0965 | LR: 8.97e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9310s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8260 | max_loss: 0.1503 | max_backward: 1.9603 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6124.9 | uss_MB: 5185.7 | child_rss_MB: 5918.3 | allocated_MB: 451.8 | reserved_MB: 30628.0 | inactive_split_MB: 996.6 +rank6 [2026-10-28 20:25:37,479] INFO : Running validation on rank6 +rank6 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 20:26:32,847] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:26:33,361] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 20:27:00,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 20:27:01,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 20:27:02,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 20:27:03,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 20:27:13,521] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 20:27:14,319] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:27:14,833] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 20:27:31,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 20:27:32,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 20:27:33,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 20:27:35,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 20:27:45,257] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 20:27:46,111] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 20:27:46,648] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 20:28:02,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 20:28:03,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 20:28:04,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 20:28:04,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 20:28:14,985] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 20:33:29,189] INFO : Step 11100/50000 rank6 | Train Loss: 1.6214 | Train Loss Unweighted: 0.8328 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.2186 | Classification: 0.1350 | Classification_binary: 0.2802 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2042 | Regression_eta: 0.0030 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0804 | Classification_binary: 0.2818 | Regression_cos_phi: 0.1108 | Regression_energy: 0.0971 | Regression_eta: 0.0710 | Regression_pt: 0.1076 | Regression_sin_phi: 0.1173 | LR: 8.95e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.8750s | Phase Times: max_pre_step_barrier: 0.0088 | max_batch_to_device: 0.0068 | max_forward: 0.8488 | max_loss: 0.1548 | max_backward: 1.7645 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 1.1046 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7069.9 | uss_MB: 6125.7 | child_rss_MB: 5940.1 | allocated_MB: 448.1 | reserved_MB: 24340.0 | inactive_split_MB: 1081.9 +rank6 [2026-10-28 20:38:18,706] INFO : Step 11200/50000 rank6 | Train Loss: 1.7050 | Train Loss Unweighted: 0.8927 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2640 | Classification: 0.1534 | Classification_binary: 0.2523 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2404 | Regression_eta: 0.0030 | Regression_pt: 0.2410 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.2718 | Regression_cos_phi: 0.1170 | Regression_energy: 0.1470 | Regression_eta: 0.0838 | Regression_pt: 0.1626 | Regression_sin_phi: 0.1010 | LR: 8.93e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.8971s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8707 | max_loss: 0.2001 | max_backward: 2.2037 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.7912 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 6163.4 | uss_MB: 5218.5 | child_rss_MB: 5949.2 | allocated_MB: 447.8 | reserved_MB: 26392.0 | inactive_split_MB: 938.7 +rank6 [2026-10-28 20:43:01,723] INFO : Step 11300/50000 rank6 | Train Loss: 1.3885 | Train Loss Unweighted: 0.7511 | Train Loss Components: Auxiliary: 0.2594 | Cardinality: 0.1357 | Classification: 0.1316 | Classification_binary: 0.2357 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1874 | Regression_eta: 0.0024 | Regression_pt: 0.1915 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0854 | Classification_binary: 0.2046 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0993 | Regression_eta: 0.0490 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0764 | LR: 8.91e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.4284s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8053 | max_loss: 0.1469 | max_backward: 2.0062 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.4618 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1345 | Memory: rss_MB: 6165.0 | uss_MB: 5222.0 | child_rss_MB: 5948.8 | allocated_MB: 450.6 | reserved_MB: 26400.0 | inactive_split_MB: 1092.9 +rank6 [2026-10-28 20:48:05,522] INFO : Step 11400/50000 rank6 | Train Loss: 1.5865 | Train Loss Unweighted: 0.8323 | Train Loss Components: Auxiliary: 0.2981 | Cardinality: 0.1866 | Classification: 0.1338 | Classification_binary: 0.2899 | HitGrouping: 0.1840 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1999 | Regression_eta: 0.0028 | Regression_pt: 0.2030 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.2693 | Regression_cos_phi: 0.1018 | Regression_energy: 0.1075 | Regression_eta: 0.0636 | Regression_pt: 0.1173 | Regression_sin_phi: 0.1275 | LR: 8.89e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.7653s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0769 | max_forward: 1.3800 | max_loss: 0.6715 | max_backward: 3.0339 | max_optimizer_step: 0.2660 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6163.4 | uss_MB: 5219.6 | child_rss_MB: 5946.4 | allocated_MB: 448.1 | reserved_MB: 27144.0 | inactive_split_MB: 1081.9 +rank6 [2026-10-28 20:52:57,976] INFO : Step 11500/50000 rank6 | Train Loss: 1.4825 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2779 | Cardinality: 0.1999 | Classification: 0.1273 | Classification_binary: 0.2463 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1956 | Regression_eta: 0.0026 | Regression_pt: 0.1972 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0801 | Classification_binary: 0.2542 | Regression_cos_phi: 0.0974 | Regression_energy: 0.0993 | Regression_eta: 0.0848 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0949 | LR: 8.88e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.0564s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.7822 | max_loss: 0.1471 | max_backward: 1.7308 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 1.4583 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6164.2 | uss_MB: 5219.4 | child_rss_MB: 5944.4 | allocated_MB: 450.8 | reserved_MB: 27144.0 | inactive_split_MB: 1342.3 +rank6 [2026-10-28 20:57:41,223] INFO : Step 11600/50000 rank6 | Train Loss: 1.4219 | Train Loss Unweighted: 0.7444 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1679 | Classification: 0.1415 | Classification_binary: 0.2106 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1945 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0798 | Classification_binary: 0.1880 | Regression_cos_phi: 0.0954 | Regression_energy: 0.0939 | Regression_eta: 0.0500 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0792 | LR: 8.86e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7009s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8362 | max_loss: 0.1428 | max_backward: 2.0109 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6156.9 | uss_MB: 5220.2 | child_rss_MB: 5947.3 | allocated_MB: 448.6 | reserved_MB: 27146.0 | inactive_split_MB: 1315.4 +rank6 [2026-10-28 21:02:35,116] INFO : Step 11700/50000 rank6 | Train Loss: 1.5203 | Train Loss Unweighted: 0.8017 | Train Loss Components: Auxiliary: 0.2866 | Cardinality: 0.1889 | Classification: 0.1302 | Classification_binary: 0.2577 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2024 | Regression_eta: 0.0026 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0760 | Classification_binary: 0.2667 | Regression_cos_phi: 0.1149 | Regression_energy: 0.1099 | Regression_eta: 0.0891 | Regression_pt: 0.1211 | Regression_sin_phi: 0.1244 | LR: 8.84e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.3171s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 1.0014 | max_loss: 0.1975 | max_backward: 2.5347 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0688 | Memory: rss_MB: 6157.0 | uss_MB: 5144.6 | child_rss_MB: 5922.6 | allocated_MB: 448.7 | reserved_MB: 27266.0 | inactive_split_MB: 1090.9 +rank6 [2026-10-28 21:07:42,591] INFO : Step 11800/50000 rank6 | Train Loss: 1.5591 | Train Loss Unweighted: 0.7907 | Train Loss Components: Auxiliary: 0.2873 | Cardinality: 0.2051 | Classification: 0.1346 | Classification_binary: 0.2655 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1918 | Regression_eta: 0.0030 | Regression_pt: 0.1929 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.2892 | Regression_cos_phi: 0.1241 | Regression_energy: 0.0875 | Regression_eta: 0.0825 | Regression_pt: 0.0958 | Regression_sin_phi: 0.1494 | LR: 8.82e-04 | DataLoad Time: 0.0186s | Model Forward Time: 7.4773s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.0966 | max_forward: 1.7907 | max_loss: 0.6496 | max_backward: 2.7103 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 2.3587 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6136.9 | uss_MB: 5209.7 | child_rss_MB: 5946.4 | allocated_MB: 450.4 | reserved_MB: 27266.0 | inactive_split_MB: 1187.2 +rank6 [2026-10-28 21:12:51,258] INFO : Step 11900/50000 rank6 | Train Loss: 1.3283 | Train Loss Unweighted: 0.7113 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1476 | Classification: 0.1193 | Classification_binary: 0.2094 | HitGrouping: 0.1824 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0021 | Regression_pt: 0.1901 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0779 | Classification_binary: 0.2079 | Regression_cos_phi: 0.1295 | Regression_energy: 0.0973 | Regression_eta: 0.0550 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0997 | LR: 8.80e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.1700s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0785 | max_forward: 0.7917 | max_loss: 0.1465 | max_backward: 1.5595 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6137.9 | uss_MB: 5216.4 | child_rss_MB: 5947.1 | allocated_MB: 443.9 | reserved_MB: 27266.0 | inactive_split_MB: 955.3 +rank6 [2026-10-28 21:17:48,735] INFO : Step 12000/50000 rank6 | Train Loss: 1.4074 | Train Loss Unweighted: 0.7415 | Train Loss Components: Auxiliary: 0.2683 | Cardinality: 0.1719 | Classification: 0.1235 | Classification_binary: 0.2283 | HitGrouping: 0.1812 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1909 | Regression_eta: 0.0022 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0950 | Classification_binary: 0.2632 | Regression_cos_phi: 0.1154 | Regression_energy: 0.1138 | Regression_eta: 0.0846 | Regression_pt: 0.1247 | Regression_sin_phi: 0.1010 | LR: 8.78e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.2532s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0303 | max_forward: 0.7589 | max_loss: 0.1459 | max_backward: 1.9073 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6136.6 | uss_MB: 5219.4 | child_rss_MB: 5952.0 | allocated_MB: 447.6 | reserved_MB: 27266.0 | inactive_split_MB: 1310.4 +rank6 [2026-10-28 21:17:49,718] INFO : Running validation on rank6 +rank6 [2026-10-28 21:17:49,718] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 21:17:49,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 21:18:44,924] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:18:45,602] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 21:19:11,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 21:19:12,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 21:19:13,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 21:19:14,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 21:19:24,776] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 21:19:25,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:19:25,829] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 21:19:41,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 21:19:42,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 21:19:43,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 21:19:44,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 21:19:54,471] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 21:19:55,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 21:19:56,387] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 21:20:12,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 21:20:12,937] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 21:20:13,728] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 21:20:14,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 21:20:24,517] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 21:25:33,195] INFO : Step 12100/50000 rank6 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8521 | Train Loss Components: Auxiliary: 0.3093 | Cardinality: 0.2088 | Classification: 0.1337 | Classification_binary: 0.2971 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0029 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0712 | Classification_binary: 0.2669 | Regression_cos_phi: 0.0958 | Regression_energy: 0.1027 | Regression_eta: 0.0659 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0951 | LR: 8.76e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.9403s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8139 | max_loss: 0.1586 | max_backward: 1.9999 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9439 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7027.3 | uss_MB: 6104.0 | child_rss_MB: 5925.8 | allocated_MB: 450.8 | reserved_MB: 28008.0 | inactive_split_MB: 1125.6 +rank6 [2026-10-28 21:30:25,391] INFO : Step 12200/50000 rank6 | Train Loss: 1.4655 | Train Loss Unweighted: 0.7431 | Train Loss Components: Auxiliary: 0.2757 | Cardinality: 0.1845 | Classification: 0.1184 | Classification_binary: 0.2450 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1862 | Regression_eta: 0.0026 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0858 | Classification_binary: 0.2834 | Regression_cos_phi: 0.1461 | Regression_energy: 0.1005 | Regression_eta: 0.1438 | Regression_pt: 0.1111 | Regression_sin_phi: 0.1589 | LR: 8.74e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4993s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0364 | max_forward: 0.7684 | max_loss: 0.1438 | max_backward: 1.8240 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.7243 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0697 | Memory: rss_MB: 6148.5 | uss_MB: 5233.4 | child_rss_MB: 5949.1 | allocated_MB: 445.9 | reserved_MB: 28008.0 | inactive_split_MB: 995.3 +rank6 [2026-10-28 21:35:16,327] INFO : Step 12300/50000 rank6 | Train Loss: 1.3845 | Train Loss Unweighted: 0.7270 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1523 | Classification: 0.1341 | Classification_binary: 0.2109 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1869 | Regression_eta: 0.0026 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0928 | Classification_binary: 0.2161 | Regression_cos_phi: 0.1044 | Regression_energy: 0.0808 | Regression_eta: 0.0812 | Regression_pt: 0.0878 | Regression_sin_phi: 0.1256 | LR: 8.72e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5873s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7264 | max_loss: 0.1976 | max_backward: 1.8534 | max_optimizer_step: 0.2993 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6147.3 | uss_MB: 5233.4 | child_rss_MB: 5944.1 | allocated_MB: 448.1 | reserved_MB: 29988.0 | inactive_split_MB: 1298.2 +rank6 [2026-10-28 21:40:15,076] INFO : Step 12400/50000 rank6 | Train Loss: 1.4006 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.2654 | Cardinality: 0.1602 | Classification: 0.1310 | Classification_binary: 0.2363 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0927 | Classification_binary: 0.2609 | Regression_cos_phi: 0.1286 | Regression_energy: 0.0959 | Regression_eta: 0.0798 | Regression_pt: 0.1068 | Regression_sin_phi: 0.1348 | LR: 8.70e-04 | DataLoad Time: 0.0169s | Model Forward Time: 7.1202s | Phase Times: max_pre_step_barrier: 0.0107 | max_batch_to_device: 0.0207 | max_forward: 1.2284 | max_loss: 0.2740 | max_backward: 2.6428 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8714 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 6147.7 | uss_MB: 5022.1 | child_rss_MB: 5936.9 | allocated_MB: 448.8 | reserved_MB: 29988.0 | inactive_split_MB: 1332.2 +rank6 [2026-10-28 21:45:07,595] INFO : Step 12500/50000 rank6 | Train Loss: 1.4931 | Train Loss Unweighted: 0.7691 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1768 | Classification: 0.1197 | Classification_binary: 0.2587 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1910 | Regression_eta: 0.0024 | Regression_pt: 0.1944 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0801 | Classification_binary: 0.2554 | Regression_cos_phi: 0.1075 | Regression_energy: 0.1009 | Regression_eta: 0.0614 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0792 | LR: 8.68e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7422s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0333 | max_forward: 0.8806 | max_loss: 0.1518 | max_backward: 2.1554 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.5983 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6144.5 | uss_MB: 5232.3 | child_rss_MB: 5946.8 | allocated_MB: 452.8 | reserved_MB: 29990.0 | inactive_split_MB: 1046.7 +rank6 [2026-10-28 21:50:03,650] INFO : Step 12600/50000 rank6 | Train Loss: 1.3547 | Train Loss Unweighted: 0.7028 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1629 | Classification: 0.1033 | Classification_binary: 0.2112 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1912 | Regression_eta: 0.0024 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0840 | Classification_binary: 0.2503 | Regression_cos_phi: 0.0988 | Regression_energy: 0.1110 | Regression_eta: 0.0863 | Regression_pt: 0.1197 | Regression_sin_phi: 0.1331 | LR: 8.66e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5165s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.7639 | max_loss: 0.2516 | max_backward: 1.8947 | max_optimizer_step: 0.2049 | max_tensorboard_logging: 0.7350 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6141.1 | uss_MB: 5233.0 | child_rss_MB: 5920.5 | allocated_MB: 446.9 | reserved_MB: 29990.0 | inactive_split_MB: 1034.9 +rank6 [2026-10-28 21:54:54,645] INFO : Step 12700/50000 rank6 | Train Loss: 1.3557 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1573 | Classification: 0.1262 | Classification_binary: 0.2132 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1863 | Regression_eta: 0.0022 | Regression_pt: 0.1887 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0723 | Classification_binary: 0.2072 | Regression_cos_phi: 0.0984 | Regression_energy: 0.1146 | Regression_eta: 0.0700 | Regression_pt: 0.1270 | Regression_sin_phi: 0.1277 | LR: 8.64e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.1264s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7499 | max_loss: 0.2453 | max_backward: 3.4334 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6139.6 | uss_MB: 5232.1 | child_rss_MB: 5948.4 | allocated_MB: 450.2 | reserved_MB: 29990.0 | inactive_split_MB: 1245.6 +rank6 [2026-10-28 21:59:39,371] INFO : Step 12800/50000 rank6 | Train Loss: 1.5298 | Train Loss Unweighted: 0.7754 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1939 | Classification: 0.1237 | Classification_binary: 0.2821 | HitGrouping: 0.1822 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1810 | Regression_eta: 0.0027 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.2901 | Regression_cos_phi: 0.0815 | Regression_energy: 0.1021 | Regression_eta: 0.0615 | Regression_pt: 0.1111 | Regression_sin_phi: 0.1051 | LR: 8.62e-04 | DataLoad Time: 0.0188s | Model Forward Time: 5.2926s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.5299 | max_backward: 2.3185 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6133.4 | uss_MB: 4813.1 | child_rss_MB: 5927.2 | allocated_MB: 449.9 | reserved_MB: 32006.0 | inactive_split_MB: 889.3 +rank6 [2026-10-28 22:04:39,768] INFO : Step 12900/50000 rank6 | Train Loss: 1.3302 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1576 | Classification: 0.1054 | Classification_binary: 0.2343 | HitGrouping: 0.1811 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1758 | Regression_eta: 0.0020 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.2132 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0926 | Regression_eta: 0.0419 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0740 | LR: 8.60e-04 | DataLoad Time: 0.0209s | Model Forward Time: 5.6384s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0137 | max_forward: 0.9836 | max_loss: 0.2293 | max_backward: 2.2888 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.9302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6139.6 | uss_MB: 5231.4 | child_rss_MB: 5950.5 | allocated_MB: 445.9 | reserved_MB: 32006.0 | inactive_split_MB: 1247.6 +rank6 [2026-10-28 22:09:25,338] INFO : Step 13000/50000 rank6 | Train Loss: 1.3124 | Train Loss Unweighted: 0.7073 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1406 | Classification: 0.1147 | Classification_binary: 0.1973 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1944 | Regression_eta: 0.0023 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0876 | Classification_binary: 0.1886 | Regression_cos_phi: 0.1311 | Regression_energy: 0.1128 | Regression_eta: 0.0846 | Regression_pt: 0.1222 | Regression_sin_phi: 0.1095 | LR: 8.58e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2802s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0065 | max_forward: 0.7321 | max_loss: 0.1616 | max_backward: 1.7374 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6140.2 | uss_MB: 5231.6 | child_rss_MB: 5949.6 | allocated_MB: 444.7 | reserved_MB: 32006.0 | inactive_split_MB: 1300.2 +rank6 [2026-10-28 22:09:26,329] INFO : Running validation on rank6 +rank6 [2026-10-28 22:09:26,329] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 22:09:26,330] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 22:10:21,902] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:10:22,475] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 22:10:48,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 22:10:49,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 22:10:50,821] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 22:10:51,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 22:11:01,840] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 22:11:02,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:11:02,876] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 22:11:19,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 22:11:20,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 22:11:21,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 22:11:22,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 22:11:32,533] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 22:11:32,691] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 22:11:33,218] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 22:11:48,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 22:11:49,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 22:11:50,319] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 22:11:51,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 22:12:01,136] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-28 22:17:08,485] INFO : Step 13100/50000 rank6 | Train Loss: 1.6165 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3082 | Cardinality: 0.2635 | Classification: 0.1364 | Classification_binary: 0.2628 | HitGrouping: 0.1835 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0026 | Regression_pt: 0.1957 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0944 | Classification_binary: 0.2451 | Regression_cos_phi: 0.1415 | Regression_energy: 0.0984 | Regression_eta: 0.0924 | Regression_pt: 0.1062 | Regression_sin_phi: 0.1311 | LR: 8.56e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2699s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0479 | max_forward: 0.8994 | max_loss: 0.2807 | max_backward: 2.1108 | max_optimizer_step: 0.2760 | max_tensorboard_logging: 0.9690 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 7057.8 | uss_MB: 6143.5 | child_rss_MB: 5948.1 | allocated_MB: 446.5 | reserved_MB: 23120.0 | inactive_split_MB: 920.0 +rank6 [2026-10-28 22:22:01,650] INFO : Step 13200/50000 rank6 | Train Loss: 1.4928 | Train Loss Unweighted: 0.7469 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.2211 | Classification: 0.1278 | Classification_binary: 0.2505 | HitGrouping: 0.1746 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1808 | Regression_eta: 0.0025 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0814 | Classification_binary: 0.2674 | Regression_cos_phi: 0.1193 | Regression_energy: 0.0817 | Regression_eta: 0.0724 | Regression_pt: 0.0883 | Regression_sin_phi: 0.1012 | LR: 8.54e-04 | DataLoad Time: 0.0204s | Model Forward Time: 6.5974s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.7755 | max_loss: 0.2864 | max_backward: 3.4589 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6222.1 | uss_MB: 5074.0 | child_rss_MB: 5943.3 | allocated_MB: 448.3 | reserved_MB: 25156.0 | inactive_split_MB: 978.0 +rank6 [2026-10-28 22:26:52,972] INFO : Step 13300/50000 rank6 | Train Loss: 1.3251 | Train Loss Unweighted: 0.6741 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1607 | Classification: 0.1132 | Classification_binary: 0.1927 | HitGrouping: 0.1773 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1805 | Regression_eta: 0.0024 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.1969 | Regression_cos_phi: 0.0936 | Regression_energy: 0.0907 | Regression_eta: 0.0720 | Regression_pt: 0.0998 | Regression_sin_phi: 0.0898 | LR: 8.52e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3790s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8118 | max_loss: 0.1467 | max_backward: 1.5503 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.9039 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6221.5 | uss_MB: 5307.8 | child_rss_MB: 5952.8 | allocated_MB: 446.0 | reserved_MB: 26736.0 | inactive_split_MB: 965.8 +rank6 [2026-10-28 22:31:44,118] INFO : Step 13400/50000 rank6 | Train Loss: 1.2635 | Train Loss Unweighted: 0.6770 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1436 | Classification: 0.1317 | Classification_binary: 0.2016 | HitGrouping: 0.1790 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1692 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1113 | Classification_binary: 0.1910 | Regression_cos_phi: 0.1427 | Regression_energy: 0.0933 | Regression_eta: 0.0638 | Regression_pt: 0.1015 | Regression_sin_phi: 0.1088 | LR: 8.50e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7357s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8396 | max_loss: 0.1412 | max_backward: 2.0130 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.8047 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6225.0 | uss_MB: 5310.9 | child_rss_MB: 5925.4 | allocated_MB: 445.8 | reserved_MB: 26736.0 | inactive_split_MB: 1014.4 +rank6 [2026-10-28 22:36:27,627] INFO : Step 13500/50000 rank6 | Train Loss: 1.3935 | Train Loss Unweighted: 0.7401 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1713 | Classification: 0.1245 | Classification_binary: 0.2541 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0021 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0721 | Classification_binary: 0.2539 | Regression_cos_phi: 0.1098 | Regression_energy: 0.0962 | Regression_eta: 0.0567 | Regression_pt: 0.1039 | Regression_sin_phi: 0.1312 | LR: 8.48e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5755s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0859 | max_forward: 0.8721 | max_loss: 0.1498 | max_backward: 1.7581 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.7855 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6221.6 | uss_MB: 5308.9 | child_rss_MB: 5951.2 | allocated_MB: 449.1 | reserved_MB: 26736.0 | inactive_split_MB: 922.5 +rank6 [2026-10-28 22:41:19,303] INFO : Step 13600/50000 rank6 | Train Loss: 1.3997 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1870 | Classification: 0.1206 | Classification_binary: 0.2286 | HitGrouping: 0.1765 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1702 | Regression_eta: 0.0026 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.2615 | Regression_cos_phi: 0.0918 | Regression_energy: 0.1132 | Regression_eta: 0.0867 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0937 | LR: 8.45e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.2263s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7703 | max_loss: 0.2484 | max_backward: 1.6943 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 6210.7 | uss_MB: 5299.9 | child_rss_MB: 5953.4 | allocated_MB: 450.3 | reserved_MB: 26736.0 | inactive_split_MB: 922.0 +rank6 [2026-10-28 22:46:12,326] INFO : Step 13700/50000 rank6 | Train Loss: 1.2648 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2420 | Cardinality: 0.1315 | Classification: 0.1136 | Classification_binary: 0.1964 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1793 | Regression_eta: 0.0020 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0905 | Classification_binary: 0.2088 | Regression_cos_phi: 0.1111 | Regression_energy: 0.0926 | Regression_eta: 0.0561 | Regression_pt: 0.1011 | Regression_sin_phi: 0.1283 | LR: 8.43e-04 | DataLoad Time: 0.0166s | Model Forward Time: 6.2736s | Phase Times: max_pre_step_barrier: 0.0406 | max_batch_to_device: 0.0119 | max_forward: 1.2281 | max_loss: 0.2480 | max_backward: 3.6472 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.6460 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0767 | Memory: rss_MB: 6216.0 | uss_MB: 5307.4 | child_rss_MB: 5948.4 | allocated_MB: 446.9 | reserved_MB: 26742.0 | inactive_split_MB: 1058.7 +rank6 [2026-10-28 22:51:09,234] INFO : Step 13800/50000 rank6 | Train Loss: 1.3469 | Train Loss Unweighted: 0.7266 | Train Loss Components: Auxiliary: 0.2604 | Cardinality: 0.1436 | Classification: 0.1184 | Classification_binary: 0.2382 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1812 | Regression_eta: 0.0021 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.2264 | Regression_cos_phi: 0.0915 | Regression_energy: 0.1074 | Regression_eta: 0.0587 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0784 | LR: 8.41e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.8538s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7995 | max_loss: 0.1458 | max_backward: 3.4735 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6221.0 | uss_MB: 5310.1 | child_rss_MB: 5919.7 | allocated_MB: 446.4 | reserved_MB: 27082.0 | inactive_split_MB: 1370.3 +rank6 [2026-10-28 22:56:04,349] INFO : Step 13900/50000 rank6 | Train Loss: 1.4882 | Train Loss Unweighted: 0.7682 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1631 | Classification: 0.1199 | Classification_binary: 0.2624 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1900 | Regression_eta: 0.0029 | Regression_pt: 0.1902 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0892 | Classification_binary: 0.2433 | Regression_cos_phi: 0.1000 | Regression_energy: 0.0952 | Regression_eta: 0.0689 | Regression_pt: 0.1035 | Regression_sin_phi: 0.0999 | LR: 8.39e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4538s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.7750 | max_loss: 0.1563 | max_backward: 1.5716 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.8868 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6217.9 | uss_MB: 5307.2 | child_rss_MB: 5946.5 | allocated_MB: 449.9 | reserved_MB: 27316.0 | inactive_split_MB: 918.0 +rank6 [2026-10-28 23:00:54,751] INFO : Step 14000/50000 rank6 | Train Loss: 1.2239 | Train Loss Unweighted: 0.6384 | Train Loss Components: Auxiliary: 0.2362 | Cardinality: 0.1573 | Classification: 0.0898 | Classification_binary: 0.1853 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0021 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2104 | Regression_cos_phi: 0.1051 | Regression_energy: 0.0964 | Regression_eta: 0.0675 | Regression_pt: 0.1053 | Regression_sin_phi: 0.0892 | LR: 8.37e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.5027s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7611 | max_loss: 0.1442 | max_backward: 1.7887 | max_optimizer_step: 0.2976 | max_tensorboard_logging: 0.7418 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6219.2 | uss_MB: 5309.9 | child_rss_MB: 5948.9 | allocated_MB: 446.8 | reserved_MB: 29264.0 | inactive_split_MB: 1150.9 +rank6 [2026-10-28 23:00:55,745] INFO : Running validation on rank6 +rank6 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 23:01:50,596] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:01:51,150] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 23:02:17,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 23:02:19,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 23:02:20,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 23:02:22,199] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 23:02:32,210] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-28 23:02:33,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:02:33,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 23:02:49,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 23:02:50,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 23:02:51,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 23:02:52,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 23:03:02,885] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 23:03:03,407] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:03:03,862] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 23:03:19,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 23:03:20,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 23:03:21,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 23:03:22,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 23:03:32,220] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 23:08:22,314] INFO : Step 14100/50000 rank6 | Train Loss: 1.3472 | Train Loss Unweighted: 0.6964 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1511 | Classification: 0.1164 | Classification_binary: 0.2090 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1819 | Regression_eta: 0.0023 | Regression_pt: 0.1843 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1799 | Regression_cos_phi: 0.1291 | Regression_energy: 0.1065 | Regression_eta: 0.0631 | Regression_pt: 0.1180 | Regression_sin_phi: 0.1033 | LR: 8.35e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.2193s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8885 | max_loss: 0.1430 | max_backward: 2.1784 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 1.1214 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7149.4 | uss_MB: 6238.2 | child_rss_MB: 5960.3 | allocated_MB: 451.8 | reserved_MB: 25710.0 | inactive_split_MB: 1103.7 +rank6 [2026-10-28 23:12:52,862] INFO : Step 14200/50000 rank6 | Train Loss: 1.4358 | Train Loss Unweighted: 0.7354 | Train Loss Components: Auxiliary: 0.2744 | Cardinality: 0.1685 | Classification: 0.1248 | Classification_binary: 0.2623 | HitGrouping: 0.1814 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1709 | Regression_eta: 0.0023 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2230 | Regression_cos_phi: 0.0892 | Regression_energy: 0.0892 | Regression_eta: 0.0412 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0720 | LR: 8.32e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.3441s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0875 | max_forward: 0.8579 | max_loss: 0.1577 | max_backward: 1.8303 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6333.7 | uss_MB: 5422.0 | child_rss_MB: 5960.5 | allocated_MB: 449.2 | reserved_MB: 27638.0 | inactive_split_MB: 1319.6 +rank6 [2026-10-28 23:17:26,476] INFO : Step 14300/50000 rank6 | Train Loss: 1.2642 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2452 | Cardinality: 0.1652 | Classification: 0.1069 | Classification_binary: 0.2108 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0018 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.2371 | Regression_cos_phi: 0.0923 | Regression_energy: 0.0999 | Regression_eta: 0.0562 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0822 | LR: 8.30e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3452s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7599 | max_loss: 0.1477 | max_backward: 1.7369 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6331.9 | uss_MB: 5419.4 | child_rss_MB: 5959.7 | allocated_MB: 446.3 | reserved_MB: 29582.0 | inactive_split_MB: 949.7 +rank6 [2026-10-28 23:22:05,870] INFO : Step 14400/50000 rank6 | Train Loss: 1.2457 | Train Loss Unweighted: 0.6253 | Train Loss Components: Auxiliary: 0.2300 | Cardinality: 0.1408 | Classification: 0.1072 | Classification_binary: 0.1847 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1629 | Regression_eta: 0.0022 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.2049 | Regression_cos_phi: 0.0867 | Regression_energy: 0.0937 | Regression_eta: 0.0569 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0898 | LR: 8.28e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.9713s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.9667 | max_loss: 0.2306 | max_backward: 2.7434 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6331.8 | uss_MB: 4969.8 | child_rss_MB: 5956.2 | allocated_MB: 450.7 | reserved_MB: 29584.0 | inactive_split_MB: 1252.8 +rank6 [2026-10-28 23:26:50,539] INFO : Step 14500/50000 rank6 | Train Loss: 1.4093 | Train Loss Unweighted: 0.7194 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1803 | Classification: 0.1211 | Classification_binary: 0.2406 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.2775 | Regression_cos_phi: 0.0904 | Regression_energy: 0.0892 | Regression_eta: 0.0670 | Regression_pt: 0.0979 | Regression_sin_phi: 0.1545 | LR: 8.26e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.8480s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8204 | max_loss: 0.1455 | max_backward: 2.0838 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7592 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1066 | Memory: rss_MB: 6329.0 | uss_MB: 5423.5 | child_rss_MB: 5932.6 | allocated_MB: 449.1 | reserved_MB: 29584.0 | inactive_split_MB: 1312.1 +rank6 [2026-10-28 23:31:26,129] INFO : Step 14600/50000 rank6 | Train Loss: 1.4466 | Train Loss Unweighted: 0.7628 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1607 | Classification: 0.1257 | Classification_binary: 0.2761 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1767 | Regression_eta: 0.0023 | Regression_pt: 0.1796 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1007 | Classification_binary: 0.3154 | Regression_cos_phi: 0.1230 | Regression_energy: 0.0898 | Regression_eta: 0.0595 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0918 | LR: 8.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8555s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0622 | max_forward: 0.8658 | max_loss: 0.1496 | max_backward: 1.9094 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.9894 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6320.2 | uss_MB: 5414.6 | child_rss_MB: 5962.0 | allocated_MB: 450.2 | reserved_MB: 29584.0 | inactive_split_MB: 939.8 +rank6 [2026-10-28 23:35:59,706] INFO : Step 14700/50000 rank6 | Train Loss: 1.2298 | Train Loss Unweighted: 0.6544 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1321 | Classification: 0.0995 | Classification_binary: 0.1882 | HitGrouping: 0.1788 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1813 | Regression_eta: 0.0021 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2137 | Regression_cos_phi: 0.0714 | Regression_energy: 0.1115 | Regression_eta: 0.0452 | Regression_pt: 0.1196 | Regression_sin_phi: 0.1001 | LR: 8.21e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.9555s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8361 | max_loss: 0.2462 | max_backward: 1.9700 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.8314 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6323.9 | uss_MB: 5421.4 | child_rss_MB: 5964.3 | allocated_MB: 445.8 | reserved_MB: 31548.0 | inactive_split_MB: 1127.1 +rank6 [2026-10-28 23:40:34,504] INFO : Step 14800/50000 rank6 | Train Loss: 1.2682 | Train Loss Unweighted: 0.6574 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1516 | Classification: 0.1063 | Classification_binary: 0.2120 | HitGrouping: 0.1767 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1664 | Regression_eta: 0.0020 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.2100 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0823 | Regression_eta: 0.0505 | Regression_pt: 0.0900 | Regression_sin_phi: 0.0755 | LR: 8.19e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.6940s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7530 | max_loss: 0.1476 | max_backward: 1.8229 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6323.4 | uss_MB: 5420.5 | child_rss_MB: 5957.9 | allocated_MB: 449.4 | reserved_MB: 33554.0 | inactive_split_MB: 1332.6 +rank6 [2026-10-28 23:45:11,571] INFO : Step 14900/50000 rank6 | Train Loss: 1.2753 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2409 | Cardinality: 0.1305 | Classification: 0.1126 | Classification_binary: 0.2387 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1497 | Regression_eta: 0.0022 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0572 | Classification_binary: 0.2395 | Regression_cos_phi: 0.1231 | Regression_energy: 0.0777 | Regression_eta: 0.0535 | Regression_pt: 0.0849 | Regression_sin_phi: 0.1108 | LR: 8.17e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8746s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.7955 | max_loss: 0.1633 | max_backward: 1.8376 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.9623 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1206 | Memory: rss_MB: 6322.3 | uss_MB: 5420.8 | child_rss_MB: 5961.8 | allocated_MB: 449.7 | reserved_MB: 33554.0 | inactive_split_MB: 1310.1 +rank6 [2026-10-28 23:49:45,306] INFO : Step 15000/50000 rank6 | Train Loss: 1.2701 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1516 | Classification: 0.1012 | Classification_binary: 0.2121 | HitGrouping: 0.1755 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0715 | Classification_binary: 0.2383 | Regression_cos_phi: 0.1062 | Regression_energy: 0.1047 | Regression_eta: 0.0590 | Regression_pt: 0.1144 | Regression_sin_phi: 0.1285 | LR: 8.15e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1017s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7505 | max_loss: 0.2449 | max_backward: 1.7335 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.4870 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6318.2 | uss_MB: 5419.0 | child_rss_MB: 5953.6 | allocated_MB: 447.3 | reserved_MB: 33554.0 | inactive_split_MB: 1038.0 +rank6 [2026-10-28 23:49:46,286] INFO : Running validation on rank6 +rank6 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-28 23:50:37,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:50:38,005] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-28 23:51:04,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-28 23:51:05,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-28 23:51:06,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-28 23:51:07,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-28 23:51:17,883] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 23:51:18,491] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:51:18,969] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-28 23:51:34,699] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-28 23:51:35,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-28 23:51:36,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-28 23:51:37,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-28 23:51:47,597] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-28 23:51:48,549] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-28 23:51:49,026] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-28 23:52:04,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-28 23:52:05,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-28 23:52:06,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-28 23:52:07,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-28 23:52:17,028] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-28 23:57:18,880] INFO : Step 15100/50000 rank6 | Train Loss: 1.2388 | Train Loss Unweighted: 0.6590 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1366 | Classification: 0.1178 | Classification_binary: 0.1841 | HitGrouping: 0.1810 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1743 | Regression_eta: 0.0018 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1019 | Classification_binary: 0.1694 | Regression_cos_phi: 0.1032 | Regression_energy: 0.0862 | Regression_eta: 0.0509 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0969 | LR: 8.12e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.8262s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.8010 | max_loss: 0.1660 | max_backward: 1.9832 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.8565 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7422.7 | uss_MB: 6518.8 | child_rss_MB: 5938.1 | allocated_MB: 446.2 | reserved_MB: 25230.0 | inactive_split_MB: 1087.3 +rank6 [2026-10-29 00:02:05,558] INFO : Step 15200/50000 rank6 | Train Loss: 1.3362 | Train Loss Unweighted: 0.6611 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1596 | Classification: 0.1086 | Classification_binary: 0.2256 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1600 | Regression_eta: 0.0022 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.2232 | Regression_cos_phi: 0.1031 | Regression_energy: 0.0948 | Regression_eta: 0.0680 | Regression_pt: 0.1021 | Regression_sin_phi: 0.1060 | LR: 8.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6155s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1077 | max_forward: 0.8572 | max_loss: 0.1505 | max_backward: 1.8951 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7089 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6402.3 | uss_MB: 5500.4 | child_rss_MB: 5939.7 | allocated_MB: 449.9 | reserved_MB: 25448.0 | inactive_split_MB: 1183.4 +rank6 [2026-10-29 00:06:54,749] INFO : Step 15300/50000 rank6 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6783 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1400 | Classification: 0.1062 | Classification_binary: 0.2357 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1646 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0698 | Classification_binary: 0.2408 | Regression_cos_phi: 0.0926 | Regression_energy: 0.0809 | Regression_eta: 0.0562 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0892 | LR: 8.08e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7470s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7962 | max_loss: 0.2606 | max_backward: 1.9831 | max_optimizer_step: 0.2491 | max_tensorboard_logging: 0.7840 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6402.1 | uss_MB: 5500.3 | child_rss_MB: 5940.3 | allocated_MB: 446.7 | reserved_MB: 27282.0 | inactive_split_MB: 1189.4 +rank6 [2026-10-29 00:11:34,440] INFO : Step 15400/50000 rank6 | Train Loss: 1.2329 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1583 | Classification: 0.1127 | Classification_binary: 0.1891 | HitGrouping: 0.1750 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1648 | Regression_eta: 0.0018 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.1796 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0961 | Regression_eta: 0.0450 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0832 | LR: 8.05e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2505s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.7153 | max_loss: 0.1508 | max_backward: 1.6501 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.5965 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6395.9 | uss_MB: 5491.2 | child_rss_MB: 5941.0 | allocated_MB: 444.7 | reserved_MB: 27520.0 | inactive_split_MB: 1102.9 +rank6 [2026-10-29 00:16:23,331] INFO : Step 15500/50000 rank6 | Train Loss: 1.2944 | Train Loss Unweighted: 0.6657 | Train Loss Components: Auxiliary: 0.2480 | Cardinality: 0.1645 | Classification: 0.1174 | Classification_binary: 0.2036 | HitGrouping: 0.1739 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1691 | Regression_eta: 0.0021 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.1690 | Regression_cos_phi: 0.0734 | Regression_energy: 0.0835 | Regression_eta: 0.0608 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0832 | LR: 8.03e-04 | DataLoad Time: 0.0202s | Model Forward Time: 5.2292s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.9970 | max_loss: 0.1740 | max_backward: 2.3916 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.8759 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 6406.1 | uss_MB: 5472.3 | child_rss_MB: 5938.5 | allocated_MB: 449.9 | reserved_MB: 27520.0 | inactive_split_MB: 1174.5 +rank6 [2026-10-29 00:21:07,253] INFO : Step 15600/50000 rank6 | Train Loss: 1.4613 | Train Loss Unweighted: 0.7617 | Train Loss Components: Auxiliary: 0.2872 | Cardinality: 0.1801 | Classification: 0.1276 | Classification_binary: 0.2641 | HitGrouping: 0.1785 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1795 | Regression_eta: 0.0025 | Regression_pt: 0.1856 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0877 | Classification_binary: 0.2732 | Regression_cos_phi: 0.0996 | Regression_energy: 0.1028 | Regression_eta: 0.0727 | Regression_pt: 0.1152 | Regression_sin_phi: 0.1003 | LR: 8.01e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.7223s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.9893 | max_loss: 0.1680 | max_backward: 2.1731 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.0419 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6396.7 | uss_MB: 5034.1 | child_rss_MB: 5938.2 | allocated_MB: 448.1 | reserved_MB: 27520.0 | inactive_split_MB: 1080.9 +rank6 [2026-10-29 00:25:54,169] INFO : Step 15700/50000 rank6 | Train Loss: 1.3299 | Train Loss Unweighted: 0.6547 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1880 | Classification: 0.1192 | Classification_binary: 0.2203 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1547 | Regression_eta: 0.0020 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0913 | Classification_binary: 0.2305 | Regression_cos_phi: 0.1057 | Regression_energy: 0.0726 | Regression_eta: 0.0733 | Regression_pt: 0.0790 | Regression_sin_phi: 0.1053 | LR: 7.98e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.0277s | Phase Times: max_pre_step_barrier: 0.0061 | max_batch_to_device: 0.0068 | max_forward: 0.9761 | max_loss: 0.2543 | max_backward: 2.0225 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8016 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.1020 | Memory: rss_MB: 6402.6 | uss_MB: 5496.7 | child_rss_MB: 5941.3 | allocated_MB: 448.3 | reserved_MB: 27520.0 | inactive_split_MB: 1390.5 +rank6 [2026-10-29 00:30:33,526] INFO : Step 15800/50000 rank6 | Train Loss: 1.2079 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1572 | Classification: 0.0909 | Classification_binary: 0.1814 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1704 | Regression_eta: 0.0019 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1898 | Regression_cos_phi: 0.0701 | Regression_energy: 0.0825 | Regression_eta: 0.0521 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0788 | LR: 7.96e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3432s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1066 | max_forward: 0.8154 | max_loss: 0.1440 | max_backward: 1.6020 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.7740 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6405.3 | uss_MB: 5497.2 | child_rss_MB: 5941.3 | allocated_MB: 447.1 | reserved_MB: 27520.0 | inactive_split_MB: 1038.3 +rank6 [2026-10-29 00:35:19,265] INFO : Step 15900/50000 rank6 | Train Loss: 1.4055 | Train Loss Unweighted: 0.7313 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1808 | Classification: 0.1304 | Classification_binary: 0.2426 | HitGrouping: 0.1763 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1752 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.1776 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0880 | Regression_eta: 0.0453 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0588 | LR: 7.94e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7736s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 1.0653 | max_loss: 0.1515 | max_backward: 2.0024 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.6957 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6402.5 | uss_MB: 5500.0 | child_rss_MB: 5940.1 | allocated_MB: 447.4 | reserved_MB: 27520.0 | inactive_split_MB: 1172.5 +rank6 [2026-10-29 00:40:00,683] INFO : Step 16000/50000 rank6 | Train Loss: 1.3107 | Train Loss Unweighted: 0.6580 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1760 | Classification: 0.1019 | Classification_binary: 0.2391 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1548 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.2381 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0834 | Regression_eta: 0.0510 | Regression_pt: 0.0916 | Regression_sin_phi: 0.0734 | LR: 7.91e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7637s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.1053 | max_forward: 1.1703 | max_loss: 0.1516 | max_backward: 1.8974 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6403.0 | uss_MB: 5502.7 | child_rss_MB: 5941.0 | allocated_MB: 447.2 | reserved_MB: 27520.0 | inactive_split_MB: 1195.6 +rank6 [2026-10-29 00:40:01,696] INFO : Running validation on rank6 +rank6 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 00:40:58,072] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:40:58,783] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 00:41:23,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 00:41:24,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 00:41:25,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 00:41:26,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 00:41:36,724] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 00:41:36,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:41:37,530] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 00:41:53,487] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 00:41:54,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 00:41:55,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 00:41:56,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 00:42:06,673] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 00:42:07,291] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 00:42:07,840] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 00:42:23,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 00:42:24,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 00:42:25,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 00:42:25,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 00:42:35,954] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 00:47:34,082] INFO : Step 16100/50000 rank6 | Train Loss: 1.2037 | Train Loss Unweighted: 0.6025 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1467 | Classification: 0.1080 | Classification_binary: 0.1878 | HitGrouping: 0.1747 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1512 | Regression_eta: 0.0018 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.1840 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0854 | Regression_eta: 0.0434 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0910 | LR: 7.89e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.0441s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7665 | max_loss: 0.1405 | max_backward: 1.4602 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7547.3 | uss_MB: 6643.5 | child_rss_MB: 5941.5 | allocated_MB: 446.1 | reserved_MB: 27680.0 | inactive_split_MB: 1038.2 +rank6 [2026-10-29 00:52:15,155] INFO : Step 16200/50000 rank6 | Train Loss: 1.2083 | Train Loss Unweighted: 0.6192 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1499 | Classification: 0.1107 | Classification_binary: 0.2017 | HitGrouping: 0.1711 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1506 | Regression_eta: 0.0018 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1846 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0632 | Regression_eta: 0.0476 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0633 | LR: 7.86e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9508s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0473 | max_forward: 0.8343 | max_loss: 0.2530 | max_backward: 2.1136 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6458.0 | uss_MB: 5555.6 | child_rss_MB: 5961.3 | allocated_MB: 446.7 | reserved_MB: 27682.0 | inactive_split_MB: 1157.2 +rank6 [2026-10-29 00:56:51,717] INFO : Step 16300/50000 rank6 | Train Loss: 1.2745 | Train Loss Unweighted: 0.6291 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1604 | Classification: 0.1062 | Classification_binary: 0.2417 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1373 | Regression_eta: 0.0022 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0794 | Classification_binary: 0.2500 | Regression_cos_phi: 0.1015 | Regression_energy: 0.0822 | Regression_eta: 0.0695 | Regression_pt: 0.0897 | Regression_sin_phi: 0.1150 | LR: 7.84e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.8102s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.8059 | max_loss: 0.2506 | max_backward: 2.1763 | max_optimizer_step: 0.5213 | max_tensorboard_logging: 0.5923 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6459.9 | uss_MB: 5555.3 | child_rss_MB: 5936.4 | allocated_MB: 450.9 | reserved_MB: 27682.0 | inactive_split_MB: 950.8 +rank6 [2026-10-29 01:01:26,580] INFO : Step 16400/50000 rank6 | Train Loss: 1.2654 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2380 | Cardinality: 0.1554 | Classification: 0.1095 | Classification_binary: 0.2018 | HitGrouping: 0.1699 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1561 | Regression_eta: 0.0023 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.2116 | Regression_cos_phi: 0.1009 | Regression_energy: 0.0681 | Regression_eta: 0.0564 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0886 | LR: 7.81e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.2781s | Phase Times: max_pre_step_barrier: 0.0533 | max_batch_to_device: 0.0064 | max_forward: 0.7459 | max_loss: 0.1486 | max_backward: 1.5352 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.6677 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6452.2 | uss_MB: 5545.1 | child_rss_MB: 5962.3 | allocated_MB: 446.4 | reserved_MB: 27682.0 | inactive_split_MB: 949.1 +rank6 [2026-10-29 01:06:05,048] INFO : Step 16500/50000 rank6 | Train Loss: 1.1728 | Train Loss Unweighted: 0.5918 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1444 | Classification: 0.0927 | Classification_binary: 0.1686 | HitGrouping: 0.1723 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0021 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0875 | Regression_energy: 0.0762 | Regression_eta: 0.0371 | Regression_pt: 0.0830 | Regression_sin_phi: 0.0576 | LR: 7.79e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4905s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8043 | max_loss: 0.1772 | max_backward: 2.0250 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6458.1 | uss_MB: 5554.8 | child_rss_MB: 5927.7 | allocated_MB: 448.6 | reserved_MB: 29698.0 | inactive_split_MB: 1167.6 +rank6 [2026-10-29 01:10:53,703] INFO : Step 16600/50000 rank6 | Train Loss: 1.2920 | Train Loss Unweighted: 0.6427 | Train Loss Components: Auxiliary: 0.2455 | Cardinality: 0.1794 | Classification: 0.1148 | Classification_binary: 0.2098 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1561 | Regression_eta: 0.0021 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.2102 | Regression_cos_phi: 0.1022 | Regression_energy: 0.0877 | Regression_eta: 0.0580 | Regression_pt: 0.0946 | Regression_sin_phi: 0.0923 | LR: 7.77e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2547s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0377 | max_forward: 1.0830 | max_loss: 0.1480 | max_backward: 1.7919 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6457.2 | uss_MB: 5552.9 | child_rss_MB: 5951.6 | allocated_MB: 449.3 | reserved_MB: 29698.0 | inactive_split_MB: 1014.7 +rank6 [2026-10-29 01:15:45,846] INFO : Step 16700/50000 rank6 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6325 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1731 | Classification: 0.1072 | Classification_binary: 0.2252 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1469 | Regression_eta: 0.0019 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.2411 | Regression_cos_phi: 0.0725 | Regression_energy: 0.0809 | Regression_eta: 0.0751 | Regression_pt: 0.0883 | Regression_sin_phi: 0.1068 | LR: 7.74e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.4535s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0722 | max_loss: 0.1506 | max_backward: 2.0365 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5067 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6456.8 | uss_MB: 5555.5 | child_rss_MB: 5918.8 | allocated_MB: 448.3 | reserved_MB: 29698.0 | inactive_split_MB: 893.3 +rank6 [2026-10-29 01:20:20,821] INFO : Step 16800/50000 rank6 | Train Loss: 1.1947 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1542 | Classification: 0.1036 | Classification_binary: 0.1800 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0023 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1750 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0686 | Regression_eta: 0.0682 | Regression_pt: 0.0749 | Regression_sin_phi: 0.0775 | LR: 7.72e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.2730s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0166 | max_forward: 0.8224 | max_loss: 0.1466 | max_backward: 1.6608 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6428.0 | uss_MB: 5552.9 | child_rss_MB: 5812.3 | allocated_MB: 446.4 | reserved_MB: 29698.0 | inactive_split_MB: 1161.6 +rank6 [2026-10-29 01:25:06,863] INFO : Step 16900/50000 rank6 | Train Loss: 1.2025 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1406 | Classification: 0.1120 | Classification_binary: 0.1884 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1446 | Regression_eta: 0.0023 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1625 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0926 | Regression_eta: 0.0500 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0822 | LR: 7.69e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.3236s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7445 | max_loss: 0.1453 | max_backward: 1.8211 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6418.8 | uss_MB: 5553.4 | child_rss_MB: 5783.4 | allocated_MB: 448.9 | reserved_MB: 29698.0 | inactive_split_MB: 901.3 +rank6 [2026-10-29 01:29:52,082] INFO : Step 17000/50000 rank6 | Train Loss: 1.2049 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2349 | Cardinality: 0.1467 | Classification: 0.1018 | Classification_binary: 0.2228 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.2274 | Regression_cos_phi: 0.0793 | Regression_energy: 0.0761 | Regression_eta: 0.0458 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0780 | LR: 7.67e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.1422s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0141 | max_forward: 1.3980 | max_loss: 0.5129 | max_backward: 2.5664 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.5255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1218 | Memory: rss_MB: 6382.7 | uss_MB: 5556.2 | child_rss_MB: 5576.1 | allocated_MB: 448.2 | reserved_MB: 29698.0 | inactive_split_MB: 955.8 +rank6 [2026-10-29 01:29:53,074] INFO : Running validation on rank6 +rank6 [2026-10-29 01:29:53,075] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 01:29:53,075] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 01:30:46,067] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:30:46,579] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 01:31:14,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 01:31:15,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 01:31:16,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 01:31:17,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 01:31:27,441] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 01:31:28,037] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:31:28,589] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 01:31:44,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 01:31:45,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 01:31:46,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 01:31:47,606] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 01:31:57,617] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 01:31:58,252] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 01:31:58,744] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 01:32:14,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 01:32:15,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 01:32:16,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 01:32:16,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 01:32:26,995] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 01:37:36,198] INFO : Step 17100/50000 rank6 | Train Loss: 1.2386 | Train Loss Unweighted: 0.5919 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1931 | Classification: 0.0995 | Classification_binary: 0.1969 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.2363 | Regression_cos_phi: 0.1077 | Regression_energy: 0.0758 | Regression_eta: 0.0595 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0870 | LR: 7.64e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.3600s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0067 | max_forward: 1.0304 | max_loss: 0.1869 | max_backward: 2.3467 | max_optimizer_step: 0.3175 | max_tensorboard_logging: 0.7405 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 7631.0 | uss_MB: 6798.9 | child_rss_MB: 5579.4 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 1373.9 +rank6 [2026-10-29 01:42:27,133] INFO : Step 17200/50000 rank6 | Train Loss: 1.1810 | Train Loss Unweighted: 0.5844 | Train Loss Components: Auxiliary: 0.2270 | Cardinality: 0.1495 | Classification: 0.1085 | Classification_binary: 0.1683 | HitGrouping: 0.1731 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0022 | Regression_pt: 0.1515 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1732 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0861 | Regression_eta: 0.0814 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0741 | LR: 7.62e-04 | DataLoad Time: 0.0172s | Model Forward Time: 6.1040s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7638 | max_loss: 0.2424 | max_backward: 1.9393 | max_optimizer_step: 0.2921 | max_tensorboard_logging: 2.1267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6444.8 | uss_MB: 5633.5 | child_rss_MB: 5577.8 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 988.1 +rank6 [2026-10-29 01:47:11,311] INFO : Step 17300/50000 rank6 | Train Loss: 1.2271 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2397 | Cardinality: 0.1372 | Classification: 0.1102 | Classification_binary: 0.2127 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1512 | Regression_eta: 0.0022 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.2039 | Regression_cos_phi: 0.0928 | Regression_energy: 0.0754 | Regression_eta: 0.0806 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0677 | LR: 7.59e-04 | DataLoad Time: 0.0204s | Model Forward Time: 6.1198s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0243 | max_forward: 1.4273 | max_loss: 0.6053 | max_backward: 3.4810 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6426.5 | uss_MB: 5175.0 | child_rss_MB: 5586.3 | allocated_MB: 449.6 | reserved_MB: 26752.0 | inactive_split_MB: 1120.1 +rank6 [2026-10-29 01:52:04,203] INFO : Step 17400/50000 rank6 | Train Loss: 1.2297 | Train Loss Unweighted: 0.6013 | Train Loss Components: Auxiliary: 0.2404 | Cardinality: 0.1709 | Classification: 0.1017 | Classification_binary: 0.2232 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1354 | Regression_eta: 0.0019 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2360 | Regression_cos_phi: 0.0964 | Regression_energy: 0.0880 | Regression_eta: 0.0583 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0929 | LR: 7.57e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.6391s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0185 | max_forward: 1.0000 | max_loss: 0.1504 | max_backward: 1.9673 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6331 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6409.7 | uss_MB: 5631.7 | child_rss_MB: 5583.3 | allocated_MB: 447.1 | reserved_MB: 28732.0 | inactive_split_MB: 1125.6 +rank6 [2026-10-29 01:56:48,790] INFO : Step 17500/50000 rank6 | Train Loss: 1.2152 | Train Loss Unweighted: 0.6044 | Train Loss Components: Auxiliary: 0.2290 | Cardinality: 0.1485 | Classification: 0.1023 | Classification_binary: 0.1789 | HitGrouping: 0.1757 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1590 | Regression_eta: 0.0026 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0743 | Classification_binary: 0.1936 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0854 | Regression_eta: 0.0640 | Regression_pt: 0.0923 | Regression_sin_phi: 0.1071 | LR: 7.54e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.2338s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0167 | max_forward: 0.7711 | max_loss: 0.1449 | max_backward: 1.8153 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.4947 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6388.7 | uss_MB: 5638.8 | child_rss_MB: 5515.3 | allocated_MB: 450.1 | reserved_MB: 28732.0 | inactive_split_MB: 1375.8 +rank6 [2026-10-29 02:01:39,791] INFO : Step 17600/50000 rank6 | Train Loss: 1.1560 | Train Loss Unweighted: 0.6042 | Train Loss Components: Auxiliary: 0.2286 | Cardinality: 0.1294 | Classification: 0.1057 | Classification_binary: 0.1993 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1471 | Regression_eta: 0.0018 | Regression_pt: 0.1484 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0695 | Classification_binary: 0.2110 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0645 | Regression_eta: 0.0519 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0746 | LR: 7.52e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.8128s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.8868 | max_loss: 0.1705 | max_backward: 2.0183 | max_optimizer_step: 0.2066 | max_tensorboard_logging: 0.7222 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1349 | Memory: rss_MB: 6384.6 | uss_MB: 5639.3 | child_rss_MB: 5506.2 | allocated_MB: 448.3 | reserved_MB: 28732.0 | inactive_split_MB: 1319.7 +rank6 [2026-10-29 02:06:28,890] INFO : Step 17700/50000 rank6 | Train Loss: 1.7848 | Train Loss Unweighted: 0.7433 | Train Loss Components: Auxiliary: 0.3415 | Cardinality: 0.4306 | Classification: 0.1200 | Classification_binary: 0.2341 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1920 | Regression_eta: 0.0037 | Regression_pt: 0.1910 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.2148 | Regression_cos_phi: 0.1136 | Regression_energy: 0.1069 | Regression_eta: 0.0931 | Regression_pt: 0.1165 | Regression_sin_phi: 0.1044 | LR: 7.49e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.5778s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8788 | max_loss: 0.4738 | max_backward: 1.9859 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.5305 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6359.7 | uss_MB: 5640.4 | child_rss_MB: 5400.4 | allocated_MB: 450.1 | reserved_MB: 28738.0 | inactive_split_MB: 1125.6 +rank6 [2026-10-29 02:11:14,773] INFO : Step 17800/50000 rank6 | Train Loss: 1.1828 | Train Loss Unweighted: 0.6014 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1456 | Classification: 0.0982 | Classification_binary: 0.1996 | HitGrouping: 0.1749 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1503 | Regression_eta: 0.0019 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0750 | Regression_eta: 0.0358 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0811 | LR: 7.47e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.1920s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.3167 | max_backward: 2.2751 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 6343.6 | uss_MB: 5261.8 | child_rss_MB: 5331.5 | allocated_MB: 444.9 | reserved_MB: 28738.0 | inactive_split_MB: 1155.9 +rank6 [2026-10-29 02:16:10,312] INFO : Step 17900/50000 rank6 | Train Loss: 1.0765 | Train Loss Unweighted: 0.5646 | Train Loss Components: Auxiliary: 0.2108 | Cardinality: 0.1097 | Classification: 0.0932 | Classification_binary: 0.1585 | HitGrouping: 0.1686 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1536 | Regression_eta: 0.0016 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1389 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0840 | Regression_eta: 0.0428 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0488 | LR: 7.44e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2960s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7593 | max_loss: 0.4918 | max_backward: 1.7987 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.6052 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6290.9 | uss_MB: 5640.2 | child_rss_MB: 5109.0 | allocated_MB: 447.1 | reserved_MB: 28738.0 | inactive_split_MB: 1154.4 +rank6 [2026-10-29 02:20:55,875] INFO : Step 18000/50000 rank6 | Train Loss: 1.2227 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2303 | Cardinality: 0.1450 | Classification: 0.1088 | Classification_binary: 0.2099 | HitGrouping: 0.1650 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1447 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.1540 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0738 | Regression_eta: 0.0428 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0767 | LR: 7.42e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7002s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9293 | max_loss: 0.1567 | max_backward: 2.2476 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6283.0 | uss_MB: 5642.9 | child_rss_MB: 5063.5 | allocated_MB: 449.4 | reserved_MB: 28738.0 | inactive_split_MB: 1324.2 +rank6 [2026-10-29 02:20:56,967] INFO : Running validation on rank6 +rank6 [2026-10-29 02:20:56,967] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 02:20:56,967] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 02:21:57,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:21:58,045] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 02:22:25,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 02:22:26,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 02:22:27,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 02:22:28,349] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 02:22:38,359] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 02:22:38,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:22:39,419] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 02:22:55,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 02:22:56,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 02:22:57,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 02:22:58,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 02:23:08,294] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 02:23:08,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 02:23:09,544] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 02:23:25,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 02:23:26,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 02:23:26,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 02:23:27,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 02:23:37,603] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 02:28:43,002] INFO : Step 18100/50000 rank6 | Train Loss: 1.2922 | Train Loss Unweighted: 0.6330 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1763 | Classification: 0.1061 | Classification_binary: 0.2257 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1467 | Regression_eta: 0.0022 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0842 | Classification_binary: 0.2672 | Regression_cos_phi: 0.1107 | Regression_energy: 0.0856 | Regression_eta: 0.0845 | Regression_pt: 0.0941 | Regression_sin_phi: 0.1335 | LR: 7.39e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.0675s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.9150 | max_loss: 0.1573 | max_backward: 2.1915 | max_optimizer_step: 0.2256 | max_tensorboard_logging: 0.8426 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7551.6 | uss_MB: 6907.4 | child_rss_MB: 5063.5 | allocated_MB: 448.8 | reserved_MB: 25396.0 | inactive_split_MB: 1192.4 +rank6 [2026-10-29 02:33:21,589] INFO : Step 18200/50000 rank6 | Train Loss: 1.1403 | Train Loss Unweighted: 0.5693 | Train Loss Components: Auxiliary: 0.2140 | Cardinality: 0.1362 | Classification: 0.1099 | Classification_binary: 0.1658 | HitGrouping: 0.1698 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1444 | Regression_eta: 0.0021 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1631 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0870 | Regression_eta: 0.0538 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0592 | LR: 7.36e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5273s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0059 | max_forward: 0.7843 | max_loss: 0.1427 | max_backward: 1.4948 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 1.0509 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6418.0 | uss_MB: 5773.9 | child_rss_MB: 5038.4 | allocated_MB: 446.0 | reserved_MB: 27340.0 | inactive_split_MB: 922.1 +rank6 [2026-10-29 02:37:53,622] INFO : Step 18300/50000 rank6 | Train Loss: 1.1938 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2339 | Cardinality: 0.1526 | Classification: 0.1133 | Classification_binary: 0.1878 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1493 | Regression_eta: 0.0019 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.1604 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0648 | Regression_eta: 0.0518 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0954 | LR: 7.34e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9186s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.9058 | max_loss: 0.1527 | max_backward: 2.1689 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6422.1 | uss_MB: 5784.1 | child_rss_MB: 5032.3 | allocated_MB: 448.4 | reserved_MB: 27340.0 | inactive_split_MB: 1185.8 +rank6 [2026-10-29 02:42:24,375] INFO : Step 18400/50000 rank6 | Train Loss: 1.1959 | Train Loss Unweighted: 0.6109 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1130 | Classification: 0.1042 | Classification_binary: 0.2213 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1400 | Regression_eta: 0.0022 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.2101 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0702 | Regression_eta: 0.0549 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0786 | LR: 7.31e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.7088s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8434 | max_loss: 0.1474 | max_backward: 1.9441 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6423.0 | uss_MB: 5783.9 | child_rss_MB: 5008.1 | allocated_MB: 448.8 | reserved_MB: 29306.0 | inactive_split_MB: 1091.4 +rank6 [2026-10-29 02:47:17,191] INFO : Step 18500/50000 rank6 | Train Loss: 1.2719 | Train Loss Unweighted: 0.6036 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1990 | Classification: 0.1087 | Classification_binary: 0.1793 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1538 | Regression_eta: 0.0024 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0640 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0917 | Regression_energy: 0.0727 | Regression_eta: 0.0747 | Regression_pt: 0.0784 | Regression_sin_phi: 0.0995 | LR: 7.29e-04 | DataLoad Time: 0.0169s | Model Forward Time: 6.4222s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 1.6245 | max_loss: 0.3527 | max_backward: 3.4112 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.7932 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 6423.2 | uss_MB: 5781.7 | child_rss_MB: 5035.9 | allocated_MB: 447.9 | reserved_MB: 29306.0 | inactive_split_MB: 1193.4 +rank6 [2026-10-29 02:52:01,281] INFO : Step 18600/50000 rank6 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5476 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1351 | Classification: 0.0870 | Classification_binary: 0.1697 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1433 | Regression_eta: 0.0017 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0748 | Regression_energy: 0.1006 | Regression_eta: 0.0455 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0535 | LR: 7.26e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2407s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0527 | max_forward: 0.7639 | max_loss: 0.1373 | max_backward: 1.5273 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.7424 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6420.8 | uss_MB: 5779.1 | child_rss_MB: 5038.0 | allocated_MB: 445.6 | reserved_MB: 29306.0 | inactive_split_MB: 1194.4 +rank6 [2026-10-29 02:56:35,338] INFO : Step 18700/50000 rank6 | Train Loss: 1.2376 | Train Loss Unweighted: 0.5906 | Train Loss Components: Auxiliary: 0.2405 | Cardinality: 0.1841 | Classification: 0.1039 | Classification_binary: 0.2012 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1400 | Regression_eta: 0.0020 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0726 | Classification_binary: 0.1852 | Regression_cos_phi: 0.0853 | Regression_energy: 0.0634 | Regression_eta: 0.0454 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0885 | LR: 7.24e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7557s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8903 | max_loss: 0.1469 | max_backward: 2.1433 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4822 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1128 | Memory: rss_MB: 6419.7 | uss_MB: 5784.3 | child_rss_MB: 5013.1 | allocated_MB: 449.2 | reserved_MB: 31324.0 | inactive_split_MB: 1093.4 +rank6 [2026-10-29 03:01:09,007] INFO : Step 18800/50000 rank6 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1585 | Classification: 0.0943 | Classification_binary: 0.2038 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1342 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0635 | Classification_binary: 0.1924 | Regression_cos_phi: 0.0638 | Regression_energy: 0.0678 | Regression_eta: 0.0366 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0547 | LR: 7.21e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4219s | Phase Times: max_pre_step_barrier: 0.1142 | max_batch_to_device: 0.0068 | max_forward: 0.7808 | max_loss: 0.2502 | max_backward: 1.8229 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0996 | Memory: rss_MB: 6421.2 | uss_MB: 5781.7 | child_rss_MB: 5036.2 | allocated_MB: 450.4 | reserved_MB: 31336.0 | inactive_split_MB: 1098.4 +rank6 [2026-10-29 03:05:45,770] INFO : Step 18900/50000 rank6 | Train Loss: 1.0747 | Train Loss Unweighted: 0.5385 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.1306 | Classification: 0.0887 | Classification_binary: 0.1613 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1414 | Regression_eta: 0.0017 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1872 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0856 | Regression_eta: 0.0458 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0676 | LR: 7.18e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3939s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7400 | max_loss: 0.1656 | max_backward: 1.8003 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.7596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6421.5 | uss_MB: 5781.2 | child_rss_MB: 5039.7 | allocated_MB: 443.5 | reserved_MB: 31336.0 | inactive_split_MB: 928.5 +rank6 [2026-10-29 03:10:20,955] INFO : Step 19000/50000 rank6 | Train Loss: 1.1726 | Train Loss Unweighted: 0.5966 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1418 | Classification: 0.1102 | Classification_binary: 0.1870 | HitGrouping: 0.1705 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1463 | Regression_eta: 0.0019 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0804 | Classification_binary: 0.1793 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0765 | Regression_eta: 0.0499 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0929 | LR: 7.16e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8903s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7637 | max_loss: 0.5203 | max_backward: 1.8720 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 1.1055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6422.5 | uss_MB: 5783.8 | child_rss_MB: 5041.8 | allocated_MB: 446.6 | reserved_MB: 31336.0 | inactive_split_MB: 1097.4 +rank6 [2026-10-29 03:10:21,983] INFO : Running validation on rank6 +rank6 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 03:11:18,711] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:11:19,247] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 03:11:45,746] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 03:11:46,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 03:11:48,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 03:11:50,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 03:12:00,318] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-29 03:12:01,384] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:12:01,928] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 03:12:18,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 03:12:19,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 03:12:20,138] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 03:12:21,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 03:12:31,109] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 03:12:31,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 03:12:31,971] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 03:12:47,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 03:12:48,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 03:12:49,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 03:12:50,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 03:13:00,323] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 03:17:58,240] INFO : Step 19100/50000 rank6 | Train Loss: 1.2150 | Train Loss Unweighted: 0.6107 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1439 | Classification: 0.1037 | Classification_binary: 0.2230 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1395 | Regression_eta: 0.0020 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0730 | Classification_binary: 0.2210 | Regression_cos_phi: 0.1229 | Regression_energy: 0.0734 | Regression_eta: 0.0891 | Regression_pt: 0.0782 | Regression_sin_phi: 0.1144 | LR: 7.13e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7844s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8630 | max_loss: 0.1532 | max_backward: 1.9622 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.7474 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7697.6 | uss_MB: 7053.9 | child_rss_MB: 5049.6 | allocated_MB: 449.4 | reserved_MB: 23972.0 | inactive_split_MB: 1371.8 +rank6 [2026-10-29 03:22:35,411] INFO : Step 19200/50000 rank6 | Train Loss: 1.1075 | Train Loss Unweighted: 0.5303 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1382 | Classification: 0.0958 | Classification_binary: 0.1758 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1270 | Regression_eta: 0.0019 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1896 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0788 | Regression_eta: 0.0511 | Regression_pt: 0.0868 | Regression_sin_phi: 0.0677 | LR: 7.10e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.0040s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8287 | max_loss: 0.1556 | max_backward: 1.8127 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6504.0 | uss_MB: 5863.3 | child_rss_MB: 5048.7 | allocated_MB: 445.3 | reserved_MB: 27640.0 | inactive_split_MB: 940.7 +rank6 [2026-10-29 03:27:09,997] INFO : Step 19300/50000 rank6 | Train Loss: 1.0792 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2111 | Cardinality: 0.1254 | Classification: 0.0911 | Classification_binary: 0.1561 | HitGrouping: 0.1662 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1539 | Regression_cos_phi: 0.0891 | Regression_energy: 0.0698 | Regression_eta: 0.0674 | Regression_pt: 0.0764 | Regression_sin_phi: 0.0703 | LR: 7.08e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.5154s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8891 | max_loss: 0.1513 | max_backward: 1.7637 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6508.5 | uss_MB: 5869.2 | child_rss_MB: 5044.9 | allocated_MB: 445.1 | reserved_MB: 27640.0 | inactive_split_MB: 1366.9 +rank6 [2026-10-29 03:31:44,571] INFO : Step 19400/50000 rank6 | Train Loss: 1.2411 | Train Loss Unweighted: 0.6183 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1555 | Classification: 0.1055 | Classification_binary: 0.2145 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1463 | Regression_eta: 0.0020 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.1899 | Regression_cos_phi: 0.0957 | Regression_energy: 0.0747 | Regression_eta: 0.0698 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0806 | LR: 7.05e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.6746s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0092 | max_forward: 0.8079 | max_loss: 0.1550 | max_backward: 1.7710 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 2.8282 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6511.0 | uss_MB: 5872.2 | child_rss_MB: 5045.1 | allocated_MB: 452.5 | reserved_MB: 28010.0 | inactive_split_MB: 1195.9 +rank6 [2026-10-29 03:36:21,472] INFO : Step 19500/50000 rank6 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6008 | Train Loss Components: Auxiliary: 0.2448 | Cardinality: 0.1836 | Classification: 0.1087 | Classification_binary: 0.2123 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1374 | Regression_eta: 0.0023 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0582 | Classification_binary: 0.2056 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0625 | Regression_eta: 0.0452 | Regression_pt: 0.0679 | Regression_sin_phi: 0.0696 | LR: 7.02e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.6206s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7880 | max_loss: 0.1506 | max_backward: 1.7760 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.7620 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6510.7 | uss_MB: 5870.2 | child_rss_MB: 5041.2 | allocated_MB: 449.4 | reserved_MB: 29680.0 | inactive_split_MB: 1368.8 +rank6 [2026-10-29 03:40:54,624] INFO : Step 19600/50000 rank6 | Train Loss: 1.1451 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1464 | Classification: 0.0933 | Classification_binary: 0.1736 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1533 | Regression_eta: 0.0018 | Regression_pt: 0.1557 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.2322 | Regression_cos_phi: 0.0878 | Regression_energy: 0.0761 | Regression_eta: 0.0583 | Regression_pt: 0.0852 | Regression_sin_phi: 0.0743 | LR: 7.00e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.3538s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8082 | max_loss: 0.1437 | max_backward: 1.6649 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.6567 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6510.9 | uss_MB: 5869.4 | child_rss_MB: 5045.0 | allocated_MB: 444.5 | reserved_MB: 29680.0 | inactive_split_MB: 1016.8 +rank6 [2026-10-29 03:45:30,311] INFO : Step 19700/50000 rank6 | Train Loss: 1.1424 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2236 | Cardinality: 0.1385 | Classification: 0.1093 | Classification_binary: 0.1797 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1379 | Regression_eta: 0.0020 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0847 | Regression_eta: 0.0476 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0709 | LR: 6.97e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4874s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7612 | max_loss: 0.2297 | max_backward: 1.9668 | max_optimizer_step: 0.5229 | max_tensorboard_logging: 0.5756 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6508.5 | uss_MB: 5869.4 | child_rss_MB: 5047.4 | allocated_MB: 449.1 | reserved_MB: 29680.0 | inactive_split_MB: 1100.9 +rank6 [2026-10-29 03:50:08,779] INFO : Step 19800/50000 rank6 | Train Loss: 1.2806 | Train Loss Unweighted: 0.6390 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1432 | Classification: 0.1110 | Classification_binary: 0.2282 | HitGrouping: 0.1668 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0030 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1918 | Regression_cos_phi: 0.0713 | Regression_energy: 0.0763 | Regression_eta: 0.0705 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0804 | LR: 6.94e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.5049s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8034 | max_loss: 0.1812 | max_backward: 1.9502 | max_optimizer_step: 0.2119 | max_tensorboard_logging: 0.5857 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6510.7 | uss_MB: 5873.6 | child_rss_MB: 5045.7 | allocated_MB: 452.3 | reserved_MB: 29680.0 | inactive_split_MB: 1110.9 +rank6 [2026-10-29 03:54:46,662] INFO : Step 19900/50000 rank6 | Train Loss: 1.0692 | Train Loss Unweighted: 0.5407 | Train Loss Components: Auxiliary: 0.2077 | Cardinality: 0.1171 | Classification: 0.0922 | Classification_binary: 0.1812 | HitGrouping: 0.1610 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0561 | Classification_binary: 0.1715 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0621 | Regression_eta: 0.0415 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0860 | LR: 6.92e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.6113s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7533 | max_loss: 0.2412 | max_backward: 2.0720 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5848 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6505.8 | uss_MB: 5870.0 | child_rss_MB: 5057.5 | allocated_MB: 444.7 | reserved_MB: 29682.0 | inactive_split_MB: 1000.0 +rank6 [2026-10-29 03:59:25,167] INFO : Step 20000/50000 rank6 | Train Loss: 1.0304 | Train Loss Unweighted: 0.5275 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1060 | Classification: 0.0940 | Classification_binary: 0.1604 | HitGrouping: 0.1656 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1344 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1283 | Regression_cos_phi: 0.0592 | Regression_energy: 0.0584 | Regression_eta: 0.0319 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0361 | LR: 6.89e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.8353s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.8721 | max_loss: 0.1507 | max_backward: 2.1861 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6505.7 | uss_MB: 5870.7 | child_rss_MB: 5049.5 | allocated_MB: 445.9 | reserved_MB: 30966.0 | inactive_split_MB: 1179.2 +rank6 [2026-10-29 03:59:26,239] INFO : Running validation on rank6 +rank6 [2026-10-29 03:59:26,239] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 03:59:26,239] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 04:00:19,031] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:00:19,534] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 04:00:45,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 04:00:46,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 04:00:47,191] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 04:00:48,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 04:00:58,323] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 04:00:59,045] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:00:59,501] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 04:01:15,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 04:01:16,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 04:01:17,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 04:01:18,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 04:01:28,586] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:01:28,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:01:29,372] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 04:01:45,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 04:01:46,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 04:01:46,811] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 04:01:47,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 04:01:57,602] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 04:06:53,362] INFO : Step 20100/50000 rank6 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2437 | Cardinality: 0.1586 | Classification: 0.1055 | Classification_binary: 0.2238 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1360 | Regression_eta: 0.0021 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.2102 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0675 | Regression_eta: 0.0329 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0655 | LR: 6.86e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.6028s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0372 | max_forward: 0.8293 | max_loss: 0.1584 | max_backward: 1.7738 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.7139 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7674.6 | uss_MB: 7034.7 | child_rss_MB: 5034.6 | allocated_MB: 448.4 | reserved_MB: 29328.0 | inactive_split_MB: 969.2 +rank6 [2026-10-29 04:11:25,835] INFO : Step 20200/50000 rank6 | Train Loss: 1.3163 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.1695 | Classification: 0.1000 | Classification_binary: 0.2250 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1433 | Regression_eta: 0.0029 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.2436 | Regression_cos_phi: 0.1017 | Regression_energy: 0.0818 | Regression_eta: 0.0915 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0736 | LR: 6.84e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6416s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8823 | max_loss: 0.1595 | max_backward: 2.1030 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5136 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7674.6 | uss_MB: 7036.8 | child_rss_MB: 5027.8 | allocated_MB: 449.6 | reserved_MB: 29340.0 | inactive_split_MB: 938.7 +rank6 [2026-10-29 04:16:03,708] INFO : Step 20300/50000 rank6 | Train Loss: 1.1420 | Train Loss Unweighted: 0.5734 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1487 | Classification: 0.1087 | Classification_binary: 0.1713 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1438 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1829 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0788 | Regression_eta: 0.0426 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0820 | LR: 6.81e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.9960s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.9650 | max_loss: 0.1904 | max_backward: 2.0801 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.8679 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6555.4 | uss_MB: 5913.2 | child_rss_MB: 5021.5 | allocated_MB: 444.8 | reserved_MB: 29344.0 | inactive_split_MB: 986.2 +rank6 [2026-10-29 04:20:44,975] INFO : Step 20400/50000 rank6 | Train Loss: 1.1926 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1642 | Classification: 0.1091 | Classification_binary: 0.1922 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1329 | Regression_eta: 0.0018 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1919 | Regression_cos_phi: 0.0800 | Regression_energy: 0.0684 | Regression_eta: 0.0381 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0453 | LR: 6.78e-04 | DataLoad Time: 0.0194s | Model Forward Time: 5.2524s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0167 | max_forward: 1.2639 | max_loss: 0.1873 | max_backward: 2.4261 | max_optimizer_step: 0.3105 | max_tensorboard_logging: 0.7562 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6556.4 | uss_MB: 5916.7 | child_rss_MB: 5031.4 | allocated_MB: 447.7 | reserved_MB: 29344.0 | inactive_split_MB: 985.2 +rank6 [2026-10-29 04:25:27,116] INFO : Step 20500/50000 rank6 | Train Loss: 1.2110 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1591 | Classification: 0.1022 | Classification_binary: 0.2117 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1333 | Regression_eta: 0.0017 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.2185 | Regression_cos_phi: 0.1239 | Regression_energy: 0.0680 | Regression_eta: 0.0593 | Regression_pt: 0.0751 | Regression_sin_phi: 0.1480 | LR: 6.76e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5482s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7705 | max_loss: 0.1525 | max_backward: 1.8060 | max_optimizer_step: 0.2622 | max_tensorboard_logging: 0.7595 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6553.9 | uss_MB: 5915.2 | child_rss_MB: 5013.6 | allocated_MB: 450.6 | reserved_MB: 29344.0 | inactive_split_MB: 1123.4 +rank6 [2026-10-29 04:30:07,866] INFO : Step 20600/50000 rank6 | Train Loss: 1.1161 | Train Loss Unweighted: 0.5564 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1436 | Classification: 0.0978 | Classification_binary: 0.1795 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1916 | Regression_cos_phi: 0.1032 | Regression_energy: 0.0705 | Regression_eta: 0.0556 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0841 | LR: 6.73e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4377s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8358 | max_loss: 0.5148 | max_backward: 1.8393 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6556.9 | uss_MB: 5916.8 | child_rss_MB: 5024.7 | allocated_MB: 445.4 | reserved_MB: 31380.0 | inactive_split_MB: 917.0 +rank6 [2026-10-29 04:34:51,728] INFO : Step 20700/50000 rank6 | Train Loss: 1.0740 | Train Loss Unweighted: 0.5459 | Train Loss Components: Auxiliary: 0.2086 | Cardinality: 0.1252 | Classification: 0.0962 | Classification_binary: 0.1560 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0393 | Classification_binary: 0.1433 | Regression_cos_phi: 0.0650 | Regression_energy: 0.0805 | Regression_eta: 0.0342 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0553 | LR: 6.70e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0699s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7111 | max_loss: 0.2396 | max_backward: 1.6829 | max_optimizer_step: 0.2121 | max_tensorboard_logging: 0.4996 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6553.0 | uss_MB: 5912.9 | child_rss_MB: 5017.8 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1185.7 +rank6 [2026-10-29 04:39:29,326] INFO : Step 20800/50000 rank6 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5426 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1263 | Classification: 0.0965 | Classification_binary: 0.1999 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1207 | Regression_eta: 0.0017 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1971 | Regression_cos_phi: 0.0719 | Regression_energy: 0.0649 | Regression_eta: 0.0456 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0811 | LR: 6.67e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7617s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.9079 | max_loss: 0.1550 | max_backward: 2.1083 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.5897 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6553.9 | uss_MB: 5918.9 | child_rss_MB: 5012.5 | allocated_MB: 448.2 | reserved_MB: 31380.0 | inactive_split_MB: 1157.8 +rank6 [2026-10-29 04:44:09,835] INFO : Step 20900/50000 rank6 | Train Loss: 1.2188 | Train Loss Unweighted: 0.5970 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1525 | Classification: 0.1062 | Classification_binary: 0.2022 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0022 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0763 | Classification_binary: 0.2398 | Regression_cos_phi: 0.0896 | Regression_energy: 0.0735 | Regression_eta: 0.0592 | Regression_pt: 0.0803 | Regression_sin_phi: 0.1031 | LR: 6.65e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.7895s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0138 | max_forward: 0.8922 | max_loss: 0.1535 | max_backward: 2.0483 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6550.7 | uss_MB: 5916.4 | child_rss_MB: 5027.7 | allocated_MB: 447.8 | reserved_MB: 31380.0 | inactive_split_MB: 1126.3 +rank6 [2026-10-29 04:48:48,418] INFO : Step 21000/50000 rank6 | Train Loss: 1.0503 | Train Loss Unweighted: 0.5338 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1267 | Classification: 0.0900 | Classification_binary: 0.1573 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1405 | Regression_eta: 0.0016 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0685 | Classification_binary: 0.1648 | Regression_cos_phi: 0.0790 | Regression_energy: 0.0801 | Regression_eta: 0.0482 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0595 | LR: 6.62e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.8550s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7596 | max_loss: 0.2536 | max_backward: 1.8713 | max_optimizer_step: 0.3177 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6551.7 | uss_MB: 5915.9 | child_rss_MB: 5032.5 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1148.0 +rank6 [2026-10-29 04:48:49,285] INFO : Running validation on rank6 +rank6 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 04:49:44,289] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:49:44,825] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 04:50:11,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 04:50:12,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 04:50:13,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 04:50:14,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 04:50:24,697] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:50:25,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:50:25,767] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 04:50:41,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 04:50:42,548] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 04:50:43,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 04:50:44,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 04:50:54,516] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 04:50:55,260] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 04:50:55,906] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 04:51:11,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 04:51:12,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 04:51:13,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 04:51:13,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 04:51:23,927] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 04:56:25,805] INFO : Step 21100/50000 rank6 | Train Loss: 1.1646 | Train Loss Unweighted: 0.5785 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1401 | Classification: 0.1095 | Classification_binary: 0.1927 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1344 | Regression_eta: 0.0022 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1589 | Regression_cos_phi: 0.0705 | Regression_energy: 0.0864 | Regression_eta: 0.0790 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0682 | LR: 6.59e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.4037s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.9258 | max_loss: 0.1506 | max_backward: 2.4347 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.8145 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7847.0 | uss_MB: 5890.3 | child_rss_MB: 5018.3 | allocated_MB: 448.7 | reserved_MB: 23438.0 | inactive_split_MB: 911.7 +rank6 [2026-10-29 04:56:59,983] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank6 [2026-10-29 04:56:59,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-29 05:01:22,612] INFO : Step 21200/50000 rank6 | Train Loss: 1.1437 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1413 | Classification: 0.0985 | Classification_binary: 0.1768 | HitGrouping: 0.1669 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0020 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1939 | Regression_cos_phi: 0.1237 | Regression_energy: 0.0768 | Regression_eta: 0.0482 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0652 | LR: 6.56e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5648s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7993 | max_loss: 0.1450 | max_backward: 1.7427 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7842.9 | uss_MB: 7208.8 | child_rss_MB: 5012.9 | allocated_MB: 446.9 | reserved_MB: 23590.0 | inactive_split_MB: 1086.9 +rank6 [2026-10-29 05:06:02,214] INFO : Step 21300/50000 rank6 | Train Loss: 1.1586 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1648 | Classification: 0.1118 | Classification_binary: 0.1760 | HitGrouping: 0.1681 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1392 | Regression_eta: 0.0021 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0722 | Classification_binary: 0.1611 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0926 | Regression_eta: 0.0448 | Regression_pt: 0.1002 | Regression_sin_phi: 0.0688 | LR: 6.54e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.2679s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0558 | max_forward: 0.9498 | max_loss: 0.1699 | max_backward: 2.4288 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.7788 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1270 | Memory: rss_MB: 6605.4 | uss_MB: 5978.0 | child_rss_MB: 5012.8 | allocated_MB: 447.4 | reserved_MB: 25440.0 | inactive_split_MB: 1003.9 +rank6 [2026-10-29 05:10:41,727] INFO : Step 21400/50000 rank6 | Train Loss: 1.1843 | Train Loss Unweighted: 0.5823 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1516 | Classification: 0.1038 | Classification_binary: 0.2065 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1335 | Regression_eta: 0.0020 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.2065 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0728 | Regression_eta: 0.0392 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0547 | LR: 6.51e-04 | DataLoad Time: 0.0186s | Model Forward Time: 5.0974s | Phase Times: max_pre_step_barrier: 0.0085 | max_batch_to_device: 0.0640 | max_forward: 0.9920 | max_loss: 0.5036 | max_backward: 2.0711 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.8200 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6606.3 | uss_MB: 5978.8 | child_rss_MB: 5018.5 | allocated_MB: 448.9 | reserved_MB: 25440.0 | inactive_split_MB: 994.4 +rank6 [2026-10-29 05:15:15,087] INFO : Step 21500/50000 rank6 | Train Loss: 1.1484 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1478 | Classification: 0.0953 | Classification_binary: 0.2062 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1299 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1957 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0586 | Regression_eta: 0.0562 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0733 | LR: 6.48e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.9380s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8371 | max_loss: 0.1537 | max_backward: 2.1150 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.8633 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6603.8 | uss_MB: 5979.6 | child_rss_MB: 5017.0 | allocated_MB: 447.2 | reserved_MB: 27056.0 | inactive_split_MB: 1468.9 +rank6 [2026-10-29 05:19:51,738] INFO : Step 21600/50000 rank6 | Train Loss: 1.0171 | Train Loss Unweighted: 0.5098 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1167 | Classification: 0.0973 | Classification_binary: 0.1559 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1265 | Regression_eta: 0.0014 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.1928 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0716 | Regression_eta: 0.0481 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0661 | LR: 6.45e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5909s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0062 | max_forward: 0.7285 | max_loss: 0.1940 | max_backward: 1.8148 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.8819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6602.2 | uss_MB: 5978.2 | child_rss_MB: 5014.0 | allocated_MB: 445.7 | reserved_MB: 27056.0 | inactive_split_MB: 1196.5 +rank6 [2026-10-29 05:24:30,476] INFO : Step 21700/50000 rank6 | Train Loss: 1.1623 | Train Loss Unweighted: 0.5535 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1498 | Classification: 0.1025 | Classification_binary: 0.1819 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.1626 | Regression_cos_phi: 0.0914 | Regression_energy: 0.0708 | Regression_eta: 0.0351 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0607 | LR: 6.43e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4936s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7947 | max_loss: 0.1379 | max_backward: 1.9808 | max_optimizer_step: 0.2157 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6600.2 | uss_MB: 5978.2 | child_rss_MB: 5010.8 | allocated_MB: 448.2 | reserved_MB: 27056.0 | inactive_split_MB: 1045.2 +rank6 [2026-10-29 05:29:08,470] INFO : Step 21800/50000 rank6 | Train Loss: 1.0898 | Train Loss Unweighted: 0.5518 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1233 | Classification: 0.0977 | Classification_binary: 0.1967 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1269 | Regression_eta: 0.0015 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.1542 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0554 | Regression_eta: 0.0346 | Regression_pt: 0.0614 | Regression_sin_phi: 0.0732 | LR: 6.40e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.7216s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8952 | max_loss: 0.1765 | max_backward: 2.1628 | max_optimizer_step: 0.2138 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6602.0 | uss_MB: 5872.4 | child_rss_MB: 5012.0 | allocated_MB: 448.4 | reserved_MB: 27056.0 | inactive_split_MB: 1001.4 +rank6 [2026-10-29 05:33:46,454] INFO : Step 21900/50000 rank6 | Train Loss: 1.0699 | Train Loss Unweighted: 0.5306 | Train Loss Components: Auxiliary: 0.2121 | Cardinality: 0.1436 | Classification: 0.0991 | Classification_binary: 0.1704 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1276 | Regression_eta: 0.0015 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1767 | Regression_cos_phi: 0.0625 | Regression_energy: 0.0603 | Regression_eta: 0.0356 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0606 | LR: 6.37e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4513s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7813 | max_loss: 0.1495 | max_backward: 1.8703 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 6600.0 | uss_MB: 5976.9 | child_rss_MB: 5013.0 | allocated_MB: 447.4 | reserved_MB: 27056.0 | inactive_split_MB: 1002.4 +rank6 [2026-10-29 05:38:23,934] INFO : Step 22000/50000 rank6 | Train Loss: 1.0659 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2100 | Cardinality: 0.1294 | Classification: 0.0955 | Classification_binary: 0.1521 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1360 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1295 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0844 | Regression_eta: 0.0415 | Regression_pt: 0.0903 | Regression_sin_phi: 0.0786 | LR: 6.34e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4503s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7855 | max_loss: 0.1383 | max_backward: 1.9648 | max_optimizer_step: 0.2132 | max_tensorboard_logging: 0.5884 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6605.4 | uss_MB: 5979.9 | child_rss_MB: 5015.2 | allocated_MB: 449.1 | reserved_MB: 27056.0 | inactive_split_MB: 1180.9 +rank6 [2026-10-29 05:38:24,835] INFO : Running validation on rank6 +rank6 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 05:39:16,673] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:39:17,215] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 05:39:43,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 05:39:44,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 05:39:45,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 05:39:46,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 05:39:56,313] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 05:39:56,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:39:57,074] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 05:40:13,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 05:40:14,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 05:40:15,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 05:40:16,757] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 05:40:26,768] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 05:40:27,479] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 05:40:27,967] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 05:40:43,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 05:40:44,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 05:40:45,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 05:40:45,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 05:40:55,969] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 05:45:56,688] INFO : Step 22100/50000 rank6 | Train Loss: 1.1235 | Train Loss Unweighted: 0.5530 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1528 | Classification: 0.0997 | Classification_binary: 0.1905 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1304 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1774 | Regression_cos_phi: 0.0591 | Regression_energy: 0.0588 | Regression_eta: 0.0342 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0566 | LR: 6.32e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5232s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8135 | max_loss: 0.1488 | max_backward: 1.7877 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5979 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7900.1 | uss_MB: 7271.0 | child_rss_MB: 5019.5 | allocated_MB: 451.8 | reserved_MB: 29136.0 | inactive_split_MB: 1136.2 +rank6 [2026-10-29 05:50:45,341] INFO : Step 22200/50000 rank6 | Train Loss: 1.1922 | Train Loss Unweighted: 0.5684 | Train Loss Components: Auxiliary: 0.2361 | Cardinality: 0.1699 | Classification: 0.0906 | Classification_binary: 0.2085 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0668 | Classification_binary: 0.2123 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0768 | Regression_eta: 0.0630 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0723 | LR: 6.29e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5686s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8616 | max_loss: 0.1545 | max_backward: 1.6969 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 0.7120 | max_tensorboard_flush: 0.0150 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7899.1 | uss_MB: 7269.8 | child_rss_MB: 5025.9 | allocated_MB: 448.1 | reserved_MB: 29136.0 | inactive_split_MB: 1371.0 +rank6 [2026-10-29 05:55:21,940] INFO : Step 22300/50000 rank6 | Train Loss: 1.0221 | Train Loss Unweighted: 0.5006 | Train Loss Components: Auxiliary: 0.1973 | Cardinality: 0.1169 | Classification: 0.0876 | Classification_binary: 0.1576 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1252 | Regression_eta: 0.0021 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1379 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0667 | Regression_eta: 0.0473 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0556 | LR: 6.26e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.5032s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7903 | max_loss: 0.1446 | max_backward: 1.7078 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 0.8083 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6690.1 | uss_MB: 6062.0 | child_rss_MB: 5023.7 | allocated_MB: 446.4 | reserved_MB: 29136.0 | inactive_split_MB: 1159.3 +rank6 [2026-10-29 06:00:08,367] INFO : Step 22400/50000 rank6 | Train Loss: 1.1170 | Train Loss Unweighted: 0.5609 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1391 | Classification: 0.1065 | Classification_binary: 0.1807 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1348 | Regression_eta: 0.0019 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1903 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0802 | Regression_eta: 0.0602 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0931 | LR: 6.23e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.9524s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7658 | max_loss: 0.2479 | max_backward: 1.9808 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9928 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6691.0 | uss_MB: 6065.0 | child_rss_MB: 5027.0 | allocated_MB: 447.1 | reserved_MB: 31152.0 | inactive_split_MB: 925.7 +rank6 [2026-10-29 06:04:41,560] INFO : Step 22500/50000 rank6 | Train Loss: 1.1245 | Train Loss Unweighted: 0.5482 | Train Loss Components: Auxiliary: 0.2231 | Cardinality: 0.1463 | Classification: 0.0856 | Classification_binary: 0.2083 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1244 | Regression_eta: 0.0016 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0520 | Classification_binary: 0.1911 | Regression_cos_phi: 0.0620 | Regression_energy: 0.0665 | Regression_eta: 0.0485 | Regression_pt: 0.0727 | Regression_sin_phi: 0.0880 | LR: 6.20e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.0100s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 1.1916 | max_loss: 0.1613 | max_backward: 2.1225 | max_optimizer_step: 0.2176 | max_tensorboard_logging: 0.7664 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6685.2 | uss_MB: 6063.6 | child_rss_MB: 5003.5 | allocated_MB: 447.4 | reserved_MB: 31152.0 | inactive_split_MB: 959.7 +rank6 [2026-10-29 06:09:13,551] INFO : Step 22600/50000 rank6 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5177 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1260 | Classification: 0.0928 | Classification_binary: 0.1789 | HitGrouping: 0.1619 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1208 | Regression_eta: 0.0019 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.2153 | Regression_cos_phi: 0.0749 | Regression_energy: 0.0752 | Regression_eta: 0.0467 | Regression_pt: 0.0817 | Regression_sin_phi: 0.0771 | LR: 6.18e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.8016s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0651 | max_forward: 1.1064 | max_loss: 0.1531 | max_backward: 1.9236 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.8515 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6676.3 | uss_MB: 6063.0 | child_rss_MB: 5024.2 | allocated_MB: 448.3 | reserved_MB: 31154.0 | inactive_split_MB: 961.5 +rank6 [2026-10-29 06:13:51,719] INFO : Step 22700/50000 rank6 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5090 | Train Loss Components: Auxiliary: 0.1959 | Cardinality: 0.1101 | Classification: 0.0877 | Classification_binary: 0.1479 | HitGrouping: 0.1634 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1339 | Regression_eta: 0.0015 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1594 | Regression_cos_phi: 0.0699 | Regression_energy: 0.0655 | Regression_eta: 0.0469 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0551 | LR: 6.15e-04 | DataLoad Time: 0.0215s | Model Forward Time: 4.4581s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 1.0743 | max_loss: 0.1383 | max_backward: 1.7235 | max_optimizer_step: 0.2204 | max_tensorboard_logging: 0.5170 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 6672.9 | uss_MB: 6061.8 | child_rss_MB: 5030.8 | allocated_MB: 446.5 | reserved_MB: 31154.0 | inactive_split_MB: 972.7 +rank6 [2026-10-29 06:18:32,627] INFO : Step 22800/50000 rank6 | Train Loss: 1.0730 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2215 | Cardinality: 0.1332 | Classification: 0.1095 | Classification_binary: 0.1782 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1289 | Regression_eta: 0.0014 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.1595 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0653 | Regression_eta: 0.0291 | Regression_pt: 0.0719 | Regression_sin_phi: 0.0671 | LR: 6.12e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7955s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.2500 | max_backward: 2.0107 | max_optimizer_step: 0.2761 | max_tensorboard_logging: 0.6168 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 6673.9 | uss_MB: 6062.6 | child_rss_MB: 5029.6 | allocated_MB: 447.2 | reserved_MB: 31154.0 | inactive_split_MB: 922.1 +rank6 [2026-10-29 06:23:13,210] INFO : Step 22900/50000 rank6 | Train Loss: 1.1830 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2389 | Cardinality: 0.1788 | Classification: 0.0948 | Classification_binary: 0.2023 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1393 | Regression_eta: 0.0018 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.2381 | Regression_cos_phi: 0.0567 | Regression_energy: 0.0857 | Regression_eta: 0.0453 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0688 | LR: 6.09e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.9938s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0073 | max_forward: 0.7992 | max_loss: 0.1478 | max_backward: 1.9051 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.9858 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 6664.2 | uss_MB: 6055.9 | child_rss_MB: 5028.8 | allocated_MB: 448.8 | reserved_MB: 31154.0 | inactive_split_MB: 1109.2 +rank6 [2026-10-29 06:27:51,863] INFO : Step 23000/50000 rank6 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5014 | Train Loss Components: Auxiliary: 0.1951 | Cardinality: 0.1184 | Classification: 0.0781 | Classification_binary: 0.1455 | HitGrouping: 0.1653 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0661 | Regression_eta: 0.0403 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0526 | LR: 6.06e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.6095s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7528 | max_loss: 0.2292 | max_backward: 1.7818 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.8884 | max_tensorboard_flush: 0.0060 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6672.0 | uss_MB: 6063.2 | child_rss_MB: 5028.5 | allocated_MB: 443.9 | reserved_MB: 31154.0 | inactive_split_MB: 985.1 +rank6 [2026-10-29 06:27:52,750] INFO : Running validation on rank6 +rank6 [2026-10-29 06:27:52,751] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 06:27:52,751] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 06:28:45,239] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:28:45,726] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 06:29:12,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 06:29:13,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 06:29:14,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 06:29:15,510] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 06:29:25,520] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 06:29:26,163] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:29:26,632] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 06:29:42,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 06:29:43,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 06:29:44,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 06:29:45,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 06:29:55,492] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 06:29:56,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 06:29:56,644] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 06:30:12,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 06:30:13,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 06:30:14,382] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 06:30:15,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 06:30:25,181] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 06:35:29,455] INFO : Step 23100/50000 rank6 | Train Loss: 1.1240 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1320 | Classification: 0.1122 | Classification_binary: 0.1745 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1274 | Regression_eta: 0.0021 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.1612 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0658 | Regression_eta: 0.0408 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0570 | LR: 6.04e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8781s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7914 | max_loss: 0.1430 | max_backward: 1.7255 | max_optimizer_step: 0.3295 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0695 | Memory: rss_MB: 7843.1 | uss_MB: 7233.5 | child_rss_MB: 5030.9 | allocated_MB: 449.4 | reserved_MB: 25680.0 | inactive_split_MB: 1046.0 +rank6 [2026-10-29 06:40:11,955] INFO : Step 23200/50000 rank6 | Train Loss: 1.1736 | Train Loss Unweighted: 0.5745 | Train Loss Components: Auxiliary: 0.2341 | Cardinality: 0.1586 | Classification: 0.1057 | Classification_binary: 0.2033 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1304 | Regression_eta: 0.0018 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2366 | Regression_cos_phi: 0.1028 | Regression_energy: 0.0702 | Regression_eta: 0.0654 | Regression_pt: 0.0766 | Regression_sin_phi: 0.1393 | LR: 6.01e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.1123s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9381 | max_loss: 0.1692 | max_backward: 2.2545 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6161 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 7846.1 | uss_MB: 5861.1 | child_rss_MB: 5025.2 | allocated_MB: 447.8 | reserved_MB: 25680.0 | inactive_split_MB: 1169.7 +rank6 [2026-10-29 06:44:52,828] INFO : Step 23300/50000 rank6 | Train Loss: 1.0647 | Train Loss Unweighted: 0.5262 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1268 | Classification: 0.0958 | Classification_binary: 0.1665 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1301 | Regression_eta: 0.0017 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1671 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0610 | Regression_eta: 0.0451 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0849 | LR: 5.98e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6094s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8613 | max_loss: 0.1501 | max_backward: 2.0035 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 0.6274 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6729.0 | uss_MB: 6118.5 | child_rss_MB: 5031.7 | allocated_MB: 447.3 | reserved_MB: 29702.0 | inactive_split_MB: 1252.4 +rank6 [2026-10-29 06:49:32,127] INFO : Step 23400/50000 rank6 | Train Loss: 0.9916 | Train Loss Unweighted: 0.5023 | Train Loss Components: Auxiliary: 0.1957 | Cardinality: 0.1164 | Classification: 0.1006 | Classification_binary: 0.1374 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1273 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0499 | Regression_eta: 0.0444 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0368 | LR: 5.95e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3943s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7229 | max_loss: 0.2412 | max_backward: 1.6798 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.4992 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1172 | Memory: rss_MB: 6727.8 | uss_MB: 6117.0 | child_rss_MB: 5031.9 | allocated_MB: 445.7 | reserved_MB: 29702.0 | inactive_split_MB: 1158.3 +rank6 [2026-10-29 06:54:42,356] INFO : Step 23500/50000 rank6 | Train Loss: 1.0349 | Train Loss Unweighted: 0.5234 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1083 | Classification: 0.0851 | Classification_binary: 0.1870 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1235 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1802 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0582 | Regression_eta: 0.0510 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0485 | LR: 5.92e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7752s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7924 | max_loss: 0.2020 | max_backward: 1.9787 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.7987 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6728.4 | uss_MB: 6111.2 | child_rss_MB: 5026.6 | allocated_MB: 446.9 | reserved_MB: 31774.0 | inactive_split_MB: 1042.0 +rank6 [2026-10-29 06:59:38,280] INFO : Step 23600/50000 rank6 | Train Loss: 1.1326 | Train Loss Unweighted: 0.5418 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1458 | Classification: 0.0933 | Classification_binary: 0.1900 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1267 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2146 | Regression_cos_phi: 0.0694 | Regression_energy: 0.0612 | Regression_eta: 0.0482 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0983 | LR: 5.90e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9833s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0063 | max_forward: 0.7946 | max_loss: 0.1855 | max_backward: 1.9114 | max_optimizer_step: 0.2664 | max_tensorboard_logging: 0.9547 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6730.1 | uss_MB: 6120.3 | child_rss_MB: 5028.3 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1333.8 +rank6 [2026-10-29 07:04:20,062] INFO : Step 23700/50000 rank6 | Train Loss: 1.0167 | Train Loss Unweighted: 0.4968 | Train Loss Components: Auxiliary: 0.1932 | Cardinality: 0.1158 | Classification: 0.0853 | Classification_binary: 0.1475 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1285 | Regression_eta: 0.0017 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1725 | Regression_cos_phi: 0.0859 | Regression_energy: 0.0592 | Regression_eta: 0.0735 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0983 | LR: 5.87e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.7141s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0221 | max_forward: 0.7952 | max_loss: 0.1461 | max_backward: 1.8308 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7718 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6725.6 | uss_MB: 6118.0 | child_rss_MB: 5028.2 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1160.4 +rank6 [2026-10-29 07:08:57,914] INFO : Step 23800/50000 rank6 | Train Loss: 1.0651 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1325 | Classification: 0.0886 | Classification_binary: 0.1616 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1318 | Regression_eta: 0.0015 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1454 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0714 | Regression_eta: 0.0281 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0506 | LR: 5.84e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.6715s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8562 | max_loss: 0.1426 | max_backward: 1.8700 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6730.2 | uss_MB: 6120.2 | child_rss_MB: 5026.7 | allocated_MB: 449.4 | reserved_MB: 31776.0 | inactive_split_MB: 1046.7 +rank6 [2026-10-29 07:13:41,573] INFO : Step 23900/50000 rank6 | Train Loss: 1.1515 | Train Loss Unweighted: 0.5718 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1646 | Classification: 0.0962 | Classification_binary: 0.2086 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0744 | Classification_binary: 0.2160 | Regression_cos_phi: 0.0913 | Regression_energy: 0.0697 | Regression_eta: 0.0394 | Regression_pt: 0.0761 | Regression_sin_phi: 0.0563 | LR: 5.81e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.8392s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.8887 | max_loss: 0.2019 | max_backward: 2.1185 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6224 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6714.4 | uss_MB: 6108.3 | child_rss_MB: 5032.9 | allocated_MB: 447.1 | reserved_MB: 31776.0 | inactive_split_MB: 1168.8 +rank6 [2026-10-29 07:18:20,710] INFO : Step 24000/50000 rank6 | Train Loss: 1.0174 | Train Loss Unweighted: 0.5187 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1133 | Classification: 0.0873 | Classification_binary: 0.1674 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0014 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1836 | Regression_cos_phi: 0.0845 | Regression_energy: 0.0609 | Regression_eta: 0.0389 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0836 | LR: 5.78e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.6288s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 1.3237 | max_loss: 0.2403 | max_backward: 1.8319 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5068 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6721.2 | uss_MB: 6114.9 | child_rss_MB: 5029.4 | allocated_MB: 446.4 | reserved_MB: 31776.0 | inactive_split_MB: 957.1 +rank6 [2026-10-29 07:18:21,624] INFO : Running validation on rank6 +rank6 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 07:19:13,461] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:19:13,969] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 07:19:41,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 07:19:42,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 07:19:43,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 07:19:44,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 07:19:54,551] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 07:19:55,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:19:55,705] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 07:20:12,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 07:20:13,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 07:20:14,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 07:20:15,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 07:20:25,027] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 07:20:25,040] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 07:20:25,604] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 07:20:41,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 07:20:42,139] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 07:20:42,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 07:20:43,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 07:20:53,734] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 07:26:06,616] INFO : Step 24100/50000 rank6 | Train Loss: 1.0669 | Train Loss Unweighted: 0.5331 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1324 | Classification: 0.0928 | Classification_binary: 0.1578 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1390 | Regression_eta: 0.0017 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0727 | Classification_binary: 0.1418 | Regression_cos_phi: 0.0521 | Regression_energy: 0.0733 | Regression_eta: 0.0412 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0605 | LR: 5.75e-04 | DataLoad Time: 0.0171s | Model Forward Time: 7.5198s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.7836 | max_loss: 0.1442 | max_backward: 1.8747 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 3.6030 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0626 | Memory: rss_MB: 7857.5 | uss_MB: 7243.1 | child_rss_MB: 5051.0 | allocated_MB: 448.1 | reserved_MB: 24490.0 | inactive_split_MB: 1198.5 +rank6 [2026-10-29 07:30:59,926] INFO : Step 24200/50000 rank6 | Train Loss: 1.1691 | Train Loss Unweighted: 0.5645 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1546 | Classification: 0.1031 | Classification_binary: 0.1915 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1325 | Regression_eta: 0.0019 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1950 | Regression_cos_phi: 0.0890 | Regression_energy: 0.0635 | Regression_eta: 0.0625 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0832 | LR: 5.73e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2337s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.9207 | max_loss: 0.1533 | max_backward: 2.2288 | max_optimizer_step: 0.2436 | max_tensorboard_logging: 0.8522 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7858.2 | uss_MB: 7247.2 | child_rss_MB: 5039.0 | allocated_MB: 450.2 | reserved_MB: 26490.0 | inactive_split_MB: 976.9 +rank6 [2026-10-29 07:35:42,203] INFO : Step 24300/50000 rank6 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5336 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1380 | Classification: 0.0927 | Classification_binary: 0.1863 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1247 | Regression_eta: 0.0020 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0563 | Classification_binary: 0.1927 | Regression_cos_phi: 0.0963 | Regression_energy: 0.0622 | Regression_eta: 0.0745 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0975 | LR: 5.70e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.6731s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8648 | max_loss: 0.1481 | max_backward: 1.7152 | max_optimizer_step: 0.2781 | max_tensorboard_logging: 0.7831 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6810.0 | uss_MB: 6199.1 | child_rss_MB: 5043.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 983.3 +rank6 [2026-10-29 07:40:19,153] INFO : Step 24400/50000 rank6 | Train Loss: 0.9707 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0879 | Classification_binary: 0.1454 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0014 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0953 | Regression_energy: 0.0532 | Regression_eta: 0.0342 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0456 | LR: 5.67e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.8581s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7628 | max_loss: 0.1940 | max_backward: 2.4253 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.9481 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6811.6 | uss_MB: 5803.5 | child_rss_MB: 5042.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 1129.3 +rank6 [2026-10-29 07:45:00,050] INFO : Step 24500/50000 rank6 | Train Loss: 1.0937 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1319 | Classification: 0.0941 | Classification_binary: 0.1676 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1334 | Regression_eta: 0.0016 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1726 | Regression_cos_phi: 0.0901 | Regression_energy: 0.0708 | Regression_eta: 0.0425 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0495 | LR: 5.64e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4276s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7565 | max_loss: 0.5105 | max_backward: 1.8057 | max_optimizer_step: 0.2534 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6807.0 | uss_MB: 6198.6 | child_rss_MB: 5045.1 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 945.2 +rank6 [2026-10-29 07:49:44,236] INFO : Step 24600/50000 rank6 | Train Loss: 1.0924 | Train Loss Unweighted: 0.5511 | Train Loss Components: Auxiliary: 0.2237 | Cardinality: 0.1233 | Classification: 0.0963 | Classification_binary: 0.1991 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1255 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1939 | Regression_cos_phi: 0.0817 | Regression_energy: 0.0578 | Regression_eta: 0.0419 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0713 | LR: 5.61e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6467s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.9060 | max_loss: 0.1523 | max_backward: 2.0894 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.5102 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6811.1 | uss_MB: 6200.6 | child_rss_MB: 5046.3 | allocated_MB: 449.5 | reserved_MB: 26492.0 | inactive_split_MB: 943.2 +rank6 [2026-10-29 07:54:29,278] INFO : Step 24700/50000 rank6 | Train Loss: 0.9972 | Train Loss Unweighted: 0.4984 | Train Loss Components: Auxiliary: 0.1950 | Cardinality: 0.1160 | Classification: 0.0782 | Classification_binary: 0.1692 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0304 | Classification_binary: 0.1900 | Regression_cos_phi: 0.0585 | Regression_energy: 0.0672 | Regression_eta: 0.0380 | Regression_pt: 0.0740 | Regression_sin_phi: 0.0483 | LR: 5.58e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5117s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0060 | max_forward: 0.7847 | max_loss: 0.1519 | max_backward: 1.7159 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.8325 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6813.0 | uss_MB: 6199.7 | child_rss_MB: 5050.6 | allocated_MB: 447.9 | reserved_MB: 27708.0 | inactive_split_MB: 1169.6 +rank6 [2026-10-29 07:59:09,067] INFO : Step 24800/50000 rank6 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4722 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1202 | Classification: 0.0808 | Classification_binary: 0.1484 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0016 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1355 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0669 | Regression_eta: 0.0376 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0552 | LR: 5.56e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7168s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0127 | max_forward: 0.7903 | max_loss: 0.1427 | max_backward: 1.7798 | max_optimizer_step: 0.2260 | max_tensorboard_logging: 0.8943 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6813.0 | uss_MB: 6199.9 | child_rss_MB: 5051.5 | allocated_MB: 447.6 | reserved_MB: 28508.0 | inactive_split_MB: 989.1 +rank6 [2026-10-29 08:03:50,793] INFO : Step 24900/50000 rank6 | Train Loss: 1.0773 | Train Loss Unweighted: 0.5300 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1506 | Classification: 0.0964 | Classification_binary: 0.1776 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1257 | Regression_eta: 0.0016 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1571 | Regression_cos_phi: 0.0857 | Regression_energy: 0.0607 | Regression_eta: 0.0329 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0542 | LR: 5.53e-04 | DataLoad Time: 0.0152s | Model Forward Time: 5.3163s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8100 | max_loss: 0.1483 | max_backward: 1.9423 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 1.0834 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6802.2 | uss_MB: 5593.7 | child_rss_MB: 5050.0 | allocated_MB: 449.1 | reserved_MB: 28508.0 | inactive_split_MB: 993.6 +rank6 [2026-10-29 08:08:37,679] INFO : Step 25000/50000 rank6 | Train Loss: 1.1315 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1671 | Classification: 0.0915 | Classification_binary: 0.1899 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1201 | Regression_eta: 0.0020 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1909 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0640 | Regression_eta: 0.0458 | Regression_pt: 0.0704 | Regression_sin_phi: 0.0692 | LR: 5.50e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7054s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8152 | max_loss: 0.2476 | max_backward: 1.7290 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.8750 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6807.0 | uss_MB: 6195.6 | child_rss_MB: 5052.2 | allocated_MB: 448.5 | reserved_MB: 28508.0 | inactive_split_MB: 953.9 +rank6 [2026-10-29 08:08:38,676] INFO : Running validation on rank6 +rank6 [2026-10-29 08:08:38,676] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 08:08:38,676] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 08:09:30,805] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:09:31,355] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 08:09:57,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 08:09:59,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 08:10:00,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 08:10:01,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 08:10:11,124] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 08:10:11,661] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:10:12,184] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 08:10:28,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 08:10:29,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 08:10:30,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 08:10:31,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 08:10:41,926] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 08:10:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 08:10:42,693] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 08:10:58,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 08:10:59,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 08:11:00,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 08:11:01,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 08:11:11,022] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 08:16:19,774] INFO : Step 25100/50000 rank6 | Train Loss: 1.0264 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1288 | Classification: 0.0850 | Classification_binary: 0.1497 | HitGrouping: 0.1664 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1378 | Regression_eta: 0.0016 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0361 | Classification_binary: 0.1516 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0783 | Regression_eta: 0.0432 | Regression_pt: 0.0853 | Regression_sin_phi: 0.0902 | LR: 5.47e-04 | DataLoad Time: 0.0175s | Model Forward Time: 5.5422s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0059 | max_forward: 0.7878 | max_loss: 0.1437 | max_backward: 2.2383 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9692 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7903.8 | uss_MB: 7286.7 | child_rss_MB: 5042.9 | allocated_MB: 446.8 | reserved_MB: 24062.0 | inactive_split_MB: 1159.3 +rank6 [2026-10-29 08:21:20,699] INFO : Step 25200/50000 rank6 | Train Loss: 1.0775 | Train Loss Unweighted: 0.5434 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1397 | Classification: 0.0996 | Classification_binary: 0.1706 | HitGrouping: 0.1584 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1340 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1649 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0710 | Regression_eta: 0.0564 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0548 | LR: 5.44e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.2307s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8831 | max_loss: 0.1456 | max_backward: 2.1320 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7903.3 | uss_MB: 7287.2 | child_rss_MB: 5042.2 | allocated_MB: 448.0 | reserved_MB: 26042.0 | inactive_split_MB: 1093.3 +rank6 [2026-10-29 08:26:17,211] INFO : Step 25300/50000 rank6 | Train Loss: 1.0761 | Train Loss Unweighted: 0.5109 | Train Loss Components: Auxiliary: 0.2132 | Cardinality: 0.1361 | Classification: 0.0961 | Classification_binary: 0.1821 | HitGrouping: 0.1562 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1138 | Regression_eta: 0.0018 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0750 | Classification_binary: 0.2104 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0676 | Regression_eta: 0.0612 | Regression_pt: 0.0741 | Regression_sin_phi: 0.0957 | LR: 5.42e-04 | DataLoad Time: 0.0197s | Model Forward Time: 6.1296s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.9122 | max_loss: 0.1821 | max_backward: 2.1499 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.9506 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6897.2 | uss_MB: 5222.8 | child_rss_MB: 5036.8 | allocated_MB: 446.7 | reserved_MB: 28062.0 | inactive_split_MB: 1098.3 +rank6 [2026-10-29 08:31:10,695] INFO : Step 25400/50000 rank6 | Train Loss: 1.0222 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1238 | Classification: 0.0928 | Classification_binary: 0.1653 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1221 | Regression_eta: 0.0015 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0648 | Regression_eta: 0.0391 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0670 | LR: 5.39e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.7093s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1954 | max_backward: 1.8415 | max_optimizer_step: 0.3277 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6896.2 | uss_MB: 6278.8 | child_rss_MB: 5052.2 | allocated_MB: 449.3 | reserved_MB: 28062.0 | inactive_split_MB: 945.6 +rank6 [2026-10-29 08:36:13,578] INFO : Step 25500/50000 rank6 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4596 | Train Loss Components: Auxiliary: 0.1817 | Cardinality: 0.1001 | Classification: 0.0953 | Classification_binary: 0.1334 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0016 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1345 | Regression_cos_phi: 0.0411 | Regression_energy: 0.0632 | Regression_eta: 0.0318 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0371 | LR: 5.36e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.7846s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7884 | max_loss: 0.1475 | max_backward: 2.0272 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.7862 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0483 | Memory: rss_MB: 6898.2 | uss_MB: 6283.2 | child_rss_MB: 5050.7 | allocated_MB: 450.7 | reserved_MB: 28062.0 | inactive_split_MB: 1166.9 +rank6 [2026-10-29 08:41:08,951] INFO : Step 25600/50000 rank6 | Train Loss: 1.0724 | Train Loss Unweighted: 0.5330 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1190 | Classification: 0.0985 | Classification_binary: 0.1843 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1231 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1790 | Regression_cos_phi: 0.0450 | Regression_energy: 0.0627 | Regression_eta: 0.0352 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0551 | LR: 5.33e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.1549s | Phase Times: max_pre_step_barrier: 0.0194 | max_batch_to_device: 0.0083 | max_forward: 1.0622 | max_loss: 0.3311 | max_backward: 2.5999 | max_optimizer_step: 0.3019 | max_tensorboard_logging: 0.8123 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6895.2 | uss_MB: 5585.2 | child_rss_MB: 5023.7 | allocated_MB: 450.4 | reserved_MB: 28064.0 | inactive_split_MB: 1163.5 +rank6 [2026-10-29 08:46:09,676] INFO : Step 25700/50000 rank6 | Train Loss: 1.0595 | Train Loss Unweighted: 0.5120 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.1394 | Classification: 0.0920 | Classification_binary: 0.1853 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.2472 | Regression_cos_phi: 0.1275 | Regression_energy: 0.0591 | Regression_eta: 0.0666 | Regression_pt: 0.0625 | Regression_sin_phi: 0.1075 | LR: 5.30e-04 | DataLoad Time: 0.0199s | Model Forward Time: 5.8072s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.8320 | max_loss: 0.2520 | max_backward: 2.6962 | max_optimizer_step: 0.2699 | max_tensorboard_logging: 0.7057 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6885.0 | uss_MB: 6272.6 | child_rss_MB: 5051.6 | allocated_MB: 448.6 | reserved_MB: 29942.0 | inactive_split_MB: 1165.2 +rank6 [2026-10-29 08:51:40,159] INFO : Step 25800/50000 rank6 | Train Loss: 0.9592 | Train Loss Unweighted: 0.4733 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1038 | Classification: 0.0787 | Classification_binary: 0.1531 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1182 | Regression_eta: 0.0016 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1754 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0734 | Regression_eta: 0.0493 | Regression_pt: 0.0816 | Regression_sin_phi: 0.0866 | LR: 5.27e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4827s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7549 | max_loss: 0.1480 | max_backward: 1.5296 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 0.8796 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1361 | Memory: rss_MB: 6889.1 | uss_MB: 6277.5 | child_rss_MB: 5023.6 | allocated_MB: 445.9 | reserved_MB: 30100.0 | inactive_split_MB: 961.8 +rank6 [2026-10-29 08:56:50,110] INFO : Step 25900/50000 rank6 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4613 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1142 | Classification: 0.0768 | Classification_binary: 0.1501 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1145 | Regression_eta: 0.0015 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0648 | Regression_eta: 0.0366 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0654 | LR: 5.25e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8322s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0124 | max_forward: 0.7901 | max_loss: 0.1430 | max_backward: 1.9362 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8933 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 6880.5 | uss_MB: 6281.6 | child_rss_MB: 5008.5 | allocated_MB: 447.7 | reserved_MB: 30100.0 | inactive_split_MB: 1167.6 +rank6 [2026-10-29 09:01:47,521] INFO : Step 26000/50000 rank6 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5169 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1347 | Classification: 0.0896 | Classification_binary: 0.1938 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0015 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1964 | Regression_cos_phi: 0.0764 | Regression_energy: 0.0549 | Regression_eta: 0.0417 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0660 | LR: 5.22e-04 | DataLoad Time: 0.0151s | Model Forward Time: 5.4970s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8125 | max_loss: 0.2612 | max_backward: 1.8771 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.8972 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6845.2 | uss_MB: 5700.0 | child_rss_MB: 4882.0 | allocated_MB: 450.3 | reserved_MB: 30108.0 | inactive_split_MB: 1161.3 +rank6 [2026-10-29 09:01:48,415] INFO : Running validation on rank6 +rank6 [2026-10-29 09:01:48,415] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 09:01:48,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 09:02:42,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:02:42,921] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 09:03:09,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 09:03:10,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 09:03:11,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 09:03:12,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 09:03:22,325] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 09:03:22,720] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:03:23,246] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 09:03:39,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 09:03:40,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 09:03:41,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 09:03:42,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 09:03:52,190] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 09:03:52,918] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:03:53,429] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 09:04:08,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 09:04:09,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 09:04:10,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 09:04:11,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 09:04:21,339] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 09:09:37,593] INFO : Step 26100/50000 rank6 | Train Loss: 0.9519 | Train Loss Unweighted: 0.4738 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1091 | Classification: 0.0871 | Classification_binary: 0.1547 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1147 | Regression_eta: 0.0013 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0564 | Regression_eta: 0.0437 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0624 | LR: 5.19e-04 | DataLoad Time: 0.0176s | Model Forward Time: 5.2077s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0321 | max_forward: 0.8143 | max_loss: 0.1540 | max_backward: 1.7669 | max_optimizer_step: 0.2382 | max_tensorboard_logging: 1.1185 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 8060.0 | uss_MB: 7433.5 | child_rss_MB: 4850.5 | allocated_MB: 445.1 | reserved_MB: 25850.0 | inactive_split_MB: 1376.9 +rank6 [2026-10-29 09:14:36,072] INFO : Step 26200/50000 rank6 | Train Loss: 0.9758 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.0970 | Classification: 0.0858 | Classification_binary: 0.1501 | HitGrouping: 0.1665 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1499 | Regression_cos_phi: 0.0464 | Regression_energy: 0.0648 | Regression_eta: 0.0423 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0557 | LR: 5.16e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6740s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7728 | max_loss: 0.1384 | max_backward: 1.8195 | max_optimizer_step: 0.3036 | max_tensorboard_logging: 0.7010 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0602 | Memory: rss_MB: 8060.1 | uss_MB: 7488.7 | child_rss_MB: 4848.5 | allocated_MB: 447.4 | reserved_MB: 27830.0 | inactive_split_MB: 1117.7 +rank6 [2026-10-29 09:19:51,625] INFO : Step 26300/50000 rank6 | Train Loss: 1.0568 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2146 | Cardinality: 0.1416 | Classification: 0.0935 | Classification_binary: 0.1772 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1417 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0556 | Regression_eta: 0.0305 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0574 | LR: 5.13e-04 | DataLoad Time: 0.0202s | Model Forward Time: 6.1497s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.8205 | max_loss: 0.1808 | max_backward: 2.8872 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 1.1024 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6927.6 | uss_MB: 6354.4 | child_rss_MB: 4842.0 | allocated_MB: 450.8 | reserved_MB: 29830.0 | inactive_split_MB: 1114.5 +rank6 [2026-10-29 09:24:55,631] INFO : Step 26400/50000 rank6 | Train Loss: 1.0536 | Train Loss Unweighted: 0.5239 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1323 | Classification: 0.0937 | Classification_binary: 0.1845 | HitGrouping: 0.1559 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1209 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.2301 | Regression_cos_phi: 0.0757 | Regression_energy: 0.0662 | Regression_eta: 0.0492 | Regression_pt: 0.0688 | Regression_sin_phi: 0.0702 | LR: 5.10e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.5424s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8639 | max_loss: 0.1569 | max_backward: 2.4710 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 1.0049 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6924.7 | uss_MB: 6353.2 | child_rss_MB: 4853.6 | allocated_MB: 447.6 | reserved_MB: 29832.0 | inactive_split_MB: 1186.4 +rank6 [2026-10-29 09:29:59,513] INFO : Step 26500/50000 rank6 | Train Loss: 0.9812 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1299 | Classification: 0.0807 | Classification_binary: 0.1400 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1217 | Regression_eta: 0.0017 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1423 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0692 | Regression_eta: 0.0526 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0511 | LR: 5.08e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.8054s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7436 | max_loss: 0.1579 | max_backward: 1.8417 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.9265 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6921.4 | uss_MB: 6354.1 | child_rss_MB: 4851.7 | allocated_MB: 444.4 | reserved_MB: 29832.0 | inactive_split_MB: 1229.5 +rank6 [2026-10-29 09:34:58,506] INFO : Step 26600/50000 rank6 | Train Loss: 1.0108 | Train Loss Unweighted: 0.5066 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.1182 | Classification: 0.0957 | Classification_binary: 0.1588 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1242 | Regression_eta: 0.0014 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0701 | Regression_eta: 0.0431 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0725 | LR: 5.05e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6966s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7561 | max_loss: 0.1430 | max_backward: 1.8427 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6921.4 | uss_MB: 6356.0 | child_rss_MB: 4827.0 | allocated_MB: 447.9 | reserved_MB: 29832.0 | inactive_split_MB: 1037.2 +rank6 [2026-10-29 09:40:07,140] INFO : Step 26700/50000 rank6 | Train Loss: 1.0878 | Train Loss Unweighted: 0.5266 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1269 | Classification: 0.0956 | Classification_binary: 0.1961 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1144 | Regression_eta: 0.0018 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.1867 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0622 | Regression_eta: 0.0521 | Regression_pt: 0.0685 | Regression_sin_phi: 0.0631 | LR: 5.02e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1893s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.9788 | max_loss: 0.1540 | max_backward: 1.9407 | max_optimizer_step: 0.2295 | max_tensorboard_logging: 1.0076 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6911.8 | uss_MB: 6344.9 | child_rss_MB: 4859.9 | allocated_MB: 450.1 | reserved_MB: 29832.0 | inactive_split_MB: 1190.4 +rank6 [2026-10-29 09:48:41,511] INFO : Step 26800/50000 rank6 | Train Loss: 1.0077 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1102 | Classification: 0.0787 | Classification_binary: 0.1553 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1359 | Regression_eta: 0.0015 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1587 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0923 | Regression_eta: 0.0388 | Regression_pt: 0.1056 | Regression_sin_phi: 0.0683 | LR: 4.99e-04 | DataLoad Time: 0.0210s | Model Forward Time: 4.8125s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.7774 | max_loss: 0.2523 | max_backward: 1.8741 | max_optimizer_step: 0.2873 | max_tensorboard_logging: 0.8206 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6909.2 | uss_MB: 6351.0 | child_rss_MB: 4816.3 | allocated_MB: 444.7 | reserved_MB: 29832.0 | inactive_split_MB: 1121.3 +rank6 [2026-10-29 09:53:40,048] INFO : Step 26900/50000 rank6 | Train Loss: 0.9524 | Train Loss Unweighted: 0.4793 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1099 | Classification: 0.0833 | Classification_binary: 0.1305 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1313 | Regression_eta: 0.0016 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0575 | Classification_binary: 0.1313 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0681 | Regression_eta: 0.0351 | Regression_pt: 0.0728 | Regression_sin_phi: 0.0724 | LR: 4.96e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.4175s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7292 | max_loss: 0.1373 | max_backward: 1.7442 | max_optimizer_step: 0.3152 | max_tensorboard_logging: 0.6764 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 6905.2 | uss_MB: 5520.6 | child_rss_MB: 4788.1 | allocated_MB: 446.9 | reserved_MB: 29832.0 | inactive_split_MB: 1016.7 +rank6 [2026-10-29 09:58:37,162] INFO : Step 27000/50000 rank6 | Train Loss: 1.0285 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1243 | Classification: 0.0862 | Classification_binary: 0.1888 | HitGrouping: 0.1539 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0016 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.2166 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0589 | Regression_eta: 0.0524 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0552 | LR: 4.94e-04 | DataLoad Time: 0.0218s | Model Forward Time: 5.1607s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.9134 | max_loss: 0.1497 | max_backward: 2.1803 | max_optimizer_step: 0.2254 | max_tensorboard_logging: 0.9490 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6904.1 | uss_MB: 6354.5 | child_rss_MB: 4772.3 | allocated_MB: 449.7 | reserved_MB: 29832.0 | inactive_split_MB: 1024.3 +rank6 [2026-10-29 09:58:38,108] INFO : Running validation on rank6 +rank6 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 09:58:38,109] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 09:59:32,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 09:59:33,287] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 09:59:59,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 10:00:00,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 10:00:02,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 10:00:03,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 10:00:13,090] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 10:00:14,009] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:00:14,537] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 10:00:31,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 10:00:33,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 10:00:34,544] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 10:00:35,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 10:00:45,531] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-29 10:00:46,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:00:46,864] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 10:01:02,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 10:01:03,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 10:01:04,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 10:01:05,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 10:01:15,011] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 10:06:37,477] INFO : Step 27100/50000 rank6 | Train Loss: 1.0614 | Train Loss Unweighted: 0.4989 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1449 | Classification: 0.0752 | Classification_binary: 0.1877 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.1812 | Regression_cos_phi: 0.0755 | Regression_energy: 0.0608 | Regression_eta: 0.0413 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0474 | LR: 4.91e-04 | DataLoad Time: 0.0216s | Model Forward Time: 7.1159s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0966 | max_forward: 2.1724 | max_loss: 0.6705 | max_backward: 3.1723 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 1.5888 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 8047.0 | uss_MB: 7496.7 | child_rss_MB: 4778.7 | allocated_MB: 447.0 | reserved_MB: 27292.0 | inactive_split_MB: 983.4 +rank6 [2026-10-29 10:11:19,567] INFO : Step 27200/50000 rank6 | Train Loss: 0.9045 | Train Loss Unweighted: 0.4389 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1028 | Classification: 0.0770 | Classification_binary: 0.1287 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0544 | Regression_eta: 0.0327 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0485 | LR: 4.88e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.7894s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7725 | max_loss: 0.5032 | max_backward: 1.6788 | max_optimizer_step: 0.2815 | max_tensorboard_logging: 0.9686 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8047.0 | uss_MB: 7493.7 | child_rss_MB: 4786.7 | allocated_MB: 445.7 | reserved_MB: 29236.0 | inactive_split_MB: 1232.8 +rank6 [2026-10-29 10:15:58,638] INFO : Step 27300/50000 rank6 | Train Loss: 1.0726 | Train Loss Unweighted: 0.5095 | Train Loss Components: Auxiliary: 0.2138 | Cardinality: 0.1337 | Classification: 0.1081 | Classification_binary: 0.1640 | HitGrouping: 0.1596 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1159 | Regression_eta: 0.0019 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0853 | Classification_binary: 0.1792 | Regression_cos_phi: 0.0905 | Regression_energy: 0.0654 | Regression_eta: 0.1014 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0890 | LR: 4.85e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.1426s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8896 | max_loss: 0.1460 | max_backward: 2.0153 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6974.0 | uss_MB: 6424.6 | child_rss_MB: 4784.6 | allocated_MB: 450.9 | reserved_MB: 29330.0 | inactive_split_MB: 1103.1 +rank6 [2026-10-29 10:20:31,815] INFO : Step 27400/50000 rank6 | Train Loss: 1.0758 | Train Loss Unweighted: 0.5282 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1344 | Classification: 0.0930 | Classification_binary: 0.1899 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1207 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1665 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0851 | Regression_eta: 0.0333 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0881 | LR: 4.82e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.0220s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8446 | max_loss: 0.1488 | max_backward: 2.0519 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.8725 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6977.0 | uss_MB: 5627.2 | child_rss_MB: 4777.8 | allocated_MB: 448.4 | reserved_MB: 31310.0 | inactive_split_MB: 895.7 +rank6 [2026-10-29 10:25:07,332] INFO : Step 27500/50000 rank6 | Train Loss: 0.9697 | Train Loss Unweighted: 0.4574 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1111 | Classification: 0.0831 | Classification_binary: 0.1473 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1114 | Regression_eta: 0.0018 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1733 | Regression_cos_phi: 0.0849 | Regression_energy: 0.0736 | Regression_eta: 0.0585 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0774 | LR: 4.80e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4614s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.7904 | max_loss: 0.1563 | max_backward: 1.6291 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.7457 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6966.1 | uss_MB: 4710.9 | child_rss_MB: 4769.8 | allocated_MB: 447.3 | reserved_MB: 31310.0 | inactive_split_MB: 986.8 +rank6 [2026-10-29 10:30:03,961] INFO : Step 27600/50000 rank6 | Train Loss: 0.9616 | Train Loss Unweighted: 0.4618 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1092 | Classification: 0.0865 | Classification_binary: 0.1284 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0019 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1008 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0507 | Regression_eta: 0.0390 | Regression_pt: 0.0555 | Regression_sin_phi: 0.0633 | LR: 4.77e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5709s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0380 | max_forward: 0.7674 | max_loss: 0.1463 | max_backward: 1.9164 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.5145 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6974.1 | uss_MB: 6422.9 | child_rss_MB: 4778.4 | allocated_MB: 446.7 | reserved_MB: 31310.0 | inactive_split_MB: 1182.0 +rank6 [2026-10-29 10:34:46,993] INFO : Step 27700/50000 rank6 | Train Loss: 1.0693 | Train Loss Unweighted: 0.5190 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1414 | Classification: 0.0939 | Classification_binary: 0.1783 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0015 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1783 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0650 | Regression_eta: 0.0474 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0682 | LR: 4.74e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.6747s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.8151 | max_loss: 0.1750 | max_backward: 2.0398 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.5213 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0659 | Memory: rss_MB: 6974.1 | uss_MB: 6424.8 | child_rss_MB: 4756.0 | allocated_MB: 448.4 | reserved_MB: 33312.0 | inactive_split_MB: 986.3 +rank6 [2026-10-29 10:39:29,302] INFO : Step 27800/50000 rank6 | Train Loss: 1.0875 | Train Loss Unweighted: 0.5294 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1226 | Classification: 0.0924 | Classification_binary: 0.1842 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1237 | Regression_eta: 0.0019 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1709 | Regression_cos_phi: 0.0819 | Regression_energy: 0.0544 | Regression_eta: 0.0722 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0587 | LR: 4.71e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8085s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0152 | max_forward: 0.8463 | max_loss: 0.1744 | max_backward: 1.7379 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.9397 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 6971.2 | uss_MB: 6421.2 | child_rss_MB: 4773.7 | allocated_MB: 452.2 | reserved_MB: 33312.0 | inactive_split_MB: 1320.1 +rank6 [2026-10-29 10:44:11,939] INFO : Step 27900/50000 rank6 | Train Loss: 0.9483 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0988 | Classification: 0.0859 | Classification_binary: 0.1439 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1244 | Regression_eta: 0.0014 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0591 | Regression_eta: 0.0439 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0475 | LR: 4.68e-04 | DataLoad Time: 0.0204s | Model Forward Time: 4.4745s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.7626 | max_loss: 0.1370 | max_backward: 1.8424 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6972.2 | uss_MB: 6423.3 | child_rss_MB: 4773.1 | allocated_MB: 447.1 | reserved_MB: 33312.0 | inactive_split_MB: 1112.6 +rank6 [2026-10-29 10:48:55,971] INFO : Step 28000/50000 rank6 | Train Loss: 0.9840 | Train Loss Unweighted: 0.4736 | Train Loss Components: Auxiliary: 0.1949 | Cardinality: 0.1114 | Classification: 0.0927 | Classification_binary: 0.1564 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1099 | Regression_eta: 0.0015 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0379 | Regression_energy: 0.0547 | Regression_eta: 0.0323 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0634 | LR: 4.66e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9417s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8192 | max_loss: 0.2552 | max_backward: 2.1299 | max_optimizer_step: 0.2920 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 6969.4 | uss_MB: 6424.2 | child_rss_MB: 4776.8 | allocated_MB: 448.2 | reserved_MB: 33312.0 | inactive_split_MB: 1178.0 +rank6 [2026-10-29 10:48:56,836] INFO : Running validation on rank6 +rank6 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 10:49:51,449] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:49:52,005] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 10:50:18,715] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 10:50:19,755] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 10:50:20,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 10:50:21,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 10:50:31,858] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 10:50:32,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:50:32,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 10:50:49,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 10:50:50,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 10:50:51,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 10:50:52,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 10:51:02,210] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 10:51:02,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 10:51:03,445] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 10:51:19,093] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 10:51:19,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 10:51:20,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 10:51:21,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 10:51:31,518] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 10:56:32,744] INFO : Step 28100/50000 rank6 | Train Loss: 1.0915 | Train Loss Unweighted: 0.5085 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1649 | Classification: 0.0833 | Classification_binary: 0.1834 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1187 | Regression_eta: 0.0017 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2087 | Regression_cos_phi: 0.0774 | Regression_energy: 0.0633 | Regression_eta: 0.0683 | Regression_pt: 0.0694 | Regression_sin_phi: 0.0930 | LR: 4.63e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.8656s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8571 | max_loss: 0.1603 | max_backward: 1.9112 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 0.7670 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0838 | Memory: rss_MB: 8288.3 | uss_MB: 7733.7 | child_rss_MB: 4773.7 | allocated_MB: 448.9 | reserved_MB: 25946.0 | inactive_split_MB: 1313.1 +rank6 [2026-10-29 11:01:18,173] INFO : Step 28200/50000 rank6 | Train Loss: 0.9241 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1144 | Classification: 0.0770 | Classification_binary: 0.1441 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1163 | Regression_eta: 0.0014 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0417 | Classification_binary: 0.1515 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0580 | Regression_eta: 0.0272 | Regression_pt: 0.0644 | Regression_sin_phi: 0.0503 | LR: 4.60e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8974s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0751 | max_forward: 0.8691 | max_loss: 0.5236 | max_backward: 1.8483 | max_optimizer_step: 0.2480 | max_tensorboard_logging: 0.8546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 8289.0 | uss_MB: 7733.3 | child_rss_MB: 4770.2 | allocated_MB: 445.0 | reserved_MB: 25948.0 | inactive_split_MB: 976.4 +rank6 [2026-10-29 11:07:40,702] INFO : Step 28300/50000 rank6 | Train Loss: 0.9513 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1028 | Classification: 0.0882 | Classification_binary: 0.1368 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1147 | Regression_eta: 0.0016 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0621 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0602 | Regression_energy: 0.0696 | Regression_eta: 0.0380 | Regression_pt: 0.0758 | Regression_sin_phi: 0.0675 | LR: 4.57e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.9458s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.7916 | max_loss: 0.1449 | max_backward: 2.0850 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.8583 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7055.4 | uss_MB: 6496.0 | child_rss_MB: 4774.9 | allocated_MB: 447.2 | reserved_MB: 25952.0 | inactive_split_MB: 1165.3 +rank6 [2026-10-29 11:12:27,615] INFO : Step 28400/50000 rank6 | Train Loss: 1.0604 | Train Loss Unweighted: 0.4982 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1382 | Classification: 0.0847 | Classification_binary: 0.1862 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1111 | Regression_eta: 0.0018 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1533 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0656 | Regression_eta: 0.0523 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0457 | LR: 4.55e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.0302s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9139 | max_loss: 0.1474 | max_backward: 2.1323 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.6846 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0517 | Memory: rss_MB: 7055.5 | uss_MB: 6496.2 | child_rss_MB: 4770.0 | allocated_MB: 448.2 | reserved_MB: 26968.0 | inactive_split_MB: 835.9 +rank6 [2026-10-29 11:17:11,503] INFO : Step 28500/50000 rank6 | Train Loss: 1.0282 | Train Loss Unweighted: 0.5005 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1237 | Classification: 0.0911 | Classification_binary: 0.1756 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0013 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1853 | Regression_cos_phi: 0.0789 | Regression_energy: 0.0611 | Regression_eta: 0.0310 | Regression_pt: 0.0686 | Regression_sin_phi: 0.0671 | LR: 4.52e-04 | DataLoad Time: 0.0212s | Model Forward Time: 9.4586s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0886 | max_forward: 0.9758 | max_loss: 0.5123 | max_backward: 2.0811 | max_optimizer_step: 0.2797 | max_tensorboard_logging: 1.5644 | max_tensorboard_flush: 0.0047 | max_post_step_barrier: 0.0889 | Memory: rss_MB: 7042.2 | uss_MB: 4349.9 | child_rss_MB: 4774.7 | allocated_MB: 452.0 | reserved_MB: 27968.0 | inactive_split_MB: 923.2 +rank6 [2026-10-29 11:21:56,248] INFO : Step 28600/50000 rank6 | Train Loss: 0.9753 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1202 | Classification: 0.0804 | Classification_binary: 0.1446 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1132 | Regression_eta: 0.0017 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0605 | Regression_eta: 0.0569 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0985 | LR: 4.49e-04 | DataLoad Time: 0.0232s | Model Forward Time: 4.9417s | Phase Times: max_pre_step_barrier: 0.1127 | max_batch_to_device: 0.0146 | max_forward: 0.7731 | max_loss: 0.2404 | max_backward: 1.6352 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 1.0006 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7048.8 | uss_MB: 5469.7 | child_rss_MB: 4776.7 | allocated_MB: 446.8 | reserved_MB: 30056.0 | inactive_split_MB: 896.4 +rank6 [2026-10-29 11:26:34,428] INFO : Step 28700/50000 rank6 | Train Loss: 1.0558 | Train Loss Unweighted: 0.5173 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1082 | Classification: 0.1065 | Classification_binary: 0.1612 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1217 | Regression_eta: 0.0024 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0627 | Classification_binary: 0.1436 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0657 | Regression_eta: 0.0384 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0627 | LR: 4.46e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.9565s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0686 | max_forward: 0.8423 | max_loss: 0.1475 | max_backward: 2.0170 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.8830 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 7053.8 | uss_MB: 6496.4 | child_rss_MB: 4774.2 | allocated_MB: 447.9 | reserved_MB: 30056.0 | inactive_split_MB: 925.2 +rank6 [2026-10-29 11:31:18,243] INFO : Step 28800/50000 rank6 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5167 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1474 | Classification: 0.0969 | Classification_binary: 0.1939 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1784 | Regression_cos_phi: 0.0761 | Regression_energy: 0.0543 | Regression_eta: 0.0679 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0769 | LR: 4.44e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.8743s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0075 | max_forward: 0.8427 | max_loss: 0.2661 | max_backward: 2.8464 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7050.1 | uss_MB: 4315.7 | child_rss_MB: 4755.7 | allocated_MB: 448.2 | reserved_MB: 30058.0 | inactive_split_MB: 976.8 +rank6 [2026-10-29 11:35:53,801] INFO : Step 28900/50000 rank6 | Train Loss: 0.9389 | Train Loss Unweighted: 0.4547 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.1078 | Classification: 0.0733 | Classification_binary: 0.1521 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0491 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0762 | Regression_eta: 0.0416 | Regression_pt: 0.0829 | Regression_sin_phi: 0.0712 | LR: 4.41e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.5677s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0062 | max_forward: 0.7499 | max_loss: 0.2426 | max_backward: 1.8556 | max_optimizer_step: 0.3135 | max_tensorboard_logging: 0.5831 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7048.4 | uss_MB: 6494.0 | child_rss_MB: 4774.7 | allocated_MB: 444.3 | reserved_MB: 30058.0 | inactive_split_MB: 1133.2 +rank6 [2026-10-29 11:40:53,318] INFO : Step 29000/50000 rank6 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4535 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0939 | Classification: 0.0855 | Classification_binary: 0.1308 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0015 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1346 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0613 | Regression_eta: 0.0422 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0848 | LR: 4.38e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.8471s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0062 | max_forward: 0.9270 | max_loss: 0.2073 | max_backward: 2.1896 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6044 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7047.7 | uss_MB: 6496.6 | child_rss_MB: 4759.3 | allocated_MB: 450.2 | reserved_MB: 30060.0 | inactive_split_MB: 1402.9 +rank6 [2026-10-29 11:40:54,336] INFO : Running validation on rank6 +rank6 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 11:41:46,666] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:41:47,176] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 11:42:16,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 11:42:18,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 11:42:19,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 11:42:20,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 11:42:30,590] INFO : Time taken to make predictions on device 6 is: 0.29 min +rank6 [2026-10-29 11:42:31,032] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:42:31,573] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 11:42:47,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 11:42:48,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 11:42:49,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 11:42:50,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 11:43:00,785] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 11:43:00,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 11:43:01,398] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 11:43:16,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 11:43:17,385] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 11:43:18,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 11:43:18,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 11:43:28,962] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 11:48:35,107] INFO : Step 29100/50000 rank6 | Train Loss: 1.0559 | Train Loss Unweighted: 0.5040 | Train Loss Components: Auxiliary: 0.2142 | Cardinality: 0.1307 | Classification: 0.0939 | Classification_binary: 0.1851 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1098 | Regression_eta: 0.0015 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0795 | Classification_binary: 0.1686 | Regression_cos_phi: 0.0971 | Regression_energy: 0.0646 | Regression_eta: 0.0555 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0982 | LR: 4.35e-04 | DataLoad Time: 0.0157s | Model Forward Time: 7.6725s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.6779 | max_loss: 0.3815 | max_backward: 4.2661 | max_optimizer_step: 0.3470 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8253.0 | uss_MB: 6395.3 | child_rss_MB: 4770.6 | allocated_MB: 448.8 | reserved_MB: 24188.0 | inactive_split_MB: 1162.1 +rank6 [2026-10-29 11:53:34,049] INFO : Step 29200/50000 rank6 | Train Loss: 0.9909 | Train Loss Unweighted: 0.4801 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1204 | Classification: 0.0923 | Classification_binary: 0.1746 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1054 | Regression_eta: 0.0014 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0495 | Classification_binary: 0.2064 | Regression_cos_phi: 0.0813 | Regression_energy: 0.0563 | Regression_eta: 0.0358 | Regression_pt: 0.0613 | Regression_sin_phi: 0.0746 | LR: 4.33e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7773s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8354 | max_loss: 0.1558 | max_backward: 2.1201 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0677 | Memory: rss_MB: 8251.2 | uss_MB: 7692.4 | child_rss_MB: 4763.1 | allocated_MB: 447.9 | reserved_MB: 24190.0 | inactive_split_MB: 1166.5 +rank6 [2026-10-29 11:58:27,982] INFO : Step 29300/50000 rank6 | Train Loss: 0.9337 | Train Loss Unweighted: 0.4462 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1121 | Classification: 0.0773 | Classification_binary: 0.1359 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1146 | Regression_eta: 0.0015 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1643 | Regression_cos_phi: 0.0776 | Regression_energy: 0.0612 | Regression_eta: 0.0530 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0726 | LR: 4.30e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.9137s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0277 | max_forward: 0.8212 | max_loss: 0.1457 | max_backward: 1.8414 | max_optimizer_step: 0.2550 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7135.1 | uss_MB: 5997.0 | child_rss_MB: 4761.5 | allocated_MB: 444.9 | reserved_MB: 24190.0 | inactive_split_MB: 1038.6 +rank6 [2026-10-29 12:03:07,470] INFO : Step 29400/50000 rank6 | Train Loss: 1.0186 | Train Loss Unweighted: 0.4675 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1372 | Classification: 0.0948 | Classification_binary: 0.1561 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0020 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1537 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0777 | Regression_eta: 0.0536 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0824 | LR: 4.27e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8019s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8495 | max_loss: 0.1434 | max_backward: 1.9359 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7145.4 | uss_MB: 6588.5 | child_rss_MB: 4762.3 | allocated_MB: 447.9 | reserved_MB: 26208.0 | inactive_split_MB: 957.3 +rank6 [2026-10-29 12:08:16,203] INFO : Step 29500/50000 rank6 | Train Loss: 1.0909 | Train Loss Unweighted: 0.5362 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.1071 | Classification_binary: 0.1896 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1176 | Regression_eta: 0.0019 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0746 | Classification_binary: 0.1642 | Regression_cos_phi: 0.1034 | Regression_energy: 0.0504 | Regression_eta: 0.0462 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0751 | LR: 4.24e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9668s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0876 | max_forward: 0.8925 | max_loss: 0.1527 | max_backward: 1.9621 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.8162 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7137.1 | uss_MB: 6580.0 | child_rss_MB: 4760.9 | allocated_MB: 448.9 | reserved_MB: 26214.0 | inactive_split_MB: 1166.5 +rank6 [2026-10-29 12:13:12,652] INFO : Step 29600/50000 rank6 | Train Loss: 0.9608 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.1935 | Cardinality: 0.1093 | Classification: 0.0850 | Classification_binary: 0.1542 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1571 | Regression_cos_phi: 0.0570 | Regression_energy: 0.0525 | Regression_eta: 0.0442 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0582 | LR: 4.22e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8672s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7848 | max_loss: 0.1495 | max_backward: 1.8042 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.0178 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7115.2 | uss_MB: 6585.9 | child_rss_MB: 4743.4 | allocated_MB: 445.5 | reserved_MB: 26214.0 | inactive_split_MB: 1171.9 +rank6 [2026-10-29 12:18:07,442] INFO : Step 29700/50000 rank6 | Train Loss: 0.9511 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1260 | Classification: 0.0829 | Classification_binary: 0.1357 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1183 | Regression_eta: 0.0014 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0354 | Classification_binary: 0.1139 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0586 | Regression_eta: 0.0296 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0355 | LR: 4.19e-04 | DataLoad Time: 0.0221s | Model Forward Time: 5.2346s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7531 | max_loss: 0.2308 | max_backward: 2.2313 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.9708 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7117.3 | uss_MB: 6587.7 | child_rss_MB: 4741.9 | allocated_MB: 446.6 | reserved_MB: 26214.0 | inactive_split_MB: 1160.6 +rank6 [2026-10-29 12:23:08,927] INFO : Step 29800/50000 rank6 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4915 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1104 | Classification: 0.0864 | Classification_binary: 0.1759 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0736 | Classification_binary: 0.1818 | Regression_cos_phi: 0.0612 | Regression_energy: 0.0666 | Regression_eta: 0.0592 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0749 | LR: 4.16e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.4223s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8210 | max_loss: 0.1528 | max_backward: 2.4753 | max_optimizer_step: 0.3576 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0038 | max_post_step_barrier: 0.1798 | Memory: rss_MB: 7115.5 | uss_MB: 5536.1 | child_rss_MB: 4741.9 | allocated_MB: 450.3 | reserved_MB: 26214.0 | inactive_split_MB: 938.8 +rank6 [2026-10-29 12:28:10,142] INFO : Step 29900/50000 rank6 | Train Loss: 0.9604 | Train Loss Unweighted: 0.4559 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1243 | Classification: 0.0825 | Classification_binary: 0.1616 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1040 | Regression_eta: 0.0015 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1937 | Regression_cos_phi: 0.0681 | Regression_energy: 0.0605 | Regression_eta: 0.0435 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0618 | LR: 4.14e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9481s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1142 | max_forward: 0.9271 | max_loss: 0.1531 | max_backward: 1.8698 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.8620 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7091.0 | uss_MB: 6437.5 | child_rss_MB: 4745.0 | allocated_MB: 450.0 | reserved_MB: 26214.0 | inactive_split_MB: 1166.0 +rank6 [2026-10-29 12:32:58,888] INFO : Step 30000/50000 rank6 | Train Loss: 0.9450 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.1132 | Classification: 0.0847 | Classification_binary: 0.1394 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0335 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0376 | Regression_energy: 0.0544 | Regression_eta: 0.0245 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0368 | LR: 4.11e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.1888s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 1.0327 | max_loss: 0.1419 | max_backward: 1.6503 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.4878 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7039.0 | uss_MB: 6586.5 | child_rss_MB: 4745.4 | allocated_MB: 443.8 | reserved_MB: 26214.0 | inactive_split_MB: 1152.2 +rank6 [2026-10-29 12:47:12,790] INFO : Running validation on rank6 +rank6 [2026-10-29 12:47:12,790] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 12:47:12,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 14:49:00,086] INFO : Initializing model with seed=12345; process seed=12351 +rank6 [2026-10-29 14:49:00,097] INFO : configured dtype=torch.bfloat16 for autocast +rank6 [2026-10-29 14:49:20,929] INFO : Instantiating model +rank6 [2026-10-29 14:49:20,929] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank6 [2026-10-29 14:49:20,930] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank6 [2026-10-29 14:49:20,948] INFO : Input encoding initialization took 0.02s +rank6 [2026-10-29 14:49:20,948] INFO : nn0 parameters: 171776 +rank6 [2026-10-29 14:49:20,948] INFO : Initializing shared backbone layers, num_convs=6 +rank6 [2026-10-29 14:49:20,948] INFO : Initializing backbone layer 0 +rank6 [2026-10-29 14:49:20,983] INFO : Initializing backbone layer 1 +rank6 [2026-10-29 14:49:20,992] INFO : Initializing backbone layer 2 +rank6 [2026-10-29 14:49:21,000] INFO : Initializing backbone layer 3 +rank6 [2026-10-29 14:49:21,009] INFO : Initializing backbone layer 4 +rank6 [2026-10-29 14:49:21,017] INFO : Initializing backbone layer 5 +rank6 [2026-10-29 14:49:21,026] INFO : Backbone initialization took 0.08s +rank6 [2026-10-29 14:49:21,026] INFO : backbone parameters: 7901376 +rank6 [2026-10-29 14:49:21,026] INFO : Initializing output DNNs +rank6 [2026-10-29 14:49:21,063] INFO : Output DNNs initialization took 0.04s +rank6 [2026-10-29 14:49:21,063] INFO : backbone_mode=BackboneMode.SHARED +rank6 [2026-10-29 14:49:21,063] INFO : jagged_attention enabled=False +rank6 [2026-10-29 14:49:21,063] INFO : flash_attn_varlen enabled=False +rank6 [2026-10-29 14:49:21,063] INFO : task_queries enabled=False +rank6 [2026-10-29 14:49:21,063] INFO : classification_norm parameters: 0 +rank6 [2026-10-29 14:49:21,063] INFO : regression_norm parameters: 0 +rank6 [2026-10-29 14:49:21,063] INFO : classification_query parameters: 0 +rank6 [2026-10-29 14:49:21,063] INFO : regression_query parameters: 0 +rank6 [2026-10-29 14:49:21,063] INFO : classification_readout parameters: 0 +rank6 [2026-10-29 14:49:21,063] INFO : regression_readout parameters: 0 +rank6 [2026-10-29 14:49:21,063] INFO : output_mode=set +rank6 [2026-10-29 14:49:21,063] INFO : set_decoder parameters: 4713812 +rank6 [2026-10-29 14:49:21,064] INFO : nn_binary_particle parameters: 0 +rank6 [2026-10-29 14:49:21,064] INFO : nn_pid parameters: 0 +rank6 [2026-10-29 14:49:21,064] INFO : nn_pt parameters: 0 +rank6 [2026-10-29 14:49:21,064] INFO : nn_eta parameters: 0 +rank6 [2026-10-29 14:49:21,064] INFO : nn_sin_phi parameters: 0 +rank6 [2026-10-29 14:49:21,064] INFO : nn_cos_phi parameters: 0 +rank6 [2026-10-29 14:49:21,064] INFO : nn_energy parameters: 0 +rank6 [2026-10-29 14:49:21,064] INFO : Total MLPF parameters: 12786964 +rank6 [2026-10-29 14:49:21,065] INFO : MLPF __init__ done +rank6 [2026-10-29 14:49:21,065] INFO : Instantiated model +rank6 [2026-10-29 14:49:21,065] INFO : Moving model to device rank=6 +rank6 [2026-10-29 14:49:21,087] INFO : Moved model to device rank=6 +rank6 [2026-10-29 14:49:21,089] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank6 [2026-10-29 14:49:21,561] INFO : Restoring training from step 30001 +rank6 [2026-10-29 14:49:21,569] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth +rank6 [2026-10-29 14:49:21,569] INFO : Configured model for SyncBatchNorm rank=6 +rank6 [2026-10-29 14:49:22,135] INFO : Configured model for DistributedDataParallel rank=6 +rank6 [2026-10-29 14:49:22,289] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank6 [2026-10-29 14:49:22,289] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank6 [2026-10-29 14:49:22,291] INFO : Trainable parameters: 12786964 +rank6 [2026-10-29 14:49:22,291] INFO : Non-trainable parameters: 23268 +rank6 [2026-10-29 14:49:22,291] INFO : Total parameters: 12810232 +rank6 [2026-10-29 14:49:22,294] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank6 [2026-10-29 14:49:22,294] INFO : Getting dataloaders +rank6 [2026-10-29 14:49:22,294] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:49:28,732] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:49:29,729] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:49:30,958] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank6 [2026-10-29 14:49:30,958] INFO : Creating WeightedMixIterator with 1 data loaders. +rank6 [2026-10-29 14:49:30,958] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:49:31,234] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:49:31,488] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 14:49:31,740] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank6 [2026-10-29 14:49:31,740] INFO : Creating InterleavedIterator with 1 data loaders. +rank6 [2026-10-29 14:49:45,498] INFO : Got dataloaders +rank6 [2026-10-29 14:49:45,498] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [8888] of epochs [1] +rank6 [2026-10-29 14:49:45,498] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank6 [2026-10-29 14:49:45,498] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank6 [2026-10-29 14:49:45,498] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank6 [2026-10-29 14:49:47,370] INFO : loader split=train rank=6 len=21112 +rank6 [2026-10-29 14:49:47,370] INFO : loader split=valid rank=6 len=12 +rank6 [2026-10-29 14:49:47,370] INFO : Starting training from step 30001 to 50000 on rank 6 of 8 +rank6 [2026-10-29 14:49:47,370] INFO : Creating train_iterator +rank6 [2026-10-29 14:49:47,370] INFO : ResumableSampler physical:train loading state: start_index=142208 +rank6 [2026-10-29 14:50:01,778] INFO : Created train_iterator +rank6 [2026-10-29 14:54:42,004] INFO : Step 30100/50000 rank6 | Train Loss: 0.9854 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.2024 | Cardinality: 0.1295 | Classification: 0.0965 | Classification_binary: 0.1561 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1758 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0628 | Regression_eta: 0.0420 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0848 | LR: 4.08e-04 | DataLoad Time: 0.0186s | Model Forward Time: 3.6352s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7011 | max_loss: 0.1473 | max_backward: 1.5273 | max_optimizer_step: 0.1574 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0061 | Memory: rss_MB: 4775.9 | uss_MB: 3472.5 | child_rss_MB: 6376.2 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 850.1 +rank6 [2026-10-29 14:59:12,793] INFO : Step 30200/50000 rank6 | Train Loss: 1.0165 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2044 | Cardinality: 0.1520 | Classification: 0.0856 | Classification_binary: 0.1718 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1461 | Regression_cos_phi: 0.0391 | Regression_energy: 0.0447 | Regression_eta: 0.0299 | Regression_pt: 0.0493 | Regression_sin_phi: 0.0497 | LR: 4.06e-04 | DataLoad Time: 0.0160s | Model Forward Time: 3.9883s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.7915 | max_loss: 0.2568 | max_backward: 1.8926 | max_optimizer_step: 0.1551 | max_tensorboard_logging: 0.4604 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4776.3 | uss_MB: 3476.7 | child_rss_MB: 6367.6 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 842.0 +rank6 [2026-10-29 15:03:42,966] INFO : Step 30300/50000 rank6 | Train Loss: 0.9183 | Train Loss Unweighted: 0.4449 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1232 | Classification: 0.0820 | Classification_binary: 0.1459 | HitGrouping: 0.1516 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0012 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0641 | Regression_energy: 0.0584 | Regression_eta: 0.0425 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0628 | LR: 4.03e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.8582s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7525 | max_loss: 0.2476 | max_backward: 1.8262 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.4574 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0039 | Memory: rss_MB: 4773.5 | uss_MB: 3475.1 | child_rss_MB: 6385.6 | allocated_MB: 495.8 | reserved_MB: 28760.0 | inactive_split_MB: 848.2 +rank6 [2026-10-29 15:08:13,476] INFO : Step 30400/50000 rank6 | Train Loss: 0.9035 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1137 | Classification: 0.0848 | Classification_binary: 0.1358 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0361 | Regression_energy: 0.0601 | Regression_eta: 0.0331 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0389 | LR: 4.00e-04 | DataLoad Time: 0.0168s | Model Forward Time: 3.9733s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7769 | max_loss: 0.1398 | max_backward: 1.9002 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.4925 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4770.8 | uss_MB: 3475.5 | child_rss_MB: 6366.2 | allocated_MB: 498.8 | reserved_MB: 28760.0 | inactive_split_MB: 878.9 +rank6 [2026-10-29 15:12:45,607] INFO : Step 30500/50000 rank6 | Train Loss: 1.0399 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1239 | Classification: 0.0901 | Classification_binary: 0.1841 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0018 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1695 | Regression_cos_phi: 0.0777 | Regression_energy: 0.0533 | Regression_eta: 0.0409 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0850 | LR: 3.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.3857s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9688 | max_loss: 0.1627 | max_backward: 2.3054 | max_optimizer_step: 0.1555 | max_tensorboard_logging: 0.4908 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4768.4 | uss_MB: 3475.6 | child_rss_MB: 6405.5 | allocated_MB: 501.7 | reserved_MB: 28760.0 | inactive_split_MB: 944.3 +rank6 [2026-10-29 15:17:13,866] INFO : Step 30600/50000 rank6 | Train Loss: 0.9987 | Train Loss Unweighted: 0.4959 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1112 | Classification: 0.0932 | Classification_binary: 0.1720 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1137 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0728 | Regression_eta: 0.0475 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0846 | LR: 3.95e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2233s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7736 | max_loss: 0.1566 | max_backward: 1.7922 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.6994 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4770.8 | uss_MB: 3476.1 | child_rss_MB: 6399.5 | allocated_MB: 500.1 | reserved_MB: 28760.0 | inactive_split_MB: 845.9 +rank6 [2026-10-29 15:21:45,752] INFO : Step 30700/50000 rank6 | Train Loss: 0.9490 | Train Loss Unweighted: 0.4575 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1237 | Classification: 0.0844 | Classification_binary: 0.1399 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0014 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0389 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0558 | Regression_eta: 0.0424 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0405 | LR: 3.92e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1843s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7215 | max_loss: 0.5560 | max_backward: 2.0906 | max_optimizer_step: 0.1553 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4768.7 | uss_MB: 3474.9 | child_rss_MB: 6379.2 | allocated_MB: 493.8 | reserved_MB: 28760.0 | inactive_split_MB: 827.5 +rank6 [2026-10-29 15:26:16,185] INFO : Step 30800/50000 rank6 | Train Loss: 1.0320 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1331 | Classification: 0.0916 | Classification_binary: 0.1632 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1173 | Regression_eta: 0.0015 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1374 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0704 | Regression_eta: 0.0354 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0569 | LR: 3.90e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6912s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1016 | max_forward: 0.8041 | max_loss: 0.1462 | max_backward: 1.9351 | max_optimizer_step: 0.1552 | max_tensorboard_logging: 1.1007 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4769.5 | uss_MB: 3475.6 | child_rss_MB: 6407.8 | allocated_MB: 497.9 | reserved_MB: 28760.0 | inactive_split_MB: 842.0 +rank6 [2026-10-29 15:30:49,188] INFO : Step 30900/50000 rank6 | Train Loss: 1.0226 | Train Loss Unweighted: 0.4952 | Train Loss Components: Auxiliary: 0.2139 | Cardinality: 0.1339 | Classification: 0.0875 | Classification_binary: 0.1859 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1092 | Regression_eta: 0.0013 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1697 | Regression_cos_phi: 0.0709 | Regression_energy: 0.0500 | Regression_eta: 0.0402 | Regression_pt: 0.0552 | Regression_sin_phi: 0.0398 | LR: 3.87e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.4019s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7610 | max_loss: 0.2067 | max_backward: 1.7028 | max_optimizer_step: 0.1599 | max_tensorboard_logging: 1.0327 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4771.1 | uss_MB: 3476.3 | child_rss_MB: 6385.8 | allocated_MB: 497.2 | reserved_MB: 28760.0 | inactive_split_MB: 838.0 +rank6 [2026-10-29 15:35:26,546] INFO : Step 31000/50000 rank6 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4630 | Train Loss Components: Auxiliary: 0.2006 | Cardinality: 0.1217 | Classification: 0.0821 | Classification_binary: 0.1592 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1085 | Regression_eta: 0.0018 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1590 | Regression_cos_phi: 0.0486 | Regression_energy: 0.0575 | Regression_eta: 0.0511 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0456 | LR: 3.84e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.4977s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7780 | max_loss: 0.2475 | max_backward: 1.8803 | max_optimizer_step: 0.1588 | max_tensorboard_logging: 0.9947 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4772.1 | uss_MB: 3477.0 | child_rss_MB: 6402.3 | allocated_MB: 498.2 | reserved_MB: 28760.0 | inactive_split_MB: 820.4 +rank6 [2026-10-29 15:35:27,414] INFO : Running validation on rank6 +rank6 [2026-10-29 15:35:27,414] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 15:35:27,414] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 15:36:19,716] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:36:20,267] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 15:36:46,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 15:36:47,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 15:36:48,461] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 15:36:49,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 15:36:59,542] INFO : Time taken to make predictions on device 6 is: 0.28 min +rank6 [2026-10-29 15:36:59,566] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:37:00,104] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 15:37:15,563] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 15:37:16,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 15:37:17,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 15:37:18,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 15:37:28,477] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 15:37:29,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 15:37:29,838] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 15:37:45,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 15:37:46,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 15:37:46,896] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 15:37:47,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 15:37:57,690] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 15:42:50,283] INFO : Step 31100/50000 rank6 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4264 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1054 | Classification: 0.0853 | Classification_binary: 0.1288 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0481 | Classification_binary: 0.1295 | Regression_cos_phi: 0.0322 | Regression_energy: 0.0555 | Regression_eta: 0.0261 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0485 | LR: 3.82e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5234s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7353 | max_loss: 0.1410 | max_backward: 1.7959 | max_optimizer_step: 0.2754 | max_tensorboard_logging: 0.9599 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 6463.6 | uss_MB: 5188.4 | child_rss_MB: 6394.5 | allocated_MB: 495.8 | reserved_MB: 27508.0 | inactive_split_MB: 851.2 +rank6 [2026-10-29 15:47:34,509] INFO : Step 31200/50000 rank6 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4709 | Train Loss Components: Auxiliary: 0.2045 | Cardinality: 0.1286 | Classification: 0.0862 | Classification_binary: 0.1646 | HitGrouping: 0.1524 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1077 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0631 | Classification_binary: 0.1574 | Regression_cos_phi: 0.0815 | Regression_energy: 0.0647 | Regression_eta: 0.0539 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0578 | LR: 3.79e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3216s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8441 | max_loss: 0.1469 | max_backward: 2.0813 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 6462.4 | uss_MB: 5189.4 | child_rss_MB: 6422.3 | allocated_MB: 496.9 | reserved_MB: 27508.0 | inactive_split_MB: 935.6 +rank6 [2026-10-29 15:52:19,964] INFO : Step 31300/50000 rank6 | Train Loss: 1.0631 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1394 | Classification: 0.0917 | Classification_binary: 0.1731 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1780 | Regression_cos_phi: 0.1007 | Regression_energy: 0.0606 | Regression_eta: 0.0395 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0561 | LR: 3.76e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2506s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7950 | max_loss: 0.1940 | max_backward: 1.8810 | max_optimizer_step: 0.2519 | max_tensorboard_logging: 0.5719 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5602.7 | uss_MB: 4330.1 | child_rss_MB: 6421.1 | allocated_MB: 498.6 | reserved_MB: 29580.0 | inactive_split_MB: 773.5 +rank6 [2026-10-29 15:56:52,523] INFO : Step 31400/50000 rank6 | Train Loss: 0.9019 | Train Loss Unweighted: 0.4258 | Train Loss Components: Auxiliary: 0.1823 | Cardinality: 0.1101 | Classification: 0.0746 | Classification_binary: 0.1337 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1070 | Regression_eta: 0.0014 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1405 | Regression_cos_phi: 0.0484 | Regression_energy: 0.0587 | Regression_eta: 0.0288 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0526 | LR: 3.74e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2764s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8478 | max_loss: 0.2460 | max_backward: 2.0403 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.4770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5595.4 | uss_MB: 4331.0 | child_rss_MB: 6422.8 | allocated_MB: 495.2 | reserved_MB: 29580.0 | inactive_split_MB: 986.6 +rank6 [2026-10-29 16:01:34,249] INFO : Step 31500/50000 rank6 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4434 | Train Loss Components: Auxiliary: 0.1867 | Cardinality: 0.1036 | Classification: 0.0834 | Classification_binary: 0.1437 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1064 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0651 | Regression_eta: 0.0545 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0880 | LR: 3.71e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.7500s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7930 | max_loss: 0.1458 | max_backward: 1.7855 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 2.2800 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5572.8 | uss_MB: 4310.0 | child_rss_MB: 6426.4 | allocated_MB: 496.8 | reserved_MB: 29580.0 | inactive_split_MB: 850.8 +rank6 [2026-10-29 16:06:05,475] INFO : Step 31600/50000 rank6 | Train Loss: 1.0288 | Train Loss Unweighted: 0.4730 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1317 | Classification: 0.0906 | Classification_binary: 0.1743 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1012 | Regression_eta: 0.0017 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0781 | Classification_binary: 0.1764 | Regression_cos_phi: 0.0998 | Regression_energy: 0.0554 | Regression_eta: 0.0739 | Regression_pt: 0.0616 | Regression_sin_phi: 0.1085 | LR: 3.69e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1932s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1021 | max_forward: 0.8041 | max_loss: 0.5000 | max_backward: 1.8881 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0133 | Memory: rss_MB: 5567.6 | uss_MB: 4330.2 | child_rss_MB: 6421.3 | allocated_MB: 494.5 | reserved_MB: 29580.0 | inactive_split_MB: 850.5 +rank6 [2026-10-29 16:10:48,123] INFO : Step 31700/50000 rank6 | Train Loss: 0.9206 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1106 | Classification: 0.0729 | Classification_binary: 0.1465 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1687 | Regression_cos_phi: 0.0989 | Regression_energy: 0.0668 | Regression_eta: 0.0442 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0929 | LR: 3.66e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.7513s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0062 | max_forward: 0.7097 | max_loss: 0.1806 | max_backward: 1.6117 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0100 | Memory: rss_MB: 5567.0 | uss_MB: 4328.6 | child_rss_MB: 6424.1 | allocated_MB: 495.3 | reserved_MB: 29580.0 | inactive_split_MB: 969.6 +rank6 [2026-10-29 16:15:56,265] INFO : Step 31800/50000 rank6 | Train Loss: 0.9090 | Train Loss Unweighted: 0.4310 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1140 | Classification: 0.0748 | Classification_binary: 0.1332 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1094 | Regression_eta: 0.0013 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0308 | Classification_binary: 0.1232 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0526 | Regression_eta: 0.0313 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0433 | LR: 3.64e-04 | DataLoad Time: 0.0157s | Model Forward Time: 3.8758s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7102 | max_loss: 0.1436 | max_backward: 1.6093 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5562.0 | uss_MB: 4326.9 | child_rss_MB: 6424.4 | allocated_MB: 499.4 | reserved_MB: 29584.0 | inactive_split_MB: 1068.5 +rank6 [2026-10-29 16:20:39,712] INFO : Step 31900/50000 rank6 | Train Loss: 0.9919 | Train Loss Unweighted: 0.4645 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1269 | Classification: 0.0911 | Classification_binary: 0.1745 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0975 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1703 | Regression_cos_phi: 0.1034 | Regression_energy: 0.0494 | Regression_eta: 0.0394 | Regression_pt: 0.0550 | Regression_sin_phi: 0.0703 | LR: 3.61e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.3404s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.2538 | max_backward: 1.9768 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.5597 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5568.8 | uss_MB: 4333.1 | child_rss_MB: 6433.0 | allocated_MB: 496.2 | reserved_MB: 29584.0 | inactive_split_MB: 829.8 +rank6 [2026-10-29 16:25:21,505] INFO : Step 32000/50000 rank6 | Train Loss: 0.9662 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1172 | Classification: 0.0861 | Classification_binary: 0.1650 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1010 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1624 | Regression_cos_phi: 0.0741 | Regression_energy: 0.0603 | Regression_eta: 0.0310 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0572 | LR: 3.58e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2269s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7801 | max_loss: 0.1534 | max_backward: 1.8397 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5561.3 | uss_MB: 4329.3 | child_rss_MB: 6422.4 | allocated_MB: 497.7 | reserved_MB: 29584.0 | inactive_split_MB: 927.6 +rank6 [2026-10-29 16:25:22,457] INFO : Running validation on rank6 +rank6 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 16:26:16,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:26:16,711] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 16:26:42,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 16:26:43,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 16:26:45,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 16:26:46,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 16:26:56,072] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 16:26:56,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:26:57,010] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 16:27:12,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 16:27:13,976] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 16:27:14,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 16:27:15,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 16:27:25,923] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 16:27:26,913] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 16:27:27,382] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 16:27:42,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 16:27:43,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 16:27:44,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 16:27:45,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 16:27:55,357] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 16:32:59,848] INFO : Step 32100/50000 rank6 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4572 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1125 | Classification: 0.0829 | Classification_binary: 0.1389 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1159 | Regression_eta: 0.0016 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1495 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0740 | Regression_eta: 0.0737 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0559 | LR: 3.56e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8025s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0431 | max_forward: 0.7694 | max_loss: 0.1379 | max_backward: 1.5662 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 1.3675 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6985.8 | uss_MB: 5757.8 | child_rss_MB: 6447.8 | allocated_MB: 499.0 | reserved_MB: 25862.0 | inactive_split_MB: 892.0 +rank6 [2026-10-29 16:37:33,176] INFO : Step 32200/50000 rank6 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1193 | Classification: 0.0958 | Classification_binary: 0.1508 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1092 | Regression_eta: 0.0016 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1232 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0543 | Regression_eta: 0.0556 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0547 | LR: 3.53e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.1394s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8207 | max_loss: 0.1504 | max_backward: 2.0613 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.9148 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6989.2 | uss_MB: 5763.2 | child_rss_MB: 6457.2 | allocated_MB: 499.4 | reserved_MB: 25864.0 | inactive_split_MB: 778.6 +rank6 [2026-10-29 16:42:04,507] INFO : Step 32300/50000 rank6 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4909 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1295 | Classification: 0.0875 | Classification_binary: 0.1834 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0017 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1913 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0580 | Regression_eta: 0.0491 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0677 | LR: 3.51e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7847s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8382 | max_loss: 0.2412 | max_backward: 2.0637 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.9540 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5664.5 | uss_MB: 4450.0 | child_rss_MB: 6462.2 | allocated_MB: 499.0 | reserved_MB: 27716.0 | inactive_split_MB: 775.0 +rank6 [2026-10-29 16:46:37,461] INFO : Step 32400/50000 rank6 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1089 | Classification: 0.0860 | Classification_binary: 0.1494 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1059 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1382 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0522 | Regression_eta: 0.0336 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0821 | LR: 3.48e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5414s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1137 | max_forward: 0.8338 | max_loss: 0.1463 | max_backward: 1.8285 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 0.8970 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5664.1 | uss_MB: 4449.3 | child_rss_MB: 6459.6 | allocated_MB: 497.5 | reserved_MB: 27716.0 | inactive_split_MB: 964.6 +rank6 [2026-10-29 16:51:11,723] INFO : Step 32500/50000 rank6 | Train Loss: 0.9126 | Train Loss Unweighted: 0.4568 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.0989 | Classification: 0.0980 | Classification_binary: 0.1302 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1128 | Regression_eta: 0.0017 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1639 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0514 | Regression_eta: 0.0829 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0799 | LR: 3.46e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0954s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7619 | max_loss: 0.1497 | max_backward: 1.5269 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.7665 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5643.1 | uss_MB: 4429.2 | child_rss_MB: 6454.9 | allocated_MB: 495.6 | reserved_MB: 27718.0 | inactive_split_MB: 1084.2 +rank6 [2026-10-29 16:55:48,603] INFO : Step 32600/50000 rank6 | Train Loss: 0.9821 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2025 | Cardinality: 0.1270 | Classification: 0.0917 | Classification_binary: 0.1687 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1000 | Regression_eta: 0.0014 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0699 | Classification_binary: 0.1584 | Regression_cos_phi: 0.0712 | Regression_energy: 0.0582 | Regression_eta: 0.0421 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0606 | LR: 3.43e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2674s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1505 | max_backward: 1.8823 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 5658.5 | uss_MB: 4448.5 | child_rss_MB: 6453.0 | allocated_MB: 496.5 | reserved_MB: 27844.0 | inactive_split_MB: 835.5 +rank6 [2026-10-29 17:00:16,962] INFO : Step 32700/50000 rank6 | Train Loss: 1.0218 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2103 | Cardinality: 0.1308 | Classification: 0.0908 | Classification_binary: 0.1761 | HitGrouping: 0.1579 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0547 | Regression_eta: 0.0764 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0544 | LR: 3.41e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.2019s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7654 | max_loss: 0.2523 | max_backward: 1.8639 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4904 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.1101 | Memory: rss_MB: 5657.8 | uss_MB: 4448.3 | child_rss_MB: 6459.1 | allocated_MB: 495.8 | reserved_MB: 27844.0 | inactive_split_MB: 1177.8 +rank6 [2026-10-29 17:04:50,646] INFO : Step 32800/50000 rank6 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4798 | Train Loss Components: Auxiliary: 0.1977 | Cardinality: 0.1139 | Classification: 0.0957 | Classification_binary: 0.1480 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1163 | Regression_eta: 0.0019 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1512 | Regression_cos_phi: 0.0620 | Regression_energy: 0.1002 | Regression_eta: 0.0435 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0647 | LR: 3.38e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.0491s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7437 | max_loss: 0.1443 | max_backward: 1.7603 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.5989 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5618.6 | uss_MB: 4447.1 | child_rss_MB: 6328.3 | allocated_MB: 497.6 | reserved_MB: 27846.0 | inactive_split_MB: 1089.8 +rank6 [2026-10-29 17:09:25,012] INFO : Step 32900/50000 rank6 | Train Loss: 1.0585 | Train Loss Unweighted: 0.4917 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1472 | Classification: 0.0991 | Classification_binary: 0.1566 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0020 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1198 | Regression_cos_phi: 0.0435 | Regression_energy: 0.0632 | Regression_eta: 0.0618 | Regression_pt: 0.0667 | Regression_sin_phi: 0.0775 | LR: 3.36e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2363s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0141 | max_forward: 0.7853 | max_loss: 0.1442 | max_backward: 1.9716 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.5450 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5622.7 | uss_MB: 4451.0 | child_rss_MB: 6346.8 | allocated_MB: 498.5 | reserved_MB: 27846.0 | inactive_split_MB: 844.8 +rank6 [2026-10-29 17:13:57,544] INFO : Step 33000/50000 rank6 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4896 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1109 | Classification: 0.0902 | Classification_binary: 0.1757 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1096 | Regression_eta: 0.0015 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1970 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0678 | Regression_eta: 0.0410 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0673 | LR: 3.33e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.1397s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0372 | max_forward: 0.7798 | max_loss: 0.1504 | max_backward: 1.8342 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5614.6 | uss_MB: 4448.7 | child_rss_MB: 6348.5 | allocated_MB: 497.7 | reserved_MB: 27846.0 | inactive_split_MB: 857.6 +rank6 [2026-10-29 17:13:58,425] INFO : Running validation on rank6 +rank6 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 17:14:49,526] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:14:50,069] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 17:15:17,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 17:15:18,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 17:15:19,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 17:15:20,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 17:15:30,321] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 17:15:31,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:15:31,508] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 17:15:47,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 17:15:48,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 17:15:49,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 17:15:50,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 17:16:00,028] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 17:16:01,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 17:16:01,833] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 17:16:17,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 17:16:18,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 17:16:18,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 17:16:19,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 17:16:29,618] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 17:21:40,219] INFO : Step 33100/50000 rank6 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4666 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1297 | Classification: 0.0842 | Classification_binary: 0.1537 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1125 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0457 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0461 | Regression_energy: 0.0557 | Regression_eta: 0.0392 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0645 | LR: 3.31e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1407s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7607 | max_loss: 0.1464 | max_backward: 1.8352 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6888.0 | uss_MB: 5749.0 | child_rss_MB: 6329.7 | allocated_MB: 496.1 | reserved_MB: 24572.0 | inactive_split_MB: 941.9 +rank6 [2026-10-29 17:26:33,026] INFO : Step 33200/50000 rank6 | Train Loss: 0.9345 | Train Loss Unweighted: 0.4417 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.1121 | Classification: 0.0825 | Classification_binary: 0.1333 | HitGrouping: 0.1549 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0017 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.1122 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0607 | Regression_eta: 0.0242 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0279 | LR: 3.28e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.2170s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0447 | max_forward: 0.7619 | max_loss: 0.4589 | max_backward: 1.9181 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.4922 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6887.2 | uss_MB: 5750.0 | child_rss_MB: 6329.3 | allocated_MB: 497.6 | reserved_MB: 24578.0 | inactive_split_MB: 835.2 +rank6 [2026-10-29 17:31:31,554] INFO : Step 33300/50000 rank6 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1270 | Classification: 0.0780 | Classification_binary: 0.1518 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0985 | Regression_eta: 0.0012 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0574 | Regression_eta: 0.0461 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0502 | LR: 3.26e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.6820s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0334 | max_forward: 0.7851 | max_loss: 0.1434 | max_backward: 2.0284 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.7975 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5654.9 | uss_MB: 4170.1 | child_rss_MB: 6324.3 | allocated_MB: 496.9 | reserved_MB: 24578.0 | inactive_split_MB: 903.1 +rank6 [2026-10-29 17:36:22,674] INFO : Step 33400/50000 rank6 | Train Loss: 0.9537 | Train Loss Unweighted: 0.4634 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1239 | Classification: 0.0872 | Classification_binary: 0.1563 | HitGrouping: 0.1500 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0013 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1998 | Regression_cos_phi: 0.0930 | Regression_energy: 0.0523 | Regression_eta: 0.0378 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0917 | LR: 3.23e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2726s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7951 | max_loss: 0.2522 | max_backward: 1.9349 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0134 | Memory: rss_MB: 5664.7 | uss_MB: 4533.0 | child_rss_MB: 6325.6 | allocated_MB: 500.7 | reserved_MB: 26578.0 | inactive_split_MB: 843.2 +rank6 [2026-10-29 17:41:05,305] INFO : Step 33500/50000 rank6 | Train Loss: 0.9039 | Train Loss Unweighted: 0.4494 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.1057 | Classification: 0.0795 | Classification_binary: 0.1278 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0011 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1273 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0610 | Regression_eta: 0.0224 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0357 | LR: 3.21e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.1849s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0177 | max_forward: 0.7696 | max_loss: 0.1421 | max_backward: 1.9122 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 0.5562 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5654.7 | uss_MB: 4519.8 | child_rss_MB: 6324.9 | allocated_MB: 494.9 | reserved_MB: 26578.0 | inactive_split_MB: 839.3 +rank6 [2026-10-29 17:45:56,711] INFO : Step 33600/50000 rank6 | Train Loss: 0.9179 | Train Loss Unweighted: 0.4415 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.0997 | Classification: 0.0812 | Classification_binary: 0.1440 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0549 | Classification_binary: 0.1120 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0519 | Regression_eta: 0.0288 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0655 | LR: 3.19e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4035s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7783 | max_loss: 0.2418 | max_backward: 2.0392 | max_optimizer_step: 0.2787 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5667.7 | uss_MB: 4535.9 | child_rss_MB: 6324.1 | allocated_MB: 497.9 | reserved_MB: 28594.0 | inactive_split_MB: 1127.0 +rank6 [2026-10-29 17:50:43,453] INFO : Step 33700/50000 rank6 | Train Loss: 1.0564 | Train Loss Unweighted: 0.4962 | Train Loss Components: Auxiliary: 0.2235 | Cardinality: 0.1517 | Classification: 0.0917 | Classification_binary: 0.1786 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.2075 | Regression_cos_phi: 0.0863 | Regression_energy: 0.0517 | Regression_eta: 0.0545 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0900 | LR: 3.16e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.1454s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0121 | max_forward: 0.7981 | max_loss: 0.1554 | max_backward: 1.9054 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5667.6 | uss_MB: 4535.1 | child_rss_MB: 6300.3 | allocated_MB: 501.8 | reserved_MB: 28594.0 | inactive_split_MB: 1103.2 +rank6 [2026-10-29 17:55:24,637] INFO : Step 33800/50000 rank6 | Train Loss: 0.9305 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1894 | Cardinality: 0.1080 | Classification: 0.0751 | Classification_binary: 0.1369 | HitGrouping: 0.1527 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1201 | Regression_eta: 0.0014 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1555 | Regression_cos_phi: 0.0538 | Regression_energy: 0.0657 | Regression_eta: 0.0364 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0570 | LR: 3.14e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1049s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8033 | max_loss: 0.1441 | max_backward: 1.7197 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0176 | Memory: rss_MB: 5662.9 | uss_MB: 4531.8 | child_rss_MB: 6320.7 | allocated_MB: 496.9 | reserved_MB: 28594.0 | inactive_split_MB: 909.5 +rank6 [2026-10-29 18:00:01,587] INFO : Step 33900/50000 rank6 | Train Loss: 0.9360 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.1942 | Cardinality: 0.1035 | Classification: 0.0805 | Classification_binary: 0.1361 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1124 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1160 | Regression_cos_phi: 0.0682 | Regression_energy: 0.0753 | Regression_eta: 0.0350 | Regression_pt: 0.0798 | Regression_sin_phi: 0.0532 | LR: 3.11e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2985s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7557 | max_loss: 0.2439 | max_backward: 1.8923 | max_optimizer_step: 0.2623 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5665.6 | uss_MB: 4533.9 | child_rss_MB: 6306.4 | allocated_MB: 496.5 | reserved_MB: 28598.0 | inactive_split_MB: 941.5 +rank6 [2026-10-29 18:04:39,053] INFO : Step 34000/50000 rank6 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0930 | Classification: 0.0875 | Classification_binary: 0.1606 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0673 | Regression_energy: 0.0524 | Regression_eta: 0.0446 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0622 | LR: 3.09e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.3484s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0943 | max_forward: 0.8289 | max_loss: 0.4616 | max_backward: 2.0219 | max_optimizer_step: 0.1767 | max_tensorboard_logging: 0.4773 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5666.0 | uss_MB: 4534.2 | child_rss_MB: 6326.4 | allocated_MB: 496.0 | reserved_MB: 28598.0 | inactive_split_MB: 1123.0 +rank6 [2026-10-29 18:04:39,969] INFO : Running validation on rank6 +rank6 [2026-10-29 18:04:39,969] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 18:04:39,969] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 18:05:30,627] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:05:31,101] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 18:05:55,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 18:05:56,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 18:05:57,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 18:05:58,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 18:06:08,387] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:06:08,708] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:06:09,217] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 18:06:25,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 18:06:26,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 18:06:27,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 18:06:28,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 18:06:38,157] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:06:39,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:06:39,487] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 18:06:54,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 18:06:55,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 18:06:56,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 18:06:57,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 18:07:07,013] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 18:12:12,108] INFO : Step 34100/50000 rank6 | Train Loss: 1.0240 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2153 | Cardinality: 0.1202 | Classification: 0.0979 | Classification_binary: 0.1668 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1210 | Regression_eta: 0.0013 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.1595 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0620 | Regression_eta: 0.0289 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0569 | LR: 3.06e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5018s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0078 | max_forward: 0.9085 | max_loss: 0.1531 | max_backward: 2.1727 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.4680 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6900.4 | uss_MB: 5761.6 | child_rss_MB: 6326.4 | allocated_MB: 496.4 | reserved_MB: 24560.0 | inactive_split_MB: 1172.2 +rank6 [2026-10-29 18:17:11,666] INFO : Step 34200/50000 rank6 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4225 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.1030 | Classification: 0.0694 | Classification_binary: 0.1279 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0016 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1147 | Regression_cos_phi: 0.0569 | Regression_energy: 0.0629 | Regression_eta: 0.0481 | Regression_pt: 0.0720 | Regression_sin_phi: 0.0394 | LR: 3.04e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.8726s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 1.1401 | max_loss: 1.0887 | max_backward: 3.3565 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5402 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6897.3 | uss_MB: 5759.3 | child_rss_MB: 6328.4 | allocated_MB: 493.1 | reserved_MB: 27668.0 | inactive_split_MB: 842.0 +rank6 [2026-10-29 18:21:53,343] INFO : Step 34300/50000 rank6 | Train Loss: 0.8999 | Train Loss Unweighted: 0.4291 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.1046 | Classification: 0.0782 | Classification_binary: 0.1421 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0015 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0352 | Classification_binary: 0.1313 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0519 | Regression_eta: 0.0354 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0386 | LR: 3.02e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5612s | Phase Times: max_pre_step_barrier: 0.1089 | max_batch_to_device: 0.0070 | max_forward: 0.7738 | max_loss: 0.4754 | max_backward: 1.8888 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5721.4 | uss_MB: 4584.0 | child_rss_MB: 6315.8 | allocated_MB: 498.9 | reserved_MB: 29668.0 | inactive_split_MB: 777.1 +rank6 [2026-10-29 18:26:42,331] INFO : Step 34400/50000 rank6 | Train Loss: 0.9382 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2026 | Cardinality: 0.1181 | Classification: 0.0846 | Classification_binary: 0.1681 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0513 | Classification_binary: 0.1505 | Regression_cos_phi: 0.0504 | Regression_energy: 0.0538 | Regression_eta: 0.0365 | Regression_pt: 0.0603 | Regression_sin_phi: 0.0611 | LR: 2.99e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.6565s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.8411 | max_loss: 0.2739 | max_backward: 2.1385 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5734.9 | uss_MB: 4598.3 | child_rss_MB: 6315.2 | allocated_MB: 496.8 | reserved_MB: 29668.0 | inactive_split_MB: 858.8 +rank6 [2026-10-29 18:31:40,822] INFO : Step 34500/50000 rank6 | Train Loss: 0.9189 | Train Loss Unweighted: 0.4456 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1063 | Classification: 0.0767 | Classification_binary: 0.1421 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0442 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0392 | Regression_energy: 0.0585 | Regression_eta: 0.0323 | Regression_pt: 0.0637 | Regression_sin_phi: 0.0565 | LR: 2.97e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2131s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.7430 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5727.7 | uss_MB: 4595.8 | child_rss_MB: 6313.8 | allocated_MB: 494.3 | reserved_MB: 29668.0 | inactive_split_MB: 926.0 +rank6 [2026-10-29 18:36:27,181] INFO : Step 34600/50000 rank6 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4340 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1046 | Classification: 0.0794 | Classification_binary: 0.1340 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0016 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0471 | Classification_binary: 0.1428 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0674 | Regression_eta: 0.0551 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0585 | LR: 2.95e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.6946s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0170 | max_forward: 0.9895 | max_loss: 0.4622 | max_backward: 2.2541 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5731.7 | uss_MB: 4597.0 | child_rss_MB: 6311.8 | allocated_MB: 498.6 | reserved_MB: 29672.0 | inactive_split_MB: 765.1 +rank6 [2026-10-29 18:40:58,825] INFO : Step 34700/50000 rank6 | Train Loss: 0.9586 | Train Loss Unweighted: 0.4539 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1185 | Classification: 0.0862 | Classification_binary: 0.1624 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0534 | Regression_eta: 0.0668 | Regression_pt: 0.0582 | Regression_sin_phi: 0.0541 | LR: 2.92e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3662s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7862 | max_loss: 0.2628 | max_backward: 1.9231 | max_optimizer_step: 0.2860 | max_tensorboard_logging: 0.4599 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5730.7 | uss_MB: 4596.4 | child_rss_MB: 6303.0 | allocated_MB: 499.0 | reserved_MB: 29672.0 | inactive_split_MB: 1074.3 +rank6 [2026-10-29 18:45:38,653] INFO : Step 34800/50000 rank6 | Train Loss: 0.9346 | Train Loss Unweighted: 0.4373 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1276 | Classification: 0.0867 | Classification_binary: 0.1551 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0959 | Regression_eta: 0.0013 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1649 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0447 | Regression_eta: 0.0372 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0710 | LR: 2.90e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.2172s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0770 | max_forward: 0.8404 | max_loss: 0.1556 | max_backward: 1.8035 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.5351 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5727.0 | uss_MB: 4594.2 | child_rss_MB: 6312.4 | allocated_MB: 497.4 | reserved_MB: 29672.0 | inactive_split_MB: 1113.6 +rank6 [2026-10-29 18:50:17,935] INFO : Step 34900/50000 rank6 | Train Loss: 0.8719 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1154 | Classification: 0.0681 | Classification_binary: 0.1220 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1093 | Regression_eta: 0.0013 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0239 | Classification_binary: 0.1202 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0722 | Regression_eta: 0.0287 | Regression_pt: 0.0807 | Regression_sin_phi: 0.0476 | LR: 2.88e-04 | DataLoad Time: 0.0194s | Model Forward Time: 6.1446s | Phase Times: max_pre_step_barrier: 0.0080 | max_batch_to_device: 0.0693 | max_forward: 1.2127 | max_loss: 0.2834 | max_backward: 3.0996 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.6125 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 5724.6 | uss_MB: 4576.6 | child_rss_MB: 6292.2 | allocated_MB: 495.3 | reserved_MB: 29672.0 | inactive_split_MB: 864.7 +rank6 [2026-10-29 18:55:19,194] INFO : Step 35000/50000 rank6 | Train Loss: 0.9157 | Train Loss Unweighted: 0.4220 | Train Loss Components: Auxiliary: 0.1897 | Cardinality: 0.1230 | Classification: 0.0771 | Classification_binary: 0.1455 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0980 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0360 | Classification_binary: 0.0996 | Regression_cos_phi: 0.0296 | Regression_energy: 0.0594 | Regression_eta: 0.0217 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0521 | LR: 2.85e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5129s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.8243 | max_loss: 0.4759 | max_backward: 2.1181 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5728.2 | uss_MB: 4598.3 | child_rss_MB: 6307.6 | allocated_MB: 498.3 | reserved_MB: 29672.0 | inactive_split_MB: 766.0 +rank6 [2026-10-29 18:55:20,127] INFO : Running validation on rank6 +rank6 [2026-10-29 18:55:20,127] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 18:55:20,128] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 18:56:11,605] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:56:12,115] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 18:56:36,688] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 18:56:37,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 18:56:38,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 18:56:39,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 18:56:49,863] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 18:56:49,876] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:56:50,325] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 18:57:07,019] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 18:57:08,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 18:57:09,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 18:57:10,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 18:57:20,165] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 18:57:20,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 18:57:20,899] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 18:57:36,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 18:57:37,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 18:57:37,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 18:57:38,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 18:57:48,637] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 19:02:56,669] INFO : Step 35100/50000 rank6 | Train Loss: 0.9622 | Train Loss Unweighted: 0.4512 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1150 | Classification: 0.0857 | Classification_binary: 0.1724 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0948 | Regression_eta: 0.0014 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1660 | Regression_cos_phi: 0.1064 | Regression_energy: 0.0574 | Regression_eta: 0.0396 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0695 | LR: 2.83e-04 | DataLoad Time: 0.0138s | Model Forward Time: 4.8779s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1018 | max_forward: 0.8921 | max_loss: 0.1518 | max_backward: 2.0328 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8633 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5815.2 | uss_MB: 4686.8 | child_rss_MB: 6305.0 | allocated_MB: 499.4 | reserved_MB: 23366.0 | inactive_split_MB: 1243.8 +rank6 [2026-10-29 19:07:43,278] INFO : Step 35200/50000 rank6 | Train Loss: 0.9331 | Train Loss Unweighted: 0.4385 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.1234 | Classification: 0.0814 | Classification_binary: 0.1441 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1528 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0477 | Regression_eta: 0.0381 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0420 | LR: 2.81e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1480s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0577 | max_forward: 0.7693 | max_loss: 0.1423 | max_backward: 1.7209 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5822.3 | uss_MB: 4693.9 | child_rss_MB: 6304.7 | allocated_MB: 494.4 | reserved_MB: 25346.0 | inactive_split_MB: 1054.2 +rank6 [2026-10-29 19:12:30,667] INFO : Step 35300/50000 rank6 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4075 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1105 | Classification: 0.0819 | Classification_binary: 0.1245 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0989 | Regression_eta: 0.0014 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1117 | Regression_cos_phi: 0.0805 | Regression_energy: 0.0612 | Regression_eta: 0.0329 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0492 | LR: 2.79e-04 | DataLoad Time: 0.0174s | Model Forward Time: 6.8650s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 1.2856 | max_loss: 0.4079 | max_backward: 4.4969 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5749 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5814.5 | uss_MB: 4685.2 | child_rss_MB: 6307.8 | allocated_MB: 498.3 | reserved_MB: 25346.0 | inactive_split_MB: 1140.5 +rank6 [2026-10-29 19:17:14,809] INFO : Step 35400/50000 rank6 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4888 | Train Loss Components: Auxiliary: 0.2130 | Cardinality: 0.1296 | Classification: 0.0908 | Classification_binary: 0.1718 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0014 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1539 | Regression_cos_phi: 0.0662 | Regression_energy: 0.0507 | Regression_eta: 0.0344 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0453 | LR: 2.76e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3442s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8312 | max_loss: 0.1490 | max_backward: 2.0287 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.4708 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5820.8 | uss_MB: 4696.7 | child_rss_MB: 6308.5 | allocated_MB: 497.8 | reserved_MB: 25346.0 | inactive_split_MB: 1132.0 +rank6 [2026-10-29 19:21:58,249] INFO : Step 35500/50000 rank6 | Train Loss: 0.9444 | Train Loss Unweighted: 0.4557 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1201 | Classification: 0.0924 | Classification_binary: 0.1605 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1824 | Regression_cos_phi: 0.0729 | Regression_energy: 0.0575 | Regression_eta: 0.0412 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0647 | LR: 2.74e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.3603s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7670 | max_loss: 0.2318 | max_backward: 1.8405 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5820.9 | uss_MB: 4694.9 | child_rss_MB: 6312.3 | allocated_MB: 497.5 | reserved_MB: 26942.0 | inactive_split_MB: 1130.5 +rank6 [2026-10-29 19:26:41,939] INFO : Step 35600/50000 rank6 | Train Loss: 0.9280 | Train Loss Unweighted: 0.4451 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1095 | Classification: 0.0861 | Classification_binary: 0.1334 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0437 | Classification_binary: 0.1175 | Regression_cos_phi: 0.0332 | Regression_energy: 0.0573 | Regression_eta: 0.0281 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0614 | LR: 2.72e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.7395s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7460 | max_loss: 0.1440 | max_backward: 1.8206 | max_optimizer_step: 0.2911 | max_tensorboard_logging: 0.6036 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 5820.5 | uss_MB: 3919.7 | child_rss_MB: 6316.0 | allocated_MB: 496.1 | reserved_MB: 26942.0 | inactive_split_MB: 907.6 +rank6 [2026-10-29 19:31:29,665] INFO : Step 35700/50000 rank6 | Train Loss: 0.9116 | Train Loss Unweighted: 0.4301 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1148 | Classification: 0.0829 | Classification_binary: 0.1432 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0998 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1233 | Regression_cos_phi: 0.0463 | Regression_energy: 0.0509 | Regression_eta: 0.0320 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0639 | LR: 2.70e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6323s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8312 | max_loss: 0.2568 | max_backward: 2.2622 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5823.2 | uss_MB: 4699.5 | child_rss_MB: 6323.6 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1241.0 +rank6 [2026-10-29 19:36:11,896] INFO : Step 35800/50000 rank6 | Train Loss: 1.0095 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1252 | Classification: 0.0857 | Classification_binary: 0.1786 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1066 | Regression_eta: 0.0014 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1569 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0530 | Regression_eta: 0.0569 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0599 | LR: 2.68e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1745s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7644 | max_loss: 0.5090 | max_backward: 1.8855 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4861 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5816.8 | uss_MB: 4694.8 | child_rss_MB: 6319.7 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1129.0 +rank6 [2026-10-29 19:40:48,761] INFO : Step 35900/50000 rank6 | Train Loss: 0.9596 | Train Loss Unweighted: 0.4571 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1173 | Classification: 0.0913 | Classification_binary: 0.1468 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0014 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0605 | Classification_binary: 0.1449 | Regression_cos_phi: 0.0661 | Regression_energy: 0.0883 | Regression_eta: 0.0338 | Regression_pt: 0.0957 | Regression_sin_phi: 0.0701 | LR: 2.65e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.0316s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7217 | max_loss: 0.1443 | max_backward: 1.6894 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.4586 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5816.9 | uss_MB: 4695.3 | child_rss_MB: 6319.9 | allocated_MB: 495.4 | reserved_MB: 29024.0 | inactive_split_MB: 934.6 +rank6 [2026-10-29 19:45:19,092] INFO : Step 36000/50000 rank6 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0912 | Classification: 0.0938 | Classification_binary: 0.1314 | HitGrouping: 0.1587 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1135 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1587 | Regression_cos_phi: 0.0772 | Regression_energy: 0.0567 | Regression_eta: 0.0486 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0710 | LR: 2.63e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3006s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0073 | max_forward: 0.7630 | max_loss: 0.4769 | max_backward: 1.8869 | max_optimizer_step: 0.2587 | max_tensorboard_logging: 0.5842 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5815.2 | uss_MB: 4695.9 | child_rss_MB: 6324.3 | allocated_MB: 495.0 | reserved_MB: 29024.0 | inactive_split_MB: 937.5 +rank6 [2026-10-29 19:45:19,982] INFO : Running validation on rank6 +rank6 [2026-10-29 19:45:19,982] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 19:45:19,982] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 19:46:10,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:46:11,093] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 19:46:35,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 19:46:36,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 19:46:37,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 19:46:38,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 19:46:48,969] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 19:46:49,454] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:46:49,932] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 19:47:05,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 19:47:06,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 19:47:07,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 19:47:08,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 19:47:18,897] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 19:47:19,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 19:47:19,886] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 19:47:35,502] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 19:47:36,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 19:47:37,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 19:47:37,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 19:47:47,945] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 19:53:12,007] INFO : Step 36100/50000 rank6 | Train Loss: 0.9292 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1100 | Classification: 0.0906 | Classification_binary: 0.1490 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0518 | Classification_binary: 0.1554 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0622 | Regression_eta: 0.0459 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0636 | LR: 2.61e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.7842s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7768 | max_loss: 0.2455 | max_backward: 1.8077 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 1.0653 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5880.7 | uss_MB: 4756.1 | child_rss_MB: 6293.3 | allocated_MB: 499.2 | reserved_MB: 26012.0 | inactive_split_MB: 1080.2 +rank6 [2026-10-29 19:58:21,470] INFO : Step 36200/50000 rank6 | Train Loss: 0.9030 | Train Loss Unweighted: 0.4308 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1163 | Classification: 0.0762 | Classification_binary: 0.1596 | HitGrouping: 0.1506 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1679 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0435 | Regression_eta: 0.0295 | Regression_pt: 0.0482 | Regression_sin_phi: 0.0455 | LR: 2.59e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.5381s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0777 | max_forward: 0.8125 | max_loss: 0.1541 | max_backward: 1.8492 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 5884.8 | uss_MB: 4767.2 | child_rss_MB: 6290.6 | allocated_MB: 494.4 | reserved_MB: 26016.0 | inactive_split_MB: 1377.4 +rank6 [2026-10-29 20:03:20,658] INFO : Step 36300/50000 rank6 | Train Loss: 0.9576 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1236 | Classification: 0.0876 | Classification_binary: 0.1347 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1100 | Regression_eta: 0.0017 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.1324 | Regression_cos_phi: 0.0831 | Regression_energy: 0.0549 | Regression_eta: 0.0410 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0816 | LR: 2.57e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.4541s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.8065 | max_loss: 0.1484 | max_backward: 1.9315 | max_optimizer_step: 0.1930 | max_tensorboard_logging: 0.6801 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5878.7 | uss_MB: 4768.0 | child_rss_MB: 6273.7 | allocated_MB: 496.0 | reserved_MB: 28008.0 | inactive_split_MB: 1156.5 +rank6 [2026-10-29 20:08:10,865] INFO : Step 36400/50000 rank6 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4147 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.0999 | Classification: 0.0796 | Classification_binary: 0.1372 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1310 | Regression_cos_phi: 0.0370 | Regression_energy: 0.0473 | Regression_eta: 0.0301 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0433 | LR: 2.55e-04 | DataLoad Time: 0.0223s | Model Forward Time: 5.8685s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0192 | max_forward: 1.1829 | max_loss: 0.3790 | max_backward: 3.0917 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 5880.4 | uss_MB: 4768.9 | child_rss_MB: 6270.8 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 845.2 +rank6 [2026-10-29 20:12:41,929] INFO : Step 36500/50000 rank6 | Train Loss: 0.9762 | Train Loss Unweighted: 0.4543 | Train Loss Components: Auxiliary: 0.2088 | Cardinality: 0.1239 | Classification: 0.0805 | Classification_binary: 0.1682 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1772 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0490 | Regression_eta: 0.0427 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0588 | LR: 2.52e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2742s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7782 | max_loss: 0.1554 | max_backward: 1.9378 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.5103 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5877.6 | uss_MB: 4769.9 | child_rss_MB: 6273.0 | allocated_MB: 499.0 | reserved_MB: 30008.0 | inactive_split_MB: 913.8 +rank6 [2026-10-29 20:17:47,229] INFO : Step 36600/50000 rank6 | Train Loss: 0.8699 | Train Loss Unweighted: 0.4190 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.1138 | Classification: 0.0734 | Classification_binary: 0.1317 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0010 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0526 | Regression_energy: 0.0614 | Regression_eta: 0.0328 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0358 | LR: 2.50e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.2977s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7561 | max_loss: 0.1454 | max_backward: 1.9017 | max_optimizer_step: 0.5115 | max_tensorboard_logging: 0.5672 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5872.6 | uss_MB: 4766.8 | child_rss_MB: 6277.6 | allocated_MB: 494.4 | reserved_MB: 30008.0 | inactive_split_MB: 856.0 +rank6 [2026-10-29 20:22:56,452] INFO : Step 36700/50000 rank6 | Train Loss: 0.9065 | Train Loss Unweighted: 0.4327 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.1102 | Classification: 0.0849 | Classification_binary: 0.1262 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1087 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1280 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0697 | Regression_eta: 0.0368 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0765 | LR: 2.48e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.0799s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.7169 | max_loss: 0.1507 | max_backward: 1.7104 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5871.4 | uss_MB: 4766.4 | child_rss_MB: 6270.7 | allocated_MB: 496.5 | reserved_MB: 30008.0 | inactive_split_MB: 1154.6 +rank6 [2026-10-29 20:28:04,048] INFO : Step 36800/50000 rank6 | Train Loss: 0.9068 | Train Loss Unweighted: 0.4227 | Train Loss Components: Auxiliary: 0.1925 | Cardinality: 0.1100 | Classification: 0.0739 | Classification_binary: 0.1571 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0945 | Regression_eta: 0.0013 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1472 | Regression_cos_phi: 0.0465 | Regression_energy: 0.0477 | Regression_eta: 0.0480 | Regression_pt: 0.0514 | Regression_sin_phi: 0.0398 | LR: 2.46e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4655s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8487 | max_loss: 0.1525 | max_backward: 2.1121 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5877.1 | uss_MB: 4770.1 | child_rss_MB: 6274.6 | allocated_MB: 499.3 | reserved_MB: 30008.0 | inactive_split_MB: 939.6 +rank6 [2026-10-29 20:33:09,562] INFO : Step 36900/50000 rank6 | Train Loss: 0.8934 | Train Loss Unweighted: 0.4126 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1194 | Classification: 0.0721 | Classification_binary: 0.1507 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0013 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0311 | Classification_binary: 0.1429 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0524 | Regression_eta: 0.0317 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0413 | LR: 2.44e-04 | DataLoad Time: 0.0143s | Model Forward Time: 5.5694s | Phase Times: max_pre_step_barrier: 0.0139 | max_batch_to_device: 0.0068 | max_forward: 1.0975 | max_loss: 0.2241 | max_backward: 2.7278 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.6216 | max_tensorboard_flush: 0.0085 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5871.7 | uss_MB: 4750.7 | child_rss_MB: 6260.8 | allocated_MB: 498.9 | reserved_MB: 30008.0 | inactive_split_MB: 771.1 +rank6 [2026-10-29 20:38:05,960] INFO : Step 37000/50000 rank6 | Train Loss: 0.9040 | Train Loss Unweighted: 0.4182 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1192 | Classification: 0.0807 | Classification_binary: 0.1318 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1341 | Regression_cos_phi: 0.0562 | Regression_energy: 0.0486 | Regression_eta: 0.0348 | Regression_pt: 0.0529 | Regression_sin_phi: 0.0339 | LR: 2.42e-04 | DataLoad Time: 0.0174s | Model Forward Time: 3.8497s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7055 | max_loss: 0.1505 | max_backward: 1.5664 | max_optimizer_step: 0.1900 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5866.9 | uss_MB: 4763.2 | child_rss_MB: 6273.0 | allocated_MB: 493.9 | reserved_MB: 30008.0 | inactive_split_MB: 1169.8 +rank6 [2026-10-29 20:38:06,932] INFO : Running validation on rank6 +rank6 [2026-10-29 20:38:06,932] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 20:38:06,932] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 20:39:05,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:39:06,530] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 20:39:31,485] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 20:39:32,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 20:39:33,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 20:39:34,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 20:39:44,661] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 20:39:45,043] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:39:45,533] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 20:40:01,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 20:40:02,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 20:40:03,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 20:40:04,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 20:40:14,983] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 20:40:16,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 20:40:16,608] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 20:40:32,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 20:40:33,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 20:40:34,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 20:40:35,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 20:40:45,310] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 20:45:59,178] INFO : Step 37100/50000 rank6 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4614 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1255 | Classification: 0.0999 | Classification_binary: 0.1492 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1041 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0998 | Classification_binary: 0.1363 | Regression_cos_phi: 0.1190 | Regression_energy: 0.0578 | Regression_eta: 0.0613 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0790 | LR: 2.40e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.8475s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0061 | max_forward: 0.8553 | max_loss: 0.1461 | max_backward: 2.1723 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5959.4 | uss_MB: 4852.0 | child_rss_MB: 6311.9 | allocated_MB: 499.3 | reserved_MB: 25668.0 | inactive_split_MB: 852.4 +rank6 [2026-10-29 20:50:39,579] INFO : Step 37200/50000 rank6 | Train Loss: 0.9288 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.1928 | Cardinality: 0.1108 | Classification: 0.0836 | Classification_binary: 0.1643 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0015 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1770 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0536 | Regression_eta: 0.0401 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0593 | LR: 2.38e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6402s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8460 | max_loss: 0.2603 | max_backward: 2.0959 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5966.0 | uss_MB: 4861.5 | child_rss_MB: 6312.1 | allocated_MB: 497.9 | reserved_MB: 25674.0 | inactive_split_MB: 846.0 +rank6 [2026-10-29 20:55:11,587] INFO : Step 37300/50000 rank6 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1275 | Classification: 0.0781 | Classification_binary: 0.1424 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0982 | Regression_eta: 0.0013 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1497 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0447 | Regression_eta: 0.0392 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0655 | LR: 2.36e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.0198s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7653 | max_loss: 0.1472 | max_backward: 1.7105 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.4778 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0692 | Memory: rss_MB: 5962.2 | uss_MB: 4859.4 | child_rss_MB: 6302.2 | allocated_MB: 497.3 | reserved_MB: 27616.0 | inactive_split_MB: 974.4 +rank6 [2026-10-29 21:00:09,948] INFO : Step 37400/50000 rank6 | Train Loss: 0.8836 | Train Loss Unweighted: 0.4228 | Train Loss Components: Auxiliary: 0.1825 | Cardinality: 0.0983 | Classification: 0.0851 | Classification_binary: 0.1289 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1025 | Regression_eta: 0.0015 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.0927 | Regression_cos_phi: 0.0249 | Regression_energy: 0.0436 | Regression_eta: 0.0265 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0617 | LR: 2.34e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5393s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7611 | max_loss: 0.1956 | max_backward: 1.9276 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7505 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5960.0 | uss_MB: 4860.1 | child_rss_MB: 6301.8 | allocated_MB: 495.0 | reserved_MB: 27690.0 | inactive_split_MB: 915.5 +rank6 [2026-10-29 21:05:01,764] INFO : Step 37500/50000 rank6 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1260 | Classification: 0.0804 | Classification_binary: 0.1577 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1005 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1399 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0503 | Regression_eta: 0.0351 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0564 | LR: 2.32e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4453s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.9030 | max_loss: 0.1494 | max_backward: 1.9871 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5938.4 | uss_MB: 4859.4 | child_rss_MB: 6224.4 | allocated_MB: 496.1 | reserved_MB: 27690.0 | inactive_split_MB: 923.5 +rank6 [2026-10-29 21:09:54,198] INFO : Step 37600/50000 rank6 | Train Loss: 0.9091 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.0995 | Classification: 0.0683 | Classification_binary: 0.1534 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0957 | Regression_eta: 0.0015 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0324 | Classification_binary: 0.1652 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0505 | Regression_eta: 0.0309 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0673 | LR: 2.30e-04 | DataLoad Time: 0.0141s | Model Forward Time: 4.3516s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0070 | max_forward: 0.8280 | max_loss: 0.1484 | max_backward: 1.8328 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5933.4 | uss_MB: 4856.7 | child_rss_MB: 6230.2 | allocated_MB: 497.1 | reserved_MB: 27690.0 | inactive_split_MB: 927.5 +rank6 [2026-10-29 21:14:45,920] INFO : Step 37700/50000 rank6 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3995 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.0864 | Classification: 0.0813 | Classification_binary: 0.1231 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0012 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1310 | Regression_cos_phi: 0.0448 | Regression_energy: 0.0628 | Regression_eta: 0.0404 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0727 | LR: 2.28e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4212s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7582 | max_loss: 0.2486 | max_backward: 1.9197 | max_optimizer_step: 0.3005 | max_tensorboard_logging: 0.5767 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5802.8 | uss_MB: 4857.6 | child_rss_MB: 5767.0 | allocated_MB: 495.2 | reserved_MB: 29766.0 | inactive_split_MB: 925.5 +rank6 [2026-10-29 21:19:50,095] INFO : Step 37800/50000 rank6 | Train Loss: 0.8717 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1057 | Classification: 0.0822 | Classification_binary: 0.1363 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0014 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1194 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0512 | Regression_eta: 0.0273 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0291 | LR: 2.26e-04 | DataLoad Time: 0.0187s | Model Forward Time: 4.6255s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 1.2105 | max_loss: 0.2449 | max_backward: 2.0574 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 5800.5 | uss_MB: 4856.8 | child_rss_MB: 5763.9 | allocated_MB: 498.3 | reserved_MB: 29766.0 | inactive_split_MB: 930.0 +rank6 [2026-10-29 21:24:49,255] INFO : Step 37900/50000 rank6 | Train Loss: 0.9843 | Train Loss Unweighted: 0.4684 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1353 | Classification: 0.0912 | Classification_binary: 0.1705 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0014 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1787 | Regression_cos_phi: 0.0676 | Regression_energy: 0.0571 | Regression_eta: 0.0422 | Regression_pt: 0.0612 | Regression_sin_phi: 0.0619 | LR: 2.24e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.2098s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0297 | max_forward: 0.8081 | max_loss: 0.1547 | max_backward: 2.6708 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5096 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5782.9 | uss_MB: 4859.7 | child_rss_MB: 5732.7 | allocated_MB: 495.6 | reserved_MB: 29766.0 | inactive_split_MB: 912.0 +rank6 [2026-10-29 21:29:44,506] INFO : Step 38000/50000 rank6 | Train Loss: 0.8556 | Train Loss Unweighted: 0.4002 | Train Loss Components: Auxiliary: 0.1833 | Cardinality: 0.1090 | Classification: 0.0730 | Classification_binary: 0.1293 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0972 | Regression_eta: 0.0011 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0440 | Regression_energy: 0.0553 | Regression_eta: 0.0371 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0402 | LR: 2.22e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.1662s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7330 | max_loss: 0.2284 | max_backward: 1.8082 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5778.8 | uss_MB: 4857.4 | child_rss_MB: 5727.0 | allocated_MB: 494.5 | reserved_MB: 29766.0 | inactive_split_MB: 847.3 +rank6 [2026-10-29 21:29:45,441] INFO : Running validation on rank6 +rank6 [2026-10-29 21:29:45,441] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 21:29:45,441] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 21:30:41,619] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:30:42,096] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 21:31:09,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 21:31:10,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 21:31:11,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 21:31:12,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 21:31:22,765] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 21:31:23,195] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:31:23,629] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 21:31:39,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 21:31:40,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 21:31:41,358] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 21:31:42,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 21:31:52,346] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 21:31:52,734] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 21:31:53,235] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 21:32:08,917] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 21:32:09,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 21:32:10,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 21:32:11,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 21:32:21,353] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 21:37:39,070] INFO : Step 38100/50000 rank6 | Train Loss: 0.9020 | Train Loss Unweighted: 0.4337 | Train Loss Components: Auxiliary: 0.1907 | Cardinality: 0.1231 | Classification: 0.0888 | Classification_binary: 0.1339 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0014 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0535 | Classification_binary: 0.1041 | Regression_cos_phi: 0.0424 | Regression_energy: 0.0628 | Regression_eta: 0.0249 | Regression_pt: 0.0676 | Regression_sin_phi: 0.0294 | LR: 2.20e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8726s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0963 | max_forward: 0.8445 | max_loss: 0.1406 | max_backward: 1.9197 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.9697 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7028.7 | uss_MB: 6098.3 | child_rss_MB: 5734.5 | allocated_MB: 496.0 | reserved_MB: 27516.0 | inactive_split_MB: 970.5 +rank6 [2026-10-29 21:42:37,687] INFO : Step 38200/50000 rank6 | Train Loss: 0.9174 | Train Loss Unweighted: 0.4351 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.0971 | Classification: 0.0888 | Classification_binary: 0.1526 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0948 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0932 | Regression_energy: 0.0592 | Regression_eta: 0.0313 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0581 | LR: 2.18e-04 | DataLoad Time: 0.0170s | Model Forward Time: 6.4471s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.8480 | max_loss: 0.2651 | max_backward: 3.1905 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.9151 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5881.2 | uss_MB: 4342.7 | child_rss_MB: 5733.1 | allocated_MB: 499.7 | reserved_MB: 27516.0 | inactive_split_MB: 848.5 +rank6 [2026-10-29 21:47:30,433] INFO : Step 38300/50000 rank6 | Train Loss: 0.9342 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1998 | Cardinality: 0.1099 | Classification: 0.0869 | Classification_binary: 0.1470 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0013 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1633 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0494 | Regression_eta: 0.0375 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0676 | LR: 2.16e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.6875s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0664 | max_forward: 0.8187 | max_loss: 0.1464 | max_backward: 1.8154 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.9517 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5875.2 | uss_MB: 4949.5 | child_rss_MB: 5725.9 | allocated_MB: 495.5 | reserved_MB: 27516.0 | inactive_split_MB: 1085.5 +rank6 [2026-10-29 21:52:14,429] INFO : Step 38400/50000 rank6 | Train Loss: 0.8281 | Train Loss Unweighted: 0.4036 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0915 | Classification: 0.0723 | Classification_binary: 0.1160 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1069 | Regression_eta: 0.0012 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.1099 | Regression_cos_phi: 0.0452 | Regression_energy: 0.0519 | Regression_eta: 0.0250 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0368 | LR: 2.14e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.7620s | Phase Times: max_pre_step_barrier: 0.0868 | max_batch_to_device: 0.0062 | max_forward: 0.9937 | max_loss: 0.2500 | max_backward: 1.9690 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5872.2 | uss_MB: 4950.8 | child_rss_MB: 5730.9 | allocated_MB: 496.1 | reserved_MB: 27516.0 | inactive_split_MB: 972.6 +rank6 [2026-10-29 21:57:09,318] INFO : Step 38500/50000 rank6 | Train Loss: 0.9386 | Train Loss Unweighted: 0.4484 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1248 | Classification: 0.1044 | Classification_binary: 0.1389 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0830 | Classification_binary: 0.1222 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0540 | Regression_eta: 0.0450 | Regression_pt: 0.0589 | Regression_sin_phi: 0.0839 | LR: 2.12e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.2044s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.7200 | max_loss: 0.1386 | max_backward: 1.5323 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5868.3 | uss_MB: 4952.0 | child_rss_MB: 5707.9 | allocated_MB: 496.0 | reserved_MB: 27520.0 | inactive_split_MB: 776.0 +rank6 [2026-10-29 22:01:43,605] INFO : Step 38600/50000 rank6 | Train Loss: 0.9227 | Train Loss Unweighted: 0.4436 | Train Loss Components: Auxiliary: 0.1984 | Cardinality: 0.1056 | Classification: 0.0788 | Classification_binary: 0.1660 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0979 | Regression_eta: 0.0013 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0458 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0545 | Regression_energy: 0.0685 | Regression_eta: 0.0301 | Regression_pt: 0.0735 | Regression_sin_phi: 0.0523 | LR: 2.11e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.4284s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8785 | max_loss: 0.1473 | max_backward: 2.0276 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5861.9 | uss_MB: 4951.4 | child_rss_MB: 5677.1 | allocated_MB: 499.0 | reserved_MB: 29592.0 | inactive_split_MB: 827.5 +rank6 [2026-10-29 22:06:19,438] INFO : Step 38700/50000 rank6 | Train Loss: 0.8153 | Train Loss Unweighted: 0.3795 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0947 | Classification: 0.0722 | Classification_binary: 0.1256 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0893 | Regression_eta: 0.0013 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0494 | Classification_binary: 0.1249 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0397 | Regression_eta: 0.0360 | Regression_pt: 0.0438 | Regression_sin_phi: 0.0330 | LR: 2.09e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2101s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7251 | max_loss: 0.2433 | max_backward: 1.8091 | max_optimizer_step: 0.1965 | max_tensorboard_logging: 0.4744 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5858.3 | uss_MB: 4950.6 | child_rss_MB: 5664.7 | allocated_MB: 493.1 | reserved_MB: 29592.0 | inactive_split_MB: 1080.0 +rank6 [2026-10-29 22:11:06,083] INFO : Step 38800/50000 rank6 | Train Loss: 0.8386 | Train Loss Unweighted: 0.4077 | Train Loss Components: Auxiliary: 0.1776 | Cardinality: 0.0905 | Classification: 0.0849 | Classification_binary: 0.1308 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0012 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0414 | Regression_energy: 0.0569 | Regression_eta: 0.0386 | Regression_pt: 0.0640 | Regression_sin_phi: 0.0689 | LR: 2.07e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1592s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.7346 | max_loss: 0.1426 | max_backward: 1.8320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5857.6 | uss_MB: 4949.0 | child_rss_MB: 5646.6 | allocated_MB: 497.9 | reserved_MB: 29592.0 | inactive_split_MB: 1086.4 +rank6 [2026-10-29 22:15:54,683] INFO : Step 38900/50000 rank6 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1916 | Cardinality: 0.1035 | Classification: 0.0839 | Classification_binary: 0.1479 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0014 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1325 | Regression_cos_phi: 0.0794 | Regression_energy: 0.0493 | Regression_eta: 0.0367 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0501 | LR: 2.05e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.8773s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0144 | max_forward: 2.1610 | max_loss: 0.2966 | max_backward: 2.8203 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4876 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 5858.5 | uss_MB: 4951.7 | child_rss_MB: 5647.2 | allocated_MB: 500.0 | reserved_MB: 29592.0 | inactive_split_MB: 849.6 +rank6 [2026-10-29 22:20:36,724] INFO : Step 39000/50000 rank6 | Train Loss: 0.8505 | Train Loss Unweighted: 0.4196 | Train Loss Components: Auxiliary: 0.1813 | Cardinality: 0.0941 | Classification: 0.0808 | Classification_binary: 0.1368 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0993 | Regression_eta: 0.0015 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0399 | Classification_binary: 0.1444 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0584 | Regression_eta: 0.0280 | Regression_pt: 0.0642 | Regression_sin_phi: 0.0392 | LR: 2.03e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.2856s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7392 | max_loss: 0.2445 | max_backward: 1.8584 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5844.6 | uss_MB: 4939.8 | child_rss_MB: 5638.4 | allocated_MB: 496.8 | reserved_MB: 29592.0 | inactive_split_MB: 1086.0 +rank6 [2026-10-29 22:20:37,630] INFO : Running validation on rank6 +rank6 [2026-10-29 22:20:37,631] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 22:20:37,631] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 22:21:31,241] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:21:31,758] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 22:21:59,032] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 22:22:00,084] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 22:22:01,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 22:22:02,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 22:22:12,213] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 22:22:12,679] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:22:13,174] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 22:22:29,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 22:22:30,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 22:22:31,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 22:22:32,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 22:22:42,565] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-29 22:22:43,329] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 22:22:43,800] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 22:22:59,468] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 22:23:00,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 22:23:01,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 22:23:01,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 22:23:11,918] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-29 22:28:30,320] INFO : Step 39100/50000 rank6 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1110 | Classification: 0.0839 | Classification_binary: 0.1198 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0333 | Classification_binary: 0.1291 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0820 | Regression_eta: 0.0301 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0385 | LR: 2.01e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5490s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7800 | max_loss: 0.2457 | max_backward: 1.9719 | max_optimizer_step: 0.3248 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 7010.9 | uss_MB: 6103.0 | child_rss_MB: 5608.2 | allocated_MB: 497.5 | reserved_MB: 24374.0 | inactive_split_MB: 923.6 +rank6 [2026-10-29 22:33:29,807] INFO : Step 39200/50000 rank6 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4242 | Train Loss Components: Auxiliary: 0.1863 | Cardinality: 0.1210 | Classification: 0.0912 | Classification_binary: 0.1382 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0356 | Regression_energy: 0.0564 | Regression_eta: 0.0336 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0647 | LR: 2.00e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2583s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0324 | max_forward: 0.7817 | max_loss: 0.1452 | max_backward: 1.9286 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5940.0 | uss_MB: 5036.4 | child_rss_MB: 5609.8 | allocated_MB: 497.9 | reserved_MB: 26338.0 | inactive_split_MB: 912.4 +rank6 [2026-10-29 22:38:28,992] INFO : Step 39300/50000 rank6 | Train Loss: 0.9298 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1080 | Classification: 0.0882 | Classification_binary: 0.1594 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0951 | Regression_eta: 0.0013 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0450 | Classification_binary: 0.1584 | Regression_cos_phi: 0.0499 | Regression_energy: 0.0381 | Regression_eta: 0.0362 | Regression_pt: 0.0417 | Regression_sin_phi: 0.0585 | LR: 1.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.1932s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7922 | max_loss: 0.2572 | max_backward: 1.8558 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 5927.1 | uss_MB: 5036.1 | child_rss_MB: 5613.3 | allocated_MB: 499.4 | reserved_MB: 26338.0 | inactive_split_MB: 1408.0 +rank6 [2026-10-29 22:43:21,467] INFO : Step 39400/50000 rank6 | Train Loss: 0.8421 | Train Loss Unweighted: 0.3964 | Train Loss Components: Auxiliary: 0.1755 | Cardinality: 0.1006 | Classification: 0.0679 | Classification_binary: 0.1346 | HitGrouping: 0.1534 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1586 | Regression_cos_phi: 0.0543 | Regression_energy: 0.0430 | Regression_eta: 0.0286 | Regression_pt: 0.0475 | Regression_sin_phi: 0.0411 | LR: 1.96e-04 | DataLoad Time: 0.0178s | Model Forward Time: 6.2961s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1084 | max_forward: 1.1740 | max_loss: 0.3046 | max_backward: 2.6778 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8613 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5928.1 | uss_MB: 4351.2 | child_rss_MB: 5609.1 | allocated_MB: 495.0 | reserved_MB: 26338.0 | inactive_split_MB: 1095.6 +rank6 [2026-10-29 22:48:17,291] INFO : Step 39500/50000 rank6 | Train Loss: 0.8539 | Train Loss Unweighted: 0.4179 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0879 | Classification: 0.0889 | Classification_binary: 0.1243 | HitGrouping: 0.1565 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1320 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0604 | Regression_eta: 0.0460 | Regression_pt: 0.0660 | Regression_sin_phi: 0.0685 | LR: 1.94e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.6155s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7373 | max_loss: 0.1406 | max_backward: 1.8719 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.8502 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5922.7 | uss_MB: 5037.5 | child_rss_MB: 5598.5 | allocated_MB: 492.8 | reserved_MB: 28318.0 | inactive_split_MB: 1078.3 +rank6 [2026-10-29 22:53:41,207] INFO : Step 39600/50000 rank6 | Train Loss: 0.9371 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1161 | Classification: 0.0813 | Classification_binary: 0.1596 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1039 | Regression_eta: 0.0013 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1572 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0509 | Regression_eta: 0.0594 | Regression_pt: 0.0556 | Regression_sin_phi: 0.0808 | LR: 1.93e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.3238s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0173 | max_forward: 1.4879 | max_loss: 0.2520 | max_backward: 1.9863 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.8346 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5921.1 | uss_MB: 5038.4 | child_rss_MB: 5602.9 | allocated_MB: 496.0 | reserved_MB: 28318.0 | inactive_split_MB: 858.4 +rank6 [2026-10-29 22:58:48,059] INFO : Step 39700/50000 rank6 | Train Loss: 0.9204 | Train Loss Unweighted: 0.4427 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1101 | Classification: 0.0841 | Classification_binary: 0.1567 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0997 | Regression_eta: 0.0013 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0616 | Classification_binary: 0.1863 | Regression_cos_phi: 0.0595 | Regression_energy: 0.0432 | Regression_eta: 0.0427 | Regression_pt: 0.0478 | Regression_sin_phi: 0.0803 | LR: 1.91e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.2768s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.7571 | max_loss: 0.4946 | max_backward: 1.9147 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 5909.3 | uss_MB: 5037.4 | child_rss_MB: 5602.2 | allocated_MB: 497.0 | reserved_MB: 30324.0 | inactive_split_MB: 1175.5 +rank6 [2026-10-29 23:04:38,178] INFO : Step 39800/50000 rank6 | Train Loss: 0.8155 | Train Loss Unweighted: 0.3780 | Train Loss Components: Auxiliary: 0.1674 | Cardinality: 0.0963 | Classification: 0.0643 | Classification_binary: 0.1275 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1402 | Regression_cos_phi: 0.0392 | Regression_energy: 0.0425 | Regression_eta: 0.0270 | Regression_pt: 0.0440 | Regression_sin_phi: 0.0534 | LR: 1.89e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0224s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7197 | max_loss: 0.1489 | max_backward: 1.6690 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.4760 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1068 | Memory: rss_MB: 5897.7 | uss_MB: 5034.7 | child_rss_MB: 5594.3 | allocated_MB: 494.4 | reserved_MB: 30324.0 | inactive_split_MB: 897.3 +rank6 [2026-10-29 23:09:43,447] INFO : Step 39900/50000 rank6 | Train Loss: 0.8844 | Train Loss Unweighted: 0.3998 | Train Loss Components: Auxiliary: 0.1871 | Cardinality: 0.1077 | Classification: 0.0758 | Classification_binary: 0.1443 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0883 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0284 | Classification_binary: 0.1265 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0527 | Regression_eta: 0.0294 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0297 | LR: 1.88e-04 | DataLoad Time: 0.0118s | Model Forward Time: 4.9675s | Phase Times: max_pre_step_barrier: 0.0268 | max_batch_to_device: 0.0727 | max_forward: 1.7016 | max_loss: 0.4968 | max_backward: 2.0829 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.4776 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5898.5 | uss_MB: 5037.5 | child_rss_MB: 5601.6 | allocated_MB: 496.1 | reserved_MB: 30324.0 | inactive_split_MB: 1077.6 +rank6 [2026-10-29 23:14:38,167] INFO : Step 40000/50000 rank6 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1170 | Classification: 0.0871 | Classification_binary: 0.1550 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0479 | Regression_eta: 0.0305 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0492 | LR: 1.86e-04 | DataLoad Time: 0.0137s | Model Forward Time: 5.4143s | Phase Times: max_pre_step_barrier: 0.1084 | max_batch_to_device: 0.0070 | max_forward: 1.1725 | max_loss: 0.2555 | max_backward: 2.4297 | max_optimizer_step: 0.2969 | max_tensorboard_logging: 0.6818 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5889.3 | uss_MB: 5028.2 | child_rss_MB: 5597.1 | allocated_MB: 501.7 | reserved_MB: 30324.0 | inactive_split_MB: 1368.0 +rank6 [2026-10-29 23:14:39,118] INFO : Running validation on rank6 +rank6 [2026-10-29 23:14:39,119] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-29 23:14:39,119] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-29 23:15:31,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:15:32,369] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-29 23:16:03,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-29 23:16:04,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-29 23:16:06,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-29 23:16:07,446] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-29 23:16:17,454] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-29 23:16:28,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:16:28,835] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-29 23:16:44,990] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-29 23:16:45,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-29 23:16:46,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-29 23:16:47,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-29 23:16:57,962] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-29 23:16:58,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-29 23:16:59,025] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-29 23:17:14,560] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-29 23:17:15,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-29 23:17:16,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-29 23:17:17,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-29 23:17:27,153] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-29 23:22:41,683] INFO : Step 40100/50000 rank6 | Train Loss: 0.8115 | Train Loss Unweighted: 0.3794 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.1028 | Classification: 0.0708 | Classification_binary: 0.1239 | HitGrouping: 0.1460 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0909 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0555 | Regression_energy: 0.0541 | Regression_eta: 0.0410 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0552 | LR: 1.84e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.0776s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.1076 | max_forward: 0.8426 | max_loss: 0.1437 | max_backward: 1.8847 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 7273.7 | uss_MB: 5477.4 | child_rss_MB: 5617.3 | allocated_MB: 493.6 | reserved_MB: 28004.0 | inactive_split_MB: 975.8 +rank6 [2026-10-29 23:27:53,247] INFO : Step 40200/50000 rank6 | Train Loss: 0.8264 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1679 | Cardinality: 0.0984 | Classification: 0.0703 | Classification_binary: 0.1210 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0913 | Regression_eta: 0.0013 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1034 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0435 | Regression_eta: 0.0251 | Regression_pt: 0.0469 | Regression_sin_phi: 0.0398 | LR: 1.83e-04 | DataLoad Time: 0.0127s | Model Forward Time: 4.8886s | Phase Times: max_pre_step_barrier: 0.0270 | max_batch_to_device: 0.0062 | max_forward: 0.8324 | max_loss: 0.1802 | max_backward: 2.0569 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0152 | max_post_step_barrier: 0.0867 | Memory: rss_MB: 5955.5 | uss_MB: 5094.1 | child_rss_MB: 5625.1 | allocated_MB: 497.1 | reserved_MB: 28008.0 | inactive_split_MB: 935.5 +rank6 [2026-10-29 23:33:22,808] INFO : Step 40300/50000 rank6 | Train Loss: 0.9351 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1181 | Classification: 0.0959 | Classification_binary: 0.1652 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0014 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0456 | Classification_binary: 0.1588 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0449 | Regression_eta: 0.0324 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0539 | LR: 1.81e-04 | DataLoad Time: 0.0168s | Model Forward Time: 5.3113s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8191 | max_loss: 0.1494 | max_backward: 1.9817 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 0.9693 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 5955.1 | uss_MB: 5096.4 | child_rss_MB: 5610.7 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 927.5 +rank6 [2026-10-29 23:38:38,678] INFO : Step 40400/50000 rank6 | Train Loss: 0.9379 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1359 | Classification: 0.0771 | Classification_binary: 0.1554 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0944 | Regression_eta: 0.0014 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0486 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0872 | Regression_energy: 0.0558 | Regression_eta: 0.0476 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0724 | LR: 1.79e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.6743s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0500 | max_forward: 0.8111 | max_loss: 0.5067 | max_backward: 1.8144 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5951.4 | uss_MB: 5092.9 | child_rss_MB: 5616.0 | allocated_MB: 497.0 | reserved_MB: 28008.0 | inactive_split_MB: 931.6 +rank6 [2026-10-29 23:43:43,799] INFO : Step 40500/50000 rank6 | Train Loss: 0.8053 | Train Loss Unweighted: 0.3782 | Train Loss Components: Auxiliary: 0.1649 | Cardinality: 0.0788 | Classification: 0.0749 | Classification_binary: 0.1246 | HitGrouping: 0.1502 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0496 | Classification_binary: 0.1284 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0506 | Regression_eta: 0.0329 | Regression_pt: 0.0561 | Regression_sin_phi: 0.0647 | LR: 1.78e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3971s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.8420 | max_loss: 0.1414 | max_backward: 1.5396 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.9215 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5948.4 | uss_MB: 5092.0 | child_rss_MB: 5597.4 | allocated_MB: 494.5 | reserved_MB: 28012.0 | inactive_split_MB: 979.3 +rank6 [2026-10-29 23:48:43,694] INFO : Step 40600/50000 rank6 | Train Loss: 0.8980 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1108 | Classification: 0.0752 | Classification_binary: 0.1417 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0955 | Regression_eta: 0.0015 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0320 | Regression_energy: 0.0584 | Regression_eta: 0.0487 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0621 | LR: 1.76e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3170s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7855 | max_loss: 0.1488 | max_backward: 1.8742 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 5948.2 | uss_MB: 5094.3 | child_rss_MB: 5593.1 | allocated_MB: 499.7 | reserved_MB: 28012.0 | inactive_split_MB: 1102.0 +rank6 [2026-10-29 23:53:39,581] INFO : Step 40700/50000 rank6 | Train Loss: 0.9220 | Train Loss Unweighted: 0.4304 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1148 | Classification: 0.0767 | Classification_binary: 0.1645 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1443 | Regression_cos_phi: 0.0386 | Regression_energy: 0.0549 | Regression_eta: 0.0236 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0603 | LR: 1.75e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5458s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.8091 | max_loss: 0.2475 | max_backward: 2.0011 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.5972 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5949.6 | uss_MB: 5096.1 | child_rss_MB: 5593.6 | allocated_MB: 498.9 | reserved_MB: 28012.0 | inactive_split_MB: 776.0 +rank6 [2026-10-30 00:01:00,054] INFO : Step 40800/50000 rank6 | Train Loss: 0.8642 | Train Loss Unweighted: 0.4082 | Train Loss Components: Auxiliary: 0.1815 | Cardinality: 0.1013 | Classification: 0.0834 | Classification_binary: 0.1260 | HitGrouping: 0.1507 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1158 | Regression_cos_phi: 0.0311 | Regression_energy: 0.0485 | Regression_eta: 0.0279 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0499 | LR: 1.73e-04 | DataLoad Time: 0.0177s | Model Forward Time: 5.1150s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7033 | max_loss: 0.1373 | max_backward: 1.5042 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1068 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5935.0 | uss_MB: 4327.0 | child_rss_MB: 5586.6 | allocated_MB: 493.1 | reserved_MB: 28012.0 | inactive_split_MB: 897.3 +rank6 [2026-10-30 00:06:14,591] INFO : Step 40900/50000 rank6 | Train Loss: 0.8749 | Train Loss Unweighted: 0.4038 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1070 | Classification: 0.0854 | Classification_binary: 0.1300 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0923 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0306 | Classification_binary: 0.1130 | Regression_cos_phi: 0.0380 | Regression_energy: 0.0517 | Regression_eta: 0.0207 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0379 | LR: 1.72e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5710s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7515 | max_loss: 0.2142 | max_backward: 1.7950 | max_optimizer_step: 0.4960 | max_tensorboard_logging: 0.9322 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5926.9 | uss_MB: 5094.2 | child_rss_MB: 5563.3 | allocated_MB: 497.9 | reserved_MB: 30012.0 | inactive_split_MB: 893.0 +rank6 [2026-10-30 00:11:07,451] INFO : Step 41000/50000 rank6 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.0956 | Classification: 0.0893 | Classification_binary: 0.1497 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0014 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0394 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0766 | Regression_energy: 0.0450 | Regression_eta: 0.0364 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0493 | LR: 1.70e-04 | DataLoad Time: 0.0114s | Model Forward Time: 5.0869s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0071 | max_forward: 1.1963 | max_loss: 0.1519 | max_backward: 2.0474 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.0118 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 5918.6 | uss_MB: 5085.3 | child_rss_MB: 5547.4 | allocated_MB: 498.7 | reserved_MB: 30012.0 | inactive_split_MB: 1108.0 +rank6 [2026-10-30 00:11:08,374] INFO : Running validation on rank6 +rank6 [2026-10-30 00:11:08,374] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 00:11:08,374] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 00:11:59,246] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:12:00,024] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 00:12:26,075] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 00:12:27,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 00:12:28,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 00:12:29,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 00:12:39,357] INFO : Time taken to make predictions on device 6 is: 0.26 min +rank6 [2026-10-30 00:12:39,784] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:12:40,281] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 00:12:56,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 00:12:57,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 00:12:58,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 00:12:59,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 00:13:09,229] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 00:13:09,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 00:13:10,647] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 00:13:27,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 00:13:27,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 00:13:28,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 00:13:29,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 00:13:39,475] INFO : Time taken to make predictions on device 6 is: 0.24 min +rank6 [2026-10-30 00:19:15,378] INFO : Step 41100/50000 rank6 | Train Loss: 0.9430 | Train Loss Unweighted: 0.4319 | Train Loss Components: Auxiliary: 0.1995 | Cardinality: 0.1330 | Classification: 0.0721 | Classification_binary: 0.1579 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0013 | Regression_pt: 0.0998 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1425 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0456 | Regression_eta: 0.0305 | Regression_pt: 0.0488 | Regression_sin_phi: 0.0369 | LR: 1.69e-04 | DataLoad Time: 0.0148s | Model Forward Time: 6.3663s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8221 | max_loss: 0.4800 | max_backward: 2.2615 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7242.0 | uss_MB: 5759.2 | child_rss_MB: 5549.6 | allocated_MB: 497.6 | reserved_MB: 23400.0 | inactive_split_MB: 991.8 +rank6 [2026-10-30 00:24:27,079] INFO : Step 41200/50000 rank6 | Train Loss: 0.8544 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1097 | Classification: 0.0734 | Classification_binary: 0.1131 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1015 | Regression_eta: 0.0015 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0462 | Classification_binary: 0.1074 | Regression_cos_phi: 0.0536 | Regression_energy: 0.0559 | Regression_eta: 0.0866 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0624 | LR: 1.67e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2543s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7030 | max_loss: 0.2408 | max_backward: 1.7750 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5316 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6039.9 | uss_MB: 5208.8 | child_rss_MB: 5554.6 | allocated_MB: 496.1 | reserved_MB: 27308.0 | inactive_split_MB: 984.0 +rank6 [2026-10-30 00:29:20,951] INFO : Step 41300/50000 rank6 | Train Loss: 0.8221 | Train Loss Unweighted: 0.3852 | Train Loss Components: Auxiliary: 0.1744 | Cardinality: 0.0906 | Classification: 0.0754 | Classification_binary: 0.1369 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0849 | Regression_eta: 0.0012 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1298 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0485 | Regression_eta: 0.0226 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0539 | LR: 1.66e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4686s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8083 | max_loss: 0.1794 | max_backward: 1.9993 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.5906 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6040.0 | uss_MB: 5213.3 | child_rss_MB: 5551.2 | allocated_MB: 497.0 | reserved_MB: 27308.0 | inactive_split_MB: 1229.0 +rank6 [2026-10-30 00:34:25,620] INFO : Step 41400/50000 rank6 | Train Loss: 0.9216 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1171 | Classification: 0.0901 | Classification_binary: 0.1523 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1010 | Regression_eta: 0.0011 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0371 | Regression_energy: 0.0536 | Regression_eta: 0.0253 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0333 | LR: 1.64e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.6366s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0065 | max_forward: 0.7831 | max_loss: 0.4937 | max_backward: 1.9271 | max_optimizer_step: 0.3071 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6033.3 | uss_MB: 5208.0 | child_rss_MB: 5553.7 | allocated_MB: 498.4 | reserved_MB: 29288.0 | inactive_split_MB: 1180.4 +rank6 [2026-10-30 00:39:29,960] INFO : Step 41500/50000 rank6 | Train Loss: 0.8584 | Train Loss Unweighted: 0.4111 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1030 | Classification: 0.0769 | Classification_binary: 0.1210 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1392 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0554 | Regression_eta: 0.0414 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0586 | LR: 1.63e-04 | DataLoad Time: 0.0167s | Model Forward Time: 6.0424s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.1908 | max_loss: 0.3017 | max_backward: 3.1310 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 6009.7 | uss_MB: 5150.0 | child_rss_MB: 5552.7 | allocated_MB: 494.5 | reserved_MB: 29288.0 | inactive_split_MB: 976.9 +rank6 [2026-10-30 00:44:19,403] INFO : Step 41600/50000 rank6 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4026 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1032 | Classification: 0.0762 | Classification_binary: 0.1246 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0992 | Regression_eta: 0.0013 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0584 | Regression_eta: 0.0353 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0478 | LR: 1.61e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9942s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0065 | max_forward: 0.7279 | max_loss: 0.2339 | max_backward: 2.5338 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.4866 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1404 | Memory: rss_MB: 6001.6 | uss_MB: 5206.5 | child_rss_MB: 5546.1 | allocated_MB: 496.4 | reserved_MB: 29288.0 | inactive_split_MB: 761.6 +rank6 [2026-10-30 00:49:10,876] INFO : Step 41700/50000 rank6 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1941 | Cardinality: 0.1112 | Classification: 0.0767 | Classification_binary: 0.1493 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0988 | Regression_eta: 0.0013 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1358 | Regression_cos_phi: 0.0581 | Regression_energy: 0.0569 | Regression_eta: 0.0364 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0555 | LR: 1.60e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5067s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7998 | max_loss: 0.1827 | max_backward: 1.9774 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6004.5 | uss_MB: 5210.8 | child_rss_MB: 5548.7 | allocated_MB: 497.0 | reserved_MB: 29288.0 | inactive_split_MB: 1220.0 +rank6 [2026-10-30 00:53:58,972] INFO : Step 41800/50000 rank6 | Train Loss: 0.9125 | Train Loss Unweighted: 0.4285 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.1148 | Classification: 0.0784 | Classification_binary: 0.1537 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1735 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0426 | Regression_eta: 0.0592 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0615 | LR: 1.58e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8252s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7952 | max_loss: 0.1582 | max_backward: 1.8899 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9392 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1143 | Memory: rss_MB: 5984.7 | uss_MB: 5200.7 | child_rss_MB: 5542.4 | allocated_MB: 499.3 | reserved_MB: 31288.0 | inactive_split_MB: 932.9 +rank6 [2026-10-30 00:59:00,784] INFO : Step 41900/50000 rank6 | Train Loss: 0.8303 | Train Loss Unweighted: 0.3927 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1132 | Classification: 0.0782 | Classification_binary: 0.1167 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0010 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1238 | Regression_cos_phi: 0.0515 | Regression_energy: 0.0647 | Regression_eta: 0.0257 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0383 | LR: 1.57e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.1513s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7413 | max_loss: 0.1970 | max_backward: 1.7614 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5208 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5992.5 | uss_MB: 5207.2 | child_rss_MB: 5522.2 | allocated_MB: 497.1 | reserved_MB: 31288.0 | inactive_split_MB: 1188.0 +rank6 [2026-10-30 01:03:51,615] INFO : Step 42000/50000 rank6 | Train Loss: 0.8724 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1847 | Cardinality: 0.0893 | Classification: 0.0874 | Classification_binary: 0.1377 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0958 | Regression_eta: 0.0012 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.1176 | Regression_cos_phi: 0.0423 | Regression_energy: 0.0419 | Regression_eta: 0.0287 | Regression_pt: 0.0458 | Regression_sin_phi: 0.0639 | LR: 1.56e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7564s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1147 | max_forward: 0.8308 | max_loss: 0.1392 | max_backward: 1.8336 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5996.6 | uss_MB: 5209.4 | child_rss_MB: 5540.2 | allocated_MB: 496.3 | reserved_MB: 31288.0 | inactive_split_MB: 930.5 +rank6 [2026-10-30 01:03:52,597] INFO : Running validation on rank6 +rank6 [2026-10-30 01:03:52,597] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 01:03:52,597] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 01:04:46,127] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:04:46,670] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 01:05:13,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 01:05:14,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 01:05:15,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 01:05:16,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 01:05:26,972] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-30 01:05:27,369] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:05:27,884] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 01:05:43,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 01:05:44,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 01:05:45,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 01:05:46,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 01:05:56,695] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 01:05:57,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:05:57,664] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 01:06:13,687] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 01:06:14,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 01:06:15,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 01:06:16,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 01:06:26,659] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 01:11:42,847] INFO : Step 42100/50000 rank6 | Train Loss: 0.9390 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1200 | Classification: 0.0884 | Classification_binary: 0.1619 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0926 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0545 | Classification_binary: 0.1783 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0547 | Regression_eta: 0.0355 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0726 | LR: 1.54e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.0868s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0976 | max_forward: 1.0189 | max_loss: 0.3859 | max_backward: 2.5922 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 1.0368 | max_tensorboard_flush: 0.0149 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7501.8 | uss_MB: 4866.0 | child_rss_MB: 5544.9 | allocated_MB: 497.0 | reserved_MB: 25850.0 | inactive_split_MB: 892.8 +rank6 [2026-10-30 01:16:53,955] INFO : Step 42200/50000 rank6 | Train Loss: 0.9057 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1144 | Classification: 0.0879 | Classification_binary: 0.1381 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0519 | Classification_binary: 0.1338 | Regression_cos_phi: 0.0933 | Regression_energy: 0.0456 | Regression_eta: 0.0375 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0559 | LR: 1.53e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.1534s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7457 | max_loss: 0.1999 | max_backward: 1.6824 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.5916 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6096.0 | uss_MB: 5306.7 | child_rss_MB: 5537.7 | allocated_MB: 497.3 | reserved_MB: 27866.0 | inactive_split_MB: 959.8 +rank6 [2026-10-30 01:18:06,041] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank6 [2026-10-30 01:18:06,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank6 [2026-10-30 01:21:57,449] INFO : Step 42300/50000 rank6 | Train Loss: 0.9052 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1933 | Cardinality: 0.1127 | Classification: 0.0787 | Classification_binary: 0.1569 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0868 | Regression_eta: 0.0013 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1554 | Regression_cos_phi: 0.0615 | Regression_energy: 0.0474 | Regression_eta: 0.0307 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0542 | LR: 1.52e-04 | DataLoad Time: 0.0160s | Model Forward Time: 6.1171s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.9764 | max_loss: 0.1904 | max_backward: 2.7671 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.9997 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6118.1 | uss_MB: 5325.9 | child_rss_MB: 5544.6 | allocated_MB: 499.1 | reserved_MB: 27872.0 | inactive_split_MB: 1033.6 +rank6 [2026-10-30 01:26:55,612] INFO : Step 42400/50000 rank6 | Train Loss: 0.9358 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.1976 | Cardinality: 0.1220 | Classification: 0.0815 | Classification_binary: 0.1533 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0567 | Classification_binary: 0.1681 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0443 | Regression_eta: 0.0355 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0550 | LR: 1.50e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8868s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8701 | max_loss: 0.1465 | max_backward: 1.9416 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.8192 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6099.3 | uss_MB: 5308.3 | child_rss_MB: 5546.3 | allocated_MB: 499.8 | reserved_MB: 29908.0 | inactive_split_MB: 939.2 +rank6 [2026-10-30 01:31:49,012] INFO : Step 42500/50000 rank6 | Train Loss: 0.8126 | Train Loss Unweighted: 0.3791 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0975 | Classification: 0.0765 | Classification_binary: 0.1172 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0918 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1046 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0382 | Regression_eta: 0.0252 | Regression_pt: 0.0422 | Regression_sin_phi: 0.0344 | LR: 1.49e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5923s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0858 | max_forward: 0.8156 | max_loss: 0.1407 | max_backward: 1.8570 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7997 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6101.2 | uss_MB: 5308.6 | child_rss_MB: 5538.8 | allocated_MB: 497.5 | reserved_MB: 29908.0 | inactive_split_MB: 943.4 +rank6 [2026-10-30 01:36:44,534] INFO : Step 42600/50000 rank6 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0772 | Classification_binary: 0.1347 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0012 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0590 | Classification_binary: 0.1311 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0492 | Regression_eta: 0.0416 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0807 | LR: 1.48e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.0488s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7519 | max_loss: 0.1393 | max_backward: 1.8021 | max_optimizer_step: 0.3023 | max_tensorboard_logging: 0.7973 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6101.5 | uss_MB: 4501.2 | child_rss_MB: 5544.2 | allocated_MB: 497.0 | reserved_MB: 29908.0 | inactive_split_MB: 877.2 +rank6 [2026-10-30 01:41:48,354] INFO : Step 42700/50000 rank6 | Train Loss: 0.9129 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1178 | Classification: 0.0751 | Classification_binary: 0.1632 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0858 | Regression_eta: 0.0014 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.1646 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0471 | Regression_eta: 0.0306 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0619 | LR: 1.47e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.6153s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 1.0869 | max_loss: 0.1547 | max_backward: 1.6854 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7949 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6101.6 | uss_MB: 5308.9 | child_rss_MB: 5545.8 | allocated_MB: 497.3 | reserved_MB: 29908.0 | inactive_split_MB: 953.4 +rank6 [2026-10-30 01:46:39,116] INFO : Step 42800/50000 rank6 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1179 | Classification: 0.0767 | Classification_binary: 0.1362 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0013 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.1521 | Regression_cos_phi: 0.0596 | Regression_energy: 0.0472 | Regression_eta: 0.0370 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0634 | LR: 1.45e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3536s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8438 | max_loss: 0.1478 | max_backward: 1.6710 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.6821 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6089.4 | uss_MB: 5295.4 | child_rss_MB: 5546.6 | allocated_MB: 499.0 | reserved_MB: 30126.0 | inactive_split_MB: 875.7 +rank6 [2026-10-30 01:51:40,827] INFO : Step 42900/50000 rank6 | Train Loss: 0.7857 | Train Loss Unweighted: 0.3758 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0951 | Classification: 0.0820 | Classification_binary: 0.1113 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0898 | Regression_eta: 0.0010 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1092 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0496 | Regression_eta: 0.0381 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0509 | LR: 1.44e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.9235s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8792 | max_loss: 0.1461 | max_backward: 1.9928 | max_optimizer_step: 0.1903 | max_tensorboard_logging: 0.9695 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6098.2 | uss_MB: 5308.5 | child_rss_MB: 5544.6 | allocated_MB: 496.5 | reserved_MB: 30126.0 | inactive_split_MB: 951.8 +rank6 [2026-10-30 01:56:37,111] INFO : Step 43000/50000 rank6 | Train Loss: 0.8899 | Train Loss Unweighted: 0.4207 | Train Loss Components: Auxiliary: 0.1911 | Cardinality: 0.1035 | Classification: 0.0847 | Classification_binary: 0.1529 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0343 | Classification_binary: 0.1195 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0387 | Regression_eta: 0.0304 | Regression_pt: 0.0440 | Regression_sin_phi: 0.0390 | LR: 1.43e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7361s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1951 | max_backward: 1.9694 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.8418 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6086.1 | uss_MB: 5311.0 | child_rss_MB: 5508.6 | allocated_MB: 495.5 | reserved_MB: 30126.0 | inactive_split_MB: 1120.4 +rank6 [2026-10-30 01:56:38,137] INFO : Running validation on rank6 +rank6 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 01:57:33,188] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:57:33,677] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 01:57:59,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 01:58:00,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 01:58:01,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 01:58:02,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 01:58:12,649] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 01:58:13,038] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:58:13,511] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 01:58:29,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 01:58:30,339] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 01:58:31,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 01:58:32,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 01:58:42,264] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 01:58:43,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 01:58:43,758] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 01:58:59,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 01:59:00,521] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 01:59:01,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 01:59:02,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 01:59:12,119] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-30 02:04:24,631] INFO : Step 43100/50000 rank6 | Train Loss: 0.8965 | Train Loss Unweighted: 0.4137 | Train Loss Components: Auxiliary: 0.1901 | Cardinality: 0.1074 | Classification: 0.0705 | Classification_binary: 0.1472 | HitGrouping: 0.1435 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1316 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0464 | Regression_eta: 0.0392 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0725 | LR: 1.42e-04 | DataLoad Time: 0.0156s | Model Forward Time: 5.2210s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.2396 | max_backward: 1.9954 | max_optimizer_step: 0.3209 | max_tensorboard_logging: 1.0803 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7631.1 | uss_MB: 6853.7 | child_rss_MB: 5519.2 | allocated_MB: 496.0 | reserved_MB: 26194.0 | inactive_split_MB: 843.4 +rank6 [2026-10-30 02:09:06,731] INFO : Step 43200/50000 rank6 | Train Loss: 0.8164 | Train Loss Unweighted: 0.3744 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0957 | Classification: 0.0763 | Classification_binary: 0.1197 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1208 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0425 | Regression_eta: 0.0400 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0430 | LR: 1.40e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4249s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7823 | max_loss: 0.4822 | max_backward: 1.6766 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6123.7 | uss_MB: 5349.6 | child_rss_MB: 5511.0 | allocated_MB: 495.8 | reserved_MB: 26686.0 | inactive_split_MB: 848.9 +rank6 [2026-10-30 02:13:48,743] INFO : Step 43300/50000 rank6 | Train Loss: 0.9151 | Train Loss Unweighted: 0.4260 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1202 | Classification: 0.0906 | Classification_binary: 0.1467 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0013 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0345 | Regression_energy: 0.0387 | Regression_eta: 0.0362 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0386 | LR: 1.39e-04 | DataLoad Time: 0.0130s | Model Forward Time: 4.7164s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0155 | max_forward: 0.8938 | max_loss: 0.1464 | max_backward: 2.1010 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.5477 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1243 | Memory: rss_MB: 6097.4 | uss_MB: 5352.9 | child_rss_MB: 5427.7 | allocated_MB: 497.9 | reserved_MB: 28278.0 | inactive_split_MB: 1309.2 +rank6 [2026-10-30 02:18:30,112] INFO : Step 43400/50000 rank6 | Train Loss: 0.9723 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2082 | Cardinality: 0.1222 | Classification: 0.0880 | Classification_binary: 0.1684 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0015 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1558 | Regression_cos_phi: 0.0598 | Regression_energy: 0.0424 | Regression_eta: 0.0421 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0773 | LR: 1.38e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4353s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.5043 | max_backward: 2.0661 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.4824 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6077.9 | uss_MB: 5350.3 | child_rss_MB: 5335.1 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 851.9 +rank6 [2026-10-30 02:23:12,892] INFO : Step 43500/50000 rank6 | Train Loss: 0.8402 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.0947 | Classification: 0.0776 | Classification_binary: 0.1278 | HitGrouping: 0.1501 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0999 | Regression_eta: 0.0012 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0568 | Regression_eta: 0.0437 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0616 | LR: 1.37e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.8633s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8086 | max_loss: 0.1489 | max_backward: 2.0887 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.7978 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1094 | Memory: rss_MB: 6077.6 | uss_MB: 5352.8 | child_rss_MB: 5340.6 | allocated_MB: 498.6 | reserved_MB: 28278.0 | inactive_split_MB: 909.7 +rank6 [2026-10-30 02:28:09,613] INFO : Step 43600/50000 rank6 | Train Loss: 0.7985 | Train Loss Unweighted: 0.3653 | Train Loss Components: Auxiliary: 0.1621 | Cardinality: 0.0887 | Classification: 0.0728 | Classification_binary: 0.1184 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0851 | Regression_eta: 0.0013 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1067 | Regression_cos_phi: 0.0511 | Regression_energy: 0.0471 | Regression_eta: 0.0466 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0784 | LR: 1.36e-04 | DataLoad Time: 0.0200s | Model Forward Time: 5.2513s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0170 | max_forward: 1.2777 | max_loss: 0.2064 | max_backward: 1.9835 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 1.1084 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6044.0 | uss_MB: 4957.3 | child_rss_MB: 5260.0 | allocated_MB: 493.5 | reserved_MB: 28278.0 | inactive_split_MB: 1313.2 +rank6 [2026-10-30 02:33:07,296] INFO : Step 43700/50000 rank6 | Train Loss: 0.8599 | Train Loss Unweighted: 0.3884 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1089 | Classification: 0.0797 | Classification_binary: 0.1434 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0812 | Regression_eta: 0.0013 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0450 | Regression_eta: 0.0335 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0559 | LR: 1.35e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6054s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7902 | max_loss: 0.1752 | max_backward: 1.8774 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8044 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6046.0 | uss_MB: 5350.4 | child_rss_MB: 5241.5 | allocated_MB: 498.0 | reserved_MB: 28278.0 | inactive_split_MB: 1046.0 +rank6 [2026-10-30 02:37:45,726] INFO : Step 43800/50000 rank6 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1875 | Cardinality: 0.1049 | Classification: 0.0814 | Classification_binary: 0.1423 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0907 | Regression_eta: 0.0012 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0354 | Regression_energy: 0.0453 | Regression_eta: 0.0279 | Regression_pt: 0.0492 | Regression_sin_phi: 0.0330 | LR: 1.34e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5750s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7940 | max_loss: 0.4805 | max_backward: 1.8159 | max_optimizer_step: 0.2941 | max_tensorboard_logging: 0.7850 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6040.0 | uss_MB: 5349.3 | child_rss_MB: 5218.8 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 839.4 +rank6 [2026-10-30 02:42:41,736] INFO : Step 43900/50000 rank6 | Train Loss: 0.7987 | Train Loss Unweighted: 0.3885 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.0999 | Classification: 0.0753 | Classification_binary: 0.1166 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0968 | Regression_eta: 0.0010 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0337 | Classification_binary: 0.1039 | Regression_cos_phi: 0.0293 | Regression_energy: 0.0434 | Regression_eta: 0.0193 | Regression_pt: 0.0477 | Regression_sin_phi: 0.0290 | LR: 1.33e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.5938s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0846 | max_forward: 0.8891 | max_loss: 0.1768 | max_backward: 1.7357 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7516 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6030.6 | uss_MB: 5347.7 | child_rss_MB: 5179.6 | allocated_MB: 494.5 | reserved_MB: 28278.0 | inactive_split_MB: 1129.5 +rank6 [2026-10-30 02:47:33,826] INFO : Step 44000/50000 rank6 | Train Loss: 0.8263 | Train Loss Unweighted: 0.3926 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.1027 | Classification: 0.0840 | Classification_binary: 0.1318 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0868 | Regression_eta: 0.0011 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1212 | Regression_cos_phi: 0.0446 | Regression_energy: 0.0465 | Regression_eta: 0.0287 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0543 | LR: 1.32e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4526s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8816 | max_loss: 0.1426 | max_backward: 1.9616 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.4846 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6020.6 | uss_MB: 5352.2 | child_rss_MB: 5134.9 | allocated_MB: 496.3 | reserved_MB: 28762.0 | inactive_split_MB: 1307.2 +rank6 [2026-10-30 02:47:34,861] INFO : Running validation on rank6 +rank6 [2026-10-30 02:47:34,861] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 02:47:34,861] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 02:48:28,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 02:48:28,669] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 02:48:56,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 02:48:57,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 02:48:58,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 02:48:59,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 02:49:09,209] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 02:49:09,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 02:49:10,103] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 02:49:26,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 02:49:27,045] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 02:49:28,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 02:49:28,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 02:49:38,990] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 02:49:39,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 02:49:40,182] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 02:49:55,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 02:49:56,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 02:49:57,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 02:49:58,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 02:50:08,194] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-30 02:55:20,174] INFO : Step 44100/50000 rank6 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1986 | Cardinality: 0.1334 | Classification: 0.0764 | Classification_binary: 0.1516 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0433 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0395 | Regression_energy: 0.0485 | Regression_eta: 0.0289 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0350 | LR: 1.31e-04 | DataLoad Time: 0.0144s | Model Forward Time: 5.1236s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0079 | max_forward: 0.8221 | max_loss: 0.1579 | max_backward: 2.0192 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 1.0889 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7539.1 | uss_MB: 6869.4 | child_rss_MB: 5111.3 | allocated_MB: 498.6 | reserved_MB: 25920.0 | inactive_split_MB: 1073.8 +rank6 [2026-10-30 03:00:12,363] INFO : Step 44200/50000 rank6 | Train Loss: 0.9018 | Train Loss Unweighted: 0.4273 | Train Loss Components: Auxiliary: 0.1909 | Cardinality: 0.1109 | Classification: 0.0847 | Classification_binary: 0.1354 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0012 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0412 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0543 | Regression_eta: 0.0260 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0338 | LR: 1.30e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3590s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.1013 | max_forward: 0.8608 | max_loss: 0.1485 | max_backward: 1.6010 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 6119.2 | uss_MB: 5452.1 | child_rss_MB: 5083.8 | allocated_MB: 496.1 | reserved_MB: 27956.0 | inactive_split_MB: 1248.1 +rank6 [2026-10-30 03:04:47,146] INFO : Step 44300/50000 rank6 | Train Loss: 0.8239 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1722 | Cardinality: 0.1007 | Classification: 0.0789 | Classification_binary: 0.1118 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1081 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0468 | Regression_eta: 0.0381 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0549 | LR: 1.29e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3185s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8120 | max_loss: 0.1377 | max_backward: 1.4700 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9055 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6110.2 | uss_MB: 5452.8 | child_rss_MB: 5031.6 | allocated_MB: 492.8 | reserved_MB: 27956.0 | inactive_split_MB: 1035.8 +rank6 [2026-10-30 03:09:18,054] INFO : Step 44400/50000 rank6 | Train Loss: 0.8683 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.0849 | Classification: 0.0836 | Classification_binary: 0.1500 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1270 | Regression_cos_phi: 0.0517 | Regression_energy: 0.0406 | Regression_eta: 0.0255 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0353 | LR: 1.28e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9246s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8907 | max_loss: 0.1476 | max_backward: 2.0623 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6106.5 | uss_MB: 5454.6 | child_rss_MB: 5002.7 | allocated_MB: 499.5 | reserved_MB: 27956.0 | inactive_split_MB: 926.0 +rank6 [2026-10-30 03:14:01,913] INFO : Step 44500/50000 rank6 | Train Loss: 0.8672 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.0989 | Classification: 0.0738 | Classification_binary: 0.1455 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0512 | Regression_energy: 0.0452 | Regression_eta: 0.0304 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0710 | LR: 1.27e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.2929s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0597 | max_forward: 0.8669 | max_loss: 0.1547 | max_backward: 1.7327 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.5628 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6105.7 | uss_MB: 5452.5 | child_rss_MB: 5004.4 | allocated_MB: 498.3 | reserved_MB: 27956.0 | inactive_split_MB: 860.5 +rank6 [2026-10-30 03:18:59,361] INFO : Step 44600/50000 rank6 | Train Loss: 0.7900 | Train Loss Unweighted: 0.3743 | Train Loss Components: Auxiliary: 0.1688 | Cardinality: 0.0956 | Classification: 0.0724 | Classification_binary: 0.1157 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0919 | Regression_eta: 0.0010 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1072 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0720 | Regression_eta: 0.0261 | Regression_pt: 0.0787 | Regression_sin_phi: 0.0606 | LR: 1.26e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5431s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0745 | max_forward: 0.7797 | max_loss: 0.1409 | max_backward: 1.9334 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6022 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6098.6 | uss_MB: 5447.3 | child_rss_MB: 4994.6 | allocated_MB: 494.5 | reserved_MB: 27958.0 | inactive_split_MB: 927.6 +rank6 [2026-10-30 03:23:44,789] INFO : Step 44700/50000 rank6 | Train Loss: 0.8669 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1878 | Cardinality: 0.1160 | Classification: 0.0768 | Classification_binary: 0.1380 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0863 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1153 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0422 | Regression_eta: 0.0612 | Regression_pt: 0.0457 | Regression_sin_phi: 0.0640 | LR: 1.25e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4953s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8629 | max_loss: 0.1449 | max_backward: 2.0281 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.5242 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6099.2 | uss_MB: 5434.3 | child_rss_MB: 5003.0 | allocated_MB: 498.4 | reserved_MB: 27958.0 | inactive_split_MB: 781.6 +rank6 [2026-10-30 03:28:42,238] INFO : Step 44800/50000 rank6 | Train Loss: 0.8585 | Train Loss Unweighted: 0.4003 | Train Loss Components: Auxiliary: 0.1869 | Cardinality: 0.1024 | Classification: 0.0769 | Classification_binary: 0.1550 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0823 | Regression_eta: 0.0011 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1612 | Regression_cos_phi: 0.0394 | Regression_energy: 0.0403 | Regression_eta: 0.0294 | Regression_pt: 0.0456 | Regression_sin_phi: 0.0498 | LR: 1.24e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8034s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0072 | max_forward: 0.7883 | max_loss: 0.5033 | max_backward: 1.9727 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6090.2 | uss_MB: 5455.2 | child_rss_MB: 4956.4 | allocated_MB: 496.5 | reserved_MB: 27958.0 | inactive_split_MB: 778.0 +rank6 [2026-10-30 03:33:12,492] INFO : Step 44900/50000 rank6 | Train Loss: 0.8358 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.1024 | Classification: 0.0797 | Classification_binary: 0.1294 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0859 | Regression_eta: 0.0011 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1349 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0436 | Regression_eta: 0.0363 | Regression_pt: 0.0459 | Regression_sin_phi: 0.0657 | LR: 1.23e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.3690s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7360 | max_loss: 0.1482 | max_backward: 1.6886 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7825 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6084.1 | uss_MB: 5452.9 | child_rss_MB: 4956.9 | allocated_MB: 494.3 | reserved_MB: 27958.0 | inactive_split_MB: 922.0 +rank6 [2026-10-30 03:37:54,704] INFO : Step 45000/50000 rank6 | Train Loss: 0.8678 | Train Loss Unweighted: 0.3957 | Train Loss Components: Auxiliary: 0.1790 | Cardinality: 0.1036 | Classification: 0.0801 | Classification_binary: 0.1237 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0941 | Regression_eta: 0.0013 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0379 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0643 | Regression_energy: 0.0435 | Regression_eta: 0.0431 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0393 | LR: 1.22e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.5684s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7189 | max_loss: 0.2183 | max_backward: 1.8172 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.8812 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6058.9 | uss_MB: 5454.9 | child_rss_MB: 4844.5 | allocated_MB: 495.0 | reserved_MB: 27960.0 | inactive_split_MB: 978.5 +rank6 [2026-10-30 03:37:55,714] INFO : Running validation on rank6 +rank6 [2026-10-30 03:37:55,714] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 03:37:55,714] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 03:38:46,292] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 03:38:46,807] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 03:39:14,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 03:39:15,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 03:39:16,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 03:39:18,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 03:39:28,333] INFO : Time taken to make predictions on device 6 is: 0.27 min +rank6 [2026-10-30 03:39:29,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 03:39:30,322] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 03:39:46,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 03:39:47,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 03:39:48,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 03:39:49,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 03:39:59,085] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 03:40:00,144] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 03:40:00,673] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 03:40:16,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 03:40:17,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 03:40:18,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 03:40:19,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 03:40:29,186] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-30 03:45:39,355] INFO : Step 45100/50000 rank6 | Train Loss: 0.8712 | Train Loss Unweighted: 0.4129 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0997 | Classification: 0.0837 | Classification_binary: 0.1445 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0910 | Regression_eta: 0.0012 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1364 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0390 | Regression_eta: 0.0266 | Regression_pt: 0.0419 | Regression_sin_phi: 0.0435 | LR: 1.21e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.1917s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8448 | max_loss: 0.1523 | max_backward: 2.1445 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 1.0003 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7489.1 | uss_MB: 6883.8 | child_rss_MB: 4866.9 | allocated_MB: 499.4 | reserved_MB: 24746.0 | inactive_split_MB: 784.5 +rank6 [2026-10-30 03:50:46,377] INFO : Step 45200/50000 rank6 | Train Loss: 0.9130 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1058 | Classification: 0.0776 | Classification_binary: 0.1464 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0963 | Regression_eta: 0.0015 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0469 | Classification_binary: 0.1651 | Regression_cos_phi: 0.0449 | Regression_energy: 0.0397 | Regression_eta: 0.0260 | Regression_pt: 0.0436 | Regression_sin_phi: 0.0918 | LR: 1.20e-04 | DataLoad Time: 0.0159s | Model Forward Time: 6.5420s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1574 | max_backward: 1.6440 | max_optimizer_step: 0.2111 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1112 | Memory: rss_MB: 6132.8 | uss_MB: 4597.7 | child_rss_MB: 4866.1 | allocated_MB: 499.7 | reserved_MB: 26730.0 | inactive_split_MB: 1073.5 +rank6 [2026-10-30 03:55:44,004] INFO : Step 45300/50000 rank6 | Train Loss: 0.8299 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.1126 | Classification: 0.0744 | Classification_binary: 0.1207 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1281 | Regression_cos_phi: 0.0505 | Regression_energy: 0.0577 | Regression_eta: 0.0252 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0438 | LR: 1.19e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.1815s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0827 | max_forward: 0.7966 | max_loss: 0.1432 | max_backward: 1.5141 | max_optimizer_step: 0.2646 | max_tensorboard_logging: 0.6448 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6128.3 | uss_MB: 5538.9 | child_rss_MB: 4864.4 | allocated_MB: 495.4 | reserved_MB: 27170.0 | inactive_split_MB: 881.8 +rank6 [2026-10-30 04:00:47,346] INFO : Step 45400/50000 rank6 | Train Loss: 0.8255 | Train Loss Unweighted: 0.4011 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.0984 | Classification: 0.0907 | Classification_binary: 0.1338 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0870 | Regression_eta: 0.0011 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0568 | Classification_binary: 0.1271 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0535 | Regression_eta: 0.0312 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0461 | LR: 1.19e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.3452s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7846 | max_loss: 0.1934 | max_backward: 1.9857 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 0.7880 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6124.9 | uss_MB: 4519.9 | child_rss_MB: 4862.5 | allocated_MB: 499.2 | reserved_MB: 27170.0 | inactive_split_MB: 989.3 +rank6 [2026-10-30 04:05:44,853] INFO : Step 45500/50000 rank6 | Train Loss: 0.8945 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1090 | Classification: 0.0825 | Classification_binary: 0.1567 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0012 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1806 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0510 | Regression_eta: 0.0414 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0492 | LR: 1.18e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4582s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8686 | max_loss: 0.1508 | max_backward: 1.9581 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.4909 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6130.5 | uss_MB: 5543.1 | child_rss_MB: 4864.9 | allocated_MB: 498.4 | reserved_MB: 28766.0 | inactive_split_MB: 883.2 +rank6 [2026-10-30 04:10:39,744] INFO : Step 45600/50000 rank6 | Train Loss: 0.8704 | Train Loss Unweighted: 0.4121 | Train Loss Components: Auxiliary: 0.1840 | Cardinality: 0.1126 | Classification: 0.0802 | Classification_binary: 0.1324 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0983 | Regression_eta: 0.0010 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0431 | Classification_binary: 0.1399 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0518 | Regression_eta: 0.0262 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0519 | LR: 1.17e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.8865s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0163 | max_forward: 0.8161 | max_loss: 0.2729 | max_backward: 2.1846 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6122.7 | uss_MB: 5488.3 | child_rss_MB: 4871.5 | allocated_MB: 495.6 | reserved_MB: 28766.0 | inactive_split_MB: 1080.4 +rank6 [2026-10-30 04:15:37,194] INFO : Step 45700/50000 rank6 | Train Loss: 0.8006 | Train Loss Unweighted: 0.3784 | Train Loss Components: Auxiliary: 0.1681 | Cardinality: 0.0873 | Classification: 0.0773 | Classification_binary: 0.1111 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0012 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1059 | Regression_cos_phi: 0.0401 | Regression_energy: 0.0402 | Regression_eta: 0.0375 | Regression_pt: 0.0432 | Regression_sin_phi: 0.0565 | LR: 1.16e-04 | DataLoad Time: 0.0131s | Model Forward Time: 4.8141s | Phase Times: max_pre_step_barrier: 0.0327 | max_batch_to_device: 0.0149 | max_forward: 1.1858 | max_loss: 0.1429 | max_backward: 1.6968 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1366 | Memory: rss_MB: 6126.5 | uss_MB: 5540.3 | child_rss_MB: 4873.0 | allocated_MB: 496.6 | reserved_MB: 28766.0 | inactive_split_MB: 903.7 +rank6 [2026-10-30 04:20:28,800] INFO : Step 45800/50000 rank6 | Train Loss: 0.8566 | Train Loss Unweighted: 0.3982 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1097 | Classification: 0.0794 | Classification_binary: 0.1398 | HitGrouping: 0.1406 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1164 | Regression_cos_phi: 0.0366 | Regression_energy: 0.0409 | Regression_eta: 0.0178 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0434 | LR: 1.16e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.1201s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.9805 | max_loss: 0.5234 | max_backward: 2.3872 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.8153 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6131.1 | uss_MB: 5547.8 | child_rss_MB: 4874.9 | allocated_MB: 501.7 | reserved_MB: 28766.0 | inactive_split_MB: 1064.3 +rank6 [2026-10-30 04:25:16,811] INFO : Step 45900/50000 rank6 | Train Loss: 0.8754 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1042 | Classification: 0.0696 | Classification_binary: 0.1484 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0407 | Classification_binary: 0.1442 | Regression_cos_phi: 0.0427 | Regression_energy: 0.0403 | Regression_eta: 0.0216 | Regression_pt: 0.0433 | Regression_sin_phi: 0.0454 | LR: 1.15e-04 | DataLoad Time: 0.0101s | Model Forward Time: 5.8223s | Phase Times: max_pre_step_barrier: 0.0308 | max_batch_to_device: 0.0265 | max_forward: 1.7626 | max_loss: 0.1678 | max_backward: 2.6283 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 6124.4 | uss_MB: 5544.6 | child_rss_MB: 4869.8 | allocated_MB: 496.0 | reserved_MB: 29170.0 | inactive_split_MB: 983.4 +rank6 [2026-10-30 04:29:51,942] INFO : Step 46000/50000 rank6 | Train Loss: 0.7531 | Train Loss Unweighted: 0.3556 | Train Loss Components: Auxiliary: 0.1535 | Cardinality: 0.0870 | Classification: 0.0690 | Classification_binary: 0.1026 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0903 | Regression_eta: 0.0011 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1106 | Regression_cos_phi: 0.0494 | Regression_energy: 0.0568 | Regression_eta: 0.0364 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0399 | LR: 1.14e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.3117s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0329 | max_forward: 0.7497 | max_loss: 0.1389 | max_backward: 1.9140 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5104 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6122.4 | uss_MB: 5543.2 | child_rss_MB: 4869.8 | allocated_MB: 492.4 | reserved_MB: 29170.0 | inactive_split_MB: 973.8 +rank6 [2026-10-30 04:29:53,062] INFO : Running validation on rank6 +rank6 [2026-10-30 04:29:53,063] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 04:29:53,063] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 04:30:44,349] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 04:30:44,927] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 04:31:11,697] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 04:31:12,747] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 04:31:13,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 04:31:14,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 04:31:24,869] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 04:31:25,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 04:31:25,584] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 04:31:41,771] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 04:31:42,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 04:31:43,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 04:31:44,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 04:31:54,717] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 04:31:54,979] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 04:31:55,463] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 04:32:11,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 04:32:12,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 04:32:12,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 04:32:13,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 04:32:23,743] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-30 04:37:27,740] INFO : Step 46100/50000 rank6 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3769 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.0999 | Classification: 0.0743 | Classification_binary: 0.1253 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0010 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0362 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0544 | Regression_eta: 0.0285 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0624 | LR: 1.13e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.8738s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.9293 | max_loss: 0.1452 | max_backward: 2.3008 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.4968 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1219 | Memory: rss_MB: 7472.5 | uss_MB: 6889.9 | child_rss_MB: 4882.8 | allocated_MB: 498.3 | reserved_MB: 23298.0 | inactive_split_MB: 776.1 +rank6 [2026-10-30 04:42:19,615] INFO : Step 46200/50000 rank6 | Train Loss: 0.8661 | Train Loss Unweighted: 0.3994 | Train Loss Components: Auxiliary: 0.1947 | Cardinality: 0.1164 | Classification: 0.0793 | Classification_binary: 0.1511 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0833 | Regression_eta: 0.0011 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1444 | Regression_cos_phi: 0.0571 | Regression_energy: 0.0455 | Regression_eta: 0.0283 | Regression_pt: 0.0500 | Regression_sin_phi: 0.0332 | LR: 1.13e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.7276s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8070 | max_loss: 0.2423 | max_backward: 1.9860 | max_optimizer_step: 0.2482 | max_tensorboard_logging: 0.7539 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6199.1 | uss_MB: 5617.8 | child_rss_MB: 4859.8 | allocated_MB: 499.3 | reserved_MB: 25288.0 | inactive_split_MB: 1325.5 +rank6 [2026-10-30 04:46:59,453] INFO : Step 46300/50000 rank6 | Train Loss: 0.8634 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0916 | Classification: 0.0803 | Classification_binary: 0.1320 | HitGrouping: 0.1536 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1031 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0406 | Regression_eta: 0.0283 | Regression_pt: 0.0442 | Regression_sin_phi: 0.0565 | LR: 1.12e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5156s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7891 | max_loss: 0.1470 | max_backward: 1.7900 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.7709 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6201.3 | uss_MB: 5618.4 | child_rss_MB: 4887.4 | allocated_MB: 498.4 | reserved_MB: 25290.0 | inactive_split_MB: 977.6 +rank6 [2026-10-30 04:51:36,900] INFO : Step 46400/50000 rank6 | Train Loss: 0.8499 | Train Loss Unweighted: 0.4102 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.0956 | Classification: 0.0755 | Classification_binary: 0.1172 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1118 | Regression_cos_phi: 0.0750 | Regression_energy: 0.0788 | Regression_eta: 0.0395 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0484 | LR: 1.11e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9774s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7847 | max_loss: 0.2487 | max_backward: 1.9181 | max_optimizer_step: 0.3148 | max_tensorboard_logging: 0.9408 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6188.6 | uss_MB: 5608.9 | child_rss_MB: 4878.3 | allocated_MB: 501.6 | reserved_MB: 25290.0 | inactive_split_MB: 992.4 +rank6 [2026-10-30 04:56:10,902] INFO : Step 46500/50000 rank6 | Train Loss: 0.8687 | Train Loss Unweighted: 0.4055 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1002 | Classification: 0.0822 | Classification_binary: 0.1448 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0872 | Regression_eta: 0.0013 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0514 | Classification_binary: 0.1388 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0463 | Regression_eta: 0.0299 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0509 | LR: 1.11e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5410s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1839 | max_backward: 1.9629 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6193.9 | uss_MB: 5618.8 | child_rss_MB: 4880.0 | allocated_MB: 498.8 | reserved_MB: 29306.0 | inactive_split_MB: 924.4 +rank6 [2026-10-30 05:00:44,089] INFO : Step 46600/50000 rank6 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4049 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1082 | Classification: 0.0757 | Classification_binary: 0.1508 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0446 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0539 | Regression_energy: 0.0511 | Regression_eta: 0.0332 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0553 | LR: 1.10e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2126s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7939 | max_loss: 0.2001 | max_backward: 1.6859 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6188.5 | uss_MB: 5615.4 | child_rss_MB: 4846.0 | allocated_MB: 496.0 | reserved_MB: 29306.0 | inactive_split_MB: 901.7 +rank6 [2026-10-30 05:05:18,398] INFO : Step 46700/50000 rank6 | Train Loss: 0.8417 | Train Loss Unweighted: 0.3855 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0982 | Classification: 0.0807 | Classification_binary: 0.1184 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0916 | Regression_eta: 0.0015 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.1225 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0399 | Regression_eta: 0.0430 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0595 | LR: 1.10e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7619s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8551 | max_loss: 0.5124 | max_backward: 1.9004 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.7762 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1375 | Memory: rss_MB: 6185.4 | uss_MB: 5616.8 | child_rss_MB: 4868.3 | allocated_MB: 495.0 | reserved_MB: 29308.0 | inactive_split_MB: 1000.4 +rank6 [2026-10-30 05:09:48,389] INFO : Step 46800/50000 rank6 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3911 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0920 | Classification: 0.0920 | Classification_binary: 0.1296 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0828 | Regression_eta: 0.0013 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0656 | Classification_binary: 0.1139 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0413 | Regression_eta: 0.0253 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0375 | LR: 1.09e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4616s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7934 | max_loss: 0.1440 | max_backward: 1.7630 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.7219 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6183.1 | uss_MB: 5617.4 | child_rss_MB: 4871.4 | allocated_MB: 495.6 | reserved_MB: 29308.0 | inactive_split_MB: 1332.0 +rank6 [2026-10-30 05:14:19,737] INFO : Step 46900/50000 rank6 | Train Loss: 0.8496 | Train Loss Unweighted: 0.4142 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0976 | Classification: 0.0776 | Classification_binary: 0.1448 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0946 | Regression_eta: 0.0010 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1468 | Regression_cos_phi: 0.0488 | Regression_energy: 0.0534 | Regression_eta: 0.0305 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0408 | LR: 1.09e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.4515s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.9007 | max_loss: 0.1576 | max_backward: 1.8470 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6183.1 | uss_MB: 5616.1 | child_rss_MB: 4872.4 | allocated_MB: 495.4 | reserved_MB: 29310.0 | inactive_split_MB: 850.5 +rank6 [2026-10-30 05:18:49,738] INFO : Step 47000/50000 rank6 | Train Loss: 0.7980 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0982 | Classification: 0.0774 | Classification_binary: 0.1196 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0414 | Classification_binary: 0.1130 | Regression_cos_phi: 0.0373 | Regression_energy: 0.0426 | Regression_eta: 0.0181 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0490 | LR: 1.08e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8521s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 1.2405 | max_loss: 0.1502 | max_backward: 2.1273 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.7105 | max_tensorboard_flush: 0.0082 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6184.1 | uss_MB: 5617.4 | child_rss_MB: 4877.5 | allocated_MB: 494.9 | reserved_MB: 29310.0 | inactive_split_MB: 852.6 +rank6 [2026-10-30 05:18:50,808] INFO : Running validation on rank6 +rank6 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 05:18:50,808] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 05:19:41,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 05:19:42,336] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 05:20:08,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 05:20:09,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 05:20:10,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 05:20:11,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 05:20:21,177] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 05:20:21,505] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 05:20:22,014] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 05:20:37,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 05:20:38,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 05:20:39,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 05:20:40,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 05:20:50,809] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 05:20:51,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 05:20:52,202] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 05:21:07,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 05:21:08,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 05:21:09,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 05:21:10,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 05:21:20,015] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-30 05:26:12,382] INFO : Step 47100/50000 rank6 | Train Loss: 0.8046 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0851 | Classification: 0.0882 | Classification_binary: 0.1121 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0936 | Regression_eta: 0.0010 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0557 | Classification_binary: 0.1156 | Regression_cos_phi: 0.0460 | Regression_energy: 0.0521 | Regression_eta: 0.0271 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0675 | LR: 1.07e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5748s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8250 | max_loss: 0.1453 | max_backward: 1.6267 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8108 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 7449.0 | uss_MB: 6878.6 | child_rss_MB: 4837.4 | allocated_MB: 494.1 | reserved_MB: 25416.0 | inactive_split_MB: 894.5 +rank6 [2026-10-30 05:30:44,527] INFO : Step 47200/50000 rank6 | Train Loss: 0.8884 | Train Loss Unweighted: 0.4191 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1138 | Classification: 0.0793 | Classification_binary: 0.1489 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0010 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1225 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0402 | Regression_eta: 0.0231 | Regression_pt: 0.0440 | Regression_sin_phi: 0.0299 | LR: 1.07e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.9171s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7960 | max_loss: 0.1482 | max_backward: 1.9261 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.9435 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0661 | Memory: rss_MB: 6249.9 | uss_MB: 5682.5 | child_rss_MB: 4852.8 | allocated_MB: 496.8 | reserved_MB: 26534.0 | inactive_split_MB: 847.6 +rank6 [2026-10-30 05:35:15,255] INFO : Step 47300/50000 rank6 | Train Loss: 0.8547 | Train Loss Unweighted: 0.3920 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1049 | Classification: 0.0798 | Classification_binary: 0.1368 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0012 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0341 | Classification_binary: 0.1356 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0462 | Regression_eta: 0.0207 | Regression_pt: 0.0498 | Regression_sin_phi: 0.0535 | LR: 1.06e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5889s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7649 | max_loss: 0.2313 | max_backward: 1.8030 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8438 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6253.9 | uss_MB: 5688.5 | child_rss_MB: 4855.9 | allocated_MB: 496.9 | reserved_MB: 28534.0 | inactive_split_MB: 933.0 +rank6 [2026-10-30 05:39:47,015] INFO : Step 47400/50000 rank6 | Train Loss: 0.7606 | Train Loss Unweighted: 0.3755 | Train Loss Components: Auxiliary: 0.1640 | Cardinality: 0.0704 | Classification: 0.0737 | Classification_binary: 0.1072 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0960 | Regression_eta: 0.0010 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0376 | Classification_binary: 0.1097 | Regression_cos_phi: 0.0606 | Regression_energy: 0.0573 | Regression_eta: 0.0342 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0401 | LR: 1.06e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5312s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0275 | max_forward: 0.8465 | max_loss: 0.1482 | max_backward: 1.7103 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.8388 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 6241.7 | uss_MB: 5678.8 | child_rss_MB: 4860.9 | allocated_MB: 496.4 | reserved_MB: 28534.0 | inactive_split_MB: 913.7 +rank6 [2026-10-30 05:44:18,077] INFO : Step 47500/50000 rank6 | Train Loss: 0.8134 | Train Loss Unweighted: 0.3681 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1057 | Classification: 0.0739 | Classification_binary: 0.1303 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0805 | Regression_eta: 0.0011 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1193 | Regression_cos_phi: 0.0340 | Regression_energy: 0.0388 | Regression_eta: 0.0481 | Regression_pt: 0.0412 | Regression_sin_phi: 0.0695 | LR: 1.06e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7794s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1751 | max_backward: 1.9857 | max_optimizer_step: 0.3263 | max_tensorboard_logging: 0.7363 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6249.7 | uss_MB: 5690.9 | child_rss_MB: 4845.9 | allocated_MB: 498.3 | reserved_MB: 28534.0 | inactive_split_MB: 851.5 +rank6 [2026-10-30 05:48:48,524] INFO : Step 47600/50000 rank6 | Train Loss: 0.8663 | Train Loss Unweighted: 0.4076 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1090 | Classification: 0.0820 | Classification_binary: 0.1491 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0865 | Regression_eta: 0.0011 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1517 | Regression_cos_phi: 0.0594 | Regression_energy: 0.0584 | Regression_eta: 0.0310 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0419 | LR: 1.05e-04 | DataLoad Time: 0.0143s | Model Forward Time: 4.6240s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.9385 | max_loss: 0.1530 | max_backward: 2.1678 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6249.7 | uss_MB: 5691.7 | child_rss_MB: 4853.8 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 843.5 +rank6 [2026-10-30 05:53:19,839] INFO : Step 47700/50000 rank6 | Train Loss: 0.8615 | Train Loss Unweighted: 0.4088 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0954 | Classification: 0.0764 | Classification_binary: 0.1291 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1002 | Regression_eta: 0.0011 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0744 | Regression_energy: 0.0517 | Regression_eta: 0.0353 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0451 | LR: 1.05e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2186s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7406 | max_loss: 0.2391 | max_backward: 1.7059 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1444 | Memory: rss_MB: 6248.7 | uss_MB: 5687.5 | child_rss_MB: 4851.8 | allocated_MB: 493.9 | reserved_MB: 28534.0 | inactive_split_MB: 895.3 +rank6 [2026-10-30 05:57:52,842] INFO : Step 47800/50000 rank6 | Train Loss: 0.8267 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.0942 | Classification: 0.0882 | Classification_binary: 0.1186 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0870 | Regression_eta: 0.0012 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0378 | Classification_binary: 0.1222 | Regression_cos_phi: 0.0677 | Regression_energy: 0.0465 | Regression_eta: 0.0402 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0520 | LR: 1.04e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.0877s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.7869 | max_loss: 0.5030 | max_backward: 2.0522 | max_optimizer_step: 0.2267 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6249.7 | uss_MB: 5692.3 | child_rss_MB: 4857.3 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 916.9 +rank6 [2026-10-30 06:02:25,106] INFO : Step 47900/50000 rank6 | Train Loss: 0.8423 | Train Loss Unweighted: 0.4000 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.0845 | Classification: 0.0794 | Classification_binary: 0.1416 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0349 | Regression_eta: 0.0225 | Regression_pt: 0.0387 | Regression_sin_phi: 0.0340 | LR: 1.04e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5076s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7994 | max_loss: 0.1511 | max_backward: 1.9354 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.6685 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6247.4 | uss_MB: 5692.2 | child_rss_MB: 4860.2 | allocated_MB: 496.8 | reserved_MB: 30642.0 | inactive_split_MB: 779.5 +rank6 [2026-10-30 06:06:56,646] INFO : Step 48000/50000 rank6 | Train Loss: 0.8641 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0972 | Classification: 0.0768 | Classification_binary: 0.1396 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0890 | Regression_eta: 0.0015 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1483 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0486 | Regression_eta: 0.0334 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0508 | LR: 1.04e-04 | DataLoad Time: 0.0140s | Model Forward Time: 4.4102s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0057 | max_forward: 0.8017 | max_loss: 0.1550 | max_backward: 1.6990 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.7498 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6247.1 | uss_MB: 5690.2 | child_rss_MB: 4855.9 | allocated_MB: 495.1 | reserved_MB: 30642.0 | inactive_split_MB: 896.2 +rank6 [2026-10-30 06:06:57,654] INFO : Running validation on rank6 +rank6 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 06:07:49,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:07:49,560] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 06:08:14,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 06:08:15,111] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 06:08:16,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 06:08:17,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 06:08:27,248] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 06:08:28,016] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:08:28,619] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 06:08:44,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 06:08:45,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 06:08:46,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 06:08:47,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 06:08:57,388] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 06:08:58,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:08:58,714] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 06:09:14,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 06:09:14,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 06:09:15,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 06:09:16,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 06:09:26,462] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-30 06:14:16,584] INFO : Step 48100/50000 rank6 | Train Loss: 0.8226 | Train Loss Unweighted: 0.3895 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.1043 | Classification: 0.0717 | Classification_binary: 0.1116 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1023 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0579 | Classification_binary: 0.1170 | Regression_cos_phi: 0.0556 | Regression_energy: 0.0616 | Regression_eta: 0.0312 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0498 | LR: 1.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4974s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7477 | max_loss: 0.1426 | max_backward: 1.7972 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.6718 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1303 | Memory: rss_MB: 7752.6 | uss_MB: 7191.3 | child_rss_MB: 4880.7 | allocated_MB: 494.1 | reserved_MB: 23078.0 | inactive_split_MB: 1096.3 +rank6 [2026-10-30 06:18:47,187] INFO : Step 48200/50000 rank6 | Train Loss: 0.8500 | Train Loss Unweighted: 0.3835 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1027 | Classification: 0.0771 | Classification_binary: 0.1374 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0827 | Regression_eta: 0.0012 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1218 | Regression_cos_phi: 0.0583 | Regression_energy: 0.0402 | Regression_eta: 0.0250 | Regression_pt: 0.0447 | Regression_sin_phi: 0.0524 | LR: 1.03e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.2132s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8132 | max_loss: 0.2545 | max_backward: 2.1167 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1241 | Memory: rss_MB: 6310.4 | uss_MB: 5751.6 | child_rss_MB: 4867.3 | allocated_MB: 496.6 | reserved_MB: 25094.0 | inactive_split_MB: 913.4 +rank6 [2026-10-30 06:23:18,411] INFO : Step 48300/50000 rank6 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4303 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1089 | Classification: 0.0832 | Classification_binary: 0.1567 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0013 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0477 | Classification_binary: 0.1476 | Regression_cos_phi: 0.0531 | Regression_energy: 0.0511 | Regression_eta: 0.0408 | Regression_pt: 0.0563 | Regression_sin_phi: 0.0564 | LR: 1.03e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4479s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8079 | max_loss: 0.1526 | max_backward: 1.9733 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6317.4 | uss_MB: 5760.9 | child_rss_MB: 4872.1 | allocated_MB: 498.5 | reserved_MB: 25098.0 | inactive_split_MB: 771.0 +rank6 [2026-10-30 06:27:49,029] INFO : Step 48400/50000 rank6 | Train Loss: 0.7961 | Train Loss Unweighted: 0.3733 | Train Loss Components: Auxiliary: 0.1682 | Cardinality: 0.0877 | Classification: 0.0685 | Classification_binary: 0.1191 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0012 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1036 | Regression_cos_phi: 0.0788 | Regression_energy: 0.0508 | Regression_eta: 0.0477 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0705 | LR: 1.02e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2401s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7916 | max_loss: 0.1471 | max_backward: 1.6943 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5940 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6314.4 | uss_MB: 5759.4 | child_rss_MB: 4842.4 | allocated_MB: 496.3 | reserved_MB: 25098.0 | inactive_split_MB: 1087.5 +rank6 [2026-10-30 06:32:21,028] INFO : Step 48500/50000 rank6 | Train Loss: 0.8668 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0816 | Classification: 0.0832 | Classification_binary: 0.1257 | HitGrouping: 0.1542 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0423 | Classification_binary: 0.0953 | Regression_cos_phi: 0.0668 | Regression_energy: 0.0545 | Regression_eta: 0.0308 | Regression_pt: 0.0594 | Regression_sin_phi: 0.0488 | LR: 1.02e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3828s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7587 | max_loss: 0.1606 | max_backward: 1.7573 | max_optimizer_step: 0.3024 | max_tensorboard_logging: 0.5851 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6309.3 | uss_MB: 5759.8 | child_rss_MB: 4851.4 | allocated_MB: 498.9 | reserved_MB: 25098.0 | inactive_split_MB: 1192.4 +rank6 [2026-10-30 06:36:52,986] INFO : Step 48600/50000 rank6 | Train Loss: 0.8434 | Train Loss Unweighted: 0.4010 | Train Loss Components: Auxiliary: 0.1850 | Cardinality: 0.0964 | Classification: 0.0791 | Classification_binary: 0.1425 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0280 | Classification_binary: 0.1241 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0416 | Regression_eta: 0.0179 | Regression_pt: 0.0443 | Regression_sin_phi: 0.0500 | LR: 1.02e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4843s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7760 | max_loss: 0.1505 | max_backward: 1.8991 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0654 | Memory: rss_MB: 6311.3 | uss_MB: 5762.6 | child_rss_MB: 4856.0 | allocated_MB: 497.0 | reserved_MB: 26848.0 | inactive_split_MB: 777.5 +rank6 [2026-10-30 06:41:29,684] INFO : Step 48700/50000 rank6 | Train Loss: 0.9079 | Train Loss Unweighted: 0.4266 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1195 | Classification: 0.0920 | Classification_binary: 0.1456 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1509 | Regression_cos_phi: 0.0378 | Regression_energy: 0.0423 | Regression_eta: 0.0466 | Regression_pt: 0.0464 | Regression_sin_phi: 0.0458 | LR: 1.02e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5452s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7797 | max_loss: 0.1578 | max_backward: 1.7107 | max_optimizer_step: 0.5280 | max_tensorboard_logging: 0.7650 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6308.9 | uss_MB: 5759.1 | child_rss_MB: 4857.7 | allocated_MB: 496.9 | reserved_MB: 26848.0 | inactive_split_MB: 891.8 +rank6 [2026-10-30 06:46:02,598] INFO : Step 48800/50000 rank6 | Train Loss: 0.8011 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1685 | Cardinality: 0.0863 | Classification: 0.0837 | Classification_binary: 0.1116 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0965 | Regression_eta: 0.0012 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1029 | Regression_cos_phi: 0.0549 | Regression_energy: 0.0517 | Regression_eta: 0.0296 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0689 | LR: 1.01e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.4997s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7374 | max_loss: 0.2432 | max_backward: 1.8459 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.7427 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6306.9 | uss_MB: 5759.1 | child_rss_MB: 4857.0 | allocated_MB: 493.9 | reserved_MB: 26848.0 | inactive_split_MB: 991.0 +rank6 [2026-10-30 06:50:45,084] INFO : Step 48900/50000 rank6 | Train Loss: 0.8551 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.1109 | Classification: 0.0760 | Classification_binary: 0.1310 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0508 | Classification_binary: 0.1394 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0466 | Regression_eta: 0.0381 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0571 | LR: 1.01e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.5414s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8768 | max_loss: 0.1470 | max_backward: 2.0816 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6306.9 | uss_MB: 5761.7 | child_rss_MB: 4858.9 | allocated_MB: 499.5 | reserved_MB: 26852.0 | inactive_split_MB: 1398.0 +rank6 [2026-10-30 06:55:20,573] INFO : Step 49000/50000 rank6 | Train Loss: 0.9005 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.2010 | Cardinality: 0.1248 | Classification: 0.0800 | Classification_binary: 0.1499 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0939 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0424 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0421 | Regression_energy: 0.0486 | Regression_eta: 0.0235 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0405 | LR: 1.01e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.9026s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0439 | max_forward: 0.8365 | max_loss: 0.1536 | max_backward: 1.9893 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8890 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6309.3 | uss_MB: 5762.6 | child_rss_MB: 4865.2 | allocated_MB: 497.5 | reserved_MB: 26852.0 | inactive_split_MB: 943.6 +rank6 [2026-10-30 06:55:21,669] INFO : Running validation on rank6 +rank6 [2026-10-30 06:55:21,669] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 06:55:21,669] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 06:56:13,351] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:56:13,862] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 06:56:38,971] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 06:56:40,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 06:56:41,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 06:56:42,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 06:56:52,126] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 06:56:52,550] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:56:53,029] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 06:57:08,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 06:57:09,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 06:57:10,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 06:57:11,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 06:57:21,525] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 06:57:22,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 06:57:23,154] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 06:57:38,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 06:57:39,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 06:57:40,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 06:57:41,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 06:57:51,303] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-30 07:02:45,631] INFO : Step 49100/50000 rank6 | Train Loss: 0.8219 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.0880 | Classification: 0.0841 | Classification_binary: 0.1303 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0855 | Regression_eta: 0.0012 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0454 | Regression_energy: 0.0528 | Regression_eta: 0.0338 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0535 | LR: 1.01e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3720s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7738 | max_loss: 0.1431 | max_backward: 1.6939 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6276 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 7720.3 | uss_MB: 7170.2 | child_rss_MB: 4829.0 | allocated_MB: 495.9 | reserved_MB: 24730.0 | inactive_split_MB: 936.0 +rank6 [2026-10-30 07:07:18,677] INFO : Step 49200/50000 rank6 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3843 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0856 | Classification: 0.0829 | Classification_binary: 0.1188 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0892 | Regression_eta: 0.0011 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0461 | Classification_binary: 0.0985 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0481 | Regression_eta: 0.0320 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0502 | LR: 1.01e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.7724s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7590 | max_loss: 0.5032 | max_backward: 1.9826 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.7926 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6365.4 | uss_MB: 5817.0 | child_rss_MB: 4830.4 | allocated_MB: 494.5 | reserved_MB: 26752.0 | inactive_split_MB: 1117.5 +rank6 [2026-10-30 07:11:50,862] INFO : Step 49300/50000 rank6 | Train Loss: 0.9397 | Train Loss Unweighted: 0.4282 | Train Loss Components: Auxiliary: 0.2065 | Cardinality: 0.1351 | Classification: 0.0889 | Classification_binary: 0.1552 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0901 | Regression_eta: 0.0012 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1608 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0516 | Regression_eta: 0.0420 | Regression_pt: 0.0574 | Regression_sin_phi: 0.0594 | LR: 1.00e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4920s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0070 | max_forward: 0.7807 | max_loss: 0.1520 | max_backward: 1.8619 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6076 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6376.6 | uss_MB: 5827.0 | child_rss_MB: 4814.1 | allocated_MB: 496.4 | reserved_MB: 26752.0 | inactive_split_MB: 1120.6 +rank6 [2026-10-30 07:16:22,253] INFO : Step 49400/50000 rank6 | Train Loss: 0.8870 | Train Loss Unweighted: 0.4213 | Train Loss Components: Auxiliary: 0.1937 | Cardinality: 0.1005 | Classification: 0.0815 | Classification_binary: 0.1465 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1730 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0521 | Regression_eta: 0.0482 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0806 | LR: 1.00e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2722s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8046 | max_loss: 0.1540 | max_backward: 1.7244 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.6030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6373.7 | uss_MB: 5825.4 | child_rss_MB: 4813.8 | allocated_MB: 498.5 | reserved_MB: 28804.0 | inactive_split_MB: 886.2 +rank6 [2026-10-30 07:20:55,284] INFO : Step 49500/50000 rank6 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3949 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0991 | Classification: 0.0800 | Classification_binary: 0.1101 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0013 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0510 | Classification_binary: 0.1116 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0756 | Regression_eta: 0.0254 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0415 | LR: 1.00e-04 | DataLoad Time: 0.0164s | Model Forward Time: 5.2161s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0074 | max_forward: 0.9924 | max_loss: 0.1865 | max_backward: 2.5853 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.6095 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0429 | Memory: rss_MB: 6377.1 | uss_MB: 5695.1 | child_rss_MB: 4830.9 | allocated_MB: 495.9 | reserved_MB: 28804.0 | inactive_split_MB: 849.6 +rank6 [2026-10-30 07:25:48,133] INFO : Step 49600/50000 rank6 | Train Loss: 0.9075 | Train Loss Unweighted: 0.4421 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1021 | Classification: 0.0939 | Classification_binary: 0.1401 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1438 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0677 | Regression_eta: 0.0406 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0474 | LR: 1.00e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4395s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7852 | max_loss: 0.1445 | max_backward: 2.0967 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4754 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6378.7 | uss_MB: 5829.2 | child_rss_MB: 4808.2 | allocated_MB: 497.8 | reserved_MB: 28804.0 | inactive_split_MB: 845.2 +rank6 [2026-10-30 07:30:32,223] INFO : Step 49700/50000 rank6 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4307 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1074 | Classification: 0.0931 | Classification_binary: 0.1611 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0865 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0490 | Regression_energy: 0.0469 | Regression_eta: 0.0309 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0585 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6428s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.8169 | max_loss: 0.1522 | max_backward: 2.0553 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.4937 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1402 | Memory: rss_MB: 6379.2 | uss_MB: 5828.6 | child_rss_MB: 4829.3 | allocated_MB: 499.3 | reserved_MB: 30718.0 | inactive_split_MB: 838.8 +rank6 [2026-10-30 07:35:17,054] INFO : Step 49800/50000 rank6 | Train Loss: 0.8342 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.1036 | Classification: 0.0769 | Classification_binary: 0.1292 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0547 | Regression_energy: 0.0411 | Regression_eta: 0.0335 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0608 | LR: 1.00e-04 | DataLoad Time: 0.0179s | Model Forward Time: 5.1022s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0329 | max_forward: 0.7490 | max_loss: 0.1889 | max_backward: 2.2249 | max_optimizer_step: 0.2452 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6374.9 | uss_MB: 5812.4 | child_rss_MB: 4828.7 | allocated_MB: 494.4 | reserved_MB: 30718.0 | inactive_split_MB: 1134.4 +rank6 [2026-10-30 07:39:58,302] INFO : Step 49900/50000 rank6 | Train Loss: 0.8436 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.0947 | Classification: 0.0943 | Classification_binary: 0.1179 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0982 | Regression_eta: 0.0012 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.1253 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0603 | Regression_eta: 0.0419 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0675 | LR: 1.00e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.4161s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7496 | max_loss: 0.5142 | max_backward: 1.9365 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6373.9 | uss_MB: 5825.7 | child_rss_MB: 4831.8 | allocated_MB: 494.8 | reserved_MB: 30718.0 | inactive_split_MB: 1090.4 +rank6 [2026-10-30 07:44:45,887] INFO : Step 50000/50000 rank6 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4040 | Train Loss Components: Auxiliary: 0.1921 | Cardinality: 0.1113 | Classification: 0.0818 | Classification_binary: 0.1436 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0872 | Regression_eta: 0.0014 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0544 | Classification_binary: 0.1023 | Regression_cos_phi: 0.0607 | Regression_energy: 0.0566 | Regression_eta: 0.0241 | Regression_pt: 0.0639 | Regression_sin_phi: 0.0591 | LR: 1.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.6357s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 1.3997 | max_loss: 0.5678 | max_backward: 3.0655 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6373.9 | uss_MB: 5825.4 | child_rss_MB: 4827.6 | allocated_MB: 497.9 | reserved_MB: 30720.0 | inactive_split_MB: 906.1 +rank6 [2026-10-30 07:44:46,920] INFO : Running validation on rank6 +rank6 [2026-10-30 07:44:46,920] INFO : ResumableSampler physical:valid resetting +rank6 [2026-10-30 07:44:46,920] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank6 [2026-10-30 07:45:38,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 07:45:38,998] INFO : Running predictions on cld_edm_ttbar_hits +rank6 [2026-10-30 07:46:04,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_6_0.parquet +rank6 [2026-10-30 07:46:05,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_6_1.parquet +rank6 [2026-10-30 07:46:06,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_6_2.parquet +rank6 [2026-10-30 07:46:07,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_6_3.parquet +rank6 [2026-10-30 07:46:17,418] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 07:46:17,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 07:46:17,995] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank6 [2026-10-30 07:46:34,211] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_6_0.parquet +rank6 [2026-10-30 07:46:35,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_6_1.parquet +rank6 [2026-10-30 07:46:36,192] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_6_2.parquet +rank6 [2026-10-30 07:46:37,161] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_6_3.parquet +rank6 [2026-10-30 07:46:47,172] INFO : Time taken to make predictions on device 6 is: 0.25 min +rank6 [2026-10-30 07:46:47,615] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank6 [2026-10-30 07:46:48,153] INFO : Running predictions on cld_edm_qq_hits +rank6 [2026-10-30 07:47:03,956] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_6_0.parquet +rank6 [2026-10-30 07:47:04,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_6_1.parquet +rank6 [2026-10-30 07:47:05,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_6_2.parquet +rank6 [2026-10-30 07:47:06,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_6_3.parquet +rank6 [2026-10-30 07:47:16,369] INFO : Time taken to make predictions on device 6 is: 0.23 min +rank6 [2026-10-30 07:47:17,254] INFO : Training completed. Total time on device 6: 1017.498min diff --git a/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.7 b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.7 new file mode 100644 index 0000000000000000000000000000000000000000..903e6d5731b3809ca91520ed6845d7402394203c --- /dev/null +++ b/cld/hits/v3.2.0/cld_hits_heptv2_seed12345_20261028_111202_875252/train.log.7 @@ -0,0 +1,2631 @@ +rank7 [2026-10-28 11:12:32,418] INFO : Initializing model with seed=12345; process seed=12352 +rank7 [2026-10-28 11:12:32,419] INFO : configured dtype=torch.bfloat16 for autocast +rank7 [2026-10-28 11:12:54,158] INFO : Instantiating model +rank7 [2026-10-28 11:12:54,158] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank7 [2026-10-28 11:12:54,158] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank7 [2026-10-28 11:12:54,176] INFO : Input encoding initialization took 0.02s +rank7 [2026-10-28 11:12:54,176] INFO : nn0 parameters: 171776 +rank7 [2026-10-28 11:12:54,176] INFO : Initializing shared backbone layers, num_convs=6 +rank7 [2026-10-28 11:12:54,177] INFO : Initializing backbone layer 0 +rank7 [2026-10-28 11:12:54,212] INFO : Initializing backbone layer 1 +rank7 [2026-10-28 11:12:54,221] INFO : Initializing backbone layer 2 +rank7 [2026-10-28 11:12:54,230] INFO : Initializing backbone layer 3 +rank7 [2026-10-28 11:12:54,238] INFO : Initializing backbone layer 4 +rank7 [2026-10-28 11:12:54,247] INFO : Initializing backbone layer 5 +rank7 [2026-10-28 11:12:54,255] INFO : Backbone initialization took 0.08s +rank7 [2026-10-28 11:12:54,256] INFO : backbone parameters: 7901376 +rank7 [2026-10-28 11:12:54,256] INFO : Initializing output DNNs +rank7 [2026-10-28 11:12:54,292] INFO : Output DNNs initialization took 0.04s +rank7 [2026-10-28 11:12:54,292] INFO : backbone_mode=BackboneMode.SHARED +rank7 [2026-10-28 11:12:54,292] INFO : jagged_attention enabled=False +rank7 [2026-10-28 11:12:54,292] INFO : flash_attn_varlen enabled=False +rank7 [2026-10-28 11:12:54,292] INFO : task_queries enabled=False +rank7 [2026-10-28 11:12:54,292] INFO : classification_norm parameters: 0 +rank7 [2026-10-28 11:12:54,292] INFO : regression_norm parameters: 0 +rank7 [2026-10-28 11:12:54,292] INFO : classification_query parameters: 0 +rank7 [2026-10-28 11:12:54,292] INFO : regression_query parameters: 0 +rank7 [2026-10-28 11:12:54,292] INFO : classification_readout parameters: 0 +rank7 [2026-10-28 11:12:54,293] INFO : regression_readout parameters: 0 +rank7 [2026-10-28 11:12:54,293] INFO : output_mode=set +rank7 [2026-10-28 11:12:54,293] INFO : set_decoder parameters: 4713812 +rank7 [2026-10-28 11:12:54,293] INFO : nn_binary_particle parameters: 0 +rank7 [2026-10-28 11:12:54,293] INFO : nn_pid parameters: 0 +rank7 [2026-10-28 11:12:54,293] INFO : nn_pt parameters: 0 +rank7 [2026-10-28 11:12:54,293] INFO : nn_eta parameters: 0 +rank7 [2026-10-28 11:12:54,293] INFO : nn_sin_phi parameters: 0 +rank7 [2026-10-28 11:12:54,293] INFO : nn_cos_phi parameters: 0 +rank7 [2026-10-28 11:12:54,293] INFO : nn_energy parameters: 0 +rank7 [2026-10-28 11:12:54,294] INFO : Total MLPF parameters: 12786964 +rank7 [2026-10-28 11:12:54,294] INFO : MLPF __init__ done +rank7 [2026-10-28 11:12:54,294] INFO : Instantiated model +rank7 [2026-10-28 11:12:54,294] INFO : Moving model to device rank=7 +rank7 [2026-10-28 11:12:54,319] INFO : Moved model to device rank=7 +rank7 [2026-10-28 11:12:54,320] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank7 [2026-10-28 11:12:54,321] INFO : Configured model for SyncBatchNorm rank=7 +rank7 [2026-10-28 11:12:54,645] INFO : Configured model for DistributedDataParallel rank=7 +rank7 [2026-10-28 11:12:54,808] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank7 [2026-10-28 11:12:54,808] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank7 [2026-10-28 11:12:54,809] INFO : Trainable parameters: 12786964 +rank7 [2026-10-28 11:12:54,809] INFO : Non-trainable parameters: 23268 +rank7 [2026-10-28 11:12:54,809] INFO : Total parameters: 12810232 +rank7 [2026-10-28 11:12:54,813] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank7 [2026-10-28 11:12:54,813] INFO : Getting dataloaders +rank7 [2026-10-28 11:12:54,813] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:03,740] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:04,811] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,147] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-28 11:13:06,147] INFO : Creating WeightedMixIterator with 1 data loaders. +rank7 [2026-10-28 11:13:06,147] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,418] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,683] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:13:06,939] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-28 11:13:06,939] INFO : Creating InterleavedIterator with 1 data loaders. +rank7 [2026-10-28 11:13:21,356] INFO : Got dataloaders +rank7 [2026-10-28 11:13:21,356] INFO : loader split=train rank=7 len=21112 +rank7 [2026-10-28 11:13:21,356] INFO : loader split=valid rank=7 len=12 +rank7 [2026-10-28 11:13:21,356] INFO : Starting training from step 1 to 50000 on rank 7 of 8 +rank7 [2026-10-28 11:13:21,356] INFO : Creating train_iterator +rank7 [2026-10-28 11:13:21,356] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-28 11:13:36,129] INFO : Created train_iterator +rank7 [2026-10-28 11:18:11,208] INFO : Calibrated fixed task weights after 100 steps (800 rank-batches); they will apply from the next training step: losses={'Classification_binary': 0.5408100029081107, 'Classification': 0.8863177470117808, 'Regression_pt': 0.9386234817653895, 'Regression_eta': 0.019238746587652712, 'Regression_sin_phi': 0.006576384851359762, 'Regression_cos_phi': 0.006405904629500583, 'Regression_energy': 1.0145655398070812}, weights={'Classification_binary': 1.0, 'Classification': 0.6101762056350708, 'Regression_pt': 0.5761735439300537, 'Regression_eta': 28.110458374023438, 'Regression_sin_phi': 82.23515319824219, 'Regression_cos_phi': 84.42367553710938, 'Regression_energy': 0.5330458879470825} +rank7 [2026-10-28 11:18:13,719] INFO : Step 100/50000 rank7 | Train Loss: 4.2631 | Train Loss Unweighted: 2.7366 | Train Loss Components: Auxiliary: 0.7941 | Cardinality: 0.3942 | Classification: 0.6312 | Classification_binary: 0.4549 | HitGrouping: 0.3381 | Regression_cos_phi: 0.0047 | Regression_energy: 0.8263 | Regression_eta: 0.0122 | Regression_pt: 0.8026 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.6048 | Classification_binary: 0.4366 | Regression_cos_phi: 0.0046 | Regression_energy: 0.7030 | Regression_eta: 0.0128 | Regression_pt: 0.6732 | Regression_sin_phi: 0.0046 | LR: 1.00e-03 | DataLoad Time: 0.0167s | Model Forward Time: 3.5373s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.6509 | max_loss: 0.1961 | max_backward: 1.4166 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.4885 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0051 | Memory: rss_MB: 4769.1 | uss_MB: 3475.5 | child_rss_MB: 6372.1 | allocated_MB: 446.7 | reserved_MB: 24804.0 | inactive_split_MB: 1063.6 +rank7 [2026-10-28 11:22:42,536] INFO : Step 200/50000 rank7 | Train Loss: 4.2029 | Train Loss Unweighted: 2.5447 | Train Loss Components: Auxiliary: 0.8074 | Cardinality: 0.4500 | Classification: 0.5373 | Classification_binary: 0.5135 | HitGrouping: 0.3101 | Regression_cos_phi: 0.0042 | Regression_energy: 0.7508 | Regression_eta: 0.0093 | Regression_pt: 0.7252 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.3441 | Classification_binary: 0.4559 | Regression_cos_phi: 0.2486 | Regression_energy: 0.4784 | Regression_eta: 0.2465 | Regression_pt: 0.5109 | Regression_sin_phi: 0.3244 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 4.6616s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.9214 | max_loss: 0.1502 | max_backward: 2.1220 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0947 | Memory: rss_MB: 4772.0 | uss_MB: 3480.3 | child_rss_MB: 6374.7 | allocated_MB: 449.0 | reserved_MB: 26252.0 | inactive_split_MB: 1054.4 +rank7 [2026-10-28 11:27:10,389] INFO : Step 300/50000 rank7 | Train Loss: 3.5114 | Train Loss Unweighted: 2.0975 | Train Loss Components: Auxiliary: 0.6791 | Cardinality: 0.3510 | Classification: 0.4201 | Classification_binary: 0.4975 | HitGrouping: 0.2993 | Regression_cos_phi: 0.0033 | Regression_energy: 0.5752 | Regression_eta: 0.0073 | Regression_pt: 0.5905 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2923 | Classification_binary: 0.5482 | Regression_cos_phi: 0.3473 | Regression_energy: 0.3264 | Regression_eta: 0.2113 | Regression_pt: 0.3613 | Regression_sin_phi: 0.3280 | LR: 1.00e-03 | DataLoad Time: 0.0150s | Model Forward Time: 4.2603s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0064 | max_forward: 0.7369 | max_loss: 0.1526 | max_backward: 1.7330 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.9030 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 4773.0 | uss_MB: 3484.3 | child_rss_MB: 6364.6 | allocated_MB: 446.1 | reserved_MB: 26252.0 | inactive_split_MB: 1068.0 +rank7 [2026-10-28 11:31:37,117] INFO : Step 400/50000 rank7 | Train Loss: 3.3615 | Train Loss Unweighted: 1.9456 | Train Loss Components: Auxiliary: 0.6310 | Cardinality: 0.3555 | Classification: 0.4058 | Classification_binary: 0.4436 | HitGrouping: 0.3006 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5363 | Regression_eta: 0.0080 | Regression_pt: 0.5451 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2529 | Classification_binary: 0.4568 | Regression_cos_phi: 0.3398 | Regression_energy: 0.2837 | Regression_eta: 0.2701 | Regression_pt: 0.3113 | Regression_sin_phi: 0.2233 | LR: 1.00e-03 | DataLoad Time: 0.0174s | Model Forward Time: 4.5024s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1151 | max_forward: 0.7642 | max_loss: 0.1384 | max_backward: 1.7906 | max_optimizer_step: 0.1554 | max_tensorboard_logging: 0.9359 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0880 | Memory: rss_MB: 4771.8 | uss_MB: 3484.8 | child_rss_MB: 6366.3 | allocated_MB: 446.8 | reserved_MB: 26252.0 | inactive_split_MB: 1064.0 +rank7 [2026-10-28 11:36:08,659] INFO : Step 500/50000 rank7 | Train Loss: 3.8257 | Train Loss Unweighted: 2.0630 | Train Loss Components: Auxiliary: 0.7010 | Cardinality: 0.4740 | Classification: 0.4380 | Classification_binary: 0.4817 | HitGrouping: 0.2961 | Regression_cos_phi: 0.0044 | Regression_energy: 0.5611 | Regression_eta: 0.0087 | Regression_pt: 0.5645 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.3134 | Classification_binary: 0.4450 | Regression_cos_phi: 0.3882 | Regression_energy: 0.2866 | Regression_eta: 0.2917 | Regression_pt: 0.3137 | Regression_sin_phi: 0.3317 | LR: 1.00e-03 | DataLoad Time: 0.0157s | Model Forward Time: 5.1259s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0077 | max_forward: 1.2433 | max_loss: 0.1536 | max_backward: 2.4885 | max_optimizer_step: 0.1562 | max_tensorboard_logging: 1.0374 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0034 | Memory: rss_MB: 4768.9 | uss_MB: 3486.1 | child_rss_MB: 6361.5 | allocated_MB: 449.6 | reserved_MB: 26256.0 | inactive_split_MB: 929.2 +rank7 [2026-10-28 11:40:42,093] INFO : Step 600/50000 rank7 | Train Loss: 3.4702 | Train Loss Unweighted: 1.9733 | Train Loss Components: Auxiliary: 0.6509 | Cardinality: 0.3585 | Classification: 0.3825 | Classification_binary: 0.5061 | HitGrouping: 0.2915 | Regression_cos_phi: 0.0038 | Regression_energy: 0.5321 | Regression_eta: 0.0074 | Regression_pt: 0.5377 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2451 | Classification_binary: 0.4982 | Regression_cos_phi: 0.3049 | Regression_energy: 0.3034 | Regression_eta: 0.2100 | Regression_pt: 0.3300 | Regression_sin_phi: 0.3290 | LR: 1.00e-03 | DataLoad Time: 0.0140s | Model Forward Time: 4.5593s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0064 | max_forward: 0.8098 | max_loss: 0.1544 | max_backward: 1.9541 | max_optimizer_step: 0.1578 | max_tensorboard_logging: 0.9377 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0146 | Memory: rss_MB: 4772.7 | uss_MB: 3486.1 | child_rss_MB: 6366.7 | allocated_MB: 449.5 | reserved_MB: 27396.0 | inactive_split_MB: 1022.5 +rank7 [2026-10-28 11:45:16,604] INFO : Step 700/50000 rank7 | Train Loss: 2.9693 | Train Loss Unweighted: 1.8072 | Train Loss Components: Auxiliary: 0.5907 | Cardinality: 0.2768 | Classification: 0.3175 | Classification_binary: 0.3969 | HitGrouping: 0.2848 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5321 | Regression_eta: 0.0061 | Regression_pt: 0.5492 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1997 | Classification_binary: 0.4420 | Regression_cos_phi: 0.2756 | Regression_energy: 0.3160 | Regression_eta: 0.1973 | Regression_pt: 0.3589 | Regression_sin_phi: 0.2516 | LR: 1.00e-03 | DataLoad Time: 0.0214s | Model Forward Time: 3.9458s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7910 | max_loss: 0.2436 | max_backward: 1.8672 | max_optimizer_step: 0.1576 | max_tensorboard_logging: 0.4737 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0022 | Memory: rss_MB: 4773.8 | uss_MB: 3484.3 | child_rss_MB: 6369.5 | allocated_MB: 446.1 | reserved_MB: 27396.0 | inactive_split_MB: 1066.2 +rank7 [2026-10-28 11:49:48,175] INFO : Step 800/50000 rank7 | Train Loss: 3.1527 | Train Loss Unweighted: 1.8424 | Train Loss Components: Auxiliary: 0.6214 | Cardinality: 0.3518 | Classification: 0.3758 | Classification_binary: 0.3967 | HitGrouping: 0.2962 | Regression_cos_phi: 0.0028 | Regression_energy: 0.5290 | Regression_eta: 0.0063 | Regression_pt: 0.5287 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2375 | Classification_binary: 0.4154 | Regression_cos_phi: 0.2911 | Regression_energy: 0.2980 | Regression_eta: 0.1971 | Regression_pt: 0.3214 | Regression_sin_phi: 0.2675 | LR: 9.99e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6435s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0093 | max_forward: 0.8507 | max_loss: 0.1493 | max_backward: 1.9706 | max_optimizer_step: 0.1582 | max_tensorboard_logging: 1.0840 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0072 | Memory: rss_MB: 4754.0 | uss_MB: 3484.3 | child_rss_MB: 6293.3 | allocated_MB: 450.9 | reserved_MB: 27396.0 | inactive_split_MB: 920.9 +rank7 [2026-10-28 11:54:30,570] INFO : Step 900/50000 rank7 | Train Loss: 3.2552 | Train Loss Unweighted: 1.9216 | Train Loss Components: Auxiliary: 0.6208 | Cardinality: 0.2901 | Classification: 0.3818 | Classification_binary: 0.4734 | HitGrouping: 0.2881 | Regression_cos_phi: 0.0034 | Regression_energy: 0.5239 | Regression_eta: 0.0075 | Regression_pt: 0.5284 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2058 | Classification_binary: 0.4088 | Regression_cos_phi: 0.2284 | Regression_energy: 0.2547 | Regression_eta: 0.1769 | Regression_pt: 0.2809 | Regression_sin_phi: 0.2741 | LR: 9.99e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6734s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8050 | max_loss: 0.1486 | max_backward: 1.9075 | max_optimizer_step: 0.1563 | max_tensorboard_logging: 0.9729 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0029 | Memory: rss_MB: 4751.7 | uss_MB: 3485.4 | child_rss_MB: 6307.4 | allocated_MB: 449.7 | reserved_MB: 27406.0 | inactive_split_MB: 927.3 +rank7 [2026-10-28 11:59:03,649] INFO : Step 1000/50000 rank7 | Train Loss: 2.8964 | Train Loss Unweighted: 1.7179 | Train Loss Components: Auxiliary: 0.5694 | Cardinality: 0.2509 | Classification: 0.3299 | Classification_binary: 0.4129 | HitGrouping: 0.2818 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4804 | Regression_eta: 0.0063 | Regression_pt: 0.4827 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2272 | Classification_binary: 0.4492 | Regression_cos_phi: 0.2775 | Regression_energy: 0.2414 | Regression_eta: 0.2245 | Regression_pt: 0.2638 | Regression_sin_phi: 0.3539 | LR: 9.99e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4516s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0060 | max_forward: 0.7367 | max_loss: 0.2419 | max_backward: 1.8137 | max_optimizer_step: 0.1631 | max_tensorboard_logging: 1.0408 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 4746.9 | uss_MB: 3485.7 | child_rss_MB: 6281.8 | allocated_MB: 449.1 | reserved_MB: 27406.0 | inactive_split_MB: 959.2 +rank7 [2026-10-28 11:59:04,553] INFO : Running validation on rank7 +rank7 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 11:59:04,553] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 11:59:57,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 11:59:57,577] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 12:00:26,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 12:00:27,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 12:00:28,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 12:00:29,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 12:00:39,882] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-28 12:00:40,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:00:41,019] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 12:00:57,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 12:00:58,115] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 12:00:59,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 12:00:59,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 12:01:09,978] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 12:01:10,611] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:01:11,131] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 12:01:26,375] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 12:01:27,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 12:01:28,027] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 12:01:28,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_1000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 12:01:38,870] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 12:06:47,467] INFO : Step 1100/50000 rank7 | Train Loss: 2.7351 | Train Loss Unweighted: 1.6531 | Train Loss Components: Auxiliary: 0.5524 | Cardinality: 0.2470 | Classification: 0.3238 | Classification_binary: 0.3516 | HitGrouping: 0.2744 | Regression_cos_phi: 0.0026 | Regression_energy: 0.4831 | Regression_eta: 0.0054 | Regression_pt: 0.4842 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2038 | Classification_binary: 0.3667 | Regression_cos_phi: 0.2632 | Regression_energy: 0.2149 | Regression_eta: 0.1576 | Regression_pt: 0.2305 | Regression_sin_phi: 0.2210 | LR: 9.99e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.7312s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.7345 | max_loss: 0.2408 | max_backward: 1.8906 | max_optimizer_step: 0.1838 | max_tensorboard_logging: 1.1618 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0231 | Memory: rss_MB: 6205.6 | uss_MB: 4949.9 | child_rss_MB: 6308.4 | allocated_MB: 448.2 | reserved_MB: 24590.0 | inactive_split_MB: 1051.2 +rank7 [2026-10-28 12:11:28,614] INFO : Step 1200/50000 rank7 | Train Loss: 2.9931 | Train Loss Unweighted: 1.7517 | Train Loss Components: Auxiliary: 0.5886 | Cardinality: 0.2847 | Classification: 0.3460 | Classification_binary: 0.4151 | HitGrouping: 0.2795 | Regression_cos_phi: 0.0030 | Regression_energy: 0.4874 | Regression_eta: 0.0065 | Regression_pt: 0.4908 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1973 | Classification_binary: 0.3942 | Regression_cos_phi: 0.2054 | Regression_energy: 0.3026 | Regression_eta: 0.1504 | Regression_pt: 0.3320 | Regression_sin_phi: 0.1616 | LR: 9.99e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.9395s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0061 | max_forward: 0.9910 | max_loss: 0.1662 | max_backward: 2.3256 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.8355 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 6205.6 | uss_MB: 4943.2 | child_rss_MB: 6333.1 | allocated_MB: 449.6 | reserved_MB: 24590.0 | inactive_split_MB: 1155.8 +rank7 [2026-10-28 12:16:08,306] INFO : Step 1300/50000 rank7 | Train Loss: 3.0304 | Train Loss Unweighted: 1.7607 | Train Loss Components: Auxiliary: 0.5900 | Cardinality: 0.2909 | Classification: 0.3397 | Classification_binary: 0.4400 | HitGrouping: 0.2683 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4807 | Regression_eta: 0.0062 | Regression_pt: 0.4878 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1818 | Classification_binary: 0.4339 | Regression_cos_phi: 0.2146 | Regression_energy: 0.2520 | Regression_eta: 0.1498 | Regression_pt: 0.2728 | Regression_sin_phi: 0.1970 | LR: 9.98e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3303s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8206 | max_loss: 0.1504 | max_backward: 1.9515 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5688 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5595.4 | uss_MB: 4349.7 | child_rss_MB: 6286.2 | allocated_MB: 448.6 | reserved_MB: 25778.0 | inactive_split_MB: 1050.0 +rank7 [2026-10-28 12:20:48,119] INFO : Step 1400/50000 rank7 | Train Loss: 2.5732 | Train Loss Unweighted: 1.5137 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2349 | Classification: 0.2906 | Classification_binary: 0.3573 | HitGrouping: 0.2641 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4252 | Regression_eta: 0.0055 | Regression_pt: 0.4303 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2022 | Classification_binary: 0.3768 | Regression_cos_phi: 0.2136 | Regression_energy: 0.2424 | Regression_eta: 0.1637 | Regression_pt: 0.2624 | Regression_sin_phi: 0.1586 | LR: 9.98e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.1754s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.8166 | max_loss: 0.1962 | max_backward: 1.8388 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 0.4964 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0232 | Memory: rss_MB: 5588.2 | uss_MB: 3765.0 | child_rss_MB: 6275.0 | allocated_MB: 454.5 | reserved_MB: 28272.0 | inactive_split_MB: 1143.7 +rank7 [2026-10-28 12:25:37,858] INFO : Step 1500/50000 rank7 | Train Loss: 2.8564 | Train Loss Unweighted: 1.5902 | Train Loss Components: Auxiliary: 0.5576 | Cardinality: 0.3788 | Classification: 0.3167 | Classification_binary: 0.3534 | HitGrouping: 0.2644 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4517 | Regression_eta: 0.0055 | Regression_pt: 0.4576 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1583 | Classification_binary: 0.3359 | Regression_cos_phi: 0.2637 | Regression_energy: 0.2200 | Regression_eta: 0.1797 | Regression_pt: 0.2423 | Regression_sin_phi: 0.2090 | LR: 9.98e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6172s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 1.0272 | max_loss: 0.2459 | max_backward: 1.8682 | max_optimizer_step: 0.1701 | max_tensorboard_logging: 1.0883 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0105 | Memory: rss_MB: 5566.9 | uss_MB: 4329.3 | child_rss_MB: 6244.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 989.3 +rank7 [2026-10-28 12:30:20,921] INFO : Step 1600/50000 rank7 | Train Loss: 2.8293 | Train Loss Unweighted: 1.6754 | Train Loss Components: Auxiliary: 0.5537 | Cardinality: 0.2456 | Classification: 0.3224 | Classification_binary: 0.4168 | HitGrouping: 0.2726 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4572 | Regression_eta: 0.0058 | Regression_pt: 0.4676 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2208 | Classification_binary: 0.4218 | Regression_cos_phi: 0.3048 | Regression_energy: 0.2406 | Regression_eta: 0.1987 | Regression_pt: 0.2667 | Regression_sin_phi: 0.2586 | LR: 9.98e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5695s | Phase Times: max_pre_step_barrier: 0.1094 | max_batch_to_device: 0.0066 | max_forward: 0.7720 | max_loss: 0.1510 | max_backward: 1.7462 | max_optimizer_step: 0.1689 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0118 | Memory: rss_MB: 5578.6 | uss_MB: 4348.9 | child_rss_MB: 6226.8 | allocated_MB: 448.1 | reserved_MB: 28272.0 | inactive_split_MB: 989.9 +rank7 [2026-10-28 12:35:04,110] INFO : Step 1700/50000 rank7 | Train Loss: 2.7295 | Train Loss Unweighted: 1.5885 | Train Loss Components: Auxiliary: 0.5261 | Cardinality: 0.2505 | Classification: 0.2942 | Classification_binary: 0.3852 | HitGrouping: 0.2668 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4488 | Regression_eta: 0.0059 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1682 | Classification_binary: 0.3902 | Regression_cos_phi: 0.2999 | Regression_energy: 0.2562 | Regression_eta: 0.1938 | Regression_pt: 0.2778 | Regression_sin_phi: 0.2594 | LR: 9.97e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9360s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0062 | max_forward: 1.0304 | max_loss: 0.4846 | max_backward: 1.9957 | max_optimizer_step: 0.1726 | max_tensorboard_logging: 1.0278 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0185 | Memory: rss_MB: 5571.8 | uss_MB: 4349.5 | child_rss_MB: 6226.2 | allocated_MB: 446.9 | reserved_MB: 28272.0 | inactive_split_MB: 999.4 +rank7 [2026-10-28 12:39:46,514] INFO : Step 1800/50000 rank7 | Train Loss: 2.5541 | Train Loss Unweighted: 1.4989 | Train Loss Components: Auxiliary: 0.5124 | Cardinality: 0.2504 | Classification: 0.2799 | Classification_binary: 0.3357 | HitGrouping: 0.2658 | Regression_cos_phi: 0.0024 | Regression_energy: 0.4350 | Regression_eta: 0.0052 | Regression_pt: 0.4384 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1609 | Classification_binary: 0.3532 | Regression_cos_phi: 0.1897 | Regression_energy: 0.2321 | Regression_eta: 0.1637 | Regression_pt: 0.2540 | Regression_sin_phi: 0.2108 | LR: 9.97e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3586s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7316 | max_loss: 0.1436 | max_backward: 1.6570 | max_optimizer_step: 0.2625 | max_tensorboard_logging: 0.9220 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5563.3 | uss_MB: 4348.3 | child_rss_MB: 6170.2 | allocated_MB: 444.7 | reserved_MB: 28272.0 | inactive_split_MB: 991.9 +rank7 [2026-10-28 12:44:29,073] INFO : Step 1900/50000 rank7 | Train Loss: 2.7201 | Train Loss Unweighted: 1.5780 | Train Loss Components: Auxiliary: 0.5257 | Cardinality: 0.2710 | Classification: 0.3152 | Classification_binary: 0.3657 | HitGrouping: 0.2653 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4371 | Regression_eta: 0.0058 | Regression_pt: 0.4488 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2009 | Classification_binary: 0.3481 | Regression_cos_phi: 0.1846 | Regression_energy: 0.2145 | Regression_eta: 0.1582 | Regression_pt: 0.2345 | Regression_sin_phi: 0.2243 | LR: 9.97e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.4041s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0072 | max_forward: 0.7723 | max_loss: 0.1448 | max_backward: 2.0316 | max_optimizer_step: 0.1665 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5561.2 | uss_MB: 4335.4 | child_rss_MB: 6153.7 | allocated_MB: 448.9 | reserved_MB: 28272.0 | inactive_split_MB: 1166.2 +rank7 [2026-10-28 12:49:11,972] INFO : Step 2000/50000 rank7 | Train Loss: 2.7893 | Train Loss Unweighted: 1.5772 | Train Loss Components: Auxiliary: 0.5342 | Cardinality: 0.3149 | Classification: 0.3056 | Classification_binary: 0.4233 | HitGrouping: 0.2623 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4169 | Regression_eta: 0.0058 | Regression_pt: 0.4204 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2121 | Classification_binary: 0.4546 | Regression_cos_phi: 0.2483 | Regression_energy: 0.2232 | Regression_eta: 0.1600 | Regression_pt: 0.2435 | Regression_sin_phi: 0.2092 | LR: 9.96e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.5206s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0070 | max_forward: 1.0424 | max_loss: 0.2160 | max_backward: 2.3184 | max_optimizer_step: 0.1761 | max_tensorboard_logging: 0.9849 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0132 | Memory: rss_MB: 5555.4 | uss_MB: 4316.1 | child_rss_MB: 6148.5 | allocated_MB: 451.3 | reserved_MB: 28272.0 | inactive_split_MB: 992.4 +rank7 [2026-10-28 12:49:12,926] INFO : Running validation on rank7 +rank7 [2026-10-28 12:49:12,927] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 12:49:12,927] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 12:50:07,789] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:50:08,359] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 12:50:34,236] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 12:50:35,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 12:50:36,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 12:50:37,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 12:50:47,296] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 12:50:48,335] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:50:48,899] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 12:51:04,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 12:51:05,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 12:51:06,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 12:51:07,765] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 12:51:17,774] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 12:51:18,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 12:51:18,824] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 12:51:34,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 12:51:35,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 12:51:36,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 12:51:37,207] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_2000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 12:51:47,217] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 12:57:01,908] INFO : Step 2100/50000 rank7 | Train Loss: 2.3990 | Train Loss Unweighted: 1.4049 | Train Loss Components: Auxiliary: 0.4671 | Cardinality: 0.2199 | Classification: 0.2491 | Classification_binary: 0.3399 | HitGrouping: 0.2605 | Regression_cos_phi: 0.0022 | Regression_energy: 0.4015 | Regression_eta: 0.0046 | Regression_pt: 0.4053 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1453 | Classification_binary: 0.3766 | Regression_cos_phi: 0.2019 | Regression_energy: 0.2154 | Regression_eta: 0.1499 | Regression_pt: 0.2330 | Regression_sin_phi: 0.2106 | LR: 9.96e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8141s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0063 | max_forward: 0.7238 | max_loss: 0.1490 | max_backward: 1.8098 | max_optimizer_step: 0.1850 | max_tensorboard_logging: 1.2800 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0253 | Memory: rss_MB: 6608.5 | uss_MB: 5401.4 | child_rss_MB: 6187.1 | allocated_MB: 444.8 | reserved_MB: 23654.0 | inactive_split_MB: 964.6 +rank7 [2026-10-28 13:01:41,080] INFO : Step 2200/50000 rank7 | Train Loss: 2.6105 | Train Loss Unweighted: 1.4907 | Train Loss Components: Auxiliary: 0.5017 | Cardinality: 0.2489 | Classification: 0.2976 | Classification_binary: 0.3552 | HitGrouping: 0.2519 | Regression_cos_phi: 0.0028 | Regression_energy: 0.4108 | Regression_eta: 0.0054 | Regression_pt: 0.4161 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1864 | Classification_binary: 0.3427 | Regression_cos_phi: 0.1911 | Regression_energy: 0.2545 | Regression_eta: 0.1421 | Regression_pt: 0.2773 | Regression_sin_phi: 0.1854 | LR: 9.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.9415s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 1.0006 | max_loss: 0.1446 | max_backward: 1.7671 | max_optimizer_step: 0.2923 | max_tensorboard_logging: 1.2733 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6609.6 | uss_MB: 5403.3 | child_rss_MB: 6178.9 | allocated_MB: 445.7 | reserved_MB: 24780.0 | inactive_split_MB: 1132.5 +rank7 [2026-10-28 13:06:12,757] INFO : Step 2300/50000 rank7 | Train Loss: 2.7462 | Train Loss Unweighted: 1.5643 | Train Loss Components: Auxiliary: 0.5175 | Cardinality: 0.2724 | Classification: 0.3064 | Classification_binary: 0.4106 | HitGrouping: 0.2546 | Regression_cos_phi: 0.0031 | Regression_energy: 0.4145 | Regression_eta: 0.0056 | Regression_pt: 0.4213 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1571 | Classification_binary: 0.3520 | Regression_cos_phi: 0.1730 | Regression_energy: 0.2307 | Regression_eta: 0.0900 | Regression_pt: 0.2555 | Regression_sin_phi: 0.1885 | LR: 9.95e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.7156s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0353 | max_forward: 0.7930 | max_loss: 0.1572 | max_backward: 1.9535 | max_optimizer_step: 0.1750 | max_tensorboard_logging: 1.0414 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0140 | Memory: rss_MB: 5702.4 | uss_MB: 4498.0 | child_rss_MB: 6178.9 | allocated_MB: 451.2 | reserved_MB: 26626.0 | inactive_split_MB: 1036.8 +rank7 [2026-10-28 13:10:49,217] INFO : Step 2400/50000 rank7 | Train Loss: 2.5721 | Train Loss Unweighted: 1.4496 | Train Loss Components: Auxiliary: 0.4832 | Cardinality: 0.2667 | Classification: 0.2677 | Classification_binary: 0.3686 | HitGrouping: 0.2521 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3975 | Regression_eta: 0.0053 | Regression_pt: 0.4051 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1883 | Classification_binary: 0.4153 | Regression_cos_phi: 0.2560 | Regression_energy: 0.2207 | Regression_eta: 0.1698 | Regression_pt: 0.2424 | Regression_sin_phi: 0.2355 | LR: 9.95e-04 | DataLoad Time: 0.0184s | Model Forward Time: 4.0743s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0063 | max_forward: 0.7367 | max_loss: 0.1564 | max_backward: 1.6902 | max_optimizer_step: 0.2413 | max_tensorboard_logging: 0.5717 | max_tensorboard_flush: 0.0086 | max_post_step_barrier: 0.0270 | Memory: rss_MB: 5703.1 | uss_MB: 4496.2 | child_rss_MB: 6167.5 | allocated_MB: 448.4 | reserved_MB: 26626.0 | inactive_split_MB: 1138.1 +rank7 [2026-10-28 13:15:33,529] INFO : Step 2500/50000 rank7 | Train Loss: 2.3709 | Train Loss Unweighted: 1.3478 | Train Loss Components: Auxiliary: 0.4634 | Cardinality: 0.2742 | Classification: 0.2528 | Classification_binary: 0.3100 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3861 | Regression_eta: 0.0042 | Regression_pt: 0.3903 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.2039 | Classification_binary: 0.3043 | Regression_cos_phi: 0.2391 | Regression_energy: 0.2123 | Regression_eta: 0.1285 | Regression_pt: 0.2298 | Regression_sin_phi: 0.1929 | LR: 9.94e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.3390s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0059 | max_forward: 0.7598 | max_loss: 0.2442 | max_backward: 1.9617 | max_optimizer_step: 0.1778 | max_tensorboard_logging: 0.5783 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5689.7 | uss_MB: 4483.6 | child_rss_MB: 6173.2 | allocated_MB: 447.2 | reserved_MB: 27052.0 | inactive_split_MB: 1262.0 +rank7 [2026-10-28 13:20:26,044] INFO : Step 2600/50000 rank7 | Train Loss: 2.6216 | Train Loss Unweighted: 1.5111 | Train Loss Components: Auxiliary: 0.4988 | Cardinality: 0.2823 | Classification: 0.2863 | Classification_binary: 0.3723 | HitGrouping: 0.2550 | Regression_cos_phi: 0.0027 | Regression_energy: 0.4174 | Regression_eta: 0.0052 | Regression_pt: 0.4247 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1478 | Classification_binary: 0.3420 | Regression_cos_phi: 0.1436 | Regression_energy: 0.1861 | Regression_eta: 0.1334 | Regression_pt: 0.2068 | Regression_sin_phi: 0.1444 | LR: 9.94e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2862s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0069 | max_forward: 0.8125 | max_loss: 0.1494 | max_backward: 1.9790 | max_optimizer_step: 0.1938 | max_tensorboard_logging: 0.5826 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0149 | Memory: rss_MB: 5698.8 | uss_MB: 4500.2 | child_rss_MB: 6171.1 | allocated_MB: 448.2 | reserved_MB: 28734.0 | inactive_split_MB: 1300.2 +rank7 [2026-10-28 13:25:15,883] INFO : Step 2700/50000 rank7 | Train Loss: 2.5735 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4801 | Cardinality: 0.2457 | Classification: 0.2692 | Classification_binary: 0.4021 | HitGrouping: 0.2514 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3948 | Regression_eta: 0.0060 | Regression_pt: 0.3979 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1621 | Classification_binary: 0.3776 | Regression_cos_phi: 0.1749 | Regression_energy: 0.2094 | Regression_eta: 0.1656 | Regression_pt: 0.2273 | Regression_sin_phi: 0.1694 | LR: 9.94e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.3203s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0070 | max_forward: 0.8435 | max_loss: 0.1556 | max_backward: 1.9409 | max_optimizer_step: 0.1773 | max_tensorboard_logging: 0.5512 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5695.7 | uss_MB: 4500.5 | child_rss_MB: 6182.0 | allocated_MB: 447.4 | reserved_MB: 28734.0 | inactive_split_MB: 980.7 +rank7 [2026-10-28 13:29:49,852] INFO : Step 2800/50000 rank7 | Train Loss: 2.3517 | Train Loss Unweighted: 1.3315 | Train Loss Components: Auxiliary: 0.4396 | Cardinality: 0.2218 | Classification: 0.2269 | Classification_binary: 0.3206 | HitGrouping: 0.2464 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3829 | Regression_eta: 0.0052 | Regression_pt: 0.3909 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1427 | Classification_binary: 0.3449 | Regression_cos_phi: 0.1718 | Regression_energy: 0.1987 | Regression_eta: 0.1429 | Regression_pt: 0.2167 | Regression_sin_phi: 0.2549 | LR: 9.93e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.3210s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0062 | max_forward: 0.8898 | max_loss: 0.1952 | max_backward: 1.8882 | max_optimizer_step: 0.1796 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0163 | Memory: rss_MB: 5687.7 | uss_MB: 4493.8 | child_rss_MB: 6179.6 | allocated_MB: 445.9 | reserved_MB: 28734.0 | inactive_split_MB: 1208.5 +rank7 [2026-10-28 13:34:34,297] INFO : Step 2900/50000 rank7 | Train Loss: 2.4505 | Train Loss Unweighted: 1.3674 | Train Loss Components: Auxiliary: 0.4603 | Cardinality: 0.2746 | Classification: 0.2611 | Classification_binary: 0.3388 | HitGrouping: 0.2505 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3750 | Regression_eta: 0.0049 | Regression_pt: 0.3827 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1399 | Classification_binary: 0.3328 | Regression_cos_phi: 0.2038 | Regression_energy: 0.2186 | Regression_eta: 0.1188 | Regression_pt: 0.2381 | Regression_sin_phi: 0.1746 | LR: 9.93e-04 | DataLoad Time: 0.0178s | Model Forward Time: 6.3927s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7484 | max_loss: 0.6089 | max_backward: 2.2353 | max_optimizer_step: 0.1775 | max_tensorboard_logging: 2.4313 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0155 | Memory: rss_MB: 5691.8 | uss_MB: 4497.3 | child_rss_MB: 6175.9 | allocated_MB: 446.7 | reserved_MB: 28734.0 | inactive_split_MB: 1133.2 +rank7 [2026-10-28 13:39:22,258] INFO : Step 3000/50000 rank7 | Train Loss: 2.6179 | Train Loss Unweighted: 1.4750 | Train Loss Components: Auxiliary: 0.4760 | Cardinality: 0.2600 | Classification: 0.3170 | Classification_binary: 0.3907 | HitGrouping: 0.2509 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3743 | Regression_eta: 0.0054 | Regression_pt: 0.3819 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1942 | Classification_binary: 0.3553 | Regression_cos_phi: 0.2088 | Regression_energy: 0.1876 | Regression_eta: 0.1280 | Regression_pt: 0.2018 | Regression_sin_phi: 0.2027 | LR: 9.92e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8197s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0842 | max_forward: 0.8203 | max_loss: 0.1511 | max_backward: 1.9714 | max_optimizer_step: 0.1743 | max_tensorboard_logging: 1.0307 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0125 | Memory: rss_MB: 5691.4 | uss_MB: 4499.3 | child_rss_MB: 6154.8 | allocated_MB: 451.4 | reserved_MB: 28740.0 | inactive_split_MB: 1256.0 +rank7 [2026-10-28 13:39:23,129] INFO : Running validation on rank7 +rank7 [2026-10-28 13:39:23,129] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 13:39:23,129] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 13:40:15,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:40:16,491] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 13:40:43,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 13:40:44,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 13:40:45,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 13:40:46,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 13:40:56,910] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 13:40:57,942] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:40:58,515] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 13:41:14,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 13:41:15,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 13:41:16,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 13:41:17,275] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 13:41:27,285] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 13:41:28,083] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 13:41:28,597] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 13:41:44,281] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 13:41:45,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 13:41:45,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 13:41:46,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_3000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 13:41:56,797] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 13:46:51,408] INFO : Step 3100/50000 rank7 | Train Loss: 2.3456 | Train Loss Unweighted: 1.3820 | Train Loss Components: Auxiliary: 0.4397 | Cardinality: 0.2017 | Classification: 0.2549 | Classification_binary: 0.3432 | HitGrouping: 0.2399 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3852 | Regression_eta: 0.0046 | Regression_pt: 0.3892 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1889 | Classification_binary: 0.3821 | Regression_cos_phi: 0.3091 | Regression_energy: 0.2338 | Regression_eta: 0.1510 | Regression_pt: 0.2552 | Regression_sin_phi: 0.2927 | LR: 9.91e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.4920s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0425 | max_forward: 0.7824 | max_loss: 0.1439 | max_backward: 1.7183 | max_optimizer_step: 0.1970 | max_tensorboard_logging: 0.8666 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6905.9 | uss_MB: 5708.9 | child_rss_MB: 6176.0 | allocated_MB: 447.6 | reserved_MB: 24468.0 | inactive_split_MB: 1269.5 +rank7 [2026-10-28 13:51:36,981] INFO : Step 3200/50000 rank7 | Train Loss: 2.2453 | Train Loss Unweighted: 1.2729 | Train Loss Components: Auxiliary: 0.4171 | Cardinality: 0.2143 | Classification: 0.2366 | Classification_binary: 0.3116 | HitGrouping: 0.2410 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3569 | Regression_eta: 0.0047 | Regression_pt: 0.3584 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1268 | Classification_binary: 0.3024 | Regression_cos_phi: 0.1852 | Regression_energy: 0.1650 | Regression_eta: 0.1177 | Regression_pt: 0.1812 | Regression_sin_phi: 0.1261 | LR: 9.91e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6657s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7985 | max_loss: 0.1379 | max_backward: 1.7895 | max_optimizer_step: 0.2027 | max_tensorboard_logging: 0.8903 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6908.5 | uss_MB: 5710.9 | child_rss_MB: 6167.1 | allocated_MB: 448.6 | reserved_MB: 26448.0 | inactive_split_MB: 1391.4 +rank7 [2026-10-28 13:56:16,619] INFO : Step 3300/50000 rank7 | Train Loss: 2.5474 | Train Loss Unweighted: 1.4472 | Train Loss Components: Auxiliary: 0.4779 | Cardinality: 0.2732 | Classification: 0.2824 | Classification_binary: 0.3640 | HitGrouping: 0.2475 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3911 | Regression_eta: 0.0054 | Regression_pt: 0.3992 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1738 | Classification_binary: 0.3358 | Regression_cos_phi: 0.2256 | Regression_energy: 0.1961 | Regression_eta: 0.1348 | Regression_pt: 0.2169 | Regression_sin_phi: 0.1900 | LR: 9.90e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.6659s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8323 | max_loss: 0.1479 | max_backward: 2.0543 | max_optimizer_step: 0.1987 | max_tensorboard_logging: 0.7834 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0205 | Memory: rss_MB: 5785.1 | uss_MB: 4587.8 | child_rss_MB: 6143.0 | allocated_MB: 448.7 | reserved_MB: 26460.0 | inactive_split_MB: 1087.4 +rank7 [2026-10-28 14:00:49,247] INFO : Step 3400/50000 rank7 | Train Loss: 2.5743 | Train Loss Unweighted: 1.4358 | Train Loss Components: Auxiliary: 0.4720 | Cardinality: 0.2664 | Classification: 0.2699 | Classification_binary: 0.3915 | HitGrouping: 0.2461 | Regression_cos_phi: 0.0029 | Regression_energy: 0.3784 | Regression_eta: 0.0053 | Regression_pt: 0.3851 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1639 | Classification_binary: 0.3905 | Regression_cos_phi: 0.2505 | Regression_energy: 0.2002 | Regression_eta: 0.1467 | Regression_pt: 0.2188 | Regression_sin_phi: 0.1993 | LR: 9.90e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.2606s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7827 | max_loss: 0.1630 | max_backward: 1.9191 | max_optimizer_step: 0.1830 | max_tensorboard_logging: 0.5606 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0174 | Memory: rss_MB: 5794.1 | uss_MB: 4597.7 | child_rss_MB: 6161.5 | allocated_MB: 450.4 | reserved_MB: 28138.0 | inactive_split_MB: 1261.0 +rank7 [2026-10-28 14:05:23,603] INFO : Step 3500/50000 rank7 | Train Loss: 2.1916 | Train Loss Unweighted: 1.2418 | Train Loss Components: Auxiliary: 0.4069 | Cardinality: 0.2204 | Classification: 0.2212 | Classification_binary: 0.3041 | HitGrouping: 0.2426 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3517 | Regression_eta: 0.0047 | Regression_pt: 0.3558 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1770 | Classification_binary: 0.3727 | Regression_cos_phi: 0.2318 | Regression_energy: 0.2099 | Regression_eta: 0.1699 | Regression_pt: 0.2318 | Regression_sin_phi: 0.2587 | LR: 9.89e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.2266s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0062 | max_forward: 0.7972 | max_loss: 0.1441 | max_backward: 1.7398 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.6845 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5780.0 | uss_MB: 4584.6 | child_rss_MB: 6167.3 | allocated_MB: 443.7 | reserved_MB: 28532.0 | inactive_split_MB: 1176.3 +rank7 [2026-10-28 14:10:06,691] INFO : Step 3600/50000 rank7 | Train Loss: 2.3195 | Train Loss Unweighted: 1.3024 | Train Loss Components: Auxiliary: 0.4248 | Cardinality: 0.2221 | Classification: 0.2518 | Classification_binary: 0.3172 | HitGrouping: 0.2417 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3610 | Regression_eta: 0.0052 | Regression_pt: 0.3623 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1663 | Classification_binary: 0.3187 | Regression_cos_phi: 0.2643 | Regression_energy: 0.1915 | Regression_eta: 0.1693 | Regression_pt: 0.2070 | Regression_sin_phi: 0.2687 | LR: 9.89e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3066s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8456 | max_loss: 0.1388 | max_backward: 1.9466 | max_optimizer_step: 0.1811 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5787.8 | uss_MB: 4598.8 | child_rss_MB: 6163.2 | allocated_MB: 449.1 | reserved_MB: 28532.0 | inactive_split_MB: 1277.8 +rank7 [2026-10-28 14:14:47,797] INFO : Step 3700/50000 rank7 | Train Loss: 2.5861 | Train Loss Unweighted: 1.4427 | Train Loss Components: Auxiliary: 0.4700 | Cardinality: 0.2678 | Classification: 0.2978 | Classification_binary: 0.3844 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3719 | Regression_eta: 0.0058 | Regression_pt: 0.3773 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1941 | Classification_binary: 0.3352 | Regression_cos_phi: 0.2484 | Regression_energy: 0.2003 | Regression_eta: 0.1624 | Regression_pt: 0.2189 | Regression_sin_phi: 0.2052 | LR: 9.88e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5011s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 0.8001 | max_loss: 0.1519 | max_backward: 1.8239 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.7916 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0181 | Memory: rss_MB: 5775.1 | uss_MB: 4600.5 | child_rss_MB: 6165.4 | allocated_MB: 450.6 | reserved_MB: 30118.0 | inactive_split_MB: 1097.6 +rank7 [2026-10-28 14:19:24,598] INFO : Step 3800/50000 rank7 | Train Loss: 2.3269 | Train Loss Unweighted: 1.3016 | Train Loss Components: Auxiliary: 0.4225 | Cardinality: 0.2364 | Classification: 0.2400 | Classification_binary: 0.3389 | HitGrouping: 0.2359 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3534 | Regression_eta: 0.0054 | Regression_pt: 0.3591 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1571 | Classification_binary: 0.3572 | Regression_cos_phi: 0.2277 | Regression_energy: 0.2123 | Regression_eta: 0.1207 | Regression_pt: 0.2356 | Regression_sin_phi: 0.2230 | LR: 9.87e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.4704s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7555 | max_loss: 0.1601 | max_backward: 1.7612 | max_optimizer_step: 0.1817 | max_tensorboard_logging: 0.9394 | max_tensorboard_flush: 0.0120 | max_post_step_barrier: 0.0215 | Memory: rss_MB: 5772.0 | uss_MB: 4595.7 | child_rss_MB: 6158.5 | allocated_MB: 447.4 | reserved_MB: 30118.0 | inactive_split_MB: 1355.2 +rank7 [2026-10-28 14:24:05,270] INFO : Step 3900/50000 rank7 | Train Loss: 2.0276 | Train Loss Unweighted: 1.1874 | Train Loss Components: Auxiliary: 0.3861 | Cardinality: 0.1825 | Classification: 0.2116 | Classification_binary: 0.2707 | HitGrouping: 0.2342 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3485 | Regression_eta: 0.0038 | Regression_pt: 0.3489 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0940 | Classification_binary: 0.2398 | Regression_cos_phi: 0.1344 | Regression_energy: 0.2025 | Regression_eta: 0.0811 | Regression_pt: 0.2176 | Regression_sin_phi: 0.1280 | LR: 9.87e-04 | DataLoad Time: 0.0192s | Model Forward Time: 4.4478s | Phase Times: max_pre_step_barrier: 0.0095 | max_batch_to_device: 0.0140 | max_forward: 1.1007 | max_loss: 0.1867 | max_backward: 2.0952 | max_optimizer_step: 0.1816 | max_tensorboard_logging: 0.5091 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0198 | Memory: rss_MB: 5772.2 | uss_MB: 4598.5 | child_rss_MB: 6165.7 | allocated_MB: 448.6 | reserved_MB: 30118.0 | inactive_split_MB: 1119.4 +rank7 [2026-10-28 14:28:52,324] INFO : Step 4000/50000 rank7 | Train Loss: 2.3921 | Train Loss Unweighted: 1.3683 | Train Loss Components: Auxiliary: 0.4366 | Cardinality: 0.2254 | Classification: 0.2720 | Classification_binary: 0.3524 | HitGrouping: 0.2405 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3647 | Regression_eta: 0.0048 | Regression_pt: 0.3692 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1761 | Classification_binary: 0.3014 | Regression_cos_phi: 0.2011 | Regression_energy: 0.1992 | Regression_eta: 0.1382 | Regression_pt: 0.2155 | Regression_sin_phi: 0.1865 | LR: 9.86e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8901s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.8532 | max_loss: 0.1476 | max_backward: 2.4362 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.6138 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5773.5 | uss_MB: 4601.5 | child_rss_MB: 6164.9 | allocated_MB: 449.7 | reserved_MB: 30118.0 | inactive_split_MB: 899.8 +rank7 [2026-10-28 14:28:53,310] INFO : Running validation on rank7 +rank7 [2026-10-28 14:28:53,310] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 14:28:53,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 14:29:46,117] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:29:46,856] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 14:30:12,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 14:30:13,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 14:30:14,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 14:30:15,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 14:30:25,924] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 14:30:26,980] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:30:27,512] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 14:30:43,656] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 14:30:44,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 14:30:45,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 14:30:46,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 14:30:56,541] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 14:30:57,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 14:30:57,799] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 14:31:13,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 14:31:14,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 14:31:15,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 14:31:16,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_4000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 14:31:26,189] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 14:36:30,159] INFO : Step 4100/50000 rank7 | Train Loss: 2.5062 | Train Loss Unweighted: 1.3582 | Train Loss Components: Auxiliary: 0.4559 | Cardinality: 0.3208 | Classification: 0.2565 | Classification_binary: 0.3823 | HitGrouping: 0.2386 | Regression_cos_phi: 0.0025 | Regression_energy: 0.3543 | Regression_eta: 0.0046 | Regression_pt: 0.3553 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1551 | Classification_binary: 0.3837 | Regression_cos_phi: 0.2150 | Regression_energy: 0.1900 | Regression_eta: 0.1405 | Regression_pt: 0.2089 | Regression_sin_phi: 0.2400 | LR: 9.85e-04 | DataLoad Time: 0.0185s | Model Forward Time: 5.2054s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0069 | max_forward: 0.8943 | max_loss: 0.2518 | max_backward: 1.9865 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 1.1537 | max_tensorboard_flush: 0.0083 | max_post_step_barrier: 0.0318 | Memory: rss_MB: 6976.0 | uss_MB: 5798.7 | child_rss_MB: 6190.7 | allocated_MB: 449.4 | reserved_MB: 25148.0 | inactive_split_MB: 1028.6 +rank7 [2026-10-28 14:41:18,155] INFO : Step 4200/50000 rank7 | Train Loss: 2.0961 | Train Loss Unweighted: 1.2232 | Train Loss Components: Auxiliary: 0.3927 | Cardinality: 0.2101 | Classification: 0.2242 | Classification_binary: 0.2902 | HitGrouping: 0.2316 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3497 | Regression_eta: 0.0042 | Regression_pt: 0.3510 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1457 | Classification_binary: 0.3984 | Regression_cos_phi: 0.2036 | Regression_energy: 0.1935 | Regression_eta: 0.1332 | Regression_pt: 0.2110 | Regression_sin_phi: 0.1792 | LR: 9.84e-04 | DataLoad Time: 0.0184s | Model Forward Time: 5.6545s | Phase Times: max_pre_step_barrier: 0.1131 | max_batch_to_device: 0.0199 | max_forward: 1.5911 | max_loss: 0.3393 | max_backward: 3.0707 | max_optimizer_step: 0.1998 | max_tensorboard_logging: 0.5197 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0392 | Memory: rss_MB: 6973.9 | uss_MB: 5797.7 | child_rss_MB: 6186.7 | allocated_MB: 446.0 | reserved_MB: 25296.0 | inactive_split_MB: 922.1 +rank7 [2026-10-28 14:46:18,702] INFO : Step 4300/50000 rank7 | Train Loss: 2.2792 | Train Loss Unweighted: 1.3065 | Train Loss Components: Auxiliary: 0.4224 | Cardinality: 0.2158 | Classification: 0.2339 | Classification_binary: 0.3250 | HitGrouping: 0.2436 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3669 | Regression_eta: 0.0044 | Regression_pt: 0.3714 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1174 | Classification_binary: 0.3032 | Regression_cos_phi: 0.1783 | Regression_energy: 0.2122 | Regression_eta: 0.1094 | Regression_pt: 0.2261 | Regression_sin_phi: 0.2339 | LR: 9.84e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5464s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0938 | max_forward: 0.8451 | max_loss: 0.1396 | max_backward: 1.9669 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.6639 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5862.7 | uss_MB: 4685.4 | child_rss_MB: 6195.4 | allocated_MB: 448.4 | reserved_MB: 28580.0 | inactive_split_MB: 1346.0 +rank7 [2026-10-28 14:50:58,886] INFO : Step 4400/50000 rank7 | Train Loss: 2.4955 | Train Loss Unweighted: 1.3385 | Train Loss Components: Auxiliary: 0.4510 | Cardinality: 0.3020 | Classification: 0.2394 | Classification_binary: 0.3842 | HitGrouping: 0.2371 | Regression_cos_phi: 0.0028 | Regression_energy: 0.3482 | Regression_eta: 0.0051 | Regression_pt: 0.3563 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1389 | Classification_binary: 0.3594 | Regression_cos_phi: 0.1934 | Regression_energy: 0.1903 | Regression_eta: 0.1241 | Regression_pt: 0.2099 | Regression_sin_phi: 0.1760 | LR: 9.83e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2061s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0083 | max_forward: 0.7889 | max_loss: 0.1505 | max_backward: 1.8572 | max_optimizer_step: 0.1853 | max_tensorboard_logging: 0.4868 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0222 | Memory: rss_MB: 5870.1 | uss_MB: 4697.1 | child_rss_MB: 6175.1 | allocated_MB: 449.1 | reserved_MB: 28582.0 | inactive_split_MB: 1105.9 +rank7 [2026-10-28 14:55:40,224] INFO : Step 4500/50000 rank7 | Train Loss: 2.3283 | Train Loss Unweighted: 1.2750 | Train Loss Components: Auxiliary: 0.4322 | Cardinality: 0.3015 | Classification: 0.2318 | Classification_binary: 0.3495 | HitGrouping: 0.2369 | Regression_cos_phi: 0.0020 | Regression_energy: 0.3409 | Regression_eta: 0.0044 | Regression_pt: 0.3441 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1648 | Classification_binary: 0.3738 | Regression_cos_phi: 0.1987 | Regression_energy: 0.1939 | Regression_eta: 0.1385 | Regression_pt: 0.2093 | Regression_sin_phi: 0.2145 | LR: 9.82e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.0181s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7326 | max_loss: 0.2428 | max_backward: 1.7841 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0226 | Memory: rss_MB: 5868.2 | uss_MB: 4696.0 | child_rss_MB: 6178.6 | allocated_MB: 447.8 | reserved_MB: 28582.0 | inactive_split_MB: 1258.0 +rank7 [2026-10-28 15:00:16,448] INFO : Step 4600/50000 rank7 | Train Loss: 2.0353 | Train Loss Unweighted: 1.1777 | Train Loss Components: Auxiliary: 0.3804 | Cardinality: 0.1920 | Classification: 0.2053 | Classification_binary: 0.2642 | HitGrouping: 0.2341 | Regression_cos_phi: 0.0018 | Regression_energy: 0.3490 | Regression_eta: 0.0039 | Regression_pt: 0.3512 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1567 | Classification_binary: 0.2611 | Regression_cos_phi: 0.1708 | Regression_energy: 0.1669 | Regression_eta: 0.1287 | Regression_pt: 0.1807 | Regression_sin_phi: 0.1786 | LR: 9.81e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.0850s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0402 | max_forward: 0.7143 | max_loss: 0.1412 | max_backward: 1.6726 | max_optimizer_step: 0.1844 | max_tensorboard_logging: 0.5799 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 5861.4 | uss_MB: 4694.6 | child_rss_MB: 6186.8 | allocated_MB: 446.4 | reserved_MB: 28582.0 | inactive_split_MB: 981.4 +rank7 [2026-10-28 15:05:12,090] INFO : Step 4700/50000 rank7 | Train Loss: 2.2426 | Train Loss Unweighted: 1.2576 | Train Loss Components: Auxiliary: 0.4073 | Cardinality: 0.2222 | Classification: 0.2485 | Classification_binary: 0.3333 | HitGrouping: 0.2339 | Regression_cos_phi: 0.0023 | Regression_energy: 0.3307 | Regression_eta: 0.0049 | Regression_pt: 0.3356 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1235 | Classification_binary: 0.3027 | Regression_cos_phi: 0.1602 | Regression_energy: 0.1868 | Regression_eta: 0.1059 | Regression_pt: 0.2057 | Regression_sin_phi: 0.1926 | LR: 9.81e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.6205s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.1471 | max_backward: 2.0627 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.7062 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0769 | Memory: rss_MB: 5860.5 | uss_MB: 4699.2 | child_rss_MB: 6169.2 | allocated_MB: 447.7 | reserved_MB: 28652.0 | inactive_split_MB: 1164.9 +rank7 [2026-10-28 15:09:59,906] INFO : Step 4800/50000 rank7 | Train Loss: 2.2697 | Train Loss Unweighted: 1.2637 | Train Loss Components: Auxiliary: 0.4132 | Cardinality: 0.2493 | Classification: 0.2262 | Classification_binary: 0.3667 | HitGrouping: 0.2285 | Regression_cos_phi: 0.0024 | Regression_energy: 0.3287 | Regression_eta: 0.0044 | Regression_pt: 0.3332 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1538 | Classification_binary: 0.3947 | Regression_cos_phi: 0.1952 | Regression_energy: 0.1632 | Regression_eta: 0.1262 | Regression_pt: 0.1788 | Regression_sin_phi: 0.1637 | LR: 9.80e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.0692s | Phase Times: max_pre_step_barrier: 0.1096 | max_batch_to_device: 0.0065 | max_forward: 0.7626 | max_loss: 0.1571 | max_backward: 1.6465 | max_optimizer_step: 0.1876 | max_tensorboard_logging: 0.5017 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0260 | Memory: rss_MB: 5855.3 | uss_MB: 4693.4 | child_rss_MB: 6188.9 | allocated_MB: 450.7 | reserved_MB: 28652.0 | inactive_split_MB: 1088.6 +rank7 [2026-10-28 15:14:51,789] INFO : Step 4900/50000 rank7 | Train Loss: 2.0529 | Train Loss Unweighted: 1.1598 | Train Loss Components: Auxiliary: 0.3765 | Cardinality: 0.1942 | Classification: 0.2269 | Classification_binary: 0.2874 | HitGrouping: 0.2309 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3172 | Regression_eta: 0.0041 | Regression_pt: 0.3199 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1168 | Classification_binary: 0.3101 | Regression_cos_phi: 0.1224 | Regression_energy: 0.1758 | Regression_eta: 0.1152 | Regression_pt: 0.1900 | Regression_sin_phi: 0.1699 | LR: 9.79e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.3242s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0067 | max_forward: 0.7293 | max_loss: 0.1974 | max_backward: 1.8633 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.6011 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0206 | Memory: rss_MB: 5858.6 | uss_MB: 4697.0 | child_rss_MB: 6189.0 | allocated_MB: 447.4 | reserved_MB: 28652.0 | inactive_split_MB: 1266.1 +rank7 [2026-10-28 15:19:59,464] INFO : Step 5000/50000 rank7 | Train Loss: 2.0827 | Train Loss Unweighted: 1.1872 | Train Loss Components: Auxiliary: 0.3916 | Cardinality: 0.2154 | Classification: 0.2462 | Classification_binary: 0.2941 | HitGrouping: 0.2241 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3169 | Regression_eta: 0.0042 | Regression_pt: 0.3219 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1035 | Classification_binary: 0.2525 | Regression_cos_phi: 0.1348 | Regression_energy: 0.1776 | Regression_eta: 0.0953 | Regression_pt: 0.1925 | Regression_sin_phi: 0.1568 | LR: 9.78e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3251s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0063 | max_forward: 0.8382 | max_loss: 0.1450 | max_backward: 1.9472 | max_optimizer_step: 0.1804 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0203 | Memory: rss_MB: 5858.9 | uss_MB: 4697.5 | child_rss_MB: 6177.0 | allocated_MB: 446.9 | reserved_MB: 29316.0 | inactive_split_MB: 980.8 +rank7 [2026-10-28 15:20:00,329] INFO : Running validation on rank7 +rank7 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 15:20:00,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 15:20:55,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:20:56,482] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 15:21:22,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 15:21:23,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 15:21:24,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 15:21:25,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 15:21:35,795] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 15:21:36,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:21:36,967] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 15:21:53,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 15:21:54,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 15:21:55,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 15:21:56,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 15:22:06,540] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 15:22:06,552] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 15:22:07,049] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 15:22:23,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 15:22:24,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 15:22:25,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 15:22:26,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_5000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 15:22:36,274] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 15:28:09,087] INFO : Step 5100/50000 rank7 | Train Loss: 2.4098 | Train Loss Unweighted: 1.3457 | Train Loss Components: Auxiliary: 0.4318 | Cardinality: 0.2397 | Classification: 0.2711 | Classification_binary: 0.3798 | HitGrouping: 0.2336 | Regression_cos_phi: 0.0026 | Regression_energy: 0.3391 | Regression_eta: 0.0051 | Regression_pt: 0.3453 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1714 | Classification_binary: 0.3970 | Regression_cos_phi: 0.2270 | Regression_energy: 0.1942 | Regression_eta: 0.1834 | Regression_pt: 0.2148 | Regression_sin_phi: 0.2766 | LR: 9.77e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.8399s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7777 | max_loss: 0.1511 | max_backward: 1.7914 | max_optimizer_step: 0.2637 | max_tensorboard_logging: 1.0544 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 5949.7 | uss_MB: 4786.0 | child_rss_MB: 6159.3 | allocated_MB: 447.6 | reserved_MB: 25650.0 | inactive_split_MB: 1122.4 +rank7 [2026-10-28 15:32:58,661] INFO : Step 5200/50000 rank7 | Train Loss: 2.0929 | Train Loss Unweighted: 1.1974 | Train Loss Components: Auxiliary: 0.3875 | Cardinality: 0.2007 | Classification: 0.2193 | Classification_binary: 0.3278 | HitGrouping: 0.2269 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3194 | Regression_eta: 0.0041 | Regression_pt: 0.3227 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1411 | Classification_binary: 0.3665 | Regression_cos_phi: 0.1940 | Regression_energy: 0.1694 | Regression_eta: 0.1025 | Regression_pt: 0.1859 | Regression_sin_phi: 0.1551 | LR: 9.76e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4927s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.7458 | max_loss: 0.2423 | max_backward: 1.7123 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.9648 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5957.0 | uss_MB: 4796.7 | child_rss_MB: 6179.0 | allocated_MB: 450.7 | reserved_MB: 28708.0 | inactive_split_MB: 1139.3 +rank7 [2026-10-28 15:37:52,596] INFO : Step 5300/50000 rank7 | Train Loss: 1.9905 | Train Loss Unweighted: 1.1352 | Train Loss Components: Auxiliary: 0.3704 | Cardinality: 0.2083 | Classification: 0.2239 | Classification_binary: 0.2818 | HitGrouping: 0.2273 | Regression_cos_phi: 0.0019 | Regression_energy: 0.3086 | Regression_eta: 0.0037 | Regression_pt: 0.3135 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1422 | Classification_binary: 0.2800 | Regression_cos_phi: 0.1730 | Regression_energy: 0.1503 | Regression_eta: 0.1046 | Regression_pt: 0.1602 | Regression_sin_phi: 0.1935 | LR: 9.75e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.1035s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0061 | max_forward: 1.2453 | max_loss: 0.2326 | max_backward: 2.1392 | max_optimizer_step: 0.1883 | max_tensorboard_logging: 1.0947 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0236 | Memory: rss_MB: 5945.0 | uss_MB: 4784.6 | child_rss_MB: 6156.3 | allocated_MB: 445.9 | reserved_MB: 30688.0 | inactive_split_MB: 925.3 +rank7 [2026-10-28 15:42:39,662] INFO : Step 5400/50000 rank7 | Train Loss: 2.2932 | Train Loss Unweighted: 1.2957 | Train Loss Components: Auxiliary: 0.4189 | Cardinality: 0.2411 | Classification: 0.2689 | Classification_binary: 0.3420 | HitGrouping: 0.2274 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3351 | Regression_eta: 0.0045 | Regression_pt: 0.3403 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1656 | Classification_binary: 0.3101 | Regression_cos_phi: 0.1774 | Regression_energy: 0.1827 | Regression_eta: 0.1152 | Regression_pt: 0.2005 | Regression_sin_phi: 0.2257 | LR: 9.74e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5699s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7688 | max_loss: 0.1697 | max_backward: 1.8189 | max_optimizer_step: 0.2747 | max_tensorboard_logging: 0.8641 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0207 | Memory: rss_MB: 5955.2 | uss_MB: 4796.8 | child_rss_MB: 6163.2 | allocated_MB: 450.8 | reserved_MB: 30688.0 | inactive_split_MB: 915.1 +rank7 [2026-10-28 15:47:42,229] INFO : Step 5500/50000 rank7 | Train Loss: 2.1734 | Train Loss Unweighted: 1.2295 | Train Loss Components: Auxiliary: 0.3970 | Cardinality: 0.2269 | Classification: 0.2218 | Classification_binary: 0.3571 | HitGrouping: 0.2200 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3188 | Regression_eta: 0.0041 | Regression_pt: 0.3234 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1269 | Classification_binary: 0.3705 | Regression_cos_phi: 0.1579 | Regression_energy: 0.1757 | Regression_eta: 0.1175 | Regression_pt: 0.1942 | Regression_sin_phi: 0.1959 | LR: 9.73e-04 | DataLoad Time: 0.0187s | Model Forward Time: 5.9306s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0153 | max_forward: 1.1913 | max_loss: 0.6135 | max_backward: 3.0599 | max_optimizer_step: 0.1745 | max_tensorboard_logging: 0.6982 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5952.5 | uss_MB: 4797.4 | child_rss_MB: 6170.1 | allocated_MB: 446.9 | reserved_MB: 32804.0 | inactive_split_MB: 913.2 +rank7 [2026-10-28 15:52:30,289] INFO : Step 5600/50000 rank7 | Train Loss: 2.0307 | Train Loss Unweighted: 1.1169 | Train Loss Components: Auxiliary: 0.3630 | Cardinality: 0.1966 | Classification: 0.2085 | Classification_binary: 0.3031 | HitGrouping: 0.2243 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2975 | Regression_eta: 0.0047 | Regression_pt: 0.2988 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1510 | Classification_binary: 0.3499 | Regression_cos_phi: 0.2171 | Regression_energy: 0.1473 | Regression_eta: 0.1675 | Regression_pt: 0.1581 | Regression_sin_phi: 0.1924 | LR: 9.72e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.7950s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0068 | max_forward: 0.7794 | max_loss: 0.1497 | max_backward: 1.9410 | max_optimizer_step: 0.5154 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0061 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 5920.9 | uss_MB: 4797.0 | child_rss_MB: 6174.4 | allocated_MB: 445.4 | reserved_MB: 32804.0 | inactive_split_MB: 1164.2 +rank7 [2026-10-28 15:57:16,862] INFO : Step 5700/50000 rank7 | Train Loss: 2.2064 | Train Loss Unweighted: 1.2224 | Train Loss Components: Auxiliary: 0.4023 | Cardinality: 0.2320 | Classification: 0.2526 | Classification_binary: 0.3174 | HitGrouping: 0.2343 | Regression_cos_phi: 0.0022 | Regression_energy: 0.3173 | Regression_eta: 0.0045 | Regression_pt: 0.3259 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1605 | Classification_binary: 0.3272 | Regression_cos_phi: 0.1876 | Regression_energy: 0.1769 | Regression_eta: 0.1285 | Regression_pt: 0.1969 | Regression_sin_phi: 0.1736 | LR: 9.71e-04 | DataLoad Time: 0.0200s | Model Forward Time: 4.6670s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7746 | max_loss: 0.4928 | max_backward: 1.9590 | max_optimizer_step: 0.2290 | max_tensorboard_logging: 0.8719 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0086 | Memory: rss_MB: 5893.1 | uss_MB: 3755.9 | child_rss_MB: 6168.2 | allocated_MB: 448.4 | reserved_MB: 32804.0 | inactive_split_MB: 913.0 +rank7 [2026-10-28 16:02:01,174] INFO : Step 5800/50000 rank7 | Train Loss: 2.1438 | Train Loss Unweighted: 1.2123 | Train Loss Components: Auxiliary: 0.3898 | Cardinality: 0.2335 | Classification: 0.2404 | Classification_binary: 0.3575 | HitGrouping: 0.2176 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3006 | Regression_eta: 0.0042 | Regression_pt: 0.3054 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1439 | Classification_binary: 0.3229 | Regression_cos_phi: 0.1385 | Regression_energy: 0.1613 | Regression_eta: 0.0919 | Regression_pt: 0.1778 | Regression_sin_phi: 0.1266 | LR: 9.70e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.5873s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8028 | max_loss: 0.1540 | max_backward: 1.8645 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.8379 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0218 | Memory: rss_MB: 5883.5 | uss_MB: 4797.1 | child_rss_MB: 6169.4 | allocated_MB: 448.3 | reserved_MB: 32804.0 | inactive_split_MB: 916.5 +rank7 [2026-10-28 16:06:46,106] INFO : Step 5900/50000 rank7 | Train Loss: 2.2094 | Train Loss Unweighted: 1.1453 | Train Loss Components: Auxiliary: 0.3936 | Cardinality: 0.2891 | Classification: 0.2198 | Classification_binary: 0.3337 | HitGrouping: 0.2238 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2890 | Regression_eta: 0.0046 | Regression_pt: 0.2935 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1536 | Classification_binary: 0.3567 | Regression_cos_phi: 0.1923 | Regression_energy: 0.1397 | Regression_eta: 0.1471 | Regression_pt: 0.1537 | Regression_sin_phi: 0.2489 | LR: 9.69e-04 | DataLoad Time: 0.0218s | Model Forward Time: 4.5861s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0065 | max_forward: 1.0922 | max_loss: 0.1477 | max_backward: 1.8468 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.8251 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0991 | Memory: rss_MB: 5881.4 | uss_MB: 4798.1 | child_rss_MB: 6148.2 | allocated_MB: 446.5 | reserved_MB: 32804.0 | inactive_split_MB: 927.1 +rank7 [2026-10-28 16:11:31,999] INFO : Step 6000/50000 rank7 | Train Loss: 1.8458 | Train Loss Unweighted: 1.0363 | Train Loss Components: Auxiliary: 0.3398 | Cardinality: 0.1796 | Classification: 0.1819 | Classification_binary: 0.2710 | HitGrouping: 0.2261 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2865 | Regression_eta: 0.0032 | Regression_pt: 0.2900 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1104 | Classification_binary: 0.2725 | Regression_cos_phi: 0.1099 | Regression_energy: 0.1314 | Regression_eta: 0.0909 | Regression_pt: 0.1442 | Regression_sin_phi: 0.1476 | LR: 9.68e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5086s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.7380 | max_loss: 0.2483 | max_backward: 1.8094 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.9243 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5850.9 | uss_MB: 4795.2 | child_rss_MB: 6173.4 | allocated_MB: 448.1 | reserved_MB: 32804.0 | inactive_split_MB: 1232.3 +rank7 [2026-10-28 16:11:32,849] INFO : Running validation on rank7 +rank7 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 16:11:32,849] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 16:12:26,407] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:12:26,923] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 16:12:53,979] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 16:12:55,143] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 16:12:56,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 16:12:57,244] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 16:13:07,254] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 16:13:07,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:13:08,323] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 16:13:24,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 16:13:25,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 16:13:26,577] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 16:13:27,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 16:13:37,525] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 16:13:38,169] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 16:13:38,687] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 16:13:53,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 16:13:54,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 16:13:55,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 16:13:56,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_6000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 16:14:06,510] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 16:19:13,933] INFO : Step 6100/50000 rank7 | Train Loss: 2.0818 | Train Loss Unweighted: 1.1297 | Train Loss Components: Auxiliary: 0.3734 | Cardinality: 0.2287 | Classification: 0.2079 | Classification_binary: 0.3249 | HitGrouping: 0.2182 | Regression_cos_phi: 0.0022 | Regression_energy: 0.2929 | Regression_eta: 0.0044 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1022 | Classification_binary: 0.2375 | Regression_cos_phi: 0.1114 | Regression_energy: 0.1690 | Regression_eta: 0.0792 | Regression_pt: 0.1828 | Regression_sin_phi: 0.1361 | LR: 9.67e-04 | DataLoad Time: 0.0151s | Model Forward Time: 6.5166s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8130 | max_loss: 0.1524 | max_backward: 3.1921 | max_optimizer_step: 0.1924 | max_tensorboard_logging: 1.4719 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0212 | Memory: rss_MB: 5941.1 | uss_MB: 4883.2 | child_rss_MB: 6189.9 | allocated_MB: 448.4 | reserved_MB: 24434.0 | inactive_split_MB: 1287.8 +rank7 [2026-10-28 16:23:58,090] INFO : Step 6200/50000 rank7 | Train Loss: 2.3716 | Train Loss Unweighted: 1.2065 | Train Loss Components: Auxiliary: 0.4163 | Cardinality: 0.3927 | Classification: 0.2343 | Classification_binary: 0.3679 | HitGrouping: 0.2165 | Regression_cos_phi: 0.0023 | Regression_energy: 0.2973 | Regression_eta: 0.0043 | Regression_pt: 0.2981 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1532 | Classification_binary: 0.3700 | Regression_cos_phi: 0.1959 | Regression_energy: 0.1869 | Regression_eta: 0.1199 | Regression_pt: 0.2021 | Regression_sin_phi: 0.1834 | LR: 9.66e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.5581s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0076 | max_forward: 0.8556 | max_loss: 0.1996 | max_backward: 1.9593 | max_optimizer_step: 0.2732 | max_tensorboard_logging: 0.5823 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5946.6 | uss_MB: 3767.3 | child_rss_MB: 6181.8 | allocated_MB: 447.9 | reserved_MB: 27020.0 | inactive_split_MB: 1185.4 +rank7 [2026-10-28 16:28:38,484] INFO : Step 6300/50000 rank7 | Train Loss: 1.9346 | Train Loss Unweighted: 1.0816 | Train Loss Components: Auxiliary: 0.3559 | Cardinality: 0.2071 | Classification: 0.2053 | Classification_binary: 0.2781 | HitGrouping: 0.2242 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2941 | Regression_eta: 0.0037 | Regression_pt: 0.2966 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1098 | Classification_binary: 0.2916 | Regression_cos_phi: 0.1890 | Regression_energy: 0.1463 | Regression_eta: 0.1201 | Regression_pt: 0.1616 | Regression_sin_phi: 0.1688 | LR: 9.65e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.5879s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0068 | max_forward: 0.7412 | max_loss: 0.1444 | max_backward: 1.7431 | max_optimizer_step: 0.1899 | max_tensorboard_logging: 0.9045 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5941.6 | uss_MB: 4896.5 | child_rss_MB: 6164.9 | allocated_MB: 444.6 | reserved_MB: 27022.0 | inactive_split_MB: 1185.4 +rank7 [2026-10-28 16:33:23,144] INFO : Step 6400/50000 rank7 | Train Loss: 2.0123 | Train Loss Unweighted: 1.0992 | Train Loss Components: Auxiliary: 0.3659 | Cardinality: 0.2082 | Classification: 0.2291 | Classification_binary: 0.2885 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2842 | Regression_eta: 0.0040 | Regression_pt: 0.2891 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1160 | Classification_binary: 0.2574 | Regression_cos_phi: 0.1212 | Regression_energy: 0.1509 | Regression_eta: 0.0832 | Regression_pt: 0.1684 | Regression_sin_phi: 0.1421 | LR: 9.64e-04 | DataLoad Time: 0.0228s | Model Forward Time: 4.7555s | Phase Times: max_pre_step_barrier: 0.0065 | max_batch_to_device: 0.0069 | max_forward: 1.1063 | max_loss: 0.2471 | max_backward: 1.9949 | max_optimizer_step: 0.2726 | max_tensorboard_logging: 0.8233 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0246 | Memory: rss_MB: 5936.1 | uss_MB: 4896.3 | child_rss_MB: 6176.5 | allocated_MB: 451.6 | reserved_MB: 27022.0 | inactive_split_MB: 1113.0 +rank7 [2026-10-28 16:37:57,182] INFO : Step 6500/50000 rank7 | Train Loss: 2.1791 | Train Loss Unweighted: 1.2220 | Train Loss Components: Auxiliary: 0.4020 | Cardinality: 0.2499 | Classification: 0.2271 | Classification_binary: 0.3508 | HitGrouping: 0.2192 | Regression_cos_phi: 0.0021 | Regression_energy: 0.3177 | Regression_eta: 0.0042 | Regression_pt: 0.3181 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1264 | Classification_binary: 0.3669 | Regression_cos_phi: 0.2556 | Regression_energy: 0.1680 | Regression_eta: 0.1563 | Regression_pt: 0.1802 | Regression_sin_phi: 0.2165 | LR: 9.63e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.9960s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0548 | max_forward: 0.8069 | max_loss: 0.1518 | max_backward: 2.0692 | max_optimizer_step: 0.1860 | max_tensorboard_logging: 0.9414 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0385 | Memory: rss_MB: 5926.6 | uss_MB: 4895.5 | child_rss_MB: 6160.0 | allocated_MB: 449.6 | reserved_MB: 28336.0 | inactive_split_MB: 1199.2 +rank7 [2026-10-28 16:42:33,043] INFO : Step 6600/50000 rank7 | Train Loss: 1.9690 | Train Loss Unweighted: 1.0745 | Train Loss Components: Auxiliary: 0.3537 | Cardinality: 0.2132 | Classification: 0.1982 | Classification_binary: 0.3095 | HitGrouping: 0.2136 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2771 | Regression_eta: 0.0040 | Regression_pt: 0.2816 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1066 | Classification_binary: 0.3149 | Regression_cos_phi: 0.1174 | Regression_energy: 0.1615 | Regression_eta: 0.0985 | Regression_pt: 0.1751 | Regression_sin_phi: 0.1495 | LR: 9.62e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8262s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7382 | max_loss: 0.1467 | max_backward: 1.8335 | max_optimizer_step: 0.1901 | max_tensorboard_logging: 1.1664 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0220 | Memory: rss_MB: 5923.1 | uss_MB: 4893.3 | child_rss_MB: 6174.8 | allocated_MB: 447.4 | reserved_MB: 28340.0 | inactive_split_MB: 1012.8 +rank7 [2026-10-28 16:47:06,903] INFO : Step 6700/50000 rank7 | Train Loss: 1.9470 | Train Loss Unweighted: 1.0781 | Train Loss Components: Auxiliary: 0.3616 | Cardinality: 0.2363 | Classification: 0.1923 | Classification_binary: 0.2661 | HitGrouping: 0.2227 | Regression_cos_phi: 0.0017 | Regression_energy: 0.3028 | Regression_eta: 0.0035 | Regression_pt: 0.3097 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0995 | Classification_binary: 0.2796 | Regression_cos_phi: 0.1227 | Regression_energy: 0.1655 | Regression_eta: 0.0758 | Regression_pt: 0.1840 | Regression_sin_phi: 0.1246 | LR: 9.61e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.6605s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.7926 | max_loss: 0.2421 | max_backward: 2.0577 | max_optimizer_step: 0.2523 | max_tensorboard_logging: 0.7621 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5925.5 | uss_MB: 4898.4 | child_rss_MB: 6176.2 | allocated_MB: 450.1 | reserved_MB: 29806.0 | inactive_split_MB: 1002.0 +rank7 [2026-10-28 16:51:50,914] INFO : Step 6800/50000 rank7 | Train Loss: 2.1609 | Train Loss Unweighted: 1.1507 | Train Loss Components: Auxiliary: 0.3771 | Cardinality: 0.2180 | Classification: 0.2315 | Classification_binary: 0.3225 | HitGrouping: 0.2189 | Regression_cos_phi: 0.0025 | Regression_energy: 0.2916 | Regression_eta: 0.0048 | Regression_pt: 0.2952 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1464 | Classification_binary: 0.3067 | Regression_cos_phi: 0.2374 | Regression_energy: 0.1598 | Regression_eta: 0.1471 | Regression_pt: 0.1759 | Regression_sin_phi: 0.1849 | LR: 9.60e-04 | DataLoad Time: 0.0208s | Model Forward Time: 5.1900s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0070 | max_forward: 0.9144 | max_loss: 0.4866 | max_backward: 2.1484 | max_optimizer_step: 0.1873 | max_tensorboard_logging: 1.2438 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 5920.8 | uss_MB: 4896.0 | child_rss_MB: 6174.1 | allocated_MB: 451.3 | reserved_MB: 29806.0 | inactive_split_MB: 1098.9 +rank7 [2026-10-28 16:56:32,287] INFO : Step 6900/50000 rank7 | Train Loss: 1.9936 | Train Loss Unweighted: 1.1069 | Train Loss Components: Auxiliary: 0.3625 | Cardinality: 0.1991 | Classification: 0.2085 | Classification_binary: 0.3385 | HitGrouping: 0.2125 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2742 | Regression_eta: 0.0041 | Regression_pt: 0.2777 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1759 | Classification_binary: 0.3510 | Regression_cos_phi: 0.1937 | Regression_energy: 0.1325 | Regression_eta: 0.1438 | Regression_pt: 0.1465 | Regression_sin_phi: 0.1984 | LR: 9.58e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.6486s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.7764 | max_loss: 0.2225 | max_backward: 1.6637 | max_optimizer_step: 0.1926 | max_tensorboard_logging: 1.1184 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0241 | Memory: rss_MB: 5918.2 | uss_MB: 4895.4 | child_rss_MB: 6170.7 | allocated_MB: 450.8 | reserved_MB: 29806.0 | inactive_split_MB: 1193.4 +rank7 [2026-10-28 17:01:16,519] INFO : Step 7000/50000 rank7 | Train Loss: 1.9256 | Train Loss Unweighted: 1.0803 | Train Loss Components: Auxiliary: 0.3608 | Cardinality: 0.2036 | Classification: 0.1946 | Classification_binary: 0.2738 | HitGrouping: 0.2205 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2998 | Regression_eta: 0.0039 | Regression_pt: 0.3044 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1271 | Classification_binary: 0.2900 | Regression_cos_phi: 0.1421 | Regression_energy: 0.1823 | Regression_eta: 0.0959 | Regression_pt: 0.2020 | Regression_sin_phi: 0.1864 | LR: 9.57e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7622s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0067 | max_forward: 0.7896 | max_loss: 0.1557 | max_backward: 1.7415 | max_optimizer_step: 0.2946 | max_tensorboard_logging: 1.0959 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 5913.7 | uss_MB: 4893.6 | child_rss_MB: 6162.0 | allocated_MB: 450.3 | reserved_MB: 29806.0 | inactive_split_MB: 1283.4 +rank7 [2026-10-28 17:01:17,465] INFO : Running validation on rank7 +rank7 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 17:01:17,465] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 17:02:11,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:02:12,863] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 17:02:38,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 17:02:39,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 17:02:40,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 17:02:41,834] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 17:02:51,842] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 17:02:52,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:02:53,111] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 17:03:09,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 17:03:10,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 17:03:11,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 17:03:12,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 17:03:22,163] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 17:03:22,841] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:03:23,343] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 17:03:39,149] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 17:03:40,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 17:03:40,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 17:03:41,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_7000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 17:03:51,788] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 17:08:58,936] INFO : Step 7100/50000 rank7 | Train Loss: 1.8251 | Train Loss Unweighted: 1.0448 | Train Loss Components: Auxiliary: 0.3403 | Cardinality: 0.1731 | Classification: 0.2040 | Classification_binary: 0.2675 | HitGrouping: 0.2114 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2809 | Regression_eta: 0.0035 | Regression_pt: 0.2853 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0887 | Classification_binary: 0.2641 | Regression_cos_phi: 0.1095 | Regression_energy: 0.1448 | Regression_eta: 0.0839 | Regression_pt: 0.1614 | Regression_sin_phi: 0.1231 | LR: 9.56e-04 | DataLoad Time: 0.0216s | Model Forward Time: 5.2549s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0622 | max_forward: 0.8282 | max_loss: 0.1540 | max_backward: 1.9352 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 1.3584 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0370 | Memory: rss_MB: 6015.2 | uss_MB: 4988.2 | child_rss_MB: 6155.9 | allocated_MB: 449.9 | reserved_MB: 22498.0 | inactive_split_MB: 1193.4 +rank7 [2026-10-28 17:13:31,724] INFO : Step 7200/50000 rank7 | Train Loss: 2.1146 | Train Loss Unweighted: 1.1506 | Train Loss Components: Auxiliary: 0.3887 | Cardinality: 0.2411 | Classification: 0.2243 | Classification_binary: 0.3469 | HitGrouping: 0.2109 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2844 | Regression_eta: 0.0044 | Regression_pt: 0.2865 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1293 | Classification_binary: 0.3207 | Regression_cos_phi: 0.2102 | Regression_energy: 0.1479 | Regression_eta: 0.1247 | Regression_pt: 0.1612 | Regression_sin_phi: 0.1840 | LR: 9.55e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.5735s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0369 | max_forward: 0.7969 | max_loss: 0.1517 | max_backward: 1.9380 | max_optimizer_step: 0.1906 | max_tensorboard_logging: 0.5021 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6030.2 | uss_MB: 5003.5 | child_rss_MB: 6152.6 | allocated_MB: 448.6 | reserved_MB: 22906.0 | inactive_split_MB: 1087.6 +rank7 [2026-10-28 17:18:11,974] INFO : Step 7300/50000 rank7 | Train Loss: 1.8657 | Train Loss Unweighted: 1.0129 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2046 | Classification: 0.1759 | Classification_binary: 0.2802 | HitGrouping: 0.2112 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2746 | Regression_eta: 0.0037 | Regression_pt: 0.2747 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0993 | Classification_binary: 0.2924 | Regression_cos_phi: 0.1730 | Regression_energy: 0.1503 | Regression_eta: 0.0972 | Regression_pt: 0.1618 | Regression_sin_phi: 0.1247 | LR: 9.53e-04 | DataLoad Time: 0.0128s | Model Forward Time: 4.4714s | Phase Times: max_pre_step_barrier: 0.0082 | max_batch_to_device: 0.0150 | max_forward: 0.7713 | max_loss: 0.1507 | max_backward: 1.8303 | max_optimizer_step: 0.1913 | max_tensorboard_logging: 0.6113 | max_tensorboard_flush: 0.0116 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6029.9 | uss_MB: 5003.9 | child_rss_MB: 6151.8 | allocated_MB: 449.9 | reserved_MB: 25044.0 | inactive_split_MB: 918.5 +rank7 [2026-10-28 17:22:47,474] INFO : Step 7400/50000 rank7 | Train Loss: 1.8326 | Train Loss Unweighted: 1.0006 | Train Loss Components: Auxiliary: 0.3381 | Cardinality: 0.2227 | Classification: 0.1795 | Classification_binary: 0.2563 | HitGrouping: 0.2155 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2767 | Regression_eta: 0.0034 | Regression_pt: 0.2813 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1186 | Classification_binary: 0.2383 | Regression_cos_phi: 0.1259 | Regression_energy: 0.1451 | Regression_eta: 0.0832 | Regression_pt: 0.1559 | Regression_sin_phi: 0.1791 | LR: 9.52e-04 | DataLoad Time: 0.0220s | Model Forward Time: 4.2036s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0684 | max_forward: 0.7762 | max_loss: 0.1384 | max_backward: 1.7881 | max_optimizer_step: 0.1925 | max_tensorboard_logging: 0.5753 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6026.9 | uss_MB: 5000.5 | child_rss_MB: 6151.8 | allocated_MB: 447.4 | reserved_MB: 25044.0 | inactive_split_MB: 1152.4 +rank7 [2026-10-28 17:27:28,847] INFO : Step 7500/50000 rank7 | Train Loss: 2.1479 | Train Loss Unweighted: 1.1510 | Train Loss Components: Auxiliary: 0.3847 | Cardinality: 0.2329 | Classification: 0.2181 | Classification_binary: 0.3323 | HitGrouping: 0.2143 | Regression_cos_phi: 0.0024 | Regression_energy: 0.2947 | Regression_eta: 0.0044 | Regression_pt: 0.2968 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1492 | Classification_binary: 0.3226 | Regression_cos_phi: 0.2672 | Regression_energy: 0.1634 | Regression_eta: 0.1395 | Regression_pt: 0.1759 | Regression_sin_phi: 0.2601 | LR: 9.51e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2509s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7726 | max_loss: 0.2518 | max_backward: 1.8458 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1040 | Memory: rss_MB: 6026.8 | uss_MB: 5002.5 | child_rss_MB: 6153.2 | allocated_MB: 452.3 | reserved_MB: 26846.0 | inactive_split_MB: 1117.1 +rank7 [2026-10-28 17:32:29,655] INFO : Step 7600/50000 rank7 | Train Loss: 1.9984 | Train Loss Unweighted: 1.0873 | Train Loss Components: Auxiliary: 0.3667 | Cardinality: 0.2088 | Classification: 0.2053 | Classification_binary: 0.3405 | HitGrouping: 0.2080 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2647 | Regression_eta: 0.0038 | Regression_pt: 0.2689 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1327 | Classification_binary: 0.3524 | Regression_cos_phi: 0.1848 | Regression_energy: 0.1539 | Regression_eta: 0.1148 | Regression_pt: 0.1720 | Regression_sin_phi: 0.1607 | LR: 9.50e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5790s | Phase Times: max_pre_step_barrier: 0.0040 | max_batch_to_device: 0.0067 | max_forward: 0.7834 | max_loss: 0.1641 | max_backward: 1.8407 | max_optimizer_step: 0.2477 | max_tensorboard_logging: 0.8255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0293 | Memory: rss_MB: 6029.1 | uss_MB: 5003.6 | child_rss_MB: 6150.1 | allocated_MB: 448.9 | reserved_MB: 26846.0 | inactive_split_MB: 1232.3 +rank7 [2026-10-28 17:37:19,853] INFO : Step 7700/50000 rank7 | Train Loss: 1.7524 | Train Loss Unweighted: 0.9761 | Train Loss Components: Auxiliary: 0.3200 | Cardinality: 0.1783 | Classification: 0.1756 | Classification_binary: 0.2521 | HitGrouping: 0.2025 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2705 | Regression_eta: 0.0035 | Regression_pt: 0.2709 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1354 | Classification_binary: 0.3286 | Regression_cos_phi: 0.1962 | Regression_energy: 0.1560 | Regression_eta: 0.1386 | Regression_pt: 0.1666 | Regression_sin_phi: 0.2025 | LR: 9.48e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.4853s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7659 | max_loss: 0.1551 | max_backward: 1.8103 | max_optimizer_step: 0.2942 | max_tensorboard_logging: 0.8042 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0272 | Memory: rss_MB: 6027.2 | uss_MB: 5004.4 | child_rss_MB: 6154.6 | allocated_MB: 448.8 | reserved_MB: 26846.0 | inactive_split_MB: 1042.2 +rank7 [2026-10-28 17:42:10,706] INFO : Step 7800/50000 rank7 | Train Loss: 1.8039 | Train Loss Unweighted: 1.0226 | Train Loss Components: Auxiliary: 0.3378 | Cardinality: 0.1686 | Classification: 0.1822 | Classification_binary: 0.2556 | HitGrouping: 0.2102 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2863 | Regression_eta: 0.0034 | Regression_pt: 0.2914 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1032 | Classification_binary: 0.2251 | Regression_cos_phi: 0.1196 | Regression_energy: 0.1504 | Regression_eta: 0.0670 | Regression_pt: 0.1679 | Regression_sin_phi: 0.1129 | LR: 9.47e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.6869s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.8680 | max_loss: 0.1424 | max_backward: 1.9949 | max_optimizer_step: 0.1912 | max_tensorboard_logging: 0.8163 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0631 | Memory: rss_MB: 6024.0 | uss_MB: 5003.5 | child_rss_MB: 6156.5 | allocated_MB: 448.6 | reserved_MB: 26846.0 | inactive_split_MB: 1396.9 +rank7 [2026-10-28 17:47:05,102] INFO : Step 7900/50000 rank7 | Train Loss: 2.0641 | Train Loss Unweighted: 1.0960 | Train Loss Components: Auxiliary: 0.3855 | Cardinality: 0.2401 | Classification: 0.2031 | Classification_binary: 0.3367 | HitGrouping: 0.2085 | Regression_cos_phi: 0.0021 | Regression_energy: 0.2721 | Regression_eta: 0.0041 | Regression_pt: 0.2759 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1284 | Classification_binary: 0.3419 | Regression_cos_phi: 0.1592 | Regression_energy: 0.1424 | Regression_eta: 0.1034 | Regression_pt: 0.1545 | Regression_sin_phi: 0.1281 | LR: 9.46e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.6368s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0070 | max_forward: 0.7971 | max_loss: 0.2430 | max_backward: 1.9131 | max_optimizer_step: 0.2681 | max_tensorboard_logging: 0.7971 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6025.3 | uss_MB: 5004.3 | child_rss_MB: 6161.5 | allocated_MB: 451.8 | reserved_MB: 27044.0 | inactive_split_MB: 827.4 +rank7 [2026-10-28 17:51:56,121] INFO : Step 8000/50000 rank7 | Train Loss: 1.8391 | Train Loss Unweighted: 0.9813 | Train Loss Components: Auxiliary: 0.3355 | Cardinality: 0.2153 | Classification: 0.1625 | Classification_binary: 0.2896 | HitGrouping: 0.2081 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2605 | Regression_eta: 0.0035 | Regression_pt: 0.2616 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0919 | Classification_binary: 0.3034 | Regression_cos_phi: 0.1232 | Regression_energy: 0.1664 | Regression_eta: 0.1211 | Regression_pt: 0.1830 | Regression_sin_phi: 0.2330 | LR: 9.44e-04 | DataLoad Time: 0.0184s | Model Forward Time: 6.5527s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 1.2188 | max_loss: 0.4570 | max_backward: 3.9408 | max_optimizer_step: 0.2852 | max_tensorboard_logging: 0.5566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0228 | Memory: rss_MB: 6013.9 | uss_MB: 4999.4 | child_rss_MB: 6155.0 | allocated_MB: 448.4 | reserved_MB: 27044.0 | inactive_split_MB: 1156.9 +rank7 [2026-10-28 17:51:56,979] INFO : Running validation on rank7 +rank7 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 17:51:56,979] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 17:52:50,096] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:52:50,645] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 17:53:16,792] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 17:53:17,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 17:53:18,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 17:53:19,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 17:53:29,755] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 17:53:30,251] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:53:30,808] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 17:53:47,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 17:53:48,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 17:53:49,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 17:53:50,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 17:54:00,724] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-28 17:54:01,631] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 17:54:02,178] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 17:54:17,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 17:54:18,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 17:54:19,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 17:54:19,922] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_8000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 17:54:29,932] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 17:59:43,743] INFO : Step 8100/50000 rank7 | Train Loss: 1.7101 | Train Loss Unweighted: 0.9276 | Train Loss Components: Auxiliary: 0.3166 | Cardinality: 0.1787 | Classification: 0.1664 | Classification_binary: 0.2381 | HitGrouping: 0.2072 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2564 | Regression_eta: 0.0032 | Regression_pt: 0.2599 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0828 | Classification_binary: 0.2394 | Regression_cos_phi: 0.1284 | Regression_energy: 0.1407 | Regression_eta: 0.0754 | Regression_pt: 0.1541 | Regression_sin_phi: 0.0898 | LR: 9.43e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6982s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0071 | max_forward: 0.7628 | max_loss: 0.1353 | max_backward: 2.0123 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.8300 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6970.2 | uss_MB: 5951.6 | child_rss_MB: 6190.8 | allocated_MB: 447.2 | reserved_MB: 25306.0 | inactive_split_MB: 1261.3 +rank7 [2026-10-28 18:04:56,111] INFO : Step 8200/50000 rank7 | Train Loss: 1.8745 | Train Loss Unweighted: 1.0293 | Train Loss Components: Auxiliary: 0.3395 | Cardinality: 0.1946 | Classification: 0.1986 | Classification_binary: 0.2966 | HitGrouping: 0.2016 | Regression_cos_phi: 0.0020 | Regression_energy: 0.2609 | Regression_eta: 0.0035 | Regression_pt: 0.2657 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1220 | Classification_binary: 0.2373 | Regression_cos_phi: 0.1553 | Regression_energy: 0.1670 | Regression_eta: 0.0864 | Regression_pt: 0.1865 | Regression_sin_phi: 0.1493 | LR: 9.42e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3066s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.7790 | max_loss: 0.1535 | max_backward: 1.9181 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.5352 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0256 | Memory: rss_MB: 6080.6 | uss_MB: 5064.4 | child_rss_MB: 6184.5 | allocated_MB: 448.2 | reserved_MB: 25360.0 | inactive_split_MB: 1159.7 +rank7 [2026-10-28 18:09:37,299] INFO : Step 8300/50000 rank7 | Train Loss: 1.8727 | Train Loss Unweighted: 1.0119 | Train Loss Components: Auxiliary: 0.3401 | Cardinality: 0.1867 | Classification: 0.1728 | Classification_binary: 0.3272 | HitGrouping: 0.2062 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2513 | Regression_eta: 0.0033 | Regression_pt: 0.2533 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1064 | Classification_binary: 0.3063 | Regression_cos_phi: 0.1503 | Regression_energy: 0.1172 | Regression_eta: 0.0991 | Regression_pt: 0.1289 | Regression_sin_phi: 0.1606 | LR: 9.40e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5455s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0436 | max_forward: 1.1303 | max_loss: 0.1503 | max_backward: 2.0213 | max_optimizer_step: 0.1916 | max_tensorboard_logging: 0.6132 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0276 | Memory: rss_MB: 6082.4 | uss_MB: 5067.6 | child_rss_MB: 6182.5 | allocated_MB: 449.6 | reserved_MB: 27362.0 | inactive_split_MB: 1355.2 +rank7 [2026-10-28 18:14:10,521] INFO : Step 8400/50000 rank7 | Train Loss: 1.6705 | Train Loss Unweighted: 0.9205 | Train Loss Components: Auxiliary: 0.3113 | Cardinality: 0.1938 | Classification: 0.1656 | Classification_binary: 0.2512 | HitGrouping: 0.2055 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2485 | Regression_eta: 0.0028 | Regression_pt: 0.2494 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1014 | Classification_binary: 0.2857 | Regression_cos_phi: 0.1085 | Regression_energy: 0.1333 | Regression_eta: 0.0965 | Regression_pt: 0.1441 | Regression_sin_phi: 0.1219 | LR: 9.39e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4477s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.8079 | max_loss: 0.1514 | max_backward: 1.6942 | max_optimizer_step: 0.2670 | max_tensorboard_logging: 0.7366 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6074.8 | uss_MB: 5060.4 | child_rss_MB: 6188.9 | allocated_MB: 446.4 | reserved_MB: 27366.0 | inactive_split_MB: 1386.9 +rank7 [2026-10-28 18:18:45,092] INFO : Step 8500/50000 rank7 | Train Loss: 1.8447 | Train Loss Unweighted: 0.9684 | Train Loss Components: Auxiliary: 0.3299 | Cardinality: 0.2197 | Classification: 0.1758 | Classification_binary: 0.2823 | HitGrouping: 0.2067 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2478 | Regression_eta: 0.0040 | Regression_pt: 0.2547 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0805 | Classification_binary: 0.2500 | Regression_cos_phi: 0.1066 | Regression_energy: 0.1197 | Regression_eta: 0.0763 | Regression_pt: 0.1317 | Regression_sin_phi: 0.1111 | LR: 9.37e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.1670s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0066 | max_forward: 0.7185 | max_loss: 0.1465 | max_backward: 1.6783 | max_optimizer_step: 0.2239 | max_tensorboard_logging: 0.6269 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 6077.8 | uss_MB: 5064.0 | child_rss_MB: 6185.3 | allocated_MB: 448.4 | reserved_MB: 27368.0 | inactive_split_MB: 1257.8 +rank7 [2026-10-28 18:23:30,152] INFO : Step 8600/50000 rank7 | Train Loss: 1.9431 | Train Loss Unweighted: 1.0237 | Train Loss Components: Auxiliary: 0.3540 | Cardinality: 0.2483 | Classification: 0.1814 | Classification_binary: 0.3328 | HitGrouping: 0.2024 | Regression_cos_phi: 0.0019 | Regression_energy: 0.2495 | Regression_eta: 0.0034 | Regression_pt: 0.2527 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0917 | Classification_binary: 0.2946 | Regression_cos_phi: 0.1320 | Regression_energy: 0.1299 | Regression_eta: 0.0821 | Regression_pt: 0.1417 | Regression_sin_phi: 0.0988 | LR: 9.36e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.5571s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.1011 | max_forward: 0.9340 | max_loss: 0.1508 | max_backward: 1.7704 | max_optimizer_step: 0.1931 | max_tensorboard_logging: 0.7686 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0286 | Memory: rss_MB: 6082.0 | uss_MB: 5062.1 | child_rss_MB: 6191.5 | allocated_MB: 448.9 | reserved_MB: 27368.0 | inactive_split_MB: 1268.3 +rank7 [2026-10-28 18:28:08,445] INFO : Step 8700/50000 rank7 | Train Loss: 1.9739 | Train Loss Unweighted: 1.0707 | Train Loss Components: Auxiliary: 0.3725 | Cardinality: 0.2322 | Classification: 0.1854 | Classification_binary: 0.2934 | HitGrouping: 0.2130 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2900 | Regression_eta: 0.0037 | Regression_pt: 0.2944 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1245 | Classification_binary: 0.3168 | Regression_cos_phi: 0.1463 | Regression_energy: 0.1608 | Regression_eta: 0.1144 | Regression_pt: 0.1755 | Regression_sin_phi: 0.2289 | LR: 9.34e-04 | DataLoad Time: 0.0183s | Model Forward Time: 6.1645s | Phase Times: max_pre_step_barrier: 0.0890 | max_batch_to_device: 0.0067 | max_forward: 0.7726 | max_loss: 0.1574 | max_backward: 1.6332 | max_optimizer_step: 0.1940 | max_tensorboard_logging: 2.5948 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0221 | Memory: rss_MB: 6082.7 | uss_MB: 5062.6 | child_rss_MB: 6185.4 | allocated_MB: 448.0 | reserved_MB: 29420.0 | inactive_split_MB: 876.0 +rank7 [2026-10-28 18:32:55,587] INFO : Step 8800/50000 rank7 | Train Loss: 1.5247 | Train Loss Unweighted: 0.8421 | Train Loss Components: Auxiliary: 0.2808 | Cardinality: 0.1524 | Classification: 0.1581 | Classification_binary: 0.2323 | HitGrouping: 0.1969 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2217 | Regression_eta: 0.0029 | Regression_pt: 0.2242 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0769 | Classification_binary: 0.2544 | Regression_cos_phi: 0.1328 | Regression_energy: 0.1240 | Regression_eta: 0.0819 | Regression_pt: 0.1342 | Regression_sin_phi: 0.1372 | LR: 9.33e-04 | DataLoad Time: 0.0165s | Model Forward Time: 3.9915s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0064 | max_forward: 0.7382 | max_loss: 0.2414 | max_backward: 1.7741 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.4630 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0278 | Memory: rss_MB: 6078.3 | uss_MB: 5061.0 | child_rss_MB: 6166.0 | allocated_MB: 447.9 | reserved_MB: 29420.0 | inactive_split_MB: 1260.3 +rank7 [2026-10-28 18:37:36,890] INFO : Step 8900/50000 rank7 | Train Loss: 1.7540 | Train Loss Unweighted: 0.9436 | Train Loss Components: Auxiliary: 0.3227 | Cardinality: 0.2072 | Classification: 0.1816 | Classification_binary: 0.2735 | HitGrouping: 0.1904 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2393 | Regression_eta: 0.0031 | Regression_pt: 0.2426 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0918 | Classification_binary: 0.2337 | Regression_cos_phi: 0.0908 | Regression_energy: 0.1230 | Regression_eta: 0.0733 | Regression_pt: 0.1338 | Regression_sin_phi: 0.1467 | LR: 9.31e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4476s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0068 | max_forward: 0.8077 | max_loss: 0.2089 | max_backward: 1.9976 | max_optimizer_step: 0.2211 | max_tensorboard_logging: 0.5747 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 6080.9 | uss_MB: 5000.4 | child_rss_MB: 6188.7 | allocated_MB: 451.8 | reserved_MB: 29420.0 | inactive_split_MB: 1133.8 +rank7 [2026-10-28 18:42:20,745] INFO : Step 9000/50000 rank7 | Train Loss: 1.7807 | Train Loss Unweighted: 0.9795 | Train Loss Components: Auxiliary: 0.3326 | Cardinality: 0.1942 | Classification: 0.1600 | Classification_binary: 0.3157 | HitGrouping: 0.1975 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2461 | Regression_eta: 0.0033 | Regression_pt: 0.2511 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1028 | Classification_binary: 0.3217 | Regression_cos_phi: 0.1644 | Regression_energy: 0.1295 | Regression_eta: 0.0961 | Regression_pt: 0.1423 | Regression_sin_phi: 0.1652 | LR: 9.30e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8579s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.8696 | max_loss: 0.1559 | max_backward: 1.8245 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.9520 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0271 | Memory: rss_MB: 6064.6 | uss_MB: 5051.1 | child_rss_MB: 6187.5 | allocated_MB: 448.9 | reserved_MB: 29420.0 | inactive_split_MB: 1051.6 +rank7 [2026-10-28 18:42:21,625] INFO : Running validation on rank7 +rank7 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 18:42:21,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 18:43:19,669] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:43:20,246] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 18:43:46,999] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 18:43:48,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 18:43:49,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 18:43:49,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 18:43:59,994] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 18:44:00,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:44:01,306] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 18:44:17,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 18:44:18,732] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 18:44:19,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 18:44:20,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 18:44:30,628] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 18:44:31,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 18:44:31,605] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 18:44:47,495] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 18:44:48,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 18:44:49,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 18:44:50,186] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_9000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 18:45:00,197] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 18:50:14,977] INFO : Step 9100/50000 rank7 | Train Loss: 1.6002 | Train Loss Unweighted: 0.8685 | Train Loss Components: Auxiliary: 0.3013 | Cardinality: 0.1975 | Classification: 0.1658 | Classification_binary: 0.2349 | HitGrouping: 0.1971 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2304 | Regression_eta: 0.0027 | Regression_pt: 0.2318 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1049 | Classification_binary: 0.2538 | Regression_cos_phi: 0.1508 | Regression_energy: 0.1381 | Regression_eta: 0.0856 | Regression_pt: 0.1492 | Regression_sin_phi: 0.1071 | LR: 9.28e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5177s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7981 | max_loss: 0.1460 | max_backward: 1.6475 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.9178 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0518 | Memory: rss_MB: 7065.6 | uss_MB: 6052.1 | child_rss_MB: 6167.4 | allocated_MB: 449.7 | reserved_MB: 25788.0 | inactive_split_MB: 957.3 +rank7 [2026-10-28 18:55:02,311] INFO : Step 9200/50000 rank7 | Train Loss: 1.6859 | Train Loss Unweighted: 0.8867 | Train Loss Components: Auxiliary: 0.3120 | Cardinality: 0.1906 | Classification: 0.1447 | Classification_binary: 0.2542 | HitGrouping: 0.1974 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2389 | Regression_eta: 0.0033 | Regression_pt: 0.2421 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1012 | Classification_binary: 0.2413 | Regression_cos_phi: 0.1514 | Regression_energy: 0.1225 | Regression_eta: 0.0813 | Regression_pt: 0.1350 | Regression_sin_phi: 0.1151 | LR: 9.27e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3384s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7865 | max_loss: 0.1441 | max_backward: 1.8922 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0299 | Memory: rss_MB: 6128.5 | uss_MB: 5123.6 | child_rss_MB: 6161.7 | allocated_MB: 450.8 | reserved_MB: 25788.0 | inactive_split_MB: 1040.1 +rank7 [2026-10-28 18:59:49,907] INFO : Step 9300/50000 rank7 | Train Loss: 1.8980 | Train Loss Unweighted: 0.9669 | Train Loss Components: Auxiliary: 0.3566 | Cardinality: 0.3104 | Classification: 0.1729 | Classification_binary: 0.3073 | HitGrouping: 0.1936 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2397 | Regression_eta: 0.0030 | Regression_pt: 0.2407 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1068 | Classification_binary: 0.2889 | Regression_cos_phi: 0.1146 | Regression_energy: 0.1301 | Regression_eta: 0.0740 | Regression_pt: 0.1416 | Regression_sin_phi: 0.1361 | LR: 9.25e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.1832s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0143 | max_forward: 0.7819 | max_loss: 0.1515 | max_backward: 1.8209 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6071.1 | uss_MB: 5121.9 | child_rss_MB: 5951.4 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1031.1 +rank7 [2026-10-28 19:04:46,837] INFO : Step 9400/50000 rank7 | Train Loss: 1.5878 | Train Loss Unweighted: 0.8563 | Train Loss Components: Auxiliary: 0.2880 | Cardinality: 0.1552 | Classification: 0.1580 | Classification_binary: 0.2625 | HitGrouping: 0.1887 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2141 | Regression_eta: 0.0030 | Regression_pt: 0.2153 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1333 | Classification_binary: 0.2955 | Regression_cos_phi: 0.1462 | Regression_energy: 0.1146 | Regression_eta: 0.0883 | Regression_pt: 0.1254 | Regression_sin_phi: 0.1538 | LR: 9.24e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.0189s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.7342 | max_loss: 0.2471 | max_backward: 1.7320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.4671 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0255 | Memory: rss_MB: 6067.3 | uss_MB: 5118.7 | child_rss_MB: 5945.7 | allocated_MB: 448.9 | reserved_MB: 25950.0 | inactive_split_MB: 1121.1 +rank7 [2026-10-28 19:09:47,342] INFO : Step 9500/50000 rank7 | Train Loss: 1.5519 | Train Loss Unweighted: 0.8141 | Train Loss Components: Auxiliary: 0.2846 | Cardinality: 0.1719 | Classification: 0.1393 | Classification_binary: 0.2430 | HitGrouping: 0.1977 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2129 | Regression_eta: 0.0030 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.2407 | Regression_cos_phi: 0.0852 | Regression_energy: 0.1044 | Regression_eta: 0.0597 | Regression_pt: 0.1133 | Regression_sin_phi: 0.1160 | LR: 9.22e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.1287s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0076 | max_forward: 0.7261 | max_loss: 0.1376 | max_backward: 1.8267 | max_optimizer_step: 0.1992 | max_tensorboard_logging: 0.4699 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0312 | Memory: rss_MB: 6069.1 | uss_MB: 5122.8 | child_rss_MB: 5952.7 | allocated_MB: 446.7 | reserved_MB: 27752.0 | inactive_split_MB: 1128.4 +rank7 [2026-10-28 19:14:55,454] INFO : Step 9600/50000 rank7 | Train Loss: 1.7379 | Train Loss Unweighted: 0.9171 | Train Loss Components: Auxiliary: 0.3221 | Cardinality: 0.2199 | Classification: 0.1679 | Classification_binary: 0.2886 | HitGrouping: 0.1923 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2245 | Regression_eta: 0.0032 | Regression_pt: 0.2297 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0918 | Classification_binary: 0.2521 | Regression_cos_phi: 0.1082 | Regression_energy: 0.1159 | Regression_eta: 0.0802 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0944 | LR: 9.21e-04 | DataLoad Time: 0.0173s | Model Forward Time: 7.7248s | Phase Times: max_pre_step_barrier: 0.0081 | max_batch_to_device: 0.0183 | max_forward: 3.1116 | max_loss: 0.7194 | max_backward: 4.4829 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0387 | Memory: rss_MB: 6068.7 | uss_MB: 5120.4 | child_rss_MB: 5924.4 | allocated_MB: 449.4 | reserved_MB: 29732.0 | inactive_split_MB: 1322.6 +rank7 [2026-10-28 19:20:02,940] INFO : Step 9700/50000 rank7 | Train Loss: 1.8683 | Train Loss Unweighted: 0.9602 | Train Loss Components: Auxiliary: 0.3449 | Cardinality: 0.2461 | Classification: 0.1512 | Classification_binary: 0.3271 | HitGrouping: 0.1997 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2364 | Regression_eta: 0.0037 | Regression_pt: 0.2383 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0910 | Classification_binary: 0.3041 | Regression_cos_phi: 0.1389 | Regression_energy: 0.1200 | Regression_eta: 0.0879 | Regression_pt: 0.1302 | Regression_sin_phi: 0.1702 | LR: 9.19e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0356s | Phase Times: max_pre_step_barrier: 0.0056 | max_batch_to_device: 0.0069 | max_forward: 0.8299 | max_loss: 0.7776 | max_backward: 2.5571 | max_optimizer_step: 0.2010 | max_tensorboard_logging: 0.5726 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0275 | Memory: rss_MB: 6067.5 | uss_MB: 5123.7 | child_rss_MB: 5943.1 | allocated_MB: 450.3 | reserved_MB: 29734.0 | inactive_split_MB: 1128.1 +rank7 [2026-10-28 19:25:00,573] INFO : Step 9800/50000 rank7 | Train Loss: 1.6119 | Train Loss Unweighted: 0.8200 | Train Loss Components: Auxiliary: 0.3036 | Cardinality: 0.2266 | Classification: 0.1371 | Classification_binary: 0.2410 | HitGrouping: 0.1941 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2176 | Regression_eta: 0.0027 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0793 | Classification_binary: 0.2692 | Regression_cos_phi: 0.1420 | Regression_energy: 0.1275 | Regression_eta: 0.0892 | Regression_pt: 0.1372 | Regression_sin_phi: 0.1077 | LR: 9.17e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.5938s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7430 | max_loss: 0.2467 | max_backward: 1.8961 | max_optimizer_step: 0.2008 | max_tensorboard_logging: 0.9056 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0321 | Memory: rss_MB: 6064.8 | uss_MB: 5122.0 | child_rss_MB: 5925.9 | allocated_MB: 446.9 | reserved_MB: 29734.0 | inactive_split_MB: 905.1 +rank7 [2026-10-28 19:29:40,210] INFO : Step 9900/50000 rank7 | Train Loss: 1.5408 | Train Loss Unweighted: 0.8391 | Train Loss Components: Auxiliary: 0.2884 | Cardinality: 0.1631 | Classification: 0.1532 | Classification_binary: 0.2416 | HitGrouping: 0.1919 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2178 | Regression_eta: 0.0029 | Regression_pt: 0.2207 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0770 | Classification_binary: 0.2010 | Regression_cos_phi: 0.1091 | Regression_energy: 0.1270 | Regression_eta: 0.1357 | Regression_pt: 0.1404 | Regression_sin_phi: 0.1325 | LR: 9.16e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.6031s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7611 | max_loss: 0.1425 | max_backward: 1.8994 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.9048 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 6066.1 | uss_MB: 5122.8 | child_rss_MB: 5945.5 | allocated_MB: 448.3 | reserved_MB: 29734.0 | inactive_split_MB: 1119.7 +rank7 [2026-10-28 19:34:23,378] INFO : Step 10000/50000 rank7 | Train Loss: 1.7147 | Train Loss Unweighted: 0.9225 | Train Loss Components: Auxiliary: 0.3261 | Cardinality: 0.1969 | Classification: 0.1590 | Classification_binary: 0.2912 | HitGrouping: 0.1934 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2305 | Regression_eta: 0.0029 | Regression_pt: 0.2356 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0849 | Classification_binary: 0.2661 | Regression_cos_phi: 0.1289 | Regression_energy: 0.1126 | Regression_eta: 0.0754 | Regression_pt: 0.1244 | Regression_sin_phi: 0.1236 | LR: 9.14e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8790s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.1140 | max_forward: 0.8951 | max_loss: 0.1695 | max_backward: 1.8837 | max_optimizer_step: 0.1961 | max_tensorboard_logging: 1.0339 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 6055.8 | uss_MB: 5112.2 | child_rss_MB: 5922.2 | allocated_MB: 449.4 | reserved_MB: 29734.0 | inactive_split_MB: 1320.6 +rank7 [2026-10-28 19:34:24,225] INFO : Running validation on rank7 +rank7 [2026-10-28 19:34:24,225] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 19:34:24,226] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 19:35:21,693] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:35:22,179] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 19:35:48,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 19:35:49,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 19:35:50,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 19:35:51,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 19:36:01,634] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 19:36:02,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:36:02,954] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 19:36:18,972] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 19:36:20,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 19:36:21,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 19:36:22,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 19:36:32,060] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 19:36:36,471] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 19:36:36,974] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 19:36:52,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 19:36:53,569] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 19:36:54,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 19:36:55,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_10000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 19:37:05,211] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 19:42:23,380] INFO : Step 10100/50000 rank7 | Train Loss: 1.6697 | Train Loss Unweighted: 0.8627 | Train Loss Components: Auxiliary: 0.3045 | Cardinality: 0.2287 | Classification: 0.1283 | Classification_binary: 0.2543 | HitGrouping: 0.1964 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2359 | Regression_eta: 0.0032 | Regression_pt: 0.2378 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0711 | Classification_binary: 0.2492 | Regression_cos_phi: 0.1159 | Regression_energy: 0.1105 | Regression_eta: 0.0807 | Regression_pt: 0.1202 | Regression_sin_phi: 0.1255 | LR: 9.12e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7396s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7652 | max_loss: 0.1496 | max_backward: 1.8791 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 0.9326 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0393 | Memory: rss_MB: 7065.4 | uss_MB: 6119.6 | child_rss_MB: 5927.2 | allocated_MB: 448.1 | reserved_MB: 26360.0 | inactive_split_MB: 1080.0 +rank7 [2026-10-28 19:47:10,587] INFO : Step 10200/50000 rank7 | Train Loss: 1.3968 | Train Loss Unweighted: 0.7619 | Train Loss Components: Auxiliary: 0.2598 | Cardinality: 0.1483 | Classification: 0.1293 | Classification_binary: 0.2070 | HitGrouping: 0.1867 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2090 | Regression_eta: 0.0026 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2232 | Regression_cos_phi: 0.1075 | Regression_energy: 0.1149 | Regression_eta: 0.0678 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0783 | LR: 9.11e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.4830s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0330 | max_forward: 0.8960 | max_loss: 0.5080 | max_backward: 1.9040 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.6816 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0424 | Memory: rss_MB: 6137.8 | uss_MB: 5194.1 | child_rss_MB: 5912.5 | allocated_MB: 450.3 | reserved_MB: 27130.0 | inactive_split_MB: 1088.5 +rank7 [2026-10-28 19:51:56,122] INFO : Step 10300/50000 rank7 | Train Loss: 1.6039 | Train Loss Unweighted: 0.8519 | Train Loss Components: Auxiliary: 0.2943 | Cardinality: 0.1656 | Classification: 0.1487 | Classification_binary: 0.2679 | HitGrouping: 0.1881 | Regression_cos_phi: 0.0017 | Regression_energy: 0.2132 | Regression_eta: 0.0030 | Regression_pt: 0.2158 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0927 | Classification_binary: 0.2229 | Regression_cos_phi: 0.1407 | Regression_energy: 0.1209 | Regression_eta: 0.0806 | Regression_pt: 0.1291 | Regression_sin_phi: 0.1583 | LR: 9.09e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7920s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8152 | max_loss: 0.2478 | max_backward: 2.0483 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.8541 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0326 | Memory: rss_MB: 6143.7 | uss_MB: 5198.5 | child_rss_MB: 5930.4 | allocated_MB: 448.7 | reserved_MB: 28404.0 | inactive_split_MB: 991.9 +rank7 [2026-10-28 19:56:39,451] INFO : Step 10400/50000 rank7 | Train Loss: 1.7188 | Train Loss Unweighted: 0.8798 | Train Loss Components: Auxiliary: 0.3209 | Cardinality: 0.2578 | Classification: 0.1544 | Classification_binary: 0.2910 | HitGrouping: 0.1869 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2139 | Regression_eta: 0.0029 | Regression_pt: 0.2147 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0944 | Classification_binary: 0.3022 | Regression_cos_phi: 0.1197 | Regression_energy: 0.1159 | Regression_eta: 0.0702 | Regression_pt: 0.1258 | Regression_sin_phi: 0.1109 | LR: 9.07e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6745s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7821 | max_loss: 0.2448 | max_backward: 1.8782 | max_optimizer_step: 0.2932 | max_tensorboard_logging: 0.8061 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0290 | Memory: rss_MB: 6134.3 | uss_MB: 5193.2 | child_rss_MB: 5938.4 | allocated_MB: 448.9 | reserved_MB: 28404.0 | inactive_split_MB: 983.3 +rank7 [2026-10-28 20:01:29,564] INFO : Step 10500/50000 rank7 | Train Loss: 1.4652 | Train Loss Unweighted: 0.7986 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1697 | Classification: 0.1280 | Classification_binary: 0.2159 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2232 | Regression_eta: 0.0026 | Regression_pt: 0.2263 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0792 | Classification_binary: 0.2385 | Regression_cos_phi: 0.1355 | Regression_energy: 0.1161 | Regression_eta: 0.0684 | Regression_pt: 0.1262 | Regression_sin_phi: 0.1245 | LR: 9.06e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.2142s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.7113 | max_loss: 0.1455 | max_backward: 1.6308 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.7760 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6134.1 | uss_MB: 5192.0 | child_rss_MB: 5916.0 | allocated_MB: 444.7 | reserved_MB: 28404.0 | inactive_split_MB: 1307.3 +rank7 [2026-10-28 20:06:14,521] INFO : Step 10600/50000 rank7 | Train Loss: 1.5657 | Train Loss Unweighted: 0.8341 | Train Loss Components: Auxiliary: 0.2802 | Cardinality: 0.1297 | Classification: 0.1617 | Classification_binary: 0.2471 | HitGrouping: 0.1902 | Regression_cos_phi: 0.0018 | Regression_energy: 0.2084 | Regression_eta: 0.0037 | Regression_pt: 0.2096 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0901 | Classification_binary: 0.2331 | Regression_cos_phi: 0.1719 | Regression_energy: 0.0994 | Regression_eta: 0.0934 | Regression_pt: 0.1089 | Regression_sin_phi: 0.1557 | LR: 9.04e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.9398s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0173 | max_forward: 0.7875 | max_loss: 0.2211 | max_backward: 1.9019 | max_optimizer_step: 0.2814 | max_tensorboard_logging: 0.8518 | max_tensorboard_flush: 0.0072 | max_post_step_barrier: 0.0268 | Memory: rss_MB: 6133.2 | uss_MB: 5193.0 | child_rss_MB: 5934.7 | allocated_MB: 449.8 | reserved_MB: 28404.0 | inactive_split_MB: 1130.2 +rank7 [2026-10-28 20:10:59,505] INFO : Step 10700/50000 rank7 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8676 | Train Loss Components: Auxiliary: 0.3003 | Cardinality: 0.1749 | Classification: 0.1453 | Classification_binary: 0.2956 | HitGrouping: 0.1880 | Regression_cos_phi: 0.0015 | Regression_energy: 0.2085 | Regression_eta: 0.0033 | Regression_pt: 0.2116 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.2312 | Regression_cos_phi: 0.1218 | Regression_energy: 0.1135 | Regression_eta: 0.0740 | Regression_pt: 0.1256 | Regression_sin_phi: 0.1165 | LR: 9.02e-04 | DataLoad Time: 0.0119s | Model Forward Time: 5.1535s | Phase Times: max_pre_step_barrier: 0.0305 | max_batch_to_device: 0.0154 | max_forward: 2.0246 | max_loss: 0.2864 | max_backward: 2.7424 | max_optimizer_step: 0.1980 | max_tensorboard_logging: 0.4734 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0336 | Memory: rss_MB: 6136.9 | uss_MB: 5195.8 | child_rss_MB: 5906.4 | allocated_MB: 446.2 | reserved_MB: 29148.0 | inactive_split_MB: 1134.5 +rank7 [2026-10-28 20:15:41,623] INFO : Step 10800/50000 rank7 | Train Loss: 1.5849 | Train Loss Unweighted: 0.7936 | Train Loss Components: Auxiliary: 0.2895 | Cardinality: 0.1942 | Classification: 0.1383 | Classification_binary: 0.2480 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0016 | Regression_energy: 0.2007 | Regression_eta: 0.0032 | Regression_pt: 0.2001 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0969 | Classification_binary: 0.2746 | Regression_cos_phi: 0.1265 | Regression_energy: 0.1228 | Regression_eta: 0.0808 | Regression_pt: 0.1314 | Regression_sin_phi: 0.1178 | LR: 9.00e-04 | DataLoad Time: 0.0174s | Model Forward Time: 8.1980s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7684 | max_loss: 0.2408 | max_backward: 1.7977 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 1.7470 | max_tensorboard_flush: 0.0028 | max_post_step_barrier: 0.0522 | Memory: rss_MB: 6126.4 | uss_MB: 4363.9 | child_rss_MB: 5910.3 | allocated_MB: 448.6 | reserved_MB: 30628.0 | inactive_split_MB: 1292.6 +rank7 [2026-10-28 20:20:32,832] INFO : Step 10900/50000 rank7 | Train Loss: 1.3552 | Train Loss Unweighted: 0.7247 | Train Loss Components: Auxiliary: 0.2531 | Cardinality: 0.1449 | Classification: 0.1275 | Classification_binary: 0.2074 | HitGrouping: 0.1874 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1917 | Regression_eta: 0.0024 | Regression_pt: 0.1933 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0639 | Classification_binary: 0.1952 | Regression_cos_phi: 0.0861 | Regression_energy: 0.1080 | Regression_eta: 0.0651 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0933 | LR: 8.99e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4556s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0065 | max_forward: 0.7076 | max_loss: 0.2425 | max_backward: 1.6536 | max_optimizer_step: 0.2002 | max_tensorboard_logging: 0.9432 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0316 | Memory: rss_MB: 6131.8 | uss_MB: 5191.3 | child_rss_MB: 5932.5 | allocated_MB: 446.4 | reserved_MB: 30628.0 | inactive_split_MB: 984.9 +rank7 [2026-10-28 20:25:36,607] INFO : Step 11000/50000 rank7 | Train Loss: 1.5999 | Train Loss Unweighted: 0.8322 | Train Loss Components: Auxiliary: 0.2959 | Cardinality: 0.1737 | Classification: 0.1477 | Classification_binary: 0.2877 | HitGrouping: 0.1892 | Regression_cos_phi: 0.0017 | Regression_energy: 0.1943 | Regression_eta: 0.0030 | Regression_pt: 0.1965 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0879 | Classification_binary: 0.2713 | Regression_cos_phi: 0.1369 | Regression_energy: 0.1072 | Regression_eta: 0.0764 | Regression_pt: 0.1172 | Regression_sin_phi: 0.1783 | LR: 8.97e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.9308s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8260 | max_loss: 0.1503 | max_backward: 1.9603 | max_optimizer_step: 0.1969 | max_tensorboard_logging: 1.0383 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 6124.9 | uss_MB: 5185.7 | child_rss_MB: 5918.3 | allocated_MB: 451.8 | reserved_MB: 30628.0 | inactive_split_MB: 996.6 +rank7 [2026-10-28 20:25:37,479] INFO : Running validation on rank7 +rank7 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 20:25:37,479] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 20:26:32,845] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:26:33,362] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 20:27:00,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 20:27:01,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 20:27:02,354] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 20:27:03,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 20:27:13,327] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 20:27:14,319] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:27:14,834] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 20:27:31,233] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 20:27:32,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 20:27:34,048] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 20:27:35,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 20:27:45,116] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-28 20:27:46,111] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 20:27:46,649] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 20:28:02,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 20:28:02,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 20:28:03,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 20:28:04,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_11000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 20:28:14,518] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 20:33:29,189] INFO : Step 11100/50000 rank7 | Train Loss: 1.6214 | Train Loss Unweighted: 0.8328 | Train Loss Components: Auxiliary: 0.3021 | Cardinality: 0.2186 | Classification: 0.1350 | Classification_binary: 0.2802 | HitGrouping: 0.1851 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2042 | Regression_eta: 0.0030 | Regression_pt: 0.2075 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.2320 | Regression_cos_phi: 0.1004 | Regression_energy: 0.1057 | Regression_eta: 0.0726 | Regression_pt: 0.1156 | Regression_sin_phi: 0.1122 | LR: 8.95e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.8750s | Phase Times: max_pre_step_barrier: 0.0088 | max_batch_to_device: 0.0068 | max_forward: 0.8488 | max_loss: 0.1548 | max_backward: 1.7645 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 1.1046 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0433 | Memory: rss_MB: 7069.9 | uss_MB: 6125.7 | child_rss_MB: 5940.1 | allocated_MB: 448.1 | reserved_MB: 24340.0 | inactive_split_MB: 1081.9 +rank7 [2026-10-28 20:38:18,707] INFO : Step 11200/50000 rank7 | Train Loss: 1.7050 | Train Loss Unweighted: 0.8927 | Train Loss Components: Auxiliary: 0.3271 | Cardinality: 0.2640 | Classification: 0.1534 | Classification_binary: 0.2523 | HitGrouping: 0.1990 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2404 | Regression_eta: 0.0030 | Regression_pt: 0.2410 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1221 | Classification_binary: 0.2939 | Regression_cos_phi: 0.1356 | Regression_energy: 0.1254 | Regression_eta: 0.1010 | Regression_pt: 0.1357 | Regression_sin_phi: 0.1643 | LR: 8.93e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.8971s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8707 | max_loss: 0.2001 | max_backward: 2.2037 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.7912 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0771 | Memory: rss_MB: 6163.4 | uss_MB: 5218.5 | child_rss_MB: 5949.2 | allocated_MB: 447.8 | reserved_MB: 26392.0 | inactive_split_MB: 938.7 +rank7 [2026-10-28 20:43:01,723] INFO : Step 11300/50000 rank7 | Train Loss: 1.3885 | Train Loss Unweighted: 0.7511 | Train Loss Components: Auxiliary: 0.2594 | Cardinality: 0.1357 | Classification: 0.1316 | Classification_binary: 0.2357 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1874 | Regression_eta: 0.0024 | Regression_pt: 0.1915 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.2263 | Regression_cos_phi: 0.0835 | Regression_energy: 0.1091 | Regression_eta: 0.0710 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0816 | LR: 8.91e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.4287s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8053 | max_loss: 0.1469 | max_backward: 2.0062 | max_optimizer_step: 0.2031 | max_tensorboard_logging: 0.4618 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.1345 | Memory: rss_MB: 6165.0 | uss_MB: 5222.0 | child_rss_MB: 5948.8 | allocated_MB: 450.6 | reserved_MB: 26400.0 | inactive_split_MB: 1092.9 +rank7 [2026-10-28 20:48:05,522] INFO : Step 11400/50000 rank7 | Train Loss: 1.5865 | Train Loss Unweighted: 0.8323 | Train Loss Components: Auxiliary: 0.2981 | Cardinality: 0.1866 | Classification: 0.1338 | Classification_binary: 0.2899 | HitGrouping: 0.1840 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1999 | Regression_eta: 0.0028 | Regression_pt: 0.2030 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0748 | Classification_binary: 0.2576 | Regression_cos_phi: 0.1089 | Regression_energy: 0.0971 | Regression_eta: 0.0727 | Regression_pt: 0.1075 | Regression_sin_phi: 0.1168 | LR: 8.89e-04 | DataLoad Time: 0.0206s | Model Forward Time: 5.7649s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0769 | max_forward: 1.3800 | max_loss: 0.6715 | max_backward: 3.0339 | max_optimizer_step: 0.2660 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6163.4 | uss_MB: 5219.6 | child_rss_MB: 5946.4 | allocated_MB: 448.1 | reserved_MB: 27144.0 | inactive_split_MB: 1081.9 +rank7 [2026-10-28 20:52:57,974] INFO : Step 11500/50000 rank7 | Train Loss: 1.4825 | Train Loss Unweighted: 0.7715 | Train Loss Components: Auxiliary: 0.2779 | Cardinality: 0.1999 | Classification: 0.1273 | Classification_binary: 0.2463 | HitGrouping: 0.1823 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1956 | Regression_eta: 0.0026 | Regression_pt: 0.1972 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0841 | Classification_binary: 0.2737 | Regression_cos_phi: 0.1368 | Regression_energy: 0.1293 | Regression_eta: 0.0781 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0862 | LR: 8.88e-04 | DataLoad Time: 0.0163s | Model Forward Time: 5.0557s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.7822 | max_loss: 0.1471 | max_backward: 1.7308 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 1.4583 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0307 | Memory: rss_MB: 6164.2 | uss_MB: 5219.4 | child_rss_MB: 5944.4 | allocated_MB: 450.8 | reserved_MB: 27144.0 | inactive_split_MB: 1342.3 +rank7 [2026-10-28 20:57:41,223] INFO : Step 11600/50000 rank7 | Train Loss: 1.4219 | Train Loss Unweighted: 0.7444 | Train Loss Components: Auxiliary: 0.2603 | Cardinality: 0.1679 | Classification: 0.1415 | Classification_binary: 0.2106 | HitGrouping: 0.1856 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1928 | Regression_eta: 0.0022 | Regression_pt: 0.1945 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.1037 | Classification_binary: 0.2087 | Regression_cos_phi: 0.0990 | Regression_energy: 0.1017 | Regression_eta: 0.0608 | Regression_pt: 0.1116 | Regression_sin_phi: 0.1204 | LR: 8.86e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6999s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8362 | max_loss: 0.1428 | max_backward: 2.0109 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6156.9 | uss_MB: 5220.2 | child_rss_MB: 5947.3 | allocated_MB: 448.6 | reserved_MB: 27146.0 | inactive_split_MB: 1315.4 +rank7 [2026-10-28 21:02:35,116] INFO : Step 11700/50000 rank7 | Train Loss: 1.5203 | Train Loss Unweighted: 0.8017 | Train Loss Components: Auxiliary: 0.2866 | Cardinality: 0.1889 | Classification: 0.1302 | Classification_binary: 0.2577 | HitGrouping: 0.1852 | Regression_cos_phi: 0.0013 | Regression_energy: 0.2024 | Regression_eta: 0.0026 | Regression_pt: 0.2061 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0740 | Classification_binary: 0.2258 | Regression_cos_phi: 0.1147 | Regression_energy: 0.1077 | Regression_eta: 0.0687 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0881 | LR: 8.84e-04 | DataLoad Time: 0.0192s | Model Forward Time: 5.3159s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 1.0014 | max_loss: 0.1975 | max_backward: 2.5347 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.8286 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0688 | Memory: rss_MB: 6157.0 | uss_MB: 5144.6 | child_rss_MB: 5922.6 | allocated_MB: 448.7 | reserved_MB: 27266.0 | inactive_split_MB: 1090.9 +rank7 [2026-10-28 21:07:42,591] INFO : Step 11800/50000 rank7 | Train Loss: 1.5591 | Train Loss Unweighted: 0.7907 | Train Loss Components: Auxiliary: 0.2873 | Cardinality: 0.2051 | Classification: 0.1346 | Classification_binary: 0.2655 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1918 | Regression_eta: 0.0030 | Regression_pt: 0.1929 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0974 | Classification_binary: 0.3132 | Regression_cos_phi: 0.1495 | Regression_energy: 0.1084 | Regression_eta: 0.0952 | Regression_pt: 0.1187 | Regression_sin_phi: 0.1393 | LR: 8.82e-04 | DataLoad Time: 0.0180s | Model Forward Time: 7.4783s | Phase Times: max_pre_step_barrier: 0.0092 | max_batch_to_device: 0.0966 | max_forward: 1.7907 | max_loss: 0.6496 | max_backward: 2.7103 | max_optimizer_step: 0.2018 | max_tensorboard_logging: 2.3587 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 6136.9 | uss_MB: 5209.7 | child_rss_MB: 5946.4 | allocated_MB: 450.4 | reserved_MB: 27266.0 | inactive_split_MB: 1187.2 +rank7 [2026-10-28 21:12:51,258] INFO : Step 11900/50000 rank7 | Train Loss: 1.3283 | Train Loss Unweighted: 0.7113 | Train Loss Components: Auxiliary: 0.2509 | Cardinality: 0.1476 | Classification: 0.1193 | Classification_binary: 0.2094 | HitGrouping: 0.1824 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1880 | Regression_eta: 0.0021 | Regression_pt: 0.1901 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0807 | Classification_binary: 0.2347 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0989 | Regression_eta: 0.0697 | Regression_pt: 0.1085 | Regression_sin_phi: 0.0935 | LR: 8.80e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.1694s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0785 | max_forward: 0.7917 | max_loss: 0.1465 | max_backward: 1.5595 | max_optimizer_step: 0.2394 | max_tensorboard_logging: 0.6251 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 6137.9 | uss_MB: 5216.4 | child_rss_MB: 5947.1 | allocated_MB: 443.9 | reserved_MB: 27266.0 | inactive_split_MB: 955.3 +rank7 [2026-10-28 21:17:48,734] INFO : Step 12000/50000 rank7 | Train Loss: 1.4074 | Train Loss Unweighted: 0.7415 | Train Loss Components: Auxiliary: 0.2683 | Cardinality: 0.1719 | Classification: 0.1235 | Classification_binary: 0.2283 | HitGrouping: 0.1812 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1909 | Regression_eta: 0.0022 | Regression_pt: 0.1941 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0651 | Classification_binary: 0.1915 | Regression_cos_phi: 0.0618 | Regression_energy: 0.0950 | Regression_eta: 0.0403 | Regression_pt: 0.1037 | Regression_sin_phi: 0.0691 | LR: 8.78e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.2510s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0303 | max_forward: 0.7589 | max_loss: 0.1459 | max_backward: 1.9073 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.5003 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0279 | Memory: rss_MB: 6136.6 | uss_MB: 5219.4 | child_rss_MB: 5952.0 | allocated_MB: 447.6 | reserved_MB: 27266.0 | inactive_split_MB: 1310.4 +rank7 [2026-10-28 21:17:49,718] INFO : Running validation on rank7 +rank7 [2026-10-28 21:17:49,719] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 21:17:49,719] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 21:18:44,923] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:18:45,601] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 21:19:11,629] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 21:19:12,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 21:19:13,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 21:19:14,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 21:19:24,610] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 21:19:25,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:19:25,829] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 21:19:42,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 21:19:43,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 21:19:44,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 21:19:45,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 21:19:55,036] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 21:19:55,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 21:19:56,382] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 21:20:12,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 21:20:12,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 21:20:13,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 21:20:14,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_12000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 21:20:24,507] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-28 21:25:33,195] INFO : Step 12100/50000 rank7 | Train Loss: 1.6436 | Train Loss Unweighted: 0.8521 | Train Loss Components: Auxiliary: 0.3093 | Cardinality: 0.2088 | Classification: 0.1337 | Classification_binary: 0.2971 | HitGrouping: 0.1871 | Regression_cos_phi: 0.0014 | Regression_energy: 0.2069 | Regression_eta: 0.0029 | Regression_pt: 0.2085 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0864 | Classification_binary: 0.2814 | Regression_cos_phi: 0.0996 | Regression_energy: 0.1083 | Regression_eta: 0.0798 | Regression_pt: 0.1176 | Regression_sin_phi: 0.1271 | LR: 8.76e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9399s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8139 | max_loss: 0.1586 | max_backward: 1.9999 | max_optimizer_step: 0.2141 | max_tensorboard_logging: 0.9439 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 7027.3 | uss_MB: 6104.0 | child_rss_MB: 5925.8 | allocated_MB: 450.8 | reserved_MB: 28008.0 | inactive_split_MB: 1125.6 +rank7 [2026-10-28 21:30:25,390] INFO : Step 12200/50000 rank7 | Train Loss: 1.4655 | Train Loss Unweighted: 0.7431 | Train Loss Components: Auxiliary: 0.2757 | Cardinality: 0.1845 | Classification: 0.1184 | Classification_binary: 0.2450 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1862 | Regression_eta: 0.0026 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0800 | Classification_binary: 0.2778 | Regression_cos_phi: 0.1466 | Regression_energy: 0.0883 | Regression_eta: 0.0700 | Regression_pt: 0.0956 | Regression_sin_phi: 0.1050 | LR: 8.74e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4992s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0364 | max_forward: 0.7684 | max_loss: 0.1438 | max_backward: 1.8240 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.7243 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0697 | Memory: rss_MB: 6148.5 | uss_MB: 5233.4 | child_rss_MB: 5949.1 | allocated_MB: 445.9 | reserved_MB: 28008.0 | inactive_split_MB: 995.3 +rank7 [2026-10-28 21:35:16,327] INFO : Step 12300/50000 rank7 | Train Loss: 1.3845 | Train Loss Unweighted: 0.7270 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1523 | Classification: 0.1341 | Classification_binary: 0.2109 | HitGrouping: 0.1827 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1869 | Regression_eta: 0.0026 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.1936 | Regression_cos_phi: 0.0940 | Regression_energy: 0.0993 | Regression_eta: 0.0684 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0716 | LR: 8.72e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5876s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7264 | max_loss: 0.1976 | max_backward: 1.8534 | max_optimizer_step: 0.2993 | max_tensorboard_logging: 0.8107 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 6147.3 | uss_MB: 5233.4 | child_rss_MB: 5944.1 | allocated_MB: 448.1 | reserved_MB: 29988.0 | inactive_split_MB: 1298.2 +rank7 [2026-10-28 21:40:15,082] INFO : Step 12400/50000 rank7 | Train Loss: 1.4006 | Train Loss Unweighted: 0.7353 | Train Loss Components: Auxiliary: 0.2654 | Cardinality: 0.1602 | Classification: 0.1310 | Classification_binary: 0.2363 | HitGrouping: 0.1768 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1802 | Regression_eta: 0.0024 | Regression_pt: 0.1828 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0913 | Classification_binary: 0.2487 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0973 | Regression_eta: 0.0597 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0797 | LR: 8.70e-04 | DataLoad Time: 0.0178s | Model Forward Time: 7.1200s | Phase Times: max_pre_step_barrier: 0.0107 | max_batch_to_device: 0.0207 | max_forward: 1.2284 | max_loss: 0.2740 | max_backward: 2.6428 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.8714 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 6147.7 | uss_MB: 5022.1 | child_rss_MB: 5936.9 | allocated_MB: 448.8 | reserved_MB: 29988.0 | inactive_split_MB: 1332.2 +rank7 [2026-10-28 21:45:07,595] INFO : Step 12500/50000 rank7 | Train Loss: 1.4931 | Train Loss Unweighted: 0.7691 | Train Loss Components: Auxiliary: 0.2809 | Cardinality: 0.1768 | Classification: 0.1197 | Classification_binary: 0.2587 | HitGrouping: 0.1829 | Regression_cos_phi: 0.0015 | Regression_energy: 0.1910 | Regression_eta: 0.0024 | Regression_pt: 0.1944 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.2714 | Regression_cos_phi: 0.1157 | Regression_energy: 0.1191 | Regression_eta: 0.0784 | Regression_pt: 0.1303 | Regression_sin_phi: 0.1195 | LR: 8.68e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.7414s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0333 | max_forward: 0.8806 | max_loss: 0.1518 | max_backward: 2.1554 | max_optimizer_step: 0.2180 | max_tensorboard_logging: 0.5983 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 6144.5 | uss_MB: 5232.3 | child_rss_MB: 5946.8 | allocated_MB: 452.8 | reserved_MB: 29990.0 | inactive_split_MB: 1046.7 +rank7 [2026-10-28 21:50:03,650] INFO : Step 12600/50000 rank7 | Train Loss: 1.3547 | Train Loss Unweighted: 0.7028 | Train Loss Components: Auxiliary: 0.2487 | Cardinality: 0.1629 | Classification: 0.1033 | Classification_binary: 0.2112 | HitGrouping: 0.1849 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1912 | Regression_eta: 0.0024 | Regression_pt: 0.1923 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.2143 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0959 | Regression_eta: 0.0802 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0930 | LR: 8.66e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5167s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0058 | max_forward: 0.7639 | max_loss: 0.2516 | max_backward: 1.8947 | max_optimizer_step: 0.2049 | max_tensorboard_logging: 0.7350 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6141.1 | uss_MB: 5233.0 | child_rss_MB: 5920.5 | allocated_MB: 446.9 | reserved_MB: 29990.0 | inactive_split_MB: 1034.9 +rank7 [2026-10-28 21:54:54,645] INFO : Step 12700/50000 rank7 | Train Loss: 1.3557 | Train Loss Unweighted: 0.7189 | Train Loss Components: Auxiliary: 0.2588 | Cardinality: 0.1573 | Classification: 0.1262 | Classification_binary: 0.2132 | HitGrouping: 0.1878 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1863 | Regression_eta: 0.0022 | Regression_pt: 0.1887 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0677 | Classification_binary: 0.2216 | Regression_cos_phi: 0.0789 | Regression_energy: 0.1000 | Regression_eta: 0.0475 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0736 | LR: 8.64e-04 | DataLoad Time: 0.0165s | Model Forward Time: 6.1264s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7499 | max_loss: 0.2453 | max_backward: 3.4334 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7784 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 6139.6 | uss_MB: 5232.1 | child_rss_MB: 5948.4 | allocated_MB: 450.2 | reserved_MB: 29990.0 | inactive_split_MB: 1245.6 +rank7 [2026-10-28 21:59:39,371] INFO : Step 12800/50000 rank7 | Train Loss: 1.5298 | Train Loss Unweighted: 0.7754 | Train Loss Components: Auxiliary: 0.2870 | Cardinality: 0.1939 | Classification: 0.1237 | Classification_binary: 0.2821 | HitGrouping: 0.1822 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1810 | Regression_eta: 0.0027 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2645 | Regression_cos_phi: 0.1076 | Regression_energy: 0.1012 | Regression_eta: 0.1156 | Regression_pt: 0.1115 | Regression_sin_phi: 0.1671 | LR: 8.62e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.2916s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8170 | max_loss: 0.5299 | max_backward: 2.3185 | max_optimizer_step: 0.2016 | max_tensorboard_logging: 0.9775 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0280 | Memory: rss_MB: 6133.4 | uss_MB: 4813.1 | child_rss_MB: 5927.2 | allocated_MB: 449.9 | reserved_MB: 32006.0 | inactive_split_MB: 889.3 +rank7 [2026-10-28 22:04:39,769] INFO : Step 12900/50000 rank7 | Train Loss: 1.3302 | Train Loss Unweighted: 0.6969 | Train Loss Components: Auxiliary: 0.2540 | Cardinality: 0.1576 | Classification: 0.1054 | Classification_binary: 0.2343 | HitGrouping: 0.1811 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1758 | Regression_eta: 0.0020 | Regression_pt: 0.1771 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0764 | Classification_binary: 0.2832 | Regression_cos_phi: 0.0853 | Regression_energy: 0.0976 | Regression_eta: 0.0580 | Regression_pt: 0.1068 | Regression_sin_phi: 0.1052 | LR: 8.60e-04 | DataLoad Time: 0.0205s | Model Forward Time: 5.6396s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0137 | max_forward: 0.9836 | max_loss: 0.2293 | max_backward: 2.2888 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.9302 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6139.6 | uss_MB: 5231.4 | child_rss_MB: 5950.5 | allocated_MB: 445.9 | reserved_MB: 32006.0 | inactive_split_MB: 1247.6 +rank7 [2026-10-28 22:09:25,338] INFO : Step 13000/50000 rank7 | Train Loss: 1.3124 | Train Loss Unweighted: 0.7073 | Train Loss Components: Auxiliary: 0.2484 | Cardinality: 0.1406 | Classification: 0.1147 | Classification_binary: 0.1973 | HitGrouping: 0.1820 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1944 | Regression_eta: 0.0023 | Regression_pt: 0.1962 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0829 | Classification_binary: 0.2204 | Regression_cos_phi: 0.0952 | Regression_energy: 0.1010 | Regression_eta: 0.0615 | Regression_pt: 0.1090 | Regression_sin_phi: 0.1181 | LR: 8.58e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2800s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0065 | max_forward: 0.7321 | max_loss: 0.1616 | max_backward: 1.7374 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0292 | Memory: rss_MB: 6140.2 | uss_MB: 5231.6 | child_rss_MB: 5949.6 | allocated_MB: 444.7 | reserved_MB: 32006.0 | inactive_split_MB: 1300.2 +rank7 [2026-10-28 22:09:26,329] INFO : Running validation on rank7 +rank7 [2026-10-28 22:09:26,329] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 22:09:26,330] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 22:10:21,900] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:10:22,476] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 22:10:48,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 22:10:49,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 22:10:50,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 22:10:51,588] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 22:11:01,598] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 22:11:02,352] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:11:02,875] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 22:11:19,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 22:11:20,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 22:11:21,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 22:11:22,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 22:11:32,079] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 22:11:32,691] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 22:11:33,218] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 22:11:49,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 22:11:50,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 22:11:51,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 22:11:51,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_13000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 22:12:01,978] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 22:17:08,486] INFO : Step 13100/50000 rank7 | Train Loss: 1.6165 | Train Loss Unweighted: 0.7947 | Train Loss Components: Auxiliary: 0.3082 | Cardinality: 0.2635 | Classification: 0.1364 | Classification_binary: 0.2628 | HitGrouping: 0.1835 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1945 | Regression_eta: 0.0026 | Regression_pt: 0.1957 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0874 | Classification_binary: 0.2264 | Regression_cos_phi: 0.1390 | Regression_energy: 0.1008 | Regression_eta: 0.0628 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0958 | LR: 8.56e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2707s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0479 | max_forward: 0.8994 | max_loss: 0.2807 | max_backward: 2.1108 | max_optimizer_step: 0.2760 | max_tensorboard_logging: 0.9690 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0906 | Memory: rss_MB: 7057.8 | uss_MB: 6143.5 | child_rss_MB: 5948.1 | allocated_MB: 446.5 | reserved_MB: 23120.0 | inactive_split_MB: 920.0 +rank7 [2026-10-28 22:22:01,650] INFO : Step 13200/50000 rank7 | Train Loss: 1.4928 | Train Loss Unweighted: 0.7469 | Train Loss Components: Auxiliary: 0.2817 | Cardinality: 0.2211 | Classification: 0.1278 | Classification_binary: 0.2505 | HitGrouping: 0.1746 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1808 | Regression_eta: 0.0025 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0785 | Classification_binary: 0.2544 | Regression_cos_phi: 0.1137 | Regression_energy: 0.0920 | Regression_eta: 0.0532 | Regression_pt: 0.1003 | Regression_sin_phi: 0.1110 | LR: 8.54e-04 | DataLoad Time: 0.0196s | Model Forward Time: 6.5985s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0063 | max_forward: 0.7755 | max_loss: 0.2864 | max_backward: 3.4589 | max_optimizer_step: 0.1832 | max_tensorboard_logging: 1.1887 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0261 | Memory: rss_MB: 6222.1 | uss_MB: 5074.0 | child_rss_MB: 5943.3 | allocated_MB: 448.3 | reserved_MB: 25156.0 | inactive_split_MB: 978.0 +rank7 [2026-10-28 22:26:52,973] INFO : Step 13300/50000 rank7 | Train Loss: 1.3251 | Train Loss Unweighted: 0.6741 | Train Loss Components: Auxiliary: 0.2496 | Cardinality: 0.1607 | Classification: 0.1132 | Classification_binary: 0.1927 | HitGrouping: 0.1773 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1805 | Regression_eta: 0.0024 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.2217 | Regression_cos_phi: 0.1154 | Regression_energy: 0.0943 | Regression_eta: 0.0641 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0997 | LR: 8.52e-04 | DataLoad Time: 0.0145s | Model Forward Time: 4.3806s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8118 | max_loss: 0.1467 | max_backward: 1.5503 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.9039 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0252 | Memory: rss_MB: 6221.5 | uss_MB: 5307.8 | child_rss_MB: 5952.8 | allocated_MB: 446.0 | reserved_MB: 26736.0 | inactive_split_MB: 965.8 +rank7 [2026-10-28 22:31:44,118] INFO : Step 13400/50000 rank7 | Train Loss: 1.2635 | Train Loss Unweighted: 0.6770 | Train Loss Components: Auxiliary: 0.2430 | Cardinality: 0.1436 | Classification: 0.1317 | Classification_binary: 0.2016 | HitGrouping: 0.1790 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1692 | Regression_eta: 0.0019 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0768 | Classification_binary: 0.1676 | Regression_cos_phi: 0.0519 | Regression_energy: 0.0819 | Regression_eta: 0.0350 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0551 | LR: 8.50e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7359s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8396 | max_loss: 0.1412 | max_backward: 2.0130 | max_optimizer_step: 0.2029 | max_tensorboard_logging: 0.8047 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0298 | Memory: rss_MB: 6225.0 | uss_MB: 5310.9 | child_rss_MB: 5925.4 | allocated_MB: 445.8 | reserved_MB: 26736.0 | inactive_split_MB: 1014.4 +rank7 [2026-10-28 22:36:27,627] INFO : Step 13500/50000 rank7 | Train Loss: 1.3935 | Train Loss Unweighted: 0.7401 | Train Loss Components: Auxiliary: 0.2660 | Cardinality: 0.1713 | Classification: 0.1245 | Classification_binary: 0.2541 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1777 | Regression_eta: 0.0021 | Regression_pt: 0.1793 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2430 | Regression_cos_phi: 0.0826 | Regression_energy: 0.0887 | Regression_eta: 0.0635 | Regression_pt: 0.0953 | Regression_sin_phi: 0.0899 | LR: 8.48e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5753s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0859 | max_forward: 0.8721 | max_loss: 0.1498 | max_backward: 1.7581 | max_optimizer_step: 0.2023 | max_tensorboard_logging: 0.7855 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6221.6 | uss_MB: 5308.9 | child_rss_MB: 5951.2 | allocated_MB: 449.1 | reserved_MB: 26736.0 | inactive_split_MB: 922.5 +rank7 [2026-10-28 22:41:19,302] INFO : Step 13600/50000 rank7 | Train Loss: 1.3997 | Train Loss Unweighted: 0.6971 | Train Loss Components: Auxiliary: 0.2584 | Cardinality: 0.1870 | Classification: 0.1206 | Classification_binary: 0.2286 | HitGrouping: 0.1765 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1702 | Regression_eta: 0.0026 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.2324 | Regression_cos_phi: 0.1208 | Regression_energy: 0.0801 | Regression_eta: 0.0847 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0997 | LR: 8.45e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.2256s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0070 | max_forward: 0.7703 | max_loss: 0.2484 | max_backward: 1.6943 | max_optimizer_step: 0.2034 | max_tensorboard_logging: 0.5873 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0755 | Memory: rss_MB: 6210.7 | uss_MB: 5299.9 | child_rss_MB: 5953.4 | allocated_MB: 450.3 | reserved_MB: 26736.0 | inactive_split_MB: 922.0 +rank7 [2026-10-28 22:46:12,326] INFO : Step 13700/50000 rank7 | Train Loss: 1.2648 | Train Loss Unweighted: 0.6745 | Train Loss Components: Auxiliary: 0.2420 | Cardinality: 0.1315 | Classification: 0.1136 | Classification_binary: 0.1964 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1793 | Regression_eta: 0.0020 | Regression_pt: 0.1809 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.2120 | Regression_cos_phi: 0.0970 | Regression_energy: 0.0951 | Regression_eta: 0.0589 | Regression_pt: 0.1020 | Regression_sin_phi: 0.1166 | LR: 8.43e-04 | DataLoad Time: 0.0169s | Model Forward Time: 6.2736s | Phase Times: max_pre_step_barrier: 0.0406 | max_batch_to_device: 0.0119 | max_forward: 1.2281 | max_loss: 0.2480 | max_backward: 3.6472 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.6460 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0767 | Memory: rss_MB: 6216.0 | uss_MB: 5307.4 | child_rss_MB: 5948.4 | allocated_MB: 446.9 | reserved_MB: 26742.0 | inactive_split_MB: 1058.7 +rank7 [2026-10-28 22:51:09,233] INFO : Step 13800/50000 rank7 | Train Loss: 1.3469 | Train Loss Unweighted: 0.7266 | Train Loss Components: Auxiliary: 0.2604 | Cardinality: 0.1436 | Classification: 0.1184 | Classification_binary: 0.2382 | HitGrouping: 0.1797 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1812 | Regression_eta: 0.0021 | Regression_pt: 0.1844 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0801 | Classification_binary: 0.2116 | Regression_cos_phi: 0.0806 | Regression_energy: 0.0966 | Regression_eta: 0.0475 | Regression_pt: 0.1055 | Regression_sin_phi: 0.1108 | LR: 8.41e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.8519s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.7995 | max_loss: 0.1458 | max_backward: 3.4735 | max_optimizer_step: 0.1984 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0362 | Memory: rss_MB: 6221.0 | uss_MB: 5310.1 | child_rss_MB: 5919.7 | allocated_MB: 446.4 | reserved_MB: 27082.0 | inactive_split_MB: 1370.3 +rank7 [2026-10-28 22:56:04,349] INFO : Step 13900/50000 rank7 | Train Loss: 1.4882 | Train Loss Unweighted: 0.7682 | Train Loss Components: Auxiliary: 0.2793 | Cardinality: 0.1631 | Classification: 0.1199 | Classification_binary: 0.2624 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1900 | Regression_eta: 0.0029 | Regression_pt: 0.1902 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0657 | Classification_binary: 0.2593 | Regression_cos_phi: 0.0918 | Regression_energy: 0.0904 | Regression_eta: 0.0550 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0914 | LR: 8.39e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4533s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0071 | max_forward: 0.7750 | max_loss: 0.1563 | max_backward: 1.5716 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.8868 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0322 | Memory: rss_MB: 6217.9 | uss_MB: 5307.2 | child_rss_MB: 5946.5 | allocated_MB: 449.9 | reserved_MB: 27316.0 | inactive_split_MB: 918.0 +rank7 [2026-10-28 23:00:54,751] INFO : Step 14000/50000 rank7 | Train Loss: 1.2239 | Train Loss Unweighted: 0.6384 | Train Loss Components: Auxiliary: 0.2362 | Cardinality: 0.1573 | Classification: 0.0898 | Classification_binary: 0.1853 | HitGrouping: 0.1730 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1780 | Regression_eta: 0.0021 | Regression_pt: 0.1814 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0408 | Classification_binary: 0.2016 | Regression_cos_phi: 0.1123 | Regression_energy: 0.0827 | Regression_eta: 0.0506 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0649 | LR: 8.37e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5030s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7611 | max_loss: 0.1442 | max_backward: 1.7887 | max_optimizer_step: 0.2976 | max_tensorboard_logging: 0.7418 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0334 | Memory: rss_MB: 6219.2 | uss_MB: 5309.9 | child_rss_MB: 5948.9 | allocated_MB: 446.8 | reserved_MB: 29264.0 | inactive_split_MB: 1150.9 +rank7 [2026-10-28 23:00:55,745] INFO : Running validation on rank7 +rank7 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 23:00:55,745] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 23:01:50,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:01:51,152] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 23:02:17,721] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 23:02:19,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 23:02:20,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 23:02:21,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 23:02:31,950] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-28 23:02:33,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:02:33,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 23:02:50,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 23:02:51,185] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 23:02:52,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 23:02:53,068] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 23:03:03,077] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 23:03:03,407] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:03:03,862] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 23:03:19,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 23:03:20,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 23:03:21,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 23:03:21,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_14000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 23:03:31,949] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 23:08:22,314] INFO : Step 14100/50000 rank7 | Train Loss: 1.3472 | Train Loss Unweighted: 0.6964 | Train Loss Components: Auxiliary: 0.2541 | Cardinality: 0.1511 | Classification: 0.1164 | Classification_binary: 0.2090 | HitGrouping: 0.1802 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1819 | Regression_eta: 0.0023 | Regression_pt: 0.1843 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1337 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0856 | Regression_eta: 0.0257 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0434 | LR: 8.35e-04 | DataLoad Time: 0.0157s | Model Forward Time: 5.2188s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.8885 | max_loss: 0.1430 | max_backward: 2.1784 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 1.1214 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0446 | Memory: rss_MB: 7149.4 | uss_MB: 6238.2 | child_rss_MB: 5960.3 | allocated_MB: 451.8 | reserved_MB: 25710.0 | inactive_split_MB: 1103.7 +rank7 [2026-10-28 23:12:52,861] INFO : Step 14200/50000 rank7 | Train Loss: 1.4358 | Train Loss Unweighted: 0.7354 | Train Loss Components: Auxiliary: 0.2744 | Cardinality: 0.1685 | Classification: 0.1248 | Classification_binary: 0.2623 | HitGrouping: 0.1814 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1709 | Regression_eta: 0.0023 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0787 | Classification_binary: 0.2477 | Regression_cos_phi: 0.1067 | Regression_energy: 0.0866 | Regression_eta: 0.0709 | Regression_pt: 0.0937 | Regression_sin_phi: 0.1130 | LR: 8.32e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.3439s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0875 | max_forward: 0.8579 | max_loss: 0.1577 | max_backward: 1.8303 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.4810 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6333.7 | uss_MB: 5422.0 | child_rss_MB: 5960.5 | allocated_MB: 449.2 | reserved_MB: 27638.0 | inactive_split_MB: 1319.6 +rank7 [2026-10-28 23:17:26,477] INFO : Step 14300/50000 rank7 | Train Loss: 1.2642 | Train Loss Unweighted: 0.6676 | Train Loss Components: Auxiliary: 0.2452 | Cardinality: 0.1652 | Classification: 0.1069 | Classification_binary: 0.2108 | HitGrouping: 0.1701 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1718 | Regression_eta: 0.0018 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0490 | Classification_binary: 0.2214 | Regression_cos_phi: 0.0745 | Regression_energy: 0.0942 | Regression_eta: 0.0480 | Regression_pt: 0.1046 | Regression_sin_phi: 0.0684 | LR: 8.30e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.3466s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.7599 | max_loss: 0.1477 | max_backward: 1.7369 | max_optimizer_step: 0.2044 | max_tensorboard_logging: 0.7166 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6331.9 | uss_MB: 5419.4 | child_rss_MB: 5959.7 | allocated_MB: 446.3 | reserved_MB: 29582.0 | inactive_split_MB: 949.7 +rank7 [2026-10-28 23:22:05,869] INFO : Step 14400/50000 rank7 | Train Loss: 1.2457 | Train Loss Unweighted: 0.6253 | Train Loss Components: Auxiliary: 0.2300 | Cardinality: 0.1408 | Classification: 0.1072 | Classification_binary: 0.1847 | HitGrouping: 0.1789 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1629 | Regression_eta: 0.0022 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1605 | Regression_cos_phi: 0.1107 | Regression_energy: 0.0676 | Regression_eta: 0.0442 | Regression_pt: 0.0729 | Regression_sin_phi: 0.0552 | LR: 8.28e-04 | DataLoad Time: 0.0201s | Model Forward Time: 5.9707s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.9667 | max_loss: 0.2306 | max_backward: 2.7434 | max_optimizer_step: 0.2070 | max_tensorboard_logging: 0.7966 | max_tensorboard_flush: 0.0043 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6331.8 | uss_MB: 4969.8 | child_rss_MB: 5956.2 | allocated_MB: 450.7 | reserved_MB: 29584.0 | inactive_split_MB: 1252.8 +rank7 [2026-10-28 23:26:50,539] INFO : Step 14500/50000 rank7 | Train Loss: 1.4093 | Train Loss Unweighted: 0.7194 | Train Loss Components: Auxiliary: 0.2678 | Cardinality: 0.1803 | Classification: 0.1211 | Classification_binary: 0.2406 | HitGrouping: 0.1804 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1754 | Regression_eta: 0.0024 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0795 | Classification_binary: 0.2196 | Regression_cos_phi: 0.0899 | Regression_energy: 0.0797 | Regression_eta: 0.0497 | Regression_pt: 0.0883 | Regression_sin_phi: 0.0887 | LR: 8.26e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.8490s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.8204 | max_loss: 0.1455 | max_backward: 2.0838 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.7592 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.1066 | Memory: rss_MB: 6329.0 | uss_MB: 5423.5 | child_rss_MB: 5932.6 | allocated_MB: 449.1 | reserved_MB: 29584.0 | inactive_split_MB: 1312.1 +rank7 [2026-10-28 23:31:26,129] INFO : Step 14600/50000 rank7 | Train Loss: 1.4466 | Train Loss Unweighted: 0.7628 | Train Loss Components: Auxiliary: 0.2813 | Cardinality: 0.1607 | Classification: 0.1257 | Classification_binary: 0.2761 | HitGrouping: 0.1808 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1767 | Regression_eta: 0.0023 | Regression_pt: 0.1796 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0803 | Classification_binary: 0.2644 | Regression_cos_phi: 0.1010 | Regression_energy: 0.0900 | Regression_eta: 0.0656 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0873 | LR: 8.24e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8557s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0622 | max_forward: 0.8658 | max_loss: 0.1496 | max_backward: 1.9094 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.9894 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0267 | Memory: rss_MB: 6320.2 | uss_MB: 5414.6 | child_rss_MB: 5962.0 | allocated_MB: 450.2 | reserved_MB: 29584.0 | inactive_split_MB: 939.8 +rank7 [2026-10-28 23:35:59,708] INFO : Step 14700/50000 rank7 | Train Loss: 1.2298 | Train Loss Unweighted: 0.6544 | Train Loss Components: Auxiliary: 0.2376 | Cardinality: 0.1321 | Classification: 0.0995 | Classification_binary: 0.1882 | HitGrouping: 0.1788 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1813 | Regression_eta: 0.0021 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0527 | Classification_binary: 0.1753 | Regression_cos_phi: 0.0753 | Regression_energy: 0.0807 | Regression_eta: 0.0522 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0664 | LR: 8.21e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.9568s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0066 | max_forward: 0.8361 | max_loss: 0.2462 | max_backward: 1.9700 | max_optimizer_step: 0.2061 | max_tensorboard_logging: 0.8314 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6323.9 | uss_MB: 5421.4 | child_rss_MB: 5964.3 | allocated_MB: 445.8 | reserved_MB: 31548.0 | inactive_split_MB: 1127.1 +rank7 [2026-10-28 23:40:34,505] INFO : Step 14800/50000 rank7 | Train Loss: 1.2682 | Train Loss Unweighted: 0.6574 | Train Loss Components: Auxiliary: 0.2431 | Cardinality: 0.1516 | Classification: 0.1063 | Classification_binary: 0.2120 | HitGrouping: 0.1767 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1664 | Regression_eta: 0.0020 | Regression_pt: 0.1684 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0664 | Classification_binary: 0.1734 | Regression_cos_phi: 0.1137 | Regression_energy: 0.0853 | Regression_eta: 0.0729 | Regression_pt: 0.0907 | Regression_sin_phi: 0.0950 | LR: 8.19e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.6948s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0064 | max_forward: 0.7530 | max_loss: 0.1476 | max_backward: 1.8229 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9546 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0308 | Memory: rss_MB: 6323.4 | uss_MB: 5420.5 | child_rss_MB: 5957.9 | allocated_MB: 449.4 | reserved_MB: 33554.0 | inactive_split_MB: 1332.6 +rank7 [2026-10-28 23:45:11,571] INFO : Step 14900/50000 rank7 | Train Loss: 1.2753 | Train Loss Unweighted: 0.6572 | Train Loss Components: Auxiliary: 0.2409 | Cardinality: 0.1305 | Classification: 0.1126 | Classification_binary: 0.2387 | HitGrouping: 0.1661 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1497 | Regression_eta: 0.0022 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0702 | Classification_binary: 0.2350 | Regression_cos_phi: 0.0928 | Regression_energy: 0.0780 | Regression_eta: 0.0593 | Regression_pt: 0.0859 | Regression_sin_phi: 0.1037 | LR: 8.17e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8740s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.7955 | max_loss: 0.1633 | max_backward: 1.8376 | max_optimizer_step: 0.2465 | max_tensorboard_logging: 0.9623 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1206 | Memory: rss_MB: 6322.3 | uss_MB: 5420.8 | child_rss_MB: 5961.8 | allocated_MB: 449.7 | reserved_MB: 33554.0 | inactive_split_MB: 1310.1 +rank7 [2026-10-28 23:49:45,305] INFO : Step 15000/50000 rank7 | Train Loss: 1.2701 | Train Loss Unweighted: 0.6495 | Train Loss Components: Auxiliary: 0.2450 | Cardinality: 0.1516 | Classification: 0.1012 | Classification_binary: 0.2121 | HitGrouping: 0.1755 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1655 | Regression_eta: 0.0020 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0610 | Classification_binary: 0.2118 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0868 | Regression_eta: 0.0463 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0886 | LR: 8.15e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.1007s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7505 | max_loss: 0.2449 | max_backward: 1.7335 | max_optimizer_step: 0.2058 | max_tensorboard_logging: 0.4870 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6318.2 | uss_MB: 5419.0 | child_rss_MB: 5953.6 | allocated_MB: 447.3 | reserved_MB: 33554.0 | inactive_split_MB: 1038.0 +rank7 [2026-10-28 23:49:46,286] INFO : Running validation on rank7 +rank7 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-28 23:49:46,286] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-28 23:50:37,475] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:50:38,005] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-28 23:51:04,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-28 23:51:05,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-28 23:51:06,404] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-28 23:51:07,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-28 23:51:17,374] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-28 23:51:18,491] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:51:18,973] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-28 23:51:34,864] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-28 23:51:35,830] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-28 23:51:36,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-28 23:51:37,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-28 23:51:47,691] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 23:51:48,549] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-28 23:51:49,026] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-28 23:52:04,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-28 23:52:05,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-28 23:52:06,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-28 23:52:07,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_15000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-28 23:52:17,230] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-28 23:57:18,881] INFO : Step 15100/50000 rank7 | Train Loss: 1.2388 | Train Loss Unweighted: 0.6590 | Train Loss Components: Auxiliary: 0.2428 | Cardinality: 0.1366 | Classification: 0.1178 | Classification_binary: 0.1841 | HitGrouping: 0.1810 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1743 | Regression_eta: 0.0018 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1759 | Regression_cos_phi: 0.0870 | Regression_energy: 0.0948 | Regression_eta: 0.0501 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0699 | LR: 8.12e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.8264s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.8010 | max_loss: 0.1660 | max_backward: 1.9832 | max_optimizer_step: 0.2129 | max_tensorboard_logging: 0.8565 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 7422.7 | uss_MB: 6518.8 | child_rss_MB: 5938.1 | allocated_MB: 446.2 | reserved_MB: 25230.0 | inactive_split_MB: 1087.3 +rank7 [2026-10-29 00:02:05,558] INFO : Step 15200/50000 rank7 | Train Loss: 1.3362 | Train Loss Unweighted: 0.6611 | Train Loss Components: Auxiliary: 0.2504 | Cardinality: 0.1596 | Classification: 0.1086 | Classification_binary: 0.2256 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1600 | Regression_eta: 0.0022 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0013 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1906 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0909 | Regression_eta: 0.0668 | Regression_pt: 0.0986 | Regression_sin_phi: 0.1015 | LR: 8.10e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6160s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1077 | max_forward: 0.8572 | max_loss: 0.1505 | max_backward: 1.8951 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7089 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6402.3 | uss_MB: 5500.4 | child_rss_MB: 5939.7 | allocated_MB: 449.9 | reserved_MB: 25448.0 | inactive_split_MB: 1183.4 +rank7 [2026-10-29 00:06:54,749] INFO : Step 15300/50000 rank7 | Train Loss: 1.2924 | Train Loss Unweighted: 0.6783 | Train Loss Components: Auxiliary: 0.2476 | Cardinality: 0.1400 | Classification: 0.1062 | Classification_binary: 0.2357 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1646 | Regression_eta: 0.0021 | Regression_pt: 0.1674 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0707 | Classification_binary: 0.2526 | Regression_cos_phi: 0.0967 | Regression_energy: 0.0826 | Regression_eta: 0.0659 | Regression_pt: 0.0919 | Regression_sin_phi: 0.1706 | LR: 8.08e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.7489s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7962 | max_loss: 0.2606 | max_backward: 1.9831 | max_optimizer_step: 0.2491 | max_tensorboard_logging: 0.7840 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6402.1 | uss_MB: 5500.3 | child_rss_MB: 5940.3 | allocated_MB: 446.7 | reserved_MB: 27282.0 | inactive_split_MB: 1189.4 +rank7 [2026-10-29 00:11:34,440] INFO : Step 15400/50000 rank7 | Train Loss: 1.2329 | Train Loss Unweighted: 0.6360 | Train Loss Components: Auxiliary: 0.2375 | Cardinality: 0.1583 | Classification: 0.1127 | Classification_binary: 0.1891 | HitGrouping: 0.1750 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1648 | Regression_eta: 0.0018 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0647 | Classification_binary: 0.2421 | Regression_cos_phi: 0.0935 | Regression_energy: 0.0966 | Regression_eta: 0.0695 | Regression_pt: 0.1059 | Regression_sin_phi: 0.0829 | LR: 8.05e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2518s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0059 | max_forward: 0.7153 | max_loss: 0.1508 | max_backward: 1.6501 | max_optimizer_step: 0.2073 | max_tensorboard_logging: 0.5965 | max_tensorboard_flush: 0.0078 | max_post_step_barrier: 0.0284 | Memory: rss_MB: 6395.9 | uss_MB: 5491.2 | child_rss_MB: 5941.0 | allocated_MB: 444.7 | reserved_MB: 27520.0 | inactive_split_MB: 1102.9 +rank7 [2026-10-29 00:16:23,332] INFO : Step 15500/50000 rank7 | Train Loss: 1.2944 | Train Loss Unweighted: 0.6657 | Train Loss Components: Auxiliary: 0.2480 | Cardinality: 0.1645 | Classification: 0.1174 | Classification_binary: 0.2036 | HitGrouping: 0.1739 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1691 | Regression_eta: 0.0021 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0775 | Classification_binary: 0.1909 | Regression_cos_phi: 0.0915 | Regression_energy: 0.0827 | Regression_eta: 0.0536 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0712 | LR: 8.03e-04 | DataLoad Time: 0.0211s | Model Forward Time: 5.2290s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.9970 | max_loss: 0.1740 | max_backward: 2.3916 | max_optimizer_step: 0.2056 | max_tensorboard_logging: 0.8759 | max_tensorboard_flush: 0.0124 | max_post_step_barrier: 0.0401 | Memory: rss_MB: 6406.1 | uss_MB: 5472.3 | child_rss_MB: 5938.5 | allocated_MB: 449.9 | reserved_MB: 27520.0 | inactive_split_MB: 1174.5 +rank7 [2026-10-29 00:21:07,253] INFO : Step 15600/50000 rank7 | Train Loss: 1.4613 | Train Loss Unweighted: 0.7617 | Train Loss Components: Auxiliary: 0.2872 | Cardinality: 0.1801 | Classification: 0.1276 | Classification_binary: 0.2641 | HitGrouping: 0.1785 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1795 | Regression_eta: 0.0025 | Regression_pt: 0.1856 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0860 | Classification_binary: 0.2380 | Regression_cos_phi: 0.1112 | Regression_energy: 0.0900 | Regression_eta: 0.0662 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0959 | LR: 8.01e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.7222s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.9893 | max_loss: 0.1680 | max_backward: 2.1731 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 1.0419 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0343 | Memory: rss_MB: 6396.7 | uss_MB: 5034.1 | child_rss_MB: 5938.2 | allocated_MB: 448.1 | reserved_MB: 27520.0 | inactive_split_MB: 1080.9 +rank7 [2026-10-29 00:25:54,169] INFO : Step 15700/50000 rank7 | Train Loss: 1.3299 | Train Loss Unweighted: 0.6547 | Train Loss Components: Auxiliary: 0.2527 | Cardinality: 0.1880 | Classification: 0.1192 | Classification_binary: 0.2203 | HitGrouping: 0.1710 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1547 | Regression_eta: 0.0020 | Regression_pt: 0.1561 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0741 | Classification_binary: 0.2434 | Regression_cos_phi: 0.0807 | Regression_energy: 0.0718 | Regression_eta: 0.0726 | Regression_pt: 0.0789 | Regression_sin_phi: 0.1079 | LR: 7.98e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.0298s | Phase Times: max_pre_step_barrier: 0.0061 | max_batch_to_device: 0.0068 | max_forward: 0.9761 | max_loss: 0.2543 | max_backward: 2.0225 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8016 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.1020 | Memory: rss_MB: 6402.6 | uss_MB: 5496.7 | child_rss_MB: 5941.3 | allocated_MB: 448.3 | reserved_MB: 27520.0 | inactive_split_MB: 1390.5 +rank7 [2026-10-29 00:30:33,526] INFO : Step 15800/50000 rank7 | Train Loss: 1.2079 | Train Loss Unweighted: 0.6186 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1572 | Classification: 0.0909 | Classification_binary: 0.1814 | HitGrouping: 0.1709 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1704 | Regression_eta: 0.0019 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0502 | Classification_binary: 0.1780 | Regression_cos_phi: 0.0998 | Regression_energy: 0.0792 | Regression_eta: 0.0500 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0608 | LR: 7.96e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3437s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1066 | max_forward: 0.8154 | max_loss: 0.1440 | max_backward: 1.6020 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.7740 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6405.3 | uss_MB: 5497.2 | child_rss_MB: 5941.3 | allocated_MB: 447.1 | reserved_MB: 27520.0 | inactive_split_MB: 1038.3 +rank7 [2026-10-29 00:35:19,264] INFO : Step 15900/50000 rank7 | Train Loss: 1.4055 | Train Loss Unweighted: 0.7313 | Train Loss Components: Auxiliary: 0.2716 | Cardinality: 0.1808 | Classification: 0.1304 | Classification_binary: 0.2426 | HitGrouping: 0.1763 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1752 | Regression_eta: 0.0021 | Regression_pt: 0.1786 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0836 | Classification_binary: 0.2354 | Regression_cos_phi: 0.1076 | Regression_energy: 0.0906 | Regression_eta: 0.0655 | Regression_pt: 0.0997 | Regression_sin_phi: 0.1156 | LR: 7.94e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.7724s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 1.0653 | max_loss: 0.1515 | max_backward: 2.0024 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.6957 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0363 | Memory: rss_MB: 6402.5 | uss_MB: 5500.0 | child_rss_MB: 5940.1 | allocated_MB: 447.4 | reserved_MB: 27520.0 | inactive_split_MB: 1172.5 +rank7 [2026-10-29 00:40:00,683] INFO : Step 16000/50000 rank7 | Train Loss: 1.3107 | Train Loss Unweighted: 0.6580 | Train Loss Components: Auxiliary: 0.2519 | Cardinality: 0.1760 | Classification: 0.1019 | Classification_binary: 0.2391 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1548 | Regression_eta: 0.0022 | Regression_pt: 0.1580 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.2590 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0907 | Regression_eta: 0.0644 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0974 | LR: 7.91e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.7629s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.1053 | max_forward: 1.1703 | max_loss: 0.1516 | max_backward: 1.8974 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.8149 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6403.0 | uss_MB: 5502.7 | child_rss_MB: 5941.0 | allocated_MB: 447.2 | reserved_MB: 27520.0 | inactive_split_MB: 1195.6 +rank7 [2026-10-29 00:40:01,696] INFO : Running validation on rank7 +rank7 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 00:40:01,696] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 00:40:58,072] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:40:58,626] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 00:41:23,366] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 00:41:24,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 00:41:25,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 00:41:26,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 00:41:36,366] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 00:41:36,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:41:37,530] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 00:41:53,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 00:41:55,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 00:41:56,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 00:41:57,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 00:42:07,129] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 00:42:07,290] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 00:42:07,840] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 00:42:23,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 00:42:24,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 00:42:25,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 00:42:25,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_16000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 00:42:35,858] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 00:47:34,083] INFO : Step 16100/50000 rank7 | Train Loss: 1.2037 | Train Loss Unweighted: 0.6025 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1467 | Classification: 0.1080 | Classification_binary: 0.1878 | HitGrouping: 0.1747 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1512 | Regression_eta: 0.0018 | Regression_pt: 0.1516 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.2056 | Regression_cos_phi: 0.1001 | Regression_energy: 0.0810 | Regression_eta: 0.0498 | Regression_pt: 0.0874 | Regression_sin_phi: 0.1084 | LR: 7.89e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.0448s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0062 | max_forward: 0.7665 | max_loss: 0.1405 | max_backward: 1.4602 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0523 | Memory: rss_MB: 7547.3 | uss_MB: 6643.5 | child_rss_MB: 5941.5 | allocated_MB: 446.1 | reserved_MB: 27680.0 | inactive_split_MB: 1038.2 +rank7 [2026-10-29 00:52:15,156] INFO : Step 16200/50000 rank7 | Train Loss: 1.2083 | Train Loss Unweighted: 0.6192 | Train Loss Components: Auxiliary: 0.2322 | Cardinality: 0.1499 | Classification: 0.1107 | Classification_binary: 0.2017 | HitGrouping: 0.1711 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1506 | Regression_eta: 0.0018 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0684 | Classification_binary: 0.1792 | Regression_cos_phi: 0.0875 | Regression_energy: 0.0805 | Regression_eta: 0.0558 | Regression_pt: 0.0880 | Regression_sin_phi: 0.1141 | LR: 7.86e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.9523s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0473 | max_forward: 0.8343 | max_loss: 0.2530 | max_backward: 2.1136 | max_optimizer_step: 0.3076 | max_tensorboard_logging: 0.7468 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6458.0 | uss_MB: 5555.6 | child_rss_MB: 5961.3 | allocated_MB: 446.7 | reserved_MB: 27682.0 | inactive_split_MB: 1157.2 +rank7 [2026-10-29 00:56:51,718] INFO : Step 16300/50000 rank7 | Train Loss: 1.2745 | Train Loss Unweighted: 0.6291 | Train Loss Components: Auxiliary: 0.2444 | Cardinality: 0.1604 | Classification: 0.1062 | Classification_binary: 0.2417 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1373 | Regression_eta: 0.0022 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0619 | Classification_binary: 0.2112 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0727 | Regression_eta: 0.0568 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0840 | LR: 7.84e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.8118s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0068 | max_forward: 0.8059 | max_loss: 0.2506 | max_backward: 2.1763 | max_optimizer_step: 0.5213 | max_tensorboard_logging: 0.5923 | max_tensorboard_flush: 0.0035 | max_post_step_barrier: 0.0418 | Memory: rss_MB: 6459.9 | uss_MB: 5555.3 | child_rss_MB: 5936.4 | allocated_MB: 450.9 | reserved_MB: 27682.0 | inactive_split_MB: 950.8 +rank7 [2026-10-29 01:01:26,581] INFO : Step 16400/50000 rank7 | Train Loss: 1.2654 | Train Loss Unweighted: 0.6311 | Train Loss Components: Auxiliary: 0.2380 | Cardinality: 0.1554 | Classification: 0.1095 | Classification_binary: 0.2018 | HitGrouping: 0.1699 | Regression_cos_phi: 0.0012 | Regression_energy: 0.1561 | Regression_eta: 0.0023 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0680 | Classification_binary: 0.2327 | Regression_cos_phi: 0.0977 | Regression_energy: 0.0929 | Regression_eta: 0.0724 | Regression_pt: 0.1001 | Regression_sin_phi: 0.1165 | LR: 7.81e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.2792s | Phase Times: max_pre_step_barrier: 0.0533 | max_batch_to_device: 0.0064 | max_forward: 0.7459 | max_loss: 0.1486 | max_backward: 1.5352 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.6677 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0345 | Memory: rss_MB: 6452.2 | uss_MB: 5545.1 | child_rss_MB: 5962.3 | allocated_MB: 446.4 | reserved_MB: 27682.0 | inactive_split_MB: 949.1 +rank7 [2026-10-29 01:06:05,049] INFO : Step 16500/50000 rank7 | Train Loss: 1.1728 | Train Loss Unweighted: 0.5918 | Train Loss Components: Auxiliary: 0.2261 | Cardinality: 0.1444 | Classification: 0.0927 | Classification_binary: 0.1686 | HitGrouping: 0.1723 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1626 | Regression_eta: 0.0021 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1583 | Regression_cos_phi: 0.0781 | Regression_energy: 0.0874 | Regression_eta: 0.0543 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0656 | LR: 7.79e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4911s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8043 | max_loss: 0.1772 | max_backward: 2.0250 | max_optimizer_step: 0.2075 | max_tensorboard_logging: 0.5945 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6458.1 | uss_MB: 5554.8 | child_rss_MB: 5927.7 | allocated_MB: 448.6 | reserved_MB: 29698.0 | inactive_split_MB: 1167.6 +rank7 [2026-10-29 01:10:53,703] INFO : Step 16600/50000 rank7 | Train Loss: 1.2920 | Train Loss Unweighted: 0.6427 | Train Loss Components: Auxiliary: 0.2455 | Cardinality: 0.1794 | Classification: 0.1148 | Classification_binary: 0.2098 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1561 | Regression_eta: 0.0021 | Regression_pt: 0.1577 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0848 | Regression_eta: 0.0544 | Regression_pt: 0.0926 | Regression_sin_phi: 0.1048 | LR: 7.77e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.2540s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0377 | max_forward: 1.0830 | max_loss: 0.1480 | max_backward: 1.7919 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 0.4907 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0302 | Memory: rss_MB: 6457.2 | uss_MB: 5552.9 | child_rss_MB: 5951.6 | allocated_MB: 449.3 | reserved_MB: 29698.0 | inactive_split_MB: 1014.7 +rank7 [2026-10-29 01:15:45,847] INFO : Step 16700/50000 rank7 | Train Loss: 1.2559 | Train Loss Unweighted: 0.6325 | Train Loss Components: Auxiliary: 0.2426 | Cardinality: 0.1731 | Classification: 0.1072 | Classification_binary: 0.2252 | HitGrouping: 0.1672 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1469 | Regression_eta: 0.0019 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.2363 | Regression_cos_phi: 0.0700 | Regression_energy: 0.0772 | Regression_eta: 0.0448 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0726 | LR: 7.74e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4538s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0067 | max_forward: 1.0722 | max_loss: 0.1506 | max_backward: 2.0365 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.5067 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6456.8 | uss_MB: 5555.5 | child_rss_MB: 5918.8 | allocated_MB: 448.3 | reserved_MB: 29698.0 | inactive_split_MB: 893.3 +rank7 [2026-10-29 01:20:20,822] INFO : Step 16800/50000 rank7 | Train Loss: 1.1947 | Train Loss Unweighted: 0.5920 | Train Loss Components: Auxiliary: 0.2279 | Cardinality: 0.1542 | Classification: 0.1036 | Classification_binary: 0.1800 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1523 | Regression_eta: 0.0023 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0669 | Classification_binary: 0.1859 | Regression_cos_phi: 0.0877 | Regression_energy: 0.1027 | Regression_eta: 0.0691 | Regression_pt: 0.1104 | Regression_sin_phi: 0.0893 | LR: 7.72e-04 | DataLoad Time: 0.0212s | Model Forward Time: 4.2729s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0166 | max_forward: 0.8224 | max_loss: 0.1466 | max_backward: 1.6608 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6428.0 | uss_MB: 5552.9 | child_rss_MB: 5812.3 | allocated_MB: 446.4 | reserved_MB: 29698.0 | inactive_split_MB: 1161.6 +rank7 [2026-10-29 01:25:06,864] INFO : Step 16900/50000 rank7 | Train Loss: 1.2025 | Train Loss Unweighted: 0.5963 | Train Loss Components: Auxiliary: 0.2301 | Cardinality: 0.1406 | Classification: 0.1120 | Classification_binary: 0.1884 | HitGrouping: 0.1677 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1446 | Regression_eta: 0.0023 | Regression_pt: 0.1469 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0709 | Classification_binary: 0.1834 | Regression_cos_phi: 0.0937 | Regression_energy: 0.0713 | Regression_eta: 0.0535 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0707 | LR: 7.69e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.3240s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0071 | max_forward: 0.7445 | max_loss: 0.1453 | max_backward: 1.8211 | max_optimizer_step: 0.2101 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 6418.8 | uss_MB: 5553.4 | child_rss_MB: 5783.4 | allocated_MB: 448.9 | reserved_MB: 29698.0 | inactive_split_MB: 901.3 +rank7 [2026-10-29 01:29:52,082] INFO : Step 17000/50000 rank7 | Train Loss: 1.2049 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2349 | Cardinality: 0.1467 | Classification: 0.1018 | Classification_binary: 0.2228 | HitGrouping: 0.1666 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.2054 | Regression_cos_phi: 0.0850 | Regression_energy: 0.0791 | Regression_eta: 0.0556 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0753 | LR: 7.67e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.1426s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0141 | max_forward: 1.3980 | max_loss: 0.5129 | max_backward: 2.5664 | max_optimizer_step: 0.2042 | max_tensorboard_logging: 0.5255 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.1218 | Memory: rss_MB: 6382.7 | uss_MB: 5556.2 | child_rss_MB: 5576.1 | allocated_MB: 448.2 | reserved_MB: 29698.0 | inactive_split_MB: 955.8 +rank7 [2026-10-29 01:29:53,074] INFO : Running validation on rank7 +rank7 [2026-10-29 01:29:53,074] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 01:29:53,074] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 01:30:46,066] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:30:46,580] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 01:31:13,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 01:31:14,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 01:31:15,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 01:31:16,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 01:31:26,945] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 01:31:28,037] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:31:28,589] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 01:31:44,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 01:31:45,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 01:31:46,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 01:31:47,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 01:31:57,298] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 01:31:58,252] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 01:31:58,744] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 01:32:14,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 01:32:15,567] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 01:32:16,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 01:32:17,226] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_17000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 01:32:27,237] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 01:37:36,199] INFO : Step 17100/50000 rank7 | Train Loss: 1.2386 | Train Loss Unweighted: 0.5919 | Train Loss Components: Auxiliary: 0.2370 | Cardinality: 0.1931 | Classification: 0.0995 | Classification_binary: 0.1969 | HitGrouping: 0.1714 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1449 | Regression_eta: 0.0019 | Regression_pt: 0.1468 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0586 | Classification_binary: 0.2112 | Regression_cos_phi: 0.0667 | Regression_energy: 0.0769 | Regression_eta: 0.0539 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0881 | LR: 7.64e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.3615s | Phase Times: max_pre_step_barrier: 0.0100 | max_batch_to_device: 0.0067 | max_forward: 1.0304 | max_loss: 0.1869 | max_backward: 2.3467 | max_optimizer_step: 0.3175 | max_tensorboard_logging: 0.7405 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0485 | Memory: rss_MB: 7631.0 | uss_MB: 6798.9 | child_rss_MB: 5579.4 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 1373.9 +rank7 [2026-10-29 01:42:27,134] INFO : Step 17200/50000 rank7 | Train Loss: 1.1810 | Train Loss Unweighted: 0.5844 | Train Loss Components: Auxiliary: 0.2270 | Cardinality: 0.1495 | Classification: 0.1085 | Classification_binary: 0.1683 | HitGrouping: 0.1731 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1519 | Regression_eta: 0.0022 | Regression_pt: 0.1515 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.1588 | Regression_cos_phi: 0.0577 | Regression_energy: 0.0813 | Regression_eta: 0.0487 | Regression_pt: 0.0850 | Regression_sin_phi: 0.0852 | LR: 7.62e-04 | DataLoad Time: 0.0174s | Model Forward Time: 6.1046s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0064 | max_forward: 0.7638 | max_loss: 0.2424 | max_backward: 1.9393 | max_optimizer_step: 0.2921 | max_tensorboard_logging: 2.1267 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 6444.8 | uss_MB: 5633.5 | child_rss_MB: 5577.8 | allocated_MB: 447.9 | reserved_MB: 25812.0 | inactive_split_MB: 988.1 +rank7 [2026-10-29 01:47:11,311] INFO : Step 17300/50000 rank7 | Train Loss: 1.2271 | Train Loss Unweighted: 0.6313 | Train Loss Components: Auxiliary: 0.2397 | Cardinality: 0.1372 | Classification: 0.1102 | Classification_binary: 0.2127 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1512 | Regression_eta: 0.0022 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0780 | Regression_energy: 0.0763 | Regression_eta: 0.0483 | Regression_pt: 0.0828 | Regression_sin_phi: 0.0793 | LR: 7.59e-04 | DataLoad Time: 0.0211s | Model Forward Time: 6.1187s | Phase Times: max_pre_step_barrier: 0.0118 | max_batch_to_device: 0.0243 | max_forward: 1.4273 | max_loss: 0.6053 | max_backward: 3.4810 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5664 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0391 | Memory: rss_MB: 6426.5 | uss_MB: 5175.0 | child_rss_MB: 5586.3 | allocated_MB: 449.6 | reserved_MB: 26752.0 | inactive_split_MB: 1120.1 +rank7 [2026-10-29 01:52:04,202] INFO : Step 17400/50000 rank7 | Train Loss: 1.2297 | Train Loss Unweighted: 0.6013 | Train Loss Components: Auxiliary: 0.2404 | Cardinality: 0.1709 | Classification: 0.1017 | Classification_binary: 0.2232 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1354 | Regression_eta: 0.0019 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.2535 | Regression_cos_phi: 0.0848 | Regression_energy: 0.0663 | Regression_eta: 0.0480 | Regression_pt: 0.0739 | Regression_sin_phi: 0.1006 | LR: 7.57e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.6389s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0185 | max_forward: 1.0000 | max_loss: 0.1504 | max_backward: 1.9673 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6331 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6409.7 | uss_MB: 5631.7 | child_rss_MB: 5583.3 | allocated_MB: 447.1 | reserved_MB: 28732.0 | inactive_split_MB: 1125.6 +rank7 [2026-10-29 01:56:48,790] INFO : Step 17500/50000 rank7 | Train Loss: 1.2152 | Train Loss Unweighted: 0.6044 | Train Loss Components: Auxiliary: 0.2290 | Cardinality: 0.1485 | Classification: 0.1023 | Classification_binary: 0.1789 | HitGrouping: 0.1757 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1590 | Regression_eta: 0.0026 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1914 | Regression_cos_phi: 0.0735 | Regression_energy: 0.0743 | Regression_eta: 0.0568 | Regression_pt: 0.0788 | Regression_sin_phi: 0.0754 | LR: 7.54e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.2345s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0167 | max_forward: 0.7711 | max_loss: 0.1449 | max_backward: 1.8153 | max_optimizer_step: 0.2118 | max_tensorboard_logging: 0.4947 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6388.7 | uss_MB: 5638.8 | child_rss_MB: 5515.3 | allocated_MB: 450.1 | reserved_MB: 28732.0 | inactive_split_MB: 1375.8 +rank7 [2026-10-29 02:01:39,792] INFO : Step 17600/50000 rank7 | Train Loss: 1.1560 | Train Loss Unweighted: 0.6042 | Train Loss Components: Auxiliary: 0.2286 | Cardinality: 0.1294 | Classification: 0.1057 | Classification_binary: 0.1993 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1471 | Regression_eta: 0.0018 | Regression_pt: 0.1484 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0733 | Classification_binary: 0.2152 | Regression_cos_phi: 0.0666 | Regression_energy: 0.0993 | Regression_eta: 0.0570 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0815 | LR: 7.52e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.8150s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0068 | max_forward: 0.8868 | max_loss: 0.1705 | max_backward: 2.0183 | max_optimizer_step: 0.2066 | max_tensorboard_logging: 0.7222 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1349 | Memory: rss_MB: 6384.6 | uss_MB: 5639.3 | child_rss_MB: 5506.2 | allocated_MB: 448.3 | reserved_MB: 28732.0 | inactive_split_MB: 1319.7 +rank7 [2026-10-29 02:06:28,890] INFO : Step 17700/50000 rank7 | Train Loss: 1.7848 | Train Loss Unweighted: 0.7433 | Train Loss Components: Auxiliary: 0.3415 | Cardinality: 0.4306 | Classification: 0.1200 | Classification_binary: 0.2341 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0014 | Regression_energy: 0.1920 | Regression_eta: 0.0037 | Regression_pt: 0.1910 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0777 | Classification_binary: 0.2192 | Regression_cos_phi: 0.1108 | Regression_energy: 0.0877 | Regression_eta: 0.1060 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0888 | LR: 7.49e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.5782s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0072 | max_forward: 0.8788 | max_loss: 0.4738 | max_backward: 1.9859 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.5305 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6359.7 | uss_MB: 5640.4 | child_rss_MB: 5400.4 | allocated_MB: 450.1 | reserved_MB: 28738.0 | inactive_split_MB: 1125.6 +rank7 [2026-10-29 02:11:14,773] INFO : Step 17800/50000 rank7 | Train Loss: 1.1828 | Train Loss Unweighted: 0.6014 | Train Loss Components: Auxiliary: 0.2316 | Cardinality: 0.1456 | Classification: 0.0982 | Classification_binary: 0.1996 | HitGrouping: 0.1749 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1503 | Regression_eta: 0.0019 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0598 | Classification_binary: 0.2057 | Regression_cos_phi: 0.0814 | Regression_energy: 0.0683 | Regression_eta: 0.0463 | Regression_pt: 0.0752 | Regression_sin_phi: 0.0851 | LR: 7.47e-04 | DataLoad Time: 0.0182s | Model Forward Time: 5.1930s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0064 | max_forward: 0.7984 | max_loss: 0.3167 | max_backward: 2.2751 | max_optimizer_step: 0.2229 | max_tensorboard_logging: 0.5340 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0196 | Memory: rss_MB: 6343.6 | uss_MB: 5261.8 | child_rss_MB: 5331.5 | allocated_MB: 444.9 | reserved_MB: 28738.0 | inactive_split_MB: 1155.9 +rank7 [2026-10-29 02:16:10,311] INFO : Step 17900/50000 rank7 | Train Loss: 1.0765 | Train Loss Unweighted: 0.5646 | Train Loss Components: Auxiliary: 0.2108 | Cardinality: 0.1097 | Classification: 0.0932 | Classification_binary: 0.1585 | HitGrouping: 0.1686 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1536 | Regression_eta: 0.0016 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0287 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0385 | Regression_energy: 0.0931 | Regression_eta: 0.0279 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0453 | LR: 7.44e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.2959s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0062 | max_forward: 0.7593 | max_loss: 0.4918 | max_backward: 1.7987 | max_optimizer_step: 0.2104 | max_tensorboard_logging: 0.6052 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6290.9 | uss_MB: 5640.2 | child_rss_MB: 5109.0 | allocated_MB: 447.1 | reserved_MB: 28738.0 | inactive_split_MB: 1154.4 +rank7 [2026-10-29 02:20:55,876] INFO : Step 18000/50000 rank7 | Train Loss: 1.2227 | Train Loss Unweighted: 0.6143 | Train Loss Components: Auxiliary: 0.2303 | Cardinality: 0.1450 | Classification: 0.1088 | Classification_binary: 0.2099 | HitGrouping: 0.1650 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1447 | Regression_eta: 0.0020 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.2200 | Regression_cos_phi: 0.1167 | Regression_energy: 0.0791 | Regression_eta: 0.0602 | Regression_pt: 0.0875 | Regression_sin_phi: 0.1142 | LR: 7.42e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7005s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0066 | max_forward: 0.9293 | max_loss: 0.1567 | max_backward: 2.2476 | max_optimizer_step: 0.2081 | max_tensorboard_logging: 0.5333 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0355 | Memory: rss_MB: 6283.0 | uss_MB: 5642.9 | child_rss_MB: 5063.5 | allocated_MB: 449.4 | reserved_MB: 28738.0 | inactive_split_MB: 1324.2 +rank7 [2026-10-29 02:20:56,967] INFO : Running validation on rank7 +rank7 [2026-10-29 02:20:56,967] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 02:20:56,967] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 02:21:57,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:21:58,045] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 02:22:25,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 02:22:26,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 02:22:27,176] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 02:22:28,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 02:22:38,147] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 02:22:38,914] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:22:39,418] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 02:22:55,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 02:22:56,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 02:22:57,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 02:22:58,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 02:23:08,486] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 02:23:08,996] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 02:23:09,547] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 02:23:25,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 02:23:26,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 02:23:27,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 02:23:27,854] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_18000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 02:23:37,862] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 02:28:43,002] INFO : Step 18100/50000 rank7 | Train Loss: 1.2922 | Train Loss Unweighted: 0.6330 | Train Loss Components: Auxiliary: 0.2514 | Cardinality: 0.1763 | Classification: 0.1061 | Classification_binary: 0.2257 | HitGrouping: 0.1663 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1467 | Regression_eta: 0.0022 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0593 | Classification_binary: 0.2195 | Regression_cos_phi: 0.0818 | Regression_energy: 0.0806 | Regression_eta: 0.0492 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0748 | LR: 7.39e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0677s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.9150 | max_loss: 0.1573 | max_backward: 2.1915 | max_optimizer_step: 0.2256 | max_tensorboard_logging: 0.8426 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0520 | Memory: rss_MB: 7551.6 | uss_MB: 6907.4 | child_rss_MB: 5063.5 | allocated_MB: 448.8 | reserved_MB: 25396.0 | inactive_split_MB: 1192.4 +rank7 [2026-10-29 02:33:21,589] INFO : Step 18200/50000 rank7 | Train Loss: 1.1403 | Train Loss Unweighted: 0.5693 | Train Loss Components: Auxiliary: 0.2140 | Cardinality: 0.1362 | Classification: 0.1099 | Classification_binary: 0.1658 | HitGrouping: 0.1698 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1444 | Regression_eta: 0.0021 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0688 | Classification_binary: 0.1899 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0670 | Regression_eta: 0.0542 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0838 | LR: 7.36e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.5299s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0059 | max_forward: 0.7843 | max_loss: 0.1427 | max_backward: 1.4948 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 1.0509 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0341 | Memory: rss_MB: 6418.0 | uss_MB: 5773.9 | child_rss_MB: 5038.4 | allocated_MB: 446.0 | reserved_MB: 27340.0 | inactive_split_MB: 922.1 +rank7 [2026-10-29 02:37:53,621] INFO : Step 18300/50000 rank7 | Train Loss: 1.1938 | Train Loss Unweighted: 0.6047 | Train Loss Components: Auxiliary: 0.2339 | Cardinality: 0.1526 | Classification: 0.1133 | Classification_binary: 0.1878 | HitGrouping: 0.1719 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1493 | Regression_eta: 0.0019 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1992 | Regression_cos_phi: 0.0893 | Regression_energy: 0.0738 | Regression_eta: 0.0561 | Regression_pt: 0.0786 | Regression_sin_phi: 0.0810 | LR: 7.34e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9185s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.9058 | max_loss: 0.1527 | max_backward: 2.1689 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8500 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0399 | Memory: rss_MB: 6422.1 | uss_MB: 5784.1 | child_rss_MB: 5032.3 | allocated_MB: 448.4 | reserved_MB: 27340.0 | inactive_split_MB: 1185.8 +rank7 [2026-10-29 02:42:24,376] INFO : Step 18400/50000 rank7 | Train Loss: 1.1959 | Train Loss Unweighted: 0.6109 | Train Loss Components: Auxiliary: 0.2305 | Cardinality: 0.1130 | Classification: 0.1042 | Classification_binary: 0.2213 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1400 | Regression_eta: 0.0022 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0660 | Classification_binary: 0.2136 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0643 | Regression_eta: 0.0666 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0825 | LR: 7.31e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7073s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.8434 | max_loss: 0.1474 | max_backward: 1.9441 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.8146 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6423.0 | uss_MB: 5783.9 | child_rss_MB: 5008.1 | allocated_MB: 448.8 | reserved_MB: 29306.0 | inactive_split_MB: 1091.4 +rank7 [2026-10-29 02:47:17,191] INFO : Step 18500/50000 rank7 | Train Loss: 1.2719 | Train Loss Unweighted: 0.6036 | Train Loss Components: Auxiliary: 0.2506 | Cardinality: 0.1990 | Classification: 0.1087 | Classification_binary: 0.1793 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1538 | Regression_eta: 0.0024 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0674 | Classification_binary: 0.2329 | Regression_cos_phi: 0.1043 | Regression_energy: 0.0834 | Regression_eta: 0.0813 | Regression_pt: 0.0917 | Regression_sin_phi: 0.0991 | LR: 7.29e-04 | DataLoad Time: 0.0168s | Model Forward Time: 6.4222s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0070 | max_forward: 1.6245 | max_loss: 0.3527 | max_backward: 3.4112 | max_optimizer_step: 0.2124 | max_tensorboard_logging: 0.7932 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.1171 | Memory: rss_MB: 6423.2 | uss_MB: 5781.7 | child_rss_MB: 5035.9 | allocated_MB: 447.9 | reserved_MB: 29306.0 | inactive_split_MB: 1193.4 +rank7 [2026-10-29 02:52:01,280] INFO : Step 18600/50000 rank7 | Train Loss: 1.0849 | Train Loss Unweighted: 0.5476 | Train Loss Components: Auxiliary: 0.2056 | Cardinality: 0.1351 | Classification: 0.0870 | Classification_binary: 0.1697 | HitGrouping: 0.1724 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1433 | Regression_eta: 0.0017 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0603 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0966 | Regression_energy: 0.0649 | Regression_eta: 0.0643 | Regression_pt: 0.0731 | Regression_sin_phi: 0.0812 | LR: 7.26e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2410s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0527 | max_forward: 0.7639 | max_loss: 0.1373 | max_backward: 1.5273 | max_optimizer_step: 0.2388 | max_tensorboard_logging: 0.7424 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0368 | Memory: rss_MB: 6420.8 | uss_MB: 5779.1 | child_rss_MB: 5038.0 | allocated_MB: 445.6 | reserved_MB: 29306.0 | inactive_split_MB: 1194.4 +rank7 [2026-10-29 02:56:35,338] INFO : Step 18700/50000 rank7 | Train Loss: 1.2376 | Train Loss Unweighted: 0.5906 | Train Loss Components: Auxiliary: 0.2405 | Cardinality: 0.1841 | Classification: 0.1039 | Classification_binary: 0.2012 | HitGrouping: 0.1638 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1400 | Regression_eta: 0.0020 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0673 | Classification_binary: 0.2232 | Regression_cos_phi: 0.1112 | Regression_energy: 0.0871 | Regression_eta: 0.0515 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0958 | LR: 7.24e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.7553s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.8903 | max_loss: 0.1469 | max_backward: 2.1433 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4822 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1128 | Memory: rss_MB: 6419.7 | uss_MB: 5784.3 | child_rss_MB: 5013.1 | allocated_MB: 449.2 | reserved_MB: 31324.0 | inactive_split_MB: 1093.4 +rank7 [2026-10-29 03:01:09,007] INFO : Step 18800/50000 rank7 | Train Loss: 1.1642 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2276 | Cardinality: 0.1585 | Classification: 0.0943 | Classification_binary: 0.2038 | HitGrouping: 0.1635 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1342 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0500 | Classification_binary: 0.2008 | Regression_cos_phi: 0.0693 | Regression_energy: 0.0821 | Regression_eta: 0.0409 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0542 | LR: 7.21e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4221s | Phase Times: max_pre_step_barrier: 0.1142 | max_batch_to_device: 0.0068 | max_forward: 0.7808 | max_loss: 0.2502 | max_backward: 1.8229 | max_optimizer_step: 0.2086 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0996 | Memory: rss_MB: 6421.2 | uss_MB: 5781.7 | child_rss_MB: 5036.2 | allocated_MB: 450.4 | reserved_MB: 31336.0 | inactive_split_MB: 1098.4 +rank7 [2026-10-29 03:05:45,769] INFO : Step 18900/50000 rank7 | Train Loss: 1.0747 | Train Loss Unweighted: 0.5385 | Train Loss Components: Auxiliary: 0.2078 | Cardinality: 0.1306 | Classification: 0.0887 | Classification_binary: 0.1613 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1414 | Regression_eta: 0.0017 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1797 | Regression_cos_phi: 0.0809 | Regression_energy: 0.0732 | Regression_eta: 0.0485 | Regression_pt: 0.0790 | Regression_sin_phi: 0.0947 | LR: 7.18e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.3932s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0061 | max_forward: 0.7400 | max_loss: 0.1656 | max_backward: 1.8003 | max_optimizer_step: 0.2072 | max_tensorboard_logging: 0.7596 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0359 | Memory: rss_MB: 6421.5 | uss_MB: 5781.2 | child_rss_MB: 5039.7 | allocated_MB: 443.5 | reserved_MB: 31336.0 | inactive_split_MB: 928.5 +rank7 [2026-10-29 03:10:20,956] INFO : Step 19000/50000 rank7 | Train Loss: 1.1726 | Train Loss Unweighted: 0.5966 | Train Loss Components: Auxiliary: 0.2325 | Cardinality: 0.1418 | Classification: 0.1102 | Classification_binary: 0.1870 | HitGrouping: 0.1705 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1463 | Regression_eta: 0.0019 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1826 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0785 | Regression_eta: 0.0427 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0464 | LR: 7.16e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.8906s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.7637 | max_loss: 0.5203 | max_backward: 1.8720 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 1.1055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6422.5 | uss_MB: 5783.8 | child_rss_MB: 5041.8 | allocated_MB: 446.6 | reserved_MB: 31336.0 | inactive_split_MB: 1097.4 +rank7 [2026-10-29 03:10:21,983] INFO : Running validation on rank7 +rank7 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 03:10:21,983] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 03:11:18,709] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:11:19,247] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 03:11:45,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 03:11:46,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 03:11:47,867] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 03:11:49,408] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 03:11:59,418] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 03:12:01,384] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:12:01,927] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 03:12:17,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 03:12:18,895] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 03:12:19,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 03:12:20,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 03:12:30,779] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 03:12:31,456] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 03:12:31,972] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 03:12:47,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 03:12:48,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 03:12:49,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 03:12:50,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_19000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 03:13:00,474] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 03:17:58,240] INFO : Step 19100/50000 rank7 | Train Loss: 1.2150 | Train Loss Unweighted: 0.6107 | Train Loss Components: Auxiliary: 0.2359 | Cardinality: 0.1439 | Classification: 0.1037 | Classification_binary: 0.2230 | HitGrouping: 0.1629 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1395 | Regression_eta: 0.0020 | Regression_pt: 0.1406 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0632 | Classification_binary: 0.2526 | Regression_cos_phi: 0.0889 | Regression_energy: 0.0781 | Regression_eta: 0.0760 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0902 | LR: 7.13e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.7830s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.8630 | max_loss: 0.1532 | max_backward: 1.9622 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.7474 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7697.6 | uss_MB: 7053.9 | child_rss_MB: 5049.6 | allocated_MB: 449.4 | reserved_MB: 23972.0 | inactive_split_MB: 1371.8 +rank7 [2026-10-29 03:22:35,410] INFO : Step 19200/50000 rank7 | Train Loss: 1.1075 | Train Loss Unweighted: 0.5303 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1382 | Classification: 0.0958 | Classification_binary: 0.1758 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1270 | Regression_eta: 0.0019 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1708 | Regression_cos_phi: 0.0756 | Regression_energy: 0.0654 | Regression_eta: 0.0568 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0740 | LR: 7.10e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.0040s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.8287 | max_loss: 0.1556 | max_backward: 1.8127 | max_optimizer_step: 0.2350 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6504.0 | uss_MB: 5863.3 | child_rss_MB: 5048.7 | allocated_MB: 445.3 | reserved_MB: 27640.0 | inactive_split_MB: 940.7 +rank7 [2026-10-29 03:27:09,996] INFO : Step 19300/50000 rank7 | Train Loss: 1.0792 | Train Loss Unweighted: 0.5477 | Train Loss Components: Auxiliary: 0.2111 | Cardinality: 0.1254 | Classification: 0.0911 | Classification_binary: 0.1561 | HitGrouping: 0.1662 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1477 | Regression_eta: 0.0018 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0474 | Classification_binary: 0.1453 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0970 | Regression_eta: 0.0331 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0456 | LR: 7.08e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.5129s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0060 | max_forward: 0.8891 | max_loss: 0.1513 | max_backward: 1.7637 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.7290 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6508.5 | uss_MB: 5869.2 | child_rss_MB: 5044.9 | allocated_MB: 445.1 | reserved_MB: 27640.0 | inactive_split_MB: 1366.9 +rank7 [2026-10-29 03:31:44,570] INFO : Step 19400/50000 rank7 | Train Loss: 1.2411 | Train Loss Unweighted: 0.6183 | Train Loss Components: Auxiliary: 0.2458 | Cardinality: 0.1555 | Classification: 0.1055 | Classification_binary: 0.2145 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1463 | Regression_eta: 0.0020 | Regression_pt: 0.1479 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0762 | Classification_binary: 0.1853 | Regression_cos_phi: 0.0784 | Regression_energy: 0.0641 | Regression_eta: 0.0576 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0586 | LR: 7.05e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.6743s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0092 | max_forward: 0.8079 | max_loss: 0.1550 | max_backward: 1.7710 | max_optimizer_step: 0.2177 | max_tensorboard_logging: 2.8282 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6511.0 | uss_MB: 5872.2 | child_rss_MB: 5045.1 | allocated_MB: 452.5 | reserved_MB: 28010.0 | inactive_split_MB: 1195.9 +rank7 [2026-10-29 03:36:21,472] INFO : Step 19500/50000 rank7 | Train Loss: 1.2576 | Train Loss Unweighted: 0.6008 | Train Loss Components: Auxiliary: 0.2448 | Cardinality: 0.1836 | Classification: 0.1087 | Classification_binary: 0.2123 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1374 | Regression_eta: 0.0023 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0765 | Classification_binary: 0.2191 | Regression_cos_phi: 0.1076 | Regression_energy: 0.0861 | Regression_eta: 0.0678 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0975 | LR: 7.02e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6186s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7880 | max_loss: 0.1506 | max_backward: 1.7760 | max_optimizer_step: 0.2145 | max_tensorboard_logging: 0.7620 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0397 | Memory: rss_MB: 6510.7 | uss_MB: 5870.2 | child_rss_MB: 5041.2 | allocated_MB: 449.4 | reserved_MB: 29680.0 | inactive_split_MB: 1368.8 +rank7 [2026-10-29 03:40:54,623] INFO : Step 19600/50000 rank7 | Train Loss: 1.1451 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2260 | Cardinality: 0.1464 | Classification: 0.0933 | Classification_binary: 0.1736 | HitGrouping: 0.1759 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1533 | Regression_eta: 0.0018 | Regression_pt: 0.1557 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1959 | Regression_cos_phi: 0.0621 | Regression_energy: 0.0877 | Regression_eta: 0.0502 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0718 | LR: 7.00e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.3545s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0062 | max_forward: 0.8082 | max_loss: 0.1437 | max_backward: 1.6649 | max_optimizer_step: 0.2340 | max_tensorboard_logging: 0.6567 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6510.9 | uss_MB: 5869.4 | child_rss_MB: 5045.0 | allocated_MB: 444.5 | reserved_MB: 29680.0 | inactive_split_MB: 1016.8 +rank7 [2026-10-29 03:45:30,310] INFO : Step 19700/50000 rank7 | Train Loss: 1.1424 | Train Loss Unweighted: 0.5703 | Train Loss Components: Auxiliary: 0.2236 | Cardinality: 0.1385 | Classification: 0.1093 | Classification_binary: 0.1797 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1379 | Regression_eta: 0.0020 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.1460 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0773 | Regression_eta: 0.0366 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0541 | LR: 6.97e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4864s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0068 | max_forward: 0.7612 | max_loss: 0.2297 | max_backward: 1.9668 | max_optimizer_step: 0.5229 | max_tensorboard_logging: 0.5756 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0374 | Memory: rss_MB: 6508.5 | uss_MB: 5869.4 | child_rss_MB: 5047.4 | allocated_MB: 449.1 | reserved_MB: 29680.0 | inactive_split_MB: 1100.9 +rank7 [2026-10-29 03:50:08,780] INFO : Step 19800/50000 rank7 | Train Loss: 1.2806 | Train Loss Unweighted: 0.6390 | Train Loss Components: Auxiliary: 0.2490 | Cardinality: 0.1432 | Classification: 0.1110 | Classification_binary: 0.2282 | HitGrouping: 0.1668 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1466 | Regression_eta: 0.0030 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0645 | Classification_binary: 0.2093 | Regression_cos_phi: 0.1029 | Regression_energy: 0.0881 | Regression_eta: 0.0810 | Regression_pt: 0.0949 | Regression_sin_phi: 0.1410 | LR: 6.94e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5062s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0067 | max_forward: 0.8034 | max_loss: 0.1812 | max_backward: 1.9502 | max_optimizer_step: 0.2119 | max_tensorboard_logging: 0.5857 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6510.7 | uss_MB: 5873.6 | child_rss_MB: 5045.7 | allocated_MB: 452.3 | reserved_MB: 29680.0 | inactive_split_MB: 1110.9 +rank7 [2026-10-29 03:54:46,662] INFO : Step 19900/50000 rank7 | Train Loss: 1.0692 | Train Loss Unweighted: 0.5407 | Train Loss Components: Auxiliary: 0.2077 | Cardinality: 0.1171 | Classification: 0.0922 | Classification_binary: 0.1812 | HitGrouping: 0.1610 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1314 | Regression_eta: 0.0015 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.1917 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0747 | Regression_eta: 0.0463 | Regression_pt: 0.0769 | Regression_sin_phi: 0.0838 | LR: 6.92e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6127s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7533 | max_loss: 0.2412 | max_backward: 2.0720 | max_optimizer_step: 0.2135 | max_tensorboard_logging: 0.5848 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6505.8 | uss_MB: 5870.0 | child_rss_MB: 5057.5 | allocated_MB: 444.7 | reserved_MB: 29682.0 | inactive_split_MB: 1000.0 +rank7 [2026-10-29 03:59:25,168] INFO : Step 20000/50000 rank7 | Train Loss: 1.0304 | Train Loss Unweighted: 0.5275 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1060 | Classification: 0.0940 | Classification_binary: 0.1604 | HitGrouping: 0.1656 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1344 | Regression_eta: 0.0015 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1579 | Regression_cos_phi: 0.0732 | Regression_energy: 0.0785 | Regression_eta: 0.0352 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0711 | LR: 6.89e-04 | DataLoad Time: 0.0219s | Model Forward Time: 4.8363s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0064 | max_forward: 0.8721 | max_loss: 0.1507 | max_backward: 2.1861 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6505.7 | uss_MB: 5870.7 | child_rss_MB: 5049.5 | allocated_MB: 445.9 | reserved_MB: 30966.0 | inactive_split_MB: 1179.2 +rank7 [2026-10-29 03:59:26,239] INFO : Running validation on rank7 +rank7 [2026-10-29 03:59:26,240] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 03:59:26,240] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 04:00:19,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:00:19,531] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 04:00:44,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 04:00:45,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 04:00:46,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 04:00:48,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 04:00:58,062] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:00:59,045] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:00:59,501] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 04:01:15,645] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 04:01:16,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 04:01:17,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 04:01:18,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 04:01:28,520] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:01:28,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:01:29,373] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 04:01:45,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 04:01:46,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 04:01:46,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 04:01:47,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_20000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 04:01:57,831] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 04:06:53,362] INFO : Step 20100/50000 rank7 | Train Loss: 1.2400 | Train Loss Unweighted: 0.6086 | Train Loss Components: Auxiliary: 0.2437 | Cardinality: 0.1586 | Classification: 0.1055 | Classification_binary: 0.2238 | HitGrouping: 0.1685 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1360 | Regression_eta: 0.0021 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0649 | Classification_binary: 0.2021 | Regression_cos_phi: 0.0844 | Regression_energy: 0.0675 | Regression_eta: 0.0479 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0597 | LR: 6.86e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6013s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0372 | max_forward: 0.8293 | max_loss: 0.1584 | max_backward: 1.7738 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.7139 | max_tensorboard_flush: 0.0090 | max_post_step_barrier: 0.0516 | Memory: rss_MB: 7674.6 | uss_MB: 7034.7 | child_rss_MB: 5034.6 | allocated_MB: 448.4 | reserved_MB: 29328.0 | inactive_split_MB: 969.2 +rank7 [2026-10-29 04:11:25,835] INFO : Step 20200/50000 rank7 | Train Loss: 1.3163 | Train Loss Unweighted: 0.6194 | Train Loss Components: Auxiliary: 0.2434 | Cardinality: 0.1695 | Classification: 0.1000 | Classification_binary: 0.2250 | HitGrouping: 0.1720 | Regression_cos_phi: 0.0013 | Regression_energy: 0.1433 | Regression_eta: 0.0029 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0012 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0714 | Classification_binary: 0.2342 | Regression_cos_phi: 0.1274 | Regression_energy: 0.0730 | Regression_eta: 0.0648 | Regression_pt: 0.0808 | Regression_sin_phi: 0.1266 | LR: 6.84e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.6418s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8823 | max_loss: 0.1595 | max_backward: 2.1030 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.5136 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0536 | Memory: rss_MB: 7674.6 | uss_MB: 7036.8 | child_rss_MB: 5027.8 | allocated_MB: 449.6 | reserved_MB: 29340.0 | inactive_split_MB: 938.7 +rank7 [2026-10-29 04:16:03,709] INFO : Step 20300/50000 rank7 | Train Loss: 1.1420 | Train Loss Unweighted: 0.5734 | Train Loss Components: Auxiliary: 0.2246 | Cardinality: 0.1487 | Classification: 0.1087 | Classification_binary: 0.1713 | HitGrouping: 0.1696 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1438 | Regression_eta: 0.0018 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.2194 | Regression_cos_phi: 0.1086 | Regression_energy: 0.0789 | Regression_eta: 0.0707 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0756 | LR: 6.81e-04 | DataLoad Time: 0.0185s | Model Forward Time: 4.9986s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0068 | max_forward: 0.9650 | max_loss: 0.1904 | max_backward: 2.0801 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.8679 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0390 | Memory: rss_MB: 6555.4 | uss_MB: 5913.2 | child_rss_MB: 5021.5 | allocated_MB: 444.8 | reserved_MB: 29344.0 | inactive_split_MB: 986.2 +rank7 [2026-10-29 04:20:44,977] INFO : Step 20400/50000 rank7 | Train Loss: 1.1926 | Train Loss Unweighted: 0.5719 | Train Loss Components: Auxiliary: 0.2317 | Cardinality: 0.1642 | Classification: 0.1091 | Classification_binary: 0.1922 | HitGrouping: 0.1692 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1329 | Regression_eta: 0.0018 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0670 | Classification_binary: 0.1724 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0764 | Regression_eta: 0.0552 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0735 | LR: 6.78e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.2534s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0167 | max_forward: 1.2639 | max_loss: 0.1873 | max_backward: 2.4261 | max_optimizer_step: 0.3105 | max_tensorboard_logging: 0.7562 | max_tensorboard_flush: 0.0089 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6556.4 | uss_MB: 5916.7 | child_rss_MB: 5031.4 | allocated_MB: 447.7 | reserved_MB: 29344.0 | inactive_split_MB: 985.2 +rank7 [2026-10-29 04:25:27,116] INFO : Step 20500/50000 rank7 | Train Loss: 1.2110 | Train Loss Unweighted: 0.5861 | Train Loss Components: Auxiliary: 0.2336 | Cardinality: 0.1591 | Classification: 0.1022 | Classification_binary: 0.2117 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1333 | Regression_eta: 0.0017 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 0.1752 | Regression_cos_phi: 0.0720 | Regression_energy: 0.0730 | Regression_eta: 0.0501 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0779 | LR: 6.76e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5475s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 0.7705 | max_loss: 0.1525 | max_backward: 1.8060 | max_optimizer_step: 0.2622 | max_tensorboard_logging: 0.7595 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6553.9 | uss_MB: 5915.2 | child_rss_MB: 5013.6 | allocated_MB: 450.6 | reserved_MB: 29344.0 | inactive_split_MB: 1123.4 +rank7 [2026-10-29 04:30:07,866] INFO : Step 20600/50000 rank7 | Train Loss: 1.1161 | Train Loss Unweighted: 0.5564 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1436 | Classification: 0.0978 | Classification_binary: 0.1795 | HitGrouping: 0.1659 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1373 | Regression_eta: 0.0017 | Regression_pt: 0.1383 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0591 | Classification_binary: 0.1953 | Regression_cos_phi: 0.0644 | Regression_energy: 0.0613 | Regression_eta: 0.0472 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0571 | LR: 6.73e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.4371s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0076 | max_forward: 0.8358 | max_loss: 0.5148 | max_backward: 1.8393 | max_optimizer_step: 0.2146 | max_tensorboard_logging: 0.5518 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6556.9 | uss_MB: 5916.8 | child_rss_MB: 5024.7 | allocated_MB: 445.4 | reserved_MB: 31380.0 | inactive_split_MB: 917.0 +rank7 [2026-10-29 04:34:51,728] INFO : Step 20700/50000 rank7 | Train Loss: 1.0740 | Train Loss Unweighted: 0.5459 | Train Loss Components: Auxiliary: 0.2086 | Cardinality: 0.1252 | Classification: 0.0962 | Classification_binary: 0.1560 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1437 | Regression_eta: 0.0016 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.1536 | Regression_cos_phi: 0.0854 | Regression_energy: 0.1022 | Regression_eta: 0.0389 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0751 | LR: 6.70e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.0708s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0060 | max_forward: 0.7111 | max_loss: 0.2396 | max_backward: 1.6829 | max_optimizer_step: 0.2121 | max_tensorboard_logging: 0.4996 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0360 | Memory: rss_MB: 6553.0 | uss_MB: 5912.9 | child_rss_MB: 5017.8 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1185.7 +rank7 [2026-10-29 04:39:29,327] INFO : Step 20800/50000 rank7 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5426 | Train Loss Components: Auxiliary: 0.2163 | Cardinality: 0.1263 | Classification: 0.0965 | Classification_binary: 0.1999 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1207 | Regression_eta: 0.0017 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.2052 | Regression_cos_phi: 0.0823 | Regression_energy: 0.0572 | Regression_eta: 0.0356 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0696 | LR: 6.67e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.7626s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.9079 | max_loss: 0.1550 | max_backward: 2.1083 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.5897 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6553.9 | uss_MB: 5918.9 | child_rss_MB: 5012.5 | allocated_MB: 448.2 | reserved_MB: 31380.0 | inactive_split_MB: 1157.8 +rank7 [2026-10-29 04:44:09,835] INFO : Step 20900/50000 rank7 | Train Loss: 1.2188 | Train Loss Unweighted: 0.5970 | Train Loss Components: Auxiliary: 0.2329 | Cardinality: 0.1525 | Classification: 0.1062 | Classification_binary: 0.2022 | HitGrouping: 0.1671 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0022 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0587 | Classification_binary: 0.1993 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0773 | Regression_eta: 0.0469 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0766 | LR: 6.65e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.7891s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0138 | max_forward: 0.8922 | max_loss: 0.1535 | max_backward: 2.0483 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.7490 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6550.7 | uss_MB: 5916.4 | child_rss_MB: 5027.7 | allocated_MB: 447.8 | reserved_MB: 31380.0 | inactive_split_MB: 1126.3 +rank7 [2026-10-29 04:48:48,418] INFO : Step 21000/50000 rank7 | Train Loss: 1.0503 | Train Loss Unweighted: 0.5338 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1267 | Classification: 0.0900 | Classification_binary: 0.1573 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1405 | Regression_eta: 0.0016 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1625 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0832 | Regression_eta: 0.0467 | Regression_pt: 0.0914 | Regression_sin_phi: 0.0834 | LR: 6.62e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.8563s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7596 | max_loss: 0.2536 | max_backward: 1.8713 | max_optimizer_step: 0.3177 | max_tensorboard_logging: 0.9396 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0303 | Memory: rss_MB: 6551.7 | uss_MB: 5915.9 | child_rss_MB: 5032.5 | allocated_MB: 445.0 | reserved_MB: 31380.0 | inactive_split_MB: 1148.0 +rank7 [2026-10-29 04:48:49,285] INFO : Running validation on rank7 +rank7 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 04:48:49,285] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 04:49:44,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:49:44,824] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 04:50:11,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 04:50:12,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 04:50:13,403] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 04:50:14,346] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 04:50:24,357] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:50:25,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:50:25,769] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 04:50:41,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 04:50:42,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 04:50:43,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 04:50:44,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 04:50:54,417] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 04:50:55,259] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 04:50:55,826] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 04:51:11,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 04:51:12,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 04:51:12,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 04:51:13,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_21000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 04:51:23,844] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 04:56:25,805] INFO : Step 21100/50000 rank7 | Train Loss: 1.1646 | Train Loss Unweighted: 0.5785 | Train Loss Components: Auxiliary: 0.2274 | Cardinality: 0.1401 | Classification: 0.1095 | Classification_binary: 0.1927 | HitGrouping: 0.1649 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1344 | Regression_eta: 0.0022 | Regression_pt: 0.1378 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0725 | Classification_binary: 0.1791 | Regression_cos_phi: 0.0825 | Regression_energy: 0.0901 | Regression_eta: 0.0567 | Regression_pt: 0.0970 | Regression_sin_phi: 0.0763 | LR: 6.59e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.4038s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.9258 | max_loss: 0.1506 | max_backward: 2.4347 | max_optimizer_step: 0.3109 | max_tensorboard_logging: 0.8145 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7847.0 | uss_MB: 5890.3 | child_rss_MB: 5018.3 | allocated_MB: 448.7 | reserved_MB: 23438.0 | inactive_split_MB: 911.7 +rank7 [2026-10-29 04:56:59,983] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank7 [2026-10-29 04:56:59,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-29 05:01:22,613] INFO : Step 21200/50000 rank7 | Train Loss: 1.1437 | Train Loss Unweighted: 0.5637 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1413 | Classification: 0.0985 | Classification_binary: 0.1768 | HitGrouping: 0.1669 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1416 | Regression_eta: 0.0020 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0692 | Classification_binary: 0.1923 | Regression_cos_phi: 0.0743 | Regression_energy: 0.0856 | Regression_eta: 0.0572 | Regression_pt: 0.0926 | Regression_sin_phi: 0.0730 | LR: 6.56e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5646s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0068 | max_forward: 0.7993 | max_loss: 0.1450 | max_backward: 1.7427 | max_optimizer_step: 0.3444 | max_tensorboard_logging: 0.6617 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0614 | Memory: rss_MB: 7842.9 | uss_MB: 7208.8 | child_rss_MB: 5012.9 | allocated_MB: 446.9 | reserved_MB: 23590.0 | inactive_split_MB: 1086.9 +rank7 [2026-10-29 05:06:02,215] INFO : Step 21300/50000 rank7 | Train Loss: 1.1586 | Train Loss Unweighted: 0.5721 | Train Loss Components: Auxiliary: 0.2195 | Cardinality: 0.1648 | Classification: 0.1118 | Classification_binary: 0.1760 | HitGrouping: 0.1681 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1392 | Regression_eta: 0.0021 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0584 | Classification_binary: 0.1701 | Regression_cos_phi: 0.0522 | Regression_energy: 0.0727 | Regression_eta: 0.0474 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0581 | LR: 6.54e-04 | DataLoad Time: 0.0213s | Model Forward Time: 5.2679s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0558 | max_forward: 0.9498 | max_loss: 0.1699 | max_backward: 2.4288 | max_optimizer_step: 0.2160 | max_tensorboard_logging: 0.7788 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1270 | Memory: rss_MB: 6605.4 | uss_MB: 5978.0 | child_rss_MB: 5012.8 | allocated_MB: 447.4 | reserved_MB: 25440.0 | inactive_split_MB: 1003.9 +rank7 [2026-10-29 05:10:41,727] INFO : Step 21400/50000 rank7 | Train Loss: 1.1843 | Train Loss Unweighted: 0.5823 | Train Loss Components: Auxiliary: 0.2312 | Cardinality: 0.1516 | Classification: 0.1038 | Classification_binary: 0.2065 | HitGrouping: 0.1620 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1335 | Regression_eta: 0.0020 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1761 | Regression_cos_phi: 0.1022 | Regression_energy: 0.0797 | Regression_eta: 0.0565 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0919 | LR: 6.51e-04 | DataLoad Time: 0.0189s | Model Forward Time: 5.0965s | Phase Times: max_pre_step_barrier: 0.0085 | max_batch_to_device: 0.0640 | max_forward: 0.9920 | max_loss: 0.5036 | max_backward: 2.0711 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.8200 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0457 | Memory: rss_MB: 6606.3 | uss_MB: 5978.8 | child_rss_MB: 5018.5 | allocated_MB: 448.9 | reserved_MB: 25440.0 | inactive_split_MB: 994.4 +rank7 [2026-10-29 05:15:15,087] INFO : Step 21500/50000 rank7 | Train Loss: 1.1484 | Train Loss Unweighted: 0.5661 | Train Loss Components: Auxiliary: 0.2269 | Cardinality: 0.1478 | Classification: 0.0953 | Classification_binary: 0.2062 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1299 | Regression_eta: 0.0018 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0622 | Classification_binary: 0.2027 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0682 | Regression_eta: 0.0356 | Regression_pt: 0.0743 | Regression_sin_phi: 0.0680 | LR: 6.48e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.9361s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8371 | max_loss: 0.1537 | max_backward: 2.1150 | max_optimizer_step: 0.2158 | max_tensorboard_logging: 1.8633 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6603.8 | uss_MB: 5979.6 | child_rss_MB: 5017.0 | allocated_MB: 447.2 | reserved_MB: 27056.0 | inactive_split_MB: 1468.9 +rank7 [2026-10-29 05:19:51,738] INFO : Step 21600/50000 rank7 | Train Loss: 1.0171 | Train Loss Unweighted: 0.5098 | Train Loss Components: Auxiliary: 0.2004 | Cardinality: 0.1167 | Classification: 0.0973 | Classification_binary: 0.1559 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1265 | Regression_eta: 0.0014 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1371 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0802 | Regression_eta: 0.0375 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0456 | LR: 6.45e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5913s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0062 | max_forward: 0.7285 | max_loss: 0.1940 | max_backward: 1.8148 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.8819 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6602.2 | uss_MB: 5978.2 | child_rss_MB: 5014.0 | allocated_MB: 445.7 | reserved_MB: 27056.0 | inactive_split_MB: 1196.5 +rank7 [2026-10-29 05:24:30,476] INFO : Step 21700/50000 rank7 | Train Loss: 1.1623 | Train Loss Unweighted: 0.5535 | Train Loss Components: Auxiliary: 0.2253 | Cardinality: 0.1498 | Classification: 0.1025 | Classification_binary: 0.1819 | HitGrouping: 0.1684 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0799 | Classification_binary: 0.1684 | Regression_cos_phi: 0.0795 | Regression_energy: 0.0691 | Regression_eta: 0.0476 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0656 | LR: 6.43e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4943s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0067 | max_forward: 0.7947 | max_loss: 0.1379 | max_backward: 1.9808 | max_optimizer_step: 0.2157 | max_tensorboard_logging: 0.6079 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6600.2 | uss_MB: 5978.2 | child_rss_MB: 5010.8 | allocated_MB: 448.2 | reserved_MB: 27056.0 | inactive_split_MB: 1045.2 +rank7 [2026-10-29 05:29:08,470] INFO : Step 21800/50000 rank7 | Train Loss: 1.0898 | Train Loss Unweighted: 0.5518 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1233 | Classification: 0.0977 | Classification_binary: 0.1967 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1269 | Regression_eta: 0.0015 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1775 | Regression_cos_phi: 0.0552 | Regression_energy: 0.0672 | Regression_eta: 0.0474 | Regression_pt: 0.0715 | Regression_sin_phi: 0.0966 | LR: 6.40e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.7210s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0066 | max_forward: 0.8952 | max_loss: 0.1765 | max_backward: 2.1628 | max_optimizer_step: 0.2138 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6602.0 | uss_MB: 5872.4 | child_rss_MB: 5012.0 | allocated_MB: 448.4 | reserved_MB: 27056.0 | inactive_split_MB: 1001.4 +rank7 [2026-10-29 05:33:46,455] INFO : Step 21900/50000 rank7 | Train Loss: 1.0699 | Train Loss Unweighted: 0.5306 | Train Loss Components: Auxiliary: 0.2121 | Cardinality: 0.1436 | Classification: 0.0991 | Classification_binary: 0.1704 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1276 | Regression_eta: 0.0015 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0816 | Classification_binary: 0.1996 | Regression_cos_phi: 0.1030 | Regression_energy: 0.0731 | Regression_eta: 0.0581 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0990 | LR: 6.37e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.4507s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7813 | max_loss: 0.1495 | max_backward: 1.8703 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0668 | Memory: rss_MB: 6600.0 | uss_MB: 5976.9 | child_rss_MB: 5013.0 | allocated_MB: 447.4 | reserved_MB: 27056.0 | inactive_split_MB: 1002.4 +rank7 [2026-10-29 05:38:23,934] INFO : Step 22000/50000 rank7 | Train Loss: 1.0659 | Train Loss Unweighted: 0.5232 | Train Loss Components: Auxiliary: 0.2100 | Cardinality: 0.1294 | Classification: 0.0955 | Classification_binary: 0.1521 | HitGrouping: 0.1687 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1360 | Regression_eta: 0.0016 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0628 | Classification_binary: 0.1411 | Regression_cos_phi: 0.0968 | Regression_energy: 0.0756 | Regression_eta: 0.0444 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0706 | LR: 6.34e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4508s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7855 | max_loss: 0.1383 | max_backward: 1.9648 | max_optimizer_step: 0.2132 | max_tensorboard_logging: 0.5884 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0409 | Memory: rss_MB: 6605.4 | uss_MB: 5979.9 | child_rss_MB: 5015.2 | allocated_MB: 449.1 | reserved_MB: 27056.0 | inactive_split_MB: 1180.9 +rank7 [2026-10-29 05:38:24,835] INFO : Running validation on rank7 +rank7 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 05:38:24,835] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 05:39:16,672] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:39:17,216] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 05:39:42,764] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 05:39:43,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 05:39:44,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 05:39:45,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 05:39:55,724] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 05:39:56,594] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:39:57,074] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 05:40:13,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 05:40:14,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 05:40:15,590] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 05:40:16,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 05:40:26,658] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 05:40:27,479] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 05:40:27,972] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 05:40:43,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 05:40:44,420] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 05:40:45,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 05:40:46,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_22000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 05:40:56,062] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 05:45:56,687] INFO : Step 22100/50000 rank7 | Train Loss: 1.1235 | Train Loss Unweighted: 0.5530 | Train Loss Components: Auxiliary: 0.2250 | Cardinality: 0.1528 | Classification: 0.0997 | Classification_binary: 0.1905 | HitGrouping: 0.1627 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1290 | Regression_eta: 0.0016 | Regression_pt: 0.1304 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0624 | Classification_binary: 0.1802 | Regression_cos_phi: 0.0959 | Regression_energy: 0.0666 | Regression_eta: 0.0719 | Regression_pt: 0.0737 | Regression_sin_phi: 0.1007 | LR: 6.32e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5240s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8135 | max_loss: 0.1488 | max_backward: 1.7877 | max_optimizer_step: 0.2571 | max_tensorboard_logging: 0.5979 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 7900.1 | uss_MB: 7271.0 | child_rss_MB: 5019.5 | allocated_MB: 451.8 | reserved_MB: 29136.0 | inactive_split_MB: 1136.2 +rank7 [2026-10-29 05:50:45,342] INFO : Step 22200/50000 rank7 | Train Loss: 1.1922 | Train Loss Unweighted: 0.5684 | Train Loss Components: Auxiliary: 0.2361 | Cardinality: 0.1699 | Classification: 0.0906 | Classification_binary: 0.2085 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0019 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.2070 | Regression_cos_phi: 0.0480 | Regression_energy: 0.0810 | Regression_eta: 0.0424 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0835 | LR: 6.29e-04 | DataLoad Time: 0.0144s | Model Forward Time: 4.5700s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0069 | max_forward: 0.8616 | max_loss: 0.1545 | max_backward: 1.6969 | max_optimizer_step: 0.2424 | max_tensorboard_logging: 0.7120 | max_tensorboard_flush: 0.0150 | max_post_step_barrier: 0.0608 | Memory: rss_MB: 7899.1 | uss_MB: 7269.8 | child_rss_MB: 5025.9 | allocated_MB: 448.1 | reserved_MB: 29136.0 | inactive_split_MB: 1371.0 +rank7 [2026-10-29 05:55:21,940] INFO : Step 22300/50000 rank7 | Train Loss: 1.0221 | Train Loss Unweighted: 0.5006 | Train Loss Components: Auxiliary: 0.1973 | Cardinality: 0.1169 | Classification: 0.0876 | Classification_binary: 0.1576 | HitGrouping: 0.1631 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1252 | Regression_eta: 0.0021 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0599 | Classification_binary: 0.2162 | Regression_cos_phi: 0.0988 | Regression_energy: 0.0645 | Regression_eta: 0.0733 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0907 | LR: 6.26e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.5036s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7903 | max_loss: 0.1446 | max_backward: 1.7078 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 0.8083 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0461 | Memory: rss_MB: 6690.1 | uss_MB: 6062.0 | child_rss_MB: 5023.7 | allocated_MB: 446.4 | reserved_MB: 29136.0 | inactive_split_MB: 1159.3 +rank7 [2026-10-29 06:00:08,367] INFO : Step 22400/50000 rank7 | Train Loss: 1.1170 | Train Loss Unweighted: 0.5609 | Train Loss Components: Auxiliary: 0.2245 | Cardinality: 0.1391 | Classification: 0.1065 | Classification_binary: 0.1807 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1348 | Regression_eta: 0.0019 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1844 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0575 | Regression_eta: 0.1034 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0740 | LR: 6.23e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.9526s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0066 | max_forward: 0.7658 | max_loss: 0.2479 | max_backward: 1.9808 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.9928 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6691.0 | uss_MB: 6065.0 | child_rss_MB: 5027.0 | allocated_MB: 447.1 | reserved_MB: 31152.0 | inactive_split_MB: 925.7 +rank7 [2026-10-29 06:04:41,559] INFO : Step 22500/50000 rank7 | Train Loss: 1.1245 | Train Loss Unweighted: 0.5482 | Train Loss Components: Auxiliary: 0.2231 | Cardinality: 0.1463 | Classification: 0.0856 | Classification_binary: 0.2083 | HitGrouping: 0.1607 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1244 | Regression_eta: 0.0016 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1820 | Regression_cos_phi: 0.0587 | Regression_energy: 0.0618 | Regression_eta: 0.0449 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0744 | LR: 6.20e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.0089s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0068 | max_forward: 1.1916 | max_loss: 0.1613 | max_backward: 2.1225 | max_optimizer_step: 0.2176 | max_tensorboard_logging: 0.7664 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0423 | Memory: rss_MB: 6685.2 | uss_MB: 6063.6 | child_rss_MB: 5003.5 | allocated_MB: 447.4 | reserved_MB: 31152.0 | inactive_split_MB: 959.7 +rank7 [2026-10-29 06:09:13,551] INFO : Step 22600/50000 rank7 | Train Loss: 1.0653 | Train Loss Unweighted: 0.5177 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1260 | Classification: 0.0928 | Classification_binary: 0.1789 | HitGrouping: 0.1619 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1208 | Regression_eta: 0.0019 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 0.1973 | Regression_cos_phi: 0.0950 | Regression_energy: 0.0576 | Regression_eta: 0.0607 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0764 | LR: 6.18e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.8011s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0651 | max_forward: 1.1064 | max_loss: 0.1531 | max_backward: 1.9236 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.8515 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6676.3 | uss_MB: 6063.0 | child_rss_MB: 5024.2 | allocated_MB: 448.3 | reserved_MB: 31154.0 | inactive_split_MB: 961.5 +rank7 [2026-10-29 06:13:51,719] INFO : Step 22700/50000 rank7 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5090 | Train Loss Components: Auxiliary: 0.1959 | Cardinality: 0.1101 | Classification: 0.0877 | Classification_binary: 0.1479 | HitGrouping: 0.1634 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1339 | Regression_eta: 0.0015 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0570 | Classification_binary: 0.1357 | Regression_cos_phi: 0.1033 | Regression_energy: 0.0582 | Regression_eta: 0.0545 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0727 | LR: 6.15e-04 | DataLoad Time: 0.0217s | Model Forward Time: 4.4577s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 1.0743 | max_loss: 0.1383 | max_backward: 1.7235 | max_optimizer_step: 0.2204 | max_tensorboard_logging: 0.5170 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 6672.9 | uss_MB: 6061.8 | child_rss_MB: 5030.8 | allocated_MB: 446.5 | reserved_MB: 31154.0 | inactive_split_MB: 972.7 +rank7 [2026-10-29 06:18:32,626] INFO : Step 22800/50000 rank7 | Train Loss: 1.0730 | Train Loss Unweighted: 0.5497 | Train Loss Components: Auxiliary: 0.2215 | Cardinality: 0.1332 | Classification: 0.1095 | Classification_binary: 0.1782 | HitGrouping: 0.1573 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1289 | Regression_eta: 0.0014 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0615 | Classification_binary: 0.1614 | Regression_cos_phi: 0.0479 | Regression_energy: 0.0636 | Regression_eta: 0.0344 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0560 | LR: 6.12e-04 | DataLoad Time: 0.0209s | Model Forward Time: 4.7940s | Phase Times: max_pre_step_barrier: 0.0043 | max_batch_to_device: 0.0064 | max_forward: 0.8406 | max_loss: 0.2500 | max_backward: 2.0107 | max_optimizer_step: 0.2761 | max_tensorboard_logging: 0.6168 | max_tensorboard_flush: 0.0129 | max_post_step_barrier: 0.0539 | Memory: rss_MB: 6673.9 | uss_MB: 6062.6 | child_rss_MB: 5029.6 | allocated_MB: 447.2 | reserved_MB: 31154.0 | inactive_split_MB: 922.1 +rank7 [2026-10-29 06:23:13,203] INFO : Step 22900/50000 rank7 | Train Loss: 1.1830 | Train Loss Unweighted: 0.5795 | Train Loss Components: Auxiliary: 0.2389 | Cardinality: 0.1788 | Classification: 0.0948 | Classification_binary: 0.2023 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1393 | Regression_eta: 0.0018 | Regression_pt: 0.1397 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0522 | Classification_binary: 0.2073 | Regression_cos_phi: 0.0696 | Regression_energy: 0.0704 | Regression_eta: 0.0743 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0632 | LR: 6.09e-04 | DataLoad Time: 0.0199s | Model Forward Time: 4.9935s | Phase Times: max_pre_step_barrier: 0.0042 | max_batch_to_device: 0.0073 | max_forward: 0.7992 | max_loss: 0.1478 | max_backward: 1.9051 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.9858 | max_tensorboard_flush: 0.0123 | max_post_step_barrier: 0.0497 | Memory: rss_MB: 6664.2 | uss_MB: 6055.9 | child_rss_MB: 5028.8 | allocated_MB: 448.8 | reserved_MB: 31154.0 | inactive_split_MB: 1109.2 +rank7 [2026-10-29 06:27:51,864] INFO : Step 23000/50000 rank7 | Train Loss: 0.9959 | Train Loss Unweighted: 0.5014 | Train Loss Components: Auxiliary: 0.1951 | Cardinality: 0.1184 | Classification: 0.0781 | Classification_binary: 0.1455 | HitGrouping: 0.1653 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1367 | Regression_eta: 0.0017 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0355 | Classification_binary: 0.1516 | Regression_cos_phi: 0.0516 | Regression_energy: 0.0727 | Regression_eta: 0.0353 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0568 | LR: 6.06e-04 | DataLoad Time: 0.0205s | Model Forward Time: 4.6101s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0065 | max_forward: 0.7528 | max_loss: 0.2292 | max_backward: 1.7818 | max_optimizer_step: 0.2179 | max_tensorboard_logging: 0.8884 | max_tensorboard_flush: 0.0060 | max_post_step_barrier: 0.0417 | Memory: rss_MB: 6672.0 | uss_MB: 6063.2 | child_rss_MB: 5028.5 | allocated_MB: 443.9 | reserved_MB: 31154.0 | inactive_split_MB: 985.1 +rank7 [2026-10-29 06:27:52,750] INFO : Running validation on rank7 +rank7 [2026-10-29 06:27:52,751] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 06:27:52,751] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 06:28:45,238] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:28:45,978] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 06:29:12,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 06:29:13,213] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 06:29:14,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 06:29:15,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 06:29:25,142] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 06:29:26,163] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:29:26,632] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 06:29:42,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 06:29:43,309] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 06:29:44,260] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 06:29:45,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 06:29:55,207] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 06:29:56,165] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 06:29:56,644] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 06:30:12,440] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 06:30:13,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 06:30:14,094] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 06:30:14,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_23000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 06:30:24,941] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 06:35:29,455] INFO : Step 23100/50000 rank7 | Train Loss: 1.1240 | Train Loss Unweighted: 0.5473 | Train Loss Components: Auxiliary: 0.2202 | Cardinality: 0.1320 | Classification: 0.1122 | Classification_binary: 0.1745 | HitGrouping: 0.1636 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1274 | Regression_eta: 0.0021 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1503 | Regression_cos_phi: 0.0647 | Regression_energy: 0.0650 | Regression_eta: 0.0359 | Regression_pt: 0.0705 | Regression_sin_phi: 0.0555 | LR: 6.04e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.8779s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0066 | max_forward: 0.7914 | max_loss: 0.1430 | max_backward: 1.7255 | max_optimizer_step: 0.3295 | max_tensorboard_logging: 0.9822 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0695 | Memory: rss_MB: 7843.1 | uss_MB: 7233.5 | child_rss_MB: 5030.9 | allocated_MB: 449.4 | reserved_MB: 25680.0 | inactive_split_MB: 1046.0 +rank7 [2026-10-29 06:40:11,955] INFO : Step 23200/50000 rank7 | Train Loss: 1.1736 | Train Loss Unweighted: 0.5745 | Train Loss Components: Auxiliary: 0.2341 | Cardinality: 0.1586 | Classification: 0.1057 | Classification_binary: 0.2033 | HitGrouping: 0.1597 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1304 | Regression_eta: 0.0018 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0533 | Classification_binary: 0.1490 | Regression_cos_phi: 0.0481 | Regression_energy: 0.0826 | Regression_eta: 0.0408 | Regression_pt: 0.0905 | Regression_sin_phi: 0.0709 | LR: 6.01e-04 | DataLoad Time: 0.0196s | Model Forward Time: 5.1111s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0065 | max_forward: 0.9381 | max_loss: 0.1692 | max_backward: 2.2545 | max_optimizer_step: 0.2525 | max_tensorboard_logging: 0.6161 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 7846.1 | uss_MB: 5861.1 | child_rss_MB: 5025.2 | allocated_MB: 447.8 | reserved_MB: 25680.0 | inactive_split_MB: 1169.7 +rank7 [2026-10-29 06:44:52,828] INFO : Step 23300/50000 rank7 | Train Loss: 1.0647 | Train Loss Unweighted: 0.5262 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1268 | Classification: 0.0958 | Classification_binary: 0.1665 | HitGrouping: 0.1615 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1301 | Regression_eta: 0.0017 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1915 | Regression_cos_phi: 0.0616 | Regression_energy: 0.0866 | Regression_eta: 0.0393 | Regression_pt: 0.0931 | Regression_sin_phi: 0.0522 | LR: 5.98e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6098s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8613 | max_loss: 0.1501 | max_backward: 2.0035 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 0.6274 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6729.0 | uss_MB: 6118.5 | child_rss_MB: 5031.7 | allocated_MB: 447.3 | reserved_MB: 29702.0 | inactive_split_MB: 1252.4 +rank7 [2026-10-29 06:49:32,124] INFO : Step 23400/50000 rank7 | Train Loss: 0.9916 | Train Loss Unweighted: 0.5023 | Train Loss Components: Auxiliary: 0.1957 | Cardinality: 0.1164 | Classification: 0.1006 | Classification_binary: 0.1374 | HitGrouping: 0.1642 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1310 | Regression_eta: 0.0015 | Regression_pt: 0.1301 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0701 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0711 | Regression_energy: 0.0730 | Regression_eta: 0.0301 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0698 | LR: 5.95e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.3943s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7229 | max_loss: 0.2412 | max_backward: 1.6798 | max_optimizer_step: 0.2245 | max_tensorboard_logging: 0.4992 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.1172 | Memory: rss_MB: 6727.8 | uss_MB: 6117.0 | child_rss_MB: 5031.9 | allocated_MB: 445.7 | reserved_MB: 29702.0 | inactive_split_MB: 1158.3 +rank7 [2026-10-29 06:54:42,355] INFO : Step 23500/50000 rank7 | Train Loss: 1.0349 | Train Loss Unweighted: 0.5234 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1083 | Classification: 0.0851 | Classification_binary: 0.1870 | HitGrouping: 0.1599 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1235 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1807 | Regression_cos_phi: 0.0862 | Regression_energy: 0.0688 | Regression_eta: 0.0541 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0766 | LR: 5.92e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.7747s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7924 | max_loss: 0.2020 | max_backward: 1.9787 | max_optimizer_step: 0.2210 | max_tensorboard_logging: 0.7987 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6728.4 | uss_MB: 6111.2 | child_rss_MB: 5026.6 | allocated_MB: 446.9 | reserved_MB: 31774.0 | inactive_split_MB: 1042.0 +rank7 [2026-10-29 06:59:38,280] INFO : Step 23600/50000 rank7 | Train Loss: 1.1326 | Train Loss Unweighted: 0.5418 | Train Loss Components: Auxiliary: 0.2224 | Cardinality: 0.1458 | Classification: 0.0933 | Classification_binary: 0.1900 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1267 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0011 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.2067 | Regression_cos_phi: 0.0785 | Regression_energy: 0.0677 | Regression_eta: 0.0382 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0583 | LR: 5.90e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9843s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0063 | max_forward: 0.7946 | max_loss: 0.1855 | max_backward: 1.9114 | max_optimizer_step: 0.2664 | max_tensorboard_logging: 0.9547 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.1192 | Memory: rss_MB: 6730.1 | uss_MB: 6120.3 | child_rss_MB: 5028.3 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1333.8 +rank7 [2026-10-29 07:04:20,062] INFO : Step 23700/50000 rank7 | Train Loss: 1.0167 | Train Loss Unweighted: 0.4968 | Train Loss Components: Auxiliary: 0.1932 | Cardinality: 0.1158 | Classification: 0.0853 | Classification_binary: 0.1475 | HitGrouping: 0.1645 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1285 | Regression_eta: 0.0017 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0652 | Classification_binary: 0.1553 | Regression_cos_phi: 0.0855 | Regression_energy: 0.0657 | Regression_eta: 0.0433 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0711 | LR: 5.87e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.7146s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0221 | max_forward: 0.7952 | max_loss: 0.1461 | max_backward: 1.8308 | max_optimizer_step: 0.2225 | max_tensorboard_logging: 0.7718 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0468 | Memory: rss_MB: 6725.6 | uss_MB: 6118.0 | child_rss_MB: 5028.2 | allocated_MB: 446.9 | reserved_MB: 31776.0 | inactive_split_MB: 1160.4 +rank7 [2026-10-29 07:08:57,915] INFO : Step 23800/50000 rank7 | Train Loss: 1.0651 | Train Loss Unweighted: 0.5178 | Train Loss Components: Auxiliary: 0.2090 | Cardinality: 0.1325 | Classification: 0.0886 | Classification_binary: 0.1616 | HitGrouping: 0.1657 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1318 | Regression_eta: 0.0015 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1606 | Regression_cos_phi: 0.0736 | Regression_energy: 0.0762 | Regression_eta: 0.0447 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0730 | LR: 5.84e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.6722s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0067 | max_forward: 0.8562 | max_loss: 0.1426 | max_backward: 1.8700 | max_optimizer_step: 0.2244 | max_tensorboard_logging: 0.7637 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0463 | Memory: rss_MB: 6730.2 | uss_MB: 6120.2 | child_rss_MB: 5026.7 | allocated_MB: 449.4 | reserved_MB: 31776.0 | inactive_split_MB: 1046.7 +rank7 [2026-10-29 07:13:41,573] INFO : Step 23900/50000 rank7 | Train Loss: 1.1515 | Train Loss Unweighted: 0.5718 | Train Loss Components: Auxiliary: 0.2353 | Cardinality: 0.1646 | Classification: 0.0962 | Classification_binary: 0.2086 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1316 | Regression_eta: 0.0016 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1817 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0697 | Regression_eta: 0.0359 | Regression_pt: 0.0768 | Regression_sin_phi: 0.0601 | LR: 5.81e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.8398s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0067 | max_forward: 0.8887 | max_loss: 0.2019 | max_backward: 2.1185 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.6224 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6714.4 | uss_MB: 6108.3 | child_rss_MB: 5032.9 | allocated_MB: 447.1 | reserved_MB: 31776.0 | inactive_split_MB: 1168.8 +rank7 [2026-10-29 07:18:20,710] INFO : Step 24000/50000 rank7 | Train Loss: 1.0174 | Train Loss Unweighted: 0.5187 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1133 | Classification: 0.0873 | Classification_binary: 0.1674 | HitGrouping: 0.1602 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1298 | Regression_eta: 0.0014 | Regression_pt: 0.1313 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0729 | Classification_binary: 0.1798 | Regression_cos_phi: 0.0824 | Regression_energy: 0.0634 | Regression_eta: 0.0454 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0862 | LR: 5.78e-04 | DataLoad Time: 0.0202s | Model Forward Time: 4.6299s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0073 | max_forward: 1.3237 | max_loss: 0.2403 | max_backward: 1.8319 | max_optimizer_step: 0.2240 | max_tensorboard_logging: 0.5068 | max_tensorboard_flush: 0.0158 | max_post_step_barrier: 0.0435 | Memory: rss_MB: 6721.2 | uss_MB: 6114.9 | child_rss_MB: 5029.4 | allocated_MB: 446.4 | reserved_MB: 31776.0 | inactive_split_MB: 957.1 +rank7 [2026-10-29 07:18:21,624] INFO : Running validation on rank7 +rank7 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 07:18:21,624] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 07:19:13,460] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:19:13,969] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 07:19:40,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 07:19:42,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 07:19:43,322] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 07:19:44,284] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 07:19:54,294] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 07:19:55,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:19:55,706] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 07:20:11,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 07:20:12,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 07:20:13,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 07:20:14,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 07:20:24,753] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 07:20:25,040] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 07:20:25,604] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 07:20:41,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 07:20:42,202] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 07:20:43,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 07:20:43,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_24000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 07:20:53,877] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 07:26:06,615] INFO : Step 24100/50000 rank7 | Train Loss: 1.0669 | Train Loss Unweighted: 0.5331 | Train Loss Components: Auxiliary: 0.2097 | Cardinality: 0.1324 | Classification: 0.0928 | Classification_binary: 0.1578 | HitGrouping: 0.1639 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1390 | Regression_eta: 0.0017 | Regression_pt: 0.1400 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0650 | Classification_binary: 0.1439 | Regression_cos_phi: 0.0597 | Regression_energy: 0.0644 | Regression_eta: 0.0465 | Regression_pt: 0.0704 | Regression_sin_phi: 0.1096 | LR: 5.75e-04 | DataLoad Time: 0.0168s | Model Forward Time: 7.5195s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.7836 | max_loss: 0.1442 | max_backward: 1.8747 | max_optimizer_step: 0.2345 | max_tensorboard_logging: 3.6030 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0626 | Memory: rss_MB: 7857.5 | uss_MB: 7243.1 | child_rss_MB: 5051.0 | allocated_MB: 448.1 | reserved_MB: 24490.0 | inactive_split_MB: 1198.5 +rank7 [2026-10-29 07:30:59,926] INFO : Step 24200/50000 rank7 | Train Loss: 1.1691 | Train Loss Unweighted: 0.5645 | Train Loss Components: Auxiliary: 0.2345 | Cardinality: 0.1546 | Classification: 0.1031 | Classification_binary: 0.1915 | HitGrouping: 0.1644 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1325 | Regression_eta: 0.0019 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0795 | Classification_binary: 0.1641 | Regression_cos_phi: 0.0779 | Regression_energy: 0.0827 | Regression_eta: 0.0624 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0843 | LR: 5.73e-04 | DataLoad Time: 0.0166s | Model Forward Time: 5.2333s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.9207 | max_loss: 0.1533 | max_backward: 2.2288 | max_optimizer_step: 0.2436 | max_tensorboard_logging: 0.8522 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0603 | Memory: rss_MB: 7858.2 | uss_MB: 7247.2 | child_rss_MB: 5039.0 | allocated_MB: 450.2 | reserved_MB: 26490.0 | inactive_split_MB: 976.9 +rank7 [2026-10-29 07:35:42,204] INFO : Step 24300/50000 rank7 | Train Loss: 1.1023 | Train Loss Unweighted: 0.5336 | Train Loss Components: Auxiliary: 0.2203 | Cardinality: 0.1380 | Classification: 0.0927 | Classification_binary: 0.1863 | HitGrouping: 0.1592 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1247 | Regression_eta: 0.0020 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0485 | Classification_binary: 0.2171 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0669 | Regression_eta: 0.0548 | Regression_pt: 0.0730 | Regression_sin_phi: 0.0693 | LR: 5.70e-04 | DataLoad Time: 0.0207s | Model Forward Time: 4.6717s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0065 | max_forward: 0.8648 | max_loss: 0.1481 | max_backward: 1.7152 | max_optimizer_step: 0.2781 | max_tensorboard_logging: 0.7831 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 6810.0 | uss_MB: 6199.1 | child_rss_MB: 5043.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 983.3 +rank7 [2026-10-29 07:40:19,152] INFO : Step 24400/50000 rank7 | Train Loss: 0.9707 | Train Loss Unweighted: 0.4873 | Train Loss Components: Auxiliary: 0.1913 | Cardinality: 0.1108 | Classification: 0.0879 | Classification_binary: 0.1454 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1254 | Regression_eta: 0.0014 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0404 | Classification_binary: 0.1500 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0701 | Regression_eta: 0.0351 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0574 | LR: 5.67e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.8568s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7628 | max_loss: 0.1940 | max_backward: 2.4253 | max_optimizer_step: 0.2200 | max_tensorboard_logging: 0.9481 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0548 | Memory: rss_MB: 6811.6 | uss_MB: 5803.5 | child_rss_MB: 5042.0 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 1129.3 +rank7 [2026-10-29 07:45:00,050] INFO : Step 24500/50000 rank7 | Train Loss: 1.0937 | Train Loss Unweighted: 0.5343 | Train Loss Components: Auxiliary: 0.2156 | Cardinality: 0.1319 | Classification: 0.0941 | Classification_binary: 0.1676 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0011 | Regression_energy: 0.1334 | Regression_eta: 0.0016 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0459 | Classification_binary: 0.1459 | Regression_cos_phi: 0.1292 | Regression_energy: 0.0600 | Regression_eta: 0.0475 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0750 | LR: 5.64e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4273s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7565 | max_loss: 0.5105 | max_backward: 1.8057 | max_optimizer_step: 0.2534 | max_tensorboard_logging: 0.5891 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6807.0 | uss_MB: 6198.6 | child_rss_MB: 5045.1 | allocated_MB: 447.9 | reserved_MB: 26492.0 | inactive_split_MB: 945.2 +rank7 [2026-10-29 07:49:44,236] INFO : Step 24600/50000 rank7 | Train Loss: 1.0924 | Train Loss Unweighted: 0.5511 | Train Loss Components: Auxiliary: 0.2237 | Cardinality: 0.1233 | Classification: 0.0963 | Classification_binary: 0.1991 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1255 | Regression_eta: 0.0015 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0708 | Classification_binary: 0.2272 | Regression_cos_phi: 0.0908 | Regression_energy: 0.0748 | Regression_eta: 0.0654 | Regression_pt: 0.0819 | Regression_sin_phi: 0.1222 | LR: 5.61e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.6468s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.9060 | max_loss: 0.1523 | max_backward: 2.0894 | max_optimizer_step: 0.2230 | max_tensorboard_logging: 0.5102 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6811.1 | uss_MB: 6200.6 | child_rss_MB: 5046.3 | allocated_MB: 449.5 | reserved_MB: 26492.0 | inactive_split_MB: 943.2 +rank7 [2026-10-29 07:54:29,279] INFO : Step 24700/50000 rank7 | Train Loss: 0.9972 | Train Loss Unweighted: 0.4984 | Train Loss Components: Auxiliary: 0.1950 | Cardinality: 0.1160 | Classification: 0.0782 | Classification_binary: 0.1692 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1234 | Regression_eta: 0.0015 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0538 | Classification_binary: 0.1702 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0684 | Regression_eta: 0.0530 | Regression_pt: 0.0745 | Regression_sin_phi: 0.0744 | LR: 5.58e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.5118s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0060 | max_forward: 0.7847 | max_loss: 0.1519 | max_backward: 1.7159 | max_optimizer_step: 0.2234 | max_tensorboard_logging: 0.8325 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6813.0 | uss_MB: 6199.7 | child_rss_MB: 5050.6 | allocated_MB: 447.9 | reserved_MB: 27708.0 | inactive_split_MB: 1169.6 +rank7 [2026-10-29 07:59:09,068] INFO : Step 24800/50000 rank7 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4722 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1202 | Classification: 0.0808 | Classification_binary: 0.1484 | HitGrouping: 0.1630 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1192 | Regression_eta: 0.0016 | Regression_pt: 0.1206 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1434 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0762 | Regression_eta: 0.0334 | Regression_pt: 0.0846 | Regression_sin_phi: 0.0738 | LR: 5.56e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.7178s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0127 | max_forward: 0.7903 | max_loss: 0.1427 | max_backward: 1.7798 | max_optimizer_step: 0.2260 | max_tensorboard_logging: 0.8943 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6813.0 | uss_MB: 6199.9 | child_rss_MB: 5051.5 | allocated_MB: 447.6 | reserved_MB: 28508.0 | inactive_split_MB: 989.1 +rank7 [2026-10-29 08:03:50,794] INFO : Step 24900/50000 rank7 | Train Loss: 1.0773 | Train Loss Unweighted: 0.5300 | Train Loss Components: Auxiliary: 0.2170 | Cardinality: 0.1506 | Classification: 0.0964 | Classification_binary: 0.1776 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1257 | Regression_eta: 0.0016 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0562 | Classification_binary: 0.1544 | Regression_cos_phi: 0.0721 | Regression_energy: 0.0613 | Regression_eta: 0.0432 | Regression_pt: 0.0683 | Regression_sin_phi: 0.0776 | LR: 5.53e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.3167s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8100 | max_loss: 0.1483 | max_backward: 1.9423 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 1.0834 | max_tensorboard_flush: 0.0000 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 6802.2 | uss_MB: 5593.7 | child_rss_MB: 5050.0 | allocated_MB: 449.1 | reserved_MB: 28508.0 | inactive_split_MB: 993.6 +rank7 [2026-10-29 08:08:37,677] INFO : Step 25000/50000 rank7 | Train Loss: 1.1315 | Train Loss Unweighted: 0.5264 | Train Loss Components: Auxiliary: 0.2254 | Cardinality: 0.1671 | Classification: 0.0915 | Classification_binary: 0.1899 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1201 | Regression_eta: 0.0020 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0648 | Classification_binary: 0.2252 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0616 | Regression_eta: 0.0625 | Regression_pt: 0.0678 | Regression_sin_phi: 0.0772 | LR: 5.50e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.7049s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0063 | max_forward: 0.8152 | max_loss: 0.2476 | max_backward: 1.7290 | max_optimizer_step: 0.2214 | max_tensorboard_logging: 0.8750 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0501 | Memory: rss_MB: 6807.0 | uss_MB: 6195.6 | child_rss_MB: 5052.2 | allocated_MB: 448.5 | reserved_MB: 28508.0 | inactive_split_MB: 953.9 +rank7 [2026-10-29 08:08:38,676] INFO : Running validation on rank7 +rank7 [2026-10-29 08:08:38,676] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 08:08:38,676] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 08:09:30,804] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:09:31,356] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 08:09:58,009] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 08:09:59,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 08:10:00,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 08:10:00,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 08:10:10,984] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 08:10:11,661] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:10:12,184] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 08:10:28,589] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 08:10:29,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 08:10:30,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 08:10:31,726] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 08:10:41,734] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 08:10:42,219] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 08:10:42,694] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 08:10:58,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 08:10:59,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 08:11:00,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 08:11:01,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_25000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 08:11:11,269] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 08:16:19,774] INFO : Step 25100/50000 rank7 | Train Loss: 1.0264 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1288 | Classification: 0.0850 | Classification_binary: 0.1497 | HitGrouping: 0.1664 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1378 | Regression_eta: 0.0016 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0465 | Classification_binary: 0.1408 | Regression_cos_phi: 0.0658 | Regression_energy: 0.0638 | Regression_eta: 0.0315 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0596 | LR: 5.47e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.5428s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0059 | max_forward: 0.7878 | max_loss: 0.1437 | max_backward: 2.2383 | max_optimizer_step: 0.2415 | max_tensorboard_logging: 0.9692 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0559 | Memory: rss_MB: 7903.8 | uss_MB: 7286.7 | child_rss_MB: 5042.9 | allocated_MB: 446.8 | reserved_MB: 24062.0 | inactive_split_MB: 1159.3 +rank7 [2026-10-29 08:21:20,699] INFO : Step 25200/50000 rank7 | Train Loss: 1.0775 | Train Loss Unweighted: 0.5434 | Train Loss Components: Auxiliary: 0.2221 | Cardinality: 0.1397 | Classification: 0.0996 | Classification_binary: 0.1706 | HitGrouping: 0.1584 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1340 | Regression_eta: 0.0016 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0483 | Classification_binary: 0.1384 | Regression_cos_phi: 0.0578 | Regression_energy: 0.0761 | Regression_eta: 0.0335 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0522 | LR: 5.44e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.2313s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8831 | max_loss: 0.1456 | max_backward: 2.1320 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 1.0888 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0568 | Memory: rss_MB: 7903.3 | uss_MB: 7287.2 | child_rss_MB: 5042.2 | allocated_MB: 448.0 | reserved_MB: 26042.0 | inactive_split_MB: 1093.3 +rank7 [2026-10-29 08:26:17,211] INFO : Step 25300/50000 rank7 | Train Loss: 1.0761 | Train Loss Unweighted: 0.5109 | Train Loss Components: Auxiliary: 0.2132 | Cardinality: 0.1361 | Classification: 0.0961 | Classification_binary: 0.1821 | HitGrouping: 0.1562 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1138 | Regression_eta: 0.0018 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0691 | Classification_binary: 0.1848 | Regression_cos_phi: 0.0910 | Regression_energy: 0.0612 | Regression_eta: 0.0513 | Regression_pt: 0.0663 | Regression_sin_phi: 0.0556 | LR: 5.42e-04 | DataLoad Time: 0.0191s | Model Forward Time: 6.1305s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.9122 | max_loss: 0.1821 | max_backward: 2.1499 | max_optimizer_step: 0.3106 | max_tensorboard_logging: 0.9506 | max_tensorboard_flush: 0.0076 | max_post_step_barrier: 0.0472 | Memory: rss_MB: 6897.2 | uss_MB: 5222.8 | child_rss_MB: 5036.8 | allocated_MB: 446.7 | reserved_MB: 28062.0 | inactive_split_MB: 1098.3 +rank7 [2026-10-29 08:31:10,694] INFO : Step 25400/50000 rank7 | Train Loss: 1.0222 | Train Loss Unweighted: 0.5069 | Train Loss Components: Auxiliary: 0.2039 | Cardinality: 0.1238 | Classification: 0.0928 | Classification_binary: 0.1653 | HitGrouping: 0.1606 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1221 | Regression_eta: 0.0015 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0574 | Classification_binary: 0.1886 | Regression_cos_phi: 0.0697 | Regression_energy: 0.0655 | Regression_eta: 0.0387 | Regression_pt: 0.0709 | Regression_sin_phi: 0.1068 | LR: 5.39e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7086s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1954 | max_backward: 1.8415 | max_optimizer_step: 0.3277 | max_tensorboard_logging: 0.7733 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6896.2 | uss_MB: 6278.8 | child_rss_MB: 5052.2 | allocated_MB: 449.3 | reserved_MB: 28062.0 | inactive_split_MB: 945.6 +rank7 [2026-10-29 08:36:13,578] INFO : Step 25500/50000 rank7 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4596 | Train Loss Components: Auxiliary: 0.1817 | Cardinality: 0.1001 | Classification: 0.0953 | Classification_binary: 0.1334 | HitGrouping: 0.1531 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0016 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0434 | Classification_binary: 0.1177 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0537 | Regression_eta: 0.0338 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0535 | LR: 5.36e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7848s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0064 | max_forward: 0.7884 | max_loss: 0.1475 | max_backward: 2.0272 | max_optimizer_step: 0.2266 | max_tensorboard_logging: 0.7862 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0483 | Memory: rss_MB: 6898.2 | uss_MB: 6283.2 | child_rss_MB: 5050.7 | allocated_MB: 450.7 | reserved_MB: 28062.0 | inactive_split_MB: 1166.9 +rank7 [2026-10-29 08:41:08,950] INFO : Step 25600/50000 rank7 | Train Loss: 1.0724 | Train Loss Unweighted: 0.5330 | Train Loss Components: Auxiliary: 0.2209 | Cardinality: 0.1190 | Classification: 0.0985 | Classification_binary: 0.1843 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1231 | Regression_eta: 0.0016 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1635 | Regression_cos_phi: 0.0775 | Regression_energy: 0.0649 | Regression_eta: 0.0426 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0878 | LR: 5.33e-04 | DataLoad Time: 0.0154s | Model Forward Time: 6.1554s | Phase Times: max_pre_step_barrier: 0.0194 | max_batch_to_device: 0.0083 | max_forward: 1.0622 | max_loss: 0.3311 | max_backward: 2.5999 | max_optimizer_step: 0.3019 | max_tensorboard_logging: 0.8123 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6895.2 | uss_MB: 5585.2 | child_rss_MB: 5023.7 | allocated_MB: 450.4 | reserved_MB: 28064.0 | inactive_split_MB: 1163.5 +rank7 [2026-10-29 08:46:09,676] INFO : Step 25700/50000 rank7 | Train Loss: 1.0595 | Train Loss Unweighted: 0.5120 | Train Loss Components: Auxiliary: 0.2157 | Cardinality: 0.1394 | Classification: 0.0920 | Classification_binary: 0.1853 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1153 | Regression_eta: 0.0015 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.2215 | Regression_cos_phi: 0.0659 | Regression_energy: 0.0591 | Regression_eta: 0.0447 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0743 | LR: 5.30e-04 | DataLoad Time: 0.0203s | Model Forward Time: 5.8071s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0067 | max_forward: 1.8320 | max_loss: 0.2520 | max_backward: 2.6962 | max_optimizer_step: 0.2699 | max_tensorboard_logging: 0.7057 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0406 | Memory: rss_MB: 6885.0 | uss_MB: 6272.6 | child_rss_MB: 5051.6 | allocated_MB: 448.6 | reserved_MB: 29942.0 | inactive_split_MB: 1165.2 +rank7 [2026-10-29 08:51:40,159] INFO : Step 25800/50000 rank7 | Train Loss: 0.9592 | Train Loss Unweighted: 0.4733 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1038 | Classification: 0.0787 | Classification_binary: 0.1531 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1182 | Regression_eta: 0.0016 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0387 | Classification_binary: 0.1827 | Regression_cos_phi: 0.0690 | Regression_energy: 0.0764 | Regression_eta: 0.0390 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0557 | LR: 5.27e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.4832s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0066 | max_forward: 0.7549 | max_loss: 0.1480 | max_backward: 1.5296 | max_optimizer_step: 0.2198 | max_tensorboard_logging: 0.8796 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.1361 | Memory: rss_MB: 6889.1 | uss_MB: 6277.5 | child_rss_MB: 5023.6 | allocated_MB: 445.9 | reserved_MB: 30100.0 | inactive_split_MB: 961.8 +rank7 [2026-10-29 08:56:50,110] INFO : Step 25900/50000 rank7 | Train Loss: 0.9846 | Train Loss Unweighted: 0.4613 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1142 | Classification: 0.0768 | Classification_binary: 0.1501 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1145 | Regression_eta: 0.0015 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0289 | Classification_binary: 0.1170 | Regression_cos_phi: 0.0345 | Regression_energy: 0.0489 | Regression_eta: 0.0207 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0533 | LR: 5.25e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.8322s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0124 | max_forward: 0.7901 | max_loss: 0.1430 | max_backward: 1.9362 | max_optimizer_step: 0.2253 | max_tensorboard_logging: 0.8933 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0467 | Memory: rss_MB: 6880.5 | uss_MB: 6281.6 | child_rss_MB: 5008.5 | allocated_MB: 447.7 | reserved_MB: 30100.0 | inactive_split_MB: 1167.6 +rank7 [2026-10-29 09:01:47,514] INFO : Step 26000/50000 rank7 | Train Loss: 1.0575 | Train Loss Unweighted: 0.5169 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1347 | Classification: 0.0896 | Classification_binary: 0.1938 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0015 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.2016 | Regression_cos_phi: 0.0728 | Regression_energy: 0.0586 | Regression_eta: 0.0434 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0656 | LR: 5.22e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.4841s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.8125 | max_loss: 0.2612 | max_backward: 1.8771 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.8972 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0432 | Memory: rss_MB: 6845.2 | uss_MB: 5700.0 | child_rss_MB: 4882.0 | allocated_MB: 450.3 | reserved_MB: 30108.0 | inactive_split_MB: 1161.3 +rank7 [2026-10-29 09:01:48,415] INFO : Running validation on rank7 +rank7 [2026-10-29 09:01:48,415] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 09:01:48,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 09:02:42,436] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:02:42,920] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 09:03:09,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 09:03:10,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 09:03:11,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 09:03:11,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 09:03:22,007] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 09:03:22,719] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:03:23,246] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 09:03:39,435] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 09:03:40,472] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 09:03:41,419] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 09:03:42,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 09:03:52,366] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 09:03:52,918] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:03:53,429] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 09:04:09,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 09:04:10,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 09:04:10,889] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 09:04:11,738] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_26000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 09:04:21,748] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 09:09:37,602] INFO : Step 26100/50000 rank7 | Train Loss: 0.9519 | Train Loss Unweighted: 0.4738 | Train Loss Components: Auxiliary: 0.1914 | Cardinality: 0.1091 | Classification: 0.0871 | Classification_binary: 0.1547 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1147 | Regression_eta: 0.0013 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0542 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0437 | Regression_energy: 0.0547 | Regression_eta: 0.0366 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0553 | LR: 5.19e-04 | DataLoad Time: 0.0170s | Model Forward Time: 5.2089s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0321 | max_forward: 0.8143 | max_loss: 0.1540 | max_backward: 1.7669 | max_optimizer_step: 0.2382 | max_tensorboard_logging: 1.1185 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0611 | Memory: rss_MB: 8060.0 | uss_MB: 7433.5 | child_rss_MB: 4850.5 | allocated_MB: 445.1 | reserved_MB: 25850.0 | inactive_split_MB: 1376.9 +rank7 [2026-10-29 09:14:36,072] INFO : Step 26200/50000 rank7 | Train Loss: 0.9758 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.0970 | Classification: 0.0858 | Classification_binary: 0.1501 | HitGrouping: 0.1665 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1278 | Regression_eta: 0.0017 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1661 | Regression_cos_phi: 0.0656 | Regression_energy: 0.0598 | Regression_eta: 0.0594 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0522 | LR: 5.16e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6740s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0066 | max_forward: 0.7728 | max_loss: 0.1384 | max_backward: 1.8195 | max_optimizer_step: 0.3036 | max_tensorboard_logging: 0.7010 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0602 | Memory: rss_MB: 8060.1 | uss_MB: 7488.7 | child_rss_MB: 4848.5 | allocated_MB: 447.4 | reserved_MB: 27830.0 | inactive_split_MB: 1117.7 +rank7 [2026-10-29 09:19:51,626] INFO : Step 26300/50000 rank7 | Train Loss: 1.0568 | Train Loss Unweighted: 0.5138 | Train Loss Components: Auxiliary: 0.2146 | Cardinality: 0.1416 | Classification: 0.0935 | Classification_binary: 0.1772 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1194 | Regression_eta: 0.0015 | Regression_pt: 0.1205 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0626 | Classification_binary: 0.1848 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0778 | Regression_eta: 0.0405 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0816 | LR: 5.13e-04 | DataLoad Time: 0.0200s | Model Forward Time: 6.1508s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0072 | max_forward: 0.8205 | max_loss: 0.1808 | max_backward: 2.8872 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 1.1024 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6927.6 | uss_MB: 6354.4 | child_rss_MB: 4842.0 | allocated_MB: 450.8 | reserved_MB: 29830.0 | inactive_split_MB: 1114.5 +rank7 [2026-10-29 09:24:55,630] INFO : Step 26400/50000 rank7 | Train Loss: 1.0536 | Train Loss Unweighted: 0.5239 | Train Loss Components: Auxiliary: 0.2125 | Cardinality: 0.1323 | Classification: 0.0937 | Classification_binary: 0.1845 | HitGrouping: 0.1559 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1209 | Regression_eta: 0.0015 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0578 | Classification_binary: 0.1753 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0637 | Regression_eta: 0.0406 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0541 | LR: 5.10e-04 | DataLoad Time: 0.0159s | Model Forward Time: 5.5431s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.8639 | max_loss: 0.1569 | max_backward: 2.4710 | max_optimizer_step: 0.2228 | max_tensorboard_logging: 1.0049 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0452 | Memory: rss_MB: 6924.7 | uss_MB: 6353.2 | child_rss_MB: 4853.6 | allocated_MB: 447.6 | reserved_MB: 29832.0 | inactive_split_MB: 1186.4 +rank7 [2026-10-29 09:29:59,514] INFO : Step 26500/50000 rank7 | Train Loss: 0.9812 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.1877 | Cardinality: 0.1299 | Classification: 0.0807 | Classification_binary: 0.1400 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1217 | Regression_eta: 0.0017 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0511 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0685 | Regression_eta: 0.0501 | Regression_pt: 0.0742 | Regression_sin_phi: 0.0831 | LR: 5.08e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.8063s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0061 | max_forward: 0.7436 | max_loss: 0.1579 | max_backward: 1.8417 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.9265 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0477 | Memory: rss_MB: 6921.4 | uss_MB: 6354.1 | child_rss_MB: 4851.7 | allocated_MB: 444.4 | reserved_MB: 29832.0 | inactive_split_MB: 1229.5 +rank7 [2026-10-29 09:34:58,506] INFO : Step 26600/50000 rank7 | Train Loss: 1.0108 | Train Loss Unweighted: 0.5066 | Train Loss Components: Auxiliary: 0.2023 | Cardinality: 0.1182 | Classification: 0.0957 | Classification_binary: 0.1588 | HitGrouping: 0.1547 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1242 | Regression_eta: 0.0014 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0523 | Classification_binary: 0.1520 | Regression_cos_phi: 0.0752 | Regression_energy: 0.0641 | Regression_eta: 0.0455 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0691 | LR: 5.05e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.6973s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0062 | max_forward: 0.7561 | max_loss: 0.1430 | max_backward: 1.8427 | max_optimizer_step: 0.2304 | max_tensorboard_logging: 0.8696 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0471 | Memory: rss_MB: 6921.4 | uss_MB: 6356.0 | child_rss_MB: 4827.0 | allocated_MB: 447.9 | reserved_MB: 29832.0 | inactive_split_MB: 1037.2 +rank7 [2026-10-29 09:40:07,140] INFO : Step 26700/50000 rank7 | Train Loss: 1.0878 | Train Loss Unweighted: 0.5266 | Train Loss Components: Auxiliary: 0.2159 | Cardinality: 0.1269 | Classification: 0.0956 | Classification_binary: 0.1961 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1144 | Regression_eta: 0.0018 | Regression_pt: 0.1168 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0609 | Classification_binary: 0.1806 | Regression_cos_phi: 0.1319 | Regression_energy: 0.0529 | Regression_eta: 0.0758 | Regression_pt: 0.0582 | Regression_sin_phi: 0.1081 | LR: 5.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.1895s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0069 | max_forward: 0.9788 | max_loss: 0.1540 | max_backward: 1.9407 | max_optimizer_step: 0.2295 | max_tensorboard_logging: 1.0076 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6911.8 | uss_MB: 6344.9 | child_rss_MB: 4859.9 | allocated_MB: 450.1 | reserved_MB: 29832.0 | inactive_split_MB: 1190.4 +rank7 [2026-10-29 09:48:41,511] INFO : Step 26800/50000 rank7 | Train Loss: 1.0077 | Train Loss Unweighted: 0.5111 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1102 | Classification: 0.0787 | Classification_binary: 0.1553 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1359 | Regression_eta: 0.0015 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0695 | Regression_energy: 0.0569 | Regression_eta: 0.0441 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0771 | LR: 4.99e-04 | DataLoad Time: 0.0189s | Model Forward Time: 4.8152s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0063 | max_forward: 0.7774 | max_loss: 0.2523 | max_backward: 1.8741 | max_optimizer_step: 0.2873 | max_tensorboard_logging: 0.8206 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6909.2 | uss_MB: 6351.0 | child_rss_MB: 4816.3 | allocated_MB: 444.7 | reserved_MB: 29832.0 | inactive_split_MB: 1121.3 +rank7 [2026-10-29 09:53:40,046] INFO : Step 26900/50000 rank7 | Train Loss: 0.9524 | Train Loss Unweighted: 0.4793 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1099 | Classification: 0.0833 | Classification_binary: 0.1305 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1313 | Regression_eta: 0.0016 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0413 | Classification_binary: 0.1478 | Regression_cos_phi: 0.0588 | Regression_energy: 0.0760 | Regression_eta: 0.0738 | Regression_pt: 0.0823 | Regression_sin_phi: 0.0548 | LR: 4.96e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.4192s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0064 | max_forward: 0.7292 | max_loss: 0.1373 | max_backward: 1.7442 | max_optimizer_step: 0.3152 | max_tensorboard_logging: 0.6764 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0476 | Memory: rss_MB: 6905.2 | uss_MB: 5520.6 | child_rss_MB: 4788.1 | allocated_MB: 446.9 | reserved_MB: 29832.0 | inactive_split_MB: 1016.7 +rank7 [2026-10-29 09:58:37,161] INFO : Step 27000/50000 rank7 | Train Loss: 1.0285 | Train Loss Unweighted: 0.4948 | Train Loss Components: Auxiliary: 0.2018 | Cardinality: 0.1243 | Classification: 0.0862 | Classification_binary: 0.1888 | HitGrouping: 0.1539 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1075 | Regression_eta: 0.0016 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1748 | Regression_cos_phi: 0.0706 | Regression_energy: 0.0515 | Regression_eta: 0.0711 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0701 | LR: 4.94e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.1597s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0061 | max_forward: 0.9134 | max_loss: 0.1497 | max_backward: 2.1803 | max_optimizer_step: 0.2254 | max_tensorboard_logging: 0.9490 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0470 | Memory: rss_MB: 6904.1 | uss_MB: 6354.5 | child_rss_MB: 4772.3 | allocated_MB: 449.7 | reserved_MB: 29832.0 | inactive_split_MB: 1024.3 +rank7 [2026-10-29 09:58:38,108] INFO : Running validation on rank7 +rank7 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 09:58:38,108] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 09:59:32,783] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 09:59:33,287] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 10:00:00,101] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 10:00:01,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 10:00:02,173] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 10:00:03,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 10:00:13,147] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 10:00:14,009] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:00:14,537] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 10:00:31,162] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 10:00:33,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 10:00:34,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 10:00:35,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 10:00:45,234] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 10:00:46,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:00:46,864] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 10:01:02,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 10:01:03,110] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 10:01:03,916] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 10:01:04,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_27000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 10:01:14,778] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 10:06:37,478] INFO : Step 27100/50000 rank7 | Train Loss: 1.0614 | Train Loss Unweighted: 0.4989 | Train Loss Components: Auxiliary: 0.2148 | Cardinality: 0.1449 | Classification: 0.0752 | Classification_binary: 0.1877 | HitGrouping: 0.1621 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1155 | Regression_eta: 0.0015 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0577 | Regression_eta: 0.0363 | Regression_pt: 0.0632 | Regression_sin_phi: 0.0477 | LR: 4.91e-04 | DataLoad Time: 0.0198s | Model Forward Time: 7.1184s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0966 | max_forward: 2.1724 | max_loss: 0.6705 | max_backward: 3.1723 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 1.5888 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0630 | Memory: rss_MB: 8047.0 | uss_MB: 7496.7 | child_rss_MB: 4778.7 | allocated_MB: 447.0 | reserved_MB: 27292.0 | inactive_split_MB: 983.4 +rank7 [2026-10-29 10:11:19,567] INFO : Step 27200/50000 rank7 | Train Loss: 0.9045 | Train Loss Unweighted: 0.4389 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1028 | Classification: 0.0770 | Classification_binary: 0.1287 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1139 | Regression_eta: 0.0015 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0314 | Classification_binary: 0.1152 | Regression_cos_phi: 0.0561 | Regression_energy: 0.0481 | Regression_eta: 0.0322 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0544 | LR: 4.88e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.7902s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0065 | max_forward: 0.7725 | max_loss: 0.5032 | max_backward: 1.6788 | max_optimizer_step: 0.2815 | max_tensorboard_logging: 0.9686 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0675 | Memory: rss_MB: 8047.0 | uss_MB: 7493.7 | child_rss_MB: 4786.7 | allocated_MB: 445.7 | reserved_MB: 29236.0 | inactive_split_MB: 1232.8 +rank7 [2026-10-29 10:15:58,638] INFO : Step 27300/50000 rank7 | Train Loss: 1.0726 | Train Loss Unweighted: 0.5095 | Train Loss Components: Auxiliary: 0.2138 | Cardinality: 0.1337 | Classification: 0.1081 | Classification_binary: 0.1640 | HitGrouping: 0.1596 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1159 | Regression_eta: 0.0019 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0665 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0613 | Regression_energy: 0.0609 | Regression_eta: 0.0502 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0753 | LR: 4.85e-04 | DataLoad Time: 0.0153s | Model Forward Time: 5.1441s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0065 | max_forward: 0.8896 | max_loss: 0.1460 | max_backward: 2.0153 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.9918 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0511 | Memory: rss_MB: 6974.0 | uss_MB: 6424.6 | child_rss_MB: 4784.6 | allocated_MB: 450.9 | reserved_MB: 29330.0 | inactive_split_MB: 1103.1 +rank7 [2026-10-29 10:20:31,815] INFO : Step 27400/50000 rank7 | Train Loss: 1.0758 | Train Loss Unweighted: 0.5282 | Train Loss Components: Auxiliary: 0.2205 | Cardinality: 0.1344 | Classification: 0.0930 | Classification_binary: 0.1899 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1207 | Regression_eta: 0.0015 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0643 | Classification_binary: 0.2078 | Regression_cos_phi: 0.0660 | Regression_energy: 0.0677 | Regression_eta: 0.0440 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0881 | LR: 4.82e-04 | DataLoad Time: 0.0204s | Model Forward Time: 5.0212s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0065 | max_forward: 0.8446 | max_loss: 0.1488 | max_backward: 2.0519 | max_optimizer_step: 0.2277 | max_tensorboard_logging: 0.8725 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0369 | Memory: rss_MB: 6977.0 | uss_MB: 5627.2 | child_rss_MB: 4777.8 | allocated_MB: 448.4 | reserved_MB: 31310.0 | inactive_split_MB: 895.7 +rank7 [2026-10-29 10:25:07,332] INFO : Step 27500/50000 rank7 | Train Loss: 0.9697 | Train Loss Unweighted: 0.4574 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1111 | Classification: 0.0831 | Classification_binary: 0.1473 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1114 | Regression_eta: 0.0018 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1512 | Regression_cos_phi: 0.0632 | Regression_energy: 0.0521 | Regression_eta: 0.0331 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0792 | LR: 4.80e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.4604s | Phase Times: max_pre_step_barrier: 0.0032 | max_batch_to_device: 0.0065 | max_forward: 0.7904 | max_loss: 0.1563 | max_backward: 1.6291 | max_optimizer_step: 0.2352 | max_tensorboard_logging: 0.7457 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0245 | Memory: rss_MB: 6966.1 | uss_MB: 4710.9 | child_rss_MB: 4769.8 | allocated_MB: 447.3 | reserved_MB: 31310.0 | inactive_split_MB: 986.8 +rank7 [2026-10-29 10:30:03,960] INFO : Step 27600/50000 rank7 | Train Loss: 0.9616 | Train Loss Unweighted: 0.4618 | Train Loss Components: Auxiliary: 0.1872 | Cardinality: 0.1092 | Classification: 0.0865 | Classification_binary: 0.1284 | HitGrouping: 0.1590 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0019 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0543 | Classification_binary: 0.1217 | Regression_cos_phi: 0.0626 | Regression_energy: 0.0556 | Regression_eta: 0.0287 | Regression_pt: 0.0599 | Regression_sin_phi: 0.0631 | LR: 4.77e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.5702s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0380 | max_forward: 0.7674 | max_loss: 0.1463 | max_backward: 1.9164 | max_optimizer_step: 0.2322 | max_tensorboard_logging: 0.5145 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0491 | Memory: rss_MB: 6974.1 | uss_MB: 6422.9 | child_rss_MB: 4778.4 | allocated_MB: 446.7 | reserved_MB: 31310.0 | inactive_split_MB: 1182.0 +rank7 [2026-10-29 10:34:46,993] INFO : Step 27700/50000 rank7 | Train Loss: 1.0693 | Train Loss Unweighted: 0.5190 | Train Loss Components: Auxiliary: 0.2200 | Cardinality: 0.1414 | Classification: 0.0939 | Classification_binary: 0.1783 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1214 | Regression_eta: 0.0015 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1744 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0654 | Regression_eta: 0.0376 | Regression_pt: 0.0713 | Regression_sin_phi: 0.0549 | LR: 4.74e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.6745s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.8151 | max_loss: 0.1750 | max_backward: 2.0398 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.5213 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0659 | Memory: rss_MB: 6974.1 | uss_MB: 6424.8 | child_rss_MB: 4756.0 | allocated_MB: 448.4 | reserved_MB: 33312.0 | inactive_split_MB: 986.3 +rank7 [2026-10-29 10:39:29,303] INFO : Step 27800/50000 rank7 | Train Loss: 1.0875 | Train Loss Unweighted: 0.5294 | Train Loss Components: Auxiliary: 0.2193 | Cardinality: 0.1226 | Classification: 0.0924 | Classification_binary: 0.1842 | HitGrouping: 0.1594 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1237 | Regression_eta: 0.0019 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0449 | Classification_binary: 0.1955 | Regression_cos_phi: 0.0580 | Regression_energy: 0.0631 | Regression_eta: 0.0499 | Regression_pt: 0.0688 | Regression_sin_phi: 0.1206 | LR: 4.71e-04 | DataLoad Time: 0.0194s | Model Forward Time: 4.8105s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0152 | max_forward: 0.8463 | max_loss: 0.1744 | max_backward: 1.7379 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.9397 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0600 | Memory: rss_MB: 6971.2 | uss_MB: 6421.2 | child_rss_MB: 4773.7 | allocated_MB: 452.2 | reserved_MB: 33312.0 | inactive_split_MB: 1320.1 +rank7 [2026-10-29 10:44:11,938] INFO : Step 27900/50000 rank7 | Train Loss: 0.9483 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0988 | Classification: 0.0859 | Classification_binary: 0.1439 | HitGrouping: 0.1651 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1244 | Regression_eta: 0.0014 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0569 | Classification_binary: 0.1496 | Regression_cos_phi: 0.0600 | Regression_energy: 0.0725 | Regression_eta: 0.0336 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0410 | LR: 4.68e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.4749s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0063 | max_forward: 0.7626 | max_loss: 0.1370 | max_backward: 1.8424 | max_optimizer_step: 0.2303 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0625 | Memory: rss_MB: 6972.2 | uss_MB: 6423.3 | child_rss_MB: 4773.1 | allocated_MB: 447.1 | reserved_MB: 33312.0 | inactive_split_MB: 1112.6 +rank7 [2026-10-29 10:48:55,971] INFO : Step 28000/50000 rank7 | Train Loss: 0.9840 | Train Loss Unweighted: 0.4736 | Train Loss Components: Auxiliary: 0.1949 | Cardinality: 0.1114 | Classification: 0.0927 | Classification_binary: 0.1564 | HitGrouping: 0.1628 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1099 | Regression_eta: 0.0015 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0812 | Classification_binary: 0.1592 | Regression_cos_phi: 0.0683 | Regression_energy: 0.0658 | Regression_eta: 0.0384 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0838 | LR: 4.66e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.9423s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0070 | max_forward: 0.8192 | max_loss: 0.2552 | max_backward: 2.1299 | max_optimizer_step: 0.2920 | max_tensorboard_logging: 0.7069 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 6969.4 | uss_MB: 6424.2 | child_rss_MB: 4776.8 | allocated_MB: 448.2 | reserved_MB: 33312.0 | inactive_split_MB: 1178.0 +rank7 [2026-10-29 10:48:56,836] INFO : Running validation on rank7 +rank7 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 10:48:56,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 10:49:51,447] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:49:52,005] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 10:50:18,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 10:50:19,519] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 10:50:20,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 10:50:21,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 10:50:31,446] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 10:50:32,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:50:32,999] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 10:50:49,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 10:50:50,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 10:50:51,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 10:50:51,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 10:51:01,972] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 10:51:02,906] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 10:51:03,445] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 10:51:18,947] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 10:51:19,816] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 10:51:20,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 10:51:21,478] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_28000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 10:51:31,489] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 10:56:32,745] INFO : Step 28100/50000 rank7 | Train Loss: 1.0915 | Train Loss Unweighted: 0.5085 | Train Loss Components: Auxiliary: 0.2225 | Cardinality: 0.1649 | Classification: 0.0833 | Classification_binary: 0.1834 | HitGrouping: 0.1567 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1187 | Regression_eta: 0.0017 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0531 | Classification_binary: 0.1908 | Regression_cos_phi: 0.0689 | Regression_energy: 0.0596 | Regression_eta: 0.0404 | Regression_pt: 0.0655 | Regression_sin_phi: 0.0664 | LR: 4.63e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.8666s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8571 | max_loss: 0.1603 | max_backward: 1.9112 | max_optimizer_step: 0.2510 | max_tensorboard_logging: 0.7670 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0838 | Memory: rss_MB: 8288.3 | uss_MB: 7733.7 | child_rss_MB: 4773.7 | allocated_MB: 448.9 | reserved_MB: 25946.0 | inactive_split_MB: 1313.1 +rank7 [2026-10-29 11:01:18,173] INFO : Step 28200/50000 rank7 | Train Loss: 0.9241 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1144 | Classification: 0.0770 | Classification_binary: 0.1441 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1163 | Regression_eta: 0.0014 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0499 | Classification_binary: 0.1535 | Regression_cos_phi: 0.0544 | Regression_energy: 0.0620 | Regression_eta: 0.0465 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0487 | LR: 4.60e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.8980s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0751 | max_forward: 0.8691 | max_loss: 0.5236 | max_backward: 1.8483 | max_optimizer_step: 0.2480 | max_tensorboard_logging: 0.8546 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0640 | Memory: rss_MB: 8289.0 | uss_MB: 7733.3 | child_rss_MB: 4770.2 | allocated_MB: 445.0 | reserved_MB: 25948.0 | inactive_split_MB: 976.4 +rank7 [2026-10-29 11:07:40,703] INFO : Step 28300/50000 rank7 | Train Loss: 0.9513 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1028 | Classification: 0.0882 | Classification_binary: 0.1368 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1147 | Regression_eta: 0.0016 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1225 | Regression_cos_phi: 0.0787 | Regression_energy: 0.0534 | Regression_eta: 0.0753 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0745 | LR: 4.57e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.9464s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0066 | max_forward: 0.7916 | max_loss: 0.1449 | max_backward: 2.0850 | max_optimizer_step: 0.2328 | max_tensorboard_logging: 0.8583 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0453 | Memory: rss_MB: 7055.4 | uss_MB: 6496.0 | child_rss_MB: 4774.9 | allocated_MB: 447.2 | reserved_MB: 25952.0 | inactive_split_MB: 1165.3 +rank7 [2026-10-29 11:12:27,615] INFO : Step 28400/50000 rank7 | Train Loss: 1.0604 | Train Loss Unweighted: 0.4982 | Train Loss Components: Auxiliary: 0.2104 | Cardinality: 0.1382 | Classification: 0.0847 | Classification_binary: 0.1862 | HitGrouping: 0.1552 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1111 | Regression_eta: 0.0018 | Regression_pt: 0.1126 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1765 | Regression_cos_phi: 0.0691 | Regression_energy: 0.0544 | Regression_eta: 0.0673 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0767 | LR: 4.55e-04 | DataLoad Time: 0.0215s | Model Forward Time: 5.0302s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0070 | max_forward: 0.9139 | max_loss: 0.1474 | max_backward: 2.1323 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.6846 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0517 | Memory: rss_MB: 7055.5 | uss_MB: 6496.2 | child_rss_MB: 4770.0 | allocated_MB: 448.2 | reserved_MB: 26968.0 | inactive_split_MB: 835.9 +rank7 [2026-10-29 11:17:11,504] INFO : Step 28500/50000 rank7 | Train Loss: 1.0282 | Train Loss Unweighted: 0.5005 | Train Loss Components: Auxiliary: 0.2122 | Cardinality: 0.1237 | Classification: 0.0911 | Classification_binary: 0.1756 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0013 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0811 | Classification_binary: 0.1908 | Regression_cos_phi: 0.0829 | Regression_energy: 0.0571 | Regression_eta: 0.0426 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0676 | LR: 4.52e-04 | DataLoad Time: 0.0195s | Model Forward Time: 9.4610s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0886 | max_forward: 0.9758 | max_loss: 0.5123 | max_backward: 2.0811 | max_optimizer_step: 0.2797 | max_tensorboard_logging: 1.5644 | max_tensorboard_flush: 0.0047 | max_post_step_barrier: 0.0889 | Memory: rss_MB: 7042.2 | uss_MB: 4349.9 | child_rss_MB: 4774.7 | allocated_MB: 452.0 | reserved_MB: 27968.0 | inactive_split_MB: 923.2 +rank7 [2026-10-29 11:21:56,249] INFO : Step 28600/50000 rank7 | Train Loss: 0.9753 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1202 | Classification: 0.0804 | Classification_binary: 0.1446 | HitGrouping: 0.1648 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1132 | Regression_eta: 0.0017 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.1592 | Regression_cos_phi: 0.0871 | Regression_energy: 0.0566 | Regression_eta: 0.0448 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0532 | LR: 4.49e-04 | DataLoad Time: 0.0227s | Model Forward Time: 4.9425s | Phase Times: max_pre_step_barrier: 0.1127 | max_batch_to_device: 0.0146 | max_forward: 0.7731 | max_loss: 0.2404 | max_backward: 1.6352 | max_optimizer_step: 0.2346 | max_tensorboard_logging: 1.0006 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0479 | Memory: rss_MB: 7048.8 | uss_MB: 5469.7 | child_rss_MB: 4776.7 | allocated_MB: 446.8 | reserved_MB: 30056.0 | inactive_split_MB: 896.4 +rank7 [2026-10-29 11:26:34,428] INFO : Step 28700/50000 rank7 | Train Loss: 1.0558 | Train Loss Unweighted: 0.5173 | Train Loss Components: Auxiliary: 0.2079 | Cardinality: 0.1082 | Classification: 0.1065 | Classification_binary: 0.1612 | HitGrouping: 0.1622 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1217 | Regression_eta: 0.0024 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0600 | Classification_binary: 0.1473 | Regression_cos_phi: 0.0589 | Regression_energy: 0.0679 | Regression_eta: 0.0695 | Regression_pt: 0.0744 | Regression_sin_phi: 0.0589 | LR: 4.46e-04 | DataLoad Time: 0.0216s | Model Forward Time: 4.9563s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0686 | max_forward: 0.8423 | max_loss: 0.1475 | max_backward: 2.0170 | max_optimizer_step: 0.2341 | max_tensorboard_logging: 0.8830 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0486 | Memory: rss_MB: 7053.8 | uss_MB: 6496.4 | child_rss_MB: 4774.2 | allocated_MB: 447.9 | reserved_MB: 30056.0 | inactive_split_MB: 925.2 +rank7 [2026-10-29 11:31:18,243] INFO : Step 28800/50000 rank7 | Train Loss: 1.1020 | Train Loss Unweighted: 0.5167 | Train Loss Components: Auxiliary: 0.2228 | Cardinality: 0.1474 | Classification: 0.0969 | Classification_binary: 0.1939 | HitGrouping: 0.1563 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1105 | Regression_eta: 0.0017 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0654 | Classification_binary: 0.1805 | Regression_cos_phi: 0.0655 | Regression_energy: 0.0563 | Regression_eta: 0.0404 | Regression_pt: 0.0620 | Regression_sin_phi: 0.0833 | LR: 4.44e-04 | DataLoad Time: 0.0207s | Model Forward Time: 5.8745s | Phase Times: max_pre_step_barrier: 0.0041 | max_batch_to_device: 0.0075 | max_forward: 0.8427 | max_loss: 0.2661 | max_backward: 2.8464 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.6103 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0451 | Memory: rss_MB: 7050.1 | uss_MB: 4315.7 | child_rss_MB: 4755.7 | allocated_MB: 448.2 | reserved_MB: 30058.0 | inactive_split_MB: 976.8 +rank7 [2026-10-29 11:35:53,801] INFO : Step 28900/50000 rank7 | Train Loss: 0.9389 | Train Loss Unweighted: 0.4547 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.1078 | Classification: 0.0733 | Classification_binary: 0.1521 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1129 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0504 | Classification_binary: 0.1454 | Regression_cos_phi: 0.0472 | Regression_energy: 0.0516 | Regression_eta: 0.0546 | Regression_pt: 0.0558 | Regression_sin_phi: 0.0726 | LR: 4.41e-04 | DataLoad Time: 0.0206s | Model Forward Time: 4.5676s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0062 | max_forward: 0.7499 | max_loss: 0.2426 | max_backward: 1.8556 | max_optimizer_step: 0.3135 | max_tensorboard_logging: 0.5831 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0508 | Memory: rss_MB: 7048.4 | uss_MB: 6494.0 | child_rss_MB: 4774.7 | allocated_MB: 444.3 | reserved_MB: 30058.0 | inactive_split_MB: 1133.2 +rank7 [2026-10-29 11:40:53,318] INFO : Step 29000/50000 rank7 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4535 | Train Loss Components: Auxiliary: 0.1826 | Cardinality: 0.0939 | Classification: 0.0855 | Classification_binary: 0.1308 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1169 | Regression_eta: 0.0015 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0489 | Classification_binary: 0.1398 | Regression_cos_phi: 0.0441 | Regression_energy: 0.0621 | Regression_eta: 0.0308 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0636 | LR: 4.38e-04 | DataLoad Time: 0.0214s | Model Forward Time: 4.8474s | Phase Times: max_pre_step_barrier: 0.0037 | max_batch_to_device: 0.0062 | max_forward: 0.9270 | max_loss: 0.2073 | max_backward: 2.1896 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6044 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0475 | Memory: rss_MB: 7047.7 | uss_MB: 6496.6 | child_rss_MB: 4759.3 | allocated_MB: 450.2 | reserved_MB: 30060.0 | inactive_split_MB: 1402.9 +rank7 [2026-10-29 11:40:54,336] INFO : Running validation on rank7 +rank7 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 11:40:54,336] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 11:41:46,664] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:41:47,174] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 11:42:16,321] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 11:42:18,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 11:42:19,397] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 11:42:20,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 11:42:30,370] INFO : Time taken to make predictions on device 7 is: 0.29 min +rank7 [2026-10-29 11:42:31,032] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:42:31,846] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 11:42:47,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 11:42:48,827] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 11:42:49,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 11:42:50,760] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 11:43:00,771] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 11:43:00,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 11:43:01,399] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 11:43:16,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 11:43:17,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 11:43:18,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 11:43:19,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_29000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 11:43:29,535] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 11:48:35,107] INFO : Step 29100/50000 rank7 | Train Loss: 1.0559 | Train Loss Unweighted: 0.5040 | Train Loss Components: Auxiliary: 0.2142 | Cardinality: 0.1307 | Classification: 0.0939 | Classification_binary: 0.1851 | HitGrouping: 0.1578 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1098 | Regression_eta: 0.0015 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1628 | Regression_cos_phi: 0.0688 | Regression_energy: 0.0569 | Regression_eta: 0.0345 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0647 | LR: 4.35e-04 | DataLoad Time: 0.0161s | Model Forward Time: 7.6727s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 1.6779 | max_loss: 0.3815 | max_backward: 4.2661 | max_optimizer_step: 0.3470 | max_tensorboard_logging: 0.7414 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0663 | Memory: rss_MB: 8253.0 | uss_MB: 6395.3 | child_rss_MB: 4770.6 | allocated_MB: 448.8 | reserved_MB: 24188.0 | inactive_split_MB: 1162.1 +rank7 [2026-10-29 11:53:34,049] INFO : Step 29200/50000 rank7 | Train Loss: 0.9909 | Train Loss Unweighted: 0.4801 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1204 | Classification: 0.0923 | Classification_binary: 0.1746 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1054 | Regression_eta: 0.0014 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1756 | Regression_cos_phi: 0.0534 | Regression_energy: 0.0462 | Regression_eta: 0.0322 | Regression_pt: 0.0495 | Regression_sin_phi: 0.0722 | LR: 4.33e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.7772s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0069 | max_forward: 0.8354 | max_loss: 0.1558 | max_backward: 2.1201 | max_optimizer_step: 0.2507 | max_tensorboard_logging: 0.5049 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0677 | Memory: rss_MB: 8251.2 | uss_MB: 7692.4 | child_rss_MB: 4763.1 | allocated_MB: 447.9 | reserved_MB: 24190.0 | inactive_split_MB: 1166.5 +rank7 [2026-10-29 11:58:27,982] INFO : Step 29300/50000 rank7 | Train Loss: 0.9337 | Train Loss Unweighted: 0.4462 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1121 | Classification: 0.0773 | Classification_binary: 0.1359 | HitGrouping: 0.1548 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1146 | Regression_eta: 0.0015 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0869 | Regression_energy: 0.0573 | Regression_eta: 0.0483 | Regression_pt: 0.0625 | Regression_sin_phi: 0.0689 | LR: 4.30e-04 | DataLoad Time: 0.0208s | Model Forward Time: 4.9132s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0277 | max_forward: 0.8212 | max_loss: 0.1457 | max_backward: 1.8414 | max_optimizer_step: 0.2550 | max_tensorboard_logging: 0.8797 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 7135.1 | uss_MB: 5997.0 | child_rss_MB: 4761.5 | allocated_MB: 444.9 | reserved_MB: 24190.0 | inactive_split_MB: 1038.6 +rank7 [2026-10-29 12:03:07,469] INFO : Step 29400/50000 rank7 | Train Loss: 1.0186 | Train Loss Unweighted: 0.4675 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1372 | Classification: 0.0948 | Classification_binary: 0.1561 | HitGrouping: 0.1614 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1059 | Regression_eta: 0.0020 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0757 | Classification_binary: 0.1359 | Regression_cos_phi: 0.0631 | Regression_energy: 0.0438 | Regression_eta: 0.0318 | Regression_pt: 0.0480 | Regression_sin_phi: 0.0498 | LR: 4.27e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.8015s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8495 | max_loss: 0.1434 | max_backward: 1.9359 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.5959 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0528 | Memory: rss_MB: 7145.4 | uss_MB: 6588.5 | child_rss_MB: 4762.3 | allocated_MB: 447.9 | reserved_MB: 26208.0 | inactive_split_MB: 957.3 +rank7 [2026-10-29 12:08:16,203] INFO : Step 29500/50000 rank7 | Train Loss: 1.0909 | Train Loss Unweighted: 0.5362 | Train Loss Components: Auxiliary: 0.2174 | Cardinality: 0.1141 | Classification: 0.1071 | Classification_binary: 0.1896 | HitGrouping: 0.1557 | Regression_cos_phi: 0.0010 | Regression_energy: 0.1176 | Regression_eta: 0.0019 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0010 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0835 | Classification_binary: 0.2128 | Regression_cos_phi: 0.1003 | Regression_energy: 0.0700 | Regression_eta: 0.0592 | Regression_pt: 0.0764 | Regression_sin_phi: 0.1139 | LR: 4.24e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.9681s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0876 | max_forward: 0.8925 | max_loss: 0.1527 | max_backward: 1.9621 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.8162 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0504 | Memory: rss_MB: 7137.1 | uss_MB: 6580.0 | child_rss_MB: 4760.9 | allocated_MB: 448.9 | reserved_MB: 26214.0 | inactive_split_MB: 1166.5 +rank7 [2026-10-29 12:13:12,652] INFO : Step 29600/50000 rank7 | Train Loss: 0.9608 | Train Loss Unweighted: 0.4723 | Train Loss Components: Auxiliary: 0.1935 | Cardinality: 0.1093 | Classification: 0.0850 | Classification_binary: 0.1542 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0655 | Classification_binary: 0.1489 | Regression_cos_phi: 0.0501 | Regression_energy: 0.0580 | Regression_eta: 0.0452 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0643 | LR: 4.22e-04 | DataLoad Time: 0.0188s | Model Forward Time: 4.8688s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.7848 | max_loss: 0.1495 | max_backward: 1.8042 | max_optimizer_step: 0.2380 | max_tensorboard_logging: 1.0178 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0510 | Memory: rss_MB: 7115.2 | uss_MB: 6585.9 | child_rss_MB: 4743.4 | allocated_MB: 445.5 | reserved_MB: 26214.0 | inactive_split_MB: 1171.9 +rank7 [2026-10-29 12:18:07,450] INFO : Step 29700/50000 rank7 | Train Loss: 0.9511 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1260 | Classification: 0.0829 | Classification_binary: 0.1357 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1183 | Regression_eta: 0.0014 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0377 | Classification_binary: 0.1293 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0703 | Regression_eta: 0.0279 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0505 | LR: 4.19e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.2345s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7531 | max_loss: 0.2308 | max_backward: 2.2313 | max_optimizer_step: 0.2356 | max_tensorboard_logging: 0.9708 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0550 | Memory: rss_MB: 7117.3 | uss_MB: 6587.7 | child_rss_MB: 4741.9 | allocated_MB: 446.6 | reserved_MB: 26214.0 | inactive_split_MB: 1160.6 +rank7 [2026-10-29 12:23:08,924] INFO : Step 29800/50000 rank7 | Train Loss: 0.9950 | Train Loss Unweighted: 0.4915 | Train Loss Components: Auxiliary: 0.2030 | Cardinality: 0.1104 | Classification: 0.0864 | Classification_binary: 0.1759 | HitGrouping: 0.1558 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1122 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0564 | Classification_binary: 0.1826 | Regression_cos_phi: 0.0737 | Regression_energy: 0.0617 | Regression_eta: 0.0366 | Regression_pt: 0.0654 | Regression_sin_phi: 0.0937 | LR: 4.16e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.4227s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8210 | max_loss: 0.1528 | max_backward: 2.4753 | max_optimizer_step: 0.3576 | max_tensorboard_logging: 0.8689 | max_tensorboard_flush: 0.0038 | max_post_step_barrier: 0.1798 | Memory: rss_MB: 7115.5 | uss_MB: 5536.1 | child_rss_MB: 4741.9 | allocated_MB: 450.3 | reserved_MB: 26214.0 | inactive_split_MB: 938.8 +rank7 [2026-10-29 12:28:10,142] INFO : Step 29900/50000 rank7 | Train Loss: 0.9604 | Train Loss Unweighted: 0.4559 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1243 | Classification: 0.0825 | Classification_binary: 0.1616 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1040 | Regression_eta: 0.0015 | Regression_pt: 0.1049 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1903 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0569 | Regression_eta: 0.0335 | Regression_pt: 0.0634 | Regression_sin_phi: 0.0469 | LR: 4.14e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.9482s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.1142 | max_forward: 0.9271 | max_loss: 0.1531 | max_backward: 1.8698 | max_optimizer_step: 0.2331 | max_tensorboard_logging: 0.8620 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0482 | Memory: rss_MB: 7091.0 | uss_MB: 6437.5 | child_rss_MB: 4745.0 | allocated_MB: 450.0 | reserved_MB: 26214.0 | inactive_split_MB: 1166.0 +rank7 [2026-10-29 12:32:58,888] INFO : Step 30000/50000 rank7 | Train Loss: 0.9450 | Train Loss Unweighted: 0.4656 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.1132 | Classification: 0.0847 | Classification_binary: 0.1394 | HitGrouping: 0.1604 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1187 | Regression_eta: 0.0014 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0739 | Classification_binary: 0.1230 | Regression_cos_phi: 0.0685 | Regression_energy: 0.0662 | Regression_eta: 0.0310 | Regression_pt: 0.0699 | Regression_sin_phi: 0.0516 | LR: 4.11e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.1882s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 1.0327 | max_loss: 0.1419 | max_backward: 1.6503 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.4878 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0513 | Memory: rss_MB: 7039.0 | uss_MB: 6586.5 | child_rss_MB: 4745.4 | allocated_MB: 443.8 | reserved_MB: 26214.0 | inactive_split_MB: 1152.2 +rank7 [2026-10-29 12:47:12,790] INFO : Running validation on rank7 +rank7 [2026-10-29 12:47:12,790] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 12:47:12,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 14:49:00,090] INFO : Initializing model with seed=12345; process seed=12352 +rank7 [2026-10-29 14:49:00,096] INFO : configured dtype=torch.bfloat16 for autocast +rank7 [2026-10-29 14:49:20,929] INFO : Instantiating model +rank7 [2026-10-29 14:49:20,929] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=15 engineered_input_dim=154 +rank7 [2026-10-29 14:49:20,930] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank7 [2026-10-29 14:49:20,948] INFO : Input encoding initialization took 0.02s +rank7 [2026-10-29 14:49:20,948] INFO : nn0 parameters: 171776 +rank7 [2026-10-29 14:49:20,948] INFO : Initializing shared backbone layers, num_convs=6 +rank7 [2026-10-29 14:49:20,948] INFO : Initializing backbone layer 0 +rank7 [2026-10-29 14:49:20,983] INFO : Initializing backbone layer 1 +rank7 [2026-10-29 14:49:20,992] INFO : Initializing backbone layer 2 +rank7 [2026-10-29 14:49:21,000] INFO : Initializing backbone layer 3 +rank7 [2026-10-29 14:49:21,008] INFO : Initializing backbone layer 4 +rank7 [2026-10-29 14:49:21,017] INFO : Initializing backbone layer 5 +rank7 [2026-10-29 14:49:21,025] INFO : Backbone initialization took 0.08s +rank7 [2026-10-29 14:49:21,025] INFO : backbone parameters: 7901376 +rank7 [2026-10-29 14:49:21,025] INFO : Initializing output DNNs +rank7 [2026-10-29 14:49:21,062] INFO : Output DNNs initialization took 0.04s +rank7 [2026-10-29 14:49:21,062] INFO : backbone_mode=BackboneMode.SHARED +rank7 [2026-10-29 14:49:21,062] INFO : jagged_attention enabled=False +rank7 [2026-10-29 14:49:21,062] INFO : flash_attn_varlen enabled=False +rank7 [2026-10-29 14:49:21,062] INFO : task_queries enabled=False +rank7 [2026-10-29 14:49:21,062] INFO : classification_norm parameters: 0 +rank7 [2026-10-29 14:49:21,062] INFO : regression_norm parameters: 0 +rank7 [2026-10-29 14:49:21,062] INFO : classification_query parameters: 0 +rank7 [2026-10-29 14:49:21,062] INFO : regression_query parameters: 0 +rank7 [2026-10-29 14:49:21,062] INFO : classification_readout parameters: 0 +rank7 [2026-10-29 14:49:21,062] INFO : regression_readout parameters: 0 +rank7 [2026-10-29 14:49:21,063] INFO : output_mode=set +rank7 [2026-10-29 14:49:21,063] INFO : set_decoder parameters: 4713812 +rank7 [2026-10-29 14:49:21,063] INFO : nn_binary_particle parameters: 0 +rank7 [2026-10-29 14:49:21,063] INFO : nn_pid parameters: 0 +rank7 [2026-10-29 14:49:21,063] INFO : nn_pt parameters: 0 +rank7 [2026-10-29 14:49:21,063] INFO : nn_eta parameters: 0 +rank7 [2026-10-29 14:49:21,063] INFO : nn_sin_phi parameters: 0 +rank7 [2026-10-29 14:49:21,063] INFO : nn_cos_phi parameters: 0 +rank7 [2026-10-29 14:49:21,063] INFO : nn_energy parameters: 0 +rank7 [2026-10-29 14:49:21,064] INFO : Total MLPF parameters: 12786964 +rank7 [2026-10-29 14:49:21,064] INFO : MLPF __init__ done +rank7 [2026-10-29 14:49:21,064] INFO : Instantiated model +rank7 [2026-10-29 14:49:21,064] INFO : Moving model to device rank=7 +rank7 [2026-10-29 14:49:21,086] INFO : Moved model to device rank=7 +rank7 [2026-10-29 14:49:21,087] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank7 [2026-10-29 14:49:21,561] INFO : Restoring training from step 30001 +rank7 [2026-10-29 14:49:21,569] INFO : Loaded model weights from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/checkpoints/checkpoint-30000.pth +rank7 [2026-10-29 14:49:21,569] INFO : Configured model for SyncBatchNorm rank=7 +rank7 [2026-10-29 14:49:22,135] INFO : Configured model for DistributedDataParallel rank=7 +rank7 [2026-10-29 14:49:22,286] INFO : Modules ... Non-trainable parameters +0 module.nn0.0.weight ... 0 +1 module.nn0.0.bias ... 0 +2 module.nn0.2.weight ... 0 +3 module.nn0.2.bias ... 0 +4 module.nn0.4.weight ... 0 +.. ... ... ... +272 module.set_decoder.tracker_momentum_head.embed... ... 0 +273 module.set_decoder.tracker_momentum_head.hidde... ... 0 +274 module.set_decoder.tracker_momentum_head.hidde... ... 0 +275 module.set_decoder.tracker_momentum_head.outpu... ... 0 +276 module.set_decoder.tracker_momentum_head.outpu... ... 0 + +[277 rows x 3 columns] +rank7 [2026-10-29 14:49:22,287] INFO : DistributedDataParallel( + (module): MLPF( + (feature_engineering): HitFeatureEngineering( + (tracker_neighborhood): TrackerNeighborhoodFeatures() + (tracker_doublets): TrackerDoubletFeatures() + (calorimeter_neighborhood): CalorimeterNeighborhoodFeatures() + ) + (nn0): Sequential( + (0): Linear(in_features=154, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (set_decoder): ParticleSetDecoder( + (layers): ModuleList( + (0-3): 4 x ParticleSetDecoderLayer( + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (memory_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (cross_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (self_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (self_attention): MultiheadAttention( + (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True) + ) + (ffn_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=1024, bias=True) + (1): GELU(approximate='none') + (2): Dropout(p=0.0, inplace=False) + (3): Linear(in_features=1024, out_features=256, bias=True) + (4): Dropout(p=0.0, inplace=False) + ) + ) + ) + (output_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hit_grouping_hit): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_slot): Linear(in_features=256, out_features=32, bias=True) + (hit_grouping_background): Linear(in_features=256, out_features=1, bias=True) + (hit_grouping_projection): Linear(in_features=515, out_features=256, bias=True) + (presence_head): Linear(in_features=256, out_features=2, bias=True) + (pid_head): Linear(in_features=256, out_features=6, bias=True) + (seed_projection): Linear(in_features=256, out_features=256, bias=True) + (reference_embedding): Sequential( + (0): Linear(in_features=3, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (reference_delta_heads): ModuleList( + (0-3): 4 x Linear(in_features=256, out_features=2, bias=True) + ) + (scale_head): Linear(in_features=256, out_features=2, bias=True) + (aggregate_projection): Sequential( + (0): Linear(in_features=8, out_features=256, bias=True) + (1): GELU(approximate='none') + (2): Linear(in_features=256, out_features=256, bias=True) + ) + (tracker_momentum_head): TrackerMomentumReadout( + (embedding_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (hidden): Linear(in_features=300, out_features=256, bias=True) + (output): Linear(in_features=256, out_features=1, bias=True) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() + ) +) +rank7 [2026-10-29 14:49:22,288] INFO : Trainable parameters: 12786964 +rank7 [2026-10-29 14:49:22,288] INFO : Non-trainable parameters: 23268 +rank7 [2026-10-29 14:49:22,288] INFO : Total parameters: 12810232 +rank7 [2026-10-29 14:49:22,291] INFO : Modules Trainable parameters Non-trainable parameters + module.nn0.0.weight 39424 0 + module.nn0.0.bias 256 0 + module.nn0.2.weight 256 0 + module.nn0.2.bias 256 0 + module.nn0.4.weight 131072 0 + module.nn0.4.bias 512 0 + module.backbone.0.regions 0 96 + module.backbone.0.attn.out_linear.weight 65536 0 + module.backbone.0.attn.out_linear.bias 256 0 + module.backbone.0.attn.e2lsh.alpha 0 864 + module.backbone.0.attn.e2lsh.beta 0 3 + module.backbone.0.attn.e2lsh_new.alpha 0 96 + module.backbone.0.attn.e2lsh_new.beta 0 3 + module.backbone.0.w_q.weight 65536 0 + module.backbone.0.w_k.weight 65536 0 + module.backbone.0.w_v.weight 65536 0 + module.backbone.0.w_rpe.weight 0 2560 + module.backbone.0.w_rpe.bias 0 256 + module.backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.0.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.0.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.0.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.0.norm1.weight 256 0 + module.backbone.0.norm2.weight 256 0 + module.backbone.0.q_norm.weight 16 0 + module.backbone.0.k_norm.weight 16 0 + module.backbone.0.ff.gate_proj.weight 262144 0 + module.backbone.0.ff.up_proj.weight 262144 0 + module.backbone.0.ff.down_proj.weight 262144 0 + module.backbone.1.regions 0 96 + module.backbone.1.attn.out_linear.weight 65536 0 + module.backbone.1.attn.out_linear.bias 256 0 + module.backbone.1.attn.e2lsh.alpha 0 864 + module.backbone.1.attn.e2lsh.beta 0 3 + module.backbone.1.attn.e2lsh_new.alpha 0 96 + module.backbone.1.attn.e2lsh_new.beta 0 3 + module.backbone.1.w_q.weight 65536 0 + module.backbone.1.w_k.weight 65536 0 + module.backbone.1.w_v.weight 65536 0 + module.backbone.1.w_rpe.weight 0 2560 + module.backbone.1.w_rpe.bias 0 256 + module.backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.1.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.1.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.1.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.1.norm1.weight 256 0 + module.backbone.1.norm2.weight 256 0 + module.backbone.1.q_norm.weight 16 0 + module.backbone.1.k_norm.weight 16 0 + module.backbone.1.ff.gate_proj.weight 262144 0 + module.backbone.1.ff.up_proj.weight 262144 0 + module.backbone.1.ff.down_proj.weight 262144 0 + module.backbone.2.regions 0 96 + module.backbone.2.attn.out_linear.weight 65536 0 + module.backbone.2.attn.out_linear.bias 256 0 + module.backbone.2.attn.e2lsh.alpha 0 864 + module.backbone.2.attn.e2lsh.beta 0 3 + module.backbone.2.attn.e2lsh_new.alpha 0 96 + module.backbone.2.attn.e2lsh_new.beta 0 3 + module.backbone.2.w_q.weight 65536 0 + module.backbone.2.w_k.weight 65536 0 + module.backbone.2.w_v.weight 65536 0 + module.backbone.2.w_rpe.weight 0 2560 + module.backbone.2.w_rpe.bias 0 256 + module.backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.2.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.2.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.2.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.2.norm1.weight 256 0 + module.backbone.2.norm2.weight 256 0 + module.backbone.2.q_norm.weight 16 0 + module.backbone.2.k_norm.weight 16 0 + module.backbone.2.ff.gate_proj.weight 262144 0 + module.backbone.2.ff.up_proj.weight 262144 0 + module.backbone.2.ff.down_proj.weight 262144 0 + module.backbone.3.regions 0 96 + module.backbone.3.attn.out_linear.weight 65536 0 + module.backbone.3.attn.out_linear.bias 256 0 + module.backbone.3.attn.e2lsh.alpha 0 864 + module.backbone.3.attn.e2lsh.beta 0 3 + module.backbone.3.attn.e2lsh_new.alpha 0 96 + module.backbone.3.attn.e2lsh_new.beta 0 3 + module.backbone.3.w_q.weight 65536 0 + module.backbone.3.w_k.weight 65536 0 + module.backbone.3.w_v.weight 65536 0 + module.backbone.3.w_rpe.weight 0 2560 + module.backbone.3.w_rpe.bias 0 256 + module.backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.3.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.3.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.3.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.3.norm1.weight 256 0 + module.backbone.3.norm2.weight 256 0 + module.backbone.3.q_norm.weight 16 0 + module.backbone.3.k_norm.weight 16 0 + module.backbone.3.ff.gate_proj.weight 262144 0 + module.backbone.3.ff.up_proj.weight 262144 0 + module.backbone.3.ff.down_proj.weight 262144 0 + module.backbone.4.regions 0 96 + module.backbone.4.attn.out_linear.weight 65536 0 + module.backbone.4.attn.out_linear.bias 256 0 + module.backbone.4.attn.e2lsh.alpha 0 864 + module.backbone.4.attn.e2lsh.beta 0 3 + module.backbone.4.attn.e2lsh_new.alpha 0 96 + module.backbone.4.attn.e2lsh_new.beta 0 3 + module.backbone.4.w_q.weight 65536 0 + module.backbone.4.w_k.weight 65536 0 + module.backbone.4.w_v.weight 65536 0 + module.backbone.4.w_rpe.weight 0 2560 + module.backbone.4.w_rpe.bias 0 256 + module.backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.4.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.4.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.4.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.4.norm1.weight 256 0 + module.backbone.4.norm2.weight 256 0 + module.backbone.4.q_norm.weight 16 0 + module.backbone.4.k_norm.weight 16 0 + module.backbone.4.ff.gate_proj.weight 262144 0 + module.backbone.4.ff.up_proj.weight 262144 0 + module.backbone.4.ff.down_proj.weight 262144 0 + module.backbone.5.regions 0 96 + module.backbone.5.attn.out_linear.weight 65536 0 + module.backbone.5.attn.out_linear.bias 256 0 + module.backbone.5.attn.e2lsh.alpha 0 864 + module.backbone.5.attn.e2lsh.beta 0 3 + module.backbone.5.attn.e2lsh_new.alpha 0 96 + module.backbone.5.attn.e2lsh_new.beta 0 3 + module.backbone.5.w_q.weight 65536 0 + module.backbone.5.w_k.weight 65536 0 + module.backbone.5.w_v.weight 65536 0 + module.backbone.5.w_rpe.weight 0 2560 + module.backbone.5.w_rpe.bias 0 256 + module.backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + module.backbone.5.pe_func.position_embedding_head.0.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + module.backbone.5.pe_func.position_embedding_head.1.bias 1024 0 + module.backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + module.backbone.5.pe_func.position_embedding_head.3.bias 256 0 + module.backbone.5.norm1.weight 256 0 + module.backbone.5.norm2.weight 256 0 + module.backbone.5.q_norm.weight 16 0 + module.backbone.5.k_norm.weight 16 0 + module.backbone.5.ff.gate_proj.weight 262144 0 + module.backbone.5.ff.up_proj.weight 262144 0 + module.backbone.5.ff.down_proj.weight 262144 0 + module.set_decoder.queries 65536 0 + module.set_decoder.layers.0.query_norm.weight 256 0 + module.set_decoder.layers.0.query_norm.bias 256 0 + module.set_decoder.layers.0.memory_norm.weight 256 0 + module.set_decoder.layers.0.memory_norm.bias 256 0 + module.set_decoder.layers.0.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.self_norm.weight 256 0 + module.set_decoder.layers.0.self_norm.bias 256 0 + module.set_decoder.layers.0.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.0.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.0.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.0.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.0.ffn_norm.weight 256 0 + module.set_decoder.layers.0.ffn_norm.bias 256 0 + module.set_decoder.layers.0.ffn.0.weight 262144 0 + module.set_decoder.layers.0.ffn.0.bias 1024 0 + module.set_decoder.layers.0.ffn.3.weight 262144 0 + module.set_decoder.layers.0.ffn.3.bias 256 0 + module.set_decoder.layers.1.query_norm.weight 256 0 + module.set_decoder.layers.1.query_norm.bias 256 0 + module.set_decoder.layers.1.memory_norm.weight 256 0 + module.set_decoder.layers.1.memory_norm.bias 256 0 + module.set_decoder.layers.1.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.self_norm.weight 256 0 + module.set_decoder.layers.1.self_norm.bias 256 0 + module.set_decoder.layers.1.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.1.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.1.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.1.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.1.ffn_norm.weight 256 0 + module.set_decoder.layers.1.ffn_norm.bias 256 0 + module.set_decoder.layers.1.ffn.0.weight 262144 0 + module.set_decoder.layers.1.ffn.0.bias 1024 0 + module.set_decoder.layers.1.ffn.3.weight 262144 0 + module.set_decoder.layers.1.ffn.3.bias 256 0 + module.set_decoder.layers.2.query_norm.weight 256 0 + module.set_decoder.layers.2.query_norm.bias 256 0 + module.set_decoder.layers.2.memory_norm.weight 256 0 + module.set_decoder.layers.2.memory_norm.bias 256 0 + module.set_decoder.layers.2.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.self_norm.weight 256 0 + module.set_decoder.layers.2.self_norm.bias 256 0 + module.set_decoder.layers.2.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.2.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.2.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.2.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.2.ffn_norm.weight 256 0 + module.set_decoder.layers.2.ffn_norm.bias 256 0 + module.set_decoder.layers.2.ffn.0.weight 262144 0 + module.set_decoder.layers.2.ffn.0.bias 1024 0 + module.set_decoder.layers.2.ffn.3.weight 262144 0 + module.set_decoder.layers.2.ffn.3.bias 256 0 + module.set_decoder.layers.3.query_norm.weight 256 0 + module.set_decoder.layers.3.query_norm.bias 256 0 + module.set_decoder.layers.3.memory_norm.weight 256 0 + module.set_decoder.layers.3.memory_norm.bias 256 0 + module.set_decoder.layers.3.cross_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.cross_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.cross_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.cross_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.self_norm.weight 256 0 + module.set_decoder.layers.3.self_norm.bias 256 0 + module.set_decoder.layers.3.self_attention.in_proj_weight 196608 0 + module.set_decoder.layers.3.self_attention.in_proj_bias 768 0 + module.set_decoder.layers.3.self_attention.out_proj.weight 65536 0 + module.set_decoder.layers.3.self_attention.out_proj.bias 256 0 + module.set_decoder.layers.3.ffn_norm.weight 256 0 + module.set_decoder.layers.3.ffn_norm.bias 256 0 + module.set_decoder.layers.3.ffn.0.weight 262144 0 + module.set_decoder.layers.3.ffn.0.bias 1024 0 + module.set_decoder.layers.3.ffn.3.weight 262144 0 + module.set_decoder.layers.3.ffn.3.bias 256 0 + module.set_decoder.output_norm.weight 256 0 + module.set_decoder.output_norm.bias 256 0 + module.set_decoder.hit_grouping_hit.weight 8192 0 + module.set_decoder.hit_grouping_hit.bias 32 0 + module.set_decoder.hit_grouping_slot.weight 8192 0 + module.set_decoder.hit_grouping_slot.bias 32 0 + module.set_decoder.hit_grouping_background.weight 256 0 + module.set_decoder.hit_grouping_background.bias 1 0 + module.set_decoder.hit_grouping_projection.weight 131840 0 + module.set_decoder.hit_grouping_projection.bias 256 0 + module.set_decoder.presence_head.weight 512 0 + module.set_decoder.presence_head.bias 2 0 + module.set_decoder.pid_head.weight 1536 0 + module.set_decoder.pid_head.bias 6 0 + module.set_decoder.seed_projection.weight 65536 0 + module.set_decoder.seed_projection.bias 256 0 + module.set_decoder.reference_embedding.0.weight 768 0 + module.set_decoder.reference_embedding.0.bias 256 0 + module.set_decoder.reference_embedding.2.weight 65536 0 + module.set_decoder.reference_embedding.2.bias 256 0 + module.set_decoder.reference_delta_heads.0.weight 512 0 + module.set_decoder.reference_delta_heads.0.bias 2 0 + module.set_decoder.reference_delta_heads.1.weight 512 0 + module.set_decoder.reference_delta_heads.1.bias 2 0 + module.set_decoder.reference_delta_heads.2.weight 512 0 + module.set_decoder.reference_delta_heads.2.bias 2 0 + module.set_decoder.reference_delta_heads.3.weight 512 0 + module.set_decoder.reference_delta_heads.3.bias 2 0 + module.set_decoder.scale_head.weight 512 0 + module.set_decoder.scale_head.bias 2 0 + module.set_decoder.aggregate_projection.0.weight 2048 0 + module.set_decoder.aggregate_projection.0.bias 256 0 + module.set_decoder.aggregate_projection.2.weight 65536 0 + module.set_decoder.aggregate_projection.2.bias 256 0 +module.set_decoder.tracker_momentum_head.embedding_norm.weight 256 0 + module.set_decoder.tracker_momentum_head.embedding_norm.bias 256 0 + module.set_decoder.tracker_momentum_head.hidden.weight 76800 0 + module.set_decoder.tracker_momentum_head.hidden.bias 256 0 + module.set_decoder.tracker_momentum_head.output.weight 256 0 + module.set_decoder.tracker_momentum_head.output.bias 1 0 +rank7 [2026-10-29 14:49:22,292] INFO : Getting dataloaders +rank7 [2026-10-29 14:49:22,292] INFO : sample=cld_edm_ttbar_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:49:28,709] INFO : sample=cld_edm_ww_fullhad_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:49:29,731] INFO : sample=cld_edm_qq_hits split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:49:30,958] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank7 [2026-10-29 14:49:30,958] INFO : Creating WeightedMixIterator with 1 data loaders. +rank7 [2026-10-29 14:49:30,959] INFO : sample=cld_edm_ttbar_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:49:31,243] INFO : sample=cld_edm_ww_fullhad_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:49:31,501] INFO : sample=cld_edm_qq_hits split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 14:49:31,761] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank7 [2026-10-29 14:49:31,761] INFO : Creating InterleavedIterator with 1 data loaders. +rank7 [2026-10-29 14:49:45,683] INFO : Got dataloaders +rank7 [2026-10-29 14:49:45,683] INFO : WeightedMixIterator train: groups ['physical'] resume at batches [8888] of epochs [1] +rank7 [2026-10-29 14:49:45,683] INFO : InterleavedIterator valid loading state: cur_index=12, batches_yielded_per_loader=[12] +rank7 [2026-10-29 14:49:45,683] INFO : InterleavedIterator valid advancing sampler 0 to 192 +rank7 [2026-10-29 14:49:45,683] INFO : ResumableSampler physical:valid loading state: start_index=192 +rank7 [2026-10-29 14:49:47,303] INFO : loader split=train rank=7 len=21112 +rank7 [2026-10-29 14:49:47,303] INFO : loader split=valid rank=7 len=12 +rank7 [2026-10-29 14:49:47,303] INFO : Starting training from step 30001 to 50000 on rank 7 of 8 +rank7 [2026-10-29 14:49:47,303] INFO : Creating train_iterator +rank7 [2026-10-29 14:49:47,304] INFO : ResumableSampler physical:train loading state: start_index=142208 +rank7 [2026-10-29 14:50:01,417] INFO : Created train_iterator +rank7 [2026-10-29 14:54:42,005] INFO : Step 30100/50000 rank7 | Train Loss: 0.9854 | Train Loss Unweighted: 0.4821 | Train Loss Components: Auxiliary: 0.2024 | Cardinality: 0.1295 | Classification: 0.0965 | Classification_binary: 0.1561 | HitGrouping: 0.1545 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1127 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0700 | Classification_binary: 0.1224 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0546 | Regression_eta: 0.0335 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0491 | LR: 4.08e-04 | DataLoad Time: 0.0173s | Model Forward Time: 3.6377s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0061 | max_forward: 0.7011 | max_loss: 0.1473 | max_backward: 1.5273 | max_optimizer_step: 0.1574 | max_tensorboard_logging: 0.4772 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0061 | Memory: rss_MB: 4775.9 | uss_MB: 3472.5 | child_rss_MB: 6376.2 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 850.1 +rank7 [2026-10-29 14:59:12,794] INFO : Step 30200/50000 rank7 | Train Loss: 1.0165 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2044 | Cardinality: 0.1520 | Classification: 0.0856 | Classification_binary: 0.1718 | HitGrouping: 0.1479 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1037 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0509 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0562 | Regression_eta: 0.0400 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0594 | LR: 4.06e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.9885s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.7915 | max_loss: 0.2568 | max_backward: 1.8926 | max_optimizer_step: 0.1551 | max_tensorboard_logging: 0.4604 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0043 | Memory: rss_MB: 4776.3 | uss_MB: 3476.7 | child_rss_MB: 6367.6 | allocated_MB: 497.3 | reserved_MB: 26672.0 | inactive_split_MB: 842.0 +rank7 [2026-10-29 15:03:42,966] INFO : Step 30300/50000 rank7 | Train Loss: 0.9183 | Train Loss Unweighted: 0.4449 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1232 | Classification: 0.0820 | Classification_binary: 0.1459 | HitGrouping: 0.1516 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1073 | Regression_eta: 0.0012 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1493 | Regression_cos_phi: 0.0468 | Regression_energy: 0.0489 | Regression_eta: 0.0338 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0513 | LR: 4.03e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8583s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.7525 | max_loss: 0.2476 | max_backward: 1.8262 | max_optimizer_step: 0.1580 | max_tensorboard_logging: 0.4574 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0039 | Memory: rss_MB: 4773.5 | uss_MB: 3475.1 | child_rss_MB: 6385.6 | allocated_MB: 495.8 | reserved_MB: 28760.0 | inactive_split_MB: 848.2 +rank7 [2026-10-29 15:08:13,476] INFO : Step 30400/50000 rank7 | Train Loss: 0.9035 | Train Loss Unweighted: 0.4372 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1137 | Classification: 0.0848 | Classification_binary: 0.1358 | HitGrouping: 0.1514 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0410 | Classification_binary: 0.1208 | Regression_cos_phi: 0.0355 | Regression_energy: 0.0493 | Regression_eta: 0.0198 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0354 | LR: 4.00e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.9726s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0073 | max_forward: 0.7769 | max_loss: 0.1398 | max_backward: 1.9002 | max_optimizer_step: 0.1557 | max_tensorboard_logging: 0.4925 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0050 | Memory: rss_MB: 4770.8 | uss_MB: 3475.5 | child_rss_MB: 6366.2 | allocated_MB: 498.8 | reserved_MB: 28760.0 | inactive_split_MB: 878.9 +rank7 [2026-10-29 15:12:45,608] INFO : Step 30500/50000 rank7 | Train Loss: 1.0399 | Train Loss Unweighted: 0.4899 | Train Loss Components: Auxiliary: 0.2123 | Cardinality: 0.1239 | Classification: 0.0901 | Classification_binary: 0.1841 | HitGrouping: 0.1572 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1053 | Regression_eta: 0.0018 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1722 | Regression_cos_phi: 0.0868 | Regression_energy: 0.0603 | Regression_eta: 0.0652 | Regression_pt: 0.0643 | Regression_sin_phi: 0.0778 | LR: 3.98e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.3867s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0067 | max_forward: 0.9688 | max_loss: 0.1627 | max_backward: 2.3054 | max_optimizer_step: 0.1555 | max_tensorboard_logging: 0.4908 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0049 | Memory: rss_MB: 4768.4 | uss_MB: 3475.6 | child_rss_MB: 6405.5 | allocated_MB: 501.7 | reserved_MB: 28760.0 | inactive_split_MB: 944.3 +rank7 [2026-10-29 15:17:13,866] INFO : Step 30600/50000 rank7 | Train Loss: 0.9987 | Train Loss Unweighted: 0.4959 | Train Loss Components: Auxiliary: 0.2035 | Cardinality: 0.1112 | Classification: 0.0932 | Classification_binary: 0.1720 | HitGrouping: 0.1575 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1137 | Regression_eta: 0.0014 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1869 | Regression_cos_phi: 0.0530 | Regression_energy: 0.0581 | Regression_eta: 0.0397 | Regression_pt: 0.0636 | Regression_sin_phi: 0.0607 | LR: 3.95e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2238s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0070 | max_forward: 0.7736 | max_loss: 0.1566 | max_backward: 1.7922 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.6994 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0040 | Memory: rss_MB: 4770.8 | uss_MB: 3476.1 | child_rss_MB: 6399.5 | allocated_MB: 500.1 | reserved_MB: 28760.0 | inactive_split_MB: 845.9 +rank7 [2026-10-29 15:21:45,752] INFO : Step 30700/50000 rank7 | Train Loss: 0.9490 | Train Loss Unweighted: 0.4575 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1237 | Classification: 0.0844 | Classification_binary: 0.1399 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1149 | Regression_eta: 0.0014 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0438 | Classification_binary: 0.1429 | Regression_cos_phi: 0.0560 | Regression_energy: 0.0702 | Regression_eta: 0.0353 | Regression_pt: 0.0762 | Regression_sin_phi: 0.0433 | LR: 3.92e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1843s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.7215 | max_loss: 0.5560 | max_backward: 2.0906 | max_optimizer_step: 0.1553 | max_tensorboard_logging: 0.5054 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0044 | Memory: rss_MB: 4768.7 | uss_MB: 3474.9 | child_rss_MB: 6379.2 | allocated_MB: 493.8 | reserved_MB: 28760.0 | inactive_split_MB: 827.5 +rank7 [2026-10-29 15:26:16,185] INFO : Step 30800/50000 rank7 | Train Loss: 1.0320 | Train Loss Unweighted: 0.4949 | Train Loss Components: Auxiliary: 0.2087 | Cardinality: 0.1331 | Classification: 0.0916 | Classification_binary: 0.1632 | HitGrouping: 0.1591 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1173 | Regression_eta: 0.0015 | Regression_pt: 0.1196 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0675 | Classification_binary: 0.1559 | Regression_cos_phi: 0.0921 | Regression_energy: 0.0673 | Regression_eta: 0.0618 | Regression_pt: 0.0728 | Regression_sin_phi: 0.1323 | LR: 3.90e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6902s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.1016 | max_forward: 0.8041 | max_loss: 0.1462 | max_backward: 1.9351 | max_optimizer_step: 0.1552 | max_tensorboard_logging: 1.1007 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0026 | Memory: rss_MB: 4769.5 | uss_MB: 3475.6 | child_rss_MB: 6407.8 | allocated_MB: 497.9 | reserved_MB: 28760.0 | inactive_split_MB: 842.0 +rank7 [2026-10-29 15:30:49,188] INFO : Step 30900/50000 rank7 | Train Loss: 1.0226 | Train Loss Unweighted: 0.4952 | Train Loss Components: Auxiliary: 0.2139 | Cardinality: 0.1339 | Classification: 0.0875 | Classification_binary: 0.1859 | HitGrouping: 0.1518 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1092 | Regression_eta: 0.0013 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0397 | Classification_binary: 0.1706 | Regression_cos_phi: 0.0584 | Regression_energy: 0.0588 | Regression_eta: 0.0335 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0548 | LR: 3.87e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.4018s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7610 | max_loss: 0.2067 | max_backward: 1.7028 | max_optimizer_step: 0.1599 | max_tensorboard_logging: 1.0327 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0017 | Memory: rss_MB: 4771.1 | uss_MB: 3476.3 | child_rss_MB: 6385.8 | allocated_MB: 497.2 | reserved_MB: 28760.0 | inactive_split_MB: 838.0 +rank7 [2026-10-29 15:35:26,546] INFO : Step 31000/50000 rank7 | Train Loss: 0.9922 | Train Loss Unweighted: 0.4630 | Train Loss Components: Auxiliary: 0.2006 | Cardinality: 0.1217 | Classification: 0.0821 | Classification_binary: 0.1592 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1085 | Regression_eta: 0.0018 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1743 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0636 | Regression_eta: 0.0622 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0604 | LR: 3.84e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4977s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.7780 | max_loss: 0.2475 | max_backward: 1.8803 | max_optimizer_step: 0.1588 | max_tensorboard_logging: 0.9947 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0028 | Memory: rss_MB: 4772.1 | uss_MB: 3477.0 | child_rss_MB: 6402.3 | allocated_MB: 498.2 | reserved_MB: 28760.0 | inactive_split_MB: 820.4 +rank7 [2026-10-29 15:35:27,414] INFO : Running validation on rank7 +rank7 [2026-10-29 15:35:27,414] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 15:35:27,415] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 15:36:19,715] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:36:20,267] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 15:36:45,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 15:36:46,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 15:36:47,871] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 15:36:48,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 15:36:58,829] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 15:36:59,566] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:37:00,103] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 15:37:16,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 15:37:17,106] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 15:37:18,042] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 15:37:18,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 15:37:28,989] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 15:37:29,392] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 15:37:29,837] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 15:37:45,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 15:37:46,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 15:37:46,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 15:37:47,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_31000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 15:37:57,814] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 15:42:50,283] INFO : Step 31100/50000 rank7 | Train Loss: 0.8852 | Train Loss Unweighted: 0.4264 | Train Loss Components: Auxiliary: 0.1803 | Cardinality: 0.1054 | Classification: 0.0853 | Classification_binary: 0.1288 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1311 | Regression_cos_phi: 0.0944 | Regression_energy: 0.0453 | Regression_eta: 0.0542 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0495 | LR: 3.82e-04 | DataLoad Time: 0.0172s | Model Forward Time: 4.5227s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7353 | max_loss: 0.1410 | max_backward: 1.7959 | max_optimizer_step: 0.2754 | max_tensorboard_logging: 0.9599 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0182 | Memory: rss_MB: 6463.6 | uss_MB: 5188.4 | child_rss_MB: 6394.5 | allocated_MB: 495.8 | reserved_MB: 27508.0 | inactive_split_MB: 851.2 +rank7 [2026-10-29 15:47:34,509] INFO : Step 31200/50000 rank7 | Train Loss: 0.9857 | Train Loss Unweighted: 0.4709 | Train Loss Components: Auxiliary: 0.2045 | Cardinality: 0.1286 | Classification: 0.0862 | Classification_binary: 0.1646 | HitGrouping: 0.1524 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1077 | Regression_eta: 0.0014 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1248 | Regression_cos_phi: 0.0617 | Regression_energy: 0.0432 | Regression_eta: 0.0262 | Regression_pt: 0.0467 | Regression_sin_phi: 0.0613 | LR: 3.79e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.3224s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8441 | max_loss: 0.1469 | max_backward: 2.0813 | max_optimizer_step: 0.1798 | max_tensorboard_logging: 0.4963 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0227 | Memory: rss_MB: 6462.4 | uss_MB: 5189.4 | child_rss_MB: 6422.3 | allocated_MB: 496.9 | reserved_MB: 27508.0 | inactive_split_MB: 935.6 +rank7 [2026-10-29 15:52:19,963] INFO : Step 31300/50000 rank7 | Train Loss: 1.0631 | Train Loss Unweighted: 0.4971 | Train Loss Components: Auxiliary: 0.2115 | Cardinality: 0.1394 | Classification: 0.0917 | Classification_binary: 0.1731 | HitGrouping: 0.1556 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1142 | Regression_eta: 0.0017 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0532 | Classification_binary: 0.1705 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0631 | Regression_eta: 0.0222 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0491 | LR: 3.76e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.2498s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0065 | max_forward: 0.7950 | max_loss: 0.1940 | max_backward: 1.8810 | max_optimizer_step: 0.2519 | max_tensorboard_logging: 0.5719 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5602.7 | uss_MB: 4330.1 | child_rss_MB: 6421.1 | allocated_MB: 498.6 | reserved_MB: 29580.0 | inactive_split_MB: 773.5 +rank7 [2026-10-29 15:56:52,523] INFO : Step 31400/50000 rank7 | Train Loss: 0.9019 | Train Loss Unweighted: 0.4258 | Train Loss Components: Auxiliary: 0.1823 | Cardinality: 0.1101 | Classification: 0.0746 | Classification_binary: 0.1337 | HitGrouping: 0.1581 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1070 | Regression_eta: 0.0014 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0330 | Classification_binary: 0.1620 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0551 | Regression_eta: 0.0316 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0398 | LR: 3.74e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.2766s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0063 | max_forward: 0.8478 | max_loss: 0.2460 | max_backward: 2.0403 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.4770 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0139 | Memory: rss_MB: 5595.4 | uss_MB: 4331.0 | child_rss_MB: 6422.8 | allocated_MB: 495.2 | reserved_MB: 29580.0 | inactive_split_MB: 986.6 +rank7 [2026-10-29 16:01:34,249] INFO : Step 31500/50000 rank7 | Train Loss: 0.9009 | Train Loss Unweighted: 0.4434 | Train Loss Components: Auxiliary: 0.1867 | Cardinality: 0.1036 | Classification: 0.0834 | Classification_binary: 0.1437 | HitGrouping: 0.1499 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1064 | Regression_eta: 0.0013 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1056 | Regression_cos_phi: 0.0444 | Regression_energy: 0.0490 | Regression_eta: 0.0239 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0467 | LR: 3.71e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.7507s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0072 | max_forward: 0.7930 | max_loss: 0.1458 | max_backward: 1.7855 | max_optimizer_step: 0.1694 | max_tensorboard_logging: 2.2800 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5572.8 | uss_MB: 4310.0 | child_rss_MB: 6426.4 | allocated_MB: 496.8 | reserved_MB: 29580.0 | inactive_split_MB: 850.8 +rank7 [2026-10-29 16:06:05,474] INFO : Step 31600/50000 rank7 | Train Loss: 1.0288 | Train Loss Unweighted: 0.4730 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1317 | Classification: 0.0906 | Classification_binary: 0.1743 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1012 | Regression_eta: 0.0017 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1575 | Regression_cos_phi: 0.0565 | Regression_energy: 0.0557 | Regression_eta: 0.0265 | Regression_pt: 0.0615 | Regression_sin_phi: 0.0668 | LR: 3.69e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.1934s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1021 | max_forward: 0.8041 | max_loss: 0.5000 | max_backward: 1.8881 | max_optimizer_step: 0.1677 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0133 | Memory: rss_MB: 5567.6 | uss_MB: 4330.2 | child_rss_MB: 6421.3 | allocated_MB: 494.5 | reserved_MB: 29580.0 | inactive_split_MB: 850.5 +rank7 [2026-10-29 16:10:48,122] INFO : Step 31700/50000 rank7 | Train Loss: 0.9206 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.1890 | Cardinality: 0.1106 | Classification: 0.0729 | Classification_binary: 0.1465 | HitGrouping: 0.1568 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1095 | Regression_eta: 0.0013 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0617 | Classification_binary: 0.1610 | Regression_cos_phi: 0.0839 | Regression_energy: 0.0550 | Regression_eta: 0.0388 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0773 | LR: 3.66e-04 | DataLoad Time: 0.0176s | Model Forward Time: 3.7508s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0062 | max_forward: 0.7097 | max_loss: 0.1806 | max_backward: 1.6117 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0100 | Memory: rss_MB: 5567.0 | uss_MB: 4328.6 | child_rss_MB: 6424.1 | allocated_MB: 495.3 | reserved_MB: 29580.0 | inactive_split_MB: 969.6 +rank7 [2026-10-29 16:15:56,265] INFO : Step 31800/50000 rank7 | Train Loss: 0.9090 | Train Loss Unweighted: 0.4310 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1140 | Classification: 0.0748 | Classification_binary: 0.1332 | HitGrouping: 0.1595 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1094 | Regression_eta: 0.0013 | Regression_pt: 0.1110 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0350 | Classification_binary: 0.1276 | Regression_cos_phi: 0.0540 | Regression_energy: 0.0633 | Regression_eta: 0.0245 | Regression_pt: 0.0687 | Regression_sin_phi: 0.0761 | LR: 3.64e-04 | DataLoad Time: 0.0163s | Model Forward Time: 3.8751s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7102 | max_loss: 0.1436 | max_backward: 1.6093 | max_optimizer_step: 0.1693 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0099 | Memory: rss_MB: 5562.0 | uss_MB: 4326.9 | child_rss_MB: 6424.4 | allocated_MB: 499.4 | reserved_MB: 29584.0 | inactive_split_MB: 1068.5 +rank7 [2026-10-29 16:20:39,712] INFO : Step 31900/50000 rank7 | Train Loss: 0.9919 | Train Loss Unweighted: 0.4645 | Train Loss Components: Auxiliary: 0.2053 | Cardinality: 0.1269 | Classification: 0.0911 | Classification_binary: 0.1745 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0975 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0614 | Classification_binary: 0.1607 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0580 | Regression_eta: 0.0321 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0515 | LR: 3.61e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3405s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.2538 | max_backward: 1.9768 | max_optimizer_step: 0.2376 | max_tensorboard_logging: 0.5597 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5568.8 | uss_MB: 4333.1 | child_rss_MB: 6433.0 | allocated_MB: 496.2 | reserved_MB: 29584.0 | inactive_split_MB: 829.8 +rank7 [2026-10-29 16:25:21,505] INFO : Step 32000/50000 rank7 | Train Loss: 0.9662 | Train Loss Unweighted: 0.4570 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1172 | Classification: 0.0861 | Classification_binary: 0.1650 | HitGrouping: 0.1508 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1010 | Regression_eta: 0.0015 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0432 | Classification_binary: 0.1563 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0411 | Regression_eta: 0.0304 | Regression_pt: 0.0441 | Regression_sin_phi: 0.0354 | LR: 3.58e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.2268s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7801 | max_loss: 0.1534 | max_backward: 1.8397 | max_optimizer_step: 0.2695 | max_tensorboard_logging: 0.5734 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0138 | Memory: rss_MB: 5561.3 | uss_MB: 4329.3 | child_rss_MB: 6422.4 | allocated_MB: 497.7 | reserved_MB: 29584.0 | inactive_split_MB: 927.6 +rank7 [2026-10-29 16:25:22,457] INFO : Running validation on rank7 +rank7 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 16:25:22,457] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 16:26:16,177] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:26:16,710] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 16:26:42,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 16:26:43,891] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 16:26:44,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 16:26:45,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 16:26:55,845] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 16:26:56,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:26:57,009] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 16:27:12,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 16:27:13,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 16:27:14,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 16:27:15,790] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 16:27:25,800] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 16:27:26,913] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 16:27:27,382] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 16:27:43,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 16:27:44,039] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 16:27:44,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 16:27:45,756] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_32000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 16:27:55,767] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 16:32:59,849] INFO : Step 32100/50000 rank7 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4572 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1125 | Classification: 0.0829 | Classification_binary: 0.1389 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1159 | Regression_eta: 0.0016 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0385 | Classification_binary: 0.1404 | Regression_cos_phi: 0.0593 | Regression_energy: 0.0610 | Regression_eta: 0.0342 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0409 | LR: 3.56e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.8035s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0431 | max_forward: 0.7694 | max_loss: 0.1379 | max_backward: 1.5662 | max_optimizer_step: 0.2199 | max_tensorboard_logging: 1.3675 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6985.8 | uss_MB: 5757.8 | child_rss_MB: 6447.8 | allocated_MB: 499.0 | reserved_MB: 25862.0 | inactive_split_MB: 892.0 +rank7 [2026-10-29 16:37:33,176] INFO : Step 32200/50000 rank7 | Train Loss: 0.9700 | Train Loss Unweighted: 0.4696 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1193 | Classification: 0.0958 | Classification_binary: 0.1508 | HitGrouping: 0.1529 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1092 | Regression_eta: 0.0016 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0644 | Classification_binary: 0.1334 | Regression_cos_phi: 0.0532 | Regression_energy: 0.0484 | Regression_eta: 0.0445 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0607 | LR: 3.53e-04 | DataLoad Time: 0.0150s | Model Forward Time: 5.1398s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8207 | max_loss: 0.1504 | max_backward: 2.0613 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.9148 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0371 | Memory: rss_MB: 6989.2 | uss_MB: 5763.2 | child_rss_MB: 6457.2 | allocated_MB: 499.4 | reserved_MB: 25864.0 | inactive_split_MB: 778.6 +rank7 [2026-10-29 16:42:04,508] INFO : Step 32300/50000 rank7 | Train Loss: 1.0322 | Train Loss Unweighted: 0.4909 | Train Loss Components: Auxiliary: 0.2110 | Cardinality: 0.1295 | Classification: 0.0875 | Classification_binary: 0.1834 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0017 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0667 | Classification_binary: 0.1865 | Regression_cos_phi: 0.0554 | Regression_energy: 0.0581 | Regression_eta: 0.0451 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0837 | LR: 3.51e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.7855s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0068 | max_forward: 0.8382 | max_loss: 0.2412 | max_backward: 2.0637 | max_optimizer_step: 0.1754 | max_tensorboard_logging: 0.9540 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5664.5 | uss_MB: 4450.0 | child_rss_MB: 6462.2 | allocated_MB: 499.0 | reserved_MB: 27716.0 | inactive_split_MB: 775.0 +rank7 [2026-10-29 16:46:37,461] INFO : Step 32400/50000 rank7 | Train Loss: 0.9272 | Train Loss Unweighted: 0.4507 | Train Loss Components: Auxiliary: 0.1874 | Cardinality: 0.1089 | Classification: 0.0860 | Classification_binary: 0.1494 | HitGrouping: 0.1533 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1059 | Regression_eta: 0.0012 | Regression_pt: 0.1067 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1674 | Regression_cos_phi: 0.0687 | Regression_energy: 0.0555 | Regression_eta: 0.0326 | Regression_pt: 0.0626 | Regression_sin_phi: 0.0488 | LR: 3.48e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5427s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.1137 | max_forward: 0.8338 | max_loss: 0.1463 | max_backward: 1.8285 | max_optimizer_step: 0.1707 | max_tensorboard_logging: 0.8970 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0117 | Memory: rss_MB: 5664.1 | uss_MB: 4449.3 | child_rss_MB: 6459.6 | allocated_MB: 497.5 | reserved_MB: 27716.0 | inactive_split_MB: 964.6 +rank7 [2026-10-29 16:51:11,723] INFO : Step 32500/50000 rank7 | Train Loss: 0.9126 | Train Loss Unweighted: 0.4568 | Train Loss Components: Auxiliary: 0.1866 | Cardinality: 0.0989 | Classification: 0.0980 | Classification_binary: 0.1302 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1128 | Regression_eta: 0.0017 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0602 | Classification_binary: 0.1258 | Regression_cos_phi: 0.0483 | Regression_energy: 0.0495 | Regression_eta: 0.0379 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0543 | LR: 3.46e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.0960s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0061 | max_forward: 0.7619 | max_loss: 0.1497 | max_backward: 1.5269 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.7665 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0144 | Memory: rss_MB: 5643.1 | uss_MB: 4429.2 | child_rss_MB: 6454.9 | allocated_MB: 495.6 | reserved_MB: 27718.0 | inactive_split_MB: 1084.2 +rank7 [2026-10-29 16:55:48,603] INFO : Step 32600/50000 rank7 | Train Loss: 0.9821 | Train Loss Unweighted: 0.4631 | Train Loss Components: Auxiliary: 0.2025 | Cardinality: 0.1270 | Classification: 0.0917 | Classification_binary: 0.1687 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1000 | Regression_eta: 0.0014 | Regression_pt: 0.0997 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1570 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0576 | Regression_eta: 0.0323 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0537 | LR: 3.43e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.2674s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.8022 | max_loss: 0.1505 | max_backward: 1.8823 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 0.5840 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0173 | Memory: rss_MB: 5658.5 | uss_MB: 4448.5 | child_rss_MB: 6453.0 | allocated_MB: 496.5 | reserved_MB: 27844.0 | inactive_split_MB: 835.5 +rank7 [2026-10-29 17:00:16,963] INFO : Step 32700/50000 rank7 | Train Loss: 1.0218 | Train Loss Unweighted: 0.4931 | Train Loss Components: Auxiliary: 0.2103 | Cardinality: 0.1308 | Classification: 0.0908 | Classification_binary: 0.1761 | HitGrouping: 0.1579 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1115 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.2181 | Regression_cos_phi: 0.0733 | Regression_energy: 0.0532 | Regression_eta: 0.0368 | Regression_pt: 0.0579 | Regression_sin_phi: 0.0832 | LR: 3.41e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2042s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7654 | max_loss: 0.2523 | max_backward: 1.8639 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.4904 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.1101 | Memory: rss_MB: 5657.8 | uss_MB: 4448.3 | child_rss_MB: 6459.1 | allocated_MB: 495.8 | reserved_MB: 27844.0 | inactive_split_MB: 1177.8 +rank7 [2026-10-29 17:04:50,647] INFO : Step 32800/50000 rank7 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4798 | Train Loss Components: Auxiliary: 0.1977 | Cardinality: 0.1139 | Classification: 0.0957 | Classification_binary: 0.1480 | HitGrouping: 0.1618 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1163 | Regression_eta: 0.0019 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0539 | Classification_binary: 0.1530 | Regression_cos_phi: 0.0679 | Regression_energy: 0.0521 | Regression_eta: 0.0500 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0603 | LR: 3.38e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.0489s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7437 | max_loss: 0.1443 | max_backward: 1.7603 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.5989 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0137 | Memory: rss_MB: 5618.6 | uss_MB: 4447.1 | child_rss_MB: 6328.3 | allocated_MB: 497.6 | reserved_MB: 27846.0 | inactive_split_MB: 1089.8 +rank7 [2026-10-29 17:09:25,012] INFO : Step 32900/50000 rank7 | Train Loss: 1.0585 | Train Loss Unweighted: 0.4917 | Train Loss Components: Auxiliary: 0.2098 | Cardinality: 0.1472 | Classification: 0.0991 | Classification_binary: 0.1566 | HitGrouping: 0.1569 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1157 | Regression_eta: 0.0020 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0009 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0671 | Classification_binary: 0.1383 | Regression_cos_phi: 0.0786 | Regression_energy: 0.0564 | Regression_eta: 0.0455 | Regression_pt: 0.0607 | Regression_sin_phi: 0.0671 | LR: 3.36e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.2352s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0141 | max_forward: 0.7853 | max_loss: 0.1442 | max_backward: 1.9716 | max_optimizer_step: 0.1768 | max_tensorboard_logging: 0.5450 | max_tensorboard_flush: 0.0119 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5622.7 | uss_MB: 4451.0 | child_rss_MB: 6346.8 | allocated_MB: 498.5 | reserved_MB: 27846.0 | inactive_split_MB: 844.8 +rank7 [2026-10-29 17:13:57,544] INFO : Step 33000/50000 rank7 | Train Loss: 0.9900 | Train Loss Unweighted: 0.4896 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1109 | Classification: 0.0902 | Classification_binary: 0.1757 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1096 | Regression_eta: 0.0015 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0445 | Classification_binary: 0.1488 | Regression_cos_phi: 0.0451 | Regression_energy: 0.0435 | Regression_eta: 0.0249 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0533 | LR: 3.33e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.1403s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0372 | max_forward: 0.7798 | max_loss: 0.1504 | max_backward: 1.8342 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.5055 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0159 | Memory: rss_MB: 5614.6 | uss_MB: 4448.7 | child_rss_MB: 6348.5 | allocated_MB: 497.7 | reserved_MB: 27846.0 | inactive_split_MB: 857.6 +rank7 [2026-10-29 17:13:58,425] INFO : Running validation on rank7 +rank7 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 17:13:58,425] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 17:14:49,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:14:50,068] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 17:15:17,219] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 17:15:18,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 17:15:19,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 17:15:20,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 17:15:30,193] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 17:15:31,023] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:15:31,507] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 17:15:47,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 17:15:48,187] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 17:15:49,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 17:15:50,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 17:16:00,056] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 17:16:01,328] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 17:16:01,836] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 17:16:17,238] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 17:16:18,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 17:16:18,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 17:16:19,723] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_33000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 17:16:29,729] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 17:21:40,218] INFO : Step 33100/50000 rank7 | Train Loss: 0.9799 | Train Loss Unweighted: 0.4666 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1297 | Classification: 0.0842 | Classification_binary: 0.1537 | HitGrouping: 0.1574 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1125 | Regression_eta: 0.0015 | Regression_pt: 0.1134 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0487 | Classification_binary: 0.1733 | Regression_cos_phi: 0.0637 | Regression_energy: 0.0536 | Regression_eta: 0.0415 | Regression_pt: 0.0586 | Regression_sin_phi: 0.0511 | LR: 3.31e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.1407s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0059 | max_forward: 0.7607 | max_loss: 0.1464 | max_backward: 1.8352 | max_optimizer_step: 0.1989 | max_tensorboard_logging: 0.4758 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6888.0 | uss_MB: 5749.0 | child_rss_MB: 6329.7 | allocated_MB: 496.1 | reserved_MB: 24572.0 | inactive_split_MB: 941.9 +rank7 [2026-10-29 17:26:33,026] INFO : Step 33200/50000 rank7 | Train Loss: 0.9345 | Train Loss Unweighted: 0.4417 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.1121 | Classification: 0.0825 | Classification_binary: 0.1333 | HitGrouping: 0.1549 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0017 | Regression_pt: 0.1118 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0515 | Classification_binary: 0.1324 | Regression_cos_phi: 0.1012 | Regression_energy: 0.0532 | Regression_eta: 0.1475 | Regression_pt: 0.0573 | Regression_sin_phi: 0.0700 | LR: 3.28e-04 | DataLoad Time: 0.0176s | Model Forward Time: 4.2167s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0447 | max_forward: 0.7619 | max_loss: 0.4589 | max_backward: 1.9181 | max_optimizer_step: 0.1943 | max_tensorboard_logging: 0.4922 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6887.2 | uss_MB: 5750.0 | child_rss_MB: 6329.3 | allocated_MB: 497.6 | reserved_MB: 24578.0 | inactive_split_MB: 835.2 +rank7 [2026-10-29 17:31:31,562] INFO : Step 33300/50000 rank7 | Train Loss: 0.9171 | Train Loss Unweighted: 0.4300 | Train Loss Components: Auxiliary: 0.1936 | Cardinality: 0.1270 | Classification: 0.0780 | Classification_binary: 0.1518 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0985 | Regression_eta: 0.0012 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0440 | Classification_binary: 0.1431 | Regression_cos_phi: 0.0652 | Regression_energy: 0.0545 | Regression_eta: 0.0350 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0664 | LR: 3.26e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.6817s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0334 | max_forward: 0.7851 | max_loss: 0.1434 | max_backward: 2.0284 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.7975 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0309 | Memory: rss_MB: 5654.9 | uss_MB: 4170.1 | child_rss_MB: 6324.3 | allocated_MB: 496.9 | reserved_MB: 24578.0 | inactive_split_MB: 903.1 +rank7 [2026-10-29 17:36:22,674] INFO : Step 33400/50000 rank7 | Train Loss: 0.9537 | Train Loss Unweighted: 0.4634 | Train Loss Components: Auxiliary: 0.2000 | Cardinality: 0.1239 | Classification: 0.0872 | Classification_binary: 0.1563 | HitGrouping: 0.1500 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1085 | Regression_eta: 0.0013 | Regression_pt: 0.1086 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0452 | Classification_binary: 0.1543 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0561 | Regression_eta: 0.0409 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0452 | LR: 3.23e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.2723s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7951 | max_loss: 0.2522 | max_backward: 1.9349 | max_optimizer_step: 0.1762 | max_tensorboard_logging: 0.4815 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0134 | Memory: rss_MB: 5664.7 | uss_MB: 4533.0 | child_rss_MB: 6325.6 | allocated_MB: 500.7 | reserved_MB: 26578.0 | inactive_split_MB: 843.2 +rank7 [2026-10-29 17:41:05,304] INFO : Step 33500/50000 rank7 | Train Loss: 0.9039 | Train Loss Unweighted: 0.4494 | Train Loss Components: Auxiliary: 0.1864 | Cardinality: 0.1057 | Classification: 0.0795 | Classification_binary: 0.1278 | HitGrouping: 0.1605 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1202 | Regression_eta: 0.0011 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0550 | Classification_binary: 0.1635 | Regression_cos_phi: 0.0669 | Regression_energy: 0.0714 | Regression_eta: 0.0310 | Regression_pt: 0.0778 | Regression_sin_phi: 0.0565 | LR: 3.21e-04 | DataLoad Time: 0.0177s | Model Forward Time: 4.1838s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0177 | max_forward: 0.7696 | max_loss: 0.1421 | max_backward: 1.9122 | max_optimizer_step: 0.1772 | max_tensorboard_logging: 0.5562 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0158 | Memory: rss_MB: 5654.7 | uss_MB: 4519.8 | child_rss_MB: 6324.9 | allocated_MB: 494.9 | reserved_MB: 26578.0 | inactive_split_MB: 839.3 +rank7 [2026-10-29 17:45:56,712] INFO : Step 33600/50000 rank7 | Train Loss: 0.9179 | Train Loss Unweighted: 0.4415 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.0997 | Classification: 0.0812 | Classification_binary: 0.1440 | HitGrouping: 0.1544 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1058 | Regression_eta: 0.0012 | Regression_pt: 0.1077 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0573 | Classification_binary: 0.1546 | Regression_cos_phi: 0.0834 | Regression_energy: 0.0536 | Regression_eta: 0.0334 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0811 | LR: 3.19e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4044s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0063 | max_forward: 0.7783 | max_loss: 0.2418 | max_backward: 2.0392 | max_optimizer_step: 0.2787 | max_tensorboard_logging: 0.5781 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0188 | Memory: rss_MB: 5667.7 | uss_MB: 4535.9 | child_rss_MB: 6324.1 | allocated_MB: 497.9 | reserved_MB: 28594.0 | inactive_split_MB: 1127.0 +rank7 [2026-10-29 17:50:43,454] INFO : Step 33700/50000 rank7 | Train Loss: 1.0564 | Train Loss Unweighted: 0.4962 | Train Loss Components: Auxiliary: 0.2235 | Cardinality: 0.1517 | Classification: 0.0917 | Classification_binary: 0.1786 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0382 | Regression_energy: 0.0503 | Regression_eta: 0.0332 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0582 | LR: 3.16e-04 | DataLoad Time: 0.0137s | Model Forward Time: 4.1480s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0121 | max_forward: 0.7981 | max_loss: 0.1554 | max_backward: 1.9054 | max_optimizer_step: 0.1714 | max_tensorboard_logging: 0.4685 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0147 | Memory: rss_MB: 5667.6 | uss_MB: 4535.1 | child_rss_MB: 6300.3 | allocated_MB: 501.8 | reserved_MB: 28594.0 | inactive_split_MB: 1103.2 +rank7 [2026-10-29 17:55:24,637] INFO : Step 33800/50000 rank7 | Train Loss: 0.9305 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1894 | Cardinality: 0.1080 | Classification: 0.0751 | Classification_binary: 0.1369 | HitGrouping: 0.1527 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1201 | Regression_eta: 0.0014 | Regression_pt: 0.1228 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1415 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0556 | Regression_eta: 0.0340 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0362 | LR: 3.14e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.1054s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.8033 | max_loss: 0.1441 | max_backward: 1.7197 | max_optimizer_step: 0.2188 | max_tensorboard_logging: 0.5686 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0176 | Memory: rss_MB: 5662.9 | uss_MB: 4531.8 | child_rss_MB: 6320.7 | allocated_MB: 496.9 | reserved_MB: 28594.0 | inactive_split_MB: 909.5 +rank7 [2026-10-29 18:00:01,586] INFO : Step 33900/50000 rank7 | Train Loss: 0.9360 | Train Loss Unweighted: 0.4438 | Train Loss Components: Auxiliary: 0.1942 | Cardinality: 0.1035 | Classification: 0.0805 | Classification_binary: 0.1361 | HitGrouping: 0.1588 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1124 | Regression_eta: 0.0016 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.1441 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0583 | Regression_eta: 0.0383 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0624 | LR: 3.11e-04 | DataLoad Time: 0.0182s | Model Forward Time: 4.2965s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7557 | max_loss: 0.2439 | max_backward: 1.8923 | max_optimizer_step: 0.2623 | max_tensorboard_logging: 0.5765 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0168 | Memory: rss_MB: 5665.6 | uss_MB: 4533.9 | child_rss_MB: 6306.4 | allocated_MB: 496.5 | reserved_MB: 28598.0 | inactive_split_MB: 941.5 +rank7 [2026-10-29 18:04:39,053] INFO : Step 34000/50000 rank7 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4437 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0930 | Classification: 0.0875 | Classification_binary: 0.1606 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0611 | Classification_binary: 0.1282 | Regression_cos_phi: 0.0566 | Regression_energy: 0.0493 | Regression_eta: 0.0351 | Regression_pt: 0.0538 | Regression_sin_phi: 0.0642 | LR: 3.09e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3497s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0943 | max_forward: 0.8289 | max_loss: 0.4616 | max_backward: 2.0219 | max_optimizer_step: 0.1767 | max_tensorboard_logging: 0.4773 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0154 | Memory: rss_MB: 5666.0 | uss_MB: 4534.2 | child_rss_MB: 6326.4 | allocated_MB: 496.0 | reserved_MB: 28598.0 | inactive_split_MB: 1123.0 +rank7 [2026-10-29 18:04:39,969] INFO : Running validation on rank7 +rank7 [2026-10-29 18:04:39,969] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 18:04:39,969] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 18:05:30,627] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:05:31,101] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 18:05:54,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 18:05:55,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 18:05:56,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 18:05:57,784] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 18:06:07,793] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:06:08,708] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:06:09,213] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 18:06:25,410] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 18:06:26,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 18:06:27,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 18:06:28,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 18:06:38,281] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:06:39,030] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:06:39,482] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 18:06:54,943] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 18:06:55,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 18:06:56,598] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 18:06:57,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_34000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 18:07:07,450] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 18:12:12,107] INFO : Step 34100/50000 rank7 | Train Loss: 1.0240 | Train Loss Unweighted: 0.5104 | Train Loss Components: Auxiliary: 0.2153 | Cardinality: 0.1202 | Classification: 0.0979 | Classification_binary: 0.1668 | HitGrouping: 0.1551 | Regression_cos_phi: 0.0009 | Regression_energy: 0.1210 | Regression_eta: 0.0013 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0847 | Classification_binary: 0.2168 | Regression_cos_phi: 0.0952 | Regression_energy: 0.0721 | Regression_eta: 0.0667 | Regression_pt: 0.0780 | Regression_sin_phi: 0.0995 | LR: 3.06e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.5019s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0078 | max_forward: 0.9085 | max_loss: 0.1531 | max_backward: 2.1727 | max_optimizer_step: 0.2013 | max_tensorboard_logging: 0.4680 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0358 | Memory: rss_MB: 6900.4 | uss_MB: 5761.6 | child_rss_MB: 6326.4 | allocated_MB: 496.4 | reserved_MB: 24560.0 | inactive_split_MB: 1172.2 +rank7 [2026-10-29 18:17:11,666] INFO : Step 34200/50000 rank7 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4225 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.1030 | Classification: 0.0694 | Classification_binary: 0.1279 | HitGrouping: 0.1585 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1105 | Regression_eta: 0.0016 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1492 | Regression_cos_phi: 0.0639 | Regression_energy: 0.0492 | Regression_eta: 0.0607 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0541 | LR: 3.04e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.8722s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0065 | max_forward: 1.1401 | max_loss: 1.0887 | max_backward: 3.3565 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5402 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6897.3 | uss_MB: 5759.3 | child_rss_MB: 6328.4 | allocated_MB: 493.1 | reserved_MB: 27668.0 | inactive_split_MB: 842.0 +rank7 [2026-10-29 18:21:53,344] INFO : Step 34300/50000 rank7 | Train Loss: 0.8999 | Train Loss Unweighted: 0.4291 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.1046 | Classification: 0.0782 | Classification_binary: 0.1421 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0015 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0548 | Classification_binary: 0.1253 | Regression_cos_phi: 0.0368 | Regression_energy: 0.0652 | Regression_eta: 0.0390 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0577 | LR: 3.02e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.5619s | Phase Times: max_pre_step_barrier: 0.1089 | max_batch_to_device: 0.0070 | max_forward: 0.7738 | max_loss: 0.4754 | max_backward: 1.8888 | max_optimizer_step: 0.1995 | max_tensorboard_logging: 0.7281 | max_tensorboard_flush: 0.0114 | max_post_step_barrier: 0.0177 | Memory: rss_MB: 5721.4 | uss_MB: 4584.0 | child_rss_MB: 6315.8 | allocated_MB: 498.9 | reserved_MB: 29668.0 | inactive_split_MB: 777.1 +rank7 [2026-10-29 18:26:42,340] INFO : Step 34400/50000 rank7 | Train Loss: 0.9382 | Train Loss Unweighted: 0.4552 | Train Loss Components: Auxiliary: 0.2026 | Cardinality: 0.1181 | Classification: 0.0846 | Classification_binary: 0.1681 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1005 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0436 | Classification_binary: 0.1666 | Regression_cos_phi: 0.0419 | Regression_energy: 0.0493 | Regression_eta: 0.0385 | Regression_pt: 0.0543 | Regression_sin_phi: 0.0624 | LR: 2.99e-04 | DataLoad Time: 0.0174s | Model Forward Time: 4.6561s | Phase Times: max_pre_step_barrier: 0.1128 | max_batch_to_device: 0.0069 | max_forward: 0.8411 | max_loss: 0.2739 | max_backward: 2.1385 | max_optimizer_step: 0.1865 | max_tensorboard_logging: 0.5835 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0201 | Memory: rss_MB: 5734.9 | uss_MB: 4598.3 | child_rss_MB: 6315.2 | allocated_MB: 496.8 | reserved_MB: 29668.0 | inactive_split_MB: 858.8 +rank7 [2026-10-29 18:31:40,822] INFO : Step 34500/50000 rank7 | Train Loss: 0.9189 | Train Loss Unweighted: 0.4456 | Train Loss Components: Auxiliary: 0.1917 | Cardinality: 0.1063 | Classification: 0.0767 | Classification_binary: 0.1421 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1120 | Regression_eta: 0.0014 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0270 | Classification_binary: 0.1345 | Regression_cos_phi: 0.0537 | Regression_energy: 0.0475 | Regression_eta: 0.0437 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0400 | LR: 2.97e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.2124s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0063 | max_forward: 0.8471 | max_loss: 0.1476 | max_backward: 1.7430 | max_optimizer_step: 0.1840 | max_tensorboard_logging: 0.5872 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0224 | Memory: rss_MB: 5727.7 | uss_MB: 4595.8 | child_rss_MB: 6313.8 | allocated_MB: 494.3 | reserved_MB: 29668.0 | inactive_split_MB: 926.0 +rank7 [2026-10-29 18:36:27,181] INFO : Step 34600/50000 rank7 | Train Loss: 0.9180 | Train Loss Unweighted: 0.4340 | Train Loss Components: Auxiliary: 0.1857 | Cardinality: 0.1046 | Classification: 0.0794 | Classification_binary: 0.1340 | HitGrouping: 0.1586 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1081 | Regression_eta: 0.0016 | Regression_pt: 0.1094 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1269 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0565 | Regression_eta: 0.0392 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0334 | LR: 2.95e-04 | DataLoad Time: 0.0203s | Model Forward Time: 4.6946s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0170 | max_forward: 0.9895 | max_loss: 0.4622 | max_backward: 2.2541 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0139 | max_post_step_barrier: 0.0178 | Memory: rss_MB: 5731.7 | uss_MB: 4597.0 | child_rss_MB: 6311.8 | allocated_MB: 498.6 | reserved_MB: 29672.0 | inactive_split_MB: 765.1 +rank7 [2026-10-29 18:40:58,824] INFO : Step 34700/50000 rank7 | Train Loss: 0.9586 | Train Loss Unweighted: 0.4539 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1185 | Classification: 0.0862 | Classification_binary: 0.1624 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0015 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0601 | Classification_binary: 0.1867 | Regression_cos_phi: 0.0663 | Regression_energy: 0.0637 | Regression_eta: 0.0431 | Regression_pt: 0.0695 | Regression_sin_phi: 0.0773 | LR: 2.92e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3652s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0071 | max_forward: 0.7862 | max_loss: 0.2628 | max_backward: 1.9231 | max_optimizer_step: 0.2860 | max_tensorboard_logging: 0.4599 | max_tensorboard_flush: 0.0062 | max_post_step_barrier: 0.0180 | Memory: rss_MB: 5730.7 | uss_MB: 4596.4 | child_rss_MB: 6303.0 | allocated_MB: 499.0 | reserved_MB: 29672.0 | inactive_split_MB: 1074.3 +rank7 [2026-10-29 18:45:38,651] INFO : Step 34800/50000 rank7 | Train Loss: 0.9346 | Train Loss Unweighted: 0.4373 | Train Loss Components: Auxiliary: 0.1943 | Cardinality: 0.1276 | Classification: 0.0867 | Classification_binary: 0.1551 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0959 | Regression_eta: 0.0013 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1629 | Regression_cos_phi: 0.0542 | Regression_energy: 0.0486 | Regression_eta: 0.0267 | Regression_pt: 0.0535 | Regression_sin_phi: 0.0471 | LR: 2.90e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2173s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0770 | max_forward: 0.8404 | max_loss: 0.1556 | max_backward: 1.8035 | max_optimizer_step: 0.1843 | max_tensorboard_logging: 0.5351 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0211 | Memory: rss_MB: 5727.0 | uss_MB: 4594.2 | child_rss_MB: 6312.4 | allocated_MB: 497.4 | reserved_MB: 29672.0 | inactive_split_MB: 1113.6 +rank7 [2026-10-29 18:50:17,935] INFO : Step 34900/50000 rank7 | Train Loss: 0.8719 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.1154 | Classification: 0.0681 | Classification_binary: 0.1220 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1093 | Regression_eta: 0.0013 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0370 | Classification_binary: 0.1210 | Regression_cos_phi: 0.0738 | Regression_energy: 0.0449 | Regression_eta: 0.0781 | Regression_pt: 0.0472 | Regression_sin_phi: 0.0591 | LR: 2.88e-04 | DataLoad Time: 0.0189s | Model Forward Time: 6.1449s | Phase Times: max_pre_step_barrier: 0.0080 | max_batch_to_device: 0.0693 | max_forward: 1.2127 | max_loss: 0.2834 | max_backward: 3.0996 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.6125 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 5724.6 | uss_MB: 4576.6 | child_rss_MB: 6292.2 | allocated_MB: 495.3 | reserved_MB: 29672.0 | inactive_split_MB: 864.7 +rank7 [2026-10-29 18:55:19,194] INFO : Step 35000/50000 rank7 | Train Loss: 0.9157 | Train Loss Unweighted: 0.4220 | Train Loss Components: Auxiliary: 0.1897 | Cardinality: 0.1230 | Classification: 0.0771 | Classification_binary: 0.1455 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0980 | Regression_eta: 0.0014 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1151 | Regression_cos_phi: 0.0832 | Regression_energy: 0.0511 | Regression_eta: 0.0342 | Regression_pt: 0.0553 | Regression_sin_phi: 0.0542 | LR: 2.85e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5123s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0071 | max_forward: 0.8243 | max_loss: 0.4759 | max_backward: 2.1181 | max_optimizer_step: 0.1874 | max_tensorboard_logging: 0.5976 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5728.2 | uss_MB: 4598.3 | child_rss_MB: 6307.6 | allocated_MB: 498.3 | reserved_MB: 29672.0 | inactive_split_MB: 766.0 +rank7 [2026-10-29 18:55:20,127] INFO : Running validation on rank7 +rank7 [2026-10-29 18:55:20,128] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 18:55:20,128] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 18:56:11,604] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:56:12,431] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 18:56:36,669] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 18:56:37,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 18:56:38,685] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 18:56:39,650] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 18:56:49,660] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:56:49,876] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:56:50,323] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 18:57:06,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 18:57:07,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 18:57:08,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 18:57:09,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 18:57:19,555] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 18:57:20,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 18:57:20,899] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 18:57:36,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 18:57:37,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 18:57:38,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 18:57:38,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_35000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 18:57:49,005] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 19:02:56,670] INFO : Step 35100/50000 rank7 | Train Loss: 0.9622 | Train Loss Unweighted: 0.4512 | Train Loss Components: Auxiliary: 0.2038 | Cardinality: 0.1150 | Classification: 0.0857 | Classification_binary: 0.1724 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0948 | Regression_eta: 0.0014 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0706 | Classification_binary: 0.1988 | Regression_cos_phi: 0.0654 | Regression_energy: 0.0485 | Regression_eta: 0.0424 | Regression_pt: 0.0507 | Regression_sin_phi: 0.0672 | LR: 2.83e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.8775s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.1018 | max_forward: 0.8921 | max_loss: 0.1518 | max_backward: 2.0328 | max_optimizer_step: 0.2000 | max_tensorboard_logging: 0.8633 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0213 | Memory: rss_MB: 5815.2 | uss_MB: 4686.8 | child_rss_MB: 6305.0 | allocated_MB: 499.4 | reserved_MB: 23366.0 | inactive_split_MB: 1243.8 +rank7 [2026-10-29 19:07:43,277] INFO : Step 35200/50000 rank7 | Train Loss: 0.9331 | Train Loss Unweighted: 0.4385 | Train Loss Components: Auxiliary: 0.1900 | Cardinality: 0.1234 | Classification: 0.0814 | Classification_binary: 0.1441 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0422 | Classification_binary: 0.1268 | Regression_cos_phi: 0.0397 | Regression_energy: 0.0586 | Regression_eta: 0.0288 | Regression_pt: 0.0630 | Regression_sin_phi: 0.0345 | LR: 2.81e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.1459s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0577 | max_forward: 0.7693 | max_loss: 0.1423 | max_backward: 1.7209 | max_optimizer_step: 0.1882 | max_tensorboard_logging: 0.5574 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5822.3 | uss_MB: 4693.9 | child_rss_MB: 6304.7 | allocated_MB: 494.4 | reserved_MB: 25346.0 | inactive_split_MB: 1054.2 +rank7 [2026-10-29 19:12:30,669] INFO : Step 35300/50000 rank7 | Train Loss: 0.8693 | Train Loss Unweighted: 0.4075 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1105 | Classification: 0.0819 | Classification_binary: 0.1245 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0989 | Regression_eta: 0.0014 | Regression_pt: 0.0996 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0478 | Classification_binary: 0.1447 | Regression_cos_phi: 0.0418 | Regression_energy: 0.0590 | Regression_eta: 0.0287 | Regression_pt: 0.0648 | Regression_sin_phi: 0.0405 | LR: 2.79e-04 | DataLoad Time: 0.0182s | Model Forward Time: 6.8660s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0067 | max_forward: 1.2856 | max_loss: 0.4079 | max_backward: 4.4969 | max_optimizer_step: 0.1886 | max_tensorboard_logging: 0.5749 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0195 | Memory: rss_MB: 5814.5 | uss_MB: 4685.2 | child_rss_MB: 6307.8 | allocated_MB: 498.3 | reserved_MB: 25346.0 | inactive_split_MB: 1140.5 +rank7 [2026-10-29 19:17:14,808] INFO : Step 35400/50000 rank7 | Train Loss: 1.0147 | Train Loss Unweighted: 0.4888 | Train Loss Components: Auxiliary: 0.2130 | Cardinality: 0.1296 | Classification: 0.0908 | Classification_binary: 0.1718 | HitGrouping: 0.1530 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1110 | Regression_eta: 0.0014 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0551 | Classification_binary: 0.1685 | Regression_cos_phi: 0.0852 | Regression_energy: 0.0639 | Regression_eta: 0.0458 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0649 | LR: 2.76e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.3428s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0065 | max_forward: 0.8312 | max_loss: 0.1490 | max_backward: 2.0287 | max_optimizer_step: 0.1878 | max_tensorboard_logging: 0.4708 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5820.8 | uss_MB: 4696.7 | child_rss_MB: 6308.5 | allocated_MB: 497.8 | reserved_MB: 25346.0 | inactive_split_MB: 1132.0 +rank7 [2026-10-29 19:21:58,249] INFO : Step 35500/50000 rank7 | Train Loss: 0.9444 | Train Loss Unweighted: 0.4557 | Train Loss Components: Auxiliary: 0.1983 | Cardinality: 0.1201 | Classification: 0.0924 | Classification_binary: 0.1605 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0012 | Regression_pt: 0.1009 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.2068 | Regression_cos_phi: 0.0636 | Regression_energy: 0.0487 | Regression_eta: 0.0463 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0641 | LR: 2.74e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.3608s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7670 | max_loss: 0.2318 | max_backward: 1.8405 | max_optimizer_step: 0.2844 | max_tensorboard_logging: 0.5731 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5820.9 | uss_MB: 4694.9 | child_rss_MB: 6312.3 | allocated_MB: 497.5 | reserved_MB: 26942.0 | inactive_split_MB: 1130.5 +rank7 [2026-10-29 19:26:41,939] INFO : Step 35600/50000 rank7 | Train Loss: 0.9280 | Train Loss Unweighted: 0.4451 | Train Loss Components: Auxiliary: 0.1883 | Cardinality: 0.1095 | Classification: 0.0861 | Classification_binary: 0.1334 | HitGrouping: 0.1526 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1106 | Regression_eta: 0.0015 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0662 | Classification_binary: 0.1548 | Regression_cos_phi: 0.0674 | Regression_energy: 0.0487 | Regression_eta: 0.0670 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0728 | LR: 2.72e-04 | DataLoad Time: 0.0196s | Model Forward Time: 4.7397s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0066 | max_forward: 0.7460 | max_loss: 0.1440 | max_backward: 1.8206 | max_optimizer_step: 0.2911 | max_tensorboard_logging: 0.6036 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0454 | Memory: rss_MB: 5820.5 | uss_MB: 3919.7 | child_rss_MB: 6316.0 | allocated_MB: 496.1 | reserved_MB: 26942.0 | inactive_split_MB: 907.6 +rank7 [2026-10-29 19:31:29,665] INFO : Step 35700/50000 rank7 | Train Loss: 0.9116 | Train Loss Unweighted: 0.4301 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1148 | Classification: 0.0829 | Classification_binary: 0.1432 | HitGrouping: 0.1503 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0998 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0480 | Classification_binary: 0.1259 | Regression_cos_phi: 0.0476 | Regression_energy: 0.0428 | Regression_eta: 0.0209 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0547 | LR: 2.70e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.6312s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8312 | max_loss: 0.2568 | max_backward: 2.2622 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.5771 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0210 | Memory: rss_MB: 5823.2 | uss_MB: 4699.5 | child_rss_MB: 6323.6 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1241.0 +rank7 [2026-10-29 19:36:11,896] INFO : Step 35800/50000 rank7 | Train Loss: 1.0095 | Train Loss Unweighted: 0.4797 | Train Loss Components: Auxiliary: 0.2106 | Cardinality: 0.1252 | Classification: 0.0857 | Classification_binary: 0.1786 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1066 | Regression_eta: 0.0014 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0630 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0948 | Regression_energy: 0.0637 | Regression_eta: 0.0486 | Regression_pt: 0.0668 | Regression_sin_phi: 0.1054 | LR: 2.68e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.1755s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7644 | max_loss: 0.5090 | max_backward: 1.8855 | max_optimizer_step: 0.1867 | max_tensorboard_logging: 0.4861 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0230 | Memory: rss_MB: 5816.8 | uss_MB: 4694.8 | child_rss_MB: 6319.7 | allocated_MB: 499.0 | reserved_MB: 29024.0 | inactive_split_MB: 1129.0 +rank7 [2026-10-29 19:40:48,762] INFO : Step 35900/50000 rank7 | Train Loss: 0.9596 | Train Loss Unweighted: 0.4571 | Train Loss Components: Auxiliary: 0.2003 | Cardinality: 0.1173 | Classification: 0.0913 | Classification_binary: 0.1468 | HitGrouping: 0.1582 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1072 | Regression_eta: 0.0014 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0534 | Classification_binary: 0.1766 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0528 | Regression_eta: 0.0432 | Regression_pt: 0.0565 | Regression_sin_phi: 0.0728 | LR: 2.65e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.0321s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.7217 | max_loss: 0.1443 | max_backward: 1.6894 | max_optimizer_step: 0.1870 | max_tensorboard_logging: 0.4586 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0223 | Memory: rss_MB: 5816.9 | uss_MB: 4695.3 | child_rss_MB: 6319.9 | allocated_MB: 495.4 | reserved_MB: 29024.0 | inactive_split_MB: 934.6 +rank7 [2026-10-29 19:45:19,092] INFO : Step 36000/50000 rank7 | Train Loss: 0.9231 | Train Loss Unweighted: 0.4578 | Train Loss Components: Auxiliary: 0.1926 | Cardinality: 0.0912 | Classification: 0.0938 | Classification_binary: 0.1314 | HitGrouping: 0.1587 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1135 | Regression_eta: 0.0015 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0606 | Classification_binary: 0.1349 | Regression_cos_phi: 0.0551 | Regression_energy: 0.0800 | Regression_eta: 0.0294 | Regression_pt: 0.0867 | Regression_sin_phi: 0.0333 | LR: 2.63e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.3007s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0073 | max_forward: 0.7630 | max_loss: 0.4769 | max_backward: 1.8869 | max_optimizer_step: 0.2587 | max_tensorboard_logging: 0.5842 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0237 | Memory: rss_MB: 5815.2 | uss_MB: 4695.9 | child_rss_MB: 6324.3 | allocated_MB: 495.0 | reserved_MB: 29024.0 | inactive_split_MB: 937.5 +rank7 [2026-10-29 19:45:19,982] INFO : Running validation on rank7 +rank7 [2026-10-29 19:45:19,982] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 19:45:19,982] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 19:46:10,598] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:46:11,093] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 19:46:35,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 19:46:36,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 19:46:37,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 19:46:38,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 19:46:48,813] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 19:46:49,454] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:46:49,932] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 19:47:05,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 19:47:06,814] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 19:47:07,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 19:47:08,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 19:47:18,682] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 19:47:19,430] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 19:47:20,140] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 19:47:35,923] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 19:47:36,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 19:47:37,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 19:47:38,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_36000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 19:47:48,461] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 19:53:12,007] INFO : Step 36100/50000 rank7 | Train Loss: 0.9292 | Train Loss Unweighted: 0.4529 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1100 | Classification: 0.0906 | Classification_binary: 0.1490 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1047 | Regression_eta: 0.0013 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0758 | Classification_binary: 0.1464 | Regression_cos_phi: 0.0445 | Regression_energy: 0.0561 | Regression_eta: 0.0458 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0684 | LR: 2.61e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.7833s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7768 | max_loss: 0.2455 | max_backward: 1.8077 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 1.0653 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0229 | Memory: rss_MB: 5880.7 | uss_MB: 4756.1 | child_rss_MB: 6293.3 | allocated_MB: 499.2 | reserved_MB: 26012.0 | inactive_split_MB: 1080.2 +rank7 [2026-10-29 19:58:21,470] INFO : Step 36200/50000 rank7 | Train Loss: 0.9030 | Train Loss Unweighted: 0.4308 | Train Loss Components: Auxiliary: 0.1898 | Cardinality: 0.1163 | Classification: 0.0762 | Classification_binary: 0.1596 | HitGrouping: 0.1506 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1811 | Regression_cos_phi: 0.0619 | Regression_energy: 0.0551 | Regression_eta: 0.0259 | Regression_pt: 0.0590 | Regression_sin_phi: 0.0488 | LR: 2.59e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.5379s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0777 | max_forward: 0.8125 | max_loss: 0.1541 | max_backward: 1.8492 | max_optimizer_step: 0.1955 | max_tensorboard_logging: 0.7787 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 5884.8 | uss_MB: 4767.2 | child_rss_MB: 6290.6 | allocated_MB: 494.4 | reserved_MB: 26016.0 | inactive_split_MB: 1377.4 +rank7 [2026-10-29 20:03:20,658] INFO : Step 36300/50000 rank7 | Train Loss: 0.9576 | Train Loss Unweighted: 0.4463 | Train Loss Components: Auxiliary: 0.1895 | Cardinality: 0.1236 | Classification: 0.0876 | Classification_binary: 0.1347 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1100 | Regression_eta: 0.0017 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1421 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0595 | Regression_eta: 0.0444 | Regression_pt: 0.0656 | Regression_sin_phi: 0.0480 | LR: 2.57e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.4520s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0069 | max_forward: 0.8065 | max_loss: 0.1484 | max_backward: 1.9315 | max_optimizer_step: 0.1930 | max_tensorboard_logging: 0.6801 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0266 | Memory: rss_MB: 5878.7 | uss_MB: 4768.0 | child_rss_MB: 6273.7 | allocated_MB: 496.0 | reserved_MB: 28008.0 | inactive_split_MB: 1156.5 +rank7 [2026-10-29 20:08:10,865] INFO : Step 36400/50000 rank7 | Train Loss: 0.8761 | Train Loss Unweighted: 0.4147 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.0999 | Classification: 0.0796 | Classification_binary: 0.1372 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0981 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1167 | Regression_cos_phi: 0.0716 | Regression_energy: 0.0472 | Regression_eta: 0.0483 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0446 | LR: 2.55e-04 | DataLoad Time: 0.0219s | Model Forward Time: 5.8691s | Phase Times: max_pre_step_barrier: 0.0052 | max_batch_to_device: 0.0192 | max_forward: 1.1829 | max_loss: 0.3790 | max_backward: 3.0917 | max_optimizer_step: 0.1920 | max_tensorboard_logging: 0.5072 | max_tensorboard_flush: 0.0135 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 5880.4 | uss_MB: 4768.9 | child_rss_MB: 6270.8 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 845.2 +rank7 [2026-10-29 20:12:41,930] INFO : Step 36500/50000 rank7 | Train Loss: 0.9762 | Train Loss Unweighted: 0.4543 | Train Loss Components: Auxiliary: 0.2088 | Cardinality: 0.1239 | Classification: 0.0805 | Classification_binary: 0.1682 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1020 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1467 | Regression_cos_phi: 0.0380 | Regression_energy: 0.0516 | Regression_eta: 0.0339 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0475 | LR: 2.52e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.2755s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7782 | max_loss: 0.1554 | max_backward: 1.9378 | max_optimizer_step: 0.1892 | max_tensorboard_logging: 0.5103 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5877.6 | uss_MB: 4769.9 | child_rss_MB: 6273.0 | allocated_MB: 499.0 | reserved_MB: 30008.0 | inactive_split_MB: 913.8 +rank7 [2026-10-29 20:17:47,229] INFO : Step 36600/50000 rank7 | Train Loss: 0.8699 | Train Loss Unweighted: 0.4190 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.1138 | Classification: 0.0734 | Classification_binary: 0.1317 | HitGrouping: 0.1510 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1055 | Regression_eta: 0.0010 | Regression_pt: 0.1061 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1487 | Regression_cos_phi: 0.0389 | Regression_energy: 0.0459 | Regression_eta: 0.0242 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0424 | LR: 2.50e-04 | DataLoad Time: 0.0195s | Model Forward Time: 4.2966s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0071 | max_forward: 0.7561 | max_loss: 0.1454 | max_backward: 1.9017 | max_optimizer_step: 0.5115 | max_tensorboard_logging: 0.5672 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0274 | Memory: rss_MB: 5872.6 | uss_MB: 4766.8 | child_rss_MB: 6277.6 | allocated_MB: 494.4 | reserved_MB: 30008.0 | inactive_split_MB: 856.0 +rank7 [2026-10-29 20:22:56,452] INFO : Step 36700/50000 rank7 | Train Loss: 0.9065 | Train Loss Unweighted: 0.4327 | Train Loss Components: Auxiliary: 0.1802 | Cardinality: 0.1102 | Classification: 0.0849 | Classification_binary: 0.1262 | HitGrouping: 0.1553 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1087 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0464 | Classification_binary: 0.0976 | Regression_cos_phi: 0.1175 | Regression_energy: 0.0471 | Regression_eta: 0.0273 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0626 | LR: 2.48e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.0797s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.7169 | max_loss: 0.1507 | max_backward: 1.7104 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.5647 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0233 | Memory: rss_MB: 5871.4 | uss_MB: 4766.4 | child_rss_MB: 6270.7 | allocated_MB: 496.5 | reserved_MB: 30008.0 | inactive_split_MB: 1154.6 +rank7 [2026-10-29 20:28:04,049] INFO : Step 36800/50000 rank7 | Train Loss: 0.9068 | Train Loss Unweighted: 0.4227 | Train Loss Components: Auxiliary: 0.1925 | Cardinality: 0.1100 | Classification: 0.0739 | Classification_binary: 0.1571 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0945 | Regression_eta: 0.0013 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0528 | Classification_binary: 0.1624 | Regression_cos_phi: 0.0622 | Regression_energy: 0.0537 | Regression_eta: 0.0480 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0992 | LR: 2.46e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4672s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.8487 | max_loss: 0.1525 | max_backward: 2.1121 | max_optimizer_step: 0.1921 | max_tensorboard_logging: 0.4805 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0264 | Memory: rss_MB: 5877.1 | uss_MB: 4770.1 | child_rss_MB: 6274.6 | allocated_MB: 499.3 | reserved_MB: 30008.0 | inactive_split_MB: 939.6 +rank7 [2026-10-29 20:33:09,546] INFO : Step 36900/50000 rank7 | Train Loss: 0.8934 | Train Loss Unweighted: 0.4126 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1194 | Classification: 0.0721 | Classification_binary: 0.1507 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0013 | Regression_pt: 0.0940 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0473 | Classification_binary: 0.1598 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0472 | Regression_eta: 0.0232 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0394 | LR: 2.44e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.5616s | Phase Times: max_pre_step_barrier: 0.0139 | max_batch_to_device: 0.0068 | max_forward: 1.0975 | max_loss: 0.2241 | max_backward: 2.7278 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.6216 | max_tensorboard_flush: 0.0085 | max_post_step_barrier: 0.0282 | Memory: rss_MB: 5871.7 | uss_MB: 4750.7 | child_rss_MB: 6260.8 | allocated_MB: 498.9 | reserved_MB: 30008.0 | inactive_split_MB: 771.1 +rank7 [2026-10-29 20:38:05,960] INFO : Step 37000/50000 rank7 | Train Loss: 0.9040 | Train Loss Unweighted: 0.4182 | Train Loss Components: Auxiliary: 0.1842 | Cardinality: 0.1192 | Classification: 0.0807 | Classification_binary: 0.1318 | HitGrouping: 0.1571 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1024 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0454 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0492 | Regression_energy: 0.0533 | Regression_eta: 0.0341 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0614 | LR: 2.42e-04 | DataLoad Time: 0.0153s | Model Forward Time: 3.8519s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7055 | max_loss: 0.1505 | max_backward: 1.5664 | max_optimizer_step: 0.1900 | max_tensorboard_logging: 0.4972 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0197 | Memory: rss_MB: 5866.9 | uss_MB: 4763.2 | child_rss_MB: 6273.0 | allocated_MB: 493.9 | reserved_MB: 30008.0 | inactive_split_MB: 1169.8 +rank7 [2026-10-29 20:38:06,932] INFO : Running validation on rank7 +rank7 [2026-10-29 20:38:06,932] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 20:38:06,932] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 20:39:05,980] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:39:06,530] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 20:39:31,278] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 20:39:32,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 20:39:33,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 20:39:34,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 20:39:44,276] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 20:39:45,043] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:39:45,533] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 20:40:02,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 20:40:03,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 20:40:04,034] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 20:40:05,121] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 20:40:15,132] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 20:40:16,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 20:40:16,608] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 20:40:32,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 20:40:33,070] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 20:40:34,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 20:40:35,182] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_37000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 20:40:45,193] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 20:45:59,178] INFO : Step 37100/50000 rank7 | Train Loss: 0.9796 | Train Loss Unweighted: 0.4614 | Train Loss Components: Auxiliary: 0.2033 | Cardinality: 0.1255 | Classification: 0.0999 | Classification_binary: 0.1492 | HitGrouping: 0.1543 | Regression_cos_phi: 0.0008 | Regression_energy: 0.1041 | Regression_eta: 0.0013 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0666 | Classification_binary: 0.1319 | Regression_cos_phi: 0.0884 | Regression_energy: 0.0559 | Regression_eta: 0.0297 | Regression_pt: 0.0602 | Regression_sin_phi: 0.0680 | LR: 2.40e-04 | DataLoad Time: 0.0197s | Model Forward Time: 4.8458s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0061 | max_forward: 0.8553 | max_loss: 0.1461 | max_backward: 2.1723 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0262 | Memory: rss_MB: 5959.4 | uss_MB: 4852.0 | child_rss_MB: 6311.9 | allocated_MB: 499.3 | reserved_MB: 25668.0 | inactive_split_MB: 852.4 +rank7 [2026-10-29 20:50:39,578] INFO : Step 37200/50000 rank7 | Train Loss: 0.9288 | Train Loss Unweighted: 0.4455 | Train Loss Components: Auxiliary: 0.1928 | Cardinality: 0.1108 | Classification: 0.0836 | Classification_binary: 0.1643 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0015 | Regression_pt: 0.0980 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0468 | Classification_binary: 0.1627 | Regression_cos_phi: 0.0604 | Regression_energy: 0.0483 | Regression_eta: 0.0356 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0579 | LR: 2.38e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.6391s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0072 | max_forward: 0.8460 | max_loss: 0.2603 | max_backward: 2.0959 | max_optimizer_step: 0.2743 | max_tensorboard_logging: 0.5697 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0250 | Memory: rss_MB: 5966.0 | uss_MB: 4861.5 | child_rss_MB: 6312.1 | allocated_MB: 497.9 | reserved_MB: 25674.0 | inactive_split_MB: 846.0 +rank7 [2026-10-29 20:55:11,587] INFO : Step 37300/50000 rank7 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1275 | Classification: 0.0781 | Classification_binary: 0.1424 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0982 | Regression_eta: 0.0013 | Regression_pt: 0.0994 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0555 | Classification_binary: 0.1726 | Regression_cos_phi: 0.0956 | Regression_energy: 0.0481 | Regression_eta: 0.0446 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0580 | LR: 2.36e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.0213s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7653 | max_loss: 0.1472 | max_backward: 1.7105 | max_optimizer_step: 0.1908 | max_tensorboard_logging: 0.4778 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0692 | Memory: rss_MB: 5962.2 | uss_MB: 4859.4 | child_rss_MB: 6302.2 | allocated_MB: 497.3 | reserved_MB: 27616.0 | inactive_split_MB: 974.4 +rank7 [2026-10-29 21:00:09,947] INFO : Step 37400/50000 rank7 | Train Loss: 0.8836 | Train Loss Unweighted: 0.4228 | Train Loss Components: Auxiliary: 0.1825 | Cardinality: 0.0983 | Classification: 0.0851 | Classification_binary: 0.1289 | HitGrouping: 0.1554 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1025 | Regression_eta: 0.0015 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0470 | Classification_binary: 0.1145 | Regression_cos_phi: 0.0495 | Regression_energy: 0.0758 | Regression_eta: 0.0263 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0839 | LR: 2.34e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.5375s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0065 | max_forward: 0.7611 | max_loss: 0.1956 | max_backward: 1.9276 | max_optimizer_step: 0.2590 | max_tensorboard_logging: 0.7505 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0294 | Memory: rss_MB: 5960.0 | uss_MB: 4860.1 | child_rss_MB: 6301.8 | allocated_MB: 495.0 | reserved_MB: 27690.0 | inactive_split_MB: 915.5 +rank7 [2026-10-29 21:05:01,763] INFO : Step 37500/50000 rank7 | Train Loss: 0.9412 | Train Loss Unweighted: 0.4428 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1260 | Classification: 0.0804 | Classification_binary: 0.1577 | HitGrouping: 0.1472 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1005 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0324 | Classification_binary: 0.1409 | Regression_cos_phi: 0.0475 | Regression_energy: 0.0595 | Regression_eta: 0.0328 | Regression_pt: 0.0646 | Regression_sin_phi: 0.0437 | LR: 2.32e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.4451s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.9030 | max_loss: 0.1494 | max_backward: 1.9871 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.4738 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0273 | Memory: rss_MB: 5938.4 | uss_MB: 4859.4 | child_rss_MB: 6224.4 | allocated_MB: 496.1 | reserved_MB: 27690.0 | inactive_split_MB: 923.5 +rank7 [2026-10-29 21:09:54,197] INFO : Step 37600/50000 rank7 | Train Loss: 0.9091 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1882 | Cardinality: 0.0995 | Classification: 0.0683 | Classification_binary: 0.1534 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0008 | Regression_energy: 0.0957 | Regression_eta: 0.0015 | Regression_pt: 0.0967 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0344 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0498 | Regression_energy: 0.0438 | Regression_eta: 0.0529 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0561 | LR: 2.30e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3495s | Phase Times: max_pre_step_barrier: 0.0034 | max_batch_to_device: 0.0070 | max_forward: 0.8280 | max_loss: 0.1484 | max_backward: 1.8328 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.5743 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0259 | Memory: rss_MB: 5933.4 | uss_MB: 4856.7 | child_rss_MB: 6230.2 | allocated_MB: 497.1 | reserved_MB: 27690.0 | inactive_split_MB: 927.5 +rank7 [2026-10-29 21:14:45,920] INFO : Step 37700/50000 rank7 | Train Loss: 0.8328 | Train Loss Unweighted: 0.3995 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.0864 | Classification: 0.0813 | Classification_binary: 0.1231 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0012 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1447 | Regression_cos_phi: 0.0686 | Regression_energy: 0.0447 | Regression_eta: 0.0352 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0544 | LR: 2.28e-04 | DataLoad Time: 0.0181s | Model Forward Time: 4.4199s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0064 | max_forward: 0.7582 | max_loss: 0.2486 | max_backward: 1.9197 | max_optimizer_step: 0.3005 | max_tensorboard_logging: 0.5767 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0311 | Memory: rss_MB: 5802.8 | uss_MB: 4857.6 | child_rss_MB: 5767.0 | allocated_MB: 495.2 | reserved_MB: 29766.0 | inactive_split_MB: 925.5 +rank7 [2026-10-29 21:19:50,095] INFO : Step 37800/50000 rank7 | Train Loss: 0.8717 | Train Loss Unweighted: 0.4164 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1057 | Classification: 0.0822 | Classification_binary: 0.1363 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0014 | Regression_pt: 0.0987 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0349 | Classification_binary: 0.1410 | Regression_cos_phi: 0.0778 | Regression_energy: 0.0489 | Regression_eta: 0.0386 | Regression_pt: 0.0559 | Regression_sin_phi: 0.0429 | LR: 2.26e-04 | DataLoad Time: 0.0193s | Model Forward Time: 4.6251s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 1.2105 | max_loss: 0.2449 | max_backward: 2.0574 | max_optimizer_step: 0.1927 | max_tensorboard_logging: 0.4891 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0499 | Memory: rss_MB: 5800.5 | uss_MB: 4856.8 | child_rss_MB: 5763.9 | allocated_MB: 498.3 | reserved_MB: 29766.0 | inactive_split_MB: 930.0 +rank7 [2026-10-29 21:24:49,255] INFO : Step 37900/50000 rank7 | Train Loss: 0.9843 | Train Loss Unweighted: 0.4684 | Train Loss Components: Auxiliary: 0.2089 | Cardinality: 0.1353 | Classification: 0.0912 | Classification_binary: 0.1705 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1017 | Regression_eta: 0.0014 | Regression_pt: 0.1021 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0541 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0803 | Regression_energy: 0.0580 | Regression_eta: 0.0472 | Regression_pt: 0.0633 | Regression_sin_phi: 0.0755 | LR: 2.24e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.2096s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0297 | max_forward: 0.8081 | max_loss: 0.1547 | max_backward: 2.6708 | max_optimizer_step: 0.1942 | max_tensorboard_logging: 0.5096 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 5782.9 | uss_MB: 4859.7 | child_rss_MB: 5732.7 | allocated_MB: 495.6 | reserved_MB: 29766.0 | inactive_split_MB: 912.0 +rank7 [2026-10-29 21:29:44,507] INFO : Step 38000/50000 rank7 | Train Loss: 0.8556 | Train Loss Unweighted: 0.4002 | Train Loss Components: Auxiliary: 0.1833 | Cardinality: 0.1090 | Classification: 0.0730 | Classification_binary: 0.1293 | HitGrouping: 0.1496 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0972 | Regression_eta: 0.0011 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1327 | Regression_cos_phi: 0.0433 | Regression_energy: 0.0552 | Regression_eta: 0.0230 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0405 | LR: 2.22e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.1654s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.7330 | max_loss: 0.2284 | max_backward: 1.8082 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.4851 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0576 | Memory: rss_MB: 5778.8 | uss_MB: 4857.4 | child_rss_MB: 5727.0 | allocated_MB: 494.5 | reserved_MB: 29766.0 | inactive_split_MB: 847.3 +rank7 [2026-10-29 21:29:45,441] INFO : Running validation on rank7 +rank7 [2026-10-29 21:29:45,441] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 21:29:45,442] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 21:30:41,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:30:42,096] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 21:31:09,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 21:31:10,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 21:31:11,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 21:31:12,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 21:31:22,482] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 21:31:23,195] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:31:23,629] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 21:31:39,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 21:31:40,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 21:31:41,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 21:31:42,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 21:31:52,178] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 21:31:52,734] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 21:31:53,234] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 21:32:08,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 21:32:09,759] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 21:32:10,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 21:32:11,415] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_38000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 21:32:21,425] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 21:37:39,070] INFO : Step 38100/50000 rank7 | Train Loss: 0.9020 | Train Loss Unweighted: 0.4337 | Train Loss Components: Auxiliary: 0.1907 | Cardinality: 0.1231 | Classification: 0.0888 | Classification_binary: 0.1339 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1038 | Regression_eta: 0.0014 | Regression_pt: 0.1045 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0187 | Classification_binary: 0.1233 | Regression_cos_phi: 0.0342 | Regression_energy: 0.0395 | Regression_eta: 0.0202 | Regression_pt: 0.0434 | Regression_sin_phi: 0.0357 | LR: 2.20e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.8732s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0963 | max_forward: 0.8445 | max_loss: 0.1406 | max_backward: 1.9197 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.9697 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0494 | Memory: rss_MB: 7028.7 | uss_MB: 6098.3 | child_rss_MB: 5734.5 | allocated_MB: 496.0 | reserved_MB: 27516.0 | inactive_split_MB: 970.5 +rank7 [2026-10-29 21:42:37,687] INFO : Step 38200/50000 rank7 | Train Loss: 0.9174 | Train Loss Unweighted: 0.4351 | Train Loss Components: Auxiliary: 0.1906 | Cardinality: 0.0971 | Classification: 0.0888 | Classification_binary: 0.1526 | HitGrouping: 0.1454 | Regression_cos_phi: 0.0009 | Regression_energy: 0.0948 | Regression_eta: 0.0013 | Regression_pt: 0.0960 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1486 | Regression_cos_phi: 0.0513 | Regression_energy: 0.0411 | Regression_eta: 0.0323 | Regression_pt: 0.0446 | Regression_sin_phi: 0.0651 | LR: 2.18e-04 | DataLoad Time: 0.0176s | Model Forward Time: 6.4463s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.8480 | max_loss: 0.2651 | max_backward: 3.1905 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.9151 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0445 | Memory: rss_MB: 5881.2 | uss_MB: 4342.7 | child_rss_MB: 5733.1 | allocated_MB: 499.7 | reserved_MB: 27516.0 | inactive_split_MB: 848.5 +rank7 [2026-10-29 21:47:30,432] INFO : Step 38300/50000 rank7 | Train Loss: 0.9342 | Train Loss Unweighted: 0.4594 | Train Loss Components: Auxiliary: 0.1998 | Cardinality: 0.1099 | Classification: 0.0869 | Classification_binary: 0.1470 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1109 | Regression_eta: 0.0013 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1249 | Regression_cos_phi: 0.0269 | Regression_energy: 0.0486 | Regression_eta: 0.0248 | Regression_pt: 0.0525 | Regression_sin_phi: 0.0552 | LR: 2.16e-04 | DataLoad Time: 0.0191s | Model Forward Time: 4.6869s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0664 | max_forward: 0.8187 | max_loss: 0.1464 | max_backward: 1.8154 | max_optimizer_step: 0.1981 | max_tensorboard_logging: 0.9517 | max_tensorboard_flush: 0.0070 | max_post_step_barrier: 0.0300 | Memory: rss_MB: 5875.2 | uss_MB: 4949.5 | child_rss_MB: 5725.9 | allocated_MB: 495.5 | reserved_MB: 27516.0 | inactive_split_MB: 1085.5 +rank7 [2026-10-29 21:52:14,428] INFO : Step 38400/50000 rank7 | Train Loss: 0.8281 | Train Loss Unweighted: 0.4036 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.0915 | Classification: 0.0723 | Classification_binary: 0.1160 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1069 | Regression_eta: 0.0012 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0391 | Classification_binary: 0.1192 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0410 | Regression_eta: 0.0260 | Regression_pt: 0.0444 | Regression_sin_phi: 0.0669 | LR: 2.14e-04 | DataLoad Time: 0.0201s | Model Forward Time: 4.7619s | Phase Times: max_pre_step_barrier: 0.0868 | max_batch_to_device: 0.0062 | max_forward: 0.9937 | max_loss: 0.2500 | max_backward: 1.9690 | max_optimizer_step: 0.1974 | max_tensorboard_logging: 0.9198 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0217 | Memory: rss_MB: 5872.2 | uss_MB: 4950.8 | child_rss_MB: 5730.9 | allocated_MB: 496.1 | reserved_MB: 27516.0 | inactive_split_MB: 972.6 +rank7 [2026-10-29 21:57:09,316] INFO : Step 38500/50000 rank7 | Train Loss: 0.9386 | Train Loss Unweighted: 0.4484 | Train Loss Components: Auxiliary: 0.1967 | Cardinality: 0.1248 | Classification: 0.1044 | Classification_binary: 0.1389 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0014 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0731 | Classification_binary: 0.1376 | Regression_cos_phi: 0.0509 | Regression_energy: 0.0483 | Regression_eta: 0.0460 | Regression_pt: 0.0513 | Regression_sin_phi: 0.0545 | LR: 2.12e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2033s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0064 | max_forward: 0.7200 | max_loss: 0.1386 | max_backward: 1.5323 | max_optimizer_step: 0.1958 | max_tensorboard_logging: 0.8345 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0283 | Memory: rss_MB: 5868.3 | uss_MB: 4952.0 | child_rss_MB: 5707.9 | allocated_MB: 496.0 | reserved_MB: 27520.0 | inactive_split_MB: 776.0 +rank7 [2026-10-29 22:01:43,605] INFO : Step 38600/50000 rank7 | Train Loss: 0.9227 | Train Loss Unweighted: 0.4436 | Train Loss Components: Auxiliary: 0.1984 | Cardinality: 0.1056 | Classification: 0.0788 | Classification_binary: 0.1660 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0979 | Regression_eta: 0.0013 | Regression_pt: 0.0982 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0503 | Classification_binary: 0.1703 | Regression_cos_phi: 0.0657 | Regression_energy: 0.0522 | Regression_eta: 0.0463 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0634 | LR: 2.11e-04 | DataLoad Time: 0.0150s | Model Forward Time: 4.4273s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0075 | max_forward: 0.8785 | max_loss: 0.1473 | max_backward: 2.0276 | max_optimizer_step: 0.1941 | max_tensorboard_logging: 0.4789 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0285 | Memory: rss_MB: 5861.9 | uss_MB: 4951.4 | child_rss_MB: 5677.1 | allocated_MB: 499.0 | reserved_MB: 29592.0 | inactive_split_MB: 827.5 +rank7 [2026-10-29 22:06:19,437] INFO : Step 38700/50000 rank7 | Train Loss: 0.8153 | Train Loss Unweighted: 0.3795 | Train Loss Components: Auxiliary: 0.1668 | Cardinality: 0.0947 | Classification: 0.0722 | Classification_binary: 0.1256 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0893 | Regression_eta: 0.0013 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1280 | Regression_cos_phi: 0.0469 | Regression_energy: 0.0505 | Regression_eta: 0.0276 | Regression_pt: 0.0547 | Regression_sin_phi: 0.0576 | LR: 2.09e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.2102s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7251 | max_loss: 0.2433 | max_backward: 1.8091 | max_optimizer_step: 0.1965 | max_tensorboard_logging: 0.4744 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0277 | Memory: rss_MB: 5858.3 | uss_MB: 4950.6 | child_rss_MB: 5664.7 | allocated_MB: 493.1 | reserved_MB: 29592.0 | inactive_split_MB: 1080.0 +rank7 [2026-10-29 22:11:06,082] INFO : Step 38800/50000 rank7 | Train Loss: 0.8386 | Train Loss Unweighted: 0.4077 | Train Loss Components: Auxiliary: 0.1776 | Cardinality: 0.0905 | Classification: 0.0849 | Classification_binary: 0.1308 | HitGrouping: 0.1475 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0012 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0784 | Classification_binary: 0.1172 | Regression_cos_phi: 0.0448 | Regression_energy: 0.0483 | Regression_eta: 0.0399 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0536 | LR: 2.07e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.1586s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0074 | max_forward: 0.7346 | max_loss: 0.1426 | max_backward: 1.8320 | max_optimizer_step: 0.1952 | max_tensorboard_logging: 0.5128 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5857.6 | uss_MB: 4949.0 | child_rss_MB: 5646.6 | allocated_MB: 497.9 | reserved_MB: 29592.0 | inactive_split_MB: 1086.4 +rank7 [2026-10-29 22:15:54,682] INFO : Step 38900/50000 rank7 | Train Loss: 0.9077 | Train Loss Unweighted: 0.4219 | Train Loss Components: Auxiliary: 0.1916 | Cardinality: 0.1035 | Classification: 0.0839 | Classification_binary: 0.1479 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0934 | Regression_eta: 0.0014 | Regression_pt: 0.0938 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0488 | Classification_binary: 0.1370 | Regression_cos_phi: 0.0508 | Regression_energy: 0.0467 | Regression_eta: 0.0245 | Regression_pt: 0.0502 | Regression_sin_phi: 0.0410 | LR: 2.05e-04 | DataLoad Time: 0.0217s | Model Forward Time: 5.8762s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0144 | max_forward: 2.1610 | max_loss: 0.2966 | max_backward: 2.8203 | max_optimizer_step: 0.1951 | max_tensorboard_logging: 0.4876 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0567 | Memory: rss_MB: 5858.5 | uss_MB: 4951.7 | child_rss_MB: 5647.2 | allocated_MB: 500.0 | reserved_MB: 29592.0 | inactive_split_MB: 849.6 +rank7 [2026-10-29 22:20:36,724] INFO : Step 39000/50000 rank7 | Train Loss: 0.8505 | Train Loss Unweighted: 0.4196 | Train Loss Components: Auxiliary: 0.1813 | Cardinality: 0.0941 | Classification: 0.0808 | Classification_binary: 0.1368 | HitGrouping: 0.1428 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0993 | Regression_eta: 0.0015 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0558 | Classification_binary: 0.1496 | Regression_cos_phi: 0.0400 | Regression_energy: 0.0460 | Regression_eta: 0.0342 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0478 | LR: 2.03e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.2851s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0061 | max_forward: 0.7392 | max_loss: 0.2445 | max_backward: 1.8584 | max_optimizer_step: 0.1956 | max_tensorboard_logging: 0.5886 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5844.6 | uss_MB: 4939.8 | child_rss_MB: 5638.4 | allocated_MB: 496.8 | reserved_MB: 29592.0 | inactive_split_MB: 1086.0 +rank7 [2026-10-29 22:20:37,630] INFO : Running validation on rank7 +rank7 [2026-10-29 22:20:37,631] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 22:20:37,631] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 22:21:31,240] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:21:31,759] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 22:21:58,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 22:21:59,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 22:22:00,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 22:22:01,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 22:22:11,904] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 22:22:12,680] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:22:13,174] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 22:22:29,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 22:22:30,513] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 22:22:31,615] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 22:22:32,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 22:22:42,658] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-29 22:22:43,329] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 22:22:43,799] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 22:22:59,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 22:23:00,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 22:23:00,879] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 22:23:01,752] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_39000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 22:23:11,762] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-29 22:28:30,320] INFO : Step 39100/50000 rank7 | Train Loss: 0.8773 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1110 | Classification: 0.0839 | Classification_binary: 0.1198 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1052 | Regression_eta: 0.0014 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0466 | Classification_binary: 0.1502 | Regression_cos_phi: 0.0791 | Regression_energy: 0.0452 | Regression_eta: 0.0590 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0687 | LR: 2.01e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.5500s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0065 | max_forward: 0.7800 | max_loss: 0.2457 | max_backward: 1.9719 | max_optimizer_step: 0.3248 | max_tensorboard_logging: 0.5839 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0462 | Memory: rss_MB: 7010.9 | uss_MB: 6103.0 | child_rss_MB: 5608.2 | allocated_MB: 497.5 | reserved_MB: 24374.0 | inactive_split_MB: 923.6 +rank7 [2026-10-29 22:33:29,806] INFO : Step 39200/50000 rank7 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4242 | Train Loss Components: Auxiliary: 0.1863 | Cardinality: 0.1210 | Classification: 0.0912 | Classification_binary: 0.1382 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0956 | Regression_eta: 0.0013 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1208 | Regression_cos_phi: 0.0497 | Regression_energy: 0.0583 | Regression_eta: 0.0586 | Regression_pt: 0.0647 | Regression_sin_phi: 0.0458 | LR: 2.00e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.2580s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.0324 | max_forward: 0.7817 | max_loss: 0.1452 | max_backward: 1.9286 | max_optimizer_step: 0.1982 | max_tensorboard_logging: 0.5047 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0306 | Memory: rss_MB: 5940.0 | uss_MB: 5036.4 | child_rss_MB: 5609.8 | allocated_MB: 497.9 | reserved_MB: 26338.0 | inactive_split_MB: 912.4 +rank7 [2026-10-29 22:38:28,992] INFO : Step 39300/50000 rank7 | Train Loss: 0.9298 | Train Loss Unweighted: 0.4410 | Train Loss Components: Auxiliary: 0.2012 | Cardinality: 0.1080 | Classification: 0.0882 | Classification_binary: 0.1594 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0951 | Regression_eta: 0.0013 | Regression_pt: 0.0956 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0678 | Classification_binary: 0.1510 | Regression_cos_phi: 0.0415 | Regression_energy: 0.0393 | Regression_eta: 0.0292 | Regression_pt: 0.0418 | Regression_sin_phi: 0.0515 | LR: 1.98e-04 | DataLoad Time: 0.0160s | Model Forward Time: 5.1934s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0066 | max_forward: 0.7922 | max_loss: 0.2572 | max_backward: 1.8558 | max_optimizer_step: 0.2917 | max_tensorboard_logging: 0.8022 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0481 | Memory: rss_MB: 5927.1 | uss_MB: 5036.1 | child_rss_MB: 5613.3 | allocated_MB: 499.4 | reserved_MB: 26338.0 | inactive_split_MB: 1408.0 +rank7 [2026-10-29 22:43:21,462] INFO : Step 39400/50000 rank7 | Train Loss: 0.8421 | Train Loss Unweighted: 0.3964 | Train Loss Components: Auxiliary: 0.1755 | Cardinality: 0.1006 | Classification: 0.0679 | Classification_binary: 0.1346 | HitGrouping: 0.1534 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0012 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0498 | Classification_binary: 0.1630 | Regression_cos_phi: 0.0649 | Regression_energy: 0.0497 | Regression_eta: 0.0471 | Regression_pt: 0.0540 | Regression_sin_phi: 0.0617 | LR: 1.96e-04 | DataLoad Time: 0.0187s | Model Forward Time: 6.2903s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.1084 | max_forward: 1.1740 | max_loss: 0.3046 | max_backward: 2.6778 | max_optimizer_step: 0.2816 | max_tensorboard_logging: 0.8613 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.0335 | Memory: rss_MB: 5928.1 | uss_MB: 4351.2 | child_rss_MB: 5609.1 | allocated_MB: 495.0 | reserved_MB: 26338.0 | inactive_split_MB: 1095.6 +rank7 [2026-10-29 22:48:17,290] INFO : Step 39500/50000 rank7 | Train Loss: 0.8539 | Train Loss Unweighted: 0.4179 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0879 | Classification: 0.0889 | Classification_binary: 0.1243 | HitGrouping: 0.1565 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1008 | Regression_eta: 0.0013 | Regression_pt: 0.1014 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0604 | Classification_binary: 0.1206 | Regression_cos_phi: 0.0651 | Regression_energy: 0.0630 | Regression_eta: 0.0313 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0693 | LR: 1.94e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.6152s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.7373 | max_loss: 0.1406 | max_backward: 1.8719 | max_optimizer_step: 0.1999 | max_tensorboard_logging: 0.8502 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0296 | Memory: rss_MB: 5922.7 | uss_MB: 5037.5 | child_rss_MB: 5598.5 | allocated_MB: 492.8 | reserved_MB: 28318.0 | inactive_split_MB: 1078.3 +rank7 [2026-10-29 22:53:41,207] INFO : Step 39600/50000 rank7 | Train Loss: 0.9371 | Train Loss Unweighted: 0.4517 | Train Loss Components: Auxiliary: 0.2040 | Cardinality: 0.1161 | Classification: 0.0813 | Classification_binary: 0.1596 | HitGrouping: 0.1484 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1039 | Regression_eta: 0.0013 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1532 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0554 | Regression_eta: 0.0320 | Regression_pt: 0.0606 | Regression_sin_phi: 0.0496 | LR: 1.93e-04 | DataLoad Time: 0.0191s | Model Forward Time: 5.3240s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0173 | max_forward: 1.4879 | max_loss: 0.2520 | max_backward: 1.9863 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.8346 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0287 | Memory: rss_MB: 5921.1 | uss_MB: 5038.4 | child_rss_MB: 5602.9 | allocated_MB: 496.0 | reserved_MB: 28318.0 | inactive_split_MB: 858.4 +rank7 [2026-10-29 22:58:48,059] INFO : Step 39700/50000 rank7 | Train Loss: 0.9204 | Train Loss Unweighted: 0.4427 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1101 | Classification: 0.0841 | Classification_binary: 0.1567 | HitGrouping: 0.1461 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0997 | Regression_eta: 0.0013 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0521 | Classification_binary: 0.1640 | Regression_cos_phi: 0.0698 | Regression_energy: 0.0785 | Regression_eta: 0.0323 | Regression_pt: 0.0847 | Regression_sin_phi: 0.0544 | LR: 1.91e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.2754s | Phase Times: max_pre_step_barrier: 0.0038 | max_batch_to_device: 0.0067 | max_forward: 0.7571 | max_loss: 0.4946 | max_backward: 1.9147 | max_optimizer_step: 0.1966 | max_tensorboard_logging: 0.4829 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0624 | Memory: rss_MB: 5909.3 | uss_MB: 5037.4 | child_rss_MB: 5602.2 | allocated_MB: 497.0 | reserved_MB: 30324.0 | inactive_split_MB: 1175.5 +rank7 [2026-10-29 23:04:38,178] INFO : Step 39800/50000 rank7 | Train Loss: 0.8155 | Train Loss Unweighted: 0.3780 | Train Loss Components: Auxiliary: 0.1674 | Cardinality: 0.0963 | Classification: 0.0643 | Classification_binary: 0.1275 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0400 | Classification_binary: 0.1574 | Regression_cos_phi: 0.0359 | Regression_energy: 0.0579 | Regression_eta: 0.0272 | Regression_pt: 0.0628 | Regression_sin_phi: 0.0425 | LR: 1.89e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.0213s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0062 | max_forward: 0.7197 | max_loss: 0.1489 | max_backward: 1.6690 | max_optimizer_step: 0.2003 | max_tensorboard_logging: 0.4760 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.1068 | Memory: rss_MB: 5897.7 | uss_MB: 5034.7 | child_rss_MB: 5594.3 | allocated_MB: 494.4 | reserved_MB: 30324.0 | inactive_split_MB: 897.3 +rank7 [2026-10-29 23:09:43,446] INFO : Step 39900/50000 rank7 | Train Loss: 0.8844 | Train Loss Unweighted: 0.3998 | Train Loss Components: Auxiliary: 0.1871 | Cardinality: 0.1077 | Classification: 0.0758 | Classification_binary: 0.1443 | HitGrouping: 0.1458 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0883 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1466 | Regression_cos_phi: 0.0341 | Regression_energy: 0.0478 | Regression_eta: 0.0260 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0564 | LR: 1.88e-04 | DataLoad Time: 0.0121s | Model Forward Time: 4.9670s | Phase Times: max_pre_step_barrier: 0.0268 | max_batch_to_device: 0.0727 | max_forward: 1.7016 | max_loss: 0.4968 | max_backward: 2.0829 | max_optimizer_step: 0.1960 | max_tensorboard_logging: 0.4776 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0288 | Memory: rss_MB: 5898.5 | uss_MB: 5037.5 | child_rss_MB: 5601.6 | allocated_MB: 496.1 | reserved_MB: 30324.0 | inactive_split_MB: 1077.6 +rank7 [2026-10-29 23:14:38,166] INFO : Step 40000/50000 rank7 | Train Loss: 0.9365 | Train Loss Unweighted: 0.4585 | Train Loss Components: Auxiliary: 0.2047 | Cardinality: 0.1170 | Classification: 0.0871 | Classification_binary: 0.1550 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1061 | Regression_eta: 0.0013 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0697 | Classification_binary: 0.1786 | Regression_cos_phi: 0.0861 | Regression_energy: 0.0777 | Regression_eta: 0.0616 | Regression_pt: 0.0855 | Regression_sin_phi: 0.0661 | LR: 1.86e-04 | DataLoad Time: 0.0143s | Model Forward Time: 5.4137s | Phase Times: max_pre_step_barrier: 0.1084 | max_batch_to_device: 0.0070 | max_forward: 1.1725 | max_loss: 0.2555 | max_backward: 2.4297 | max_optimizer_step: 0.2969 | max_tensorboard_logging: 0.6818 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0319 | Memory: rss_MB: 5889.3 | uss_MB: 5028.2 | child_rss_MB: 5597.1 | allocated_MB: 501.7 | reserved_MB: 30324.0 | inactive_split_MB: 1368.0 +rank7 [2026-10-29 23:14:39,119] INFO : Running validation on rank7 +rank7 [2026-10-29 23:14:39,119] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-29 23:14:39,119] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-29 23:15:31,812] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:15:32,369] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-29 23:16:03,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-29 23:16:04,466] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-29 23:16:05,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-29 23:16:07,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-29 23:16:17,334] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-29 23:16:28,295] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:16:28,871] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-29 23:16:44,977] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-29 23:16:45,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-29 23:16:46,911] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-29 23:16:48,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-29 23:16:58,021] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-29 23:16:58,535] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-29 23:16:59,026] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-29 23:17:14,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-29 23:17:15,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-29 23:17:16,002] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-29 23:17:16,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_40000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-29 23:17:26,885] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-29 23:22:41,683] INFO : Step 40100/50000 rank7 | Train Loss: 0.8115 | Train Loss Unweighted: 0.3794 | Train Loss Components: Auxiliary: 0.1701 | Cardinality: 0.1028 | Classification: 0.0708 | Classification_binary: 0.1239 | HitGrouping: 0.1460 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0909 | Regression_eta: 0.0012 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1470 | Regression_cos_phi: 0.0514 | Regression_energy: 0.0554 | Regression_eta: 0.0341 | Regression_pt: 0.0600 | Regression_sin_phi: 0.0414 | LR: 1.84e-04 | DataLoad Time: 0.0172s | Model Forward Time: 5.0765s | Phase Times: max_pre_step_barrier: 0.0035 | max_batch_to_device: 0.1076 | max_forward: 0.8426 | max_loss: 0.1437 | max_backward: 1.8847 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 1.0136 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 7273.7 | uss_MB: 5477.4 | child_rss_MB: 5617.3 | allocated_MB: 493.6 | reserved_MB: 28004.0 | inactive_split_MB: 975.8 +rank7 [2026-10-29 23:27:53,247] INFO : Step 40200/50000 rank7 | Train Loss: 0.8264 | Train Loss Unweighted: 0.3775 | Train Loss Components: Auxiliary: 0.1679 | Cardinality: 0.0984 | Classification: 0.0703 | Classification_binary: 0.1210 | HitGrouping: 0.1474 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0913 | Regression_eta: 0.0013 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1024 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0550 | Regression_eta: 0.0268 | Regression_pt: 0.0601 | Regression_sin_phi: 0.0452 | LR: 1.83e-04 | DataLoad Time: 0.0186s | Model Forward Time: 4.8939s | Phase Times: max_pre_step_barrier: 0.0270 | max_batch_to_device: 0.0062 | max_forward: 0.8324 | max_loss: 0.1802 | max_backward: 2.0569 | max_optimizer_step: 0.2004 | max_tensorboard_logging: 0.8231 | max_tensorboard_flush: 0.0152 | max_post_step_barrier: 0.0867 | Memory: rss_MB: 5955.5 | uss_MB: 5094.1 | child_rss_MB: 5625.1 | allocated_MB: 497.1 | reserved_MB: 28008.0 | inactive_split_MB: 935.5 +rank7 [2026-10-29 23:33:22,808] INFO : Step 40300/50000 rank7 | Train Loss: 0.9351 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.1992 | Cardinality: 0.1181 | Classification: 0.0959 | Classification_binary: 0.1652 | HitGrouping: 0.1422 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0875 | Regression_eta: 0.0014 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0625 | Classification_binary: 0.1662 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0440 | Regression_eta: 0.0354 | Regression_pt: 0.0474 | Regression_sin_phi: 0.0660 | LR: 1.81e-04 | DataLoad Time: 0.0171s | Model Forward Time: 5.3105s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0075 | max_forward: 0.8191 | max_loss: 0.1494 | max_backward: 1.9817 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 0.9693 | max_tensorboard_flush: 0.0115 | max_post_step_barrier: 0.0527 | Memory: rss_MB: 5955.1 | uss_MB: 5096.4 | child_rss_MB: 5610.7 | allocated_MB: 498.6 | reserved_MB: 28008.0 | inactive_split_MB: 927.5 +rank7 [2026-10-29 23:38:38,678] INFO : Step 40400/50000 rank7 | Train Loss: 0.9379 | Train Loss Unweighted: 0.4238 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1359 | Classification: 0.0771 | Classification_binary: 0.1554 | HitGrouping: 0.1491 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0944 | Regression_eta: 0.0014 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1793 | Regression_cos_phi: 0.0558 | Regression_energy: 0.0413 | Regression_eta: 0.0268 | Regression_pt: 0.0451 | Regression_sin_phi: 0.0302 | LR: 1.79e-04 | DataLoad Time: 0.0147s | Model Forward Time: 4.6752s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0500 | max_forward: 0.8111 | max_loss: 0.5067 | max_backward: 1.8144 | max_optimizer_step: 0.2011 | max_tensorboard_logging: 0.8110 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 5951.4 | uss_MB: 5092.9 | child_rss_MB: 5616.0 | allocated_MB: 497.0 | reserved_MB: 28008.0 | inactive_split_MB: 931.6 +rank7 [2026-10-29 23:43:43,800] INFO : Step 40500/50000 rank7 | Train Loss: 0.8053 | Train Loss Unweighted: 0.3782 | Train Loss Components: Auxiliary: 0.1649 | Cardinality: 0.0788 | Classification: 0.0749 | Classification_binary: 0.1246 | HitGrouping: 0.1502 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0428 | Classification_binary: 0.1216 | Regression_cos_phi: 0.0630 | Regression_energy: 0.0484 | Regression_eta: 0.0299 | Regression_pt: 0.0516 | Regression_sin_phi: 0.0556 | LR: 1.78e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3980s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 0.8420 | max_loss: 0.1414 | max_backward: 1.5396 | max_optimizer_step: 0.2001 | max_tensorboard_logging: 0.9215 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0314 | Memory: rss_MB: 5948.4 | uss_MB: 5092.0 | child_rss_MB: 5597.4 | allocated_MB: 494.5 | reserved_MB: 28012.0 | inactive_split_MB: 979.3 +rank7 [2026-10-29 23:48:43,695] INFO : Step 40600/50000 rank7 | Train Loss: 0.8980 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1912 | Cardinality: 0.1108 | Classification: 0.0752 | Classification_binary: 0.1417 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0955 | Regression_eta: 0.0015 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0779 | Regression_energy: 0.0394 | Regression_eta: 0.0363 | Regression_pt: 0.0445 | Regression_sin_phi: 0.0507 | LR: 1.76e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.3178s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.7855 | max_loss: 0.1488 | max_backward: 1.8742 | max_optimizer_step: 0.2022 | max_tensorboard_logging: 0.5357 | max_tensorboard_flush: 0.0133 | max_post_step_barrier: 0.0354 | Memory: rss_MB: 5948.2 | uss_MB: 5094.3 | child_rss_MB: 5593.1 | allocated_MB: 499.7 | reserved_MB: 28012.0 | inactive_split_MB: 1102.0 +rank7 [2026-10-29 23:53:39,580] INFO : Step 40700/50000 rank7 | Train Loss: 0.9220 | Train Loss Unweighted: 0.4304 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1148 | Classification: 0.0767 | Classification_binary: 0.1645 | HitGrouping: 0.1452 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0929 | Regression_eta: 0.0012 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0374 | Classification_binary: 0.1768 | Regression_cos_phi: 0.0692 | Regression_energy: 0.0467 | Regression_eta: 0.0271 | Regression_pt: 0.0523 | Regression_sin_phi: 0.0408 | LR: 1.75e-04 | DataLoad Time: 0.0152s | Model Forward Time: 4.5458s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0076 | max_forward: 0.8091 | max_loss: 0.2475 | max_backward: 2.0011 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.5972 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0340 | Memory: rss_MB: 5949.6 | uss_MB: 5096.1 | child_rss_MB: 5593.6 | allocated_MB: 498.9 | reserved_MB: 28012.0 | inactive_split_MB: 776.0 +rank7 [2026-10-30 00:01:00,043] INFO : Step 40800/50000 rank7 | Train Loss: 0.8642 | Train Loss Unweighted: 0.4082 | Train Loss Components: Auxiliary: 0.1815 | Cardinality: 0.1013 | Classification: 0.0834 | Classification_binary: 0.1260 | HitGrouping: 0.1507 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0970 | Regression_eta: 0.0014 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1300 | Regression_cos_phi: 0.0372 | Regression_energy: 0.0474 | Regression_eta: 0.0252 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0476 | LR: 1.73e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.1147s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0063 | max_forward: 0.7033 | max_loss: 0.1373 | max_backward: 1.5042 | max_optimizer_step: 0.2025 | max_tensorboard_logging: 1.1068 | max_tensorboard_flush: 0.0079 | max_post_step_barrier: 0.0315 | Memory: rss_MB: 5935.0 | uss_MB: 4327.0 | child_rss_MB: 5586.6 | allocated_MB: 493.1 | reserved_MB: 28012.0 | inactive_split_MB: 897.3 +rank7 [2026-10-30 00:06:14,591] INFO : Step 40900/50000 rank7 | Train Loss: 0.8749 | Train Loss Unweighted: 0.4038 | Train Loss Components: Auxiliary: 0.1821 | Cardinality: 0.1070 | Classification: 0.0854 | Classification_binary: 0.1300 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0923 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1333 | Regression_cos_phi: 0.0770 | Regression_energy: 0.0409 | Regression_eta: 0.0299 | Regression_pt: 0.0444 | Regression_sin_phi: 0.0735 | LR: 1.72e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.5707s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7515 | max_loss: 0.2142 | max_backward: 1.7950 | max_optimizer_step: 0.4960 | max_tensorboard_logging: 0.9322 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 5926.9 | uss_MB: 5094.2 | child_rss_MB: 5563.3 | allocated_MB: 497.9 | reserved_MB: 30012.0 | inactive_split_MB: 893.0 +rank7 [2026-10-30 00:11:07,451] INFO : Step 41000/50000 rank7 | Train Loss: 0.9032 | Train Loss Unweighted: 0.4353 | Train Loss Components: Auxiliary: 0.1993 | Cardinality: 0.0956 | Classification: 0.0893 | Classification_binary: 0.1497 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0961 | Regression_eta: 0.0014 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0581 | Classification_binary: 0.1550 | Regression_cos_phi: 0.0799 | Regression_energy: 0.0528 | Regression_eta: 0.0468 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0464 | LR: 1.70e-04 | DataLoad Time: 0.0118s | Model Forward Time: 5.0872s | Phase Times: max_pre_step_barrier: 0.0153 | max_batch_to_device: 0.0071 | max_forward: 1.1963 | max_loss: 0.1519 | max_backward: 2.0474 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 1.0118 | max_tensorboard_flush: 0.0107 | max_post_step_barrier: 0.0327 | Memory: rss_MB: 5918.6 | uss_MB: 5085.3 | child_rss_MB: 5547.4 | allocated_MB: 498.7 | reserved_MB: 30012.0 | inactive_split_MB: 1108.0 +rank7 [2026-10-30 00:11:08,374] INFO : Running validation on rank7 +rank7 [2026-10-30 00:11:08,374] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 00:11:08,374] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 00:11:59,246] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:12:00,024] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 00:12:25,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 00:12:26,962] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 00:12:27,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 00:12:28,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 00:12:38,912] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 00:12:39,785] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:12:40,281] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 00:12:56,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 00:12:57,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 00:12:58,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 00:12:59,239] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 00:13:09,250] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 00:13:09,702] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 00:13:10,648] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 00:13:26,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 00:13:27,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 00:13:28,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 00:13:29,431] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_41000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 00:13:39,438] INFO : Time taken to make predictions on device 7 is: 0.24 min +rank7 [2026-10-30 00:19:15,378] INFO : Step 41100/50000 rank7 | Train Loss: 0.9430 | Train Loss Unweighted: 0.4319 | Train Loss Components: Auxiliary: 0.1995 | Cardinality: 0.1330 | Classification: 0.0721 | Classification_binary: 0.1579 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0994 | Regression_eta: 0.0013 | Regression_pt: 0.0998 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0553 | Classification_binary: 0.1808 | Regression_cos_phi: 0.0912 | Regression_energy: 0.0519 | Regression_eta: 0.0427 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0627 | LR: 1.69e-04 | DataLoad Time: 0.0137s | Model Forward Time: 6.3674s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0072 | max_forward: 0.8221 | max_loss: 0.4800 | max_backward: 2.2615 | max_optimizer_step: 0.2271 | max_tensorboard_logging: 1.1384 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0441 | Memory: rss_MB: 7242.0 | uss_MB: 5759.2 | child_rss_MB: 5549.6 | allocated_MB: 497.6 | reserved_MB: 23400.0 | inactive_split_MB: 991.8 +rank7 [2026-10-30 00:24:27,079] INFO : Step 41200/50000 rank7 | Train Loss: 0.8544 | Train Loss Unweighted: 0.3931 | Train Loss Components: Auxiliary: 0.1747 | Cardinality: 0.1097 | Classification: 0.0734 | Classification_binary: 0.1131 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1015 | Regression_eta: 0.0015 | Regression_pt: 0.1023 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1226 | Regression_cos_phi: 0.0648 | Regression_energy: 0.0451 | Regression_eta: 0.0301 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0688 | LR: 1.67e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.2542s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0060 | max_forward: 0.7030 | max_loss: 0.2408 | max_backward: 1.7750 | max_optimizer_step: 0.2015 | max_tensorboard_logging: 0.5316 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6039.9 | uss_MB: 5208.8 | child_rss_MB: 5554.6 | allocated_MB: 496.1 | reserved_MB: 27308.0 | inactive_split_MB: 984.0 +rank7 [2026-10-30 00:29:20,952] INFO : Step 41300/50000 rank7 | Train Loss: 0.8221 | Train Loss Unweighted: 0.3852 | Train Loss Components: Auxiliary: 0.1744 | Cardinality: 0.0906 | Classification: 0.0754 | Classification_binary: 0.1369 | HitGrouping: 0.1469 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0849 | Regression_eta: 0.0012 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0476 | Classification_binary: 0.1171 | Regression_cos_phi: 0.0383 | Regression_energy: 0.0455 | Regression_eta: 0.0433 | Regression_pt: 0.0479 | Regression_sin_phi: 0.0493 | LR: 1.66e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.4697s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0069 | max_forward: 0.8083 | max_loss: 0.1794 | max_backward: 1.9993 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.5906 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0348 | Memory: rss_MB: 6040.0 | uss_MB: 5213.3 | child_rss_MB: 5551.2 | allocated_MB: 497.0 | reserved_MB: 27308.0 | inactive_split_MB: 1229.0 +rank7 [2026-10-30 00:34:25,620] INFO : Step 41400/50000 rank7 | Train Loss: 0.9216 | Train Loss Unweighted: 0.4480 | Train Loss Components: Auxiliary: 0.2043 | Cardinality: 0.1171 | Classification: 0.0901 | Classification_binary: 0.1523 | HitGrouping: 0.1465 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1010 | Regression_eta: 0.0011 | Regression_pt: 0.1022 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0460 | Classification_binary: 0.1514 | Regression_cos_phi: 0.0671 | Regression_energy: 0.0510 | Regression_eta: 0.0321 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0375 | LR: 1.64e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.6350s | Phase Times: max_pre_step_barrier: 0.1138 | max_batch_to_device: 0.0065 | max_forward: 0.7831 | max_loss: 0.4937 | max_backward: 1.9271 | max_optimizer_step: 0.3071 | max_tensorboard_logging: 0.5915 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6033.3 | uss_MB: 5208.0 | child_rss_MB: 5553.7 | allocated_MB: 498.4 | reserved_MB: 29288.0 | inactive_split_MB: 1180.4 +rank7 [2026-10-30 00:39:29,959] INFO : Step 41500/50000 rank7 | Train Loss: 0.8584 | Train Loss Unweighted: 0.4111 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1030 | Classification: 0.0769 | Classification_binary: 0.1210 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1049 | Regression_eta: 0.0012 | Regression_pt: 0.1058 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0623 | Classification_binary: 0.1385 | Regression_cos_phi: 0.0579 | Regression_energy: 0.0482 | Regression_eta: 0.0314 | Regression_pt: 0.0527 | Regression_sin_phi: 0.0595 | LR: 1.63e-04 | DataLoad Time: 0.0162s | Model Forward Time: 6.0417s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0071 | max_forward: 1.1908 | max_loss: 0.3017 | max_backward: 3.1310 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.6207 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0561 | Memory: rss_MB: 6009.7 | uss_MB: 5150.0 | child_rss_MB: 5552.7 | allocated_MB: 494.5 | reserved_MB: 29288.0 | inactive_split_MB: 976.9 +rank7 [2026-10-30 00:44:19,402] INFO : Step 41600/50000 rank7 | Train Loss: 0.8635 | Train Loss Unweighted: 0.4026 | Train Loss Components: Auxiliary: 0.1799 | Cardinality: 0.1032 | Classification: 0.0762 | Classification_binary: 0.1246 | HitGrouping: 0.1564 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0992 | Regression_eta: 0.0013 | Regression_pt: 0.1001 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0356 | Classification_binary: 0.1133 | Regression_cos_phi: 0.0535 | Regression_energy: 0.0584 | Regression_eta: 0.0443 | Regression_pt: 0.0645 | Regression_sin_phi: 0.0458 | LR: 1.61e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.9933s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.0065 | max_forward: 0.7279 | max_loss: 0.2339 | max_backward: 2.5338 | max_optimizer_step: 0.2038 | max_tensorboard_logging: 0.4866 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1404 | Memory: rss_MB: 6001.6 | uss_MB: 5206.5 | child_rss_MB: 5546.1 | allocated_MB: 496.4 | reserved_MB: 29288.0 | inactive_split_MB: 761.6 +rank7 [2026-10-30 00:49:10,875] INFO : Step 41700/50000 rank7 | Train Loss: 0.9144 | Train Loss Unweighted: 0.4267 | Train Loss Components: Auxiliary: 0.1941 | Cardinality: 0.1112 | Classification: 0.0767 | Classification_binary: 0.1493 | HitGrouping: 0.1468 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0988 | Regression_eta: 0.0013 | Regression_pt: 0.0992 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0427 | Classification_binary: 0.1337 | Regression_cos_phi: 0.0525 | Regression_energy: 0.0565 | Regression_eta: 0.0264 | Regression_pt: 0.0610 | Regression_sin_phi: 0.0601 | LR: 1.60e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5061s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0071 | max_forward: 0.7998 | max_loss: 0.1827 | max_backward: 1.9774 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.5985 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0364 | Memory: rss_MB: 6004.5 | uss_MB: 5210.8 | child_rss_MB: 5548.7 | allocated_MB: 497.0 | reserved_MB: 29288.0 | inactive_split_MB: 1220.0 +rank7 [2026-10-30 00:53:58,972] INFO : Step 41800/50000 rank7 | Train Loss: 0.9125 | Train Loss Unweighted: 0.4285 | Train Loss Components: Auxiliary: 0.1954 | Cardinality: 0.1148 | Classification: 0.0784 | Classification_binary: 0.1537 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1776 | Regression_cos_phi: 0.0412 | Regression_energy: 0.0574 | Regression_eta: 0.0362 | Regression_pt: 0.0629 | Regression_sin_phi: 0.0748 | LR: 1.58e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.8256s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0071 | max_forward: 0.7952 | max_loss: 0.1582 | max_backward: 1.8899 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.9392 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.1143 | Memory: rss_MB: 5984.7 | uss_MB: 5200.7 | child_rss_MB: 5542.4 | allocated_MB: 499.3 | reserved_MB: 31288.0 | inactive_split_MB: 932.9 +rank7 [2026-10-30 00:59:00,784] INFO : Step 41900/50000 rank7 | Train Loss: 0.8303 | Train Loss Unweighted: 0.3927 | Train Loss Components: Auxiliary: 0.1775 | Cardinality: 0.1132 | Classification: 0.0782 | Classification_binary: 0.1167 | HitGrouping: 0.1476 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0010 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0429 | Classification_binary: 0.1516 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0503 | Regression_eta: 0.0279 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0391 | LR: 1.57e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.1521s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0063 | max_forward: 0.7413 | max_loss: 0.1970 | max_backward: 1.7614 | max_optimizer_step: 0.2041 | max_tensorboard_logging: 0.5208 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0357 | Memory: rss_MB: 5992.5 | uss_MB: 5207.2 | child_rss_MB: 5522.2 | allocated_MB: 497.1 | reserved_MB: 31288.0 | inactive_split_MB: 1188.0 +rank7 [2026-10-30 01:03:51,615] INFO : Step 42000/50000 rank7 | Train Loss: 0.8724 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.1847 | Cardinality: 0.0893 | Classification: 0.0874 | Classification_binary: 0.1377 | HitGrouping: 0.1490 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0958 | Regression_eta: 0.0012 | Regression_pt: 0.0973 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1261 | Regression_cos_phi: 0.0353 | Regression_energy: 0.0426 | Regression_eta: 0.0264 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0440 | LR: 1.56e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.7561s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.1147 | max_forward: 0.8308 | max_loss: 0.1392 | max_backward: 1.8336 | max_optimizer_step: 0.2005 | max_tensorboard_logging: 0.9249 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0329 | Memory: rss_MB: 5996.6 | uss_MB: 5209.4 | child_rss_MB: 5540.2 | allocated_MB: 496.3 | reserved_MB: 31288.0 | inactive_split_MB: 930.5 +rank7 [2026-10-30 01:03:52,597] INFO : Running validation on rank7 +rank7 [2026-10-30 01:03:52,598] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 01:03:52,598] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 01:04:46,126] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:04:46,670] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 01:05:13,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 01:05:14,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 01:05:15,926] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 01:05:16,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 01:05:26,905] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-30 01:05:27,369] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:05:27,885] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 01:05:43,980] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 01:05:44,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 01:05:45,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 01:05:46,841] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 01:05:56,852] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 01:05:57,192] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:05:57,664] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 01:06:13,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 01:06:14,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 01:06:15,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 01:06:17,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_42000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 01:06:27,058] INFO : Time taken to make predictions on device 7 is: 0.26 min +rank7 [2026-10-30 01:11:42,858] INFO : Step 42100/50000 rank7 | Train Loss: 0.9390 | Train Loss Unweighted: 0.4392 | Train Loss Components: Auxiliary: 0.2021 | Cardinality: 0.1200 | Classification: 0.0884 | Classification_binary: 0.1619 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0926 | Regression_eta: 0.0013 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0597 | Classification_binary: 0.1819 | Regression_cos_phi: 0.0969 | Regression_energy: 0.0435 | Regression_eta: 0.0537 | Regression_pt: 0.0478 | Regression_sin_phi: 0.0628 | LR: 1.54e-04 | DataLoad Time: 0.0175s | Model Forward Time: 6.0861s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0976 | max_forward: 1.0189 | max_loss: 0.3859 | max_backward: 2.5922 | max_optimizer_step: 0.2006 | max_tensorboard_logging: 1.0368 | max_tensorboard_flush: 0.0149 | max_post_step_barrier: 0.0344 | Memory: rss_MB: 7501.8 | uss_MB: 4866.0 | child_rss_MB: 5544.9 | allocated_MB: 497.0 | reserved_MB: 25850.0 | inactive_split_MB: 892.8 +rank7 [2026-10-30 01:16:53,955] INFO : Step 42200/50000 rank7 | Train Loss: 0.9057 | Train Loss Unweighted: 0.4254 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1144 | Classification: 0.0879 | Classification_binary: 0.1381 | HitGrouping: 0.1505 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0983 | Regression_eta: 0.0012 | Regression_pt: 0.0986 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0565 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0703 | Regression_energy: 0.0464 | Regression_eta: 0.0313 | Regression_pt: 0.0520 | Regression_sin_phi: 0.0513 | LR: 1.53e-04 | DataLoad Time: 0.0190s | Model Forward Time: 4.1529s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0067 | max_forward: 0.7457 | max_loss: 0.1999 | max_backward: 1.6824 | max_optimizer_step: 0.2077 | max_tensorboard_logging: 0.5916 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6096.0 | uss_MB: 5306.7 | child_rss_MB: 5537.7 | allocated_MB: 497.3 | reserved_MB: 27866.0 | inactive_split_MB: 959.8 +rank7 [2026-10-30 01:18:06,041] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank7 [2026-10-30 01:18:06,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank7 [2026-10-30 01:21:57,449] INFO : Step 42300/50000 rank7 | Train Loss: 0.9052 | Train Loss Unweighted: 0.4131 | Train Loss Components: Auxiliary: 0.1933 | Cardinality: 0.1127 | Classification: 0.0787 | Classification_binary: 0.1569 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0868 | Regression_eta: 0.0013 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0497 | Classification_binary: 0.1565 | Regression_cos_phi: 0.0724 | Regression_energy: 0.0432 | Regression_eta: 0.0358 | Regression_pt: 0.0471 | Regression_sin_phi: 0.0391 | LR: 1.52e-04 | DataLoad Time: 0.0161s | Model Forward Time: 6.1171s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0069 | max_forward: 0.9764 | max_loss: 0.1904 | max_backward: 2.7671 | max_optimizer_step: 0.1947 | max_tensorboard_logging: 0.9997 | max_tensorboard_flush: 0.0113 | max_post_step_barrier: 0.0498 | Memory: rss_MB: 6118.1 | uss_MB: 5325.9 | child_rss_MB: 5544.6 | allocated_MB: 499.1 | reserved_MB: 27872.0 | inactive_split_MB: 1033.6 +rank7 [2026-10-30 01:26:55,612] INFO : Step 42400/50000 rank7 | Train Loss: 0.9358 | Train Loss Unweighted: 0.4450 | Train Loss Components: Auxiliary: 0.1976 | Cardinality: 0.1220 | Classification: 0.0815 | Classification_binary: 0.1533 | HitGrouping: 0.1513 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1038 | Regression_eta: 0.0012 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0492 | Classification_binary: 0.1736 | Regression_cos_phi: 0.0524 | Regression_energy: 0.0524 | Regression_eta: 0.0271 | Regression_pt: 0.0564 | Regression_sin_phi: 0.0590 | LR: 1.50e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.8888s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8701 | max_loss: 0.1465 | max_backward: 1.9416 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.8192 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6099.3 | uss_MB: 5308.3 | child_rss_MB: 5546.3 | allocated_MB: 499.8 | reserved_MB: 29908.0 | inactive_split_MB: 939.2 +rank7 [2026-10-30 01:31:49,012] INFO : Step 42500/50000 rank7 | Train Loss: 0.8126 | Train Loss Unweighted: 0.3791 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0975 | Classification: 0.0765 | Classification_binary: 0.1172 | HitGrouping: 0.1487 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0918 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0339 | Classification_binary: 0.1320 | Regression_cos_phi: 0.0467 | Regression_energy: 0.0535 | Regression_eta: 0.0298 | Regression_pt: 0.0566 | Regression_sin_phi: 0.0645 | LR: 1.49e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.5926s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0858 | max_forward: 0.8156 | max_loss: 0.1407 | max_backward: 1.8570 | max_optimizer_step: 0.2039 | max_tensorboard_logging: 0.7997 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0361 | Memory: rss_MB: 6101.2 | uss_MB: 5308.6 | child_rss_MB: 5538.8 | allocated_MB: 497.5 | reserved_MB: 29908.0 | inactive_split_MB: 943.4 +rank7 [2026-10-30 01:36:44,528] INFO : Step 42600/50000 rank7 | Train Loss: 0.8493 | Train Loss Unweighted: 0.3865 | Train Loss Components: Auxiliary: 0.1782 | Cardinality: 0.1022 | Classification: 0.0772 | Classification_binary: 0.1347 | HitGrouping: 0.1410 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0858 | Regression_eta: 0.0012 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0838 | Regression_energy: 0.0425 | Regression_eta: 0.0513 | Regression_pt: 0.0466 | Regression_sin_phi: 0.0742 | LR: 1.48e-04 | DataLoad Time: 0.0173s | Model Forward Time: 5.0475s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0066 | max_forward: 0.7519 | max_loss: 0.1393 | max_backward: 1.8021 | max_optimizer_step: 0.3023 | max_tensorboard_logging: 0.7973 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0426 | Memory: rss_MB: 6101.5 | uss_MB: 4501.2 | child_rss_MB: 5544.2 | allocated_MB: 497.0 | reserved_MB: 29908.0 | inactive_split_MB: 877.2 +rank7 [2026-10-30 01:41:48,354] INFO : Step 42700/50000 rank7 | Train Loss: 0.9129 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1958 | Cardinality: 0.1178 | Classification: 0.0751 | Classification_binary: 0.1632 | HitGrouping: 0.1456 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0858 | Regression_eta: 0.0014 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0366 | Classification_binary: 0.1860 | Regression_cos_phi: 0.0510 | Regression_energy: 0.0486 | Regression_eta: 0.0385 | Regression_pt: 0.0539 | Regression_sin_phi: 0.0435 | LR: 1.47e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.6158s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0068 | max_forward: 1.0869 | max_loss: 0.1547 | max_backward: 1.6854 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.7949 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6101.6 | uss_MB: 5308.9 | child_rss_MB: 5545.8 | allocated_MB: 497.3 | reserved_MB: 29908.0 | inactive_split_MB: 953.4 +rank7 [2026-10-30 01:46:39,116] INFO : Step 42800/50000 rank7 | Train Loss: 0.9048 | Train Loss Unweighted: 0.4130 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1179 | Classification: 0.0767 | Classification_binary: 0.1362 | HitGrouping: 0.1512 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0981 | Regression_eta: 0.0013 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0583 | Classification_binary: 0.1414 | Regression_cos_phi: 0.0459 | Regression_energy: 0.0529 | Regression_eta: 0.0360 | Regression_pt: 0.0576 | Regression_sin_phi: 0.0611 | LR: 1.45e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.3538s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0066 | max_forward: 0.8438 | max_loss: 0.1478 | max_backward: 1.6710 | max_optimizer_step: 0.2547 | max_tensorboard_logging: 0.6821 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0349 | Memory: rss_MB: 6089.4 | uss_MB: 5295.4 | child_rss_MB: 5546.6 | allocated_MB: 499.0 | reserved_MB: 30126.0 | inactive_split_MB: 875.7 +rank7 [2026-10-30 01:51:40,827] INFO : Step 42900/50000 rank7 | Train Loss: 0.7857 | Train Loss Unweighted: 0.3758 | Train Loss Components: Auxiliary: 0.1628 | Cardinality: 0.0951 | Classification: 0.0820 | Classification_binary: 0.1113 | HitGrouping: 0.1423 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0898 | Regression_eta: 0.0010 | Regression_pt: 0.0906 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0448 | Classification_binary: 0.1026 | Regression_cos_phi: 0.0399 | Regression_energy: 0.0548 | Regression_eta: 0.0318 | Regression_pt: 0.0611 | Regression_sin_phi: 0.0510 | LR: 1.44e-04 | DataLoad Time: 0.0198s | Model Forward Time: 4.9235s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.8792 | max_loss: 0.1461 | max_backward: 1.9928 | max_optimizer_step: 0.1903 | max_tensorboard_logging: 0.9695 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0365 | Memory: rss_MB: 6098.2 | uss_MB: 5308.5 | child_rss_MB: 5544.6 | allocated_MB: 496.5 | reserved_MB: 30126.0 | inactive_split_MB: 951.8 +rank7 [2026-10-30 01:56:37,111] INFO : Step 43000/50000 rank7 | Train Loss: 0.8899 | Train Loss Unweighted: 0.4207 | Train Loss Components: Auxiliary: 0.1911 | Cardinality: 0.1035 | Classification: 0.0847 | Classification_binary: 0.1529 | HitGrouping: 0.1480 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0897 | Regression_eta: 0.0013 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0687 | Classification_binary: 0.1789 | Regression_cos_phi: 0.0557 | Regression_energy: 0.0586 | Regression_eta: 0.0517 | Regression_pt: 0.0635 | Regression_sin_phi: 0.0626 | LR: 1.43e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.7364s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0068 | max_forward: 0.7907 | max_loss: 0.1951 | max_backward: 1.9694 | max_optimizer_step: 0.2308 | max_tensorboard_logging: 0.8418 | max_tensorboard_flush: 0.0111 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6086.1 | uss_MB: 5311.0 | child_rss_MB: 5508.6 | allocated_MB: 495.5 | reserved_MB: 30126.0 | inactive_split_MB: 1120.4 +rank7 [2026-10-30 01:56:38,137] INFO : Running validation on rank7 +rank7 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 01:56:38,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 01:57:33,187] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:57:33,675] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 01:57:59,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 01:58:00,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 01:58:01,378] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 01:58:02,338] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 01:58:12,349] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 01:58:13,039] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:58:13,510] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 01:58:29,671] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 01:58:30,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 01:58:31,601] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 01:58:32,542] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 01:58:42,553] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 01:58:43,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 01:58:43,755] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 01:58:59,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 01:59:00,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 01:59:01,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 01:59:02,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_43000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 01:59:12,090] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 02:04:24,630] INFO : Step 43100/50000 rank7 | Train Loss: 0.8965 | Train Loss Unweighted: 0.4137 | Train Loss Components: Auxiliary: 0.1901 | Cardinality: 0.1074 | Classification: 0.0705 | Classification_binary: 0.1472 | HitGrouping: 0.1435 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0966 | Regression_eta: 0.0013 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1254 | Regression_cos_phi: 0.0575 | Regression_energy: 0.0442 | Regression_eta: 0.0313 | Regression_pt: 0.0485 | Regression_sin_phi: 0.0473 | LR: 1.42e-04 | DataLoad Time: 0.0141s | Model Forward Time: 5.2220s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0067 | max_forward: 0.8048 | max_loss: 0.2396 | max_backward: 1.9954 | max_optimizer_step: 0.3209 | max_tensorboard_logging: 1.0803 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0580 | Memory: rss_MB: 7631.1 | uss_MB: 6853.7 | child_rss_MB: 5519.2 | allocated_MB: 496.0 | reserved_MB: 26194.0 | inactive_split_MB: 843.4 +rank7 [2026-10-30 02:09:06,731] INFO : Step 43200/50000 rank7 | Train Loss: 0.8164 | Train Loss Unweighted: 0.3744 | Train Loss Components: Auxiliary: 0.1684 | Cardinality: 0.0957 | Classification: 0.0763 | Classification_binary: 0.1197 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0501 | Classification_binary: 0.1247 | Regression_cos_phi: 0.0740 | Regression_energy: 0.0478 | Regression_eta: 0.0416 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0726 | LR: 1.40e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.4245s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0063 | max_forward: 0.7823 | max_loss: 0.4822 | max_backward: 1.6766 | max_optimizer_step: 0.2305 | max_tensorboard_logging: 0.7680 | max_tensorboard_flush: 0.0094 | max_post_step_barrier: 0.0347 | Memory: rss_MB: 6123.7 | uss_MB: 5349.6 | child_rss_MB: 5511.0 | allocated_MB: 495.8 | reserved_MB: 26686.0 | inactive_split_MB: 848.9 +rank7 [2026-10-30 02:13:48,743] INFO : Step 43300/50000 rank7 | Train Loss: 0.9151 | Train Loss Unweighted: 0.4260 | Train Loss Components: Auxiliary: 0.1956 | Cardinality: 0.1202 | Classification: 0.0906 | Classification_binary: 0.1467 | HitGrouping: 0.1494 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0925 | Regression_eta: 0.0013 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1368 | Regression_cos_phi: 0.0506 | Regression_energy: 0.0527 | Regression_eta: 0.0297 | Regression_pt: 0.0562 | Regression_sin_phi: 0.0712 | LR: 1.39e-04 | DataLoad Time: 0.0125s | Model Forward Time: 4.7174s | Phase Times: max_pre_step_barrier: 0.0206 | max_batch_to_device: 0.0155 | max_forward: 0.8938 | max_loss: 0.1464 | max_backward: 2.1010 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.5477 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.1243 | Memory: rss_MB: 6097.4 | uss_MB: 5352.9 | child_rss_MB: 5427.7 | allocated_MB: 497.9 | reserved_MB: 28278.0 | inactive_split_MB: 1309.2 +rank7 [2026-10-30 02:18:30,113] INFO : Step 43400/50000 rank7 | Train Loss: 0.9723 | Train Loss Unweighted: 0.4562 | Train Loss Components: Auxiliary: 0.2082 | Cardinality: 0.1222 | Classification: 0.0880 | Classification_binary: 0.1684 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0975 | Regression_eta: 0.0015 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0576 | Classification_binary: 0.1647 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0493 | Regression_eta: 0.0372 | Regression_pt: 0.0533 | Regression_sin_phi: 0.0518 | LR: 1.38e-04 | DataLoad Time: 0.0151s | Model Forward Time: 4.4361s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0064 | max_forward: 0.8180 | max_loss: 0.5043 | max_backward: 2.0661 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.4824 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0351 | Memory: rss_MB: 6077.9 | uss_MB: 5350.3 | child_rss_MB: 5335.1 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 851.9 +rank7 [2026-10-30 02:23:12,892] INFO : Step 43500/50000 rank7 | Train Loss: 0.8402 | Train Loss Unweighted: 0.4081 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.0947 | Classification: 0.0776 | Classification_binary: 0.1278 | HitGrouping: 0.1501 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0999 | Regression_eta: 0.0012 | Regression_pt: 0.1004 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0588 | Classification_binary: 0.1430 | Regression_cos_phi: 0.0582 | Regression_energy: 0.0457 | Regression_eta: 0.0372 | Regression_pt: 0.0504 | Regression_sin_phi: 0.0582 | LR: 1.37e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8628s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0064 | max_forward: 0.8086 | max_loss: 0.1489 | max_backward: 2.0887 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.7978 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1094 | Memory: rss_MB: 6077.6 | uss_MB: 5352.8 | child_rss_MB: 5340.6 | allocated_MB: 498.6 | reserved_MB: 28278.0 | inactive_split_MB: 909.7 +rank7 [2026-10-30 02:28:09,613] INFO : Step 43600/50000 rank7 | Train Loss: 0.7985 | Train Loss Unweighted: 0.3653 | Train Loss Components: Auxiliary: 0.1621 | Cardinality: 0.0887 | Classification: 0.0728 | Classification_binary: 0.1184 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0851 | Regression_eta: 0.0013 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0659 | Classification_binary: 0.1237 | Regression_cos_phi: 0.0623 | Regression_energy: 0.0487 | Regression_eta: 0.0340 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0722 | LR: 1.36e-04 | DataLoad Time: 0.0210s | Model Forward Time: 5.2494s | Phase Times: max_pre_step_barrier: 0.0046 | max_batch_to_device: 0.0170 | max_forward: 1.2777 | max_loss: 0.2064 | max_backward: 1.9835 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 1.1084 | max_tensorboard_flush: 0.0031 | max_post_step_barrier: 0.0459 | Memory: rss_MB: 6044.0 | uss_MB: 4957.3 | child_rss_MB: 5260.0 | allocated_MB: 493.5 | reserved_MB: 28278.0 | inactive_split_MB: 1313.2 +rank7 [2026-10-30 02:33:07,295] INFO : Step 43700/50000 rank7 | Train Loss: 0.8599 | Train Loss Unweighted: 0.3884 | Train Loss Components: Auxiliary: 0.1834 | Cardinality: 0.1089 | Classification: 0.0797 | Classification_binary: 0.1434 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0812 | Regression_eta: 0.0013 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0507 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0610 | Regression_energy: 0.0454 | Regression_eta: 0.0384 | Regression_pt: 0.0508 | Regression_sin_phi: 0.0587 | LR: 1.35e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.6046s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0066 | max_forward: 0.7902 | max_loss: 0.1752 | max_backward: 1.8774 | max_optimizer_step: 0.2046 | max_tensorboard_logging: 0.8044 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6046.0 | uss_MB: 5350.4 | child_rss_MB: 5241.5 | allocated_MB: 498.0 | reserved_MB: 28278.0 | inactive_split_MB: 1046.0 +rank7 [2026-10-30 02:37:45,726] INFO : Step 43800/50000 rank7 | Train Loss: 0.8610 | Train Loss Unweighted: 0.4078 | Train Loss Components: Auxiliary: 0.1875 | Cardinality: 0.1049 | Classification: 0.0814 | Classification_binary: 0.1423 | HitGrouping: 0.1477 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0907 | Regression_eta: 0.0012 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1379 | Regression_cos_phi: 0.0367 | Regression_energy: 0.0553 | Regression_eta: 0.0210 | Regression_pt: 0.0598 | Regression_sin_phi: 0.0414 | LR: 1.34e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.5747s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0067 | max_forward: 0.7940 | max_loss: 0.4805 | max_backward: 1.8159 | max_optimizer_step: 0.2941 | max_tensorboard_logging: 0.7850 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0372 | Memory: rss_MB: 6040.0 | uss_MB: 5349.3 | child_rss_MB: 5218.8 | allocated_MB: 498.3 | reserved_MB: 28278.0 | inactive_split_MB: 839.4 +rank7 [2026-10-30 02:42:41,736] INFO : Step 43900/50000 rank7 | Train Loss: 0.7987 | Train Loss Unweighted: 0.3885 | Train Loss Components: Auxiliary: 0.1696 | Cardinality: 0.0999 | Classification: 0.0753 | Classification_binary: 0.1166 | HitGrouping: 0.1464 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0968 | Regression_eta: 0.0010 | Regression_pt: 0.0978 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0525 | Classification_binary: 0.1159 | Regression_cos_phi: 0.0406 | Regression_energy: 0.0428 | Regression_eta: 0.0204 | Regression_pt: 0.0465 | Regression_sin_phi: 0.0304 | LR: 1.33e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.5935s | Phase Times: max_pre_step_barrier: 0.0036 | max_batch_to_device: 0.0846 | max_forward: 0.8891 | max_loss: 0.1768 | max_backward: 1.7357 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7516 | max_tensorboard_flush: 0.0125 | max_post_step_barrier: 0.1343 | Memory: rss_MB: 6030.6 | uss_MB: 5347.7 | child_rss_MB: 5179.6 | allocated_MB: 494.5 | reserved_MB: 28278.0 | inactive_split_MB: 1129.5 +rank7 [2026-10-30 02:47:33,825] INFO : Step 44000/50000 rank7 | Train Loss: 0.8263 | Train Loss Unweighted: 0.3926 | Train Loss Components: Auxiliary: 0.1763 | Cardinality: 0.1027 | Classification: 0.0840 | Classification_binary: 0.1318 | HitGrouping: 0.1397 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0868 | Regression_eta: 0.0011 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0435 | Classification_binary: 0.1150 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0501 | Regression_eta: 0.0309 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0429 | LR: 1.32e-04 | DataLoad Time: 0.0166s | Model Forward Time: 4.4517s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0067 | max_forward: 0.8816 | max_loss: 0.1426 | max_backward: 1.9616 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.4846 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6020.6 | uss_MB: 5352.2 | child_rss_MB: 5134.9 | allocated_MB: 496.3 | reserved_MB: 28762.0 | inactive_split_MB: 1307.2 +rank7 [2026-10-30 02:47:34,861] INFO : Running validation on rank7 +rank7 [2026-10-30 02:47:34,861] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 02:47:34,861] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 02:48:28,154] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 02:48:28,670] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 02:48:55,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 02:48:56,910] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 02:48:57,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 02:48:58,842] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 02:49:08,853] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 02:49:09,618] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 02:49:10,103] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 02:49:25,975] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 02:49:26,949] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 02:49:27,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 02:49:28,832] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 02:49:38,842] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 02:49:39,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 02:49:40,181] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 02:49:55,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 02:49:56,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 02:49:57,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 02:49:58,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_44000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 02:50:08,497] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 02:55:20,175] INFO : Step 44100/50000 rank7 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4171 | Train Loss Components: Auxiliary: 0.1986 | Cardinality: 0.1334 | Classification: 0.0764 | Classification_binary: 0.1516 | HitGrouping: 0.1449 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0530 | Classification_binary: 0.1585 | Regression_cos_phi: 0.0457 | Regression_energy: 0.0448 | Regression_eta: 0.0341 | Regression_pt: 0.0483 | Regression_sin_phi: 0.0447 | LR: 1.31e-04 | DataLoad Time: 0.0155s | Model Forward Time: 5.1230s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0079 | max_forward: 0.8221 | max_loss: 0.1579 | max_backward: 2.0192 | max_optimizer_step: 0.2248 | max_tensorboard_logging: 1.0889 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0553 | Memory: rss_MB: 7539.1 | uss_MB: 6869.4 | child_rss_MB: 5111.3 | allocated_MB: 498.6 | reserved_MB: 25920.0 | inactive_split_MB: 1073.8 +rank7 [2026-10-30 03:00:12,363] INFO : Step 44200/50000 rank7 | Train Loss: 0.9018 | Train Loss Unweighted: 0.4273 | Train Loss Components: Auxiliary: 0.1909 | Cardinality: 0.1109 | Classification: 0.0847 | Classification_binary: 0.1354 | HitGrouping: 0.1488 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1021 | Regression_eta: 0.0012 | Regression_pt: 0.1025 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0416 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0642 | Regression_energy: 0.0455 | Regression_eta: 0.0347 | Regression_pt: 0.0496 | Regression_sin_phi: 0.0662 | LR: 1.30e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.3590s | Phase Times: max_pre_step_barrier: 0.0015 | max_batch_to_device: 0.1013 | max_forward: 0.8608 | max_loss: 0.1485 | max_backward: 1.6010 | max_optimizer_step: 0.2068 | max_tensorboard_logging: 0.7858 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0353 | Memory: rss_MB: 6119.2 | uss_MB: 5452.1 | child_rss_MB: 5083.8 | allocated_MB: 496.1 | reserved_MB: 27956.0 | inactive_split_MB: 1248.1 +rank7 [2026-10-30 03:04:47,145] INFO : Step 44300/50000 rank7 | Train Loss: 0.8239 | Train Loss Unweighted: 0.3875 | Train Loss Components: Auxiliary: 0.1722 | Cardinality: 0.1007 | Classification: 0.0789 | Classification_binary: 0.1118 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1047 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0644 | Regression_eta: 0.0309 | Regression_pt: 0.0698 | Regression_sin_phi: 0.0430 | LR: 1.29e-04 | DataLoad Time: 0.0173s | Model Forward Time: 4.3180s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0064 | max_forward: 0.8120 | max_loss: 0.1377 | max_backward: 1.4700 | max_optimizer_step: 0.2092 | max_tensorboard_logging: 0.9055 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.0381 | Memory: rss_MB: 6110.2 | uss_MB: 5452.8 | child_rss_MB: 5031.6 | allocated_MB: 492.8 | reserved_MB: 27956.0 | inactive_split_MB: 1035.8 +rank7 [2026-10-30 03:09:18,055] INFO : Step 44400/50000 rank7 | Train Loss: 0.8683 | Train Loss Unweighted: 0.4299 | Train Loss Components: Auxiliary: 0.1865 | Cardinality: 0.0849 | Classification: 0.0836 | Classification_binary: 0.1500 | HitGrouping: 0.1459 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0968 | Regression_eta: 0.0011 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0524 | Classification_binary: 0.1677 | Regression_cos_phi: 0.0466 | Regression_energy: 0.0681 | Regression_eta: 0.0378 | Regression_pt: 0.0734 | Regression_sin_phi: 0.0481 | LR: 1.28e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.9259s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0068 | max_forward: 0.8907 | max_loss: 0.1476 | max_backward: 2.0623 | max_optimizer_step: 0.2100 | max_tensorboard_logging: 0.9119 | max_tensorboard_flush: 0.0055 | max_post_step_barrier: 0.0398 | Memory: rss_MB: 6106.5 | uss_MB: 5454.6 | child_rss_MB: 5002.7 | allocated_MB: 499.5 | reserved_MB: 27956.0 | inactive_split_MB: 926.0 +rank7 [2026-10-30 03:14:01,913] INFO : Step 44500/50000 rank7 | Train Loss: 0.8672 | Train Loss Unweighted: 0.4045 | Train Loss Components: Auxiliary: 0.1880 | Cardinality: 0.0989 | Classification: 0.0738 | Classification_binary: 0.1455 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0914 | Regression_eta: 0.0011 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0398 | Classification_binary: 0.1605 | Regression_cos_phi: 0.0553 | Regression_energy: 0.0432 | Regression_eta: 0.0404 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0584 | LR: 1.27e-04 | DataLoad Time: 0.0169s | Model Forward Time: 4.2938s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0597 | max_forward: 0.8669 | max_loss: 0.1547 | max_backward: 1.7327 | max_optimizer_step: 0.2065 | max_tensorboard_logging: 0.5628 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6105.7 | uss_MB: 5452.5 | child_rss_MB: 5004.4 | allocated_MB: 498.3 | reserved_MB: 27956.0 | inactive_split_MB: 860.5 +rank7 [2026-10-30 03:18:59,363] INFO : Step 44600/50000 rank7 | Train Loss: 0.7900 | Train Loss Unweighted: 0.3743 | Train Loss Components: Auxiliary: 0.1688 | Cardinality: 0.0956 | Classification: 0.0724 | Classification_binary: 0.1157 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0919 | Regression_eta: 0.0010 | Regression_pt: 0.0923 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1275 | Regression_cos_phi: 0.0310 | Regression_energy: 0.0537 | Regression_eta: 0.0259 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0329 | LR: 1.26e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.5455s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0745 | max_forward: 0.7797 | max_loss: 0.1409 | max_backward: 1.9334 | max_optimizer_step: 0.2059 | max_tensorboard_logging: 0.6022 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1405 | Memory: rss_MB: 6098.6 | uss_MB: 5447.3 | child_rss_MB: 4994.6 | allocated_MB: 494.5 | reserved_MB: 27958.0 | inactive_split_MB: 927.6 +rank7 [2026-10-30 03:23:44,790] INFO : Step 44700/50000 rank7 | Train Loss: 0.8669 | Train Loss Unweighted: 0.3906 | Train Loss Components: Auxiliary: 0.1878 | Cardinality: 0.1160 | Classification: 0.0768 | Classification_binary: 0.1380 | HitGrouping: 0.1492 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0863 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0221 | Classification_binary: 0.1378 | Regression_cos_phi: 0.0374 | Regression_energy: 0.0407 | Regression_eta: 0.0309 | Regression_pt: 0.0453 | Regression_sin_phi: 0.0515 | LR: 1.25e-04 | DataLoad Time: 0.0183s | Model Forward Time: 4.4946s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0981 | max_forward: 0.8629 | max_loss: 0.1449 | max_backward: 2.0281 | max_optimizer_step: 0.1871 | max_tensorboard_logging: 0.5242 | max_tensorboard_flush: 0.0122 | max_post_step_barrier: 0.0444 | Memory: rss_MB: 6099.2 | uss_MB: 5434.3 | child_rss_MB: 5003.0 | allocated_MB: 498.4 | reserved_MB: 27958.0 | inactive_split_MB: 781.6 +rank7 [2026-10-30 03:28:42,238] INFO : Step 44800/50000 rank7 | Train Loss: 0.8585 | Train Loss Unweighted: 0.4003 | Train Loss Components: Auxiliary: 0.1869 | Cardinality: 0.1024 | Classification: 0.0769 | Classification_binary: 0.1550 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0823 | Regression_eta: 0.0011 | Regression_pt: 0.0838 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1299 | Regression_cos_phi: 0.0523 | Regression_energy: 0.0471 | Regression_eta: 0.0309 | Regression_pt: 0.0522 | Regression_sin_phi: 0.0411 | LR: 1.24e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.8035s | Phase Times: max_pre_step_barrier: 0.0039 | max_batch_to_device: 0.0072 | max_forward: 0.7883 | max_loss: 0.5033 | max_backward: 1.9727 | max_optimizer_step: 0.2035 | max_tensorboard_logging: 0.9083 | max_tensorboard_flush: 0.0019 | max_post_step_barrier: 0.0346 | Memory: rss_MB: 6090.2 | uss_MB: 5455.2 | child_rss_MB: 4956.4 | allocated_MB: 496.5 | reserved_MB: 27958.0 | inactive_split_MB: 778.0 +rank7 [2026-10-30 03:33:12,492] INFO : Step 44900/50000 rank7 | Train Loss: 0.8358 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1737 | Cardinality: 0.1024 | Classification: 0.0797 | Classification_binary: 0.1294 | HitGrouping: 0.1457 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0859 | Regression_eta: 0.0011 | Regression_pt: 0.0862 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0546 | Classification_binary: 0.1549 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0419 | Regression_eta: 0.0371 | Regression_pt: 0.0463 | Regression_sin_phi: 0.0951 | LR: 1.23e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.3682s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0065 | max_forward: 0.7360 | max_loss: 0.1482 | max_backward: 1.6886 | max_optimizer_step: 0.2064 | max_tensorboard_logging: 0.7825 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0384 | Memory: rss_MB: 6084.1 | uss_MB: 5452.9 | child_rss_MB: 4956.9 | allocated_MB: 494.3 | reserved_MB: 27958.0 | inactive_split_MB: 922.0 +rank7 [2026-10-30 03:37:54,705] INFO : Step 45000/50000 rank7 | Train Loss: 0.8678 | Train Loss Unweighted: 0.3957 | Train Loss Components: Auxiliary: 0.1790 | Cardinality: 0.1036 | Classification: 0.0801 | Classification_binary: 0.1237 | HitGrouping: 0.1521 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0941 | Regression_eta: 0.0013 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0451 | Classification_binary: 0.1357 | Regression_cos_phi: 0.0529 | Regression_energy: 0.0538 | Regression_eta: 0.0265 | Regression_pt: 0.0593 | Regression_sin_phi: 0.0549 | LR: 1.22e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.5697s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0069 | max_forward: 0.7189 | max_loss: 0.2183 | max_backward: 1.8172 | max_optimizer_step: 0.2060 | max_tensorboard_logging: 0.8812 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0386 | Memory: rss_MB: 6058.9 | uss_MB: 5454.9 | child_rss_MB: 4844.5 | allocated_MB: 495.0 | reserved_MB: 27960.0 | inactive_split_MB: 978.5 +rank7 [2026-10-30 03:37:55,714] INFO : Running validation on rank7 +rank7 [2026-10-30 03:37:55,715] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 03:37:55,715] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 03:38:46,292] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 03:38:46,808] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 03:39:14,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 03:39:15,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 03:39:16,893] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 03:39:18,631] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 03:39:28,642] INFO : Time taken to make predictions on device 7 is: 0.27 min +rank7 [2026-10-30 03:39:29,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 03:39:30,322] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 03:39:46,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 03:39:47,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 03:39:48,351] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 03:39:49,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 03:39:59,300] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 03:40:00,144] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 03:40:00,673] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 03:40:16,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 03:40:17,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 03:40:18,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 03:40:19,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_45000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 03:40:29,135] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 03:45:39,355] INFO : Step 45100/50000 rank7 | Train Loss: 0.8712 | Train Loss Unweighted: 0.4129 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0997 | Classification: 0.0837 | Classification_binary: 0.1445 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0910 | Regression_eta: 0.0012 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0634 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0507 | Regression_energy: 0.0537 | Regression_eta: 0.0407 | Regression_pt: 0.0585 | Regression_sin_phi: 0.0796 | LR: 1.21e-04 | DataLoad Time: 0.0154s | Model Forward Time: 5.1923s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0070 | max_forward: 0.8448 | max_loss: 0.1523 | max_backward: 2.1445 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 1.0003 | max_tensorboard_flush: 0.0117 | max_post_step_barrier: 0.0642 | Memory: rss_MB: 7489.1 | uss_MB: 6883.8 | child_rss_MB: 4866.9 | allocated_MB: 499.4 | reserved_MB: 24746.0 | inactive_split_MB: 784.5 +rank7 [2026-10-30 03:50:46,376] INFO : Step 45200/50000 rank7 | Train Loss: 0.9130 | Train Loss Unweighted: 0.4209 | Train Loss Components: Auxiliary: 0.1938 | Cardinality: 0.1058 | Classification: 0.0776 | Classification_binary: 0.1464 | HitGrouping: 0.1497 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0963 | Regression_eta: 0.0015 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0008 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0353 | Classification_binary: 0.1545 | Regression_cos_phi: 0.0474 | Regression_energy: 0.0442 | Regression_eta: 0.0384 | Regression_pt: 0.0490 | Regression_sin_phi: 0.0471 | LR: 1.20e-04 | DataLoad Time: 0.0164s | Model Forward Time: 6.5410s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1574 | max_backward: 1.6440 | max_optimizer_step: 0.2111 | max_tensorboard_logging: 0.8158 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.1112 | Memory: rss_MB: 6132.8 | uss_MB: 4597.7 | child_rss_MB: 4866.1 | allocated_MB: 499.7 | reserved_MB: 26730.0 | inactive_split_MB: 1073.5 +rank7 [2026-10-30 03:55:44,003] INFO : Step 45300/50000 rank7 | Train Loss: 0.8299 | Train Loss Unweighted: 0.3823 | Train Loss Components: Auxiliary: 0.1708 | Cardinality: 0.1126 | Classification: 0.0744 | Classification_binary: 0.1207 | HitGrouping: 0.1473 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0921 | Regression_eta: 0.0011 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0642 | Classification_binary: 0.1185 | Regression_cos_phi: 0.0684 | Regression_energy: 0.0551 | Regression_eta: 0.0385 | Regression_pt: 0.0597 | Regression_sin_phi: 0.0456 | LR: 1.19e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.1815s | Phase Times: max_pre_step_barrier: 0.0028 | max_batch_to_device: 0.0827 | max_forward: 0.7966 | max_loss: 0.1432 | max_backward: 1.5141 | max_optimizer_step: 0.2646 | max_tensorboard_logging: 0.6448 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6128.3 | uss_MB: 5538.9 | child_rss_MB: 4864.4 | allocated_MB: 495.4 | reserved_MB: 27170.0 | inactive_split_MB: 881.8 +rank7 [2026-10-30 04:00:47,340] INFO : Step 45400/50000 rank7 | Train Loss: 0.8255 | Train Loss Unweighted: 0.4011 | Train Loss Components: Auxiliary: 0.1814 | Cardinality: 0.0984 | Classification: 0.0907 | Classification_binary: 0.1338 | HitGrouping: 0.1387 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0870 | Regression_eta: 0.0011 | Regression_pt: 0.0875 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1187 | Regression_cos_phi: 0.0431 | Regression_energy: 0.0432 | Regression_eta: 0.0335 | Regression_pt: 0.0470 | Regression_sin_phi: 0.0321 | LR: 1.19e-04 | DataLoad Time: 0.0158s | Model Forward Time: 5.3449s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7846 | max_loss: 0.1934 | max_backward: 1.9857 | max_optimizer_step: 0.2498 | max_tensorboard_logging: 0.7880 | max_tensorboard_flush: 0.0013 | max_post_step_barrier: 0.0395 | Memory: rss_MB: 6124.9 | uss_MB: 4519.9 | child_rss_MB: 4862.5 | allocated_MB: 499.2 | reserved_MB: 27170.0 | inactive_split_MB: 989.3 +rank7 [2026-10-30 04:05:44,853] INFO : Step 45500/50000 rank7 | Train Loss: 0.8945 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1090 | Classification: 0.0825 | Classification_binary: 0.1567 | HitGrouping: 0.1446 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0896 | Regression_eta: 0.0012 | Regression_pt: 0.0902 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0607 | Classification_binary: 0.1919 | Regression_cos_phi: 0.0645 | Regression_energy: 0.0496 | Regression_eta: 0.0546 | Regression_pt: 0.0554 | Regression_sin_phi: 0.0699 | LR: 1.18e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.4589s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8686 | max_loss: 0.1508 | max_backward: 1.9581 | max_optimizer_step: 0.2083 | max_tensorboard_logging: 0.4909 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0378 | Memory: rss_MB: 6130.5 | uss_MB: 5543.1 | child_rss_MB: 4864.9 | allocated_MB: 498.4 | reserved_MB: 28766.0 | inactive_split_MB: 883.2 +rank7 [2026-10-30 04:10:39,743] INFO : Step 45600/50000 rank7 | Train Loss: 0.8704 | Train Loss Unweighted: 0.4121 | Train Loss Components: Auxiliary: 0.1840 | Cardinality: 0.1126 | Classification: 0.0802 | Classification_binary: 0.1324 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0983 | Regression_eta: 0.0010 | Regression_pt: 0.0990 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0345 | Classification_binary: 0.1498 | Regression_cos_phi: 0.0704 | Regression_energy: 0.0462 | Regression_eta: 0.0386 | Regression_pt: 0.0484 | Regression_sin_phi: 0.0538 | LR: 1.17e-04 | DataLoad Time: 0.0211s | Model Forward Time: 4.8855s | Phase Times: max_pre_step_barrier: 0.0016 | max_batch_to_device: 0.0163 | max_forward: 0.8161 | max_loss: 0.2729 | max_backward: 2.1846 | max_optimizer_step: 0.2108 | max_tensorboard_logging: 0.7386 | max_tensorboard_flush: 0.0121 | max_post_step_barrier: 0.0352 | Memory: rss_MB: 6122.7 | uss_MB: 5488.3 | child_rss_MB: 4871.5 | allocated_MB: 495.6 | reserved_MB: 28766.0 | inactive_split_MB: 1080.4 +rank7 [2026-10-30 04:15:37,195] INFO : Step 45700/50000 rank7 | Train Loss: 0.8006 | Train Loss Unweighted: 0.3784 | Train Loss Components: Auxiliary: 0.1681 | Cardinality: 0.0873 | Classification: 0.0773 | Classification_binary: 0.1111 | HitGrouping: 0.1443 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0933 | Regression_eta: 0.0012 | Regression_pt: 0.0942 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0506 | Classification_binary: 0.1100 | Regression_cos_phi: 0.0640 | Regression_energy: 0.0526 | Regression_eta: 0.0333 | Regression_pt: 0.0575 | Regression_sin_phi: 0.0779 | LR: 1.16e-04 | DataLoad Time: 0.0131s | Model Forward Time: 4.8147s | Phase Times: max_pre_step_barrier: 0.0327 | max_batch_to_device: 0.0149 | max_forward: 1.1858 | max_loss: 0.1429 | max_backward: 1.6968 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.7566 | max_tensorboard_flush: 0.0109 | max_post_step_barrier: 0.1366 | Memory: rss_MB: 6126.5 | uss_MB: 5540.3 | child_rss_MB: 4873.0 | allocated_MB: 496.6 | reserved_MB: 28766.0 | inactive_split_MB: 903.7 +rank7 [2026-10-30 04:20:28,800] INFO : Step 45800/50000 rank7 | Train Loss: 0.8566 | Train Loss Unweighted: 0.3982 | Train Loss Components: Auxiliary: 0.1903 | Cardinality: 0.1097 | Classification: 0.0794 | Classification_binary: 0.1398 | HitGrouping: 0.1406 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0444 | Classification_binary: 0.1330 | Regression_cos_phi: 0.0646 | Regression_energy: 0.0424 | Regression_eta: 0.0625 | Regression_pt: 0.0448 | Regression_sin_phi: 0.0481 | LR: 1.16e-04 | DataLoad Time: 0.0165s | Model Forward Time: 5.1198s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0071 | max_forward: 0.9805 | max_loss: 0.5234 | max_backward: 2.3872 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.8153 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0356 | Memory: rss_MB: 6131.1 | uss_MB: 5547.8 | child_rss_MB: 4874.9 | allocated_MB: 501.7 | reserved_MB: 28766.0 | inactive_split_MB: 1064.3 +rank7 [2026-10-30 04:25:16,811] INFO : Step 45900/50000 rank7 | Train Loss: 0.8754 | Train Loss Unweighted: 0.4143 | Train Loss Components: Auxiliary: 0.1870 | Cardinality: 0.1042 | Classification: 0.0696 | Classification_binary: 0.1484 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0962 | Regression_eta: 0.0011 | Regression_pt: 0.0975 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0415 | Classification_binary: 0.1623 | Regression_cos_phi: 0.0983 | Regression_energy: 0.0471 | Regression_eta: 0.0360 | Regression_pt: 0.0521 | Regression_sin_phi: 0.0906 | LR: 1.15e-04 | DataLoad Time: 0.0086s | Model Forward Time: 5.8315s | Phase Times: max_pre_step_barrier: 0.0308 | max_batch_to_device: 0.0265 | max_forward: 1.7626 | max_loss: 0.1678 | max_backward: 2.6283 | max_optimizer_step: 0.2109 | max_tensorboard_logging: 0.8701 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0411 | Memory: rss_MB: 6124.4 | uss_MB: 5544.6 | child_rss_MB: 4869.8 | allocated_MB: 496.0 | reserved_MB: 29170.0 | inactive_split_MB: 983.4 +rank7 [2026-10-30 04:29:51,933] INFO : Step 46000/50000 rank7 | Train Loss: 0.7531 | Train Loss Unweighted: 0.3556 | Train Loss Components: Auxiliary: 0.1535 | Cardinality: 0.0870 | Classification: 0.0690 | Classification_binary: 0.1026 | HitGrouping: 0.1447 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0903 | Regression_eta: 0.0011 | Regression_pt: 0.0915 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0392 | Classification_binary: 0.1124 | Regression_cos_phi: 0.0527 | Regression_energy: 0.0547 | Regression_eta: 0.0385 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0506 | LR: 1.14e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3114s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0329 | max_forward: 0.7497 | max_loss: 0.1389 | max_backward: 1.9140 | max_optimizer_step: 0.2088 | max_tensorboard_logging: 0.5104 | max_tensorboard_flush: 0.0134 | max_post_step_barrier: 0.0410 | Memory: rss_MB: 6122.4 | uss_MB: 5543.2 | child_rss_MB: 4869.8 | allocated_MB: 492.4 | reserved_MB: 29170.0 | inactive_split_MB: 973.8 +rank7 [2026-10-30 04:29:53,063] INFO : Running validation on rank7 +rank7 [2026-10-30 04:29:53,063] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 04:29:53,063] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 04:30:44,348] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 04:30:44,927] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 04:31:11,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 04:31:12,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 04:31:13,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 04:31:14,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 04:31:24,594] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 04:31:25,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 04:31:25,580] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 04:31:41,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 04:31:42,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 04:31:43,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 04:31:44,092] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 04:31:54,102] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 04:31:54,979] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 04:31:55,464] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 04:32:10,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 04:32:11,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 04:32:12,471] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 04:32:13,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_46000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 04:32:23,326] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 04:37:27,739] INFO : Step 46100/50000 rank7 | Train Loss: 0.8070 | Train Loss Unweighted: 0.3769 | Train Loss Components: Auxiliary: 0.1731 | Cardinality: 0.0999 | Classification: 0.0743 | Classification_binary: 0.1253 | HitGrouping: 0.1427 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0873 | Regression_eta: 0.0010 | Regression_pt: 0.0878 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0371 | Classification_binary: 0.0756 | Regression_cos_phi: 0.0318 | Regression_energy: 0.0365 | Regression_eta: 0.0140 | Regression_pt: 0.0396 | Regression_sin_phi: 0.0232 | LR: 1.13e-04 | DataLoad Time: 0.0149s | Model Forward Time: 4.8724s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0064 | max_forward: 0.9293 | max_loss: 0.1452 | max_backward: 2.3008 | max_optimizer_step: 0.2285 | max_tensorboard_logging: 0.4968 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.1219 | Memory: rss_MB: 7472.5 | uss_MB: 6889.9 | child_rss_MB: 4882.8 | allocated_MB: 498.3 | reserved_MB: 23298.0 | inactive_split_MB: 776.1 +rank7 [2026-10-30 04:42:19,615] INFO : Step 46200/50000 rank7 | Train Loss: 0.8661 | Train Loss Unweighted: 0.3994 | Train Loss Components: Auxiliary: 0.1947 | Cardinality: 0.1164 | Classification: 0.0793 | Classification_binary: 0.1511 | HitGrouping: 0.1440 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0833 | Regression_eta: 0.0011 | Regression_pt: 0.0836 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0368 | Classification_binary: 0.1202 | Regression_cos_phi: 0.0478 | Regression_energy: 0.0422 | Regression_eta: 0.0269 | Regression_pt: 0.0450 | Regression_sin_phi: 0.0392 | LR: 1.13e-04 | DataLoad Time: 0.0179s | Model Forward Time: 4.7282s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0067 | max_forward: 0.8070 | max_loss: 0.2423 | max_backward: 1.9860 | max_optimizer_step: 0.2482 | max_tensorboard_logging: 0.7539 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0449 | Memory: rss_MB: 6199.1 | uss_MB: 5617.8 | child_rss_MB: 4859.8 | allocated_MB: 499.3 | reserved_MB: 25288.0 | inactive_split_MB: 1325.5 +rank7 [2026-10-30 04:46:59,454] INFO : Step 46300/50000 rank7 | Train Loss: 0.8634 | Train Loss Unweighted: 0.4215 | Train Loss Components: Auxiliary: 0.1892 | Cardinality: 0.0916 | Classification: 0.0803 | Classification_binary: 0.1320 | HitGrouping: 0.1536 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1031 | Regression_eta: 0.0011 | Regression_pt: 0.1038 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0475 | Classification_binary: 0.1445 | Regression_cos_phi: 0.0455 | Regression_energy: 0.0566 | Regression_eta: 0.0284 | Regression_pt: 0.0624 | Regression_sin_phi: 0.0450 | LR: 1.12e-04 | DataLoad Time: 0.0158s | Model Forward Time: 4.5168s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 0.7891 | max_loss: 0.1470 | max_backward: 1.7900 | max_optimizer_step: 0.2150 | max_tensorboard_logging: 0.7709 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6201.3 | uss_MB: 5618.4 | child_rss_MB: 4887.4 | allocated_MB: 498.4 | reserved_MB: 25290.0 | inactive_split_MB: 977.6 +rank7 [2026-10-30 04:51:36,901] INFO : Step 46400/50000 rank7 | Train Loss: 0.8499 | Train Loss Unweighted: 0.4102 | Train Loss Components: Auxiliary: 0.1778 | Cardinality: 0.0956 | Classification: 0.0755 | Classification_binary: 0.1172 | HitGrouping: 0.1532 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1074 | Regression_eta: 0.0012 | Regression_pt: 0.1076 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0430 | Classification_binary: 0.1059 | Regression_cos_phi: 0.0635 | Regression_energy: 0.0365 | Regression_eta: 0.0194 | Regression_pt: 0.0394 | Regression_sin_phi: 0.0458 | LR: 1.11e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.9777s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0069 | max_forward: 0.7847 | max_loss: 0.2487 | max_backward: 1.9181 | max_optimizer_step: 0.3148 | max_tensorboard_logging: 0.9408 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6188.6 | uss_MB: 5608.9 | child_rss_MB: 4878.3 | allocated_MB: 501.6 | reserved_MB: 25290.0 | inactive_split_MB: 992.4 +rank7 [2026-10-30 04:56:10,902] INFO : Step 46500/50000 rank7 | Train Loss: 0.8687 | Train Loss Unweighted: 0.4055 | Train Loss Components: Auxiliary: 0.1899 | Cardinality: 0.1002 | Classification: 0.0822 | Classification_binary: 0.1448 | HitGrouping: 0.1394 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0872 | Regression_eta: 0.0013 | Regression_pt: 0.0886 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0441 | Classification_binary: 0.1307 | Regression_cos_phi: 0.0546 | Regression_energy: 0.0597 | Regression_eta: 0.0213 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0616 | LR: 1.11e-04 | DataLoad Time: 0.0154s | Model Forward Time: 4.5416s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0072 | max_forward: 0.8086 | max_loss: 0.1839 | max_backward: 1.9629 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.6147 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6193.9 | uss_MB: 5618.8 | child_rss_MB: 4880.0 | allocated_MB: 498.8 | reserved_MB: 29306.0 | inactive_split_MB: 924.4 +rank7 [2026-10-30 05:00:44,088] INFO : Step 46600/50000 rank7 | Train Loss: 0.8742 | Train Loss Unweighted: 0.4049 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1082 | Classification: 0.0757 | Classification_binary: 0.1508 | HitGrouping: 0.1434 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0880 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0566 | Classification_binary: 0.1652 | Regression_cos_phi: 0.0841 | Regression_energy: 0.0494 | Regression_eta: 0.0536 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0471 | LR: 1.10e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.2108s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0068 | max_forward: 0.7939 | max_loss: 0.2001 | max_backward: 1.6859 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.5735 | max_tensorboard_flush: 0.0108 | max_post_step_barrier: 0.0450 | Memory: rss_MB: 6188.5 | uss_MB: 5615.4 | child_rss_MB: 4846.0 | allocated_MB: 496.0 | reserved_MB: 29306.0 | inactive_split_MB: 901.7 +rank7 [2026-10-30 05:05:18,398] INFO : Step 46700/50000 rank7 | Train Loss: 0.8417 | Train Loss Unweighted: 0.3855 | Train Loss Components: Auxiliary: 0.1716 | Cardinality: 0.0982 | Classification: 0.0807 | Classification_binary: 0.1184 | HitGrouping: 0.1466 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0916 | Regression_eta: 0.0015 | Regression_pt: 0.0919 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0512 | Classification_binary: 0.1305 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0516 | Regression_eta: 0.0370 | Regression_pt: 0.0567 | Regression_sin_phi: 0.0563 | LR: 1.10e-04 | DataLoad Time: 0.0163s | Model Forward Time: 4.7623s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0072 | max_forward: 0.8551 | max_loss: 0.5124 | max_backward: 1.9004 | max_optimizer_step: 0.2106 | max_tensorboard_logging: 0.7762 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.1375 | Memory: rss_MB: 6185.4 | uss_MB: 5616.8 | child_rss_MB: 4868.3 | allocated_MB: 495.0 | reserved_MB: 29308.0 | inactive_split_MB: 1000.4 +rank7 [2026-10-30 05:09:48,389] INFO : Step 46800/50000 rank7 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3911 | Train Loss Components: Auxiliary: 0.1785 | Cardinality: 0.0920 | Classification: 0.0920 | Classification_binary: 0.1296 | HitGrouping: 0.1414 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0828 | Regression_eta: 0.0013 | Regression_pt: 0.0841 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0577 | Classification_binary: 0.1601 | Regression_cos_phi: 0.0853 | Regression_energy: 0.0441 | Regression_eta: 0.0310 | Regression_pt: 0.0487 | Regression_sin_phi: 0.0660 | LR: 1.09e-04 | DataLoad Time: 0.0170s | Model Forward Time: 4.4616s | Phase Times: max_pre_step_barrier: 0.0025 | max_batch_to_device: 0.0063 | max_forward: 0.7934 | max_loss: 0.1440 | max_backward: 1.7630 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.7219 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0404 | Memory: rss_MB: 6183.1 | uss_MB: 5617.4 | child_rss_MB: 4871.4 | allocated_MB: 495.6 | reserved_MB: 29308.0 | inactive_split_MB: 1332.0 +rank7 [2026-10-30 05:14:19,737] INFO : Step 46900/50000 rank7 | Train Loss: 0.8496 | Train Loss Unweighted: 0.4142 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0976 | Classification: 0.0776 | Classification_binary: 0.1448 | HitGrouping: 0.1412 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0946 | Regression_eta: 0.0010 | Regression_pt: 0.0951 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0443 | Classification_binary: 0.1397 | Regression_cos_phi: 0.0313 | Regression_energy: 0.0525 | Regression_eta: 0.0282 | Regression_pt: 0.0569 | Regression_sin_phi: 0.0703 | LR: 1.09e-04 | DataLoad Time: 0.0161s | Model Forward Time: 4.4512s | Phase Times: max_pre_step_barrier: 0.0027 | max_batch_to_device: 0.0069 | max_forward: 0.9007 | max_loss: 0.1576 | max_backward: 1.8470 | max_optimizer_step: 0.2107 | max_tensorboard_logging: 0.5469 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0382 | Memory: rss_MB: 6183.1 | uss_MB: 5616.1 | child_rss_MB: 4872.4 | allocated_MB: 495.4 | reserved_MB: 29310.0 | inactive_split_MB: 850.5 +rank7 [2026-10-30 05:18:49,739] INFO : Step 47000/50000 rank7 | Train Loss: 0.7980 | Train Loss Unweighted: 0.3866 | Train Loss Components: Auxiliary: 0.1721 | Cardinality: 0.0982 | Classification: 0.0774 | Classification_binary: 0.1196 | HitGrouping: 0.1444 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0932 | Regression_eta: 0.0010 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0346 | Classification_binary: 0.1170 | Regression_cos_phi: 0.0420 | Regression_energy: 0.0568 | Regression_eta: 0.0227 | Regression_pt: 0.0623 | Regression_sin_phi: 0.0342 | LR: 1.08e-04 | DataLoad Time: 0.0164s | Model Forward Time: 4.8528s | Phase Times: max_pre_step_barrier: 0.0029 | max_batch_to_device: 0.0066 | max_forward: 1.2405 | max_loss: 0.1502 | max_backward: 2.1273 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.7105 | max_tensorboard_flush: 0.0082 | max_post_step_barrier: 0.0434 | Memory: rss_MB: 6184.1 | uss_MB: 5617.4 | child_rss_MB: 4877.5 | allocated_MB: 494.9 | reserved_MB: 29310.0 | inactive_split_MB: 852.6 +rank7 [2026-10-30 05:18:50,808] INFO : Running validation on rank7 +rank7 [2026-10-30 05:18:50,809] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 05:18:50,809] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 05:19:41,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 05:19:42,336] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 05:20:07,843] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 05:20:08,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 05:20:09,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 05:20:10,808] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 05:20:20,817] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 05:20:21,505] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 05:20:22,014] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 05:20:37,878] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 05:20:38,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 05:20:39,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 05:20:40,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 05:20:50,746] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 05:20:51,707] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 05:20:52,198] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 05:21:07,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 05:21:08,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 05:21:09,279] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 05:21:10,114] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_47000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 05:21:20,125] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 05:26:12,382] INFO : Step 47100/50000 rank7 | Train Loss: 0.8046 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1692 | Cardinality: 0.0851 | Classification: 0.0882 | Classification_binary: 0.1121 | HitGrouping: 0.1450 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0936 | Regression_eta: 0.0010 | Regression_pt: 0.0950 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0484 | Classification_binary: 0.1035 | Regression_cos_phi: 0.0434 | Regression_energy: 0.0520 | Regression_eta: 0.0233 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0659 | LR: 1.07e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5745s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0064 | max_forward: 0.8250 | max_loss: 0.1453 | max_backward: 1.6267 | max_optimizer_step: 0.3246 | max_tensorboard_logging: 0.8108 | max_tensorboard_flush: 0.0087 | max_post_step_barrier: 0.0530 | Memory: rss_MB: 7449.0 | uss_MB: 6878.6 | child_rss_MB: 4837.4 | allocated_MB: 494.1 | reserved_MB: 25416.0 | inactive_split_MB: 894.5 +rank7 [2026-10-30 05:30:44,527] INFO : Step 47200/50000 rank7 | Train Loss: 0.8884 | Train Loss Unweighted: 0.4191 | Train Loss Components: Auxiliary: 0.2001 | Cardinality: 0.1138 | Classification: 0.0793 | Classification_binary: 0.1489 | HitGrouping: 0.1451 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0942 | Regression_eta: 0.0010 | Regression_pt: 0.0945 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0571 | Classification_binary: 0.1348 | Regression_cos_phi: 0.0568 | Regression_energy: 0.0500 | Regression_eta: 0.0360 | Regression_pt: 0.0536 | Regression_sin_phi: 0.0746 | LR: 1.07e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.9177s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0069 | max_forward: 0.7960 | max_loss: 0.1482 | max_backward: 1.9261 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.9435 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0661 | Memory: rss_MB: 6249.9 | uss_MB: 5682.5 | child_rss_MB: 4852.8 | allocated_MB: 496.8 | reserved_MB: 26534.0 | inactive_split_MB: 847.6 +rank7 [2026-10-30 05:35:15,255] INFO : Step 47300/50000 rank7 | Train Loss: 0.8547 | Train Loss Unweighted: 0.3920 | Train Loss Components: Auxiliary: 0.1854 | Cardinality: 0.1049 | Classification: 0.0798 | Classification_binary: 0.1368 | HitGrouping: 0.1436 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0863 | Regression_eta: 0.0012 | Regression_pt: 0.0866 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0608 | Classification_binary: 0.1560 | Regression_cos_phi: 0.0491 | Regression_energy: 0.0429 | Regression_eta: 0.0371 | Regression_pt: 0.0452 | Regression_sin_phi: 0.0549 | LR: 1.06e-04 | DataLoad Time: 0.0162s | Model Forward Time: 4.5885s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0062 | max_forward: 0.7649 | max_loss: 0.2313 | max_backward: 1.8030 | max_optimizer_step: 0.2112 | max_tensorboard_logging: 0.8438 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6253.9 | uss_MB: 5688.5 | child_rss_MB: 4855.9 | allocated_MB: 496.9 | reserved_MB: 28534.0 | inactive_split_MB: 933.0 +rank7 [2026-10-30 05:39:47,015] INFO : Step 47400/50000 rank7 | Train Loss: 0.7606 | Train Loss Unweighted: 0.3755 | Train Loss Components: Auxiliary: 0.1640 | Cardinality: 0.0704 | Classification: 0.0737 | Classification_binary: 0.1072 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0960 | Regression_eta: 0.0010 | Regression_pt: 0.0964 | Regression_sin_phi: 0.0005 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0696 | Classification_binary: 0.1328 | Regression_cos_phi: 0.0605 | Regression_energy: 0.0489 | Regression_eta: 0.0337 | Regression_pt: 0.0542 | Regression_sin_phi: 0.0731 | LR: 1.06e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5319s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0275 | max_forward: 0.8465 | max_loss: 0.1482 | max_backward: 1.7103 | max_optimizer_step: 0.2136 | max_tensorboard_logging: 0.8388 | max_tensorboard_flush: 0.0105 | max_post_step_barrier: 0.0542 | Memory: rss_MB: 6241.7 | uss_MB: 5678.8 | child_rss_MB: 4860.9 | allocated_MB: 496.4 | reserved_MB: 28534.0 | inactive_split_MB: 913.7 +rank7 [2026-10-30 05:44:18,077] INFO : Step 47500/50000 rank7 | Train Loss: 0.8134 | Train Loss Unweighted: 0.3681 | Train Loss Components: Auxiliary: 0.1766 | Cardinality: 0.1057 | Classification: 0.0739 | Classification_binary: 0.1303 | HitGrouping: 0.1433 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0805 | Regression_eta: 0.0011 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0381 | Classification_binary: 0.1087 | Regression_cos_phi: 0.0290 | Regression_energy: 0.0460 | Regression_eta: 0.0355 | Regression_pt: 0.0512 | Regression_sin_phi: 0.0385 | LR: 1.06e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.7785s | Phase Times: max_pre_step_barrier: 0.0033 | max_batch_to_device: 0.0067 | max_forward: 0.8045 | max_loss: 0.1751 | max_backward: 1.9857 | max_optimizer_step: 0.3263 | max_tensorboard_logging: 0.7363 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0367 | Memory: rss_MB: 6249.7 | uss_MB: 5690.9 | child_rss_MB: 4845.9 | allocated_MB: 498.3 | reserved_MB: 28534.0 | inactive_split_MB: 851.5 +rank7 [2026-10-30 05:48:48,524] INFO : Step 47600/50000 rank7 | Train Loss: 0.8663 | Train Loss Unweighted: 0.4076 | Train Loss Components: Auxiliary: 0.1919 | Cardinality: 0.1090 | Classification: 0.0820 | Classification_binary: 0.1491 | HitGrouping: 0.1409 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0865 | Regression_eta: 0.0011 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0679 | Classification_binary: 0.1480 | Regression_cos_phi: 0.0751 | Regression_energy: 0.0483 | Regression_eta: 0.0347 | Regression_pt: 0.0537 | Regression_sin_phi: 0.0650 | LR: 1.05e-04 | DataLoad Time: 0.0155s | Model Forward Time: 4.6223s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0064 | max_forward: 0.9385 | max_loss: 0.1530 | max_backward: 2.1678 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.4657 | max_tensorboard_flush: 0.0091 | max_post_step_barrier: 0.0421 | Memory: rss_MB: 6249.7 | uss_MB: 5691.7 | child_rss_MB: 4853.8 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 843.5 +rank7 [2026-10-30 05:53:19,839] INFO : Step 47700/50000 rank7 | Train Loss: 0.8615 | Train Loss Unweighted: 0.4088 | Train Loss Components: Auxiliary: 0.1822 | Cardinality: 0.0954 | Classification: 0.0764 | Classification_binary: 0.1291 | HitGrouping: 0.1522 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1002 | Regression_eta: 0.0011 | Regression_pt: 0.1006 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0580 | Classification_binary: 0.1335 | Regression_cos_phi: 0.0336 | Regression_energy: 0.0473 | Regression_eta: 0.0230 | Regression_pt: 0.0517 | Regression_sin_phi: 0.0462 | LR: 1.05e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.2182s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0060 | max_forward: 0.7406 | max_loss: 0.2391 | max_backward: 1.7059 | max_optimizer_step: 0.2137 | max_tensorboard_logging: 0.4856 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1444 | Memory: rss_MB: 6248.7 | uss_MB: 5687.5 | child_rss_MB: 4851.8 | allocated_MB: 493.9 | reserved_MB: 28534.0 | inactive_split_MB: 895.3 +rank7 [2026-10-30 05:57:52,842] INFO : Step 47800/50000 rank7 | Train Loss: 0.8267 | Train Loss Unweighted: 0.3837 | Train Loss Components: Auxiliary: 0.1750 | Cardinality: 0.0942 | Classification: 0.0882 | Classification_binary: 0.1186 | HitGrouping: 0.1482 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0870 | Regression_eta: 0.0012 | Regression_pt: 0.0874 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0472 | Classification_binary: 0.0980 | Regression_cos_phi: 0.0281 | Regression_energy: 0.0431 | Regression_eta: 0.0169 | Regression_pt: 0.0462 | Regression_sin_phi: 0.0227 | LR: 1.04e-04 | DataLoad Time: 0.0161s | Model Forward Time: 5.0867s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0060 | max_forward: 0.7869 | max_loss: 0.5030 | max_backward: 2.0522 | max_optimizer_step: 0.2267 | max_tensorboard_logging: 1.1260 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0422 | Memory: rss_MB: 6249.7 | uss_MB: 5692.3 | child_rss_MB: 4857.3 | allocated_MB: 496.0 | reserved_MB: 28534.0 | inactive_split_MB: 916.9 +rank7 [2026-10-30 06:02:25,107] INFO : Step 47900/50000 rank7 | Train Loss: 0.8423 | Train Loss Unweighted: 0.4000 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.0845 | Classification: 0.0794 | Classification_binary: 0.1416 | HitGrouping: 0.1432 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0875 | Regression_eta: 0.0013 | Regression_pt: 0.0889 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0455 | Classification_binary: 0.1397 | Regression_cos_phi: 0.0429 | Regression_energy: 0.0567 | Regression_eta: 0.0261 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0463 | LR: 1.04e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.5093s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0074 | max_forward: 0.7994 | max_loss: 0.1511 | max_backward: 1.9354 | max_optimizer_step: 0.5320 | max_tensorboard_logging: 0.6685 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0366 | Memory: rss_MB: 6247.4 | uss_MB: 5692.2 | child_rss_MB: 4860.2 | allocated_MB: 496.8 | reserved_MB: 30642.0 | inactive_split_MB: 779.5 +rank7 [2026-10-30 06:06:56,644] INFO : Step 48000/50000 rank7 | Train Loss: 0.8641 | Train Loss Unweighted: 0.3968 | Train Loss Components: Auxiliary: 0.1886 | Cardinality: 0.0972 | Classification: 0.0768 | Classification_binary: 0.1396 | HitGrouping: 0.1425 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0890 | Regression_eta: 0.0015 | Regression_pt: 0.0887 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0447 | Classification_binary: 0.1214 | Regression_cos_phi: 0.0410 | Regression_energy: 0.0396 | Regression_eta: 0.0307 | Regression_pt: 0.0431 | Regression_sin_phi: 0.0401 | LR: 1.04e-04 | DataLoad Time: 0.0148s | Model Forward Time: 4.4074s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0057 | max_forward: 0.8017 | max_loss: 0.1550 | max_backward: 1.6990 | max_optimizer_step: 0.2149 | max_tensorboard_logging: 0.7498 | max_tensorboard_flush: 0.0092 | max_post_step_barrier: 0.0383 | Memory: rss_MB: 6247.1 | uss_MB: 5690.2 | child_rss_MB: 4855.9 | allocated_MB: 495.1 | reserved_MB: 30642.0 | inactive_split_MB: 896.2 +rank7 [2026-10-30 06:06:57,654] INFO : Running validation on rank7 +rank7 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 06:06:57,654] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 06:07:49,052] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:07:49,560] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 06:08:14,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 06:08:15,065] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 06:08:16,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 06:08:16,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 06:08:27,005] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 06:08:28,016] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:08:28,619] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 06:08:44,421] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 06:08:45,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 06:08:46,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 06:08:47,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 06:08:57,304] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 06:08:58,204] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:08:58,714] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 06:09:14,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 06:09:15,033] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 06:09:15,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 06:09:16,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_48000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 06:09:26,695] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 06:14:16,583] INFO : Step 48100/50000 rank7 | Train Loss: 0.8226 | Train Loss Unweighted: 0.3895 | Train Loss Components: Auxiliary: 0.1740 | Cardinality: 0.1043 | Classification: 0.0717 | Classification_binary: 0.1116 | HitGrouping: 0.1453 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1023 | Regression_eta: 0.0013 | Regression_pt: 0.1016 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0420 | Classification_binary: 0.1298 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0478 | Regression_eta: 0.0272 | Regression_pt: 0.0509 | Regression_sin_phi: 0.0450 | LR: 1.03e-04 | DataLoad Time: 0.0167s | Model Forward Time: 4.4959s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0061 | max_forward: 0.7477 | max_loss: 0.1426 | max_backward: 1.7972 | max_optimizer_step: 0.2286 | max_tensorboard_logging: 0.6718 | max_tensorboard_flush: 0.0099 | max_post_step_barrier: 0.1303 | Memory: rss_MB: 7752.6 | uss_MB: 7191.3 | child_rss_MB: 4880.7 | allocated_MB: 494.1 | reserved_MB: 23078.0 | inactive_split_MB: 1096.3 +rank7 [2026-10-30 06:18:47,187] INFO : Step 48200/50000 rank7 | Train Loss: 0.8500 | Train Loss Unweighted: 0.3835 | Train Loss Components: Auxiliary: 0.1846 | Cardinality: 0.1027 | Classification: 0.0771 | Classification_binary: 0.1374 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0827 | Regression_eta: 0.0012 | Regression_pt: 0.0837 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0285 | Classification_binary: 0.1052 | Regression_cos_phi: 0.0822 | Regression_energy: 0.0490 | Regression_eta: 0.0376 | Regression_pt: 0.0531 | Regression_sin_phi: 0.0473 | LR: 1.03e-04 | DataLoad Time: 0.0169s | Model Forward Time: 5.2123s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0066 | max_forward: 0.8132 | max_loss: 0.2545 | max_backward: 2.1167 | max_optimizer_step: 0.3341 | max_tensorboard_logging: 0.9368 | max_tensorboard_flush: 0.0098 | max_post_step_barrier: 0.1241 | Memory: rss_MB: 6310.4 | uss_MB: 5751.6 | child_rss_MB: 4867.3 | allocated_MB: 496.6 | reserved_MB: 25094.0 | inactive_split_MB: 913.4 +rank7 [2026-10-30 06:23:18,411] INFO : Step 48300/50000 rank7 | Train Loss: 0.9053 | Train Loss Unweighted: 0.4303 | Train Loss Components: Auxiliary: 0.2017 | Cardinality: 0.1089 | Classification: 0.0832 | Classification_binary: 0.1567 | HitGrouping: 0.1470 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0935 | Regression_eta: 0.0013 | Regression_pt: 0.0944 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0425 | Classification_binary: 0.1469 | Regression_cos_phi: 0.0398 | Regression_energy: 0.0459 | Regression_eta: 0.0336 | Regression_pt: 0.0497 | Regression_sin_phi: 0.0402 | LR: 1.03e-04 | DataLoad Time: 0.0146s | Model Forward Time: 4.4477s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0068 | max_forward: 0.8079 | max_loss: 0.1526 | max_backward: 1.9733 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.4825 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0425 | Memory: rss_MB: 6317.4 | uss_MB: 5760.9 | child_rss_MB: 4872.1 | allocated_MB: 498.5 | reserved_MB: 25098.0 | inactive_split_MB: 771.0 +rank7 [2026-10-30 06:27:49,028] INFO : Step 48400/50000 rank7 | Train Loss: 0.7961 | Train Loss Unweighted: 0.3733 | Train Loss Components: Auxiliary: 0.1682 | Cardinality: 0.0877 | Classification: 0.0685 | Classification_binary: 0.1191 | HitGrouping: 0.1417 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0912 | Regression_eta: 0.0012 | Regression_pt: 0.0921 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0402 | Classification_binary: 0.1379 | Regression_cos_phi: 0.0471 | Regression_energy: 0.0359 | Regression_eta: 0.0277 | Regression_pt: 0.0391 | Regression_sin_phi: 0.0596 | LR: 1.02e-04 | DataLoad Time: 0.0153s | Model Forward Time: 4.2397s | Phase Times: max_pre_step_barrier: 0.0021 | max_batch_to_device: 0.0061 | max_forward: 0.7916 | max_loss: 0.1471 | max_backward: 1.6943 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.5940 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0413 | Memory: rss_MB: 6314.4 | uss_MB: 5759.4 | child_rss_MB: 4842.4 | allocated_MB: 496.3 | reserved_MB: 25098.0 | inactive_split_MB: 1087.5 +rank7 [2026-10-30 06:32:21,027] INFO : Step 48500/50000 rank7 | Train Loss: 0.8668 | Train Loss Unweighted: 0.4311 | Train Loss Components: Auxiliary: 0.1885 | Cardinality: 0.0816 | Classification: 0.0832 | Classification_binary: 0.1257 | HitGrouping: 0.1542 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1095 | Regression_eta: 0.0014 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0516 | Classification_binary: 0.1294 | Regression_cos_phi: 0.0417 | Regression_energy: 0.0489 | Regression_eta: 0.0369 | Regression_pt: 0.0545 | Regression_sin_phi: 0.0512 | LR: 1.02e-04 | DataLoad Time: 0.0168s | Model Forward Time: 4.3826s | Phase Times: max_pre_step_barrier: 0.0026 | max_batch_to_device: 0.0065 | max_forward: 0.7587 | max_loss: 0.1606 | max_backward: 1.7573 | max_optimizer_step: 0.3024 | max_tensorboard_logging: 0.5851 | max_tensorboard_flush: 0.0101 | max_post_step_barrier: 0.0420 | Memory: rss_MB: 6309.3 | uss_MB: 5759.8 | child_rss_MB: 4851.4 | allocated_MB: 498.9 | reserved_MB: 25098.0 | inactive_split_MB: 1192.4 +rank7 [2026-10-30 06:36:52,986] INFO : Step 48600/50000 rank7 | Train Loss: 0.8434 | Train Loss Unweighted: 0.4010 | Train Loss Components: Auxiliary: 0.1850 | Cardinality: 0.0964 | Classification: 0.0791 | Classification_binary: 0.1425 | HitGrouping: 0.1437 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0884 | Regression_eta: 0.0010 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0453 | Classification_binary: 0.1294 | Regression_cos_phi: 0.0367 | Regression_energy: 0.0492 | Regression_eta: 0.0322 | Regression_pt: 0.0534 | Regression_sin_phi: 0.0443 | LR: 1.02e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.4847s | Phase Times: max_pre_step_barrier: 0.0017 | max_batch_to_device: 0.0063 | max_forward: 0.7760 | max_loss: 0.1505 | max_backward: 1.8991 | max_optimizer_step: 0.2127 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0093 | max_post_step_barrier: 0.0654 | Memory: rss_MB: 6311.3 | uss_MB: 5762.6 | child_rss_MB: 4856.0 | allocated_MB: 497.0 | reserved_MB: 26848.0 | inactive_split_MB: 777.5 +rank7 [2026-10-30 06:41:29,684] INFO : Step 48700/50000 rank7 | Train Loss: 0.9079 | Train Loss Unweighted: 0.4266 | Train Loss Components: Auxiliary: 0.1980 | Cardinality: 0.1195 | Classification: 0.0920 | Classification_binary: 0.1456 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0932 | Regression_eta: 0.0012 | Regression_pt: 0.0934 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0529 | Classification_binary: 0.1279 | Regression_cos_phi: 0.0357 | Regression_energy: 0.0525 | Regression_eta: 0.0209 | Regression_pt: 0.0568 | Regression_sin_phi: 0.0370 | LR: 1.02e-04 | DataLoad Time: 0.0160s | Model Forward Time: 4.5459s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0062 | max_forward: 0.7797 | max_loss: 0.1578 | max_backward: 1.7107 | max_optimizer_step: 0.5280 | max_tensorboard_logging: 0.7650 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0436 | Memory: rss_MB: 6308.9 | uss_MB: 5759.1 | child_rss_MB: 4857.7 | allocated_MB: 496.9 | reserved_MB: 26848.0 | inactive_split_MB: 891.8 +rank7 [2026-10-30 06:46:02,598] INFO : Step 48800/50000 rank7 | Train Loss: 0.8011 | Train Loss Unweighted: 0.3912 | Train Loss Components: Auxiliary: 0.1685 | Cardinality: 0.0863 | Classification: 0.0837 | Classification_binary: 0.1116 | HitGrouping: 0.1478 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0965 | Regression_eta: 0.0012 | Regression_pt: 0.0971 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0467 | Classification_binary: 0.1112 | Regression_cos_phi: 0.0477 | Regression_energy: 0.0514 | Regression_eta: 0.0245 | Regression_pt: 0.0546 | Regression_sin_phi: 0.0328 | LR: 1.01e-04 | DataLoad Time: 0.0175s | Model Forward Time: 4.4998s | Phase Times: max_pre_step_barrier: 0.0031 | max_batch_to_device: 0.0065 | max_forward: 0.7374 | max_loss: 0.2432 | max_backward: 1.8459 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.7427 | max_tensorboard_flush: 0.0103 | max_post_step_barrier: 0.0439 | Memory: rss_MB: 6306.9 | uss_MB: 5759.1 | child_rss_MB: 4857.0 | allocated_MB: 493.9 | reserved_MB: 26848.0 | inactive_split_MB: 991.0 +rank7 [2026-10-30 06:50:45,083] INFO : Step 48900/50000 rank7 | Train Loss: 0.8551 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1860 | Cardinality: 0.1109 | Classification: 0.0760 | Classification_binary: 0.1310 | HitGrouping: 0.1489 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0879 | Regression_eta: 0.0012 | Regression_pt: 0.0884 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0559 | Classification_binary: 0.0917 | Regression_cos_phi: 0.0198 | Regression_energy: 0.0432 | Regression_eta: 0.0286 | Regression_pt: 0.0486 | Regression_sin_phi: 0.0350 | LR: 1.01e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.5416s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0069 | max_forward: 0.8768 | max_loss: 0.1470 | max_backward: 2.0816 | max_optimizer_step: 0.2095 | max_tensorboard_logging: 0.5428 | max_tensorboard_flush: 0.0095 | max_post_step_barrier: 0.0402 | Memory: rss_MB: 6306.9 | uss_MB: 5761.7 | child_rss_MB: 4858.9 | allocated_MB: 499.5 | reserved_MB: 26852.0 | inactive_split_MB: 1398.0 +rank7 [2026-10-30 06:55:20,573] INFO : Step 49000/50000 rank7 | Train Loss: 0.9005 | Train Loss Unweighted: 0.4208 | Train Loss Components: Auxiliary: 0.2010 | Cardinality: 0.1248 | Classification: 0.0800 | Classification_binary: 0.1499 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0939 | Regression_eta: 0.0011 | Regression_pt: 0.0948 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0418 | Classification_binary: 0.1491 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0546 | Regression_eta: 0.0254 | Regression_pt: 0.0587 | Regression_sin_phi: 0.0359 | LR: 1.01e-04 | DataLoad Time: 0.0159s | Model Forward Time: 4.9024s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0439 | max_forward: 0.8365 | max_loss: 0.1536 | max_backward: 1.9893 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.8890 | max_tensorboard_flush: 0.0110 | max_post_step_barrier: 0.0405 | Memory: rss_MB: 6309.3 | uss_MB: 5762.6 | child_rss_MB: 4865.2 | allocated_MB: 497.5 | reserved_MB: 26852.0 | inactive_split_MB: 943.6 +rank7 [2026-10-30 06:55:21,669] INFO : Running validation on rank7 +rank7 [2026-10-30 06:55:21,670] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 06:55:21,670] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 06:56:13,351] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:56:13,863] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 06:56:39,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 06:56:40,095] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 06:56:41,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 06:56:42,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 06:56:52,034] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 06:56:52,550] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:56:53,029] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 06:57:08,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 06:57:09,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 06:57:10,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 06:57:11,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 06:57:21,602] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 06:57:22,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 06:57:23,153] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 06:57:38,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 06:57:39,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 06:57:40,380] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 06:57:41,224] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_49000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 06:57:51,234] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 07:02:45,630] INFO : Step 49100/50000 rank7 | Train Loss: 0.8219 | Train Loss Unweighted: 0.3880 | Train Loss Components: Auxiliary: 0.1779 | Cardinality: 0.0880 | Classification: 0.0841 | Classification_binary: 0.1303 | HitGrouping: 0.1498 | Regression_cos_phi: 0.0005 | Regression_energy: 0.0855 | Regression_eta: 0.0012 | Regression_pt: 0.0857 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0463 | Classification_binary: 0.1401 | Regression_cos_phi: 0.0324 | Regression_energy: 0.0386 | Regression_eta: 0.0229 | Regression_pt: 0.0423 | Regression_sin_phi: 0.0348 | LR: 1.01e-04 | DataLoad Time: 0.0171s | Model Forward Time: 4.3704s | Phase Times: max_pre_step_barrier: 0.0018 | max_batch_to_device: 0.0060 | max_forward: 0.7738 | max_loss: 0.1431 | max_backward: 1.6939 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.6276 | max_tensorboard_flush: 0.0096 | max_post_step_barrier: 0.0592 | Memory: rss_MB: 7720.3 | uss_MB: 7170.2 | child_rss_MB: 4829.0 | allocated_MB: 495.9 | reserved_MB: 24730.0 | inactive_split_MB: 936.0 +rank7 [2026-10-30 07:07:18,677] INFO : Step 49200/50000 rank7 | Train Loss: 0.8036 | Train Loss Unweighted: 0.3843 | Train Loss Components: Auxiliary: 0.1726 | Cardinality: 0.0856 | Classification: 0.0829 | Classification_binary: 0.1188 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0892 | Regression_eta: 0.0011 | Regression_pt: 0.0911 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0505 | Classification_binary: 0.1314 | Regression_cos_phi: 0.0629 | Regression_energy: 0.0459 | Regression_eta: 0.0370 | Regression_pt: 0.0526 | Regression_sin_phi: 0.0469 | LR: 1.01e-04 | DataLoad Time: 0.0178s | Model Forward Time: 4.7722s | Phase Times: max_pre_step_barrier: 0.0030 | max_batch_to_device: 0.0066 | max_forward: 0.7590 | max_loss: 0.5032 | max_backward: 1.9826 | max_optimizer_step: 0.2717 | max_tensorboard_logging: 0.7926 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0396 | Memory: rss_MB: 6365.4 | uss_MB: 5817.0 | child_rss_MB: 4830.4 | allocated_MB: 494.5 | reserved_MB: 26752.0 | inactive_split_MB: 1117.5 +rank7 [2026-10-30 07:11:50,862] INFO : Step 49300/50000 rank7 | Train Loss: 0.9397 | Train Loss Unweighted: 0.4282 | Train Loss Components: Auxiliary: 0.2065 | Cardinality: 0.1351 | Classification: 0.0889 | Classification_binary: 0.1552 | HitGrouping: 0.1431 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0901 | Regression_eta: 0.0012 | Regression_pt: 0.0913 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0554 | Classification_binary: 0.1307 | Regression_cos_phi: 0.0653 | Regression_energy: 0.0413 | Regression_eta: 0.0430 | Regression_pt: 0.0452 | Regression_sin_phi: 0.1003 | LR: 1.00e-04 | DataLoad Time: 0.0165s | Model Forward Time: 4.4919s | Phase Times: max_pre_step_barrier: 0.0014 | max_batch_to_device: 0.0070 | max_forward: 0.7807 | max_loss: 0.1520 | max_backward: 1.8619 | max_optimizer_step: 0.2348 | max_tensorboard_logging: 0.6076 | max_tensorboard_flush: 0.0102 | max_post_step_barrier: 0.0437 | Memory: rss_MB: 6376.6 | uss_MB: 5827.0 | child_rss_MB: 4814.1 | allocated_MB: 496.4 | reserved_MB: 26752.0 | inactive_split_MB: 1120.6 +rank7 [2026-10-30 07:16:22,252] INFO : Step 49400/50000 rank7 | Train Loss: 0.8870 | Train Loss Unweighted: 0.4213 | Train Loss Components: Auxiliary: 0.1937 | Cardinality: 0.1005 | Classification: 0.0815 | Classification_binary: 0.1465 | HitGrouping: 0.1455 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0952 | Regression_eta: 0.0013 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0517 | Classification_binary: 0.1369 | Regression_cos_phi: 0.0442 | Regression_energy: 0.0532 | Regression_eta: 0.0225 | Regression_pt: 0.0572 | Regression_sin_phi: 0.0437 | LR: 1.00e-04 | DataLoad Time: 0.0156s | Model Forward Time: 4.2712s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0062 | max_forward: 0.8046 | max_loss: 0.1540 | max_backward: 1.7244 | max_optimizer_step: 0.2156 | max_tensorboard_logging: 0.6030 | max_tensorboard_flush: 0.0104 | max_post_step_barrier: 0.0428 | Memory: rss_MB: 6373.7 | uss_MB: 5825.4 | child_rss_MB: 4813.8 | allocated_MB: 498.5 | reserved_MB: 28804.0 | inactive_split_MB: 886.2 +rank7 [2026-10-30 07:20:55,284] INFO : Step 49500/50000 rank7 | Train Loss: 0.8340 | Train Loss Unweighted: 0.3949 | Train Loss Components: Auxiliary: 0.1770 | Cardinality: 0.0991 | Classification: 0.0800 | Classification_binary: 0.1101 | HitGrouping: 0.1493 | Regression_cos_phi: 0.0007 | Regression_energy: 0.1007 | Regression_eta: 0.0013 | Regression_pt: 0.1017 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0620 | Classification_binary: 0.1195 | Regression_cos_phi: 0.0762 | Regression_energy: 0.0683 | Regression_eta: 0.0373 | Regression_pt: 0.0746 | Regression_sin_phi: 0.0435 | LR: 1.00e-04 | DataLoad Time: 0.0174s | Model Forward Time: 5.2154s | Phase Times: max_pre_step_barrier: 0.0129 | max_batch_to_device: 0.0074 | max_forward: 0.9924 | max_loss: 0.1865 | max_backward: 2.5853 | max_optimizer_step: 0.2148 | max_tensorboard_logging: 0.6095 | max_tensorboard_flush: 0.0106 | max_post_step_barrier: 0.0429 | Memory: rss_MB: 6377.1 | uss_MB: 5695.1 | child_rss_MB: 4830.9 | allocated_MB: 495.9 | reserved_MB: 28804.0 | inactive_split_MB: 849.6 +rank7 [2026-10-30 07:25:48,132] INFO : Step 49600/50000 rank7 | Train Loss: 0.9075 | Train Loss Unweighted: 0.4421 | Train Loss Components: Auxiliary: 0.1939 | Cardinality: 0.1021 | Classification: 0.0939 | Classification_binary: 0.1401 | HitGrouping: 0.1471 | Regression_cos_phi: 0.0006 | Regression_energy: 0.1026 | Regression_eta: 0.0014 | Regression_pt: 0.1028 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0682 | Classification_binary: 0.1232 | Regression_cos_phi: 0.0518 | Regression_energy: 0.0545 | Regression_eta: 0.0627 | Regression_pt: 0.0580 | Regression_sin_phi: 0.0645 | LR: 1.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.4400s | Phase Times: max_pre_step_barrier: 0.0024 | max_batch_to_device: 0.0073 | max_forward: 0.7852 | max_loss: 0.1445 | max_backward: 2.0967 | max_optimizer_step: 0.2122 | max_tensorboard_logging: 0.4754 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.0376 | Memory: rss_MB: 6378.7 | uss_MB: 5829.2 | child_rss_MB: 4808.2 | allocated_MB: 497.8 | reserved_MB: 28804.0 | inactive_split_MB: 845.2 +rank7 [2026-10-30 07:30:32,223] INFO : Step 49700/50000 rank7 | Train Loss: 0.9248 | Train Loss Unweighted: 0.4307 | Train Loss Components: Auxiliary: 0.2027 | Cardinality: 0.1074 | Classification: 0.0931 | Classification_binary: 0.1611 | HitGrouping: 0.1405 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0865 | Regression_eta: 0.0013 | Regression_pt: 0.0871 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0637 | Classification_binary: 0.1474 | Regression_cos_phi: 0.0482 | Regression_energy: 0.0490 | Regression_eta: 0.0361 | Regression_pt: 0.0551 | Regression_sin_phi: 0.0607 | LR: 1.00e-04 | DataLoad Time: 0.0157s | Model Forward Time: 4.6430s | Phase Times: max_pre_step_barrier: 0.0019 | max_batch_to_device: 0.0070 | max_forward: 0.8169 | max_loss: 0.1522 | max_backward: 2.0553 | max_optimizer_step: 0.2131 | max_tensorboard_logging: 0.4937 | max_tensorboard_flush: 0.0097 | max_post_step_barrier: 0.1402 | Memory: rss_MB: 6379.2 | uss_MB: 5828.6 | child_rss_MB: 4829.3 | allocated_MB: 499.3 | reserved_MB: 30718.0 | inactive_split_MB: 838.8 +rank7 [2026-10-30 07:35:17,059] INFO : Step 49800/50000 rank7 | Train Loss: 0.8342 | Train Loss Unweighted: 0.3857 | Train Loss Components: Auxiliary: 0.1767 | Cardinality: 0.1036 | Classification: 0.0769 | Classification_binary: 0.1292 | HitGrouping: 0.1429 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0880 | Regression_eta: 0.0012 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0493 | Classification_binary: 0.1561 | Regression_cos_phi: 0.0609 | Regression_energy: 0.0464 | Regression_eta: 0.0370 | Regression_pt: 0.0510 | Regression_sin_phi: 0.0568 | LR: 1.00e-04 | DataLoad Time: 0.0167s | Model Forward Time: 5.1087s | Phase Times: max_pre_step_barrier: 0.0020 | max_batch_to_device: 0.0329 | max_forward: 0.7490 | max_loss: 0.1889 | max_backward: 2.2249 | max_optimizer_step: 0.2452 | max_tensorboard_logging: 0.6221 | max_tensorboard_flush: 0.0112 | max_post_step_barrier: 0.0597 | Memory: rss_MB: 6374.9 | uss_MB: 5812.4 | child_rss_MB: 4828.7 | allocated_MB: 494.4 | reserved_MB: 30718.0 | inactive_split_MB: 1134.4 +rank7 [2026-10-30 07:39:58,303] INFO : Step 49900/50000 rank7 | Train Loss: 0.8436 | Train Loss Unweighted: 0.4136 | Train Loss Components: Auxiliary: 0.1812 | Cardinality: 0.0947 | Classification: 0.0943 | Classification_binary: 0.1179 | HitGrouping: 0.1441 | Regression_cos_phi: 0.0006 | Regression_energy: 0.0982 | Regression_eta: 0.0012 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0007 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0638 | Classification_binary: 0.1051 | Regression_cos_phi: 0.0458 | Regression_energy: 0.0468 | Regression_eta: 0.0269 | Regression_pt: 0.0519 | Regression_sin_phi: 0.0524 | LR: 1.00e-04 | DataLoad Time: 0.0180s | Model Forward Time: 4.4169s | Phase Times: max_pre_step_barrier: 0.0023 | max_batch_to_device: 0.0071 | max_forward: 0.7496 | max_loss: 0.5142 | max_backward: 1.9365 | max_optimizer_step: 0.2151 | max_tensorboard_logging: 0.5703 | max_tensorboard_flush: 0.0100 | max_post_step_barrier: 0.0430 | Memory: rss_MB: 6373.9 | uss_MB: 5825.7 | child_rss_MB: 4831.8 | allocated_MB: 494.8 | reserved_MB: 30718.0 | inactive_split_MB: 1090.4 +rank7 [2026-10-30 07:44:45,886] INFO : Step 50000/50000 rank7 | Train Loss: 0.8857 | Train Loss Unweighted: 0.4040 | Train Loss Components: Auxiliary: 0.1921 | Cardinality: 0.1113 | Classification: 0.0818 | Classification_binary: 0.1436 | HitGrouping: 0.1424 | Regression_cos_phi: 0.0007 | Regression_energy: 0.0872 | Regression_eta: 0.0014 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0006 | Task Weights: Classification: 0.6102 | Classification_binary: 1.0000 | Regression_cos_phi: 84.4237 | Regression_energy: 0.5330 | Regression_eta: 28.1105 | Regression_pt: 0.5762 | Regression_sin_phi: 82.2352 | Task Weighted Losses: Classification: 0.0482 | Classification_binary: 0.1156 | Regression_cos_phi: 0.0710 | Regression_energy: 0.0487 | Regression_eta: 0.0696 | Regression_pt: 0.0528 | Regression_sin_phi: 0.0466 | LR: 1.00e-04 | DataLoad Time: 0.0162s | Model Forward Time: 5.6360s | Phase Times: max_pre_step_barrier: 0.0022 | max_batch_to_device: 0.0072 | max_forward: 1.3997 | max_loss: 0.5678 | max_backward: 3.0655 | max_optimizer_step: 0.2144 | max_tensorboard_logging: 0.5162 | max_tensorboard_flush: 0.0118 | max_post_step_barrier: 0.0408 | Memory: rss_MB: 6373.9 | uss_MB: 5825.4 | child_rss_MB: 4827.6 | allocated_MB: 497.9 | reserved_MB: 30720.0 | inactive_split_MB: 906.1 +rank7 [2026-10-30 07:44:46,920] INFO : Running validation on rank7 +rank7 [2026-10-30 07:44:46,920] INFO : ResumableSampler physical:valid resetting +rank7 [2026-10-30 07:44:46,920] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank7 [2026-10-30 07:45:38,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 07:45:38,998] INFO : Running predictions on cld_edm_ttbar_hits +rank7 [2026-10-30 07:46:04,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_7_0.parquet +rank7 [2026-10-30 07:46:05,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_7_1.parquet +rank7 [2026-10-30 07:46:06,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_7_2.parquet +rank7 [2026-10-30 07:46:07,021] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ttbar_hits/pred_7_3.parquet +rank7 [2026-10-30 07:46:17,029] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 07:46:17,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 07:46:17,995] INFO : Running predictions on cld_edm_ww_fullhad_hits +rank7 [2026-10-30 07:46:34,148] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_7_0.parquet +rank7 [2026-10-30 07:46:35,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_7_1.parquet +rank7 [2026-10-30 07:46:36,085] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_7_2.parquet +rank7 [2026-10-30 07:46:37,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_ww_fullhad_hits/pred_7_3.parquet +rank7 [2026-10-30 07:46:47,035] INFO : Time taken to make predictions on device 7 is: 0.25 min +rank7 [2026-10-30 07:46:47,615] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank7 [2026-10-30 07:46:48,153] INFO : Running predictions on cld_edm_qq_hits +rank7 [2026-10-30 07:47:03,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_7_0.parquet +rank7 [2026-10-30 07:47:04,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_7_1.parquet +rank7 [2026-10-30 07:47:05,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_7_2.parquet +rank7 [2026-10-30 07:47:06,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/cld_hits_heptv2_seed12345_20261028_111202_875252/preds_step_50000/cld_edm_qq_hits/pred_7_3.parquet +rank7 [2026-10-30 07:47:16,157] INFO : Time taken to make predictions on device 7 is: 0.23 min +rank7 [2026-10-30 07:47:17,240] INFO : Training completed. Total time on device 7: 1017.499min diff --git a/cld/hits/v3.2.0/comparison_metrics/provenance.json b/cld/hits/v3.2.0/comparison_metrics/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7dd93dc97277643919f983f16a0633812762afa4 --- /dev/null +++ b/cld/hits/v3.2.0/comparison_metrics/provenance.json @@ -0,0 +1,24 @@ +{ + "training_code_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "study": "20261030_clic_cld_checkpoint_summary", + "selected_step": 50000, + "sources": "Original per-run history JSON and train TensorBoard streams, retained alongside these summaries.", + "metrics": { + "f1": "Count-aggregated validation particle F1. Capped matching cost 2.0, acceptance deltaR<0.1 and relative pT error<0.5, no PID requirement.", + "pt_mae": "Mean absolute relative pT error among accepted pairs; conditional and truncated by acceptance, not inclusive resolution.", + "jet_iqr": "Equal mean of ttbar, WW fullhad, qq IQRs of angular-matched predicted/target jet pT ratios.", + "jet_recall": "Equal-process response-qualified jet recall, with abs(pT ratio - 1)<0.5.", + "met": "Full-validation mean absolute error of MET magnitude in GeV.", + "p3": "Detailed 512-entry subset mean norm of event three-momentum difference, GeV.", + "steady_s": "Median seconds/step across adjacent 100-step train TensorBoard intervals, excluding validation boundaries, warmup and session changes.", + "compute_s": "Median instrumented rank-zero forward+loss+backward+optimizer time, seconds.", + "peak_gib": "Maximum recorded allocator peak across ranks and sessions, per logical device. May include prior validation allocations." + }, + "limitations": [ + "One seed. No paired confidence intervals or verified cross-representation event identity.", + "PF TFDS 3.3.0 versus hits 3.2.2, with differing target populations and validation sampling.", + "PF validation has 1530 entries, distributed hit validation pads to 1536; detailed momentum diagnostics use differently selected 512-entry subsets.", + "Training resource figures are not inference or end-to-end reconstruction benchmarks.", + "CLD hit HEPTv2 has no step_30000.json in the source results." + ] +} diff --git a/cld/hits/v3.2.0/comparison_metrics/study_methodology.md b/cld/hits/v3.2.0/comparison_metrics/study_methodology.md new file mode 100644 index 0000000000000000000000000000000000000000..d40607cfea1c2d71542e91632ef20b56a9c172a0 --- /dev/null +++ b/cld/hits/v3.2.0/comparison_metrics/study_methodology.md @@ -0,0 +1,207 @@ +# CLIC and CLD checkpoint summary, 30 October 2026 + +Twenty summary slides compare the eight runs in the 28 October +`clic_cld_backbone_benchmark` campaign. The common endpoint is **50,000 optimizer +steps**. Evolution curves receive priority over endpoint-only comparisons. +This is a snapshot of local results, not a report of current remote job status. + +## Deliverables + +- [PDF slides](output/checkpoint_summary.slides.pdf) +- [Self-contained HTML slides](output/checkpoint_summary.slides.html), arrow-key navigation +- [Source notebook](checkpoint_summary.ipynb) and [executed notebook](output/checkpoint_summary.executed.ipynb) +- [Numerical scorecard](output/scorecard.csv), [summary](output/summary.json), [resource series](output/resources.json) +- `output/slides.json`: generated slide copy, figure paths and source categories +- `output/*.svg` and `output/*.png`: standalone scientific figures + +The figures are generated from archived data by `study_analysis.py`. Tables and +slide copy are HTML text. No external CDN, mutable experiment path, or earlier +study archive is needed to render. This deliverable follows the notebook/HTML/PDF +format in the parent README; it does not include a PowerPoint conversion. + +## Findings + +HEPTv2 improves final jet IQR, MET error and detailed event momentum error in +all four detector/representation pairs. On CLIC hits, F1 rises from 0.7747 to +0.8623 (+8.76 percentage points) and the equal-process jet IQR decreases from +0.1579 to 0.0995 (37.0%). On CLD hits, F1 rises from 0.8458 to 0.8624 +(+1.67 points) and jet IQR decreases from 0.1243 to 0.1040 (16.4%). + +Hit HEPTv2 has higher final particle F1 than track/cluster HEPTv2 in both +detectors. Track/cluster HEPTv2 retains lower conditional particle momentum +error, jet IQR, MET error and event momentum-vector error. These statements +refer to the recorded metrics and configured target populations, not paired +statistical significance or full inclusive momentum resolution. + +The median ordinary training-interval estimates are about 0.11 s/step for PF +attention, 0.40–0.41 s for PF HEPTv2, 2.36–2.38 s for hit attention and +2.84–2.89 s for hit HEPTv2. PF uses one logical accelerator device; hits use +eight. At 50k steps this projects to roughly 55–58 times the device cost for +hit HEPTv2 versus PF HEPTv2, excluding validation, queueing and input +preprocessing. This is not an inference or end-to-end reconstruction benchmark. +Exposed rank-zero batch wait has medians near 2 ms for PF and 17 ms for hits. +No storage byte counters, CPU utilization or bandwidth measurements support a +broader I/O-bottleneck conclusion. + +## Runs and configuration + +All runs use seed 12345, revision +`6b80b983477321cf708e5043fb1d2ec15dd398bc`, global batch 128, 50k scheduled steps, +LAMB, BF16, LR 0.001 with cosine decay, and no compilation. All completed +histories end at 50k. Run IDs record launch preparation on 28 October 2026 at +11:12; hit continuations were submitted on 29 October as recorded in their +manifests. Full log timestamps and exact commands remain archived. + +| Archive name | Exact run ID | Devices | History points | Resume checkpoint | +| --- | --- | ---: | ---: | --- | +| cld_hits_attention | `cld_hits_attention_seed12345_20261028_111201_979187` | 8 | 50 | 35000 | +| cld_hits_heptv2 | `cld_hits_heptv2_seed12345_20261028_111202_875252` | 8 | 49 | 30000 | +| cld_pf_attention | `cld_pf_attention_seed12345_20261028_111201_967206` | 1 | 50 | none | +| cld_pf_heptv2 | `cld_pf_heptv2_seed12345_20261028_111202_024020` | 1 | 50 | none | +| clic_hits_attention | `clic_hits_attention_seed12345_20261028_111202_121851` | 8 | 50 | 35000 | +| clic_hits_heptv2 | `clic_hits_heptv2_seed12345_20261028_111202_012645` | 8 | 50 | 29000 | +| clic_pf_attention | `clic_pf_attention_seed12345_20261028_111202_052283` | 1 | 50 | none | +| clic_pf_heptv2 | `clic_pf_heptv2_seed12345_20261028_111202_849554` | 1 | 50 | none | + +Hardware provenance comes from `scenario-manifest.json`: platform +`lumi_mi250x`, eight logical devices available per platform/node. Actual model +world size is one for track/cluster and eight for hits, verified against resolved +configuration and logs. The generic `gpu_type: l40` in train-config is a stale +recipe default, not evidence that these jobs ran on L40 hardware. Device-hours +count logical MI250X accelerator devices/GCDs, not whole dual-GCD packages. + +All models use width 256, six shared backbone layers. PF attention/HEPTv2 have +3,114,258 / 8,639,954 parameters. Hit attention/HEPTv2 have +7,260,244 / 12,786,964. The hit decoder uses four global-Flash layers and 256 +slots, supervised grouping, tracker-neighborhood features and dedicated tracker +momentum readout. HEPTv2 uses deterministic hashing, block size 128 and activation +checkpointing. These are fixed recipe comparisons, not equal-parameter or +pure-backbone-kernel benchmarks. Losses/objectives and readouts differ between +PF and hit models, so weighted loss is not used as cross-representation evidence. + +Both detectors use ttbar, WW fullhad and qq. Dataset names are +`{clic,cld}_edm_{ttbar,ww_fullhad,qq}_{pf,hits}`, with all ten numbered configs. +PF TFDS is 3.3.0 and hit TFDS is 3.2.2. Exact production/data locations are +preserved only as provenance in original configuration files. Rendering never +reads those locations. At 50k each run has about 2.37 training epochs / 6.4M +training-event presentations, not 6.4M unique events. + +## Metrics and sampling + +Particle curves use history `valid/metrics/particle/*` scalars. The archived +validation matcher minimizes `(deltaR/0.1)^2 + (log(pT ratio)/log(2))^2`, capped +at 2 per pair. Accepted matches require deltaR < 0.1 and relative pT error < 0.5, +without PID agreement. F1/efficiency aggregate counts through the validation +accumulator, rather than averaging process-level F1. Conditional momentum MAE +is mean absolute relative pT error among accepted pairs; the 50% acceptance cut +truncates its tails. It is not a central-68% width or inclusive resolution. + +`nvalid=512` with ten split configs selects 51 entries per config, 510 per +process, 1,530 total. The distributed sampler pads the hit validation collection +to 1,536 entries. Detailed momentum metrics apply to four batches per rank, +512 entries globally for both representations, but rank partitioning changes +which events are in this subset. `momentum/all/event/p3_error_mean` is the mean +norm of the difference between summed predicted and target three-momenta, +not the mean of individual particle momentum errors. Basic MET and energy +metrics use the full validation collection. MET error compares magnitudes. + +Jet curves use top-level `step//jet_ratio/jet_ratio_target_to_pred_pt/*` +scalars from the test/plot cycle. Ratios are predicted / target pT. IQR is +Q75−Q25 among angular-matched jets, with no additional response acceptance cut. +Response-qualified recall counts angular matches satisfying |ratio−1| < 0.5 +relative to all target jets. Each displayed curve is an equal mean of the three +process-specific summaries, not an IQR of pooled jets. `ntest=512` likewise +requests 51 events from each of ten configs per process before distributed +padding; sample identity and duplicate entries were not independently audited. + +PF and hit target means differ slightly (e.g. CLD 70.565 versus 70.655), which +can include both target-definition and sampler effects. We have not verified +exact truth-event identity across representations, nor rescored predictions +with a common event subset. Backbone comparisons within a representation have +the same nominal configuration and validation counts, but have no paired-event +confidence intervals or training-seed replication. Repeated validation/test +inspection makes the comparisons exploratory. Earlier studies have different +data and matching defaults, and their metric values are not mixed into this +campaign. CLD hit HEPTv2 lacks `step_30000.json`; plots connect 29k to 31k without +synthesizing a 30k observation. + +## Resource method + +`extract_resources.py` reads every archived train event file separately with +unbounded scalar retention. It retains the latest wall-time observation for +replayed steps, preserving stream identity to reject cross-session intervals. +All required scalar series have aligned step sets. There are 500 observations +per model at steps 100–50,000 after deduplication. + +Ordinary throughput uses elapsed wall time between adjacent 100-step TensorBoard +observations from the same event file, excluding intervals beginning at a +multiple of 1,000 (which cross validation/test work), intervals before 1,000, +and nonadjacent/restart boundaries. The median seconds per optimizer step is +robust to isolated stalls but is an estimate, not total campaign elapsed time. +Each 100-step interval still contains one diagnostic step. Projected 50k device +cost multiplies this median by 50,000 and by actual world size. It excludes +validation, test plotting, checkpoint cycles at excluded boundaries, scheduling +downtime, startup and abandoned/replayed training work. It does not include PF +tracking/clustering preprocessing. + +Instrumented-step curves and phase breakdowns use rank-zero raw phase timers, +not sums of maxima across different ranks. Compute is forward + loss + backward ++ optimizer. The enclosing `time_model_forward` includes training, backward and +logging, so it is not used as pure forward. `step/time_train_total` includes +synchronized diagnostics and memory logging. Its median can be much greater +than the ordinary interval estimate. The remaining phase is total minus listed +phases; negative residuals are rejected. It includes diagnostic and collective +work not separately plotted. + +Loader wait is the exposed wall time in `next(train_iterator)` on rank zero. +Transfer is the synchronized batch-to-device phase. Neither measures storage +service time, byte throughput, asynchronous worker effort or the slowest rank's +I/O. Resource plots use all raw sampled observations; thick timing lines are +centered 11-sample medians. Input multiplicities quoted on slide 15 are medians +of the per-step, across-rank batch-average element counts. + +Allocator memory is the maximum across ranks of historical peak allocation at +step end, and can include earlier validation allocations. Restart resets it. +This differs from reserved/cached memory and total VRAM occupancy. CPU RSS is +the mean training-process footprint across ranks; it omits child RSS and is not +unique node memory. MB tag values use MiB, divided by 1024 for GiB. The compact +resource JSON also retains child RSS, USS, current reserved memory and padding +for later inspection. No inference benchmark is inferred from these logs. + +**Timer trap:** history `step_valid_time` is calculated as current time minus +session start minus the last training-step duration. It is cumulative and +resets on resume, not a validation-cycle duration. Neither it nor a sum of +history `step_total_time` values is used as validation or total training cost. +The archived revision's `training.py` documents this behavior. + +## Archive and reproduction + +Run from the repository root: + +```bash +bash notebooks/studies/20261030_clic_cld_checkpoint_summary/render.sh +``` + +The script executes the source notebook, which re-extracts resources, regenerates +figures/tables/copy, and emits each slide. It writes the executed notebook and +self-contained HTML, then headless Chrome generates PDF and Poppler renders page +PNGs. Override `PYTHON_BIN` / `CHROME_BIN` if needed. Dependencies are nbformat, +nbclient, IPython, numpy, matplotlib, TensorBoard and a Python 3 Jupyter kernel. +Jupyter needs local socket permissions. Chrome/Chromium is optional for HTML +but required for PDF. `MPLCONFIGDIR` defaults to a temporary writable cache; +scientific inputs and outputs remain study-relative. + +`inputs//` contains complete histories, complete +`plots_step_50000`, the rank-zero `train.log.0` copied as `train.log`, separate +nonempty `tensorboard/train` and `tensorboard/valid` streams, train configuration, +hyperparameters, scenario manifest and resolved particleflow specification. +Files are byte-for-byte copies. `inputs/run_manifest.json` maps descriptive names +to exact original run IDs. `inputs/SHA256SUMS` checks every archived input. +`inputs/sources/` contains the campaign revision's metric, data-sampling and +timing code for definition auditing. No checkpoint, prediction parquet, model +pickle or dataset cache was copied. Inputs total approximately 428.6 MB, mainly +TensorBoard. There is no dependency on a previous study's local artifacts. + +New study files follow the parent directory's local-artifact/Git-ignore policy. +Preserve the full dated folder to retain reproducibility. No private-bucket +upload was requested or performed. diff --git a/cld/hits/v3.2.0/comparison_metrics/summary.json b/cld/hits/v3.2.0/comparison_metrics/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6ae320d7582c38f3070856bc1b414ac7690b973b --- /dev/null +++ b/cld/hits/v3.2.0/comparison_metrics/summary.json @@ -0,0 +1,44 @@ +[ + { + "run": "cld_hits_attention", + "detector": "CLD", + "model": "Hits attention", + "f1": 0.8457528948783875, + "efficiency": 0.8327773809432983, + "pt_mae": 0.12981022894382477, + "jet_iqr": 0.1243489058545048, + "jet_recall": 0.9420660540663063, + "met": 5.562965393066406, + "p3": 18.238574981689453, + "steady_s": 2.3781859827041627, + "devices": 8, + "params": 7260244, + "wait_ms": 16.909480094909668, + "wait_pct": 0.3544285023874039, + "transfer_ms": 5.489614559337497, + "compute_s": 2.324357531964779, + "sampled_s": 4.865405797958374, + "peak_gib": 11.692705154418945 + }, + { + "run": "cld_hits_heptv2", + "detector": "CLD", + "model": "Hits HEPTv2", + "f1": 0.8624090552330017, + "efficiency": 0.8545141220092773, + "pt_mae": 0.12283195555210114, + "jet_iqr": 0.10395381358980323, + "jet_recall": 0.9514567147761047, + "met": 4.790363788604736, + "p3": 15.765058517456055, + "steady_s": 2.8420948457717894, + "devices": 8, + "params": 12786964, + "wait_ms": 16.710877418518066, + "wait_pct": 0.3159196711868073, + "transfer_ms": 5.5203549563884735, + "compute_s": 2.839740104973316, + "sampled_s": 5.353001117706299, + "peak_gib": 14.568944931030273 + } +] diff --git a/cld/hits/v3.2.0/release.json b/cld/hits/v3.2.0/release.json new file mode 100644 index 0000000000000000000000000000000000000000..67e5b6ba9decc785838efedc5bf3fa67d02180a6 --- /dev/null +++ b/cld/hits/v3.2.0/release.json @@ -0,0 +1,74 @@ +{ + "release": "3.2.0", + "detector": "cld", + "representation": "hits", + "training_code_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "selected_step": 50000, + "dataset_version": "3.2.2", + "models": { + "attention": { + "run": "cld_hits_attention_seed12345_20261028_111201_979187", + "checkpoint": "checkpoints/checkpoint-50000.pth", + "checkpoint_sha256": "55342abbe8a974130f663b6c44fa5a5387d715e0946dfa7e2b4e281c6f1e3578", + "parameters": 7260244, + "global_batch_size": 128, + "accelerator_devices": 8, + "seed": 12345, + "history_points": 50, + "missing_history_steps": [], + "validation_metrics": { + "metrics/particle/matching/f1": 0.8457528948783875, + "metrics/particle/matched/pt_relative_abs_mean": 0.12981022894382477, + "metrics/particle/event/met_abs_error": 5.562965393066406, + "metrics/particle/momentum/all/event/p3_error_mean": 18.238574981689453 + } + }, + "heptv2": { + "run": "cld_hits_heptv2_seed12345_20261028_111202_875252", + "checkpoint": "checkpoints/checkpoint-50000.pth", + "checkpoint_sha256": "f162da4e4e90066cc41fd492f7279c8aa89180040a65358d25eb6c9f45660883", + "parameters": 12786964, + "global_batch_size": 128, + "accelerator_devices": 8, + "seed": 12345, + "history_points": 49, + "missing_history_steps": [ + 30000 + ], + "validation_metrics": { + "metrics/particle/matching/f1": 0.8624090552330017, + "metrics/particle/matched/pt_relative_abs_mean": 0.12283195555210114, + "metrics/particle/event/met_abs_error": 4.790363788604736, + "metrics/particle/momentum/all/event/p3_error_mean": 15.765058517456055 + } + } + }, + "contents": "Final 50k checkpoints, complete available histories, separate train/valid TensorBoard streams, configs, model kwargs, scenario manifests, all training logs, and complete final plots. Prediction parquet files and batch dumps are excluded.", + "checkpoint_selection": "Final scheduled 50k checkpoint; not selected as best by validation or test performance.", + "code_tag": "v3.2.0 is the intended code release label. This upload does not create or verify the Git tag. The training commit above is authoritative.", + "limitations": [ + "One seed. No paired confidence intervals or verified cross-representation event identity.", + "PF TFDS 3.3.0 versus hits 3.2.2, with differing target populations and validation sampling.", + "PF validation has 1530 entries, distributed hit validation pads to 1536; detailed momentum diagnostics use differently selected 512-entry subsets.", + "Training resource figures are not inference or end-to-end reconstruction benchmarks.", + "CLD hit HEPTv2 has no step_30000.json in the source results." + ], + "checkpoint_validation": [ + { + "run": "cld_hits_attention_seed12345_20261028_111201_979187", + "step": 50000, + "parameters": 7260244, + "strict_state_dict_load": true, + "validation_code_revision": "d2d991cea54ab47382e9de63bc900d2850e75eb4", + "scope": "CPU construction and strict state-dict loading including task-loss calibration buffers. No forward/inference or physics validation." + }, + { + "run": "cld_hits_heptv2_seed12345_20261028_111202_875252", + "step": 50000, + "parameters": 12786964, + "strict_state_dict_load": true, + "validation_code_revision": "d2d991cea54ab47382e9de63bc900d2850e75eb4", + "scope": "CPU construction and strict state-dict loading including task-loss calibration buffers. No forward/inference or physics validation." + } + ] +}